Merge #185: Rpc: Introduce call for per-block fee statistics

b142b2c Tests: Add data file (Anthony Towns)
9dde7d7 Tests: Test new getblockstats RPC (Jorge Timón)
bcb509c RPC: Introduce getblockstats (Jorge Timón)
938b509 Refactor: RPC: Separate GetBlockChecked() from getblock() (Jorge Timón)
This commit is contained in:
Gregory Sanders 2018-05-16 12:26:27 -04:00
commit c849ded4fe
No known key found for this signature in database
GPG key ID: F3F68E2D86A48FDB
5 changed files with 702 additions and 12 deletions

View file

@ -149,6 +149,7 @@ testScripts = [
'prioritise_transaction.py',
'invalidblockrequest.py',
'invalidtxrequest.py',
'rpc_getblockstats.py',
'confidential_transactions.py',
'preciousblock.py',
#'p2p-segwit.py',

File diff suppressed because one or more lines are too long

181
qa/rpc-tests/rpc_getblockstats.py Executable file
View file

@ -0,0 +1,181 @@
#!/usr/bin/env python3
# Copyright (c) 2017-2017 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Test getblockstats rpc call
#
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import (
assert_equal,
assert_raises_jsonrpc,
connect_nodes_bi,
start_nodes,
)
import logging
import json
import os
import time
TESTSDIR = os.path.dirname(os.path.realpath(__file__))
class GetblockstatsTest(BitcoinTestFramework):
start_height = 101
max_stat_pos = 2
STATS_NEED_TXINDEX = [
'utxo_size_inc',
]
def add_options(self, parser):
parser.add_option('--gen-test-data', dest='gen_test_data',
default=False, action='store_true',
help='Generate test data')
parser.add_option('--test-data', dest='test_data',
default='data/rpc_getblockstats.json',
action='store', metavar='FILE',
help='Test data file')
def __init__(self):
super().__init__()
self.num_nodes = 2
self.extra_args = [['-txindex'], ['-paytxfee=0.003']]
self.setup_clean_chain = True
def setup_network(self, split=False):
self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args=self.extra_args)
connect_nodes_bi(self.nodes, 0, 1)
self.is_network_split=False
self.sync_all()
def get_stats(self):
return [self.nodes[0].getblockstats(hash_or_height=self.start_height + i) for i in range(self.max_stat_pos+1)]
def generate_test_data(self, filename):
mocktime = time.time()
self.nodes[0].generate(101)
self.nodes[0].sendtoaddress(address=self.nodes[1].getnewaddress(), amount=10, subtractfeefromamount=True)
self.nodes[0].generate(1)
self.sync_all()
self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=10, subtractfeefromamount=True)
self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=10, subtractfeefromamount=False)
self.nodes[1].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=1, subtractfeefromamount=True)
self.sync_all()
self.nodes[0].generate(1)
self.expected_stats = self.get_stats()
blocks = []
tip = self.nodes[0].getbestblockhash()
blockhash = None
height = 0
while tip != blockhash:
blockhash = self.nodes[0].getblockhash(height)
blocks.append(self.nodes[0].getblock(blockhash, 0))
height += 1
to_dump = {
'blocks': blocks,
'mocktime': int(mocktime),
'stats': self.expected_stats,
}
with open(filename, 'w') as f:
json.dump(to_dump, f, sort_keys=True, indent=2)
def load_test_data(self, filename):
with open(filename, 'r') as f:
d = json.load(f)
blocks = d['blocks']
mocktime = d['mocktime']
self.expected_stats = d['stats']
# Set the timestamps from the file so that the nodes can get out of Initial Block Download
self.nodes[0].setmocktime(mocktime)
self.nodes[1].setmocktime(mocktime)
for b in blocks:
self.nodes[0].submitblock(b)
def run_test(self):
test_data = os.path.join(TESTSDIR, self.options.test_data)
if self.options.gen_test_data:
self.generate_test_data(test_data)
else:
self.load_test_data(test_data)
self.sync_all()
stats = self.get_stats()
expected_stats_noindex = []
for stat_row in stats:
expected_stats_noindex.append({k: v for k, v in stat_row.items() if k not in self.STATS_NEED_TXINDEX})
# Make sure all valid statistics are included but nothing else is
expected_keys = self.expected_stats[0].keys()
assert_equal(set(stats[0].keys()), set(expected_keys))
assert_equal(stats[0]['height'], self.start_height)
assert_equal(stats[self.max_stat_pos]['height'], self.start_height + self.max_stat_pos)
for i in range(self.max_stat_pos+1):
logging.info('Checking block %d\n' % (i))
assert_equal(stats[i], self.expected_stats[i])
# Check selecting block by hash too
blockhash = self.expected_stats[i]['blockhash']
stats_by_hash = self.nodes[0].getblockstats(hash_or_height=blockhash)
assert_equal(stats_by_hash, self.expected_stats[i])
# Check with the node that has no txindex
stats_no_txindex = self.nodes[1].getblockstats(hash_or_height=blockhash, stats=list(expected_stats_noindex[i].keys()))
assert_equal(stats_no_txindex, expected_stats_noindex[i])
# Make sure each stat can be queried on its own
for stat in expected_keys:
for i in range(self.max_stat_pos+1):
result = self.nodes[0].getblockstats(hash_or_height=self.start_height + i, stats=[stat])
assert_equal(list(result.keys()), [stat])
if result[stat] != self.expected_stats[i][stat]:
logging.info('result[%s] (%d) failed, %r != %r' % (
stat, i, result[stat], self.expected_stats[i][stat]))
assert_equal(result[stat], self.expected_stats[i][stat])
# Make sure only the selected statistics are included (more than one)
some_stats = {'minfee', 'maxfee'}
stats = self.nodes[0].getblockstats(hash_or_height=1, stats=list(some_stats))
assert_equal(set(stats.keys()), some_stats)
# Test invalid parameters raise the proper json exceptions
tip = self.start_height + self.max_stat_pos
assert_raises_jsonrpc(-8, 'Target block height %d after current tip %d' % (tip+1, tip),
self.nodes[0].getblockstats, hash_or_height=tip+1)
assert_raises_jsonrpc(-8, 'Target block height %d is negative' % (-1),
self.nodes[0].getblockstats, hash_or_height=-1)
# Make sure not valid stats aren't allowed
inv_sel_stat = 'asdfghjkl'
inv_stats = [
[inv_sel_stat],
['minfee' , inv_sel_stat],
[inv_sel_stat, 'minfee'],
['minfee', inv_sel_stat, 'maxfee'],
]
for inv_stat in inv_stats:
assert_raises_jsonrpc(-8, 'Invalid selected statistic %s' % inv_sel_stat,
self.nodes[0].getblockstats, hash_or_height=1, stats=inv_stat)
# Make sure we aren't always returning inv_sel_stat as the culprit stat
assert_raises_jsonrpc(-8, 'Invalid selected statistic aaa%s' % inv_sel_stat,
self.nodes[0].getblockstats, hash_or_height=1, stats=['minfee' , 'aaa%s' % inv_sel_stat])
assert_raises_jsonrpc(-8, 'One or more of the selected stats requires -txindex enabled',
self.nodes[1].getblockstats, hash_or_height=self.start_height + self.max_stat_pos)
# Mainchain's genesis block shouldn't be found on regtest
assert_raises_jsonrpc(-5, 'Block not found', self.nodes[0].getblockstats,
hash_or_height='000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f')
if __name__ == '__main__':
GetblockstatsTest().main()

View file

@ -25,6 +25,7 @@
#include <univalue.h>
#include <boost/algorithm/string.hpp>
#include <boost/thread/thread.hpp> // boost::thread::interrupt
#include <mutex>
@ -671,7 +672,26 @@ UniValue getblockheader(const JSONRPCRequest& request)
return blockheaderToJSON(pblockindex);
}
UniValue getblock(const JSONRPCRequest& request)
static CBlock GetBlockChecked(const CBlockIndex* pblockindex)
{
CBlock block;
if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) {
throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)");
}
if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) {
// Block not found on disk. This could be because we have the block
// header in our index but don't have the block (for example if a
// non-whitelisted node sends us an unrequested long chain of valid
// blocks, we add the headers to our index, but don't accept the
// block).
throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk");
}
return block;
}
static UniValue getblock(const JSONRPCRequest& request)
{
if (request.fHelp || request.params.size() < 1 || request.params.size() > 2)
throw std::runtime_error(
@ -738,18 +758,8 @@ UniValue getblock(const JSONRPCRequest& request)
throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found");
}
CBlock block;
CBlockIndex* pblockindex = mapBlockIndex[hash];
if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0)
throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)");
if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus()))
// Block not found on disk. This could be because we have the block
// header in our index but don't have the block (for example if a
// non-whitelisted node sends us an unrequested long chain of valid
// blocks, we add the headers to our index, but don't accept the
// block).
throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk");
const CBlock block = GetBlockChecked(pblockindex);
if (verbosity <= 0)
{
@ -1425,10 +1435,302 @@ UniValue reconsiderblock(const JSONRPCRequest& request)
return NullUniValue;
}
template<typename T>
static T CalculateTruncatedMedian(std::vector<T>& scores)
{
size_t size = scores.size();
if (size == 0) {
return 0;
}
std::sort(scores.begin(), scores.end());
if (size % 2 == 0) {
return (scores[size / 2 - 1] + scores[size / 2]) / 2;
} else {
return scores[size / 2];
}
}
template<typename T>
static inline bool SetHasKeys(const std::set<T>& set) {return false;}
template<typename T, typename Tk, typename... Args>
static inline bool SetHasKeys(const std::set<T>& set, const Tk& key, const Args&... args)
{
return (set.count(key) != 0) || SetHasKeys(set, args...);
}
// outpoint (needed for the utxo index) + nHeight + fCoinBase
static constexpr size_t PER_UTXO_OVERHEAD = sizeof(COutPoint) + sizeof(uint32_t) + sizeof(bool);
static UniValue getblockstats(const JSONRPCRequest& request)
{
if (request.fHelp || request.params.size() < 1 || request.params.size() > 4)
throw std::runtime_error(
"getblockstats ( hash_or_height stats )\n"
"\nCompute per block statistics for a given window. All amounts are in satoshis.\n"
"\nIt won't work for some heights with pruning.\n"
// "\nIt won't work without -txindex for utxo_size_inc, *fee or *feerate stats.\n"
"\nArguments:\n"
"1. \"hash_or_height\" (string or numeric, required) The block hash or height of the target block. If height, negative values count back from the current tip\n"
"2. \"stats\" (array, optional) Values to plot, by default all values, see result below)"
" ["
" \"height\", (string, optional) Selected statistic\n"
" \"time\", (string, optional) Selected statistic\n"
" ,...\n"
" ]\n"
"\nResult: (all values are in reverse order height-wise)\n"
"{ (json object)\n"
" \"avgfee\": xxxxx, (numeric) Average fee in the block\n"
" \"avgfeerate\": xxxxx, (numeric) Average feerate (in satoshis per virtual byte)\n"
" \"avgtxsize\": xxxxx, (numeric) Average transaction size\n"
" \"blockhash\": xxxxx, (string) The block hash (to check for potential reorgs)\n"
" \"height\": xxxxx, (numeric) The height of the block\n"
" \"ins\": xxxxx, (numeric) The number of inputs (excluding coinbase)\n"
" \"maxfee\": xxxxx, (numeric) Maximum fee in the block\n"
" \"maxfeerate\": xxxxx, (numeric) Maximum feerate (in satoshis per virtual byte)\n"
" \"maxtxsize\": xxxxx, (numeric) Maximum transaction size\n"
" \"medianfee\": xxxxx, (numeric) Truncated median fee in the block\n"
" \"medianfeerate\": xxxxx, (numeric) Truncated median feerate (in satoshis per virtual byte)\n"
" \"mediantime\": xxxxx, (numeric) The block median time past\n"
" \"mediantxsize\": xxxxx, (numeric) Truncated median transaction size\n"
" \"minfee\": xxxxx, (numeric) Minimum fee in the block\n"
" \"minfeerate\": xxxxx, (numeric) Minimum feerate (in satoshis per virtual byte)\n"
" \"mintxsize\": xxxxx, (numeric) Minimum transaction size\n"
" \"outs\": xxxxx, (numeric) The number of outputs\n"
" \"subsidy\": xxxxx, (numeric) The block subsidy\n"
" \"swtotal_size\": xxxxx, (numeric) Total size of all segwit transactions\n"
" \"swtotal_weight\": xxxxx, (numeric) Total weight of all segwit transactions divided by segwit scale factor (4)\n"
" \"swtxs\": xxxxx, (numeric) The number of segwit transactions\n"
" \"time\": xxxxx, (numeric) The block time\n"
" \"total_out\": xxxxx, (numeric) Total amount in all outputs (excluding coinbase and thus reward [ie subsidy + totalfee])\n"
" \"total_size\": xxxxx, (numeric) Total size of all non-coinbase transactions\n"
" \"total_weight\": xxxxx, (numeric) Total weight of all non-coinbase transactions divided by segwit scale factor (4)\n"
" \"totalfee\": xxxxx, (numeric) The fee total\n"
" \"txs\": xxxxx, (numeric) The number of transactions (excluding coinbase)\n"
" \"utxo_increase\": xxxxx, (numeric) The increase/decrease in the number of unspent outputs\n"
" \"utxo_size_inc\": xxxxx, (numeric) The increase/decrease in size for the utxo index (not discounting op_return and similar)\n"
"}\n"
"\nExamples:\n"
+ HelpExampleCli("getblockstats", "1000 '[\"minfeerate\",\"avgfeerate\"]'")
+ HelpExampleRpc("getblockstats", "1000 '[\"minfeerate\",\"avgfeerate\"]'")
);
LOCK(cs_main);
CBlockIndex* pindex;
if (request.params[0].isNum()) {
const int height = request.params[0].get_int();
const int current_tip = chainActive.Height();
if (height < 0) {
throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d is negative", height));
}
if (height > current_tip) {
throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d after current tip %d", height, current_tip));
}
pindex = chainActive[height];
} else {
const std::string strHash = request.params[0].get_str();
const uint256 hash(uint256S(strHash));
pindex = mapBlockIndex[hash];
}
if (!pindex) {
throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found");
}
if (!chainActive.Contains(pindex)) {
throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Block is not in chain %s", Params().NetworkIDString()));
}
assert(pindex != nullptr);
std::set<std::string> stats;
if (!request.params[1].isNull()) {
const UniValue stats_univalue = request.params[1].get_array();
for (unsigned int i = 0; i < stats_univalue.size(); i++) {
const std::string stat = stats_univalue[i].get_str();
stats.insert(stat);
}
}
const CAsset asset = policyAsset; // TODO Make configurable
const CBlock block = GetBlockChecked(pindex);
const bool do_all = stats.size() == 0; // Calculate everything if nothing selected (default)
const bool do_mediantxsize = do_all || stats.count("mediantxsize") != 0;
const bool do_medianfee = do_all || stats.count("medianfee") != 0;
const bool do_medianfeerate = do_all || stats.count("medianfeerate") != 0;
const bool loop_inputs = do_all || stats.count("utxo_size_inc");
const bool loop_outputs = do_all || do_medianfee || do_medianfeerate || loop_inputs ||
SetHasKeys(stats, "totalfee", "avgfee", "avgfeerate", "minfee", "maxfee", "minfeerate", "maxfeerate", "total_out");
const bool do_calculate_size = do_mediantxsize ||
SetHasKeys(stats, "total_size", "avgtxsize", "mintxsize", "maxtxsize", "swtotal_size");
const bool do_calculate_weight = do_all || SetHasKeys(stats, "total_weight", "avgfeerate", "swtotal_weight", "avgfeerate", "medianfeerate", "minfeerate", "maxfeerate");
const bool do_calculate_sw = do_all || SetHasKeys(stats, "swtxs", "swtotal_size", "swtotal_weight");
CAmount maxfee = 0;
CAmount maxfeerate = 0;
CAmount minfee = MAX_MONEY;
CAmount minfeerate = MAX_MONEY;
CAmount total_out = 0;
CAmount totalfee = 0;
int64_t inputs = 0;
int64_t maxtxsize = 0;
int64_t mintxsize = MAX_BLOCK_SERIALIZED_SIZE;
int64_t outputs = 0;
int64_t swtotal_size = 0;
int64_t swtotal_weight = 0;
int64_t swtxs = 0;
int64_t total_size = 0;
int64_t total_weight = 0;
int64_t utxo_size_inc = 0;
std::vector<CAmount> fee_array;
std::vector<CAmount> feerate_array;
std::vector<int64_t> txsize_array;
for (const auto& tx : block.vtx) {
outputs += tx->vout.size();
CAmount txfee = 0;
CAmount tx_total_out = 0;
if (loop_outputs) {
for (const CTxOut& out : tx->vout) {
utxo_size_inc += GetSerializeSize(out, SER_NETWORK, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD;
}
}
if (tx->IsCoinBase()) {
continue;
}
if (loop_outputs) {
for (const CTxOut& out : tx->vout) {
if (out.IsFee() && out.nAsset.GetAsset() == asset) {
txfee += out.nValue.GetAmount();
}
if (out.nValue.IsExplicit() && out.nAsset.IsExplicit() && out.nAsset.GetAsset() == asset) {
tx_total_out += out.nValue.GetAmount();
}
}
}
inputs += tx->vin.size(); // Don't count coinbase's fake input
total_out += tx_total_out; // Don't count coinbase reward
int64_t tx_size = 0;
if (do_calculate_size) {
tx_size = tx->GetTotalSize();
if (do_mediantxsize) {
txsize_array.push_back(tx_size);
}
maxtxsize = std::max(maxtxsize, tx_size);
mintxsize = std::min(mintxsize, tx_size);
total_size += tx_size;
}
int64_t weight = 0;
if (do_calculate_weight) {
weight = GetTransactionWeight(*tx);
total_weight += weight;
}
if (do_calculate_sw && tx->HasWitness()) {
++swtxs;
swtotal_size += tx_size;
swtotal_weight += weight;
}
if (loop_inputs) {
if (!fTxIndex) {
throw JSONRPCError(RPC_INVALID_PARAMETER, "One or more of the selected stats requires -txindex enabled");
}
for (const CTxIn& in : tx->vin) {
if (in.m_is_pegin) {
continue;
}
CTransactionRef tx_in;
uint256 hashBlock;
if (!GetTransaction(in.prevout.hash, tx_in, Params().GetConsensus(), hashBlock, false)) {
throw JSONRPCError(RPC_INTERNAL_ERROR, std::string("Unexpected internal error (tx index seems corrupt)"));
}
CTxOut prevoutput = tx_in->vout[in.prevout.n];
utxo_size_inc -= GetSerializeSize(prevoutput, SER_NETWORK, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD;
}
}
if (loop_outputs) {
assert(MoneyRange(txfee));
if (do_medianfee) {
fee_array.push_back(txfee);
}
maxfee = std::max(maxfee, txfee);
minfee = std::min(minfee, txfee);
totalfee += txfee;
// New feerate uses satoshis per virtual byte instead of per serialized byte
CAmount feerate = weight ? (txfee * WITNESS_SCALE_FACTOR) / weight : 0;
if (do_medianfeerate) {
feerate_array.push_back(feerate);
}
maxfeerate = std::max(maxfeerate, feerate);
minfeerate = std::min(minfeerate, feerate);
}
}
UniValue ret_all(UniValue::VOBJ);
ret_all.pushKV("avgfee", (block.vtx.size() > 1) ? totalfee / (block.vtx.size() - 1) : 0);
ret_all.pushKV("avgfeerate", total_weight ? (totalfee * WITNESS_SCALE_FACTOR) / total_weight : 0); // Unit: sat/vbyte
ret_all.pushKV("avgtxsize", (block.vtx.size() > 1) ? total_size / (block.vtx.size() - 1) : 0);
ret_all.pushKV("blockhash", pindex->GetBlockHash().GetHex());
ret_all.pushKV("height", (int64_t)pindex->nHeight);
ret_all.pushKV("ins", inputs);
ret_all.pushKV("maxfee", maxfee);
ret_all.pushKV("maxfeerate", maxfeerate);
ret_all.pushKV("maxtxsize", maxtxsize);
ret_all.pushKV("medianfee", CalculateTruncatedMedian(fee_array));
ret_all.pushKV("medianfeerate", CalculateTruncatedMedian(feerate_array));
ret_all.pushKV("mediantime", pindex->GetMedianTimePast());
ret_all.pushKV("mediantxsize", CalculateTruncatedMedian(txsize_array));
ret_all.pushKV("minfee", (minfee == MAX_MONEY) ? 0 : minfee);
ret_all.pushKV("minfeerate", (minfeerate == MAX_MONEY) ? 0 : minfeerate);
ret_all.pushKV("mintxsize", mintxsize == MAX_BLOCK_SERIALIZED_SIZE ? 0 : mintxsize);
ret_all.pushKV("outs", outputs);
ret_all.pushKV("subsidy", GetBlockSubsidy(pindex->nHeight, Params().GetConsensus()));
ret_all.pushKV("swtotal_size", swtotal_size);
ret_all.pushKV("swtotal_weight", swtotal_weight);
ret_all.pushKV("swtxs", swtxs);
ret_all.pushKV("time", pindex->GetBlockTime());
ret_all.pushKV("total_out", total_out);
ret_all.pushKV("total_size", total_size);
ret_all.pushKV("total_weight", total_weight);
ret_all.pushKV("totalfee", totalfee);
ret_all.pushKV("txs", (int64_t)block.vtx.size());
ret_all.pushKV("utxo_increase", outputs - inputs);
ret_all.pushKV("utxo_size_inc", utxo_size_inc);
if (do_all) {
return ret_all;
}
UniValue ret(UniValue::VOBJ);
for (const std::string& stat : stats) {
const UniValue& value = ret_all[stat];
if (value.isNull()) {
throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid selected statistic %s", stat));
}
ret.pushKV(stat, value);
}
return ret;
}
static const CRPCCommand commands[] =
{ // category name actor (function) okSafe argNames
// --------------------- ------------------------ ----------------------- ------ ----------
{ "blockchain", "getblockchaininfo", &getblockchaininfo, true, {} },
{ "blockchain", "getblockstats", &getblockstats, true, {"hash_or_height", "stats"} },
{ "blockchain", "getbestblockhash", &getbestblockhash, true, {} },
{ "blockchain", "getblockcount", &getblockcount, true, {} },
{ "blockchain", "getblock", &getblock, true, {"blockhash","verbosity"} },

View file

@ -117,6 +117,8 @@ static const CRPCConvertParam vRPCConvertParams[] =
{ "importmulti", 1, "options" },
{ "verifychain", 0, "checklevel" },
{ "verifychain", 1, "nblocks" },
{ "getblockstats", 0, "hash_or_height" },
{ "getblockstats", 1, "stats" },
{ "pruneblockchain", 0, "height" },
{ "keypoolrefill", 0, "newsize" },
{ "getrawmempool", 0, "verbose" },