From c3cbe6e203ed15ff51eea5b0b3757ae83da2ae41 Mon Sep 17 00:00:00 2001 From: Gregory Sanders Date: Wed, 25 Apr 2018 15:24:22 -0400 Subject: [PATCH] compact block RPC calls --- src/blockencodings.cpp | 12 ++- src/blockencodings.h | 3 +- src/miner.cpp | 1 + src/rpc/mining.cpp | 202 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 213 insertions(+), 5 deletions(-) diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 651cb80f1d..ab4c8029f4 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -45,7 +45,13 @@ uint64_t CBlockHeaderAndShortTxIDs::GetShortID(const uint256& txhash) const { return SipHashUint256(shorttxidk0, shorttxidk1, txhash) & 0xffffffffffffL; } - +std::vector PartiallyDownloadedBlock::GetAvailableTx() { + std::vector found_tx; + for (unsigned int i = 0; i < txn_available.size(); i++) { + if (txn_available[i]) found_tx.push_back(txn_available[i]); + } + return found_tx; +} ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& cmpctblock, const std::vector>& extra_txn) { if (cmpctblock.header.IsNull() || (cmpctblock.shorttxids.empty() && cmpctblock.prefilledtxn.empty())) @@ -175,7 +181,7 @@ bool PartiallyDownloadedBlock::IsTxAvailable(size_t index) const { return txn_available[index] ? true : false; } -ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector& vtx_missing) { +ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector& vtx_missing, bool check_pow) { assert(!header.IsNull()); uint256 hash = header.GetHash(); block = header; @@ -199,7 +205,7 @@ ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector< return READ_STATUS_INVALID; CValidationState state; - if (!CheckBlock(block, state, Params().GetConsensus())) { + if (!CheckBlock(block, state, Params().GetConsensus(), check_pow)) { // TODO: We really want to just check merkle tree manually here, // but that is expensive, and CheckBlock caches a block's // "checked-status" (in the CBlock?). CBlock should be able to diff --git a/src/blockencodings.h b/src/blockencodings.h index 5a1d80d421..7de491ac86 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -200,10 +200,11 @@ public: CBlockHeader header; PartiallyDownloadedBlock(CTxMemPool* poolIn) : pool(poolIn) {} + std::vector GetAvailableTx(); // extra_txn is a list of extra transactions to look at, in form ReadStatus InitData(const CBlockHeaderAndShortTxIDs& cmpctblock, const std::vector>& extra_txn); bool IsTxAvailable(size_t index) const; - ReadStatus FillBlock(CBlock& block, const std::vector& vtx_missing); + ReadStatus FillBlock(CBlock& block, const std::vector& vtx_missing, bool check_pow = true); }; #endif diff --git a/src/miner.cpp b/src/miner.cpp index fda5108189..1dc3ead687 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -6,6 +6,7 @@ #include "miner.h" #include "amount.h" +#include "blockencodings.h" #include "chain.h" #include "chainparams.h" #include "coins.h" diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index e212522f3a..98cc1dc526 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -23,6 +23,7 @@ #include "utilstrencodings.h" #include "validationinterface.h" #include "policy/policy.h" +#include "blockencodings.h" #ifdef ENABLE_WALLET #include "wallet/wallet.h" @@ -243,6 +244,202 @@ UniValue combineblocksigs(const JSONRPCRequest& request) return result; } +UniValue getcompactsketch(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 1) + throw runtime_error( + "getcompactsketch block_hex\n" + "\nGets hex representation of a proposed compact block sketch.\n" + "It is consumed by `consumecompactsketch.`\n" + "Arguments:\n" + "1. \"block_hex\" (string, required), Hex serialized block proposal from `getnewblockhex`.\n" + "\nResult\n" + "blockhex (hex) The block hex\n" + "\nExamples:\n" + + HelpExampleCli("getcompactsketch", "") + ); + + CBlock block; + std::vector block_bytes(ParseHex(request.params[0].get_str())); + CDataStream ssBlock(block_bytes, SER_NETWORK, PROTOCOL_VERSION); + ssBlock >> block; + + CBlockHeaderAndShortTxIDs cmpctblock(block, true); + + CDataStream ssCompactBlock(SER_NETWORK, PROTOCOL_VERSION); + ssCompactBlock << cmpctblock; + return HexStr(ssCompactBlock.begin(), ssCompactBlock.end()); + +} + +UniValue consumecompactsketch(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 1) + throw runtime_error( + "consumecompactsketch sketch\n" + "\nTakes hex representation of a proposed compact block sketch and fills it in\n" + "using mempool. Returns the block if complete, and a list\n" + "of missing transaction indices serialized as a native structure." + "NOTE: The latest instance of this call will have a partially filled block\n" + "cached in memory to be used in `consumegetblocktxn` to finalize the block.\n" + "Arguments:\n" + "1. \"sketch\" (string, required), Hex string of compact block sketch.\n" + "\nResult\n" + "{\n" + " blockhex (hex) The filled block hex. Only returns when block is final\n" + " block_tx_req (hex) The serialized structure of missing transaction indices, given to serving node\n" + " found_transactions (hex) The serialized list of found transactions to be used in finalizecompactblock\n" + "}\n" + "\nExamples:\n" + + HelpExampleCli("consumecompactsketch", "") + ); + + UniValue ret(UniValue::VOBJ); + + std::vector compact_block_bytes(ParseHex(request.params[0].get_str())); + CDataStream ssBlock(compact_block_bytes, SER_NETWORK, PROTOCOL_VERSION); + CBlockHeaderAndShortTxIDs cmpctblock; + ssBlock >> cmpctblock; + + PartiallyDownloadedBlock partialBlock(&mempool); + const std::vector> dummy; + ReadStatus status = partialBlock.InitData(cmpctblock, dummy); + if (status != READ_STATUS_OK) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Compact block decode failed"); + } + + BlockTransactionsRequest req; + std::vector found(partialBlock.GetAvailableTx()); + for (size_t i = 0; i < cmpctblock.BlockTxCount(); i++) { + if (!partialBlock.IsTxAvailable(i)) { + req.indexes.push_back(i); + } + } + + CDataStream ssReq(SER_NETWORK, PROTOCOL_VERSION); + ssReq << req; + + CDataStream ssFound(SER_NETWORK, PROTOCOL_VERSION); + ssFound << found; + + if (req.indexes.empty()) { + std::shared_ptr pblock = std::make_shared(); + std::vector dummy; + ReadStatus status = partialBlock.FillBlock(*pblock, dummy, false /* don't get pow */); + if (status == READ_STATUS_INVALID) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Bogus crap sketch."); + } else if (status == READ_STATUS_FAILED) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Failed to complete block though all transactions were apparently found. Could be random short ID collision; requires full block instead."); + } else if (status == READ_STATUS_CHECKBLOCK_FAILED) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Checkblock failed."); + } + CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); + ssBlock << *pblock; + + ret.pushKV("blockhex", HexStr(ssBlock.begin(), ssBlock.end())); + } else { + ret.pushKV("block_tx_req", HexStr(ssReq.begin(), ssReq.end())); + ret.pushKV("found_transactions", HexStr(ssFound.begin(), ssFound.end())); + } + return ret; +} + +UniValue consumegetblocktxn(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 2) + throw runtime_error( + "consumegetblocktxn full_block block_tx_req\n" + "Consumes a transaction request for a compact block sketch." + "Arguments:\n" + "1. \"full_block\" (string, required), Hex serialied block that corresponds to the block request `block_tx_req`.\n" + "2. \"block_tx_req\" (string, required), Hex serialied BlockTransactionsRequest, aka getblocktxn network message.\n" + "\nResult\n" + "block_transactions (hex) The serialized list of found transactions aka BlockTransactions\n" + "\nExamples:\n" + + HelpExampleCli("consumegetblocktxn", "") + ); + + + CBlock block; + std::vector block_bytes(ParseHex(request.params[0].get_str())); + CDataStream ssBlock(block_bytes, SER_NETWORK, PROTOCOL_VERSION); + ssBlock >> block; + + // Take in BlockTransactionsRequest, return BlockTransactions + std::vector block_req(ParseHex(request.params[1].get_str())); + CDataStream ssReq(block_req, SER_NETWORK, PROTOCOL_VERSION); + + BlockTransactionsRequest req; + ssReq >> req; + + BlockTransactions resp(req); + for (size_t i = 0; i < req.indexes.size(); i++) { + if (req.indexes[i] >= block.vtx.size()) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Peer sent us a getblocktxn with out-of-bounds tx indices"); + } + resp.txn[i] = block.vtx[req.indexes[i]]; + } + + CDataStream ssResp(SER_NETWORK, PROTOCOL_VERSION); + ssResp << resp; + + return HexStr(ssResp.begin(), ssResp.end()); +} + +UniValue finalizecompactblock(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 3) + throw runtime_error( + "finalizecompactblock compact_hex block_transactions found_transactions\n" + "Takes the two transaction lists, fills out the compact block and attempts to validate it." + "Arguments:\n" + "1. \"compact_hex\" (string, required), Hex serialized compact block.\n" + "2. \"block_transactions\" (string, required), Hex serialized BlockTransactions, the response to getblocktxn.\n" + "3. \"found_transactions\" (string, required), Hex serialized list of transactions that were found in response to receiving a compact sketch in `consumecompactsketch`.\n" + "\nResult\n" + "block (hex) The serialized final block.\n" + "\nExamples:\n" + + HelpExampleCli("finalizecompactblock", " ") + ); + + // Compact block + std::vector compact_block_bytes(ParseHex(request.params[0].get_str())); + CDataStream ssCompactBlock(compact_block_bytes, SER_NETWORK, PROTOCOL_VERSION); + CBlockHeaderAndShortTxIDs cmpctblock; + ssCompactBlock >> cmpctblock; + + // BlockTransactions from the server + std::vector block_tx(ParseHex(request.params[1].get_str())); + CDataStream ssResp(block_tx, SER_NETWORK, PROTOCOL_VERSION); + + BlockTransactions transactions; + ssResp >> transactions; + + // Cached transactions + std::vector found_tx(ParseHex(request.params[2].get_str())); + CDataStream ssFound(block_tx, SER_NETWORK, PROTOCOL_VERSION); + + std::vector found; + ssFound >> found; + + // Make mega-list + found.insert(found.end(), transactions.txn.begin(), transactions.txn.end()); + + // Now construct the final block! + PartiallyDownloadedBlock partialBlock(&mempool); + + const std::vector> dummy; + std::shared_ptr pblock = std::make_shared(); + if (partialBlock.InitData(cmpctblock, dummy) != READ_STATUS_OK || partialBlock.FillBlock(*pblock, found, false /* pow_check*/) != READ_STATUS_OK) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Failed to complete block though all transactions were apparently found. Could be random short ID collision; requires full block instead."); + } + + CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); + ssBlock << *pblock; + + return HexStr(ssBlock.begin(), ssBlock.end()); +} + UniValue getmininginfo(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) @@ -1020,7 +1217,10 @@ static const CRPCCommand commands[] = { "generating", "generate", &generate, true, {"nblocks","maxtries"} }, { "generating", "combineblocksigs", &combineblocksigs, true, {"blockhex","signatures"} }, { "generating", "getnewblockhex", &getnewblockhex, true, {"required_age"} }, - + { "generating", "getcompactsketch", &getcompactsketch, true, {"block_hex"} }, + { "generating", "consumecompactsketch", &consumecompactsketch, true, {"sketch"} }, + { "generating", "consumegetblocktxn", &consumegetblocktxn, true, {"full_block", "block_tx_req"} }, + { "generating", "finalizecompactblock", &finalizecompactblock, true, {"compact_hex","block_transactions","found_transactions"} }, { "util", "estimatefee", &estimatefee, true, {"nblocks"} }, { "util", "estimatepriority", &estimatepriority, true, {"nblocks"} }, { "util", "estimatesmartfee", &estimatesmartfee, true, {"nblocks"} },