diff --git a/BTC.h b/BTC.h index 4d2a095..d673a3b 100644 --- a/BTC.h +++ b/BTC.h @@ -392,13 +392,13 @@ namespace BTC /// already randomized. template struct GenericTrivialHashHasher { - std::size_t operator()(const BytesT &b) const { - if (LIKELY(size_t(b.size()) >= sizeof(size_t))) { + std::size_t operator()(const BytesT &b) const noexcept { + if (LIKELY(std::size_t(b.size()) >= sizeof(std::size_t))) { // common case, just return the first 8 bytes reinterpreted as size_t since this is already // a random hash. static_assert (std::is_scalar_v>, "GenericTrivialHasher must be used with a container type where .begin() returns a pointer to its data." ); - return *reinterpret_cast(b.begin()); + return *reinterpret_cast(b.begin()); } return qHash(b, 0xf1234567); // this should not normally be reached. } diff --git a/BlockProc.cpp b/BlockProc.cpp index 70ab240..67c8704 100644 --- a/BlockProc.cpp +++ b/BlockProc.cpp @@ -11,8 +11,7 @@ #include #include - -/*static*/ const QByteArray PreProcessedBlock::staticnull; +namespace BlockProcStatics { const TxHash nullhash; }; /// fill this struct's data with all the txdata, etc from a bitcoin CBlock. Alternative to using the second c'tor. void PreProcessedBlock::fill(unsigned blockHeight, size_t blockSize, const bitcoin::CBlock &b) { @@ -23,7 +22,6 @@ void PreProcessedBlock::fill(unsigned blockHeight, size_t blockSize, const bitco header = b.GetBlockHeader(); estimatedThisSizeBytes = sizeof(*this) + size_t(BTC::GetBlockHeaderSize()); txInfos.reserve(b.vtx.size()); - using HashHasher = BTC::QByteArrayHashHasher; std::unordered_map txHashToIndex; std::unordered_map, HashHasher> hashXOuts, hashXIns; std::unordered_set hashXsSeen; @@ -186,3 +184,23 @@ PreProcessedBlockPtr { return std::make_shared(height, size, block); } + + +// very much a work in progress. this needs to also consult the UTXO set to be complete. For now we just +// have this here for reference. +std::vector> +ProcessedBlock::hashXTouchedByTx() const +{ + std::vector> ret(txInfos.size()); + for (const auto & ag : hashXAggregated) { + // scan all outputs and add this hashX + for (const auto outIdx : ag.outs) { + ret[outputs[outIdx].txIdx].insert(ag.hashX); // cheap shallow copy + } + // scan all inputs and add this hashX + for (const auto inIdx : ag.ins) { + ret[inputs[inIdx].txIdx].insert(ag.hashX); + } + } + return ret; +} diff --git a/BlockProc.h b/BlockProc.h index b4ec20a..44aa0b1 100644 --- a/BlockProc.h +++ b/BlockProc.h @@ -1,7 +1,9 @@ #ifndef MY_BLOCKPROC_H #define MY_BLOCKPROC_H +#include "BTC.h" #include "HashX.h" +#include "TXO.h" #include "bitcoin/amount.h" #include "bitcoin/block.h" @@ -9,28 +11,41 @@ #include #include +#include #include #include +#include #include +using TxNum = std::uint64_t; +using BlockHeight = std::uint32_t; +using IONum = std::uint16_t; +using TxHash = QByteArray; + struct PreProcessedBlock; using PreProcessedBlockPtr = std::shared_ptr; ///< For clarity/convenience +using HashHasher = BTC::QByteArrayHashHasher; + +namespace BlockProcStatics +{ + extern const TxHash nullhash; +}; /// Note all hashes below are in *reversed* order from bitcoind's internal memory order. /// The reason for that is so that we have this PreProcessedBlock ready with the right format for putting into the db /// for later serving up to EX clients. struct PreProcessedBlock { - unsigned height = 0; ///< the height (block number) of the block + BlockHeight height = 0; ///< the height (block number) of the block size_t sizeBytes = 0; ///< the size of the original serialized block in bytes (not the size of this data structure which is significantly smaller) size_t estimatedThisSizeBytes = 0; ///< the estimated size of this data structure -- may be off by a bit but is useful for rough estimation of memory costs of block processing /// deserialized header as came in from bitcoind bitcoin::CBlockHeader header; struct TxInfo { - QByteArray hash; ///< 32 byte txid. These txid's are *reversed* from bitcoind's internal memory order. (so as to be closer to the final hex encoded format). - uint16_t nInputs = 0, nOutputs = 0; ///< the number of inputs and outputs in the tx -- all tx's are guaranteed to have <=65535 inputs or outputs currently and for the foreseeable future. If that changes, fixme. + TxHash hash; ///< 32 byte txid. These txid's are *reversed* from bitcoind's internal memory order. (so as to be closer to the final hex encoded format). + IONum nInputs = 0, nOutputs = 0; ///< the number of inputs and outputs in the tx -- all tx's are guaranteed to have <=65535 inputs or outputs currently and for the foreseeable future. If that changes, fixme. std::optional input0Index, output0Index; ///< if either of these have a value, they point into the `inputs` and `outputs` arrays below, respectively }; @@ -39,14 +54,14 @@ struct PreProcessedBlock struct InputPt { unsigned txIdx; ///< index into the `txInfos` vector above for the tx where this input appears - QByteArray prevoutHash; ///< 32-byte prevoutHash. In *reversed* memory order (hex-encoding ready!) (May be a shallow copy of a byte array in `txInfos` if the prevout tx was in this block.). May be empty if coinbase - uint16_t prevoutN; ///< the index in the prevout tx for this input (again, tx's can't have more than 65535 inputs -- if that changes, fixme!) + TxHash prevoutHash; ///< 32-byte prevoutHash. In *reversed* memory order (hex-encoding ready!) (May be a shallow copy of a byte array in `txInfos` if the prevout tx was in this block.). May be empty if coinbase + IONum prevoutN; ///< the index in the prevout tx for this input (again, tx's can't have more than 65535 inputs -- if that changes, fixme!) std::optional parentTxOutIdx; ///< if the input's prevout was in this block, the index into the `outputs` array declared below, otherwise undefined. }; struct OutPt { unsigned txIdx; ///< this is an index into the `txInfos` vector declared above - uint16_t outN; ///< this is an index into the tx's vout vector (*NOT* this class's `outputs`!) (again, tx's can't have more than 65535 inputs -- if that changes, fixme!) + IONum outN; ///< this is an index into the tx's vout vector (*NOT* this class's `outputs`!) (again, tx's can't have more than 65535 inputs -- if that changes, fixme!) bitcoin::Amount amount; }; @@ -83,11 +98,11 @@ struct PreProcessedBlock // c'tors, etc... note this class is trivially copyable, move constructible, etc etc PreProcessedBlock() = default; - PreProcessedBlock(unsigned blockHeight, size_t sizeBytes, const bitcoin::CBlock &b) { fill(blockHeight, sizeBytes, b); } + PreProcessedBlock(BlockHeight blockHeight, size_t sizeBytes, const bitcoin::CBlock &b) { fill(blockHeight, sizeBytes, b); } /// reset this to empty inline void clear() { *this = PreProcessedBlock(); } /// fill this block with data from bitcoin's CBlock - void fill(unsigned blockHeight, size_t sizeBytes, const bitcoin::CBlock &b); + void fill(BlockHeight blockHeight, size_t sizeBytes, const bitcoin::CBlock &b); /// convenience factory static method: given a block, return a shard_ptr instance of this struct PreProcessedBlockPtr static makeShared(unsigned height, size_t sizeBytes, const bitcoin::CBlock &block); @@ -96,33 +111,33 @@ struct PreProcessedBlock /// returns the input# as the input appeared in its tx, given a particular `inputs` array index /// We do it this way rather than store this information in the InputPt struct to save on memory - inline std::optional numForInputIdx(unsigned inputIdx) const { - std::optional ret; + inline std::optional numForInputIdx(unsigned inputIdx) const { + std::optional ret; if (inputIdx < inputs.size()) { if (const auto & opt = txInfos[inputs[inputIdx].txIdx].input0Index; opt.has_value() && inputIdx >= opt.value()) { const unsigned val = inputIdx - opt.value(); assert(val <= UINT16_MAX); // this should never happen -- all tx's are guaranteed to have <=65535 inputs or outputs currently and for the foreseeable future. If that changes, fixme. - ret.emplace( uint16_t(val) ); + ret.emplace( IONum(val) ); } } return ret; } /// returns the txHash given an index into the `inputs` array (or a null QByteArray if index is out of range). - inline const QByteArray &txHashForInputIdx(unsigned inputIdx) const { + inline const TxHash &txHashForInputIdx(unsigned inputIdx) const { if (inputIdx < inputs.size()) { if (const auto txIdx = inputs[inputIdx].txIdx; txIdx < txInfos.size()) return txInfos[txIdx].hash; } - return staticnull; + return BlockProcStatics::nullhash; } /// returns the txHash given an index into the `outputs` array (or a null QByteArray if index is out of range). - inline const QByteArray &txHashForOutputIdx(unsigned outputIdx) const { + inline const TxHash &txHashForOutputIdx(unsigned outputIdx) const { if (outputIdx < outputs.size()) { if (const auto txIdx = outputs[outputIdx].txIdx; txIdx < txInfos.size()) return txInfos[txIdx].hash; } - return staticnull; + return BlockProcStatics::nullhash; } /// Given an index into the `outputs` array, return a bool as to whether the output is a coinbase tx @@ -134,9 +149,105 @@ struct PreProcessedBlock /// debug string QString toDebugString() const; -private: - static const QByteArray staticnull; }; +struct ProcessedBlock; +using ProcessedBlockPtr = std::shared_ptr; ///< For clarity/convenience + +/// Similar to PreProcessedBlock above but with txNum and other indicesresolved to map to the global txNum +/// global txOutputNum, global txInputNum, etc +/// Note all hashes below are in *reversed* order from bitcoind's internal memory order. +struct ProcessedBlock +{ + BlockHeight height = 0; ///< the height (block number) of the block + size_t sizeBytes = 0; ///< the size of the original serialized block in bytes (not the size of this data structure which is significantly smaller) + size_t estimatedThisSizeBytes = 0; ///< the estimated size of this data structure -- may be off by a bit but is useful for rough estimation of memory costs of block processing + /// deserialized header as came in from bitcoind + bitcoin::CBlockHeader header; + + TxNum txNum0 = 0; ///< the global txNum for the first tx in this block. + + struct TxInfo { + TxHash hash; ///< 32 byte txid. These txid's are *reversed* from bitcoind's internal memory order. (so as to be closer to the final hex encoded format). + IONum nInputs = 0, nOutputs = 0; ///< the number of inputs and outputs in the tx -- all tx's are guaranteed to have <=65535 inputs or outputs currently and for the foreseeable future. If that changes, fixme. + std::optional input0Index, output0Index; ///< if either of these have a value, they point into the `inputs` and `outputs` arrays below, respectively + }; + + /// The info for all the tx's in the block, in the order in which they appeared in the block. + std::vector txInfos; + + /// convert to/from an index into the txInfos array above to a global txNum. + TxNum txIdx2Num(unsigned txIdx) const { return txNum0 + txIdx; } + unsigned txNum2Idx(TxNum n) const { + assert(n >= txNum0); + return unsigned(n - txNum0); + } + + struct InputPt { + unsigned txIdx; ///< index into the `txInfos` vector above for the tx where this input appears + TXO prevOut; //< prevoutTxNum:N, basically + TxNum prevTxNum; ///< the input's prevout txNum.. may be a tx in this block or in a previous block. + // Note we don't store the resolved HashX here.. see hashXAggregated below for that + }; + + struct OutPt { + unsigned txIdx; ///< this is an index into the `txInfos` vector declared above + uint16_t outN; ///< this is an index into the tx's vout vector (*NOT* this class's `outputs`!) (again, tx's can't have more than 65535 inputs -- if that changes, fixme!) + bitcoin::Amount amount; + // note we don't store the HashX here.. see hashXAggregated below for that + }; + + std::vector inputs; ///< all the inputs for *all* the tx's in this block, in the order they were encountered! + std::vector outputs; ///< all the outpoints for *all* the tx's in this block, in the order they were encountered! + + struct HashXAggregated { + /// The 32-byte hashX -- in "hex-encode-ready" memory order (that is, reversed). + HashX hashX; + /// collection of all outputs in this block that are *TO* this HashX (data items are indices into the `outputs` + /// array above) + std::vector outs; + /// collection of all inputs in this block that are *FROM* this HashX (data items are indices into the `inputs` + /// arrays above). (all inputs are resolved) + std::vector ins; + }; + + /// scriptHashes appearing in all of the outputs (and possibly inputs) in the txs in this block + std::vector hashXAggregated; + + /* + // If we decide to track OpReturn: + struct OpReturn { + unsigned outIdx; // index into the `outputs` vector declared above + bitcoin::CScript script; + }; + std::vector opreturns; + */ + // /OpReturn + // -- /End Data + + // -- Methods: + + // c'tors, etc... note this class is trivially copyable, move constructible, etc etc + ProcessedBlock() = default; + ProcessedBlock(const PreProcessedBlock &ppb, const UTXOSet &uset, TxNum txBaseNum) { + fill(ppb, uset, txBaseNum); + } + /// reset this to empty + inline void clear() { *this = ProcessedBlock(); } + /// fill this block with data from a pre-processed block plus the UTXO set and other data + void fill(const PreProcessedBlock &ppb, const UTXOSet &uset, TxNum txBaseNum); + + /// convenience factory static method: given a block, return a shard_ptr instance of this struct + ProcessedBlockPtr static makeShared(const PreProcessedBlock &ppb, const UTXOSet &uset, TxNum txBaseNum); + + + + /// WIP: Return a set of scripthashes (HashXs) for each txid. The size of this returned array is the same as + /// this->txInfos, and each position corresponds to the HashX's touched by that tx. + std::vector> hashXTouchedByTx() const; + + /// debug string + QString toDebugString() const; +}; #endif // MY_BLOCKPROC_H diff --git a/Common.h b/Common.h index 18f28d4..08d65e4 100644 --- a/Common.h +++ b/Common.h @@ -1,6 +1,7 @@ #ifndef COMMON_H #define COMMON_H +#include #include #include diff --git a/Fulcrum.pro b/Fulcrum.pro index 727c687..a803126 100644 --- a/Fulcrum.pro +++ b/Fulcrum.pro @@ -157,6 +157,7 @@ HEADERS += \ Servers.h \ SrvMgr.h \ Storage.h \ + TXO.h \ Util.h RESOURCES += \ diff --git a/TXO.h b/TXO.h new file mode 100644 index 0000000..be69bff --- /dev/null +++ b/TXO.h @@ -0,0 +1,52 @@ +#pragma once + +#include "HashX.h" + +#include "bitcoin/amount.h" + +#include +#include // for std::hash +#include +#include + +/// WIP +struct TXO { + // pack paranoia -- not strictly needed since this packs anyway the way we want on gcc and/or clang. +# ifdef __GNUC__ +# pragma pack(push, 1) +# endif + union { + struct { + /// tx index in the global tx table. First tx in blockchain is txNum 0, and so on (mapping of unique number to a txid hash, stored in db). + /// Note this is really a 48 bit value covering the range [0, 17592186044415] + std::uint64_t txNum : 48; + /// The the 'N' (output index) for the tx itself as per bitcoin tx format + std::uint16_t n; + } prevout; + std::uint64_t asU64; + } u; +# ifdef __GNUC__ +# pragma pack(pop) +# endif +}; + +namespace std { + // specialization of std::hash to be able to add struct UTXO to any unordered_set or unordered_map + template<> struct hash { + std::size_t operator()(const TXO &txo) const noexcept { + static_assert (sizeof(txo.u.asU64) <= sizeof(std::size_t) && sizeof(txo.u.prevout) >= 8 && sizeof(txo.u.prevout) == sizeof(txo.u.asU64), + "Possible non-64-bit platform detected or other struct packing weirdness on this compiler. Please compile with a 64-bit compiler."); + return txo.u.asU64; // just return the packed txNum:n value as this is guaranteed to be unique + } + }; +} + +/// WIP -- Spend info for a txo. +struct TXOInfo { + bitcoin::Amount amount; + HashX hashX; ///< the scripthash this output is sent to. Note in most cases this can be compactified to be a shallow-copy of existing data (such that dupes point to the same underlying data in eg UTXOSet). + std::optional confirmedHeight; ///< if unset, is mempool tx +}; + + +using UTXOSet = std::unordered_map; ///< TXO -> Info