diff --git a/BTC.h b/BTC.h index da1c6ac..04dfb50 100644 --- a/BTC.h +++ b/BTC.h @@ -362,6 +362,10 @@ namespace BTC return cs.size() > 0 && *cs.begin() == bitcoin::opcodetype::OP_RETURN; } + inline QByteArray HashXFromCScript(const bitcoin::CScript &cs) { + return QByteArray(BTC::HashRev(QByteArray::fromRawData(reinterpret_cast(cs.data()), int(cs.size())), true)); + } + /// Header Chain Verifier - /// To use: Basically keep calling operator() on it with subsequent headers and it will make sure /// hashPrevBlock of the current header matches the computed hash of the last header. diff --git a/BlockProc.cpp b/BlockProc.cpp index aa23dbc..6543422 100644 --- a/BlockProc.cpp +++ b/BlockProc.cpp @@ -41,7 +41,7 @@ void PreProcessedBlock::fill(BlockHeight blockHeight, size_t blockSize, const bi // remember output0 index for this txindex info.output0Index.emplace( unsigned(outputs.size()) ); - uint16_t outN = 0; + IONum outN = 0; for (const auto & out : tx->vout) { // save the outputs seen outputs.emplace_back( @@ -50,13 +50,17 @@ void PreProcessedBlock::fill(BlockHeight blockHeight, size_t blockSize, const bi estimatedThisSizeBytes += sizeof(OutPt); const size_t outputIdx = outputs.size()-1; if (const auto cscript = out.scriptPubKey; - cscript.size() && !BTC::IsOpReturn(cscript)) ///< skip OP_RETURN + !BTC::IsOpReturn(cscript)) ///< skip OP_RETURN { - const HashX hashX = cscript; + const HashX hashX = BTC::HashXFromCScript(cscript); // add this output to the hashX -> outputs association for later hashXOuts[ hashX ].emplace_back( outputIdx ); hashXsSeen.insert(hashX); - } /*else { + } + else { + ++nOpReturns; + }/*//use this clause if you want to actually save/process opreturn scripts: + else { // OpReturn tracking... opreturns.emplace_back(OpReturn{unsigned(outputIdx), cscript}); }*/ @@ -108,7 +112,7 @@ void PreProcessedBlock::fill(BlockHeight blockHeight, size_t blockSize, const bi !BTC::IsOpReturn(cscript)) { // mark this input as touching this hashX - const HashX hashX = cscript; + const HashX hashX = BTC::HashXFromCScript(cscript); hashXIns[ hashX ].emplace_back(inIdx); hashXsSeen.insert(hashX); } diff --git a/BlockProc.h b/BlockProc.h index 15cd6ac..25bbcdb 100644 --- a/BlockProc.h +++ b/BlockProc.h @@ -2,8 +2,8 @@ #define MY_BLOCKPROC_H #include "BTC.h" +#include "BlockProcTypes.h" #include "Common.h" -#include "HashX.h" #include "TXO.h" #include "bitcoin/amount.h" @@ -19,14 +19,6 @@ #include -using HashHasher = BTC::QByteArrayHashHasher; - -using TxNum = std::uint64_t; -using BlockHeight = std::uint32_t; -using IONum = std::uint16_t; -using TxHash = QByteArray; - - /// Note all hashes below are in *reversed* order from bitcoind's internal memory order. /// The reason for that is so that we have this PreProcessedBlock ready with the right format for putting into the db /// for later serving up to EX clients. @@ -81,6 +73,8 @@ struct BlockProcBase // /OpReturn // -- /End Data + unsigned nOpReturns = 0; ///< just keep a count of the number of opreturn outputs encountered in the block (used by sanity checkers) + // -- Methods: // misc helpers -- @@ -200,9 +194,6 @@ struct ProcessedBlock : public BlockProcBase /// -- Exception thrown by constructor if it cannot resolve inputs given the passed-in utxo set struct CannotResolveInputError : public Exception { using Exception::Exception; ~CannotResolveInputError(); }; - - using TxHash2NumResolver = std::function< std::optional(const TxHash &) >; - using Num2TxHashResolver = std::function< std::optional(TxNum) >; // -- Methods: /// the only c'tor -- note this may throw CannotResolveInputError diff --git a/BlockProcTypes.h b/BlockProcTypes.h new file mode 100644 index 0000000..cb47aba --- /dev/null +++ b/BlockProcTypes.h @@ -0,0 +1,20 @@ +#pragma once + +#include "BTC.h" // for BTC::QByteArrayHashHasher + +#include "bitcoin/amount.h" // for bitcoin::Amount + +#include + +#include + +using HashHasher = BTC::QByteArrayHashHasher; + +using TxNum = std::uint64_t; +using BlockHeight = std::uint32_t; +using IONum = std::uint16_t; +using TxHash = QByteArray; +using HashX = QByteArray; + +using TxHash2NumResolver = std::function< std::optional(const TxHash &) >; +using Num2TxHashResolver = std::function< std::optional(TxNum) >; diff --git a/Controller.cpp b/Controller.cpp index 0e82ef8..a014c20 100644 --- a/Controller.cpp +++ b/Controller.cpp @@ -21,6 +21,8 @@ Controller::~Controller() { Debug("%s", __FUNCTION__); cleanup(); } void Controller::startup() { + stopFlag = false; + storage = std::make_unique(options); storage->startup(); // may throw here @@ -91,6 +93,7 @@ void Controller::startup() void Controller::cleanup() { + stopFlag = true; stop(); tasks.clear(); // deletes all tasks asap if (srvmgr) { Log("Stopping SrvMgr ... "); srvmgr->cleanup(); srvmgr.reset(); } @@ -263,6 +266,7 @@ void DownloadBlocksTask::process() void DownloadBlocksTask::do_get(unsigned int bnum) { + if (ctl->isStopping()) return; // short-circuit early return if controller is stopping submitRequest("getblockhash", {bnum}, [this, bnum](const RPC::Message & resp){ QVariant var = resp.result(); const auto hash = Util::ParseHexFast(var.toByteArray()); @@ -429,6 +433,7 @@ CtlTaskT *Controller::newTask(bool connectErroredSignal, Args && ...args) void Controller::process(bool beSilentIfUpToDate) { + if (stopFlag) return; bool enablePollTimer = false; auto polltimeout = polltime_ms; stopTimer(pollTimerName); @@ -520,7 +525,7 @@ void Controller::putBlock(CtlTask *task, PreProcessedBlockPtr p) { // returns right away Util::AsyncOnObject(this, [this, task, p] { - if (!sm || isTaskDeleted(task) || sm->state == StateMachine::State::Failure) { + if (!sm || isTaskDeleted(task) || sm->state == StateMachine::State::Failure || stopFlag) { Debug() << "Ignoring block " << p->height << " for now-defunct task"; return; } else if (sm->state != StateMachine::State::DownloadingBlocks) { @@ -536,9 +541,9 @@ void Controller::putBlock(CtlTask *task, PreProcessedBlockPtr p) void Controller::process_DownloadingBlocks() { unsigned ct = 0; - for (auto it = sm->ppBlocks.find(sm->ppBlkHtNext); it != sm->ppBlocks.end(); it = sm->ppBlocks.find(sm->ppBlkHtNext)) { + for (auto it = sm->ppBlocks.find(sm->ppBlkHtNext); it != sm->ppBlocks.end() && !stopFlag; it = sm->ppBlocks.find(sm->ppBlkHtNext)) { auto ppb = it->second; - FatalAssert(ppb->height == sm->ppBlkHtNext) << "INTERNAL ERROR: Retrieved block has the wrong height! FIXME!"; // paranoia + assert(ppb->height == sm->ppBlkHtNext); // paranoia -- should never happen ++ct; ++sm->ppBlkHtNext; @@ -587,7 +592,9 @@ bool Controller::process_VerifyAndAddBlock(PreProcessedBlockPtr ppb) FatalAssert(rawHeader.size() == BTC::GetBlockHeaderSize()) << "INTERNAL ERROR: raw header has the wrong size!"; - { // UTXO set testing TESTING TESTING XXX + if constexpr (true) { // UTXO set testing TESTING TESTING XXX + static constexpr bool debugPrt = false; + const auto resolverFunc = [this](const TxHash &h) -> std::optional { std::optional ret; if (auto it = sm->txHash2NumMap.find(h); it != sm->txHash2NumMap.end()) { @@ -595,6 +602,7 @@ bool Controller::process_VerifyAndAddBlock(PreProcessedBlockPtr ppb) } return ret; }; + /* const auto revResolverFunc = [this](TxNum n) -> std::optional { std::optional ret; if (auto it = sm->num2TxHashMap.find(n); it != sm->num2TxHashMap.end()) { @@ -602,6 +610,7 @@ bool Controller::process_VerifyAndAddBlock(PreProcessedBlockPtr ppb) } return ret; }; + */ try { // add tx hash map auto pb = ProcessedBlock::makeShared(sm->txNumNext, *ppb, resolverFunc, sm->utxoset); @@ -613,9 +622,13 @@ bool Controller::process_VerifyAndAddBlock(PreProcessedBlockPtr ppb) } sm->txNumNext = i; + std::unordered_set outsSeen; // DEBUG REMOVE ME + // add outputs + for (const auto & ag : pb->hashXAggregated) { for (const auto oidx : ag.outs) { + outsSeen.insert(oidx); // DEBUG REMOVE ME const auto & out = pb->outputs[oidx]; TxNum num = pb->txIdx2Num(out.txIdx); TXOInfo info; @@ -624,11 +637,36 @@ bool Controller::process_VerifyAndAddBlock(PreProcessedBlockPtr ppb) info.confirmedHeight = pb->height; TXO txo(num, out.outN); sm->utxoset[txo] = info; - Debug() << "Added txo: " << txo.toString() - << " (txid: " << pb->txInfos[out.txIdx].hash.toHex() << " height: " << pb->height << ") " - << " amount: " << info.amount.ToString() << " for HashX: " << info.hashX.toHex(); + if constexpr (debugPrt) + Debug() << "Added txo: " << txo.toString() + << " (txid: " << pb->txInfos[out.txIdx].hash.toHex() << " height: " << pb->height << ") " + << " amount: " << info.amount.ToString() << " for HashX: " << info.hashX.toHex(); } } + // SANITY CHECK + if (auto totalOuts = outsSeen.size() + pb->nOpReturns; totalOuts != pb->outputs.size()) { + std::unordered_set missing; + for (unsigned i = 0; i < unsigned(pb->outputs.size()); ++i) + if (!outsSeen.count(i)) + missing.insert(i); + + QString mstr; + { + QTextStream ts(&mstr, QIODevice::WriteOnly|QIODevice::Truncate|QIODevice::Text); + for (auto idx : missing) { + const auto & out = pb->outputs[idx]; + auto txhash = pb->txInfos[out.txIdx].hash.toHex(); + auto N = out.outN; + TXO txo(pb->txIdx2Num(out.txIdx), N); + ts << "missing output #" << idx << " " << txhash << ":" << N << " (txo: " << txo.toString() << ")\n"; + } + } + + Fatal() << "block: " << pb->height << " nouts (" << totalOuts << ") != outputs.size (" << pb->outputs.size() << ") ppb outputs.size(): " << ppb->outputs.size() + << "\n" << mstr; + } + // /SANITY CHECK + // add spends (process inputs) unsigned inum = 0; for (const auto & in : pb->inputs) { @@ -636,20 +674,22 @@ bool Controller::process_VerifyAndAddBlock(PreProcessedBlockPtr ppb) if (pb->txInfos[in.txIdx].input0Index == inum && !in.prevOut.isValid()) { // coinbase.. skip } else if (const auto it = sm->utxoset.find(in.prevOut); it != sm->utxoset.end()) { - Debug() << "Spent " << it->first.toString() << " amount: " << it->second.amount.ToString() - << " in txid: " << dbgTxIdHex << " height: " << pb->height - << " input number: " << pb->numForInputIdx(pb->inputs, inum).value_or(0xffff) - << " HashX: " << it->second.hashX.toHex(); + if constexpr (debugPrt) + Debug() << "Spent " << it->first.toString() << " amount: " << it->second.amount.ToString() + << " in txid: " << dbgTxIdHex << " height: " << pb->height + << " input number: " << pb->numForInputIdx(pb->inputs, inum).value_or(0xffff) + << " HashX: " << it->second.hashX.toHex(); sm->utxoset.erase(it); } else { Debug() << "Failed to spend: " << in.prevOut.toString() << "(spending txid: " << dbgTxIdHex << ")"; } ++inum; } - Debug() << "utxoset size: " << sm->utxoset.size() << " block: " << pb->height; + if constexpr (debugPrt) + Debug() << "utxoset size: " << sm->utxoset.size() << " block: " << pb->height; //if (pb->height == 60000) // DEBUG TEST PRT - // pb->toDebugString(revResolverFunc, sm->utxoset); + // Debug() << pb->toDebugString(revResolverFunc, sm->utxoset); } catch (const Exception &e) { Fatal() << e.what(); } diff --git a/Controller.h b/Controller.h index 7b1c085..b0a4e7f 100644 --- a/Controller.h +++ b/Controller.h @@ -33,6 +33,8 @@ public: /// Note: we had to make this a public member but it's not really intended to be used from code outside this subsystem. void putBlock(CtlTask *sender, PreProcessedBlockPtr); + inline bool isStopping() const { return stopFlag; } + signals: /// Emitted whenever bitcoind is detected to be up-to-date, and everything is synched up. /// note this is not emitted during regular polling, but only after `synchronizing` was emitted previously. @@ -89,6 +91,8 @@ private: size_t nHeadersDownloadedSoFar() const; ///< not 100% accurate. call this only from this thread std::tuple nTxInOutSoFar() const; ///< not 100% accurate. call this only from this thread + + volatile bool stopFlag = false; }; /// Abstract base class for our private internal tasks. Concrete implementations are in Controller.cpp. diff --git a/Fulcrum.pro b/Fulcrum.pro index a803126..7dfa948 100644 --- a/Fulcrum.pro +++ b/Fulcrum.pro @@ -125,7 +125,6 @@ SOURCES += \ BitcoinD.cpp \ BlockProc.cpp \ Controller.cpp \ - HashX.cpp \ Mgr.cpp \ Mixins.cpp \ Options.cpp \ @@ -146,9 +145,9 @@ HEADERS += \ BTC.h \ BitcoinD.h \ BlockProc.h \ + BlockProcTypes.h \ Common.h \ Controller.h \ - HashX.h \ Logger.h \ Mgr.h \ Mixins.h \ diff --git a/HashX.cpp b/HashX.cpp deleted file mode 100644 index cddeba3..0000000 --- a/HashX.cpp +++ /dev/null @@ -1,30 +0,0 @@ -#include "BTC.h" -#include "HashX.h" -#include "Util.h" - -#include "bitcoin/uint256.h" - -#include - -// Note: fromRawData is a cheap copy (shallow copy pointing to the same data as cs), which is ok since -// it's just an rvalue temporary. -HashX::HashX(const bitcoin::CScript &cs) - : QByteArray(BTC::HashRev(QByteArray::fromRawData(reinterpret_cast(cs.data()), int(cs.size())), true)) -{ - assert(length() == bitcoin::uint256::width()); -} - -/*static*/ -HashX HashX::fromCScript(const bitcoin::CScript &cs) { return HashX(cs); } -/*static*/ -HashX HashX::fromHexFast(const QByteArray &definitelyHexData) { return HashX(Util::ParseHexFast(definitelyHexData)); } - -QByteArray HashX::toHex() const { return Util::ToHexFast(*this); } - -auto HashX::operator=(const bitcoin::CScript &cs) -> HashX & -{ - *this = HashX(cs); - return *this; -} - -bool HashX::operator==(const HashX &o) const { return QByteArray::operator==(o); } diff --git a/HashX.h b/HashX.h deleted file mode 100644 index 9d1a910..0000000 --- a/HashX.h +++ /dev/null @@ -1,57 +0,0 @@ -#ifndef HASHX_H -#define HASHX_H - -#include "bitcoin/script.h" - -#include - -#include - -/// "HashX" (ElectrumX/ElectronX style hash) which is a sha256 hash, done once, and "pre-reversed" -/// (that is, ready to be converted to hex for transmission directly as-is). -/// We inherit from QByteArray so as to benefit from its implicit sharing and other untility functions. -struct HashX : public QByteArray { - // inherit c'tors - using QByteArray::QByteArray; - /// construct from a CScript by taking the 32-byte *reversed* sha256 once of the CScript data. - HashX(const bitcoin::CScript &); - - /// This c'tor is needed to work around bugs on MinGW G++ - HashX(const HashX & other) : QByteArray(other) {} - /// Workaround for bugs in MinGW G++ - HashX(HashX && other) : QByteArray(std::move(other)) {} - - static HashX fromCScript(const bitcoin::CScript &); - /// Faster fromHex() which should be used only when you are sure the incoming data is definitely hex. - /// (calls Util::ParseHexFast()) - static HashX fromHexFast(const QByteArray &definitelyHexData); - - // shadows QByteArray::toHex(). We have a faster implementation we use (calls Util::ToHexFast()) - QByteArray toHex() const; - - HashX & operator=(const bitcoin::CScript &); - - /// Work around MinGW G++ bugs - HashX & operator=(const HashX &other) { QByteArray::operator=(other); return *this; } - HashX & operator=(HashX && other) { QByteArray::operator=(std::move(other)); return *this; } - - /// we had to explicitly redefine this because 'using' it created some compile errors about ambiguous - /// overloads (likely there are some private methods we inadvertently brought in?) - bool operator==(const HashX &o) const; - - - // inherit operators - using QByteArray::operator<; - using QByteArray::operator=; - using QByteArray::operator>; - using QByteArray::operator<=; - using QByteArray::operator>=; - using QByteArray::operator[]; - using QByteArray::operator!=; - - // remote + and += as they make no sense for us. - QByteArray & operator+=(const QByteArray &) = delete; - QByteArray & operator+(const QByteArray &) = delete; -}; - -#endif // HASHX_H diff --git a/TXO.h b/TXO.h index d0fd791..6d5eae9 100644 --- a/TXO.h +++ b/TXO.h @@ -1,8 +1,6 @@ #pragma once -#include "HashX.h" - -#include "bitcoin/amount.h" +#include "BlockProcTypes.h" #include @@ -11,9 +9,10 @@ #include #include + /// WIP struct TXO { - static constexpr std::uint64_t initval = ~0ULL; + static constexpr std::uint64_t initval = ~0ULL; ///< indicates !isValid() // pack paranoia -- not strictly needed since this packs anyway the way we want on gcc and/or clang. # ifdef __GNUC__ # pragma pack(push, 1) @@ -31,9 +30,14 @@ struct TXO { # ifdef __GNUC__ # pragma pack(pop) # endif + /// for most container types bool operator==(const TXO &o) const noexcept { return u.asU64 == o.u.asU64; } + /// for ordered sets + bool operator<(const TXO &o) const noexcept { return u.prevout.txNum < o.u.prevout.txNum && u.prevout.n < o.u.prevout.n; } + // convenience std::uint64_t txNum() const noexcept { return u.prevout.txNum; } + // convenience std::uint16_t N() const noexcept { return u.prevout.n; } TXO() = default; @@ -42,7 +46,7 @@ struct TXO { u.prevout.n = n; } TXO(const TXO &o) { u.asU64 = o.u.asU64; } - TXO &operator=(const TXO &o) { u.asU64 = o.u.asU64; return *this; } + TXO &operator=(const TXO &o) noexcept { u.asU64 = o.u.asU64; return *this; } bool isValid() const { return u.asU64 != initval; } @@ -50,7 +54,7 @@ struct TXO { }; namespace std { - // specialization of std::hash to be able to add struct UTXO to any unordered_set or unordered_map + /// specialization of std::hash to be able to add struct UTXO to any unordered_set or unordered_map template<> struct hash { std::size_t operator()(const TXO &txo) const noexcept { static_assert (sizeof(txo.u.asU64) <= sizeof(std::size_t) && sizeof(txo.u.prevout) >= 8 && sizeof(txo.u.prevout) == sizeof(txo.u.asU64),