mirror of
https://github.com/cculianu/Fulcrum.git
synced 2026-08-13 12:33:27 +02:00
409feb1994
Turns out our custom HashX class was doing some funny things when a HashX had a \0 byte. Likely QByteArray isn't really intended to be inherited from. Rather than investigate what was going on -- I just decided it's sufficient to do: using HashX = QByteArray and be done with it. Now our utxo set is working correctly. We still need to figure out how to store everything.. but this is progress.
352 lines
16 KiB
C++
352 lines
16 KiB
C++
#include "BlockProc.h"
|
|
#include "BTC.h"
|
|
#include "Util.h"
|
|
|
|
#include "bitcoin/transaction.h"
|
|
|
|
#include <QTextStream>
|
|
|
|
#include <algorithm>
|
|
#include <set>
|
|
#include <unordered_map>
|
|
#include <unordered_set>
|
|
|
|
/* static */ const TxHash BlockProcBase::nullhash;
|
|
|
|
/// fill this struct's data with all the txdata, etc from a bitcoin CBlock. Alternative to using the second c'tor.
|
|
void PreProcessedBlock::fill(BlockHeight blockHeight, size_t blockSize, const bitcoin::CBlock &b) {
|
|
if (!header.IsNull() || !txInfos.empty())
|
|
clear();
|
|
height = blockHeight;
|
|
sizeBytes = blockSize;
|
|
header = b.GetBlockHeader();
|
|
estimatedThisSizeBytes = sizeof(*this) + size_t(BTC::GetBlockHeaderSize());
|
|
txInfos.reserve(b.vtx.size());
|
|
std::unordered_map<TxHash, unsigned, HashHasher> txHashToIndex;
|
|
std::unordered_map<HashX, std::vector<unsigned>, HashHasher> hashXOuts, hashXIns;
|
|
std::unordered_set<HashX, HashHasher> hashXsSeen;
|
|
// run through all tx's, build inputs and outputs lists
|
|
size_t txIdx = 0;
|
|
for (const auto & tx : b.vtx) {
|
|
// copy tx hash data for the tx
|
|
TxInfo info;
|
|
info.hash = BTC::Hash2ByteArrayRev(tx->GetHash());
|
|
info.nInputs = IONum(tx->vin.size());
|
|
info.nOutputs = IONum(tx->vout.size());
|
|
// remember the tx hash -> index association for use later in this function
|
|
txHashToIndex[info.hash] = unsigned(txIdx); // cheap copy + cheap hash func. should make this fast.
|
|
|
|
// process outputs for this tx
|
|
if (!tx->vout.empty())
|
|
// remember output0 index for this txindex
|
|
info.output0Index.emplace( unsigned(outputs.size()) );
|
|
|
|
IONum outN = 0;
|
|
for (const auto & out : tx->vout) {
|
|
// save the outputs seen
|
|
outputs.emplace_back(
|
|
OutPt{ unsigned(txIdx), outN, out.nValue }
|
|
);
|
|
estimatedThisSizeBytes += sizeof(OutPt);
|
|
const size_t outputIdx = outputs.size()-1;
|
|
if (const auto cscript = out.scriptPubKey;
|
|
!BTC::IsOpReturn(cscript)) ///< skip OP_RETURN
|
|
{
|
|
const HashX hashX = BTC::HashXFromCScript(cscript);
|
|
// add this output to the hashX -> outputs association for later
|
|
hashXOuts[ hashX ].emplace_back( outputIdx );
|
|
hashXsSeen.insert(hashX);
|
|
}
|
|
else {
|
|
++nOpReturns;
|
|
}/*//use this clause if you want to actually save/process opreturn scripts:
|
|
else {
|
|
// OpReturn tracking...
|
|
opreturns.emplace_back(OpReturn{unsigned(outputIdx), cscript});
|
|
}*/
|
|
++outN;
|
|
}
|
|
|
|
// process inputs
|
|
if (!tx->vin.empty())
|
|
// remember input0Index position for this tx
|
|
info.input0Index.emplace( unsigned(inputs.size()) );
|
|
|
|
for (const auto & in : tx->vin) {
|
|
// note we do place the coinbase tx here even though we ignore it later on -- we keep it to have accurate indices
|
|
inputs.emplace_back(InputPt{
|
|
unsigned(txIdx),
|
|
BTC::Hash2ByteArrayRev(in.prevout.GetTxId()), // .prevoutHash
|
|
uint16_t(in.prevout.GetN()), // .prevoutN
|
|
{}, // .parentTxOutIdx (start out undefined)
|
|
});
|
|
estimatedThisSizeBytes += sizeof(InputPt);
|
|
}
|
|
estimatedThisSizeBytes += sizeof(info) + size_t(info.hash.size());
|
|
txInfos.emplace_back(std::move(info));
|
|
++txIdx;
|
|
}
|
|
|
|
// shrink inputs/outputs to fit now to conserve memory
|
|
inputs.shrink_to_fit();
|
|
outputs.shrink_to_fit();
|
|
|
|
// at this point we have a partially constructed object. we must run through all the inputs again
|
|
// and figure out which if any refer to tx's in this block, and assign those to our hashXIns.
|
|
// Also: to save memory on txhash's for such inputs, we make sure the txhash refers to the same underlying
|
|
// QByteArray data.
|
|
size_t inIdx = 0;
|
|
for (auto & inp : inputs) {
|
|
if (const auto it = txHashToIndex.find(inp.prevoutHash); it != txHashToIndex.end()) {
|
|
// this input refers to a tx in this block!
|
|
const auto txIdx = it->second;
|
|
assert(txIdx < txInfos.size() && txIdx < b.vtx.size());
|
|
const TxInfo & info = txInfos[txIdx];
|
|
inp.prevoutHash = info.hash; //<--- ensure shallow copy that points to same underlying data (saves memory)
|
|
if (info.output0Index.has_value())
|
|
inp.parentTxOutIdx.emplace( info.output0Index.value() + inp.prevoutN ); // save the index into the `outputs` array where the parent tx to this spend occurred
|
|
else { assert(0); }
|
|
const auto & tx = b.vtx[txIdx];
|
|
assert(inp.prevoutN < tx->vout.size());
|
|
if (const auto cscript = tx->vout[inp.prevoutN].scriptPubKey;
|
|
!BTC::IsOpReturn(cscript))
|
|
{
|
|
// mark this input as touching this hashX
|
|
const HashX hashX = BTC::HashXFromCScript(cscript);
|
|
hashXIns[ hashX ].emplace_back(inIdx);
|
|
hashXsSeen.insert(hashX);
|
|
}
|
|
}
|
|
++inIdx;
|
|
}
|
|
|
|
hashXAggregated.reserve(hashXsSeen.size());
|
|
for (const auto & hashX : hashXsSeen ) {
|
|
HashXAggregated ag;
|
|
ag.hashX = hashX;
|
|
if (auto it = hashXIns.find(hashX); it != hashXIns.end())
|
|
ag.ins.swap(it->second);
|
|
if (auto it = hashXOuts.find(hashX); it != hashXOuts.end())
|
|
ag.outs.swap(it->second);
|
|
|
|
ag.ins.shrink_to_fit();
|
|
ag.outs.shrink_to_fit();
|
|
estimatedThisSizeBytes += sizeof(ag) + size_t(ag.hashX.size()) + ag.ins.size() * sizeof(decltype(ag.ins)::value_type) + ag.outs.size() * sizeof(decltype(ag.outs)::value_type);
|
|
hashXAggregated.emplace_back(std::move(ag));
|
|
}
|
|
|
|
sortHashXAggregated(inputs);
|
|
}
|
|
|
|
QString PreProcessedBlock::toDebugString() const
|
|
{
|
|
QString ret;
|
|
{
|
|
QTextStream ts(&ret, QIODevice::ReadOnly|QIODevice::Truncate|QIODevice::Text);
|
|
ts << "<PreProcessedBlock --"
|
|
<< " height: " << height << " " << " size: " << sizeBytes << " header_nTime: " << header.nTime << " hash: " << header.GetHash().ToString().c_str()
|
|
<< " nTx: " << txInfos.size() << " nIns: " << inputs.size() << " nOuts: " << outputs.size() << " nScriptHash: " << hashXAggregated.size();
|
|
int i = 0;
|
|
for (const auto & ag : hashXAggregated) {
|
|
ts << " (#" << i << " - " << ag.hashX.toHex() << " - nIns: " << ag.ins.size() << " nOuts: " << ag.outs.size();
|
|
for (size_t j = 0; j < ag.ins.size(); ++j) {
|
|
const auto idx = ag.ins[j];
|
|
const auto & theInput [[maybe_unused]] = inputs[idx];
|
|
assert(theInput.parentTxOutIdx.has_value() && txHashForOutputIdx(theInput.parentTxOutIdx.value()) == theInput.prevoutHash);
|
|
ts << " {in# " << j << " - " << inputs[idx].prevoutHash.toHex() << ":" << inputs[idx].prevoutN
|
|
<< ", spent in " << txHashForInputIdx(inputs, idx).toHex() << ":" << numForInputIdx(inputs, idx).value_or(999999) << " }";
|
|
}
|
|
for (size_t j = 0; j < ag.outs.size(); ++j) {
|
|
const auto idx = ag.outs[j];
|
|
ts << " {out# " << j << " - " << txHashForOutputIdx(idx).toHex() << ":" << outputs[idx].outN << " amt: " << outputs[idx].amount.ToString().c_str() << " }";
|
|
}
|
|
ts << ")";
|
|
++i;
|
|
}
|
|
/*
|
|
ts << " opreturns: " << opreturns.size();
|
|
i = 0;
|
|
for (const auto & op : opreturns) {
|
|
ts << " (#" << i << " - " << txInfos[outputs[op.outIdx].txIdx].hash.toHex() << ")";
|
|
++i;
|
|
}*/
|
|
ts << " >";
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
/// convenience factory static method: given a block, return a shard_ptr instance of this struct
|
|
/*static*/
|
|
PreProcessedBlockPtr PreProcessedBlock::makeShared(unsigned height_, size_t size, const bitcoin::CBlock &block)
|
|
{
|
|
return std::make_shared<PreProcessedBlock>(height_, size, block);
|
|
}
|
|
|
|
/* ---- ProcessedBlock ---- */
|
|
ProcessedBlock::CannotResolveInputError::~CannotResolveInputError() {} // for weak vtable warning
|
|
|
|
|
|
/// convenience factory static method: given a block, return a shard_ptr instance of this struct
|
|
/*static*/
|
|
ProcessedBlockPtr ProcessedBlock::makeShared(TxNum txBaseNum, const PreProcessedBlock &ppb, const TxHash2NumResolver &resolverFunc, const UTXOSet &uset) // may throw
|
|
{
|
|
auto rawPtr = new ProcessedBlock(txBaseNum, ppb, resolverFunc, uset); // may throw
|
|
return ProcessedBlockPtr(rawPtr); // didn't throw, return shared_ptr
|
|
}
|
|
|
|
ProcessedBlock::ProcessedBlock(TxNum txBaseNum, const PreProcessedBlock &ppb, const TxHash2NumResolver &resolverFunc, const UTXOSet &uset)
|
|
: BlockProcBase(ppb), txNum0(txBaseNum)
|
|
{
|
|
inputs.reserve(ppb.inputs.size());
|
|
estimatedThisSizeBytes -= ppb.inputs.size() * sizeof(PreProcessedBlock::InputPt); // reduce the size estimate because we will recompute it below
|
|
std::unordered_map<HashX, std::optional<unsigned>, HashHasher> hashXRevMap; // index HashX -> its HashXAggregated index
|
|
|
|
unsigned i = 0;
|
|
// build hashX -> ag quick lookup map
|
|
for (auto & ag: hashXAggregated) {
|
|
hashXRevMap[ag.hashX] = i++;
|
|
}
|
|
|
|
// run through all of the inputs and resolve them to a HashX by consuling the utxo set and the resolverFunc
|
|
i = 0;
|
|
for (const auto & inp : ppb.inputs) {
|
|
TXO txo;
|
|
txo.u.prevout.n = inp.prevoutN;
|
|
if (inp.parentTxOutIdx.has_value()) {
|
|
assert(inp.parentTxOutIdx.value() < outputs.size());
|
|
// prevout was in this block, grab txNum quickly
|
|
const auto & prevout = outputs[inp.parentTxOutIdx.value()];
|
|
txo.u.prevout.txNum = txIdx2Num(prevout.txIdx);
|
|
// at this point we know the input in question was already pre-populated in the hashXAggregated structure,
|
|
} else if (i == 0) {
|
|
// coinbase input.. skip the scripthash stuff
|
|
txo = TXO(); // mark prevout as "invalid"
|
|
} else {
|
|
// prevout was not in this block, call the resolverFunc to figure out the 'txNum'
|
|
auto opt = resolverFunc(inp.prevoutHash);
|
|
if (UNLIKELY(!opt.has_value())) {
|
|
throw CannotResolveInputError(
|
|
QString("Unable to resolve prevoutHash %1 for input# %2 of txid %3")
|
|
.arg(QString(inp.prevoutHash.toHex()))
|
|
.arg(ppb.numForInputIdx(ppb.inputs, i).value_or(9999))
|
|
.arg(QString(ppb.txHashForInputIdx(ppb.inputs, i).toHex())));
|
|
}
|
|
txo.u.prevout.txNum = opt.value();
|
|
// look for utxo in utxo set to find the HashX and Amount.
|
|
if (auto it = uset.find(txo); UNLIKELY(it == uset.end())) {
|
|
throw CannotResolveInputError(
|
|
QString("Unable to resolve utxo %1:%2 in utxo set for input# %3 of txid %4")
|
|
.arg(QString(inp.prevoutHash.toHex()))
|
|
.arg(txo.u.prevout.n)
|
|
.arg(ppb.numForInputIdx(ppb.inputs, i).value_or(9999))
|
|
.arg(QString(ppb.txHashForInputIdx(ppb.inputs, i).toHex())));
|
|
} else {
|
|
// at this point we know the input in question was from a previous block so we need to populate the
|
|
// hashXAggregated structure now. Either by pushing an 'ins' to the end of an existing struct's
|
|
// vector, or creating a new
|
|
const auto & info = it->second;
|
|
auto & opt = hashXRevMap[info.hashX];
|
|
if (!opt.has_value()) {
|
|
// did not exist -- new hashX
|
|
hashXAggregated.emplace_back(HashXAggregated{
|
|
info.hashX, //
|
|
{}, // outs
|
|
{i}, // ins
|
|
});
|
|
opt = hashXAggregated.size()-1; // mark new hashX in aggregated structure
|
|
estimatedThisSizeBytes += sizeof(HashXAggregated);
|
|
} else {
|
|
hashXAggregated[opt.value()].ins.push_back(i);
|
|
}
|
|
estimatedThisSizeBytes += sizeof(i);
|
|
}
|
|
}
|
|
|
|
// now, push the input into our concrete class's inputs array
|
|
inputs.emplace_back(InputPt{
|
|
inp.txIdx, // .txIdx
|
|
txo, // .prevOut
|
|
});
|
|
|
|
estimatedThisSizeBytes += sizeof(InputPt);
|
|
++i;
|
|
}
|
|
|
|
// all inputs are pushed, we need to update the hashXAggregated structure by sorting it an making sure indices
|
|
// are unique
|
|
hashXAggregated.shrink_to_fit(); // shrink capacity -> size
|
|
for (auto & ag : hashXAggregated) {
|
|
// sort each ag structure again
|
|
std::sort(ag.ins.begin(), ag.ins.end());
|
|
// make sure inputs are unique
|
|
auto last = std::unique(ag.ins.begin(), ag.ins.end());
|
|
ag.ins.erase(last, ag.ins.end());
|
|
ag.ins.shrink_to_fit(); // just in case we grew its capacity too large
|
|
}
|
|
sortHashXAggregated(inputs); // sort by txid/output
|
|
}
|
|
|
|
// very much a work in progress. this needs to also consult the UTXO set to be complete. For now we just
|
|
// have this here for reference.
|
|
std::vector<std::unordered_set<HashX, HashHasher>>
|
|
ProcessedBlock::hashXTouchedByTx() const
|
|
{
|
|
std::vector<std::unordered_set<HashX, HashHasher>> ret(txInfos.size());
|
|
for (const auto & ag : hashXAggregated) {
|
|
// scan all outputs and add this hashX
|
|
for (const auto outIdx : ag.outs) {
|
|
ret[outputs[outIdx].txIdx].insert(ag.hashX); // cheap shallow copy
|
|
}
|
|
// scan all inputs and add this hashX
|
|
for (const auto inIdx : ag.ins) {
|
|
ret[inputs[inIdx].txIdx].insert(ag.hashX);
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
QString ProcessedBlock::toDebugString(const Num2TxHashResolver &resolver, const UTXOSet &uset) const
|
|
{
|
|
// TODO: implement...
|
|
QString ret;
|
|
{
|
|
QTextStream ts(&ret, QIODevice::ReadOnly|QIODevice::Truncate|QIODevice::Text);
|
|
ts << "<ProcessedBlock --"
|
|
<< " height: " << height << " " << " size: " << sizeBytes << " header_nTime: " << header.nTime << " hash: " << header.GetHash().ToString().c_str()
|
|
<< " nTx: " << txInfos.size() << " nIns: " << inputs.size() << " nOuts: " << outputs.size() << " nScriptHash: " << hashXAggregated.size();
|
|
int i = 0;
|
|
for (const auto & ag : hashXAggregated) {
|
|
ts << " (#" << i << " - " << ag.hashX.toHex() << " - nIns: " << ag.ins.size() << " nOuts: " << ag.outs.size();
|
|
for (size_t j = 0; j < ag.ins.size(); ++j) {
|
|
const auto idx = ag.ins[j];
|
|
const auto & theInput [[maybe_unused]] = inputs[idx];
|
|
const auto & inp = inputs[idx];
|
|
QByteArray h = resolver(inp.prevOut.txNum()).value_or("").toHex();
|
|
QString amt = "prevOutAmount: ???";
|
|
if (h.isEmpty()) {
|
|
h = QString("<NOTFOUND TxNum: %1>").arg(inp.prevOut.txNum()).toUtf8();
|
|
}
|
|
if (auto it = uset.find(inp.prevOut); it != uset.end()) {
|
|
amt = QString("prevOutAmount: %1 height: %2").arg(it->second.amount.ToString().c_str()).arg(int(it->second.confirmedHeight.value_or(0))-1);
|
|
}
|
|
ts << " {in# " << j << " - " << h << ":" << inp.prevOut.N()
|
|
<< ", spent in " << txHashForInputIdx(inputs, idx).toHex() << ":" << numForInputIdx(inputs, idx).value_or(999999) << " }";
|
|
}
|
|
for (size_t j = 0; j < ag.outs.size(); ++j) {
|
|
const auto idx = ag.outs[j];
|
|
ts << " {out# " << j << " - " << txHashForOutputIdx(idx).toHex() << ":" << outputs[idx].outN << " amt: " << outputs[idx].amount.ToString().c_str() << " }";
|
|
}
|
|
ts << ")";
|
|
++i;
|
|
}
|
|
/*
|
|
ts << " opreturns: " << opreturns.size();
|
|
i = 0;
|
|
for (const auto & op : opreturns) {
|
|
ts << " (#" << i << " - " << txInfos[outputs[op.outIdx].txIdx].hash.toHex() << ")";
|
|
++i;
|
|
}*/
|
|
ts << " >";
|
|
}
|
|
return ret;
|
|
}
|