mirror of
https://github.com/cculianu/Fulcrum.git
synced 2026-08-13 12:33:27 +02:00
310e0acc20
And also did some more cleanup in gentlyCloseDBs(), and put the DRAs into the right place (inside p->db which is now where they belong since it's DB-related).
5205 lines
270 KiB
C++
5205 lines
270 KiB
C++
//
|
|
// Fulcrum - A fast & nimble SPV Server for Bitcoin Cash
|
|
// Copyright (C) 2019-2025 Calin A. Culianu <calin.culianu@gmail.com>
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program (see LICENSE.txt). If not, see
|
|
// <https://www.gnu.org/licenses/>.
|
|
//
|
|
#include "App.h"
|
|
#include "BTC.h"
|
|
#include "ByteView.h"
|
|
#include "CostCache.h"
|
|
#include "CoTask.h"
|
|
#include "Mempool.h"
|
|
#include "Merkle.h"
|
|
#include "Rpa.h"
|
|
#include "Span.h"
|
|
#include "Storage.h"
|
|
#include "Storage/ConcatOperator.h"
|
|
#include "Storage/DBRecordArray.h"
|
|
#include "SubsMgr.h"
|
|
#include "VarInt.h"
|
|
|
|
#include "bitcoin/crypto/endian.h"
|
|
#include "bitcoin/hash.h"
|
|
|
|
#include "robin_hood/robin_hood.h"
|
|
|
|
#if __has_include(<rocksdb/advanced_cache.h>)
|
|
// Newer rocksdb 8.1 defines the `Cache` class in this header. :/
|
|
#include <rocksdb/advanced_cache.h>
|
|
#endif
|
|
#include <rocksdb/cache.h>
|
|
#include <rocksdb/db.h>
|
|
#include <rocksdb/iterator.h>
|
|
#include <rocksdb/merge_operator.h>
|
|
#include <rocksdb/options.h>
|
|
#include <rocksdb/slice.h>
|
|
#include <rocksdb/table.h>
|
|
#include <rocksdb/version.h>
|
|
#include <rocksdb/write_buffer_manager.h>
|
|
|
|
#include <QByteArray>
|
|
#include <QDir>
|
|
#include <QFileInfo>
|
|
#include <QSysInfo>
|
|
#include <QVector> // we use this for the Height2Hash cache to save on memcopies since it's implicitly shared.
|
|
|
|
#include <algorithm>
|
|
#include <atomic>
|
|
#include <cassert>
|
|
#include <cstddef> // for std::byte, offsetof, ptrdiff_t
|
|
#include <cstdlib>
|
|
#include <cstring> // for memcpy
|
|
#include <functional>
|
|
#include <limits>
|
|
#include <map>
|
|
#include <mutex>
|
|
#include <optional>
|
|
#include <set>
|
|
#include <string>
|
|
#include <tuple>
|
|
#include <type_traits>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
DatabaseError::~DatabaseError(){} // weak vtable warning suppression
|
|
DatabaseSerializationError::~DatabaseSerializationError() {} // weak vtable warning suppression
|
|
DatabaseFormatError::~DatabaseFormatError() {} // weak vtable warning suppression
|
|
DatabaseKeyNotFound::~DatabaseKeyNotFound() {} // weak vtable warning suppression
|
|
HeaderVerificationFailure::~HeaderVerificationFailure() {} // weak vtable warning suppression
|
|
UndoInfoMissing::~UndoInfoMissing() {} // weak vtable warning suppression
|
|
HistoryTooLarge::~HistoryTooLarge() {} // weak vtable warning suppression
|
|
|
|
namespace {
|
|
/// Encapsulates the 'meta' db table
|
|
struct Meta {
|
|
static constexpr uint32_t kCurrentVersion = 0x3u;
|
|
static constexpr uint32_t kMinSupportedVersion = 0x1u;
|
|
static constexpr uint32_t kMinBCHUpgrade9Version = 0x2u;
|
|
static constexpr uint32_t kMinHasExtraPlatformInfoVersion = 0x3u;
|
|
|
|
static constexpr uint32_t kMagic = 0xf33db33fu;
|
|
static constexpr uint16_t kPlatformBits = sizeof(void *)*8U;
|
|
|
|
uint32_t magic = kMagic, version = kCurrentVersion;
|
|
QString chain; ///< "test", "main", etc
|
|
/// We save the platform pointer size to the db. Previous to v3 (kMinHasExtraPlatformInfoVersion), this field
|
|
/// was unreliable between Windows & Linux and should be ignored for versions < kMinHasExtraPlatformInfoVersion.
|
|
uint16_t platformBits = kPlatformBits;
|
|
|
|
// -- New in 1.3.0 (this field is not in older db's)
|
|
/// "BCH", "BTC", or "". May be missing in DB data for older db's, in which case we take the default ("BCH")
|
|
/// when we deserialize, if we detect that it was missing.
|
|
///
|
|
/// On uninitialized, newly-created DB's this is present but empty "". The fact that it is empty allows us
|
|
/// to auto-detect the Coin in question in Controller.
|
|
QString coin = QString();
|
|
|
|
/// -- New in 1.11.0
|
|
/// These fields only are valid in v3 or above (.version >= kMinHasExtraPlatformInfoVersion)
|
|
/// These fields get re-saved to the DB from the current program's info each time (along with `platformBits`
|
|
/// above) in function: Storage::checkUpgradeDBVersion().
|
|
QString appName; // "Fulcrum"
|
|
QString appVersion; // e.g. "1.11.0 (Release d884cb4)"
|
|
QString rocksDBVersion; // e.g. "9.2.1-08f9322"
|
|
QString buildABI; // ABI used at build-time, e.g. "x86_64-little_endian-lp64"
|
|
QString osName; // OS name e.g. "macOS 14.5"
|
|
QString cpuArch; // CPU arch e.g. "x86_64"
|
|
|
|
// C'tor used *not for deser*; object constructed with good values
|
|
Meta() { makePlatformInfoCurrent(); }
|
|
|
|
// C'tor used for deser; a cleared object is constructed
|
|
struct ClearedForUnser_t {};
|
|
static inline constexpr ClearedForUnser_t ClearedForUnser{};
|
|
explicit Meta(ClearedForUnser_t) : magic{0}, version{0}, platformBits{0} {}
|
|
|
|
bool isVersionSupported() const { return version >= kMinSupportedVersion && version <= kCurrentVersion; }
|
|
bool isMagicOk() const { return magic == kMagic; }
|
|
bool isMinimumExtraPlatformInfoVersion() const { return version >= kMinHasExtraPlatformInfoVersion; }
|
|
|
|
// Set this instance's platform info to correspond to the current process's valid info.
|
|
void makePlatformInfoCurrent();
|
|
};
|
|
|
|
void Meta::makePlatformInfoCurrent() {
|
|
platformBits = kPlatformBits;
|
|
if (const auto *app = App::globalInstance()) {
|
|
appName = app->applicationName();
|
|
appVersion = app->applicationVersion();
|
|
}
|
|
rocksDBVersion = Storage::rocksdbVersion();
|
|
buildABI = QSysInfo::buildAbi();
|
|
osName = QSysInfo::prettyProductName();
|
|
cpuArch = QSysInfo::currentCpuArchitecture();
|
|
}
|
|
|
|
// some database keys we use -- todo: if this grows large, move it elsewhere
|
|
static const bool falseMem = false, trueMem = true;
|
|
static const rocksdb::Slice kMeta{"meta"}, kDirty{"dirty"}, kUtxoCount{"utxo_count"}, kRpaNeedsFullCheck{"rpa_needs_full_check"},
|
|
kTrue(reinterpret_cast<const char *>(&trueMem), sizeof(trueMem)),
|
|
kFalse(reinterpret_cast<const char *>(&falseMem), sizeof(falseMem));
|
|
|
|
// serialize/deser -- for basic types we use QDataStream, but we also have specializations at the end of this file
|
|
template <typename Type>
|
|
QByteArray Serialize(const Type & n) {
|
|
QByteArray ba;
|
|
if constexpr (std::is_base_of_v<QByteArray, Type>) {
|
|
ba = n;
|
|
} else {
|
|
QDataStream ds(&ba, QIODevice::WriteOnly|QIODevice::Truncate);
|
|
ds << n;
|
|
}
|
|
return ba;
|
|
}
|
|
template <typename Type>
|
|
Type Deserialize(const QByteArray &ba, bool *ok = nullptr) {
|
|
Type ret{};
|
|
if constexpr (std::is_base_of_v<QByteArray, Type>) {
|
|
ret = ba;
|
|
if (ok) *ok = true;
|
|
} else {
|
|
QDataStream ds(ba);
|
|
ds >> ret;
|
|
if (ok)
|
|
*ok = ds.status() == QDataStream::Status::Ok;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
template <typename T>
|
|
concept NonPointer = !std::is_pointer_v<std::remove_cv_t<T>>;
|
|
|
|
template <typename T>
|
|
concept TrivCopObj = std::is_trivially_copyable_v<T> && NonPointer<T>;
|
|
|
|
template <typename T>
|
|
concept Scalar = std::is_scalar_v<T> && NonPointer<T>;
|
|
|
|
/// Return a shallow, temporary copy of the memory of an object as a QByteArray. This reduces typing of
|
|
/// the boilerplate: "QByteArray::fromRawData(reinterpret_cast...." etc everywhere in this file.
|
|
/// Note: It is unsafe to use this function for anything other than obtaining a weak reference to the memory of an
|
|
/// object as a QByteArray for temporary purposes. The original object must live at least as long as this returned
|
|
/// QByteArray. Note that even copy-constructing a new QByteArray from this returned QByteArray will lead to
|
|
/// dangling pointers. See: https://doc.qt.io/qt-5/qbytearray.html#fromRawData.
|
|
template <TrivCopObj Object>
|
|
QByteArray ShallowTmp(const Object *mem, size_t size = sizeof(Object)) {
|
|
return QByteArray::fromRawData(reinterpret_cast<const char *>(mem), static_cast<QByteArray::size_type>(size));
|
|
}
|
|
|
|
/// Construct a QByteArray from a deep copy of any object's memory area. Slower than ShallowTmp above but 100% safe
|
|
/// to use after the original object expires since the returned QByteArray takes ownership of its private copy of
|
|
/// the memory it allocated.
|
|
template <TrivCopObj Object>
|
|
QByteArray DeepCpy(const Object *mem, size_t size = sizeof(Object)) {
|
|
return QByteArray(reinterpret_cast<const char *>(mem), static_cast<QByteArray::size_type>(size));
|
|
}
|
|
|
|
/// Serialize a simple value such as an int directly, without using the space overhead that QDataStream imposes.
|
|
/// This is less safe but is more compact since the bytes of the passed-in value are written directly to the
|
|
/// returned QByteArray, without any encapsulation. Note that use of this mechanism makes all data in the database
|
|
/// no longer platform-neutral, which is ok. The presumption is users can re-synch their DB if switching
|
|
/// architectures.
|
|
template <Scalar S> QByteArray SerializeScalar (const S & s) { return DeepCpy(&s); }
|
|
template <Scalar S> QByteArray SerializeScalarNoCopy (const S &s) { return ShallowTmp(&s); }
|
|
/// Inverse of above. Pass in an optional 'pos' pointer if you wish to continue reading raw scalars from the same
|
|
/// QByteArray during subsequent calls to this template function. *ok, if specified, is set to false if we ran off
|
|
/// the QByteArray's bounds, and a default-constructed value of 'Scalar' is returned. No other safety checking is
|
|
/// done. On successful deserialization of the scalar, *pos (if specified) is updated to point just past the
|
|
/// last byte of the successuflly converted item. On failure, *pos is always set to point past the end of the
|
|
/// QByteArray.
|
|
template <Scalar S>
|
|
S DeserializeScalar(const QByteArray &ba, bool *ok = nullptr, int *pos_out = nullptr) {
|
|
S ret{};
|
|
int dummy = 0;
|
|
int & pos = pos_out ? *pos_out : dummy;
|
|
if (pos >= 0 && pos + int(sizeof(ret)) <= ba.size()) {
|
|
if (ok) *ok = true;
|
|
std::memcpy(reinterpret_cast<std::byte *>(&ret), ba.constData() + pos, sizeof(ret));
|
|
pos += sizeof(ret);
|
|
} else {
|
|
if (ok) *ok = false;
|
|
pos = ba.size();
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
struct SHUnspentValue {
|
|
bool valid = false;
|
|
bitcoin::Amount amount;
|
|
bitcoin::token::OutputDataPtr tokenDataPtr;
|
|
};
|
|
|
|
// Ensures we store RPA db keys in big endian for faster scans of adjacent heights
|
|
struct RpaDBKey {
|
|
uint32_t height;
|
|
|
|
explicit RpaDBKey(uint32_t h) : height(h) {}
|
|
|
|
QByteArray toBytes() const {
|
|
const uint32_t bigEndian = htobe32(height); // swap to big endian
|
|
return QByteArray(reinterpret_cast<const char *>(&bigEndian), sizeof(bigEndian));
|
|
}
|
|
|
|
static RpaDBKey fromBytes(const QByteArray &ba, bool *ok = nullptr, bool strictSize = false) {
|
|
RpaDBKey k{0u};
|
|
if (size_t(ba.size()) < sizeof(uint32_t) || (strictSize && size_t(ba.size()) != sizeof(uint32_t))) {
|
|
if (ok) *ok = false;
|
|
return k;
|
|
}
|
|
uint32_t bigEndian;
|
|
std::memcpy(&bigEndian, ba.constData(), sizeof(uint32_t));
|
|
k.height = be32toh(bigEndian); // swap to host order
|
|
if (ok) *ok = true;
|
|
return k;
|
|
}
|
|
|
|
bool operator==(const RpaDBKey &o) const { return height == o.height; }
|
|
bool operator!=(const RpaDBKey &o) const { return ! this->operator==(o); }
|
|
};
|
|
|
|
// specializations
|
|
template <> QByteArray Serialize(const Meta &);
|
|
template <> Meta Deserialize(const QByteArray &, bool *);
|
|
template <> QByteArray Serialize(const TXO &);
|
|
template <> TXO Deserialize(const QByteArray &, bool *);
|
|
template <> QByteArray Serialize(const TXOInfo &);
|
|
template <> TXOInfo Deserialize(const QByteArray &, bool *);
|
|
template <> Rpa::PrefixTable Deserialize(const QByteArray &, bool *);
|
|
template <> QByteArray Serialize(const RpaDBKey &k) { return k.toBytes(); }
|
|
template <> RpaDBKey Deserialize(const QByteArray &ba, bool *ok) { return RpaDBKey::fromBytes(ba, ok); }
|
|
QByteArray Serialize2(const bitcoin::Amount &, const bitcoin::token::OutputData *);
|
|
template <> SHUnspentValue Deserialize(const QByteArray &, bool *);
|
|
// TxNumVec
|
|
using TxNumVec = std::vector<TxNum>;
|
|
// this serializes a vector of TxNums to a compact representation (6 bytes, eg 48 bits per TxNum), in little endian byte order
|
|
template <> QByteArray Serialize(const TxNumVec &);
|
|
// this deserializes a vector of TxNums from a compact representation (6 bytes, eg 48 bits per TxNum), assuming little endian byte order
|
|
template <> TxNumVec Deserialize(const QByteArray &, bool *);
|
|
|
|
// CompactTXO -- not currently used since we prefer toBytes() directly (TODO: remove if we end up never using this)
|
|
//template <> QByteArray Serialize(const CompactTXO &);
|
|
template <> CompactTXO Deserialize(const QByteArray &, bool *);
|
|
|
|
|
|
/// NOTE: The slice should live as long as the returned QByteArray does. The QByteArray is a weak pointer into the slice!
|
|
inline QByteArray FromSlice(const rocksdb::Slice &s) { return ShallowTmp(s.data(), s.size()); }
|
|
|
|
/// Generic conversion from any type we operate on to a rocksdb::Slice. Note that the type in question should have
|
|
/// a conversion function written (eg Serialize) if it is anything other than a QByteArray or a scalar.
|
|
template<bool safeScalar=false, typename Thing>
|
|
auto ToSlice(const Thing &thing) {
|
|
if constexpr (std::is_base_of_v<rocksdb::Slice, Thing>) {
|
|
// same type, no-op, return ref to thing (const Slice &)
|
|
return static_cast<const rocksdb::Slice &>(thing);
|
|
} else if constexpr (std::is_base_of_v<QByteArray, Thing>) {
|
|
// QByteArray conversion, return reference to data in QByteArray
|
|
return rocksdb::Slice(thing.constData(), size_t(thing.size()));
|
|
} else if constexpr (std::is_same_v<ByteView, Thing>) {
|
|
// ByteView conversion, return reference to data in ByteView
|
|
return rocksdb::Slice(thing.charData(), thing.size());
|
|
} else if constexpr (!safeScalar && std::is_scalar_v<Thing> && !std::is_pointer_v<Thing>) {
|
|
return rocksdb::Slice(reinterpret_cast<const char *>(&thing), sizeof(thing)); // returned slice points to raw scalar memory itself
|
|
} else {
|
|
// the purpose of this holder is to keep the temporary QByteArray alive for as long as the slice itself is alive
|
|
struct BagOfHolding {
|
|
QByteArray bytes;
|
|
rocksdb::Slice slice;
|
|
operator const rocksdb::Slice &() const { return slice; }
|
|
} h { Serialize(thing), ToSlice(h.bytes) };
|
|
return h; // this holder type "acts like" a Slice due to its operator const Slice &()
|
|
}
|
|
};
|
|
|
|
/// Helper to get a column family name. `cf` may be nullptr
|
|
QString CFName(const rocksdb::ColumnFamilyHandle *cf) { return QString::fromStdString(cf ? cf->GetName() : std::string{"unk"}); }
|
|
|
|
/// Helper to get db name (basename of path)
|
|
QString DBName(const rocksdb::DB *db, const rocksdb::ColumnFamilyHandle *cf = nullptr) {
|
|
const auto dbname = QFileInfo(QString::fromStdString(db ? db->GetName() : "???")).baseName();
|
|
if (!cf) return dbname;
|
|
return QString("%1 (cf: %2)").arg(dbname, CFName(cf));
|
|
}
|
|
/// Helper to just get the status error string as a QString
|
|
QString StatusString(const rocksdb::Status & status) { return QString::fromStdString(status.ToString()); }
|
|
|
|
/// DB read/write helpers
|
|
/// NOTE: these may throw DatabaseError
|
|
/// If missingOk=false, then the returned optional is guaranteed to have a value if this function returns without throwing.
|
|
/// If missingOk=true, then if there was no other database error and the key was not found, the returned optional !has_value()
|
|
///
|
|
/// Template arg "safeScalar", if true, will deserialize scalar int, float, etc data using the Deserialize<>
|
|
/// function (uses QDataStream, is platform neutral, but is slightly slower). If false, we will use the
|
|
/// DeserializeScalar<> fast function for scalars such as ints. It's important to read from the DB in the same
|
|
/// 'safeScalar' mode as was written!
|
|
template <typename RetType, bool safeScalar = false, typename KeyType>
|
|
std::optional<RetType> GenericDBGet(rocksdb::DB *db, rocksdb::ColumnFamilyHandle *cf,
|
|
const KeyType & keyIn, bool missingOk = false,
|
|
const QString & errorMsgPrefix = QString(), ///< used to specify a custom error message in the thrown exception
|
|
bool acceptExtraBytesAtEndOfData = false,
|
|
const rocksdb::ReadOptions & ropts = rocksdb::ReadOptions()) ///< if true, we are ok with extra unparsed bytes in data. otherwise we throw. (this check is only done for !safeScalar mode on basic types)
|
|
{
|
|
rocksdb::PinnableSlice datum;
|
|
std::optional<RetType> ret;
|
|
if (!db || !cf) [[unlikely]] throw InternalError("GenericDBGet was passed a null pointer!");
|
|
const auto status = db->Get(ropts, cf, ToSlice<safeScalar>(keyIn), &datum);
|
|
if (status.IsNotFound()) {
|
|
if (missingOk)
|
|
return ret; // optional will not has_value() to indicate missing key
|
|
throw DatabaseKeyNotFound(QString("%1: %2")
|
|
.arg(!errorMsgPrefix.isEmpty() ? errorMsgPrefix : QString("Key not found in db %1").arg(DBName(db, cf)))
|
|
.arg(StatusString(status)));
|
|
} else if (!status.ok()) {
|
|
throw DatabaseError(QString("%1: %2")
|
|
.arg(!errorMsgPrefix.isEmpty() ? errorMsgPrefix : QString("Error reading a key from db %1").arg(DBName(db, cf)))
|
|
.arg(StatusString(status)));
|
|
} else {
|
|
// ok status
|
|
if constexpr (std::is_base_of_v<QByteArray, std::remove_cv_t<RetType> >) {
|
|
static_assert (!safeScalar, "safeScalar=true mode is not supported for QByteArrays (it only is useful for scalar types)" );
|
|
// special compile-time case for QByteArray subclasses -- return a deep copy of the data bytes directly.
|
|
// TODO: figure out a way to do this without the 1 extra copy! (PinnableSlice -> ret).
|
|
ret.emplace( reinterpret_cast<const char *>(datum.data()), QByteArray::size_type(datum.size()) );
|
|
} else if constexpr (std::is_same_v<rocksdb::PinnableSlice, std::remove_cv_t<RetType>>) {
|
|
static_assert (!std::is_same_v<rocksdb::PinnableSlice, std::remove_cv_t<RetType>>,
|
|
"FIXME: rocksdb C++ is broken. This doesn't actually work.");
|
|
ret.emplace(std::move(datum)); // avoids an extra copy -- but it doesn't work because Facebook doesn't get how C++ works.
|
|
} else if constexpr (!safeScalar && std::is_scalar_v<RetType> && !std::is_pointer_v<RetType>) {
|
|
if (!acceptExtraBytesAtEndOfData && datum.size() > sizeof(RetType)) {
|
|
// reject extra stuff at end of data stream
|
|
throw DatabaseFormatError(QString("%1: Extra bytes at the end of data")
|
|
.arg(!errorMsgPrefix.isEmpty() ? errorMsgPrefix : QString("Database format error in db %1").arg(DBName(db, cf))));
|
|
}
|
|
bool ok{};
|
|
ret.emplace( DeserializeScalar<RetType>(FromSlice(datum), &ok) );
|
|
if (!ok) {
|
|
throw DatabaseSerializationError(
|
|
QString("%1: Key was retrieved ok, but data could not be deserialized as a scalar '%2'")
|
|
.arg((!errorMsgPrefix.isEmpty() ? errorMsgPrefix : QString("Error deserializing a scalar from db %1").arg(DBName(db, cf))),
|
|
QString(typeid (RetType).name())));
|
|
}
|
|
} else {
|
|
if (UNLIKELY(acceptExtraBytesAtEndOfData))
|
|
Debug() << "Warning: Caller misuse of function '" << __func__
|
|
<< "'. 'acceptExtraBytesAtEndOfData=true' is ignored when deserializing using QDataStream.";
|
|
bool ok{};
|
|
ret.emplace( Deserialize<RetType>(FromSlice(datum), &ok) );
|
|
if (!ok) {
|
|
throw DatabaseSerializationError(
|
|
QString("%1: Key was retrieved ok, but data could not be deserialized")
|
|
.arg(!errorMsgPrefix.isEmpty() ? errorMsgPrefix : QString("Error deserializing an object from db %1").arg(DBName(db, cf))));
|
|
}
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
/// Conveneience for above with the missingOk flag set to false. Will always throw or return a real value.
|
|
template <typename RetType, bool safeScalar = false, typename KeyType>
|
|
RetType GenericDBGetFailIfMissing(rocksdb::DB * db, rocksdb::ColumnFamilyHandle * cf,
|
|
const KeyType &k, const QString &errMsgPrefix = QString(), bool extraDataOk = false,
|
|
const rocksdb::ReadOptions & ropts = rocksdb::ReadOptions())
|
|
{
|
|
return GenericDBGet<RetType, safeScalar>(db, cf, k, false, errMsgPrefix, extraDataOk, ropts).value();
|
|
}
|
|
|
|
/// Throws on all errors. Otherwise writes to db.
|
|
template <bool safeScalar = false, typename KeyType, typename ValueType>
|
|
void GenericDBPut
|
|
(rocksdb::DB *db, rocksdb::ColumnFamilyHandle *cf, const KeyType & key, const ValueType & value,
|
|
const QString & errorMsgPrefix = QString(), ///< used to specify a custom error message in the thrown exception
|
|
const rocksdb::WriteOptions & opts = rocksdb::WriteOptions())
|
|
{
|
|
auto st = db->Put(opts, cf, ToSlice<safeScalar>(key), ToSlice<safeScalar>(value));
|
|
if (!st.ok())
|
|
throw DatabaseError(QString("%1: %2").arg(
|
|
(!errorMsgPrefix.isEmpty() ? errorMsgPrefix : QString("Error writing to db %1").arg(DBName(db, cf))),
|
|
StatusString(st)));
|
|
}
|
|
/// Throws on all errors. Otherwise enqueues a write to the batch.
|
|
template <bool safeScalar = false, typename KeyType, typename ValueType>
|
|
void GenericBatchPut
|
|
(rocksdb::WriteBatch & batch, rocksdb::ColumnFamilyHandle *cf, const KeyType & key, const ValueType & value,
|
|
const QString & errorMsgPrefix = QString()) ///< used to specify a custom error message in the thrown exception
|
|
{
|
|
auto st = batch.Put(cf, ToSlice<safeScalar>(key), ToSlice<safeScalar>(value));
|
|
if (!st.ok())
|
|
throw DatabaseError(QString("%1 (cf: %3): %2")
|
|
.arg((!errorMsgPrefix.isEmpty() ? errorMsgPrefix : "Error from WriteBatch::Put"),
|
|
StatusString(st), CFName(cf)));
|
|
}
|
|
/// Throws on all errors. Otherwise enqueues a delete to the batch.
|
|
template <bool safeScalar = false, typename KeyType>
|
|
void GenericBatchDelete
|
|
(rocksdb::WriteBatch & batch, rocksdb::ColumnFamilyHandle *cf, const KeyType & key,
|
|
const QString & errorMsgPrefix = QString()) ///< used to specify a custom error message in the thrown exception
|
|
{
|
|
auto st = batch.Delete(cf, ToSlice<safeScalar>(key));
|
|
if (!st.ok())
|
|
throw DatabaseError(QString("%1 (cf: %3): %2")
|
|
.arg((!errorMsgPrefix.isEmpty() ? errorMsgPrefix : "Error from WriteBatch::Delete"),
|
|
StatusString(st), CFName(cf)));
|
|
}
|
|
/// Throws on all errors. Otherwise deletes a key from db. It is not an error to delete a non-existing key.
|
|
template <bool safeScalar = false, typename KeyType>
|
|
void GenericDBDelete
|
|
(rocksdb::DB *db, rocksdb::ColumnFamilyHandle *cf, const KeyType & key,
|
|
const QString & errorMsgPrefix = QString(), ///< used to specify a custom error message in the thrown exception
|
|
const rocksdb::WriteOptions & opts = rocksdb::WriteOptions())
|
|
{
|
|
auto st = db->Delete(opts, cf, ToSlice<safeScalar>(key));
|
|
if (!st.ok())
|
|
throw DatabaseError(QString("%1: %2")
|
|
.arg((!errorMsgPrefix.isEmpty() ? errorMsgPrefix
|
|
: QString("Error deleting a key from db %1").arg(DBName(db, cf))),
|
|
StatusString(st)));
|
|
}
|
|
|
|
//// A helper data struct -- written to the blkinfo table. This helps localize a txnum to a specific position in
|
|
/// a block. The table is keyed off of block_height(uint32_t) -> serialized BlkInfo (raw bytes)
|
|
struct BlkInfo {
|
|
TxNum txNum0 = 0;
|
|
uint32_t nTx = 0;
|
|
BlkInfo() = default;
|
|
BlkInfo(const BlkInfo &) = default;
|
|
[[maybe_unused]] BlkInfo (TxNum txn, uint32_t ntx) : txNum0(txn), nTx(ntx) {}
|
|
bool operator==(const BlkInfo &o) const { return txNum0 == o.txNum0 && nTx == o.nTx; }
|
|
bool operator!=(const BlkInfo &o) const { return !(*this == o); }
|
|
[[maybe_unused]] bool operator<(const BlkInfo &o) const { return txNum0 == o.txNum0 ? nTx < o.nTx : txNum0 < o.txNum0; }
|
|
BlkInfo &operator=(const BlkInfo &) = default;
|
|
};
|
|
// serializes as raw bytes from struct
|
|
template <> QByteArray Serialize(const BlkInfo &);
|
|
// deserializes as raw bytes from struct
|
|
template <> BlkInfo Deserialize(const QByteArray &, bool *);
|
|
|
|
/// Block rewind/undo information. One of these is kept around in the db for the last configuredUndoDepth() blocks.
|
|
/// It basically stores a record of all the UTXO's added and removed, as well as the set of
|
|
/// scripthashes.
|
|
struct UndoInfo {
|
|
using ScriptHashSet = std::unordered_set<HashX, HashHasher>;
|
|
using UTXOAddUndo = std::tuple<TXO, HashX, CompactTXO>;
|
|
using UTXODelUndo = std::tuple<TXO, TXOInfo>;
|
|
|
|
BlockHeight height = 0; ///< we save a copy of this infomation as a sanity check
|
|
BlockHash hash; ///< we save a copy of this information as a sanity check. (bytes are in "reversed", bitcoind ToHex()-style memory order)
|
|
BlkInfo blkInfo; ///< we save a copy of this from the global value for convenience and as a sanity check.
|
|
|
|
// below is the actual critical undo information
|
|
ScriptHashSet scriptHashes;
|
|
std::vector<UTXOAddUndo> addUndos;
|
|
std::vector<UTXODelUndo> delUndos;
|
|
|
|
uint16_t deserVersion = 0u; ///< Only ever read-in from db, never written out (we write out the latest version always)
|
|
|
|
[[maybe_unused]] QString toDebugString() const;
|
|
|
|
[[maybe_unused]] bool operator==(const UndoInfo &) const; // for debug ser/deser
|
|
|
|
bool isValid() const { return hash.size() == HashLen; } ///< cheap, imperfect check for validity
|
|
void clear() { height = 0; hash.clear(); blkInfo = BlkInfo(); scriptHashes.clear(); addUndos.clear(); delUndos.clear(); deserVersion = 0u; }
|
|
};
|
|
|
|
QString UndoInfo::toDebugString() const {
|
|
QString ret;
|
|
QTextStream ts(&ret);
|
|
ts << "<Undo info for height: " << height << " addUndos: " << addUndos.size() << " delUndos: " << delUndos.size()
|
|
<< " scriptHashes: " << scriptHashes.size() << " nTx: " << blkInfo.nTx << " txNum0: " << blkInfo.txNum0
|
|
<< " hash: " << hash.toHex() << " deserVersion: " << int(deserVersion) << ">";
|
|
return ret;
|
|
}
|
|
|
|
bool UndoInfo::operator==(const UndoInfo &o) const {
|
|
return height == o.height && hash == o.hash && blkInfo == o.blkInfo && scriptHashes == o.scriptHashes
|
|
&& addUndos == o.addUndos && delUndos == o.delUndos;
|
|
}
|
|
|
|
// serialize as raw bytes mostly (no QDataStream)
|
|
template <> QByteArray Serialize(const UndoInfo &);
|
|
// serialize from raw bytes mostly (no QDataStream)
|
|
template <> UndoInfo Deserialize(const QByteArray &, bool *);
|
|
|
|
/// Thrown if user hits Ctrl-C / app gets a signal while we run the slow db checks
|
|
struct UserInterrupted : public Exception { using Exception::Exception; ~UserInterrupted() override; };
|
|
UserInterrupted::~UserInterrupted() {} // weak vtable warning suppression
|
|
|
|
/// Manages the txhash2txnum rocksdb table. The schema is:
|
|
/// Key: N bytes from POS position from the big-endian ordered (JSON ordered) txhash (default 6 from the End)
|
|
/// Value: One or more serialized VarInts. Each VarInt represents a "TxNum" (which tells us where the actual hash
|
|
/// lives in the txnum2txhash DBRecordArray).
|
|
///
|
|
/// This class is mainly a thin wrapper around the rocksdb and DBRecordArray facilities and they are both
|
|
/// thread-safe and reentrant. It takes no locks itself.
|
|
class TxHash2TxNumMgr {
|
|
rocksdb::DB * const db;
|
|
rocksdb::ColumnFamilyHandle * const cf;
|
|
const rocksdb::ReadOptions & rdOpts; // references into Storage::Pvt
|
|
const rocksdb::WriteOptions & wrOpts;
|
|
DBRecordArray * const dra;
|
|
std::shared_ptr<rocksdb::MergeOperator> mergeOp;
|
|
StorageDetail::ConcatOperator * concatOp; // this is a "weak" pointer into above, dynamic casted down. always valid.
|
|
Tic lastWarnTime; ///< this is not guarded by any locks. Assumption is calling code always holds an exclusive lock when calling truncateForUndo()
|
|
int64_t largestTxNumSeen = -1;
|
|
public:
|
|
const size_t keyBytes;
|
|
|
|
enum KeyPos : uint8_t { Beginning=0, Middle=1, End=2, KP_Invalid=3 };
|
|
const KeyPos keyPos;
|
|
|
|
TxHash2TxNumMgr(rocksdb::DB *db, rocksdb::ColumnFamilyHandle *cf,
|
|
const rocksdb::ReadOptions & rdOpts, const rocksdb::WriteOptions &wrOpts,
|
|
DBRecordArray *txnum2txhash, size_t keyBytes /*= 6*/, KeyPos keyPos /*= End*/)
|
|
: db(db), cf(cf), rdOpts(rdOpts), wrOpts(wrOpts), dra(txnum2txhash), keyBytes(keyBytes), keyPos(keyPos)
|
|
{
|
|
if (!this->db || !this->cf || !dra || !this->keyBytes || this->keyBytes > HashLen || this->keyPos >= KP_Invalid)
|
|
throw BadArgs("Bad argumnets supplied to TxHash2TxNumMgr constructor");
|
|
mergeOp = db->GetOptions(cf).merge_operator;
|
|
if (!mergeOp || ! (concatOp = dynamic_cast<StorageDetail::ConcatOperator *>(mergeOp.get())))
|
|
throw BadArgs("This db lacks a merge operator of type `ConcatOperator`");
|
|
loadLargestTxNumSeen();
|
|
Debug() << "TxHash2TxNumMgr: largestTxNumSeen = " << largestTxNumSeen;
|
|
}
|
|
|
|
std::unique_ptr<rocksdb::Iterator> newIterChecked() {
|
|
std::unique_ptr<rocksdb::Iterator> iter{db->NewIterator(rdOpts, cf)};
|
|
if (UNLIKELY(!iter)) throw DatabaseError("Unable to obtain an iterator to the txhash2txnum db"); // should never happen
|
|
return iter;
|
|
}
|
|
|
|
size_t mergeCount() const { return concatOp->merges.load(); }
|
|
|
|
QString dbName() const { return QString::fromStdString(cf->GetName()); }
|
|
|
|
/// Returns the largest tx num we have ever inserted into the db, or -1 if no txnums were inserted
|
|
int64_t maxTxNumSeenInDB() const { return largestTxNumSeen; }
|
|
|
|
/// Returned from insertForBlockPhased and/or truncateForUndoPhased. Intended to be an opaque type to capture
|
|
/// some state information for the async and then sync phase. Caller should call doAsyncPhase1() from a worker
|
|
/// thread and then when that completes, should call doSyncPhase2() from the caller's thread.
|
|
class PhasedOpBase {
|
|
friend class ::TxHash2TxNumMgr;
|
|
protected:
|
|
Tic t0;
|
|
std::function<void()> asyncPhase1, syncPhase2;
|
|
PhasedOpBase() = default;
|
|
public:
|
|
virtual ~PhasedOpBase() {
|
|
if (asyncPhase1 || syncPhase2) [[unlikely]]
|
|
Warning() << __func__ << " destructor called but the lambdas are still alive. FIXME!"; // should never happen
|
|
}
|
|
PhasedOpBase(PhasedOpBase &&) = delete;
|
|
PhasedOpBase(const PhasedOpBase &) = delete;
|
|
|
|
PhasedOpBase * doAsyncPhase1() {
|
|
if (asyncPhase1) {
|
|
asyncPhase1();
|
|
asyncPhase1 = nullptr; // clear lambda
|
|
} else [[unlikely]]
|
|
Warning() << __func__ << " called but asyncPhase1 is null!"; // defensive programming, should never happen
|
|
return this;
|
|
}
|
|
PhasedOpBase * doSyncPhase2() {
|
|
if (syncPhase2) {
|
|
syncPhase2();
|
|
syncPhase2 = nullptr; // clear lambda
|
|
} else [[unlikely]]
|
|
Warning() << __func__ << " called but syncPhase2 is null!"; // defensive programming, should never happen
|
|
return this;
|
|
}
|
|
};
|
|
|
|
using PhasedOp = std::unique_ptr<PhasedOpBase>;
|
|
|
|
private:
|
|
inline void insertForBlockInner1(const size_t i, rocksdb::WriteBatch &batch, const ByteView &key, const VarInt &val,
|
|
const std::vector<PreProcessedBlock::TxInfo> &txInfos) {
|
|
// Save by appending VarInt. Note that this uses the 'ConcatOperator' class we defined in this file,
|
|
// which requires rocksdb be compiled with RTTI.
|
|
if (auto st = batch.Merge(cf, ToSlice(key), ToSlice(val.byteView())); !st.ok()) [[unlikely]]
|
|
throw DatabaseError(QString("%1: batch merge fail for txHash %2: %3")
|
|
.arg(dbName(), QString(txInfos[i].hash.toHex()), StatusString(st)));
|
|
}
|
|
inline void insertForBlockInner2(rocksdb::WriteBatch &batch, const TxNum blockTxNum0, const qint64 elapsedNanos,
|
|
const std::vector<PreProcessedBlock::TxInfo> &txInfos) {
|
|
const Tic t2;
|
|
if (!txInfos.empty()) {
|
|
largestTxNumSeen = blockTxNum0 + txInfos.size() - 1;
|
|
saveLargestTxNumSeen(batch);
|
|
}
|
|
if (auto elapsed = elapsedNanos + t2.nsec(); elapsed >= /* 50msec */ 50'000'000)
|
|
DebugM("insertForBlock", ": inserted ", txInfos.size(), Util::Pluralize(" hash", txInfos.size()),
|
|
" in ", QString::asprintf("%1.3f msec", elapsed / 1e6));
|
|
}
|
|
|
|
public:
|
|
[[nodiscard]]
|
|
PhasedOp insertForBlockPhased(rocksdb::WriteBatch &batch, TxNum blockTxNum0, const std::vector<PreProcessedBlock::TxInfo> &txInfos) {
|
|
struct InsertPhased : PhasedOpBase {
|
|
std::vector<std::pair<ByteView, VarInt>> kv;
|
|
InsertPhased() = default;
|
|
~InsertPhased() override {}
|
|
};
|
|
auto ret = std::make_unique<InsertPhased>();
|
|
ret->asyncPhase1 = [this, self = ret.get(), &txInfos, blockTxNum0] {
|
|
self->kv.reserve(txInfos.size());
|
|
for (TxNum i = 0; i < txInfos.size(); ++i)
|
|
self->kv.emplace_back(makeKeyFromHash(txInfos[i].hash), blockTxNum0 + i);
|
|
self->t0.fin();
|
|
};
|
|
ret->syncPhase2 = [this, self = ret.get(), &batch, blockTxNum0, &txInfos] {
|
|
const Tic t1;
|
|
size_t i{};
|
|
for (const auto & [key, val] : self->kv)
|
|
insertForBlockInner1(i++, batch, key, val, txInfos); // may throw on error
|
|
insertForBlockInner2(batch, blockTxNum0, self->t0.nsec() + t1.nsec(), txInfos);
|
|
};
|
|
return ret;
|
|
}
|
|
|
|
void insertForBlock(rocksdb::WriteBatch &batch, TxNum blockTxNum0, const std::vector<PreProcessedBlock::TxInfo> &txInfos) {
|
|
// insertForBlockPhased(batch, blockTxNum0, txInfos)->doAsyncPhase1()->doSyncPhase2();
|
|
// Ideally we do the above, but the above would do some extra allocations which we want to avoid, so
|
|
// for performance we duplicate the code somewhat below ...
|
|
const Tic t0;
|
|
for (TxNum i = 0; i < txInfos.size(); ++i) {
|
|
const ByteView key = makeKeyFromHash(txInfos[i].hash);
|
|
const VarInt val(blockTxNum0 + i);
|
|
insertForBlockInner1(i, batch, key, val, txInfos); // may throw on error
|
|
}
|
|
insertForBlockInner2(batch, blockTxNum0, t0.nsec(), txInfos);
|
|
}
|
|
|
|
void insertForBlockNoBatch(TxNum blockTxNum0, const std::vector<PreProcessedBlock::TxInfo> &txInfos) {
|
|
rocksdb::WriteBatch batch;
|
|
insertForBlock(batch, blockTxNum0, txInfos);
|
|
if (auto st = db->Write(wrOpts, &batch) ; !st.ok())
|
|
throw DatabaseError(QString("%1: batch merge fail: %2").arg(dbName(), StatusString(st)));
|
|
}
|
|
|
|
/// This is called during blockundo. Returns immediately with 2 lambdas to be invoked later.
|
|
///
|
|
/// The lambdas delete records from the db having their TxNum >= `txNum`. The async almbda requires that
|
|
/// rf not yet be truncated. The second sync lambda commits the changes to the db.
|
|
///
|
|
/// This is slow so don't call it with huge numbers of records beyond what fits into a block.
|
|
[[nodiscard]]
|
|
PhasedOp truncateForUndoPhased(rocksdb::WriteBatch &batch, const TxNum txNum) {
|
|
struct UndoPhased : PhasedOpBase {
|
|
std::vector<QByteArray> recs;
|
|
std::vector<rocksdb::Slice> keySlices; ///< slices are views into above `recs`
|
|
std::vector<std::string> valsBackToDb;
|
|
std::vector<bool> ok;
|
|
unsigned dels{}, keeps{}, filts{}; // for DEBUG print
|
|
UndoPhased() = default;
|
|
~UndoPhased() override {}
|
|
};
|
|
auto ret = std::make_unique<UndoPhased>();
|
|
ret->asyncPhase1 = [this, self = ret.get(), txNum] {
|
|
const auto rfNR = dra->numRecords();
|
|
if (rfNR < txNum) [[unlikely]]
|
|
throw DatabaseError(dbName() + ": DBRecordArray does not have the hashes required for the specified truncation");
|
|
else if (rfNR == txNum) [[unlikely]] {
|
|
// defensive programming warning -- this should never happen
|
|
Warning() << "truncateForUndo: called with txNum == DBRecordArray->numRecords -- FIXME!";
|
|
return;
|
|
}
|
|
QString err;
|
|
auto &recs = self->recs = dra->readRecords(txNum, rfNR - txNum, &err);
|
|
if (recs.size() != rfNR - txNum || !err.isEmpty()) [[unlikely]]
|
|
throw DatabaseError(QString("%1: short read count or error reading DBRecordArray: %2").arg(dbName(), err));
|
|
|
|
DebugM("truncateForUndo: read ", recs.size(), Util::Pluralize(" record", recs.size()),
|
|
" from txNums file, elapsed: ", self->t0.msecStr(), " msec");
|
|
|
|
// first read all existing entries from the db -- we must delete the VarInts in their data blobs that
|
|
// have TxNums > txNum
|
|
auto &keySlices = self->keySlices;
|
|
keySlices.clear();
|
|
keySlices.reserve(recs.size());
|
|
for (const auto &rec : recs) {
|
|
const auto bv = makeKeyFromHash(rec);
|
|
keySlices.emplace_back(bv.charData(), bv.size());
|
|
}
|
|
std::vector<rocksdb::PinnableSlice> dbValues(keySlices.size());
|
|
std::vector<rocksdb::Status> statuses(keySlices.size());
|
|
db->MultiGet(rdOpts, cf, keySlices.size(), keySlices.data(), dbValues.data(), statuses.data());
|
|
DebugM("truncateForUndo: MultiGet for ", statuses.size(), Util::Pluralize(" key", statuses.size()),
|
|
" elapsed: ", self->t0.msecStr(), " msec");
|
|
|
|
// next filter out all VarInts >= txNum, deleting records that have no more VarInts left and writing
|
|
// back records that still have VarInts in them
|
|
auto &valsBackToDb = self->valsBackToDb;
|
|
valsBackToDb.resize(dbValues.size());
|
|
self->ok.resize(dbValues.size(), true);
|
|
for (size_t i = 0; i < dbValues.size(); ++i) {
|
|
if (!statuses[i].ok()) [[unlikely]] {
|
|
if (lastWarnTime.secs() >= 1.0) {
|
|
lastWarnTime = Tic();
|
|
// not sure what to do here... this should never happen. But warn anyway.
|
|
Warning() << "truncateForUndo: " << dbName() << ", got a non-ok status (" << StatusString(statuses[i])
|
|
<< ") when reading a key for txhash " << recs[i].toHex() << ". Proceeding anyway "
|
|
<< "but there may be DB corruption. Start " << APPNAME
|
|
<< " again with -C -C to check the database for consistency.";
|
|
}
|
|
self->ok[i] = false; // mark as to-be-skipped in sync lambda loop (should never happen!)
|
|
continue;
|
|
}
|
|
auto span = Span<const char>{dbValues[i]};
|
|
std::string &valBackToDb = valsBackToDb[i];
|
|
while (!span.empty()) {
|
|
try {
|
|
const VarInt val = VarInt::deserialize(span); // this may throw
|
|
if (val.value<TxNum>() >= txNum) {
|
|
// skip, filter out...
|
|
++self->filts;
|
|
} else {
|
|
// was a collision, keep
|
|
valBackToDb.append(val.byteView().charData(), val.size());
|
|
++self->keeps;
|
|
}
|
|
} catch (const std::exception &e) {
|
|
throw DatabaseFormatError(QString("%1: caught exception in truncateForUndo: %2").arg(dbName(),e.what()));
|
|
}
|
|
}
|
|
}
|
|
};
|
|
ret->syncPhase2 = [this, self = ret.get(), &batch, txNum] {
|
|
const Tic t1;
|
|
const size_t N = std::min(std::min(self->keySlices.size(), self->valsBackToDb.size()), self->ok.size());
|
|
if (!N) [[unlikely]] return; // nothing to do!
|
|
for (size_t i = 0; i < N; ++i) {
|
|
if (!self->ok[i]) [[unlikely]]
|
|
continue; // error reading from DB, already warned above in asyncPhase1, just continue (should never happen)
|
|
const rocksdb::Slice &keySlice = self->keySlices[i];
|
|
const std::string &valBackToDb = self->valsBackToDb[i];
|
|
if (valBackToDb.empty()) {
|
|
// delete, key now has no VarInts
|
|
if (auto st = batch.Delete(cf, keySlice); !st.ok()) [[unlikely]] {
|
|
if (lastWarnTime.secs() >= 1.0) {
|
|
lastWarnTime = Tic();
|
|
Warning() << "truncateForUndo: " << dbName() << " failed to delete a key from db: "
|
|
<< StatusString(st) << ". Continuing anyway ...";
|
|
}
|
|
}
|
|
++self->dels;
|
|
} else {
|
|
// keep key, key has some VarInts left
|
|
if (auto st = batch.Put(cf, keySlice, valBackToDb); !st.ok()) [[unlikely]]
|
|
throw DatabaseError(dbName() + ": failed to write back a key to the db: " + StatusString(st));
|
|
}
|
|
}
|
|
|
|
const int64_t txNumI = int64_t(txNum);
|
|
// we always add at the end and truncare at the end; this invariant should always hold
|
|
largestTxNumSeen = std::max(txNumI - 1, int64_t{-1});
|
|
saveLargestTxNumSeen(batch);
|
|
|
|
DebugM("truncateForUndo: txNum: ", txNum, ", nrecs: ", self->recs.size(), ", dels: ", self->dels,
|
|
", keeps: ", self->keeps, ", filts: ", self->filts, ", elapsed: ",
|
|
QString::asprintf("%1.3f", (self->t0.usec() + t1.usec()) / 1e3), " msec");
|
|
};
|
|
|
|
return ret;
|
|
}
|
|
|
|
/// Returns a valid optional containing the TxNum of txHash if txHash is found in the db. A nullopt otherwise.
|
|
/// May throw DatabaseError if there is a low-level deserialization error.
|
|
std::optional<TxNum> find(const TxHash &txHash) const {
|
|
std::optional<TxNum> ret;
|
|
const auto key = makeKeyFromHash(txHash);
|
|
auto optBytes = GenericDBGet<QByteArray>(db, cf, key, true, dbName(), true, rdOpts);
|
|
if (!optBytes) return ret; // missing
|
|
auto span = Span<const char>{*optBytes};
|
|
std::vector<uint64_t> txNums;
|
|
txNums.reserve(1 + span.size() / 5); // rough heuristic
|
|
try {
|
|
while (!span.empty())
|
|
txNums.push_back(VarInt::deserialize(span).value<uint64_t>()); // this may throw
|
|
if (txNums.empty()) [[unlikely]] throw DatabaseFormatError(QString("Missing data for txHash: ") + QString(txHash.toHex()));
|
|
QString errStr;
|
|
// we may get more than 1 txNum for a particular key, so examine them all
|
|
const auto recs = dra->readRandomRecords(txNums, &errStr, true);
|
|
if (recs.size() != txNums.size()) [[unlikely]] throw DatabaseError("Expected recs.size() == txNums.size()!");
|
|
size_t i = 0;
|
|
for (const auto & rec : recs) {
|
|
if (rec == txHash) {
|
|
// found!
|
|
ret = txNums[i];
|
|
return ret;
|
|
}
|
|
++i;
|
|
}
|
|
} catch (const std::exception &e) {
|
|
throw DatabaseError(dbName() + ": failed lookup for txHash " + QString(txHash.toHex()) + ": " + e.what());
|
|
}
|
|
return ret; // if we get here, ret is nullopt and txHash does not exist in db
|
|
}
|
|
|
|
/// Find the TxNums for a batch of hashes. Returns a vector that is exactly the same size as hashes.
|
|
/// Not-found hashes are std::nullopt optionals. Found hashes will have the correct TxNum for that hash
|
|
/// filled-in.
|
|
///
|
|
/// May throw DatabaseError on low-level db error.
|
|
std::vector<std::optional<TxNum>> findMany(const std::vector<TxHash> &hashes) const {
|
|
std::vector<std::optional<TxNum>> ret;
|
|
if (hashes.empty()) return ret; // short-circuit return on no work to do
|
|
const Tic t0;
|
|
ret.resize(hashes.size());
|
|
std::vector<rocksdb::Slice> keySlices;
|
|
std::vector<rocksdb::PinnableSlice> dbResults;
|
|
keySlices.reserve(hashes.size());
|
|
dbResults.reserve(hashes.size());
|
|
// build keys
|
|
for (const auto & hash : hashes) {
|
|
keySlices.push_back(ToSlice(makeKeyFromHash(hash))); // shallow view into bytes in hashes
|
|
dbResults.emplace_back();
|
|
}
|
|
std::vector<rocksdb::Status> statuses(keySlices.size());
|
|
db->MultiGet(rdOpts, cf, keySlices.size(), keySlices.data(), dbResults.data(), statuses.data()); // this should be faster than single gets..?
|
|
//DebugM(__func__, ": MultiGet of ", keySlices.size(), " items took ", t0.msecStr(), " msec");
|
|
if (statuses.size() != hashes.size() || dbResults.size() != hashes.size())
|
|
throw DatabaseError(dbName() + ": db returned an unexpected number of results"); // should never happen
|
|
std::vector<uint64_t> recNums;
|
|
std::vector<std::optional<std::pair<size_t, size_t>>> idx2RecNums;
|
|
idx2RecNums.resize(hashes.size());
|
|
recNums.reserve(hashes.size());
|
|
for (size_t i = 0; i < statuses.size(); ++i) {
|
|
auto & st = statuses[i];
|
|
if (st.IsNotFound()) continue; // skip NotFound
|
|
if (!st.ok()) throw DatabaseError(dbName() + ": got a status that is not ok in findMany: " + StatusString(st));
|
|
auto & dataBlob = dbResults[i];
|
|
if (dataBlob.empty()) {
|
|
Warning() << dbName() << ": Empty record for " << hashes[i].toHex() << ". FIXME!";
|
|
continue;
|
|
}
|
|
auto span = Span<const char>{dataBlob};
|
|
std::pair<size_t, size_t> p(recNums.size(), recNums.size());
|
|
while (!span.empty()) {
|
|
try {
|
|
recNums.push_back(VarInt::deserialize(span).value<uint64_t>());
|
|
} catch (const std::exception &e) {
|
|
throw DatabaseSerializationError(dbName() + ": failed to deserialize a VarInt: " + e.what());
|
|
}
|
|
++p.second;
|
|
}
|
|
if (p.second > p.first)
|
|
idx2RecNums[i] = p;
|
|
}
|
|
QString errStr;
|
|
const auto recs = dra->readRandomRecords(recNums, &errStr, true);
|
|
if (recs.size() != recNums.size()) [[unlikely]] throw DatabaseError(QString("Expected recs.size() == recNums.size()! Error: %1").arg(errStr));
|
|
assert(ret.size() == hashes.size() && ret.size() == idx2RecNums.size());
|
|
for (size_t i = 0; i < ret.size(); ++i) {
|
|
auto & optRange = idx2RecNums[i];
|
|
if (!optRange) continue; // key not found, skip
|
|
for (size_t j = optRange->first; j < optRange->second; ++j) {
|
|
if (recs[j] == hashes[i]) {
|
|
// found!
|
|
ret[i] = recNums[j]; // mark this in the return set
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
DebugM(__func__, ": ", ret.size(), " result(s), elapsed ", t0.msecStr(), " msec");
|
|
return ret;
|
|
}
|
|
|
|
bool exists(const TxHash &txHash) const { return bool(find(txHash)); }
|
|
|
|
private:
|
|
ByteView makeKeyFromHash(const ByteView &bv) const {
|
|
const auto len = bv.size();
|
|
if (UNLIKELY(len != HashLen))
|
|
throw DatabaseFormatError(QString("Hash \"%1\" is not %2 bytes").arg(QString(Util::ToHexFast(bv.substr(0, 80).toByteArray(false)))).arg(HashLen));
|
|
if (keyPos == End)
|
|
return bv.substr(len - keyBytes, keyBytes);
|
|
else if (keyPos == Middle)
|
|
return bv.substr(len/2 - keyBytes/2, keyBytes);
|
|
else // Beginning
|
|
return bv.substr(0, keyBytes);
|
|
}
|
|
static const QByteArray kLargestTxNumSeenKeyPrefix;
|
|
QByteArray makeLargestTxNumSeenKey() const {
|
|
auto ret = kLargestTxNumSeenKeyPrefix;
|
|
if (size_t(ret.length()) <= keyBytes)
|
|
// ensure a key that can never exist in db for a real tx hash (> keyBytes)
|
|
ret.append(keyBytes - size_t(ret.length()) + 1, '-');
|
|
return ret;
|
|
}
|
|
void loadLargestTxNumSeen() {
|
|
auto opt = GenericDBGet<int64_t>(db, cf, makeLargestTxNumSeenKey(), true, QString{}, false, rdOpts);
|
|
if (opt && *opt >= 0) largestTxNumSeen = *opt;
|
|
else largestTxNumSeen = -1;
|
|
}
|
|
void saveLargestTxNumSeen(rocksdb::WriteBatch &batch) const {
|
|
const auto key = makeLargestTxNumSeenKey();
|
|
if (largestTxNumSeen > -1)
|
|
GenericBatchPut(batch, cf, key, largestTxNumSeen, QString{});
|
|
else
|
|
GenericBatchDelete(batch, cf, key, QString{});
|
|
}
|
|
// Deletes *all* keys from db! May throw.
|
|
void deleteAllEntries() {
|
|
std::string firstKey, endKey;
|
|
{
|
|
std::unique_ptr<rocksdb::Iterator> iter = newIterChecked();
|
|
iter->SeekToFirst();
|
|
if (iter->Valid())
|
|
firstKey = iter->key().ToString();
|
|
iter->SeekToLast();
|
|
if (iter->Valid())
|
|
endKey = iter->key().ToString();
|
|
}
|
|
if (!endKey.empty()) {
|
|
endKey.insert(endKey.end(), char(0xff)); // make sure our lastKey spec is larger than the actual lastKey
|
|
Debug() << "Deleting keys in the range [" << Util::ToHexFast(QByteArray::fromStdString(firstKey)) << ", "
|
|
<< Util::ToHexFast(QByteArray::fromStdString(endKey)) << "] ...";
|
|
}
|
|
rocksdb::WriteBatch batch;
|
|
rocksdb::FlushOptions fopts;
|
|
rocksdb::Status st;
|
|
fopts.wait = true; fopts.allow_write_stall = true;
|
|
if (!(st = batch.DeleteRange(cf, firstKey, endKey)).ok())
|
|
throw DatabaseError(dbName() + ": failed to delete all keys: " + StatusString(st));
|
|
largestTxNumSeen = -1;
|
|
saveLargestTxNumSeen(batch);
|
|
if (!(st = db->Write(wrOpts, &batch)).ok())
|
|
throw DatabaseError(dbName() + ": failed to write batch when deleting all keys: " + StatusString(st));
|
|
if (!(st = db->Flush(fopts, cf)).ok())
|
|
throw DatabaseError(dbName() + ": failed to flush when deleting all keys: " + StatusString(st));
|
|
|
|
std::unique_ptr<rocksdb::Iterator> iter = newIterChecked();
|
|
iter->SeekToFirst();
|
|
if (iter->Valid())
|
|
throw InternalError(dbName() + ": delete all keys failed -- iterator still points to a row! FIXME!");
|
|
}
|
|
|
|
public:
|
|
// -- Utility / consistency check, etc ..
|
|
|
|
void consistencyCheck() { // this throws if the checks fail
|
|
const Tic t0;
|
|
Log() << "CheckDB: Verifying txhash index (this may take some time) ...";
|
|
std::unique_ptr<rocksdb::Iterator> iter = newIterChecked();
|
|
size_t i = 0, verified = 0;
|
|
QString err;
|
|
constexpr size_t batchSize = 50'000;
|
|
std::vector<std::pair<std::string, uint64_t>> batch;
|
|
std::vector<uint64_t> batchNums;
|
|
batch.reserve(batchSize + 500);
|
|
batchNums.reserve(batchSize + 500);
|
|
auto ProcBatch = [this, &batch, &batchNums, &err, &verified]{
|
|
std::sort(batch.begin(), batch.end(), [](const auto & a, const auto & b){
|
|
return a.second < b.second;
|
|
});
|
|
std::sort(batchNums.begin(), batchNums.end());
|
|
const auto recs = dra->readRandomRecords(batchNums, &err);
|
|
if (recs.size() != batchNums.size()) throw InternalError(QString("short read of records: ") + err);
|
|
for (size_t i = 0; i < recs.size(); ++i) {
|
|
const auto &hash = recs[i];
|
|
const auto txNum = batchNums[i];
|
|
if (batch[i].second != txNum) throw DatabaseError("txNum mismatch");
|
|
if (hash.length() != HashLen) throw DatabaseFormatError("bad record");
|
|
const auto expect = makeKeyFromHash(hash).toByteArray();
|
|
const auto &keyStr = batch[i].first;
|
|
const auto key = QByteArray::fromRawData(keyStr.data(), keyStr.size());
|
|
if (key != expect)
|
|
throw DatabaseError(QString("record %1 does not match key. expected: %2, got: %3")
|
|
.arg(txNum).arg(QString(expect.toHex()), QString(key.toHex())));
|
|
++verified;
|
|
}
|
|
batch.resize(0);
|
|
batchNums.resize(0);
|
|
};
|
|
App *ourApp = app();
|
|
const auto nrec = dra->numRecords();
|
|
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
|
if (UNLIKELY(0 == i % 100 && ourApp && ourApp->signalsCaught()))
|
|
throw UserInterrupted("User interrupted, aborting check"); // if the user hits Ctrl-C, stop the operation
|
|
if (i && 0 == i % 1'000'000) {
|
|
*(0 == i % 5'000'000 ? std::make_unique<Log>() : std::make_unique<Debug>())
|
|
<< "Verified " << verified << "/" << nrec << ", merge ops so far: " << mergeCount() << " ...";
|
|
}
|
|
const auto keySlice = iter->key();
|
|
const auto valSlice = iter->value();
|
|
const auto key = FromSlice(keySlice);
|
|
if (key.startsWith(kLargestTxNumSeenKeyPrefix))
|
|
continue; // skip this meta entry
|
|
const auto val = FromSlice(valSlice);
|
|
Span<const std::byte> bytes(reinterpret_cast<const std::byte *>(val.constData()), val.size());
|
|
if (bytes.empty()) throw DatabaseFormatError("Empty db data!");
|
|
while (!bytes.empty()) {
|
|
auto vint = VarInt::deserialize(bytes);
|
|
auto txNum = vint.value<uint64_t>();
|
|
batch.emplace_back(keySlice.ToString(), txNum);
|
|
batchNums.push_back(txNum);
|
|
++i;
|
|
}
|
|
if (batch.size() >= batchSize) {
|
|
ProcBatch();
|
|
}
|
|
}
|
|
if (!batch.empty()) ProcBatch();
|
|
iter->Reset();
|
|
Log() << "CheckDB: txhash index verified " << verified << " entries in " << t0.secsStr(1) << " secs";
|
|
}
|
|
|
|
void rebuildDB() {
|
|
deleteAllEntries();
|
|
|
|
constexpr size_t batchSize = 50'000;
|
|
Debug() << "Using key bytes: " << keyBytes << ", batchSize: " << batchSize;
|
|
const Tic t0;
|
|
App *ourApp = app();
|
|
const auto nrec = dra->numRecords();
|
|
std::vector<PreProcessedBlock::TxInfo> fakeInfos;
|
|
for (size_t i = 0; i < nrec; /*i += batchSize*/) {
|
|
if (UNLIKELY(0 == i % 100 && ourApp && ourApp->signalsCaught()))
|
|
throw UserInterrupted("User interrupted, aborting check"); // if the user hits Ctrl-C, stop the operation
|
|
if (i && 0 == i % 1'000'000) {
|
|
const double pct = double(i) * 100. / nrec;
|
|
Log() << "Progress: " << QString::number(pct, 'f', 1) << "%, merge ops so far: " << mergeCount();
|
|
}
|
|
QString err;
|
|
const auto recs = dra->readRecords(i, std::min<size_t>(batchSize, dra->numRecords() - i), &err);
|
|
if (!err.isEmpty()) throw InternalError(QString("Got error from DBRecordArray: ") + err);
|
|
// fake it
|
|
fakeInfos.resize(recs.size());
|
|
for (size_t j = 0; j < recs.size(); ++j)
|
|
fakeInfos[j].hash = recs[j];
|
|
insertForBlockNoBatch(i, fakeInfos); // this throws on error
|
|
i += fakeInfos.size();
|
|
}
|
|
fakeInfos.clear();
|
|
rocksdb::FlushOptions fopts;
|
|
fopts.wait = true; fopts.allow_write_stall = true;
|
|
if (auto st = db->Flush(fopts, cf); !st.ok())
|
|
Warning() << "DB Flush error: " << StatusString(st);
|
|
Log() << "Indexed " << nrec << " txhash entries, elapsed: " << t0.secsStr(2) << " sec";
|
|
}
|
|
|
|
void consistencyCheckSlowRev() {
|
|
Log() << "CheckDB: Verifying txhash index using the thorough reverse-check (this may take a long time) ...";
|
|
const Tic t0;
|
|
size_t i = 0, verified = 0;
|
|
const auto nrec = dra->numRecords();
|
|
constexpr size_t batchSize = 50'000;
|
|
App *ourApp = app();
|
|
for (i = 0; i < nrec; /*i += batchSize*/) {
|
|
if (i && 0 == i % 100'000)
|
|
Log() << "Verified: " << verified << "/" << nrec << ", merge ops so far: " << mergeCount() << " ...";
|
|
QString err;
|
|
auto recs = dra->readRecords(i, batchSize, &err);
|
|
recs.emplace_back(HashLen, char(0)); // add a dummy at the end
|
|
Util::getRandomBytes(recs.back().data(), HashLen); // put a random hash at the end
|
|
auto results = findMany(recs);
|
|
if (results.size() != recs.size()) throw InternalError("size mismatch: " + err);
|
|
for (size_t j = 0; j < results.size()-1; ++j) {
|
|
if (!results[j]) throw DatabaseError("Expected a value not nullopt");
|
|
if (*results[j] != i) {
|
|
static const std::set<QByteArray> DupeTxHashes = {
|
|
// Before BIP34, there were dupe coinbase tx's... so we tolerate those here.
|
|
Util::ParseHexFast("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599"),
|
|
Util::ParseHexFast("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468"),
|
|
};
|
|
if (!DupeTxHashes.count(recs[j]))
|
|
throw DatabaseError(QString("Mismatched TxNum for hash: ") + QString(recs[j].toHex()));
|
|
}
|
|
++verified;
|
|
++i;
|
|
if (UNLIKELY(0 == i % 10 && ourApp && ourApp->signalsCaught()))
|
|
throw UserInterrupted("User interrupted, aborting check"); // if the user hits Ctrl-C, stop the operation
|
|
}
|
|
if (results.back()) Warning() << "Expected last entry to be \"not found\"!";
|
|
}
|
|
|
|
Log() << "CheckDB: Verified " << verified << " total tx hashes, merge operations: " << mergeCount()
|
|
<< ", elapsed: " << t0.secsStr(2) << " sec";
|
|
}
|
|
}; // end class TxHash2TxNumMgr
|
|
|
|
/* static */ const QByteArray TxHash2TxNumMgr::kLargestTxNumSeenKeyPrefix = "+largestTxNumSeen";
|
|
|
|
} // namespace
|
|
|
|
struct Storage::Pvt
|
|
{
|
|
Pvt(const unsigned cacheSizeBytes)
|
|
: lruNum2Hash(std::max(unsigned(cacheSizeBytes*kLruNum2HashCacheMemoryWeight), 1u)),
|
|
lruHeight2Hashes_BitcoindMemOrder(std::max(unsigned(cacheSizeBytes*kLruHeight2HashesCacheMemoryWeight), 1u))
|
|
{}
|
|
|
|
Pvt(const Pvt &) = delete;
|
|
|
|
constexpr int blockHeaderSize() { return BTC::GetBlockHeaderSize(); }
|
|
|
|
/* NOTE: If taking multiple locks, all locks should be taken in the order they are declared, to avoid deadlocks. */
|
|
|
|
Meta meta;
|
|
RWLock metaLock;
|
|
|
|
std::atomic<std::underlying_type_t<SaveItem>> pendingSaves{0};
|
|
|
|
struct RocksDBHandlesEtc {
|
|
const rocksdb::ReadOptions defReadOpts; ///< avoid creating this each time
|
|
const rocksdb::WriteOptions defWriteOpts; ///< avoid creating this each time
|
|
|
|
rocksdb::Options opts;
|
|
rocksdb::ColumnFamilyOptions shistOpts, txhash2txnumOpts,txnum2txhashOpts, headersOpts;
|
|
std::weak_ptr<rocksdb::Cache> blockCache; ///< shared across all dbs, caps total block cache size across all db instances
|
|
std::weak_ptr<rocksdb::WriteBufferManager> writeBufferManager; ///< shared across all dbs, caps total memtable buffer size across all db instances
|
|
|
|
std::shared_ptr<StorageDetail::ConcatOperator> concatOperator, concatOperatorTxHash2TxNum,
|
|
concatOperatorTxNum2TxHash, concatOperatorHeaders;
|
|
|
|
std::unique_ptr<rocksdb::DB> db; // the single database we open.
|
|
std::vector<rocksdb::ColumnFamilyHandle *> columnFamilies;
|
|
|
|
// These are all either nullptr or pointers for handles that are also contained in the above `columnFamilies` vector
|
|
rocksdb::ColumnFamilyHandle *meta{}, *blkinfo{}, *utxoset{},
|
|
*shist{}, *shunspent{}, // scripthash_history and scripthash_unspent
|
|
*undo{}, // undo (reorg rewind)
|
|
*txnum2txhash{}, // mapping of txNum -> 32-byte hashes
|
|
*headers{}, // all blockchain headers
|
|
*txhash2txnum{}, // new: index of txhash -> txNumsFile
|
|
*rpa{}; // new: height -> Rpa::PrefixTable
|
|
|
|
struct ColFamHandleAndOptsRefs {
|
|
rocksdb::ColumnFamilyHandle *& handle;
|
|
const rocksdb::ColumnFamilyOptions & options;
|
|
};
|
|
// Some introspection here.. these tuples point to some of the above members
|
|
const std::map<std::string, ColFamHandleAndOptsRefs> colFamsTable = {
|
|
{ "meta", {.handle = meta, .options = opts} },
|
|
{ "blkinfo", {.handle = blkinfo, .options = opts} },
|
|
{ "utxoset", {.handle = utxoset, .options = opts} },
|
|
{ "scripthash_history", {.handle = shist, .options = shistOpts} },
|
|
{ "scripthash_unspent", {.handle = shunspent, .options = opts} },
|
|
{ "undo", {.handle = undo, .options = opts} },
|
|
{ "txnum2txhash", {.handle = txnum2txhash, .options = txnum2txhashOpts} },
|
|
{ "headers", {.handle = headers, .options = headersOpts} },
|
|
{ "txhash2txnum", {.handle = txhash2txnum, .options = txhash2txnumOpts} },
|
|
{ "rpa", {.handle = rpa, .options = opts} },
|
|
};
|
|
|
|
|
|
std::unique_ptr<DBRecordArray> txNumsDRA;
|
|
std::unique_ptr<DBRecordArray> headersDRA;
|
|
|
|
std::unique_ptr<TxHash2TxNumMgr> txhash2txnumMgr; ///< provides a bit of a higher-level interface into the db
|
|
|
|
/// One of these is alive if we are in an initial sync and user specified --utxo-cache
|
|
/// It caches UTXOs in memory and delays UTXO writes to DB so we don't have to do so much back-and-forth to
|
|
/// rocksdb.
|
|
std::unique_ptr<UTXOCache> utxoCache;
|
|
|
|
rocksdb::DB *get() { return db.get(); }
|
|
const rocksdb::DB *get() const { return db.get(); }
|
|
operator rocksdb::DB *() { return get(); }
|
|
operator const rocksdb::DB *() const { return get(); }
|
|
rocksdb::DB * operator->() { return get(); }
|
|
const rocksdb::DB * operator->() const { return get(); }
|
|
};
|
|
RocksDBHandlesEtc db;
|
|
|
|
/// Big lock used for block/history updates. Public methods that read the history such as getHistory and listUnspent
|
|
/// take this as read-only (shared), and addBlock and undoLatestBlock take this as read/write (exclusively).
|
|
/// This is intended to be a coarse lock. Currently the update code takes this along with headerVerifierLock and
|
|
/// blkInfoLock at the same time, so it's (as of now) equivalent to either of those two locks.
|
|
/// TODO: See about removing all the other locks and keeping one general RWLock for all updates?
|
|
mutable RWLock blocksLock;
|
|
|
|
BTC::HeaderVerifier headerVerifier;
|
|
mutable RWLock headerVerifierLock;
|
|
|
|
std::atomic<TxNum> txNumNext{0};
|
|
|
|
std::vector<BlkInfo> blkInfos;
|
|
std::map<TxNum, unsigned> blkInfosByTxNum; ///< ordered map of TxNum0 for a block -> index into above blkInfo array
|
|
RWLock blkInfoLock; ///< locks blkInfos and blkInfosByTxNum
|
|
|
|
std::atomic_int64_t utxoCt = 0;
|
|
|
|
static constexpr uint32_t InvalidUndoHeight = std::numeric_limits<uint32_t>::max();
|
|
|
|
std::atomic<uint32_t> earliestUndoHeight = InvalidUndoHeight; ///< the purpose of this is to control when we issue "delete" commands to the db for deleting expired undo infos from the undo db
|
|
|
|
// Ratios of cacheMemoryBytes that we give to each of the 2 lru caches -- we do 50/50
|
|
static constexpr double kLruNum2HashCacheMemoryWeight = 0.50;
|
|
static constexpr double kLruHeight2HashesCacheMemoryWeight = 1.0 - kLruNum2HashCacheMemoryWeight;
|
|
|
|
/// This cache is anticipated to see heavy use for get_history, so is configurable (config option: txhash_cache)
|
|
/// This gets cleared by undoLatestBlock.
|
|
CostCache<TxNum, TxHash> lruNum2Hash; // NOTE: max size in bytes initted in constructor
|
|
static constexpr unsigned lruNum2HashSizeCalc(unsigned nItems = 1) {
|
|
// NB: each TxHash (aka QByteArray) actually stores HashLen+1 bytes (QByteArray always appends a nul byte)
|
|
// NB2: each TxHash also has the QArrayData overhead (qByteArrayPvtDataSize())
|
|
return unsigned( decltype(lruNum2Hash)::itemOverheadBytes() + (nItems * (Util::qByteArrayPvtDataSize() + HashLen+1)) );
|
|
}
|
|
|
|
/// Cache BlockHeight -> vector of txHashes for the block (in bitcoind memory order -- little endian).
|
|
/// This is used by the txHashesForBlock function only (which is used by get_merkle and id_from_pos in the RPC protocol).
|
|
CostCache<BlockHeight, QVector<TxHash>> lruHeight2Hashes_BitcoindMemOrder; // NOTE: max size in bytes initted in constructor
|
|
/// returns the cost for a particular cache item based on the number of hashes in the vector
|
|
static constexpr unsigned lruHeight2HashSizeCalc(size_t nHashes) {
|
|
// each cache item with nHashes takes roughly this much memory
|
|
return unsigned( (nHashes * ((HashLen+1) + sizeof(TxHash) + Util::qByteArrayPvtDataSize()))
|
|
+ decltype(lruHeight2Hashes_BitcoindMemOrder)::itemOverheadBytes() );
|
|
}
|
|
|
|
struct LRUCacheStats {
|
|
std::atomic_size_t num2HashHits = 0, num2HashMisses = 0,
|
|
height2HashesHits = 0, height2HashesMisses = 0;
|
|
} lruCacheStats;
|
|
|
|
/// this object is thread safe, but it needs to be initialized with headers before allowing client connections.
|
|
std::unique_ptr<Merkle::Cache> merkleCache;
|
|
|
|
HeaderHash genesisHash; // written-to once by either loadHeaders code or addBlock for block 0. Guarded by headerVerifierLock.
|
|
|
|
Mempool mempool; ///< app-wide mempool data -- does not get saved to db. Controller.cpp writes to this
|
|
Mempool::FeeHistogramVec mempoolFeeHistogram; ///< refreshed periodically by refreshMempoolHistogram()
|
|
RWLock mempoolLock;
|
|
|
|
Tic lastWarned; ///< to rate-limit potentially spammy warning messages (guarded by blocksLock)
|
|
|
|
std::unique_ptr<CoTask> blocksWorker; ///< work to be done in parallel can be submitted to this co-task in addBlock and undoLatestBlock
|
|
|
|
/// Info specific to the `rpa` index
|
|
struct RpaInfo {
|
|
std::atomic_int32_t firstHeight = -1, lastHeight = -1; // inclusive height range that we have in the DB. -1 means undefined/missing.
|
|
std::atomic_uint64_t nReads{0u}, nWrites{0u}, nDeletions{0u}; // keep track of number of times we read/write/delete from this db
|
|
std::atomic_uint64_t nBytesWritten{0u}, nBytesRead{0u}; // keep track of number of bytes written and read during Storage object lifetime
|
|
mutable std::atomic_int rpaNeedsFullCheckCachedVal = -1; // if > -1, the last value written to the DB. If < 0, no cached val, just read from DB when querying isRpaNeedsFullCheck()
|
|
} rpaInfo;
|
|
|
|
/// Set of recent block txids seen, only valid if "notify" is enabled and if app-wide zmq "hashtx" notifs are enabled.
|
|
/// Guarded by `blocksLock`.
|
|
std::unordered_set<TxHash, HashHasher> recentBlockTxHashes;
|
|
};
|
|
|
|
namespace {
|
|
/// returns a key that is hashX concatenated with the serializd ctxo -> size 40 or 41 byte vector
|
|
/// Note hashX must be valid and sized HashLen otherwise this throws.
|
|
QByteArray mkShunspentKey(const QByteArray & hashX, const CompactTXO &ctxo) {
|
|
// we do it this way for performance:
|
|
const int hxlen = hashX.length();
|
|
if (UNLIKELY(hxlen != HashLen))
|
|
throw InternalError(QString("mkShunspentKey -- scripthash is not exactly %1 bytes: %2").arg(HashLen).arg(QString(hashX.toHex())));
|
|
QByteArray key(hxlen + int(ctxo.serializedSize(false /* no force wide */)), Qt::Uninitialized);
|
|
std::memcpy(key.data(), hashX.constData(), size_t(hxlen));
|
|
ctxo.toBytesInPlace(reinterpret_cast<std::byte *>(key.data()+hxlen), ctxo.serializedSize(false), false /* no force wide */);
|
|
return key;
|
|
}
|
|
/// throws if key is not the correct size (must be exactly 40 or 41 bytes)
|
|
CompactTXO extractCompactTXOFromShunspentKey(const rocksdb::Slice &key) {
|
|
static const auto ExtractHashXHex = [](const rocksdb::Slice &key) -> QString {
|
|
if (key.size() >= HashLen)
|
|
return QString(FromSlice(key).left(HashLen).toHex());
|
|
else
|
|
return "<undecipherable scripthash>";
|
|
};
|
|
if (const auto ksz = key.size();
|
|
UNLIKELY(ksz != HashLen + CompactTXO::minSize() && ksz != HashLen + CompactTXO::maxSize()))
|
|
// should never happen, indicates db corruption
|
|
throw InternalError(QString("Key size for scripthash %1 is invalid").arg(ExtractHashXHex(key)));
|
|
static_assert (sizeof(*key.data()) == 1, "Assumption is rocksdb::Slice is basically a byte vector");
|
|
const CompactTXO ctxo =
|
|
CompactTXO::fromBytesInPlaceExactSizeRequired(reinterpret_cast<const std::byte *>(key.data()) + HashLen,
|
|
key.size() - HashLen);
|
|
if (UNLIKELY(!ctxo.isValid()))
|
|
// should never happen, indicates db corruption
|
|
throw InternalError(QString("Deserialized CompactTXO is invalid for scripthash %1").arg(ExtractHashXHex(key)));
|
|
return ctxo;
|
|
}
|
|
std::pair<HashX, CompactTXO> extractShunspentKey(const rocksdb::Slice & key) {
|
|
const CompactTXO ctxo = extractCompactTXOFromShunspentKey(key); // throws if wrong size
|
|
return {DeepCpy(key.data(), HashLen), ctxo}; // if we get here size ok, can extract HashX
|
|
}
|
|
} // namespace
|
|
|
|
// Just a glorified hash table to cache recently-seen UTXOs to avoid database lookups. In previous versions of Fulcrum
|
|
// this class was more elaborate (it tried to avoid redundant DB writes for short-lived UTXOs that only were active
|
|
// for a few blocks). But it has been simplified in the interests of preserving database integrity across addBlock()
|
|
// calls. Additionally, all those elaborate techniques didn't improve performance much (and sometimes hampered it).
|
|
// This class is only used during initial sync and even then only if --utxo-cache is specified.
|
|
class Storage::UTXOCache
|
|
{
|
|
struct TableHasherAndEq {
|
|
bool operator()(const TXO &a, const TXO &b) const noexcept { return a == b; }
|
|
size_t operator()(const TXO &t) const noexcept { return std::hash<TXO>{}(t); }
|
|
};
|
|
struct TableEntry {
|
|
TXOInfo txoInfo;
|
|
size_t entrySize; ///< memory usage for this entry in bytes
|
|
TableEntry(TXOInfo &&info, size_t s) : txoInfo{std::move(info)}, entrySize{s} {}
|
|
};
|
|
using Table = robin_hood::unordered_flat_map<TXO, TableEntry, TableHasherAndEq, TableHasherAndEq>;
|
|
Table utxos;
|
|
std::vector<std::pair<TXO, TXOInfo>> deferredAdds;
|
|
size_t memUsed = 0u; ///< cumulative sum of all TableEntry::entrySize in the Table (excludes hash table size itself)
|
|
|
|
static_assert (std::is_same_v<decltype(std::declval<TXO>().txHash), QByteArray>
|
|
&& std::is_same_v<decltype(std::declval<TXOInfo>().hashX), QByteArray>,
|
|
"Below assumes we are using QByteArray");
|
|
static constexpr size_t BaseEntrySize = sizeof(Table::value_type)
|
|
+ (HashLen + Util::qByteArrayPvtDataSize()) * size_t{2U}; // account for txHash and hashX
|
|
|
|
void do_limitSize(const size_t bytes) {
|
|
if (memUsed <= bytes || utxos.empty()) return;
|
|
const size_t limitTo = static_cast<size_t>(bytes * 0.95); // limit to 95% of size to avoid ping-ponging through here for every call
|
|
const Tic t0;
|
|
DebugM(name, ": limiting size to ", limitTo, ", current size: ", memUsed);
|
|
size_t deletions = 0;
|
|
for (auto it = utxos.begin(); memUsed > limitTo && it != utxos.end(); /*-->see loop body<--*/) {
|
|
it = rm(it);
|
|
++deletions;
|
|
}
|
|
DebugM(name, ": deletions: ", deletions, ", utxos left: ", utxos.size(), ", elapsed: ", t0.msecStr(), " msec",
|
|
"; memUsage: ", QString::number(memUsed/1000.0/1000.0, 'f', 3), " MB");
|
|
}
|
|
|
|
bool add(TXO &&txo, TXOInfo &&info) {
|
|
const size_t entrySize = BaseEntrySize + (info.tokenDataPtr ? info.tokenDataPtr->GetMemSize() : 0u);
|
|
if (const auto & [it, inserted] = utxos.try_emplace(std::move(txo), std::move(info), entrySize); inserted) {
|
|
memUsed += entrySize;
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// Precondition: `it` must be a valid iterator.
|
|
// Returns the "next" iterator after erasing `it`, and after accounting for `it`'s memusage.
|
|
template <typename It>
|
|
It rm(const It &it) {
|
|
memUsed -= it->second.entrySize;
|
|
return utxos.erase(it);
|
|
}
|
|
|
|
bool contains(const TXO & t) const { return utxos.contains(t); }
|
|
|
|
const QString name;
|
|
CoTask prefetcher;
|
|
CoTask::Future prefetcherFut;
|
|
|
|
rocksdb::DB * const m_db;
|
|
rocksdb::ColumnFamilyHandle * const cf_utxo;
|
|
const rocksdb::ReadOptions & readOpts;
|
|
|
|
// persistent data structures we use in order to avoid having to continually re-reserve memory
|
|
struct PFData {
|
|
std::vector<std::pair<QByteArray, TXO>> keyData;
|
|
std::vector<rocksdb::Slice> keys;
|
|
std::vector<rocksdb::PinnableSlice> values;
|
|
std::vector<rocksdb::Status> statuses;
|
|
} pf;
|
|
|
|
void do_prefetch(PreProcessedBlockPtr ppb) {
|
|
// Below call to subitWork will throw std::domain_error if we are being called while the prefetcher is still
|
|
// active ... which is what we want here, because it indicates a programming error.
|
|
prefetcherFut = prefetcher.submitWork([this, ppb]{
|
|
const Tic t0;
|
|
const size_t nIns = ppb->inputs.size();
|
|
size_t num_ok = 0u, skipped = nIns > 0u /* count coinbase as skipped */;
|
|
Defer d([&t0, &num_ok, &skipped, nIns]{
|
|
if (t0.msec<int>() >= 50)
|
|
DebugM("Fetched ", num_ok, "/", nIns - skipped, " UTXOs from DB in ", t0.msecStr(3), " msec");
|
|
});
|
|
|
|
std::vector<std::pair<QByteArray, TXO>> & keyData = pf.keyData;
|
|
std::vector<rocksdb::Slice> & keys = pf.keys;
|
|
std::vector<rocksdb::PinnableSlice> & values = pf.values;
|
|
std::vector<rocksdb::Status> & statuses = pf.statuses;
|
|
Defer d2([&]{
|
|
statuses.clear();
|
|
values.clear();
|
|
keys.clear();
|
|
keyData.clear();
|
|
});
|
|
for (size_t inum = 1 /* coinbase, skip */; inum < nIns; ++inum) {
|
|
const auto & in = std::as_const(ppb->inputs)[inum];
|
|
if (in.parentTxOutIdx.has_value()) { ++skipped; /* spent in this block, skip */ }
|
|
else if (TXO txo{.txHash = in.prevoutHash, .outN = in.prevoutN}; !contains(txo)) {
|
|
++cacheMisses;
|
|
keyData.emplace_back(Serialize(txo), std::move(txo));
|
|
} else
|
|
++cacheHits;
|
|
}
|
|
if (keyData.empty()) return; // nothing to do!
|
|
// sort the key data (for faster rocksdb MultiGet)
|
|
std::sort(keyData.begin(), keyData.end());
|
|
// populate the key slices, etc
|
|
for (const auto & [key, txo] : keyData)
|
|
keys.emplace_back(key.constData(), static_cast<size_t>(key.size()));
|
|
values.resize(keys.size());
|
|
statuses.resize(keys.size());
|
|
m_db->MultiGet(readOpts, cf_utxo, keys.size(), keys.data(), values.data(), statuses.data(),
|
|
/* sorted_input = */ true);
|
|
for (size_t index = 0, nIndices = statuses.size(); index < nIndices; ++index) {
|
|
const auto & s = statuses[index];
|
|
TXO &txo = keyData[index].second;
|
|
if (s.ok()) [[likely]] {
|
|
bool ok;
|
|
TXOInfo info = Deserialize<TXOInfo>(FromSlice(values[index]), &ok);
|
|
if (!ok) [[unlikely]]
|
|
throw DatabaseSerializationError(QString("%1: Failed to deserialize TXOInfo for TXO \"%2\"")
|
|
.arg(name, txo.toString()));
|
|
else {
|
|
add(std::move(txo), std::move(info));
|
|
++num_ok;
|
|
}
|
|
} else {
|
|
throw DatabaseError(QString("%1: Error reading TXO \"%2\" from colfam %3: %4")
|
|
.arg(name, txo.toString(), CFName(cf_utxo), StatusString(s)));
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
|
|
void addAllDeferred() {
|
|
for (auto & [txo, txoinfo] : deferredAdds)
|
|
add(std::move(txo), std::move(txoinfo));
|
|
deferredAdds.clear();
|
|
}
|
|
|
|
public:
|
|
UTXOCache(const QString &name, rocksdb::DB *pdb, rocksdb::ColumnFamilyHandle *putxocf,
|
|
const rocksdb::ReadOptions & readOpts)
|
|
: name{name}, prefetcher{name + ".Prefetcher"}, m_db{pdb}, cf_utxo{putxocf}, readOpts{readOpts} {
|
|
DebugM(name, ": created");
|
|
}
|
|
|
|
~UTXOCache() {
|
|
DebugM(name, ": ", __func__, " - stats - cache hits: ", cacheHits, ", cache misses: ", cacheMisses);
|
|
}
|
|
|
|
void reserve(size_t hashMaps) { utxos.reserve(hashMaps); }
|
|
|
|
/// Figures out the best capacity to reserve based on a desired memory size.
|
|
void autoReserve(size_t memoryBytes) {
|
|
constexpr auto perEntryEstimatedCost = BaseEntrySize;
|
|
static_assert (perEntryEstimatedCost > 0);
|
|
reserve(memoryBytes / perEntryEstimatedCost); // ~5.6 million per GB of memory
|
|
}
|
|
|
|
/// NB: no locks on ppb are used for now. While this is alive ppb->inputs must not be mutated
|
|
/// NB2: call waitForPrefetchToComplete() after this is called sometime later.
|
|
/// Precondition: prefetcher must *not* already be running. If it is, this will throw std::domain_error.
|
|
void prefetch(const PreProcessedBlockPtr & ppb) { do_prefetch(ppb); }
|
|
|
|
/// May throw if the underlying CoTask work unit threw.
|
|
/// This is called by Storage::addBlock before we need to get and/or add UTXOs to the cache.
|
|
void waitForPrefetchToComplete() {
|
|
if (prefetcherFut.future.valid())
|
|
prefetcherFut.future.get();
|
|
|
|
// do any deferred adds now that the prefetcher is done/inactive
|
|
addAllDeferred();
|
|
}
|
|
|
|
size_t cacheMisses = 0, cacheHits = 0;
|
|
|
|
/// Get a UTXO from the cache. Will return a nullptr if the requested TXO was not in the cache. The returned
|
|
/// pointer, if not nullptr, points directly into the cache and is no longer valid if the cache later is mutated in
|
|
/// any way!
|
|
///
|
|
/// Does not fall-back to looking in the DB. Caller should explicitly call utxoGetFromDB() themselves
|
|
/// for that purpose. Note: this currently takes no locks. Assumption is calling code is locking
|
|
/// things correctly (this function is currently called only inside Storage::addBlock).
|
|
/// Precondition: prefetcher must not be running (this is not checked).
|
|
const TXOInfo * get(const TXO & txo) {
|
|
if (const auto it = utxos.find(txo); it != utxos.end()) {
|
|
++cacheHits;
|
|
return &it->second.txoInfo;
|
|
}
|
|
++cacheMisses;
|
|
return nullptr;
|
|
}
|
|
|
|
/// Add a TXO,TXOInfo pair to the cache. If the prefetcher is running, defers the add until
|
|
/// `waitForPrefetchToComplete()` is called.
|
|
bool put(TXO && txo, TXOInfo && info) {
|
|
if (prefetcherFut.future.valid()) {
|
|
// Prefetcher is busy, can't touch the data structures it is modifying now. Defer this until later.
|
|
deferredAdds.emplace_back(std::move(txo), std::move(info));
|
|
return false;
|
|
}
|
|
return add(std::move(txo), std::move(info));
|
|
}
|
|
|
|
bool remove(const TXO & txo) {
|
|
if (auto it = utxos.find(txo); it != utxos.end()) {
|
|
rm(it);
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/// Limit dynamic memory usage to `bytes`. Precondition: Prefetcher must not be running (this is not checked)
|
|
void limitSize(size_t bytes) { do_limitSize(bytes); }
|
|
}; // class Storage::UTXOCache
|
|
|
|
|
|
Storage::Storage(const std::shared_ptr<const Options> & options_)
|
|
: Mgr(nullptr), options(options_),
|
|
subsmgr(new ScriptHashSubsMgr(options, this)),
|
|
dspsubsmgr(new DSProofSubsMgr(options, this)),
|
|
txsubsmgr(new TransactionSubsMgr(options, this)),
|
|
p(std::make_unique<Pvt>(options->txHashCacheBytes))
|
|
{
|
|
setObjectName("Storage");
|
|
_thread.setObjectName(objectName());
|
|
}
|
|
|
|
Storage::~Storage() { Debug() << __func__; cleanup(); }
|
|
|
|
#if ((ROCKSDB_MAJOR << 16)|(ROCKSDB_MINOR << 8)|(ROCKSDB_PATCH)) > ((6 << 16)|(17 << 8)|(3)) // 6.17.3
|
|
#define HAS_ROCKSDB_NEW_VERSION_API 1
|
|
#else
|
|
#define HAS_ROCKSDB_NEW_VERSION_API 0
|
|
extern const char* rocksdb_build_git_sha; // internal to rocksdb lib -- if this breaks remove me
|
|
#endif
|
|
/* static */
|
|
QString Storage::rocksdbVersion()
|
|
{
|
|
#if !HAS_ROCKSDB_NEW_VERSION_API
|
|
QString sha(rocksdb_build_git_sha);
|
|
// rocksdb git commit sha: try and pop off the front part, and keep the rest and take the first 7 characters of that
|
|
if (auto l = sha.split(':'); l.size() == 2) // must match what we expect otherwise don't truncate
|
|
sha = l.back().left(7);
|
|
return QString("%1.%2.%3-%4").arg(ROCKSDB_MAJOR).arg(ROCKSDB_MINOR).arg(ROCKSDB_PATCH).arg(sha);
|
|
#else
|
|
const auto dbversion = QString::fromStdString(rocksdb::GetRocksVersionAsString(true));
|
|
const auto sha = []{
|
|
const auto &props = rocksdb::GetRocksBuildProperties();
|
|
if (auto it = props.find("rocksdb_build_git_sha"); it != props.end())
|
|
return QString::fromStdString(it->second).left(7);
|
|
return QString("unk");
|
|
}();
|
|
return QString("%1-%2").arg(dbversion, sha);
|
|
#endif
|
|
}
|
|
|
|
void Storage::startup()
|
|
{
|
|
Log() << "Loading database ...";
|
|
|
|
if (UNLIKELY(!subsmgr || !options || !dspsubsmgr || !txsubsmgr))
|
|
throw BadArgs("Storage instance constructed with nullptr for `options` and/or `subsmgr` and/or `dspsubsmgr` and/or `txsubsmgr` -- FIXME!");
|
|
|
|
subsmgr->startup(); // trivial, always succeeds if constructed correctly
|
|
dspsubsmgr->startup(); // trivial, always succeeds if constructed correctly
|
|
txsubsmgr->startup(); // trivial, always succeeds if constructed correctly
|
|
|
|
{
|
|
// set up the merkle cache object
|
|
using namespace std::placeholders;
|
|
p->merkleCache = std::make_unique<Merkle::Cache>(std::bind(&Storage::merkleCacheHelperFunc, this, _1, _2, _3));
|
|
}
|
|
|
|
// open DB and all column families ...
|
|
openOrCreateDB();
|
|
|
|
// load/check meta
|
|
{
|
|
const QString errMsg1{"Incompatible database format -- delete the datadir and resynch."};
|
|
const QString errMsg2{errMsg1 + " RocksDB error"};
|
|
if (const auto opt = GenericDBGet<Meta>(p->db, p->db.meta, kMeta, true, errMsg2);
|
|
opt.has_value())
|
|
{
|
|
const Meta &m_db = *opt;
|
|
if (!m_db.isMagicOk() || !m_db.isVersionSupported()
|
|
|| (m_db.isMinimumExtraPlatformInfoVersion() && m_db.platformBits != p->meta.platformBits)) {
|
|
throw DatabaseFormatError(errMsg1);
|
|
}
|
|
p->meta = m_db;
|
|
Debug () << "Read meta from db ok";
|
|
if (!p->meta.coin.isEmpty())
|
|
Log() << "Coin: " << p->meta.coin;
|
|
if (!p->meta.chain.isEmpty())
|
|
Log() << "Chain: " << p->meta.chain;
|
|
} else {
|
|
// ok, did not exist .. write a new one to db
|
|
saveMeta_impl();
|
|
}
|
|
if (isDirty()) {
|
|
throw DatabaseError("It appears that " APPNAME " was forcefully killed in the middle of committing a block to the db. "
|
|
"We cannot figure out where exactly in the update process " APPNAME " was killed, so we "
|
|
"cannot undo the inconsistent state caused by the unexpected shutdown. Sorry!"
|
|
"\n\nThe database has been corrupted. Please delete the datadir and resynch to bitcoind.\n");
|
|
}
|
|
}
|
|
|
|
// load headers -- may throw.. this must come first
|
|
loadCheckHeadersInDB();
|
|
// check txnums
|
|
loadCheckTxNumsDRAAndBlkInfo();
|
|
// construct the TxHash2TxNum manager -- depends on the above function having constructed the txNumFile
|
|
loadCheckTxHash2TxNumMgr();
|
|
// count utxos -- note this depends on "blkInfos" being filled in so it much be called after loadCheckTxNumsDRAAndBlkInfo()
|
|
loadCheckUTXOsInDB();
|
|
// very slow check, only runs if -C -C (specified twice)
|
|
loadCheckShunspentInDB();
|
|
// load check earliest undo to populate earliestUndoHeight
|
|
loadCheckEarliestUndo();
|
|
// load rpa data
|
|
if (isRpaEnabled()) loadCheckRpaDB();
|
|
// if user specified --compact-dbs on CLI, run the compaction now before returning
|
|
compactAllDBs();
|
|
|
|
// start up the co-task we use in addBlock and undoLatestBlock
|
|
p->blocksWorker = std::make_unique<CoTask>("Storage Worker");
|
|
|
|
// Detect old DB version and see if upgrade is permitted, and maybe do a DB upgrade...
|
|
checkUpgradeDBVersion();
|
|
|
|
start(); // starts our thread
|
|
}
|
|
|
|
void Storage::openOrCreateDB()
|
|
{
|
|
gentlyCloseDB(); // Ensure we start from a clean slate (in case this function is ever called to hot-reopen the DB)
|
|
|
|
// Optimize RocksDB. This is the easiest way to get RocksDB to perform well
|
|
rocksdb::Options & opts(p->db.opts);
|
|
rocksdb::ColumnFamilyOptions &shistOpts(p->db.shistOpts), &txhash2txnumOpts(p->db.txhash2txnumOpts),
|
|
&txnum2txhashOpts(p->db.txnum2txhashOpts), &headersOpts(p->db.headersOpts);
|
|
opts.IncreaseParallelism(int(Util::getNPhysicalProcessors()));
|
|
opts.OptimizeLevelStyleCompaction();
|
|
|
|
// setup shared block cache
|
|
rocksdb::BlockBasedTableOptions tableOptions;
|
|
tableOptions.block_cache = rocksdb::NewLRUCache(options->db.maxMem /* capacity limit */, -1, false /* strict capacity limit=off, turning it on made db writes sometimes fail */);
|
|
p->db.blockCache = tableOptions.block_cache; // save shared_ptr to weak_ptr
|
|
tableOptions.cache_index_and_filter_blocks = true; // from the docs: this may be a large consumer of memory, cost & cap its memory usage to the cache
|
|
std::shared_ptr<rocksdb::TableFactory> tableFactory{rocksdb::NewBlockBasedTableFactory(tableOptions)};
|
|
// shared TableFactory for all db instances
|
|
opts.table_factory = tableFactory;
|
|
|
|
// setup shared write buffer manager (for memtables memory budgeting)
|
|
// - TODO right now we fix the cap of the write buffer manager's buffer size at db.maxMem / 2; tweak this.
|
|
auto writeBufferManager = std::make_shared<rocksdb::WriteBufferManager>(options->db.maxMem / 2, tableOptions.block_cache /* cost to block cache: hopefully this caps memory better? it appears to use locks though so many this will be slow?! TODO: experiment with and without this!! */);
|
|
p->db.writeBufferManager = writeBufferManager; // save shared_ptr to weak_ptr
|
|
opts.write_buffer_manager = writeBufferManager; // will be shared across all DB instances
|
|
|
|
// create the DB if it's not already present
|
|
opts.create_if_missing = true;
|
|
opts.error_if_exists = false;
|
|
opts.max_open_files = options->db.maxOpenFiles <= 0 ? -1 : options->db.maxOpenFiles; ///< this affects memory usage see: https://github.com/facebook/rocksdb/issues/4112
|
|
opts.keep_log_file_num = options->db.keepLogFileNum;
|
|
opts.compression = rocksdb::CompressionType::kNoCompression; // for now we test without compression. TODO: characterize what is fastest and best..
|
|
opts.use_fsync = options->db.useFsync; // the false default is perfectly safe, but Jt asked for this as an option, so here it is.
|
|
|
|
shistOpts = opts; // copy what we just did (will implicitly copy over the shared table_factory and write_buffer_manager)
|
|
shistOpts.merge_operator = p->db.concatOperator = std::make_shared<StorageDetail::ConcatOperator>(); // this set of options uses the concat merge operator (we use this to append to history entries in the db)
|
|
|
|
txhash2txnumOpts = opts;
|
|
txhash2txnumOpts.merge_operator = p->db.concatOperatorTxHash2TxNum = std::make_shared<StorageDetail::ConcatOperator>();
|
|
|
|
txnum2txhashOpts = opts;
|
|
txnum2txhashOpts.merge_operator = p->db.concatOperatorTxNum2TxHash = std::make_shared<StorageDetail::ConcatOperator>();
|
|
|
|
headersOpts = opts;
|
|
headersOpts.merge_operator = p->db.concatOperatorHeaders = std::make_shared<StorageDetail::ConcatOperator>();
|
|
|
|
const auto &colFamsTable = p->db.colFamsTable;
|
|
auto colFamsNeeded = colFamsTable;
|
|
|
|
// First, open the DB, and then determine which column families it has, and open them all.
|
|
{
|
|
const QString mainDBName = "fulcrum_db";
|
|
const QString path = options->datadir + QDir::separator() + mainDBName;
|
|
rocksdb::Status s;
|
|
|
|
std::vector<rocksdb::ColumnFamilyDescriptor> colFamDescs;
|
|
{
|
|
std::vector<std::string> haveColFams;
|
|
s = rocksdb::DB::ListColumnFamilies(opts, path.toStdString(), &haveColFams);
|
|
if (s.ok()) {
|
|
colFamDescs.reserve(haveColFams.size());
|
|
for (const auto &colName : haveColFams) {
|
|
QString extra;
|
|
// determine option for this column family
|
|
std::optional<rocksdb::ColumnFamilyOptions> optOptions;
|
|
if (auto it = colFamsNeeded.find(colName); it != colFamsNeeded.end()) {
|
|
optOptions = std::move(it->second.options);
|
|
// mark db cols that the db has that we want as "no longer needed"
|
|
colFamsNeeded.erase(it);
|
|
} else if (colName == rocksdb::kDefaultColumnFamilyName) {
|
|
extra = " (ignored)";
|
|
} else {
|
|
extra = " (UNKNOWN)";
|
|
}
|
|
Debug() << "Found DB column family '" << QString::fromStdString(colName) << "'" << extra;
|
|
colFamDescs.emplace_back(colName, optOptions.value_or(opts));
|
|
}
|
|
} else {
|
|
QString sstr;
|
|
if (s.IsIOError() && !QFileInfo::exists(path))
|
|
sstr = " (new db)";
|
|
else
|
|
sstr = ": " + StatusString(s);
|
|
Debug() << "Failed to list column families for DB '" << mainDBName << "'" << sstr;
|
|
// Might as well specify that we want to open the "default" column family here
|
|
colFamDescs.emplace_back(rocksdb::kDefaultColumnFamilyName, opts);
|
|
}
|
|
}
|
|
|
|
rocksdb::DB *db = nullptr;
|
|
s = rocksdb::DB::Open(opts, path.toStdString(), colFamDescs, &p->db.columnFamilies, &db);
|
|
p->db.db.reset(db);
|
|
if (!s.ok() || !db)
|
|
throw DatabaseError(QString("Error opening %1 database: %2 (path: %3)")
|
|
.arg(mainDBName, StatusString(s), path));
|
|
}
|
|
|
|
rocksdb::DB * const db = p->db;
|
|
assert(db != nullptr);
|
|
|
|
// Next, for all the colFamsNeeded that weren't in the DB already (new DB, etc), create them!
|
|
{
|
|
std::vector<rocksdb::ColumnFamilyDescriptor> colFamDescs;
|
|
for (const auto & [name, params] : colFamsNeeded) {
|
|
colFamDescs.emplace_back(name, params.options);
|
|
}
|
|
if (!colFamDescs.empty()) {
|
|
std::vector<rocksdb::ColumnFamilyHandle *> handles;
|
|
auto s = db->CreateColumnFamilies(colFamDescs, &handles);
|
|
// "save" the successfully opened handles so we can close them later even on erro
|
|
for (auto *h : handles) {
|
|
p->db.columnFamilies.push_back(h);
|
|
colFamsNeeded.erase(h->GetName());
|
|
}
|
|
// error out if there was a problem
|
|
if (!s.ok()) {
|
|
throw DatabaseError(QString("Error opening %1 column families: %2").arg(quint64(colFamDescs.size()))
|
|
.arg(StatusString(s)));
|
|
}
|
|
}
|
|
}
|
|
|
|
// Lastly, assign the ColumnFamilyHandle pointers ...
|
|
for (rocksdb::ColumnFamilyHandle *h : p->db.columnFamilies) {
|
|
const auto &name = h->GetName();
|
|
if (auto it = colFamsTable.find(name); it != colFamsTable.end()) [[likely]] {
|
|
// assign ptr; this modifies members: p.db.meta, p.db.blkinfo, etc..
|
|
it->second.handle = h;
|
|
} else if (name != rocksdb::kDefaultColumnFamilyName) [[unlikely]] {
|
|
throw DatabaseError(QString("Encountered an unknown column family in DB: %1"
|
|
" -- incompatible or newer than expected database, perhaps?")
|
|
.arg(QString::fromStdString(name)));
|
|
}
|
|
}
|
|
|
|
if (!colFamsNeeded.empty()) [[unlikely]] {
|
|
QStringList names;
|
|
for (const auto & [name, _] : colFamsNeeded) names.append(QString::fromStdString(name));
|
|
throw InternalError("Missing column families: " + names.join(", "));
|
|
}
|
|
|
|
if (!p->db.blkinfo || !p->db.meta || !p->db.rpa || !p->db.shist || !p->db.shunspent || !p->db.txhash2txnum
|
|
|| !p->db.utxoset || !p->db.undo || !p->db.txnum2txhash || !p->db.headers) [[unlikely]]
|
|
throw InternalError("A required column family handle is still nullptr! FIXME!");
|
|
|
|
Log() << "DB memory: " << QString::number(options->db.maxMem / 1024. / 1024., 'f', 2) << " MiB";
|
|
}
|
|
|
|
void Storage::checkUpgradeDBVersion()
|
|
{
|
|
// Note: A precondition for this function is that database, headers, etc are already loaded.
|
|
|
|
// Original Fulcrum DB version before 1.9.0 was v1, then there was v2 which added CashToken data for BCH.
|
|
// Now we are on v3 as of 1.11.0+, whose only difference vs v2 is additional platform info saved to `Meta`.
|
|
//
|
|
// Going from v1 on BTC/LTC -> v2+ is ok without caveats. For BCH, we must warn the user if their DB is v1
|
|
// and it's after the upgrade9 activation time, because then the DB will be missing token data and may have
|
|
// token-containing UTXOs indexed to the wrong script hash.
|
|
Log() << "DB version: v" << p->meta.version;
|
|
if (p->meta.isMinimumExtraPlatformInfoVersion()) {
|
|
Debug() << "DB last written-to by: " << p->meta.appName << " " << p->meta.appVersion
|
|
<< " using rocksdb: " << p->meta.rocksDBVersion;
|
|
Debug() << "DB last written-to OS: " << p->meta.osName << ", CPU: " << p->meta.cpuArch << ", ABI: " << p->meta.buildABI
|
|
<< ", bits: " << p->meta.platformBits;
|
|
}
|
|
if (p->meta.version < Meta::kCurrentVersion) {
|
|
if (BTC::coinFromName(p->meta.coin) == BTC::Coin::BCH && p->meta.version < Meta::kMinBCHUpgrade9Version) {
|
|
// Get the latest header to detect if we are after the activation time
|
|
const Header hdr = headerVerifier().first.lastHeaderProcessed().second;
|
|
if (hdr.size() == BTC::GetBlockHeaderSize()) {
|
|
const auto bhdr = [&hdr] {
|
|
try {
|
|
return BTC::Deserialize<bitcoin::CBlockHeader>(hdr, 0, false, false, true, true);
|
|
} catch (const std::ios_base::failure &e) {
|
|
throw InternalError(QString("checkUpgradeDBVersion: Failed to deserialize the latest block"
|
|
" header: %1").arg(e.what()));
|
|
}
|
|
}();
|
|
const int64_t upgrade9ActivationTime = BTC::NetFromName(p->meta.chain) == BTC::Net::ChipNet
|
|
? 1668513600 // ChipNet: November 15, 2022 12:00:00 UTC
|
|
: 1684152000; // MainNet, etc: May 15, 2023 12:00:00 UTC
|
|
if (bhdr.GetBlockTime() >= upgrade9ActivationTime) {
|
|
// Uh-oh. They have a synched db that is v1, but the upgrade has already activated. Complain
|
|
// and abort out, insisting that the user re-synch the DB.
|
|
throw DatabaseError("This datadir was synched using an older version of " APPNAME " which lacked"
|
|
" full CashToken support, however Upgrade9 has already activated for this"
|
|
" chain.\n\nPlease delete the datadir and resynch to bitcoind.\n");
|
|
}
|
|
}
|
|
}
|
|
|
|
Log() << "DB version is older but compatible, updating version to v" << Meta::kCurrentVersion << " ...";
|
|
p->meta.version = Meta::kCurrentVersion;
|
|
}
|
|
// Set the platform info from the current process, and re-save to DB
|
|
p->meta.makePlatformInfoCurrent();
|
|
saveMeta_impl();
|
|
}
|
|
|
|
void Storage::compactAllDBs()
|
|
{
|
|
if (!options->compactDBs)
|
|
return;
|
|
auto *db = p->db.get();
|
|
if (!db) return;
|
|
size_t ctr = 0;
|
|
App *ourApp = app();
|
|
Tic t0;
|
|
Log() << "Compacting DB column families, please wait ...";
|
|
for (const auto &cf : p->db.columnFamilies) {
|
|
if (ourApp->signalsCaught())
|
|
break;
|
|
if (!cf) continue;
|
|
const auto name = CFName(cf);
|
|
Log() << "Compacting " << name << " ...";
|
|
rocksdb::CompactRangeOptions opts;
|
|
opts.allow_write_stall = true;
|
|
opts.exclusive_manual_compaction = true;
|
|
opts.change_level = true;
|
|
auto s = db->CompactRange(opts, cf, nullptr, nullptr);
|
|
if (!s.ok()) {
|
|
throw DatabaseError(QString("Error compacting column family %1: %2")
|
|
.arg(name, StatusString(s)));
|
|
}
|
|
++ctr;
|
|
}
|
|
Log() << "Compacted " << ctr << " column families in " << t0.secsStr(1) << " seconds";
|
|
}
|
|
|
|
void Storage::gentlyCloseDB()
|
|
{
|
|
p->db.utxoCache.reset();
|
|
p->db.txhash2txnumMgr.reset();
|
|
p->db.txNumsDRA.reset();
|
|
p->db.headersDRA.reset();
|
|
|
|
// do Flush of each column family, and close the handles
|
|
auto *db = p->db.get();
|
|
if (!db) return;
|
|
for (auto & cf : p->db.columnFamilies) {
|
|
if (!cf) continue;
|
|
const auto name = CFName(cf);
|
|
Debug() << "Flushing column family: " << name << " ...";
|
|
rocksdb::Status status;
|
|
rocksdb::FlushOptions fopts;
|
|
fopts.wait = true; fopts.allow_write_stall = true;
|
|
status = db->Flush(fopts, cf);
|
|
if (!status.ok())
|
|
Warning() << "Flush of " << name << ": " << StatusString(status);
|
|
status = db->DestroyColumnFamilyHandle(cf);
|
|
if (!status.ok())
|
|
Warning() << "Release of " << name << ": " << StatusString(status);
|
|
cf = nullptr;
|
|
}
|
|
p->db.columnFamilies.clear();
|
|
// Clear members (they were all pointers owned by the p->db.columnFamilies array)
|
|
p->db.blkinfo = p->db.meta = p->db.shist = p->db.shunspent = p->db.rpa = p->db.txhash2txnum = p->db.undo = p->db.utxoset = p->db.txnum2txhash = p->db.headers = nullptr;
|
|
|
|
// do SyncWAL() and Close() to gently close the DB
|
|
auto name = DBName(p->db);
|
|
Debug() << "Synching WAL: " << name << " ...";
|
|
auto status = db->SyncWAL();
|
|
if (!status.ok())
|
|
Warning() << "SyncWAL of " << name << ": " << StatusString(status);
|
|
Debug() << "Closing DB: " << name << " ...";
|
|
status = db->Close();
|
|
if (!status.ok())
|
|
Warning() << "Close of " << name << ": " << StatusString(status);
|
|
// kill the concat operators we used
|
|
p->db.concatOperator = p->db.concatOperatorTxHash2TxNum = p->db.concatOperatorTxNum2TxHash = p->db.concatOperatorHeaders = nullptr;
|
|
// delete db
|
|
p->db.db.reset(db = nullptr);
|
|
}
|
|
|
|
void Storage::cleanup()
|
|
{
|
|
stop(); // joins our thread
|
|
if (p->blocksWorker) p->blocksWorker.reset(); // stop the co-task
|
|
if (txsubsmgr) txsubsmgr->cleanup();
|
|
if (dspsubsmgr) dspsubsmgr->cleanup();
|
|
if (subsmgr) subsmgr->cleanup();
|
|
gentlyCloseDB();
|
|
}
|
|
|
|
|
|
auto Storage::stats() const -> Stats
|
|
{
|
|
// TODO ... more stuff here, perhaps
|
|
QVariantMap ret;
|
|
auto & c = p->db.concatOperator, & c2 = p->db.concatOperatorTxHash2TxNum, & c3 = p->db.concatOperatorTxNum2TxHash,
|
|
& c4 = p->db.concatOperatorHeaders;
|
|
ret["merge calls"] = c ? static_cast<quint64>(c->merges.load()) : QVariant();
|
|
ret["merge calls (txhash2txnum)"] = c2 ? static_cast<quint64>(c2->merges.load()) : QVariant();
|
|
ret["merge calls (txnum2txhash)"] = c3 ? static_cast<quint64>(c3->merges.load()) : QVariant();
|
|
ret["merge calls (headers)"] = c4 ? static_cast<quint64>(c4->merges.load()) : QVariant();
|
|
QVariantMap caches;
|
|
{
|
|
QVariantMap m;
|
|
|
|
const auto sz = p->lruNum2Hash.size(), szBytes = p->lruNum2Hash.totalCost(), maxSzBytes = p->lruNum2Hash.maxCost();
|
|
m["Size bytes"] = qlonglong(szBytes);
|
|
m["max bytes"] = qlonglong(maxSzBytes);
|
|
m["nItems"] = qlonglong(sz);
|
|
m["~hits"] = qlonglong(p->lruCacheStats.num2HashHits);
|
|
m["~misses"] = qlonglong(p->lruCacheStats.num2HashMisses);
|
|
caches["LRU Cache: TxNum -> TxHash"] = m;
|
|
}
|
|
{
|
|
QVariantMap m;
|
|
const unsigned nItems = p->lruHeight2Hashes_BitcoindMemOrder.size(), szBytes = p->lruHeight2Hashes_BitcoindMemOrder.totalCost(),
|
|
maxSzBytes = p->lruHeight2Hashes_BitcoindMemOrder.maxCost();
|
|
m["Size bytes"] = szBytes;
|
|
m["max bytes"] = qlonglong(maxSzBytes);
|
|
m["nBlocks"] = nItems;
|
|
m["~hits"] = qlonglong(p->lruCacheStats.height2HashesHits);
|
|
m["~misses"] = qlonglong(p->lruCacheStats.height2HashesMisses);
|
|
caches["LRU Cache: Block Height -> TxHashes"] = m;
|
|
}
|
|
{
|
|
const size_t nHashes = p->merkleCache->size(), bytes = nHashes * (HashLen + sizeof(HeaderHash));
|
|
caches["merkleHeaders_Size"] = qulonglong(nHashes);
|
|
caches["merkleHeaders_SizeBytes"] = qulonglong(bytes);
|
|
}
|
|
ret["caches"] = caches;
|
|
{
|
|
const auto &db = p->db.db;
|
|
// db stats
|
|
QVariantMap m;
|
|
QVariantMap m2;
|
|
const QString name = QFileInfo(QString::fromStdString(db->GetName())).fileName();
|
|
for (const auto prop : { "rocksdb.estimate-table-readers-mem", "rocksdb.cur-size-all-mem-tables"}) {
|
|
if (std::string s; LIKELY(db->GetProperty(prop, &s)) )
|
|
m2[prop] = QString::fromStdString(s);
|
|
}
|
|
if (auto fact = db->GetOptions().table_factory; fact) [[likely]] {
|
|
// parse the table factory options string, which is of the form " opt1: val1\n opt2: val2\n ... "
|
|
QVariantMap m3;
|
|
QString rocksdbOptionsString;
|
|
#if __has_include(<rocksdb/configurable.h>)
|
|
// Newer rocksdb API uses GetPrintableOptions
|
|
rocksdbOptionsString = QString::fromStdString( fact->GetPrintableOptions() );
|
|
#else
|
|
// Older rocksdb API used GetPrintableTableOptions
|
|
rocksdbOptionsString = QString::fromStdString( fact->GetPrintableTableOptions() );
|
|
#endif
|
|
for (const auto & line : rocksdbOptionsString.split("\n")) {
|
|
const auto nvp = line.split(":");
|
|
if (nvp.size() < 2)
|
|
continue;
|
|
auto n = nvp.first().trimmed().simplified();
|
|
auto v = nvp.mid(1).join(":").trimmed().simplified();
|
|
m3[n] = v;
|
|
}
|
|
m2["table factory options"] = m3;
|
|
} else
|
|
m2["table factory options"] = QVariant(); // explicitly state it was null (this branch should not normally happen)
|
|
m2["max_open_files"] = db->GetOptions().max_open_files;
|
|
m2["keep_log_file_num"] = qulonglong(db->GetOptions().keep_log_file_num);
|
|
m[name] = m2;
|
|
ret["DB Stats"] = m;
|
|
if (const auto cache = p->db.blockCache.lock(); cache) {
|
|
QVariantMap cmap;
|
|
cmap["usage"] = qulonglong(cache->GetUsage());
|
|
cmap["capacity"] = qulonglong(cache->GetCapacity());
|
|
ret["DB Shared Block Cache"] = cmap;
|
|
}
|
|
if (const auto wbm = p->db.writeBufferManager.lock(); wbm) {
|
|
QVariantMap wmap;
|
|
const bool en = wbm->enabled();
|
|
wmap["enabled"] = en;
|
|
if (en) {
|
|
// these stats are invalid if not enabled, so only add them if enabled
|
|
wmap["is costed to cache"] = wbm->cost_to_cache();
|
|
wmap["buffer size"] = qulonglong(wbm->buffer_size());
|
|
wmap["memory usage"] = qulonglong(wbm->memory_usage());
|
|
}
|
|
ret["DB Shared Write Buffer Manager"] = wmap;
|
|
}
|
|
|
|
{
|
|
// RPA-specific stats
|
|
QVariantMap rm;
|
|
rm["firstHeight"] = p->rpaInfo.firstHeight.load(std::memory_order_relaxed);
|
|
rm["lastHeight"] = p->rpaInfo.lastHeight.load(std::memory_order_relaxed);
|
|
rm["nReads"] = qulonglong(p->rpaInfo.nReads.load(std::memory_order_relaxed));
|
|
rm["nWrites"] = qulonglong(p->rpaInfo.nWrites.load(std::memory_order_relaxed));
|
|
rm["nDeletions"] = qulonglong(p->rpaInfo.nDeletions.load(std::memory_order_relaxed));
|
|
rm["nBytesRead"] = qulonglong(p->rpaInfo.nBytesRead.load(std::memory_order_relaxed));
|
|
rm["nBytesWritten"] = qulonglong(p->rpaInfo.nBytesWritten.load(std::memory_order_relaxed));
|
|
rm["needsFullCheck"] = p->rpaInfo.rpaNeedsFullCheckCachedVal.load(std::memory_order_relaxed);
|
|
ret["RPA Index Info"] = rm;
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
// Keep returned LockGuard in scope while you use the HeaderVerifier
|
|
auto Storage::headerVerifier() -> std::pair<BTC::HeaderVerifier &, ExclusiveLockGuard>
|
|
{
|
|
return std::pair<BTC::HeaderVerifier &, ExclusiveLockGuard>( p->headerVerifier, p->headerVerifierLock );
|
|
}
|
|
auto Storage::headerVerifier() const -> std::pair<const BTC::HeaderVerifier &, SharedLockGuard>
|
|
{
|
|
return std::pair<const BTC::HeaderVerifier &, SharedLockGuard>( p->headerVerifier, p->headerVerifierLock );
|
|
}
|
|
|
|
|
|
QString Storage::getChain() const
|
|
{
|
|
SharedLockGuard l(p->metaLock);
|
|
return p->meta.chain;
|
|
}
|
|
|
|
void Storage::setChain(const QString &chain)
|
|
{
|
|
{
|
|
ExclusiveLockGuard l(p->metaLock);
|
|
p->meta.chain = chain; // set chain for saving
|
|
}
|
|
if (!chain.isEmpty())
|
|
Log() << "Chain: " << chain;
|
|
save(SaveItem::Meta);
|
|
}
|
|
|
|
QString Storage::getCoin() const
|
|
{
|
|
SharedLockGuard l(p->metaLock);
|
|
return p->meta.coin;
|
|
}
|
|
|
|
void Storage::setCoin(const QString &coin) {
|
|
{
|
|
ExclusiveLockGuard l(p->metaLock);
|
|
p->meta.coin = coin;
|
|
}
|
|
if (!coin.isEmpty())
|
|
Log() << "Coin: " << coin;
|
|
save(SaveItem::Meta);
|
|
}
|
|
|
|
bool Storage::isRpaEnabled() const
|
|
{
|
|
using ES = Options::Rpa::EnabledSpec;
|
|
switch(options->rpa.enabledSpec) {
|
|
case ES::Enabled: return true;
|
|
case ES::Disabled: return false;
|
|
case ES::Auto: return BTC::coinFromName(getCoin()) == BTC::Coin::BCH;
|
|
}
|
|
return false; // not normally reached; suppress compiler warnings
|
|
}
|
|
|
|
int Storage::getConfiguredRpaStartHeight() const
|
|
{
|
|
if (!isRpaEnabled()) return -1; // -1 to caller means "rpa not enabled"
|
|
if (const int reqHt = options->rpa.requestedStartHeight; reqHt >= 0)
|
|
return reqHt; // user requested a specific start height >= 0
|
|
|
|
// otherwise, do "auto", which is 825,000 for mainnet, 0 for all other nets
|
|
if (BTC::NetFromName(getChain()) == BTC::Net::MainNet)
|
|
return Options::Rpa::defaultStartHeightForMainnet;
|
|
return Options::Rpa::defaultStartHeightOtherNets;
|
|
}
|
|
|
|
auto Storage::getRpaDBHeightRange() const -> std::optional<HeightRange>
|
|
{
|
|
std::optional<HeightRange> ret;
|
|
if (isRpaEnabled())
|
|
if (const int from = p->rpaInfo.firstHeight, to = p->rpaInfo.lastHeight; from >= 0 && to >= 0)
|
|
ret.emplace(static_cast<BlockHeight>(from), static_cast<BlockHeight>(to));
|
|
return ret;
|
|
}
|
|
|
|
/// returns the "next" TxNum
|
|
TxNum Storage::getTxNum() const { return p->txNumNext.load(); }
|
|
|
|
auto Storage::latestTip(Header *hdrOut) const -> std::pair<int, HeaderHash> {
|
|
static_assert(std::is_same_v<Header, HeaderHash> && std::is_same_v<Header, QByteArray>); // both must be QByteArray
|
|
std::pair<int, HeaderHash> ret = headerVerifier().first.lastHeaderProcessed(); // ok; lock stays locked until statement end.
|
|
if (hdrOut) *hdrOut = ret.second; // this is not a hash but the actual block header
|
|
if (ret.second.isEmpty() || ret.first < 0) {
|
|
ret.first = -1;
|
|
ret.second.clear();
|
|
if (hdrOut) hdrOut->clear();
|
|
} else {
|
|
// .ret now has the actual header but we want the hash
|
|
ret.second = BTC::HashRev(ret.second);
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
auto Storage::latestHeight() const -> std::optional<BlockHeight>
|
|
{
|
|
std::optional<BlockHeight> ret;
|
|
SharedLockGuard g(p->blkInfoLock);
|
|
if (!p->blkInfos.empty())
|
|
ret = BlockHeight(p->blkInfos.size()-1);
|
|
return ret;
|
|
}
|
|
|
|
void Storage::save(SaveSpec typed_spec)
|
|
{
|
|
using IntType = decltype(p->pendingSaves.load());
|
|
// enqueue save on event loop if not previously enqueued (we know it was previously enqueued if the p->pendingSaves
|
|
// atomic variable is not 0).
|
|
if (const auto spec = IntType(typed_spec); ! p->pendingSaves.fetch_or(spec))
|
|
{
|
|
QTimer::singleShot(0, this, [this]{save_impl();});
|
|
}
|
|
}
|
|
|
|
void Storage::save_impl(SaveSpec override)
|
|
{
|
|
if (const auto flags = SaveSpec(p->pendingSaves.exchange(0))|override; flags) { // atomic clear of flags, grab prev val
|
|
try {
|
|
if (flags & SaveItem::Meta) { // Meta
|
|
SharedLockGuard l(p->metaLock);
|
|
saveMeta_impl();
|
|
}
|
|
} catch (const std::exception & e) {
|
|
Fatal() << e.what(); // will abort app...
|
|
}
|
|
}
|
|
}
|
|
|
|
void Storage::saveMeta_impl()
|
|
{
|
|
if (!p->db.meta) return;
|
|
if (auto status = p->db->Put(p->db.defWriteOpts, p->db.meta, kMeta, ToSlice(Serialize(p->meta))); !status.ok()) {
|
|
throw DatabaseError("Failed to write meta to db");
|
|
}
|
|
|
|
DebugM("Wrote new metadata to db");
|
|
}
|
|
|
|
void Storage::appendHeader(rocksdb::WriteBatch &batch, const Header &h, BlockHeight height)
|
|
{
|
|
auto ctx = p->db.headersDRA->beginBatchWrite(batch);
|
|
const auto targetHeight = p->db.headersDRA->numRecords();
|
|
if (height != targetHeight) [[unlikely]]
|
|
throw InternalError(QString("Bad use of appendHeader -- expected height %1, got height %2").arg(targetHeight).arg(height));
|
|
QString err;
|
|
const auto res = ctx.append(h, &err);
|
|
if (!err.isEmpty()) [[unlikely]]
|
|
throw DatabaseError(QString("Failed to append header %1: %2").arg(height).arg(err));
|
|
else if (!res || p->db.headersDRA->numRecords() != height + 1u) [[unlikely]]
|
|
throw DatabaseError(QString("Failed to append header %1: result is bad").arg(height));
|
|
}
|
|
|
|
void Storage::deleteHeadersPastHeight(rocksdb::WriteBatch &batch, BlockHeight height)
|
|
{
|
|
QString err;
|
|
auto ctx = p->db.headersDRA->beginBatchWrite(batch);
|
|
const auto res = ctx.truncate(height + 1u, &err);
|
|
if (!err.isEmpty())
|
|
throw DatabaseError(QString("Failed to truncate headers past height %1: %2").arg(height).arg(err));
|
|
else if (!res || p->db.headersDRA->numRecords() != height + 1u)
|
|
throw InternalError("header truncate resulted in an unexepected value");
|
|
}
|
|
|
|
auto Storage::headerForHeight(BlockHeight height, QString *err) const -> std::optional<Header>
|
|
{
|
|
std::optional<Header> ret;
|
|
if (int(height) <= latestTip().first && int(height) >= 0) {
|
|
ret = headerForHeight_nolock(height, err);
|
|
} else if (err) { *err = QStringLiteral("Height %1 is out of range").arg(height); }
|
|
return ret;
|
|
}
|
|
|
|
auto Storage::headerForHeight_nolock(BlockHeight height, QString *err) const -> std::optional<Header>
|
|
{
|
|
std::optional<Header> ret;
|
|
try {
|
|
QString err1;
|
|
ret.emplace( p->db.headersDRA->readRecord(height, &err1) );
|
|
if (!err1.isEmpty()) {
|
|
ret.reset();
|
|
throw DatabaseError(QString("failed to read header %1: %2").arg(height).arg(err1));
|
|
}
|
|
} catch (const std::exception &e) {
|
|
if (err) *err = e.what();
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
auto Storage::headersFromHeight_nolock_nocheck(BlockHeight height, unsigned num, QString *err) const -> std::vector<Header>
|
|
{
|
|
if (err) err->clear();
|
|
std::vector<Header> ret = p->db.headersDRA->readRecords(height, num, err);
|
|
|
|
if (ret.size() != num && err && err->isEmpty())
|
|
*err = "short header count returned from headers file";
|
|
|
|
ret.shrink_to_fit();
|
|
return ret;
|
|
}
|
|
|
|
/// Convenient batched alias for above. Returns a set of headers starting at height. May return < count if not
|
|
/// all headers were found. Thead safe.
|
|
auto Storage::headersFromHeight(BlockHeight height, unsigned count, QString *err) const -> std::vector<Header>
|
|
{
|
|
std::vector<Header> ret;
|
|
SharedLockGuard g(p->blocksLock); // to ensure clients get a consistent view
|
|
int num = std::min(1 + latestTip().first - int(height), int(count)); // note this also takes a lock briefly so we need to do this after the lockguard above
|
|
if (num > 0) {
|
|
ret = headersFromHeight_nolock_nocheck(height, count, err);
|
|
} else if (err) *err = "No headers in the specified range";
|
|
return ret;
|
|
}
|
|
|
|
|
|
void Storage::loadCheckHeadersInDB()
|
|
{
|
|
assert(p->blockHeaderSize() > 0);
|
|
p->db.headersDRA = std::make_unique<DBRecordArray>(*p->db, *p->db.headers, /* recSz = */ size_t(p->blockHeaderSize()),
|
|
/* bucketNItems = */ 8, /* magic = */ 0x00f026a1); // may throw
|
|
|
|
Log() << "Verifying headers ...";
|
|
uint32_t num = static_cast<uint32_t>(p->db.headersDRA->numRecords());
|
|
std::vector<QByteArray> hVec;
|
|
const auto t0 = Util::getTimeNS();
|
|
{
|
|
if (num > MAX_HEADERS)
|
|
throw DatabaseFormatError(QString("Header count (%1) in database exceeds MAX_HEADERS! This is likely due to"
|
|
" a database format mistmatch. Delete the datadir and resynch it.")
|
|
.arg(num));
|
|
// verify headers: hashPrevBlock must match what we actually read from db
|
|
if (num) {
|
|
Debug() << "Verifying " << num << " " << Util::Pluralize("header", num) << " ...";
|
|
QString err;
|
|
hVec = headersFromHeight_nolock_nocheck(0, num, &err);
|
|
if (!err.isEmpty() || hVec.size() != num)
|
|
throw DatabaseFormatError(QString("%1. Possible databaase corruption. Delete the datadir and resynch.").arg(err.isEmpty() ? "Could not read all headers" : err));
|
|
|
|
auto [verif, lock] = headerVerifier();
|
|
// set genesis hash
|
|
p->genesisHash = BTC::HashRev(hVec.front());
|
|
|
|
err.clear();
|
|
// read db
|
|
for (uint32_t i = 0; i < num; ++i) {
|
|
auto & bytes = hVec[i];
|
|
if (!verif(bytes, &err))
|
|
throw DatabaseFormatError(QString("%1. Possible databaase corruption. Delete the datadir and resynch.").arg(err));
|
|
bytes = BTC::Hash(bytes); // replace the header in the vector with its hash because it will be needed below...
|
|
}
|
|
}
|
|
}
|
|
if (num) {
|
|
const auto elapsed = Util::getTimeNS();
|
|
|
|
Debug() << "Read & verified " << num << " " << Util::Pluralize("header", num) << " from db in " << QString::number((elapsed-t0)/1e6, 'f', 3) << " msec";
|
|
}
|
|
|
|
if (!p->merkleCache->isInitialized() && !hVec.empty())
|
|
p->merkleCache->initialize(hVec); // this may take a few seconds, and it may also throw
|
|
|
|
}
|
|
|
|
void Storage::loadCheckTxNumsDRAAndBlkInfo()
|
|
{
|
|
// may throw.
|
|
p->db.txNumsDRA = std::make_unique<DBRecordArray>(*p->db, *p->db.txnum2txhash,
|
|
/* recSize = */ HashLen, /* bucketNItems = */ 16, /* magic = */ 0x000012e2);
|
|
p->txNumNext = p->db.txNumsDRA->numRecords();
|
|
Debug() << "Read TxNumNext from file: " << p->txNumNext.load();
|
|
TxNum ct = 0;
|
|
if (const int height = latestTip().first; height >= 0)
|
|
{
|
|
p->blkInfos.reserve(std::min(size_t(height+1), MAX_HEADERS));
|
|
Log() << "Checking tx counts ...";
|
|
for (int i = 0; i <= height; ++i) {
|
|
static const QString errMsg("Failed to read a blkInfo from db, the database may be corrupted");
|
|
const auto blkInfo = GenericDBGetFailIfMissing<BlkInfo>(p->db, p->db.blkinfo, uint32_t(i), errMsg, false, p->db.defReadOpts);
|
|
if (blkInfo.txNum0 != ct)
|
|
throw DatabaseFormatError(QString("BlkInfo for height %1 does not match computed txNum of %2."
|
|
"\n\nThe database may be corrupted. Delete the datadir and resynch it.\n")
|
|
.arg(i).arg(ct));
|
|
ct += blkInfo.nTx;
|
|
p->blkInfos.emplace_back(blkInfo);
|
|
p->blkInfosByTxNum[blkInfo.txNum0] = unsigned(p->blkInfos.size()-1);
|
|
}
|
|
Log() << ct << " total transactions";
|
|
}
|
|
if (ct != p->txNumNext) {
|
|
throw DatabaseFormatError(QString("BlkInfo txNums do not add up to expected value of %1 != %2."
|
|
"\n\nThe database may be corrupted. Delete the datadir and resynch it.\n")
|
|
.arg(ct).arg(p->txNumNext.load()));
|
|
}
|
|
}
|
|
|
|
// this depends on the above function having been run already
|
|
void Storage::loadCheckTxHash2TxNumMgr()
|
|
{
|
|
// the below may throw
|
|
p->db.txhash2txnumMgr = std::make_unique<TxHash2TxNumMgr>(p->db.get(), p->db.txhash2txnum, p->db.defReadOpts, p->db.defWriteOpts,
|
|
p->db.txNumsDRA.get(), 6, TxHash2TxNumMgr::KeyPos::End);
|
|
try {
|
|
// basic sanity checks -- ensure we can read the first, middle, and last hash in the txNumsFile,
|
|
// and that those hashes exist in the txhash2txnum db
|
|
const QString errMsg = "The txhash index failed basic sanity checks -- it is missing some records.";
|
|
const auto nrecs = p->db.txNumsDRA->numRecords();
|
|
if (nrecs) {
|
|
for (auto recNum : {uint64_t(0), uint64_t(nrecs/2), uint64_t(nrecs-1)}) {
|
|
if (!p->db.txhash2txnumMgr->exists(p->db.txNumsDRA->readRecord(recNum)))
|
|
throw DatabaseError(errMsg);
|
|
}
|
|
} else {
|
|
// sanity check on empty db: if no records, db should also have no rows
|
|
std::unique_ptr<rocksdb::Iterator> it(p->db->NewIterator(p->db.defReadOpts, p->db.txhash2txnum));
|
|
if (!it) throw DatabaseError("Unable to obtain an iterator to the txhash2txnum set db");
|
|
for (it->SeekToFirst(); it->Valid(); it->Next()) {
|
|
throw DatabaseFormatError(QString("Failed invariant: empty txNum file should mean empty db; ") + errMsg);
|
|
}
|
|
}
|
|
|
|
if (p->db.txhash2txnumMgr->maxTxNumSeenInDB()+1 != int64_t(nrecs))
|
|
throw DatabaseFormatError(QString("Failed invariant: txNumCount != nrecs; ") + errMsg);
|
|
|
|
if (options->doSlowDbChecks) // require the slow check for this one
|
|
p->db.txhash2txnumMgr->consistencyCheck();
|
|
if (options->doSlowDbChecks >= 3) // the below check is very slow so we require -C -C -C
|
|
p->db.txhash2txnumMgr->consistencyCheckSlowRev();
|
|
} catch (const DatabaseError &e) {
|
|
// Database error -- user either lacks the database (upgrade needed) or they have it and it is corrupted --
|
|
// attempt to rebuild it.
|
|
if (p->db.txhash2txnumMgr->maxTxNumSeenInDB() > -1) {
|
|
Warning() << e.what();
|
|
Log() << "Rebuilding txhash index, please wait ...";
|
|
} else {
|
|
Debug() << e.what();
|
|
Log() << "Upgrading database, this may take from 1-10 minutes, please wait ...";
|
|
}
|
|
p->db.txhash2txnumMgr->rebuildDB();
|
|
}
|
|
}
|
|
|
|
// NOTE: this must be called *after* loadCheckTxNumsFileAndBlkInfo(), because it needs a valid p->txNumNext
|
|
void Storage::loadCheckUTXOsInDB()
|
|
{
|
|
FatalAssert(!!p->db.utxoset, __func__, ": Utxo set db is not open");
|
|
|
|
if (options->doSlowDbChecks) {
|
|
Log() << "CheckDB: Verifying utxo set (this may take some time) ...";
|
|
|
|
// Note: Before the BIP that imposed uniqueness on coinbase tx's,
|
|
// Bitcoin coinbase tx's for heights 91842 and 91812 both have outpoint:
|
|
// d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599:0
|
|
// And coinbase tx's for heights 91880 and 91722 both have outpoint:
|
|
// e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468:0
|
|
// Thus the counts may be off by as much as 2 here. So we must detect these
|
|
// utxos and compensate by fudging the count check a little bit if we see
|
|
// the utxo + heights in question.
|
|
const std::map<TXO, std::pair<HashX, std::set<unsigned>>> fudgeDueToBitcoinBugs = {
|
|
{TXO{Util::ParseHexFast("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468"), 0},
|
|
{ Util::ParseHexFast("49df7a6bfea6c409a5f03fd734a1f1a13cb8fafee6a3e08dd94db352498f99a6"), {91880, 91722} } },
|
|
{TXO{Util::ParseHexFast("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599"), 0},
|
|
{ Util::ParseHexFast("76d95f02197b7c685b972104f6d7688a78bdcbb6a757fd5a139a195e59505fab"), {91842, 91812} } },
|
|
};
|
|
std::set<TXO> seenExceptions;
|
|
// scan shunspent to see if our counts may be off
|
|
// (this scan guards against these coins being spent in future throwing off our counts yet again!)
|
|
for (const auto & [txo, pair] : fudgeDueToBitcoinBugs) {
|
|
const auto & [hashx, heights] = pair;
|
|
for (const auto & height : heights) {
|
|
if (height >= p->blkInfos.size())
|
|
continue;
|
|
const TxNum txNum = p->blkInfos[height].txNum0;
|
|
const CompactTXO ctxo(txNum, txo.outN);
|
|
auto opt = GenericDBGet<QByteArray>(p->db, p->db.shunspent, mkShunspentKey(hashx, ctxo), true, "", false, p->db.defReadOpts);
|
|
if (opt.has_value()) {
|
|
if (seenExceptions.insert(txo).second)
|
|
Debug() << "Seen exception: " << txo.toString() << ", height: " << height;
|
|
}
|
|
}
|
|
}
|
|
|
|
const Tic t0;
|
|
{
|
|
const qint64 currentHeight = latestTip().first;
|
|
|
|
std::unique_ptr<rocksdb::Iterator> iter(p->db->NewIterator(p->db.defReadOpts, p->db.utxoset));
|
|
if (!iter) throw DatabaseError("Unable to obtain an iterator to the utxo set db");
|
|
p->utxoCt = 0;
|
|
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
|
// TODO: the below checks may be too slow. See about removing them and just counting the iter.
|
|
const auto txo = Deserialize<TXO>(FromSlice(iter->key()));
|
|
if (!txo.isValid()) {
|
|
throw DatabaseSerializationError("Read an invalid txo from the utxo set database."
|
|
" This may be due to a database format mismatch."
|
|
"\n\nDelete the datadir and resynch to bitcoind.\n");
|
|
}
|
|
auto info = Deserialize<TXOInfo>(FromSlice(iter->value()));
|
|
if (!info.isValid())
|
|
throw DatabaseSerializationError(QString("Txo %1 has invalid metadata in the db."
|
|
" This may be due to a database format mismatch."
|
|
"\n\nDelete the datadir and resynch to bitcoind.\n")
|
|
.arg(txo.toString()));
|
|
|
|
// compensate for counts being off due to historical bugs in blockchain
|
|
// these outpoints actually generate 2 entries in shunspent and 1 entry here
|
|
// we must tolerate counts being off if we see this utxo.
|
|
if (auto it = fudgeDueToBitcoinBugs.find(txo);
|
|
it != fudgeDueToBitcoinBugs.end() && it->second.second.count(info.confirmedHeight.value_or(0))) {
|
|
if (seenExceptions.insert(txo).second)
|
|
Debug() << "Seen exception: " << txo.toString();
|
|
}
|
|
// this is a deep test: only happens if -C / --checkdb is specified on CLI or in conf.
|
|
const CompactTXO ctxo = CompactTXO(info.txNum, txo.outN);
|
|
const QByteArray shuKey = mkShunspentKey(info.hashX, ctxo);
|
|
static const QString errPrefix("Error reading scripthash_unspent");
|
|
QByteArray tmpBa;
|
|
SHUnspentValue shval;
|
|
if (bool fail1 = false, fail2 = false, fail3 = false, fail4 = false, fail5 = false;
|
|
(fail1 = (!info.confirmedHeight.has_value() || qint64(*info.confirmedHeight) > currentHeight))
|
|
|| (fail2 = info.txNum >= p->txNumNext)
|
|
|| (fail3 = (tmpBa = GenericDBGet<QByteArray>(p->db, p->db.shunspent, shuKey, true, errPrefix, false, p->db.defReadOpts).value_or("")).isEmpty())
|
|
|| (fail4 = (!(shval = Deserialize<SHUnspentValue>(tmpBa)).valid || info.amount != shval.amount))
|
|
|| (fail5 = (info.tokenDataPtr != shval.tokenDataPtr))) {
|
|
// TODO: reorg? Inconsisent db? FIXME
|
|
QString msg;
|
|
{
|
|
QTextStream ts(&msg);
|
|
ts << "Inconsistent database: txo " << txo.toString();
|
|
if (info.confirmedHeight) ts << " (height: " << *info.confirmedHeight << ")";
|
|
else ts << " (missing height)";
|
|
if (fail1) {
|
|
ts << " has unexpected height; current height: " << currentHeight << ".";
|
|
} else if (fail2) {
|
|
ts << ". TxNum: " << info.txNum << " >= " << p->txNumNext << ".";
|
|
} else if (fail3) {
|
|
ts << ". Failed to find ctxo " << ctxo.toString() << " in the scripthash_unspent db.";
|
|
} else if (fail4) {
|
|
ts << ". Utxo amount does not match the ctxo amount in the scripthash_unspent db.";
|
|
} else if (fail5) {
|
|
ts << ". Token data does not match the ctxo token_data in the scripthash_unspent db.";
|
|
}
|
|
ts << "\n\nThe database has been corrupted. Please delete the datadir and resynch to bitcoind.\n";
|
|
}
|
|
throw DatabaseError(msg);
|
|
}
|
|
if (0 == ++p->utxoCt % 100'000) {
|
|
*(0 == p->utxoCt % 2'500'000 ? std::make_unique<Log>() : std::make_unique<Debug>())
|
|
<< "CheckDB: Verified " << p->utxoCt << " utxos ...";
|
|
} else if (0 == p->utxoCt % 1'000 && app() && app()->signalsCaught()) {
|
|
throw UserInterrupted("User interrupted, aborting check");
|
|
}
|
|
}
|
|
|
|
if (const auto metact = readUtxoCtFromDB();
|
|
// counts may be slightly off due to the dupe tx's outlined above -- after this is run
|
|
// the utxoset will have the right count (although shunspent will disagree with this,
|
|
// which we also tolerate). So we tolerate being off due to the "exceptions" above.
|
|
std::abs(long(p->utxoCt) - long(metact)) > long(seenExceptions.size()))
|
|
throw DatabaseError(QString("UTXO count in meta table (%1) does not match the actual number of UTXOs in the utxoset (%2)."
|
|
"\n\nThe database has been corrupted. Please delete the datadir and resynch to bitcoind.\n")
|
|
.arg(metact).arg(p->utxoCt.load()));
|
|
|
|
}
|
|
Debug() << "CheckDB: Verified utxos in " << t0.msecStr() << " msec";
|
|
|
|
} else {
|
|
p->utxoCt = readUtxoCtFromDB();
|
|
}
|
|
|
|
if (const auto ct = utxoSetSize(); ct)
|
|
Log() << "UTXO set: " << ct << Util::Pluralize(" utxo", ct) << ", " << QString::number(utxoSetSizeMB(), 'f', 3) << " MB";
|
|
}
|
|
|
|
// NOTE: this must be called *after* loadCheckTxNumsFileAndBlkInfo(), because it needs a valid p->txNumNext
|
|
void Storage::loadCheckShunspentInDB()
|
|
{
|
|
FatalAssert(!!p->db.shunspent, __func__, ": Shunspent db is not open");
|
|
|
|
if (options->doSlowDbChecks < 2) // this is so slow it requires -C -C be specified
|
|
return;
|
|
|
|
Log() << "CheckDB: Verifying scripthash_unspent (this may take some time) ...";
|
|
|
|
const Tic t0;
|
|
|
|
std::unique_ptr<rocksdb::Iterator> iter(p->db->NewIterator(p->db.defReadOpts, p->db.shunspent));
|
|
if (!iter) throw DatabaseError("Unable to obtain an iterator to the scripthash unspent db");
|
|
|
|
// Note: Before the BIP that imposed uniqueness on coinbase tx's,
|
|
// Bitcoin coinbase tx's for heights 91842 and 91812 both have outpoint:
|
|
// d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599:0
|
|
// And coinbase tx's for heights 91880 and 91722 both have outpoint:
|
|
// e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468:0
|
|
// Thus the TXOInfo may be wrong for these entries since we get 2 shunspent
|
|
// entries for each of these coins but only 1 entry in utxodb for each coin.
|
|
// Thus, we just must ignore sanity check for these two coins outright.
|
|
const std::set<TXO> exceptionsDueToBitcoinBugs = {
|
|
TXO{Util::ParseHexFast("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468"), 0},
|
|
TXO{Util::ParseHexFast("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599"), 0},
|
|
};
|
|
std::set<TXO> seenExceptions;
|
|
|
|
constexpr auto errMsg = "This may be due to either a database format mismatch or data corruption."
|
|
"\n\nDelete the datadir and resynch to bitcoind.\n";
|
|
size_t ctr = 0;
|
|
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
|
const auto &[hashx, ctxo] = extractShunspentKey(iter->key());
|
|
if (!ctxo.isValid())
|
|
throw DatabaseError(QString("Read an invalid compact txo from the scripthash_unspent database. %1").arg(errMsg));
|
|
TXOInfo info;
|
|
{
|
|
SHUnspentValue shuval = Deserialize<SHUnspentValue>(FromSlice(iter->value()));
|
|
if (UNLIKELY(!shuval.valid || !bitcoin::MoneyRange(shuval.amount)))
|
|
throw DatabaseError(QString("Read an invalid SHUnspentValue from the scripthash_unspent database for scripthash: %1. %2")
|
|
.arg(QString(hashx.toHex()), errMsg));
|
|
info.txNum = ctxo.txNum();
|
|
info.hashX = hashx;
|
|
info.amount = shuval.amount;
|
|
info.confirmedHeight = heightForTxNum(ctxo.txNum());
|
|
info.tokenDataPtr = std::move(shuval.tokenDataPtr);
|
|
}
|
|
const TxHash txHash = hashForTxNum(ctxo.txNum(), true, nullptr, true).value_or(QByteArray()); // throws if missing
|
|
const TXO txo{txHash, ctxo.N()};
|
|
// look for this in the UTXO db
|
|
const auto optInfo = GenericDBGet<TXOInfo>(p->db, p->db.utxoset, ToSlice(Serialize(txo)), true, "", false, p->db.defReadOpts);
|
|
if (!optInfo) {
|
|
// we permit the buggy utxos above to be off -- those are due to collisions in historical blockchain
|
|
if (!exceptionsDueToBitcoinBugs.count(txo))
|
|
throw DatabaseError(QString("The scripthash_unspent table is missing a corresponding entry in the UTXO table for TXO \"%1\". %2")
|
|
.arg(txo.toString(), errMsg));
|
|
else {
|
|
seenExceptions.insert(txo);
|
|
Debug() << "Seen exception: " << txo.toString() << ", height: " << info.confirmedHeight.value_or(0);
|
|
}
|
|
}
|
|
|
|
if (!info.isValid() || !optInfo->isValid() || *optInfo != info) {
|
|
// we permit the buggy utxos above to be off -- those are due to collisions in historical blockchain
|
|
if (!exceptionsDueToBitcoinBugs.count(txo))
|
|
throw DatabaseError(QString("TXO \"%1\" mismatch between scripthash_unspent and the UTXO table. %2")
|
|
.arg(txo.toString(), errMsg));
|
|
else {
|
|
seenExceptions.insert(txo);
|
|
Debug() << "Seen exception: " << txo.toString() << ", height: " << info.confirmedHeight.value_or(0);
|
|
}
|
|
}
|
|
if (0 == ++ctr % 10000) {
|
|
*(0 == ctr % 200000 ? std::make_unique<Log>() : std::make_unique<Debug>())
|
|
<< "CheckDB: Verified " << ctr << " scripthash_unspent entries ...";
|
|
}
|
|
}
|
|
|
|
if (const auto metact = readUtxoCtFromDB();
|
|
// tolerate being off by as much as 2 in case the exceptional utxos get spent!
|
|
std::abs(long(ctr) - long(metact)) > long(seenExceptions.size()))
|
|
throw DatabaseError(QString("UTXO count in meta table (%1) does not match the actual number of UTXOs in shunspent (%2). %3")
|
|
.arg(metact).arg(ctr).arg(errMsg));
|
|
|
|
Log() << "Verified " << ctr << " scripthash_unspent " << Util::Pluralize("entry", ctr)
|
|
<< " in " << t0.secsStr() << " sec";
|
|
}
|
|
|
|
void Storage::loadCheckRpaDB()
|
|
{
|
|
FatalAssert(!!p->db.rpa, __func__, ": RPA db is not open");
|
|
|
|
const bool doSlowChecks = options->doSlowDbChecks;
|
|
const bool doNeededCheck = isRpaNeedsFullCheck();
|
|
const bool fullCheck = doSlowChecks || doNeededCheck;
|
|
|
|
if (doSlowChecks) {
|
|
Log() << "CheckDB: Verifying RPA db (this may take some time) ...";
|
|
} else if (doNeededCheck) {
|
|
Log() << "Performing required check on RPA db, please wait ...";
|
|
} else {
|
|
Log() << "Loading RPA db ...";
|
|
}
|
|
|
|
Tic t0;
|
|
bool blowAwayWholeDB = false;
|
|
std::optional<QString> excMessage;
|
|
try {
|
|
auto & firstHeight = p->rpaInfo.firstHeight, & lastHeight = p->rpaInfo.lastHeight;
|
|
firstHeight = lastHeight = -1;
|
|
int forceDeleteAfterHeight = -1; // if >=0, force a delete after this height
|
|
|
|
std::unique_ptr<rocksdb::Iterator> iter(p->db->NewIterator(p->db.defReadOpts, p->db.rpa));
|
|
if (!iter) throw DatabaseError("Unable to obtain an iterator to the rpa db");
|
|
auto ThrowIfNegativeIfCastedToSigned = [](uint32_t height) {
|
|
if (height > uint32_t(std::numeric_limits<int>::max()))
|
|
throw DatabaseFormatError(QString("Encountered a height (%1) in the RPA db that is > INT_MAX"
|
|
"; this indicates corruption or an incompatible DB format.").arg(height));
|
|
};
|
|
auto TryDeserializePFTAndUpdateCounts = [&info = p->rpaInfo](uint32_t height, const rocksdb::Slice &slice) {
|
|
try {
|
|
bool ok{};
|
|
++info.nReads;
|
|
info.nBytesRead += sizeof(height) + slice.size();
|
|
Rpa::PrefixTable pt = Deserialize<Rpa::PrefixTable>(FromSlice(slice), &ok);
|
|
} catch (const std::exception &e) {
|
|
throw DatabaseSerializationError(QString("Error deserializing Rpa::PrefixTable for height %1: %2")
|
|
.arg(height).arg(e.what()));
|
|
}
|
|
};
|
|
if (! fullCheck) {
|
|
// Normal fast startup -- just try and figure out what height range we actually have in the DB
|
|
iter->SeekToFirst();
|
|
if (iter->Valid()) {
|
|
bool ok;
|
|
RpaDBKey rk = RpaDBKey::fromBytes(FromSlice(iter->key()), &ok, true);
|
|
if (!ok) throw DatabaseSerializationError("Unable to deserialize RPA db key -> height");
|
|
ThrowIfNegativeIfCastedToSigned(rk.height);
|
|
TryDeserializePFTAndUpdateCounts(rk.height, iter->value()); // this may throw; if it does we will blow away the whole DB below and Controller will do a full resynch of RPA index
|
|
firstHeight = rk.height;
|
|
iter->SeekToLast();
|
|
if (UNLIKELY( ! iter->Valid())) throw DatabaseError("Unable to seek to last entry in RPA db. This is unexpected.");
|
|
rk = RpaDBKey::fromBytes(FromSlice(iter->key()), &ok, true);
|
|
if (!ok) throw DatabaseSerializationError("Unable to deserialize RPA db key -> height");
|
|
ThrowIfNegativeIfCastedToSigned(rk.height);
|
|
TryDeserializePFTAndUpdateCounts(rk.height, iter->value()); // this may throw; if it does we will blow away the whole DB below and Controller will do a full resynch of RPA index
|
|
lastHeight = rk.height;
|
|
if (lastHeight < firstHeight) // this should never happen and indicates some serialization format error
|
|
throw DatabaseSerializationError(QString("The last record has height less than the first record in the RPA db: first = %1, last = %2").arg(firstHeight.load()).arg(lastHeight.load()));
|
|
}
|
|
} else {
|
|
// Slower -- iterate through entire table to find gaps as well as verify data by deserializing it row by row
|
|
size_t ctr = 0;
|
|
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
|
const auto & k = iter->key();
|
|
if (k.size() == sizeof(uint32_t)) {
|
|
bool ok;
|
|
const RpaDBKey rk = Deserialize<RpaDBKey>(FromSlice(k), &ok);
|
|
if (!ok) throw DatabaseSerializationError("Unable to deserialize RPA db key -> height");
|
|
ThrowIfNegativeIfCastedToSigned(rk.height);
|
|
if (firstHeight < 0) firstHeight = rk.height;
|
|
TryDeserializePFTAndUpdateCounts(rk.height, iter->value()); // this may throw; if it does we will blow away the whole DB below and Controller will do a full resynch of RPA index
|
|
if (lastHeight > -1 && BlockHeight(lastHeight) + 1u != rk.height) { // detect gaps
|
|
Warning() << QString("Gap in RBA db encountered starting at height %1 to height %2").arg(lastHeight + 1).arg(rk.height);
|
|
forceDeleteAfterHeight = BlockHeight(lastHeight);
|
|
break;
|
|
}
|
|
lastHeight = rk.height;
|
|
++ctr;
|
|
if (0u == ctr % 1'000u && app() && app()->signalsCaught())
|
|
throw UserInterrupted("User interrupted, aborting check");
|
|
} else {
|
|
throw DatabaseFormatError(QString("Encountered a key in the RPA db that is not exactly %1 bytes! Hex for key: %2")
|
|
.arg(sizeof(uint32_t)).arg(QString(FromSlice(k).toHex())));
|
|
}
|
|
}
|
|
Debug () << "RPA db has " << ctr << " entries, " << p->rpaInfo.nBytesRead << " bytes; deserialized ok";
|
|
}
|
|
if (lastHeight < firstHeight || ((lastHeight <= -1 || firstHeight <= -1) && lastHeight != firstHeight)) // defensive programming: enforce invariant here
|
|
throw InternalError(QString("Programming error in %1. FIXME!").arg(__func__));
|
|
if (firstHeight > -1) {
|
|
const int currentHeight = latestTip().first;
|
|
if (currentHeight < lastHeight || forceDeleteAfterHeight > -1) {
|
|
// delete either form the "forceDeleteAfterHeight" height or the current height, whichever is smaller
|
|
const int delheight = forceDeleteAfterHeight > -1 ? std::min(forceDeleteAfterHeight, currentHeight)
|
|
: currentHeight;
|
|
const auto delheightplus1 = static_cast<BlockHeight>(std::max(delheight, -1) + 1);
|
|
|
|
Log() << "Deleting unneeded or gap RPA entries from height " << delheightplus1 << " ...";
|
|
// on success, updates p->rpaInfo.lastHeight, firstHeight, etc
|
|
if (!deleteRpaEntriesFromHeight(nullptr, delheightplus1, true, true))
|
|
throw DatabaseError("Failed to delete the required keys from the DB. Please report this situation to the developers.");
|
|
}
|
|
}
|
|
// Print some info -- note firstHeight can mutate above which is why we do this here last
|
|
if (firstHeight >= 0) Debug() << "RPA db data covers heights: " << firstHeight << " -> " << lastHeight;
|
|
else Debug() << "RPA db is empty";
|
|
} catch (const std::ios_base::failure &e) {
|
|
excMessage = e.what();
|
|
blowAwayWholeDB = true;
|
|
} catch (const DatabaseError &e) {
|
|
excMessage = e.what();
|
|
blowAwayWholeDB = true;
|
|
}
|
|
if (excMessage) Warning() << *excMessage;
|
|
if (blowAwayWholeDB) {
|
|
Log() << "RPA db is inconsistent and will be resynched from bitcoind. Deleting existing entries ...";
|
|
deleteRpaEntriesFromHeight(0, true, true);
|
|
p->rpaInfo.firstHeight = p->rpaInfo.lastHeight = -1;
|
|
}
|
|
|
|
// Lastly, if we were in check mode, flag the DB as clean now
|
|
if (fullCheck) setRpaNeedsFullCheck(false);
|
|
|
|
Debug() << (doSlowChecks ? "CheckDB: Verified" : (doNeededCheck ? "Checked" : "Loaded"))
|
|
<< " RPA db in " << t0.msecStr() << " msec";
|
|
}
|
|
|
|
bool Storage::deleteRpaEntriesFromHeight(rocksdb::WriteBatch *batch, const BlockHeight height, bool flush, bool force)
|
|
{
|
|
if (!force && p->rpaInfo.firstHeight <= -1) return true; // fast path for disabled or empty index)
|
|
if (height > unsigned(std::numeric_limits<int>::max())) throw InternalError(QString("Bad argument to ") + __func__);
|
|
constexpr uint32_t u32max = std::numeric_limits<uint32_t>::max();
|
|
QByteArray endKey = RpaDBKey(u32max).toBytes();
|
|
endKey.append('\0'); // ensue covers entire remaining uint32 range by appending a single '0' byte to make this endkey longer than the last uint32 possible.
|
|
|
|
rocksdb::Status status;
|
|
|
|
if (batch)
|
|
status = batch->DeleteRange(p->db.rpa, ToSlice(RpaDBKey(height)), ToSlice(endKey));
|
|
else
|
|
status = p->db->DeleteRange(p->db.defWriteOpts, p->db.rpa, ToSlice(RpaDBKey(height)), ToSlice(endKey));
|
|
|
|
if (!status.ok()) {
|
|
Warning() << __func__ << ": failed in call to db DeleteRange for height (>= " << height << "): "
|
|
<< StatusString(status);
|
|
return false;
|
|
}
|
|
// Update deletion count and firstHeight and lastHeight as necessary
|
|
p->rpaInfo.nDeletions += 1; // we have no idea how many records were deleted, just increment by 1 since most common case is the undo case, where we delete 1.
|
|
auto & firstHeight = p->rpaInfo.firstHeight, & lastHeight = p->rpaInfo.lastHeight;
|
|
if (lastHeight > -1 && BlockHeight(lastHeight) >= height)
|
|
lastHeight = height > 0u ? int(height - 1u) : -1;
|
|
if (firstHeight > lastHeight)
|
|
firstHeight = lastHeight.load();
|
|
if (flush && !batch) {
|
|
rocksdb::FlushOptions f;
|
|
f.wait = true;
|
|
f.allow_write_stall = true;
|
|
p->db->Flush(f, p->db.rpa);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool Storage::deleteRpaEntriesToHeight(rocksdb::WriteBatch *batch, const BlockHeight height, bool flush, bool force)
|
|
{
|
|
if (!force && p->rpaInfo.lastHeight <= -1) return true; // fast path for disabled or empty index)
|
|
if (height > unsigned(std::numeric_limits<int>::max())) throw InternalError(QString("Bad argument to ") + __func__);
|
|
QByteArray endKey = RpaDBKey(height).toBytes();
|
|
|
|
rocksdb::Status status;
|
|
if (batch)
|
|
status = batch->DeleteRange(p->db.rpa, ToSlice(RpaDBKey(0u)), ToSlice(RpaDBKey(height + 1u)));
|
|
else
|
|
status = p->db->DeleteRange(p->db.defWriteOpts, p->db.rpa, ToSlice(RpaDBKey(0u)), ToSlice(RpaDBKey(height + 1u)));
|
|
|
|
if (!status.ok()) {
|
|
Warning() << __func__ << ": failed in call to db DeleteRange for height (<= " << height << "): "
|
|
<< StatusString(status);
|
|
return false;
|
|
}
|
|
// Update deletion count and firstHeight and lastHeight as necessary
|
|
p->rpaInfo.nDeletions += 1; // we have no idea how many records were deleted, just increment by 1 since most common case is the undo case, where we delete 1.
|
|
auto & firstHeight = p->rpaInfo.firstHeight, & lastHeight = p->rpaInfo.lastHeight;
|
|
if (firstHeight > -1 && BlockHeight(firstHeight) <= height)
|
|
firstHeight = int(height + 1u);
|
|
if (firstHeight > lastHeight)
|
|
firstHeight = lastHeight.load();
|
|
if (flush && !batch) {
|
|
rocksdb::FlushOptions f;
|
|
f.wait = true;
|
|
f.allow_write_stall = true;
|
|
p->db->Flush(f, p->db.rpa);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
void Storage::clampRpaEntries(BlockHeight from, BlockHeight to)
|
|
{
|
|
ExclusiveLockGuard g(p->blocksLock);
|
|
rocksdb::WriteBatch batch;
|
|
clampRpaEntries_nolock(&batch, from, to);
|
|
auto s = p->db->Write(p->db.defWriteOpts, &batch);
|
|
if (!s.ok()) Warning() << __func__ << ": failed in batch write for (" << from << ", " << to << "): " << StatusString(s);
|
|
}
|
|
|
|
void Storage::clampRpaEntries_nolock(rocksdb::WriteBatch *batch, BlockHeight from, BlockHeight to)
|
|
{
|
|
if (from > 0u) deleteRpaEntriesToHeight(batch, from - 1u, true);
|
|
if (to < std::numeric_limits<uint32_t>::max()) deleteRpaEntriesFromHeight(batch, to + 1u, true);
|
|
DebugM("Clamped RPA index to: ", from, " -> ", to);
|
|
}
|
|
|
|
void Storage::loadCheckEarliestUndo()
|
|
{
|
|
FatalAssert(p->db.undo && p->db, __func__, ": Undo column family is not open");
|
|
|
|
const Tic t0;
|
|
unsigned ctr = 0;
|
|
using UIntSet = std::set<uint32_t>;
|
|
UIntSet swissCheeseDetector;
|
|
{
|
|
std::unique_ptr<rocksdb::Iterator> iter(p->db->NewIterator(p->db.defReadOpts, p->db.undo));
|
|
if (!iter) throw DatabaseError("Unable to obtain an iterator to the undo db");
|
|
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
|
const auto keySlice = iter->key();
|
|
if (keySlice.size() != sizeof(uint32_t))
|
|
throw DatabaseFormatError("Unexpected key in undo database. We expect only 32-bit unsigned ints!");
|
|
const uint32_t height = DeserializeScalar<uint32_t>(FromSlice(keySlice));
|
|
if (height < p->earliestUndoHeight) p->earliestUndoHeight = height;
|
|
swissCheeseDetector.insert(height);
|
|
++ctr;
|
|
}
|
|
}
|
|
if (ctr) {
|
|
Debug() << "Undo db contains " << ctr << " entries, earliest is " << p->earliestUndoHeight.load() << ", "
|
|
<< t0.msecStr(2) << " msec elapsed.";
|
|
}
|
|
// Detect swiss cheese holes in the height range, and delete the unusable non-contiguous area from the undo db.
|
|
// (This can happen in a very unlikely scenario where the user set max_reorg high then switched back to an old
|
|
// Fulcrum version then switched to this new version again).
|
|
if (unsigned testval{}; !swissCheeseDetector.empty()
|
|
&& (testval = (*swissCheeseDetector.rbegin() - *swissCheeseDetector.begin()) + 1) != ctr) {
|
|
// uh-oh -- there are holes! Argh! User must have run an older Fulcrum version that didn't delete entries past
|
|
// 100 properly -- we need to delete all old undo entries before the first hole
|
|
auto eraseUntil = swissCheeseDetector.end(); --eraseUntil; // point to last element as last 1 by itself is "contiguous"!
|
|
Warning() << "Hole(s) detected in undo db: range (" << testval << ") != counted size (" << ctr << ")";
|
|
for (auto rit = swissCheeseDetector.rbegin(), rprev = rit++; rit != swissCheeseDetector.rend(); rprev = rit++) {
|
|
if (*rprev - *rit == 1) {
|
|
eraseUntil = rit.base(); // no hole here, move the firstContig iterator to point to this element
|
|
--eraseUntil; // rit points to it + 1 so move back 1 (grr)
|
|
} else
|
|
break; // found a hole, abort loop
|
|
}
|
|
// delete everything up until the first contiguous height we saw
|
|
int delctr = 0;
|
|
for (auto it = swissCheeseDetector.begin(); it != eraseUntil; ++delctr) {
|
|
GenericDBDelete(p->db, p->db.undo, uint32_t(*it));
|
|
it = swissCheeseDetector.erase(it);
|
|
}
|
|
p->earliestUndoHeight = !swissCheeseDetector.empty() ? *swissCheeseDetector.begin() : p->InvalidUndoHeight;
|
|
ctr = swissCheeseDetector.size();
|
|
if (delctr) {
|
|
Warning() << "Deleted " << delctr << Util::Pluralize(" undo entry", delctr) << ", earliest undo entry is now "
|
|
<< p->earliestUndoHeight.load() << ", total undo entries now in db: " << ctr;
|
|
}
|
|
}
|
|
// heuristic to detect that the user changed the default on an already-synched dir
|
|
if (const auto legacy = Options::oldFulcrumReorgDepth; configuredUndoDepth() > legacy && ctr == legacy) {
|
|
Warning() << "You have specified max_reorg in the conf file as " << configuredUndoDepth() << "; older "
|
|
<< APPNAME << " versions may not cope well with this setting. As such, it is recommended that you "
|
|
<< "avoid using older versions of this program with this datadir now that you have set this option "
|
|
<< "beyond the default.";
|
|
}
|
|
// sanity check that the latest Undo block deserializes correctly (detects older Fulcrum loading newer db)
|
|
if (!swissCheeseDetector.empty()) {
|
|
const uint32_t height = *swissCheeseDetector.rbegin();
|
|
const QString errMsg(QString("Unable to read undo data for height %1").arg(height));
|
|
const UndoInfo undoInfo = GenericDBGetFailIfMissing<UndoInfo>(p->db, p->db.undo, height, errMsg);
|
|
if (!undoInfo.isValid()) throw DatabaseFormatError(errMsg);
|
|
Debug() << "Latest undo verified ok: " << undoInfo.toDebugString();
|
|
}
|
|
if (ctr > configuredUndoDepth()) {
|
|
// User lowered undo config -- now configured for less undo depth than before. Simply respect user wishes
|
|
// on startup and delete oldest entries if that happens. Note the assumption here is that the undo entries
|
|
// are all without holes starting at p->earliestUndoHeight. That assumption holds in the current code in
|
|
// addBlock(), as it's impossible to get holes due to the way we walk the blockchain history forward, adding
|
|
// blocks 1 at a time. However if the user runs older Fulcrum after having run this newer version on non-default
|
|
// settings, holes MAY appear, hence the warning above.
|
|
const unsigned n2del = ctr - configuredUndoDepth();
|
|
Warning() << "Found " << ctr << " undo entries in db, but max_reorg is " << configuredUndoDepth() << "; "
|
|
<< "deleting " << n2del << Util::Pluralize(" oldest entry", n2del) << " ...";
|
|
const Tic t1;
|
|
for (unsigned i = 0; i < n2del; ++i)
|
|
GenericDBDelete(p->db, p->db.undo, uint32_t(p->earliestUndoHeight++));
|
|
Warning() << n2del << Util::Pluralize(" undo entry", n2del) << " deleted from db in " << t1.msecStr() << " msec";
|
|
}
|
|
}
|
|
|
|
bool Storage::hasUndo() const {
|
|
return p->earliestUndoHeight != p->InvalidUndoHeight;
|
|
}
|
|
|
|
struct Storage::UTXOBatch::P {
|
|
rocksdb::WriteBatch &batch; ///< batch writes/deletes end up in the utxoset and shunspent column families
|
|
rocksdb::ColumnFamilyHandle &utxoset, &shunspent;
|
|
std::atomic_int64_t &utxoCtr;
|
|
UTXOCache * const cache{}; ///< if not nullptr, there is a UTXOCache active and we should give it the batch writes.
|
|
|
|
P(rocksdb::WriteBatch &b, rocksdb::ColumnFamilyHandle &u, rocksdb::ColumnFamilyHandle &s, std::atomic_int64_t &uc, UTXOCache *c)
|
|
: batch(b), utxoset(u), shunspent(s), utxoCtr{uc}, cache{c} {}
|
|
};
|
|
|
|
Storage::UTXOBatch::UTXOBatch(rocksdb::WriteBatch &b, rocksdb::ColumnFamilyHandle &u, rocksdb::ColumnFamilyHandle &s,
|
|
std::atomic_int64_t &uc, UTXOCache *c)
|
|
: p{std::make_unique<P>(b, u, s, uc, c)} {}
|
|
Storage::UTXOBatch::UTXOBatch(UTXOBatch &&o) { p.swap(o.p); }
|
|
|
|
void Storage::setInitialSync(bool b) {
|
|
// take all locks now.. since this is a Big Deal.
|
|
std::scoped_lock guard(p->blocksLock, p->headerVerifierLock, p->blkInfoLock, p->mempoolLock);
|
|
assert(bool(p->db.utxoset) && bool(p->db.shunspent));
|
|
if (b && !p->db.utxoCache) {
|
|
if (options->utxoCache > 0) {
|
|
// enforce that the limit should be the lesser of max size_t and the amount of physical RAM available
|
|
uint64_t bytes = options->utxoCache;
|
|
const uint64_t limit = std::min<uint64_t>(Util::getAvailablePhysicalRAM(), std::numeric_limits<size_t>::max());
|
|
if (bytes > limit) {
|
|
Warning() << "utxo-cache: Requested UTXO cache size of " << bytes << " bytes exceeds available"
|
|
<< " physical memory; will limit the UTXO cache size to not exceed physical RAM.";
|
|
bytes = limit;
|
|
}
|
|
Log() << "utxo-cache: Enabled; UTXO cache size set to " << bytes << " bytes (available physical RAM: " << limit << " bytes)";
|
|
p->db.utxoCache = std::make_unique<UTXOCache>("Storage UTXO Cache", p->db.get(), p->db.utxoset, p->db.defReadOpts);
|
|
// Reserve about 3.6 million entries per GB of utxoCache memory given to us
|
|
// We need to do this, despite the extra memory bloat, because it turns out rehashing is very costly.
|
|
p->db.utxoCache->autoReserve(bytes);
|
|
} else {
|
|
Log() << "utxo-cache: Not enabled";
|
|
}
|
|
} else if (!b && p->db.utxoCache) {
|
|
Log() << "Initial sync ended, deleting UTXO Cache ...";
|
|
p->db.utxoCache.reset();
|
|
}
|
|
}
|
|
|
|
void Storage::UTXOBatch::add(TXO &&txo, TXOInfo &&info, const CompactTXO &ctxo)
|
|
{
|
|
const QByteArray shukey = mkShunspentKey(info.hashX, ctxo),
|
|
shuval = Serialize2(info.amount, info.tokenDataPtr.get());
|
|
// Update db utxoset, keyed off txo -> txoinfo
|
|
static const QString errMsgPrefix("Failed to add a utxo to the utxo batch");
|
|
GenericBatchPut(p->batch, &p->utxoset, txo, info, errMsgPrefix); // may throw on failure
|
|
|
|
// Update the scripthash unspent. This is a very simple table which we scan by hashX prefix using
|
|
// an iterator in listUnspent. Each entry's key is prefixed with the HashX bytes (32) but suffixed with the
|
|
// serialized CompactTXO bytes (8 or 9). Each entry's data is a 8-byte int64_t of the amount of the utxo to save
|
|
// on lookup cost for getBalance().
|
|
static const QString errMsgPrefix2("Failed to add an entry to the scripthash_unspent batch");
|
|
GenericBatchPut(p->batch, &p->shunspent, shukey, shuval, errMsgPrefix2); // may throw, which is what we want
|
|
|
|
// handle cache (if any)
|
|
if (p->cache) p->cache->put(std::move(txo), std::move(info));
|
|
|
|
++p->utxoCtr; // tally utxo counts
|
|
}
|
|
|
|
void Storage::UTXOBatch::remove(const TXO &txo, const HashX &hashX, const CompactTXO &ctxo)
|
|
{
|
|
// enqueue delete from utxoset db -- may throw.
|
|
static const QString errMsgPrefix("Failed to issue a batch delete for a utxo");
|
|
GenericBatchDelete(p->batch, &p->utxoset, txo, errMsgPrefix);
|
|
|
|
// enqueue delete from scripthash_unspent db
|
|
static const QString errMsgPrefix2("Failed to issue a batch delete for a utxo to the scripthash_unspent db");
|
|
GenericBatchDelete(p->batch, &p->shunspent, mkShunspentKey(hashX, ctxo), errMsgPrefix2);
|
|
|
|
// handle cache (if any)
|
|
if (p->cache) p->cache->remove(txo);
|
|
|
|
--p->utxoCtr; // tally utxo counts
|
|
}
|
|
|
|
|
|
/// Thread-safe. Query db for a UTXO, and return it if found. May throw on database error.
|
|
std::optional<TXOInfo> Storage::utxoGetFromDB(const TXO &txo, bool throwIfMissing)
|
|
{
|
|
assert(bool(p->db.utxoset));
|
|
static const QString errMsgPrefix("Failed to read a utxo from the utxo db");
|
|
return GenericDBGet<TXOInfo>(p->db, p->db.utxoset, txo, !throwIfMissing, errMsgPrefix, false, p->db.defReadOpts);
|
|
}
|
|
|
|
int64_t Storage::utxoSetSize() const { return p->utxoCt; }
|
|
double Storage::utxoSetSizeMB() const {
|
|
// TODO: the below is inaccurate because it does not account for any bitcoin::token::OutputDataPtr that may be in TXOInfo
|
|
constexpr int64_t elemSize = TXOInfo::minSerSize() + TXO::minSize() /*<-- assumption is most utxos use 16-bit IONums */;
|
|
return (utxoSetSize()*elemSize) / 1e6;
|
|
}
|
|
|
|
/// Thread-safe. Query the mempool and the DB for a TXO. If the TXO is unspent, will return a valid
|
|
/// optional. If the TXO is spent or non-existant, will return an invalid optional.
|
|
std::optional<TXOInfo> Storage::utxoGet(const TXO &txo)
|
|
{
|
|
std::optional<TXOInfo> ret;
|
|
bool mempoolHit = false;
|
|
|
|
// take shared lock (ensure mempool doesn't mutate from underneath our feet)
|
|
// note that this lock is also taken by addBlock (so this is atomic w.r.t new blocks arriving).
|
|
auto [mempool, lock] = this->mempool(); // shared (read only) lock is held until scope end
|
|
|
|
// first, check mempool
|
|
if (auto txsIt = mempool.txs.find(txo.txHash); txsIt != mempool.txs.end()) {
|
|
mempoolHit = true; // flag mempool hit so that we don't redundantly check db at end of this function
|
|
const auto & tx = txsIt->second;
|
|
if (UNLIKELY(!tx)) {
|
|
// Paranoia to detect bugs. This will never happen.
|
|
throw InternalError(QString("TxRef for %1 is null! FIXME!").arg(QString(txo.txHash.toHex())));
|
|
}
|
|
if (txo.outN < tx->txos.size()) {
|
|
const TXOInfo & info = tx->txos[txo.outN];
|
|
if (auto hxIt = tx->hashXs.find(info.hashX); LIKELY(hxIt != tx->hashXs.end())) {
|
|
const auto & ioinfo = hxIt->second;
|
|
if (ioinfo.utxo.count(txo.outN)) {
|
|
// found! It's unspent!
|
|
ret = info;
|
|
}
|
|
} else {
|
|
// This happens for OP_RETURN outputs -- they aren't indexed so we can end up here.
|
|
// Silently ignore...
|
|
}
|
|
}
|
|
}
|
|
// next check DB if no mempool hit
|
|
if (!mempoolHit) {
|
|
// it's ok to call this with the mempool lock held (Controller also does this)
|
|
ret = utxoGetFromDB(txo, false);
|
|
if (ret.has_value()) {
|
|
// DB hit; but we need to check the mempool now to ensure TXO wasn't spent.
|
|
if (auto hxTxIt = mempool.hashXTxs.find(ret->hashX); hxTxIt != mempool.hashXTxs.end()) {
|
|
// slow-ish -- linear scan through all mempool tx's pertaining to this scripthash
|
|
// in practice this shouldn't be too bad since it's not often that a particular scripthash
|
|
// has more than a few mempool tx's.
|
|
for (const auto & tx : hxTxIt->second) {
|
|
if (auto hxInfoIt = tx->hashXs.find(hxTxIt->first); LIKELY(hxInfoIt != tx->hashXs.end())) {
|
|
const auto & ioinfo = hxInfoIt->second;
|
|
if (ioinfo.confirmedSpends.count(txo)) {
|
|
//Debug() << "TXO: " << txo.toString() << " was in DB but is spent in mempool";
|
|
// DB hit, but was spent in mempool, reset ret so that caller knows it was spent.
|
|
ret.reset();
|
|
break; // enclosing ranged for()
|
|
}
|
|
} else {
|
|
// should never happen
|
|
throw InternalError(QString("scripthash %1 has inconsistent mempool state for tx %2! FIXME!")
|
|
.arg(QString(hxTxIt->first.toHex()), QString(tx->hash.toHex())));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
void Storage::addBlock(PreProcessedBlockPtr ppb, bool saveUndo, unsigned nReserve, bool notifySubs, const bool trackRecentBlockTxHashes)
|
|
{
|
|
assert(bool(ppb) && bool(p));
|
|
|
|
std::unique_ptr<UndoInfo> undo;
|
|
|
|
if (saveUndo) {
|
|
undo = std::make_unique<UndoInfo>();
|
|
undo->height = ppb->height;
|
|
}
|
|
|
|
struct NotifyData {
|
|
using NotifySet = std::unordered_set<HashX, HashHasher>;
|
|
NotifySet scriptHashesAffected, dspTxsAffected, txidsAffected;
|
|
};
|
|
std::unique_ptr<NotifyData> notify;
|
|
|
|
if (notifySubs) {
|
|
notify = std::make_unique<NotifyData>(); // note we don't reserve here -- we will reserve at the end when we run through the hashXAggregated set one final time...
|
|
}
|
|
|
|
{
|
|
// take all locks now.. since this is a Big Deal. TODO: add more locks here?
|
|
std::scoped_lock guard(p->blocksLock, p->headerVerifierLock, p->blkInfoLock, p->mempoolLock);
|
|
|
|
rocksdb::WriteBatch batch; // all writes to DB go through this batch in order to ensure atomicity
|
|
|
|
if (p->db.utxoCache && p->db.utxoCache->cacheMisses) {
|
|
p->db.utxoCache->prefetch(ppb); // will prefetch inputs in a thread
|
|
}
|
|
|
|
const auto blockTxNum0 = p->txNumNext.load();
|
|
|
|
p->recentBlockTxHashes.clear();
|
|
if (notify) {
|
|
// Txs in block can never be in mempool. Ensure they are gone from mempool right away so that notifications
|
|
// to clients are as accurate as possible (notifications may happen after this function returns).
|
|
const auto sz = ppb->txInfos.size();
|
|
const auto rsvsz = static_cast<Mempool::TxHashNumMap::size_type>(sz > 0 ? sz-1 : 0);
|
|
Mempool::TxHashNumMap txidMap(/* bucket_count: */ rsvsz);
|
|
notify->txidsAffected.reserve(rsvsz);
|
|
if (trackRecentBlockTxHashes) {
|
|
p->recentBlockTxHashes.reserve(sz);
|
|
if (sz > 0u) [[likely]]
|
|
p->recentBlockTxHashes.insert(ppb->txInfos[0].hash); // add coinbase txhash to recent set
|
|
}
|
|
for (std::size_t i = 1 /* skip coinbase */; i < sz; ++i) {
|
|
const auto & txHash = ppb->txInfos[i].hash;
|
|
txidMap.emplace(txHash, blockTxNum0 + i);
|
|
notify->txidsAffected.insert(txHash); // add to notify set for txSubsMgr
|
|
if (trackRecentBlockTxHashes)
|
|
// add to "recently seen" set for the hashtx zmq notifier spam suppressor
|
|
p->recentBlockTxHashes.insert(txHash);
|
|
}
|
|
Mempool::ScriptHashesAffectedSet affected;
|
|
// Pre-reserve some capacity for the tmp affected set to avoid much rehashing.
|
|
// Use the heuristic 3 x numtxs capped at the SubsMgr::kRecommendedPendingNotificationsReserveSize (2048).
|
|
affected.reserve(std::min(txidMap.size()*3, SubsMgr::kRecommendedPendingNotificationsReserveSize));
|
|
auto res = p->mempool.confirmedInBlock(affected, txidMap, ppb->height,
|
|
Trace::isEnabled(), 0.5f /* shrink to fit load_factor threshold */);
|
|
if (const auto diff = res.oldSize - res.newSize; (diff || res.elapsedMsec > 5.) && Debug::isEnabled()) {
|
|
Debug d;
|
|
d << "addBlock: removed " << diff << " txs from mempool involving "
|
|
<< affected.size() << " addresses";
|
|
if (res.dspRmCt || res.dspTxRmCt)
|
|
d << " (also removed dsps: " << res.dspRmCt << ", dspTxs: " << res.dspTxRmCt << ")";
|
|
if (res.rpaRmCt)
|
|
d << " (also removed rpa entries: " << res.rpaRmCt << ")";
|
|
d << " in " << QString::number(res.elapsedMsec, 'f', 3) << " msec";
|
|
}
|
|
notify->scriptHashesAffected.merge(std::move(affected));
|
|
notify->dspTxsAffected.merge(std::move(res.dspTxsAffected));
|
|
// ^^ notify->txidsAffected is updated in the above loop
|
|
}
|
|
|
|
const auto verifUndo = p->headerVerifier; // keep a copy of verifier state for undo purposes in case this fails
|
|
// This object ensures that if an exception is thrown while we are in the below code, we undo the header verifier
|
|
// and return it to its previous state. Note the defer'd functor is called with the above scoped_lock held.
|
|
Defer undoVerifierOnScopeEnd([&verifUndo, this] { p->headerVerifier = verifUndo; });
|
|
|
|
// code in the below block may throw -- exceptions are propagated out to caller.
|
|
{
|
|
// Verify header chain makes sense (by checking hashes, using the shared header verifier)
|
|
QByteArray rawHeader;
|
|
{
|
|
QString errMsg;
|
|
if (!p->headerVerifier(ppb->header, &errMsg) ) {
|
|
// XXX possible reorg point. Caller will/should roll back the db state via issuing calls to undoLatestBlock()
|
|
throw HeaderVerificationFailure(errMsg);
|
|
}
|
|
// save raw header back to our buffer -- this will be used at the end of this function to add it to the db
|
|
// after everything completes successfully.
|
|
rawHeader = p->headerVerifier.lastHeaderProcessed().second;
|
|
}
|
|
|
|
setDirty(batch, true); // <-- no turning back. we set it dirty in case rocksdb atomicity fails here; if the app crashes unexpectedly while this is set, on next restart it will refuse to run and insist on a clean resynch if this is true.
|
|
|
|
{ // add txnum -> txhash association to the TxNumsFile...
|
|
auto ctx = p->db.txNumsDRA->beginBatchWrite(batch); // may throw if io error in c'tor here.
|
|
QString errStr;
|
|
for (const auto & txInfo : ppb->txInfos) {
|
|
if (!ctx.append(txInfo.hash, &errStr)) // does not throw here, but we do.
|
|
throw InternalError(QString("Batch append for txNums failed: %1.").arg(errStr));
|
|
}
|
|
// <-- The `ctx` d'tor may throw here if a low-level DB error occurs (see: DBRecordArray.cpp, ~BatchWriteContext()).
|
|
}
|
|
|
|
p->txNumNext += ppb->txInfos.size(); // update internal counter
|
|
|
|
if (p->txNumNext != p->db.txNumsDRA->numRecords())
|
|
throw InternalError("TxNum file and internal txNumNext counter disagree! FIXME!");
|
|
|
|
// Asynch task -- the future will automatically be awaited on scope end (even if we throw here!)
|
|
// NOTE: The assumption here is that ppb->txInfos is ok to share amongst threads -- that is, the assumption
|
|
// is that nothing mutates it. If that changes, please re-examine this code.
|
|
CoTask::Future fut; // if valid, will auto-wait for us on scope end
|
|
TxHash2TxNumMgr::PhasedOp txhash2txnumPhases;
|
|
if (ppb->txInfos.size() > 1000) {
|
|
// submit this to the co-task for blocks with enough txs
|
|
fut = p->blocksWorker->submitWork([&]{
|
|
txhash2txnumPhases = p->db.txhash2txnumMgr->insertForBlockPhased(batch, blockTxNum0, ppb->txInfos);
|
|
txhash2txnumPhases->doAsyncPhase1(); // do this CPU-bound part on another core
|
|
});
|
|
} else {
|
|
// otherwise just do the work ourselves immediately here since this is likely faster (less overhead)
|
|
p->db.txhash2txnumMgr->insertForBlock(batch, blockTxNum0, ppb->txInfos);
|
|
}
|
|
|
|
constexpr bool debugPrt = false;
|
|
|
|
// update utxoSet & scritphash history
|
|
{
|
|
std::unordered_set<HashX, HashHasher> newHashXInputsResolved;
|
|
newHashXInputsResolved.reserve(1024); ///< todo: tune this magic number?
|
|
|
|
{
|
|
// utxo batch block (updtes utxoset & scripthash_unspent tables)
|
|
UTXOBatch utxoBatch{batch, *p->db.utxoset, *p->db.shunspent, p->utxoCt, p->db.utxoCache.get()};
|
|
|
|
// reserve space in undo, if in saveUndo mode
|
|
if (undo) {
|
|
undo->addUndos.reserve(ppb->outputs.size());
|
|
undo->delUndos.reserve(ppb->inputs.size());
|
|
}
|
|
|
|
// add outputs
|
|
for (const auto & [hashX, ag] : std::as_const(ppb->hashXAggregated)) {
|
|
for (const auto oidx : ag.outs) {
|
|
const auto & out = ppb->outputs[oidx];
|
|
if (out.spentInInputIndex.has_value()) {
|
|
if constexpr (debugPrt)
|
|
Debug() << "Skipping output #: " << oidx << " for " << ppb->txInfos[out.txIdx].hash.toHex() << " (was spent in same block tx: " << ppb->txInfos[ppb->inputs[*out.spentInInputIndex].txIdx].hash.toHex() << ")";
|
|
continue;
|
|
}
|
|
const TxHash & hash = ppb->txInfos[out.txIdx].hash;
|
|
TXO txo{.txHash = hash, .outN = out.outN};
|
|
TXOInfo info{
|
|
.amount = out.amount,
|
|
.hashX = hashX,
|
|
.confirmedHeight = ppb->height,
|
|
.txNum = blockTxNum0 + out.txIdx,
|
|
.tokenDataPtr = out.tokenDataPtr,
|
|
};
|
|
const CompactTXO ctxo(info.txNum, txo.outN);
|
|
if (undo) { // save undo info if we are in saveUndo mode
|
|
undo->addUndos.emplace_back(txo, info.hashX, ctxo);
|
|
}
|
|
if constexpr (debugPrt)
|
|
Debug() << "Added txo: " << txo.toString()
|
|
<< " (txid: " << hash.toHex() << " height: " << ppb->height << ") "
|
|
<< " amount: " << info.amount.ToString() << " for HashX: " << info.hashX.toHex();
|
|
utxoBatch.add(std::move(txo), std::move(info), ctxo); // add to db
|
|
}
|
|
}
|
|
|
|
if (p->db.utxoCache)
|
|
// we need the inputs resolved now, so end the prefetch
|
|
// note this may stall and also will empty out p->db.utxoCache->deferredAdds
|
|
p->db.utxoCache->waitForPrefetchToComplete();
|
|
|
|
// add spends (process inputs)
|
|
unsigned inum = 0;
|
|
for (auto & in : ppb->inputs) {
|
|
const TXO txo{.txHash = in.prevoutHash, .outN = in.prevoutN};
|
|
const TXOInfo *pinfo{};
|
|
if (!inum) {
|
|
// coinbase.. skip
|
|
} else if (in.parentTxOutIdx.has_value()) {
|
|
// was an input that was spent in this block so it's ok to skip.. we never added it to utxo set
|
|
if constexpr (debugPrt)
|
|
Debug() << "Skipping input " << txo.toString() << ", spent in this block (output # " << *in.parentTxOutIdx << ")";
|
|
} else if (std::optional<TXOInfo> opt; (p->db.utxoCache && (pinfo = p->db.utxoCache->get(txo)))
|
|
|| ((opt = utxoGetFromDB(txo)) && (pinfo = &*opt)) ) {
|
|
// CAUTION: `pinfo` might be pointing into the UTXOCache, as such, removing from the
|
|
// UTXOCache should be the last action involving `info` and/or `pinfo`.
|
|
const TXOInfo & info = *pinfo;
|
|
if (info.confirmedHeight.has_value() && *info.confirmedHeight != ppb->height) {
|
|
// was a prevout from a previos block.. so the ppb didn't have it in the 'involving hashx' set..
|
|
// mark the spend as having involved this hashX for this ppb now.
|
|
auto & ag = ppb->hashXAggregated[info.hashX];
|
|
ag.ins.emplace_back(inum);
|
|
newHashXInputsResolved.insert(info.hashX);
|
|
// mark its txidx
|
|
if (auto & vec = ag.txNumsInvolvingHashX; vec.empty() || vec.back() != in.txIdx)
|
|
vec.emplace_back(in.txIdx);
|
|
|
|
}
|
|
if constexpr (debugPrt) {
|
|
const auto dbgTxIdHex = ppb->txHashForInputIdx(inum).toHex();
|
|
Debug() << "Spent " << txo.toString() << " amount: " << info.amount.ToString()
|
|
<< " in txid: " << dbgTxIdHex << " height: " << ppb->height
|
|
<< " input number: " << ppb->numForInputIdx(inum).value_or(0xffff)
|
|
<< " HashX: " << info.hashX.toHex();
|
|
}
|
|
if (undo) { // save undo info, if we are in saveUndo mode
|
|
undo->delUndos.emplace_back(txo, info);
|
|
}
|
|
// Enqueue deletion from db and/or immediate removal from UTXOCache -- note this must be
|
|
// called last as it may invalidate `pinfo` and/or `info`!
|
|
utxoBatch.remove(txo, info.hashX, CompactTXO(info.txNum, txo.outN)); // enqueue deletion
|
|
} else {
|
|
QString s;
|
|
{
|
|
const auto dbgTxIdHex = ppb->txHashForInputIdx(inum).toHex();
|
|
QTextStream ts(&s);
|
|
ts << "Failed to spend: " << in.prevoutHash.toHex() << ":" << in.prevoutN << " (spending txid: " << dbgTxIdHex << ")";
|
|
}
|
|
throw InternalError(s);
|
|
}
|
|
++inum;
|
|
}
|
|
}
|
|
|
|
// sort and shrink_to_fit new hashX inputs added
|
|
for (const auto & hashX : newHashXInputsResolved) {
|
|
auto & ag = ppb->hashXAggregated[hashX];
|
|
std::sort(ag.ins.begin(), ag.ins.end()); // make sure they are sorted
|
|
std::sort(ag.txNumsInvolvingHashX.begin(), ag.txNumsInvolvingHashX.end());
|
|
auto last = std::unique(ag.txNumsInvolvingHashX.begin(), ag.txNumsInvolvingHashX.end());
|
|
ag.txNumsInvolvingHashX.erase(last, ag.txNumsInvolvingHashX.end());
|
|
ag.ins.shrink_to_fit();
|
|
ag.txNumsInvolvingHashX.shrink_to_fit();
|
|
}
|
|
|
|
if constexpr (debugPrt)
|
|
Debug() << "utxoset size: " << utxoSetSize() << " block: " << ppb->height;
|
|
}
|
|
|
|
{
|
|
// now.. update the txNumsInvolvingHashX to be offset from txNum0 for this block, and save history to db table
|
|
// history is hashX -> TxNumVec (serialized) as a serities of 6-bytes txNums in blockchain order as they appeared.
|
|
if (notify)
|
|
// first, reserve space for notifications
|
|
notify->scriptHashesAffected.reserve(notify->scriptHashesAffected.size() + ppb->hashXAggregated.size());
|
|
for (auto & [hashX, ag] : ppb->hashXAggregated) {
|
|
if (notify) notify->scriptHashesAffected.insert(hashX); // fast O(1) insertion because we reserved the right size above.
|
|
for (auto & txNum : ag.txNumsInvolvingHashX) {
|
|
txNum += blockTxNum0; // transform local txIdx to -> txNum (global mapping)
|
|
}
|
|
// save scripthash history for this hashX, by appending to existing history. Note that this uses
|
|
// the 'ConcatOperator' class we defined in this file, which requires rocksdb be compiled with RTTI.
|
|
if (auto st = batch.Merge(p->db.shist, ToSlice(hashX), ToSlice(Serialize(ag.txNumsInvolvingHashX))); !st.ok())
|
|
throw DatabaseError(QString("batch merge fail for hashX %1, block height %2: %3")
|
|
.arg(QString(hashX.toHex())).arg(ppb->height).arg(StatusString(st)));
|
|
}
|
|
}
|
|
|
|
|
|
{
|
|
// update BlkInfo
|
|
if (nReserve) {
|
|
if (const auto size = p->blkInfos.size(); size + 1 > p->blkInfos.capacity())
|
|
p->blkInfos.reserve(size + nReserve); // reserve space for new blkinfos in 1 go to save on copying
|
|
}
|
|
|
|
const auto & blkInfo = p->blkInfos.emplace_back(
|
|
blockTxNum0, // .txNum0
|
|
unsigned(ppb->txInfos.size())
|
|
);
|
|
|
|
p->blkInfosByTxNum[blkInfo.txNum0] = unsigned(p->blkInfos.size()-1);
|
|
|
|
// save BlkInfo to db
|
|
static const QString blkInfoErrMsg("Error writing BlkInfo to db");
|
|
GenericBatchPut(batch, p->db.blkinfo, uint32_t(ppb->height), blkInfo, blkInfoErrMsg);
|
|
|
|
if (undo) {
|
|
// save blkInfo to undo information, if in saveUndo mode
|
|
undo->blkInfo = p->blkInfos.back();
|
|
}
|
|
}
|
|
|
|
// Save RPA PrefixTable record (appends a single row to DB), if RPA is enabled for this block
|
|
if (ppb->serializedRpaPrefixTable) {
|
|
addRpaDataForHeight_nolock(batch, ppb->height, *ppb->serializedRpaPrefixTable); // may throw theoretically if GenericBatchPut threw
|
|
}
|
|
|
|
// save the last of the undo info, if in saveUndo mode
|
|
if (undo) {
|
|
const auto t0 = Util::getTimeNS();
|
|
undo->hash = BTC::HashRev(rawHeader);
|
|
undo->scriptHashes = Util::keySet<decltype (undo->scriptHashes)>(ppb->hashXAggregated);
|
|
static const QString errPrefix("Error saving undo info to undo db");
|
|
|
|
GenericBatchPut(batch, p->db.undo, uint32_t(ppb->height), *undo, errPrefix); // save undo to db
|
|
if (ppb->height < p->earliestUndoHeight) {
|
|
// remember earliest for delete clause below...
|
|
p->earliestUndoHeight = ppb->height;
|
|
}
|
|
|
|
if constexpr (debugPrt) {
|
|
// testing undo ser/deser
|
|
Debug() << "Undo info 1: " << undo->toDebugString();
|
|
QByteArray ba = Serialize(*undo);
|
|
Debug() << "Undo info 1 serSize: " << ba.length();
|
|
bool ok;
|
|
auto undo2 = Deserialize<UndoInfo>(ba, &ok);
|
|
ba.fill('z'); // ensure no shallow copies of buffer exist in deserialized object. if they do below tests will fail
|
|
FatalAssert(ok && undo2.isValid(), "Deser of undo info failed!");
|
|
Debug() << "Undo info 2: " << undo2.toDebugString();
|
|
Debug() << "Undo info 1 == undo info 2: " << (*undo == undo2);
|
|
} else {
|
|
const auto elapsedms = (Util::getTimeNS() - t0)/1e6;
|
|
const size_t nTx = undo->blkInfo.nTx, nSH = undo->scriptHashes.size();
|
|
Debug() << "Saved V3 undo for block " << undo->height << ", "
|
|
<< nTx << " " << Util::Pluralize("transaction", nTx)
|
|
<< " involving " << nSH << " " << Util::Pluralize("scripthash", nSH)
|
|
<< ", in " << QString::number(elapsedms, 'f', 2) << " msec.";
|
|
}
|
|
}
|
|
// Expire old undos >configuredUndoDepth() blocks ago to keep the db tidy.
|
|
// We only do this if we know there is an old undo for said height in db.
|
|
// Note that the assumption here is that no holes exist, and that we always walk
|
|
// forward with addBlock() 1 block at a time (which is a valid assumption in this codebase).
|
|
if (const auto expireUndoHeight = int(ppb->height) - int(configuredUndoDepth());
|
|
expireUndoHeight >= 0 && unsigned(expireUndoHeight) >= p->earliestUndoHeight) {
|
|
// FIXME -- this runs for every block in between the last undo save and current tip.
|
|
// If the node was off for a while then restarted this just hits the db with useless deletes for non-existant
|
|
// keys as we catch up. It's not the end of the world, as each call here is on the order of microseconds..
|
|
// but perhaps we need to see about fixing this to not do that.
|
|
static const QString errPrefix("Error deleting old/stale undo info from undo db");
|
|
GenericBatchDelete(batch, p->db.undo, uint32_t(expireUndoHeight), errPrefix);
|
|
p->earliestUndoHeight = unsigned(expireUndoHeight + 1);
|
|
if constexpr (debugPrt) DebugM("Deleted undo for block ", expireUndoHeight, ", earliest now ", p->earliestUndoHeight.load());
|
|
}
|
|
|
|
appendHeader(batch, rawHeader, ppb->height);
|
|
|
|
if (ppb->height == 0) [[unlikely]] {
|
|
// update genesis hash now if block 0 -- this info is used by rpc method server.features
|
|
p->genesisHash = BTC::HashRev(rawHeader); // this variable is guarded by p->headerVerifierLock
|
|
}
|
|
|
|
saveUtxoCt(batch);
|
|
|
|
// Wait for the txhash2txnum asyncPhase1 to finish before we proceed.
|
|
if (fut.future.valid()) {
|
|
fut.future.get(); // wait for completion; this may throw if task threw
|
|
if (txhash2txnumPhases) txhash2txnumPhases->doSyncPhase2(); // issue write batch; may throw
|
|
}
|
|
|
|
setDirty(batch, false);
|
|
|
|
if (auto st = p->db->Write(p->db.defWriteOpts, &batch) ; !st.ok())
|
|
throw DatabaseError(QString("Batch write fail for block height %1: %2").arg(ppb->height).arg(StatusString(st)));
|
|
|
|
undoVerifierOnScopeEnd.disable(); // indicate to the "Defer" object declared at the top of this function that it shouldn't undo anything anymore as we are happy now with the db state now.
|
|
}
|
|
|
|
/* If UTXO cache is enabled, enforce size limit */
|
|
if (size_t limit; p->db.utxoCache && (limit = options->utxoCache) > 0u)
|
|
p->db.utxoCache->limitSize(limit);
|
|
|
|
} /// release locks
|
|
|
|
// now, do notifications with locks NOT held (we are being defensive: in the future we may modify below to take e.g. mempool lock)
|
|
if (notify) {
|
|
if (subsmgr && !notify->scriptHashesAffected.empty())
|
|
subsmgr->enqueueNotifications(std::move(notify->scriptHashesAffected));
|
|
if (dspsubsmgr && !notify->dspTxsAffected.empty())
|
|
dspsubsmgr->enqueueNotifications(std::move(notify->dspTxsAffected));
|
|
if (txsubsmgr && !notify->txidsAffected.empty())
|
|
txsubsmgr->enqueueNotifications(std::move(notify->txidsAffected));
|
|
}
|
|
}
|
|
|
|
/// NB: Caller should probably hold some locks to avoid consistency issues... even though this function is inherently thread-safe.
|
|
void Storage::addRpaDataForHeight_nolock(rocksdb::WriteBatch &batch, const BlockHeight height, const QByteArray &ser)
|
|
{
|
|
Tic t0;
|
|
|
|
static const QString rpaErrMsg("Error writing block RPA data to db");
|
|
GenericBatchPut(batch, p->db.rpa, RpaDBKey(height), ser, rpaErrMsg);
|
|
// Update RpaInfo stats: latest height, etc.
|
|
if (const int lh = p->rpaInfo.lastHeight; UNLIKELY(lh > -1 && lh != int(height) - 1)) {
|
|
// This should never happen. Warn if this invariant is violated to detect bugs.
|
|
Warning() << "RPA index lastHeight (" << lh << ") not as expected (" << (int(height) - 1) << ")."
|
|
<< " Flagging DB as needing a full check.";
|
|
setRpaNeedsFullCheck(true); // flag the RPA db for a full check on next run
|
|
}
|
|
if (const int fh = p->rpaInfo.firstHeight; UNLIKELY(fh > -1 && fh > int(height))) {
|
|
// This should never happen. Warn if this invariant is violated to detect bugs.
|
|
Warning() << "RPA index firstHeight (" << fh << ") not as expected (should be <= " << int(height) << ")."
|
|
<< " Flagging DB as needing a full check.";
|
|
p->rpaInfo.firstHeight = height;
|
|
setRpaNeedsFullCheck(true); // flag the RPA db for a full check on next run
|
|
}
|
|
p->rpaInfo.lastHeight = height;
|
|
if (p->rpaInfo.firstHeight < 0) p->rpaInfo.firstHeight = height;
|
|
++p->rpaInfo.nWrites;
|
|
p->rpaInfo.nBytesWritten += sizeof(uint32_t) + ser.size();
|
|
|
|
if (Debug::isEnabled() && (ser.size() >= 200'000 || t0.msec() >= 20))
|
|
Debug() << "Saved RPA height: " << height << ", size: " << ser.size() << ", elapsed: " << t0.msecStr() << " msec";
|
|
}
|
|
|
|
void Storage::addRpaDataForHeight(BlockHeight height, const QByteArray &serializedRpaPrefixTable)
|
|
{
|
|
ExclusiveLockGuard g(p->blocksLock);
|
|
rocksdb::WriteBatch batch;
|
|
addRpaDataForHeight_nolock(batch, height, serializedRpaPrefixTable);
|
|
auto s = p->db->Write(p->db.defWriteOpts, &batch);
|
|
if (!s.ok()) Warning() << __func__ << ": failed in batch write for height " << height << ": " << StatusString(s);
|
|
}
|
|
|
|
BlockHeight Storage::undoLatestBlock(bool notifySubs)
|
|
{
|
|
BlockHeight prevHeight{0};
|
|
size_t nSH = 0; // for stats printing
|
|
using NotifySet = std::unordered_set<HashX, HashHasher>;
|
|
struct NotifyData {
|
|
using NotifySet = std::unordered_set<HashX, HashHasher>;
|
|
NotifySet scriptHashesAffected, dspTxsAffected, txidsAffected;
|
|
};
|
|
std::unique_ptr<NotifyData> notify;
|
|
|
|
if (notifySubs) {
|
|
notify = std::make_unique<NotifyData>(); // note we don't reserve here -- we will reserve at the end when we run through the hashXAggregated set one final time...
|
|
}
|
|
|
|
{
|
|
// take all locks now.. since this is a Big Deal. TODO: add more locks here?
|
|
std::scoped_lock guard(p->blocksLock, p->headerVerifierLock, p->blkInfoLock, p->mempoolLock);
|
|
|
|
const auto t0 = Util::getTimeNS();
|
|
|
|
// First, disable the UTXO Cache, if it happened to be enabled.
|
|
// We must do this because the way the UTXO Cache works is fundamentally at odds with assumptions we have
|
|
// while we undo.
|
|
p->db.utxoCache.reset();
|
|
|
|
// NOTE: For very full mempools, this clear has the potential to stall the app after the reorg
|
|
// completes since the app will have to re-download the whole mempool state again.
|
|
//
|
|
// However, since reorging is (hopefully) a rare event -- the potential performance hit here
|
|
// in doing a mempool clear (and subsequent redownload) is hopefully acceptable. We have to
|
|
// pick the lesser of two evils here.
|
|
//
|
|
// We decided to clear on reorg because reorging takes a few seconds (or more) and may end up
|
|
// walking back more than 1 block.. so if we didn't clear here, clients migh get a *very*
|
|
// inconsistent view of their tx histories -- with potential spends in mempool for tx's that
|
|
// don't exist or are double-spent, etc. The safer option here is to clear, despite the
|
|
// performance hit.
|
|
//
|
|
if (notify) {
|
|
// mark ALL of mempool for notify so we can detect drops that weren't in block but also disappeared from mempool properly
|
|
notify->scriptHashesAffected.merge(Util::keySet<NotifySet>(p->mempool.hashXTxs));
|
|
if (!p->mempool.dsps.empty())
|
|
// since we will be clearing, just flag all in-mempool dspTxs as affected
|
|
notify->dspTxsAffected.merge(Util::keySet<NotifySet>(p->mempool.dsps.getTxDspsMap()));
|
|
notify->txidsAffected.merge(Util::keySet<NotifySet>(p->mempool.txs)); // for txSubsMgr
|
|
}
|
|
p->mempool.clear(); // make sure mempool is clean (see note above as to why)
|
|
p->recentBlockTxHashes.clear(); // these are no longer relevant if undoing
|
|
|
|
const auto [tip, header] = p->headerVerifier.lastHeaderProcessed();
|
|
if (tip <= 0 || header.length() != p->blockHeaderSize()) throw UndoInfoMissing("No header to undo");
|
|
prevHeight = unsigned(tip-1);
|
|
Header prevHeader;
|
|
{
|
|
// grab previous header now
|
|
QString err;
|
|
auto opt = headerForHeight_nolock(prevHeight, &err);
|
|
if (!opt.has_value()) throw UndoInfoMissing(err);
|
|
prevHeader = *opt;
|
|
}
|
|
const QString errMsg1 = QStringLiteral("Unable to retrieve undo info for %1").arg(tip);
|
|
auto undoOpt = GenericDBGet<UndoInfo>(p->db, p->db.undo, uint32_t(tip), true, errMsg1, false, p->db.defReadOpts);
|
|
if (!undoOpt.has_value())
|
|
throw UndoInfoMissing(errMsg1);
|
|
auto & undo = *undoOpt; // non-const because we swap out its scripthashes potentially below if notifySubs == true
|
|
|
|
// ensure undo info sanity
|
|
if (!undo.isValid() || undo.height != unsigned(tip) || undo.hash != BTC::HashRev(header)
|
|
|| prevHeight+1 >= p->blkInfos.size() || p->blkInfos.empty() || p->blkInfos.back() != undo.blkInfo)
|
|
throw DatabaseFormatError(QString("The undo information for height %1 was successfully retrieved from the "
|
|
"database, but it failed an internal consistency check.").arg(tip));
|
|
{
|
|
// all sanity check passed. Now, undo things in reverse order of what we did in addBlock above, rougly speaking
|
|
|
|
rocksdb::WriteBatch batch; // all writes to DB go through this batch in order to ensure atomicity
|
|
|
|
// first, undo the header
|
|
p->headerVerifier.reset(prevHeight+1, prevHeader);
|
|
setDirty(batch, true); // <-- no turning back. we clear this flag at the end
|
|
deleteHeadersPastHeight(batch, prevHeight);
|
|
p->merkleCache->truncate(prevHeight+1); // this takes a length, not a height, which is always +1 the height
|
|
|
|
// undo the blkInfo from the back
|
|
p->blkInfos.pop_back();
|
|
p->blkInfosByTxNum.erase(undo.blkInfo.txNum0);
|
|
GenericBatchDelete(batch, p->db.blkinfo, uint32_t(undo.height), "Failed to delete blkInfo in undoLatestBlock");
|
|
deleteRpaEntriesFromHeight(&batch, undo.height); // delete RPA >= undo.height (iff index is enabled)
|
|
|
|
// clear num2hash cache
|
|
p->lruNum2Hash.clear();
|
|
// remove block from txHashes cache
|
|
p->lruHeight2Hashes_BitcoindMemOrder.remove(undo.height);
|
|
|
|
const auto txNum0 = undo.blkInfo.txNum0;
|
|
|
|
// Asynch task -- the future will automatically be awaited on scope end (even if we throw here!)
|
|
// Note: we await the result later down in this function before we truncate the txNumsFile. (Assumption
|
|
// here is that the txNumsFile has all the hashes we want to delete until the below operation is done).
|
|
TxHash2TxNumMgr::PhasedOp txhash2txnumPhases;
|
|
CoTask::Future fut = p->blocksWorker->submitWork([&]{
|
|
txhash2txnumPhases = p->db.txhash2txnumMgr->truncateForUndoPhased(batch, txNum0);
|
|
txhash2txnumPhases->doAsyncPhase1();
|
|
});
|
|
|
|
// undo the scripthash histories
|
|
for (const auto & sh : undo.scriptHashes) {
|
|
const QString shHex = Util::ToHexFast(sh);
|
|
const auto vec = GenericDBGetFailIfMissing<TxNumVec>(p->db, p->db.shist, sh, QStringLiteral("Undo failed because we failed to retrieve the scripthash history for %1").arg(shHex), false, p->db.defReadOpts);
|
|
TxNumVec newVec;
|
|
newVec.reserve(vec.size());
|
|
for (const auto txNum : vec) {
|
|
if (txNum < txNum0) {
|
|
// accept only stuff in history that's before txNum0 for this block, filter out everything else
|
|
newVec.push_back(txNum);
|
|
}
|
|
}
|
|
const QString errMsg = QStringLiteral("Undo failed because we failed to write the new scripthash history for %1").arg(shHex);
|
|
if (!newVec.empty()) {
|
|
// The below is entirely unnecessary as the txnums should be already sorted and unique in the db data.
|
|
// We are doing this here to illustrate that this invariant in the data is very important.
|
|
// Block undo is intended to be an infrequent process (and thus not especially performance-critical),
|
|
// so this does no harm.
|
|
std::sort(newVec.begin(), newVec.end());
|
|
auto last = std::unique(newVec.begin(), newVec.end());
|
|
newVec.erase(last, newVec.end());
|
|
}
|
|
if (!newVec.empty()) {
|
|
// the sh still has some history, write it to db
|
|
GenericBatchPut(batch, p->db.shist, sh, newVec, errMsg);
|
|
} else {
|
|
// the sh in question lost all its history as a result of undo, just delete it from db to save space
|
|
GenericBatchDelete(batch, p->db.shist, sh, errMsg);
|
|
}
|
|
}
|
|
|
|
{
|
|
// UTXO set update
|
|
UTXOBatch utxoBatch{batch, *p->db.utxoset, *p->db.shunspent, p->utxoCt};
|
|
|
|
// now, undo the utxo deletions by re-adding them
|
|
for (auto & [txo, info] : undo.delUndos) {
|
|
// note that deletions may have an info with a txnum before this block, for obvious reasons
|
|
utxoBatch.add(std::move(txo), std::move(info), CompactTXO(info.txNum, txo.outN)); // may throw
|
|
}
|
|
|
|
// now, undo the utxo additions by deleting them
|
|
for (const auto & [txo, hashx, ctxo] : undo.addUndos) {
|
|
assert(ctxo.txNum() >= txNum0); // all of the additions must have been in this block or newer
|
|
utxoBatch.remove(txo, hashx, ctxo); // may throw
|
|
}
|
|
}
|
|
|
|
if (p->earliestUndoHeight >= undo.height)
|
|
// oops, we're out of undos now!
|
|
p->earliestUndoHeight = p->InvalidUndoHeight;
|
|
GenericBatchDelete(batch, p->db.undo, uint32_t(undo.height)); // make sure to delete this undo info since it was just applied.
|
|
|
|
// add all tx hashes that we are rolling back to the notify set for the txSubsMgr
|
|
if (notify) {
|
|
const auto txHashes = p->db.txNumsDRA->readRecords(txNum0, undo.blkInfo.nTx);
|
|
notify->txidsAffected.insert(txHashes.begin(), txHashes.end());
|
|
}
|
|
|
|
// Wait for the txhash2txnum truncate to finish before we proceed, since that co-task assumes the txNumsFile
|
|
// won't change.
|
|
if (fut.future.valid()) {
|
|
fut.future.get(); // this may throw if task threw
|
|
if (txhash2txnumPhases) txhash2txnumPhases->doSyncPhase2(); // may throw (unlikely)
|
|
}
|
|
|
|
// lastly, truncate the tx num file and re-set txNumNext to point to this block's txNum0 (thereby recycling it)
|
|
assert(long(p->txNumNext) - long(txNum0) == long(undo.blkInfo.nTx));
|
|
p->txNumNext = txNum0;
|
|
{
|
|
auto ctx = p->db.txNumsDRA->beginBatchWrite(batch);
|
|
QString err;
|
|
if (ctx.truncate(txNum0, &err) != txNum0 || !err.isEmpty()) {
|
|
throw InternalError(QString("Failed to truncate txNumsFile to %1: %2").arg(txNum0).arg(err));
|
|
}
|
|
}
|
|
|
|
saveUtxoCt(batch);
|
|
setDirty(batch, false); // phew. done.
|
|
|
|
if (auto st = p->db->Write(p->db.defWriteOpts, &batch) ; !st.ok())
|
|
throw DatabaseError(QString("Batch write fail for undo of block height %1: %2")
|
|
.arg(tip).arg(StatusString(st)));
|
|
|
|
p->db->SyncWAL(); // TODO verify this is needed, for now we can be paranoid and use this
|
|
|
|
nSH = undo.scriptHashes.size();
|
|
|
|
if (notify) {
|
|
if (notify->scriptHashesAffected.empty())
|
|
notify->scriptHashesAffected.swap(undo.scriptHashes);
|
|
else
|
|
notify->scriptHashesAffected.merge(std::move(undo.scriptHashes));
|
|
}
|
|
}
|
|
|
|
const size_t nTx = undo.blkInfo.nTx;
|
|
const auto elapsedms = (Util::getTimeNS() - t0) / 1e6;
|
|
Log() << "Applied undo for block " << undo.height << " hash " << Util::ToHexFast(undo.hash) << ", "
|
|
<< nTx << " " << Util::Pluralize("transaction", nTx)
|
|
<< " involving " << nSH << " " << Util::Pluralize("scripthash", nSH)
|
|
<< ", in " << QString::number(elapsedms, 'f', 2) << " msec, new height now: " << prevHeight;
|
|
} // release locks
|
|
|
|
// now, do notifications
|
|
if (notify) {
|
|
if (subsmgr && !notify->scriptHashesAffected.empty())
|
|
subsmgr->enqueueNotifications(std::move(notify->scriptHashesAffected));
|
|
if (dspsubsmgr && !notify->dspTxsAffected.empty())
|
|
dspsubsmgr->enqueueNotifications(std::move(notify->dspTxsAffected));
|
|
if (txsubsmgr && !notify->txidsAffected.empty())
|
|
txsubsmgr->enqueueNotifications(std::move(notify->txidsAffected));
|
|
}
|
|
|
|
return prevHeight;
|
|
}
|
|
|
|
|
|
void Storage::setDirty(rocksdb::WriteBatch &batch, bool dirtyFlag)
|
|
{
|
|
static const QString errPrefix("Error saving dirty flag to the meta db");
|
|
const auto & val = dirtyFlag ? kTrue : kFalse;
|
|
GenericBatchPut(batch, p->db.meta, kDirty, val, errPrefix);
|
|
}
|
|
|
|
bool Storage::isDirty() const
|
|
{
|
|
static const QString errPrefix("Error reading dirty flag from the meta db");
|
|
return GenericDBGet<bool>(p->db, p->db.meta, kDirty, true, errPrefix, false, p->db.defReadOpts).value_or(false);
|
|
}
|
|
|
|
void Storage::setRpaNeedsFullCheck(const bool val)
|
|
{
|
|
if (!p->db.meta) return;
|
|
static const QString errPrefix("Error saving rpa_needs_full_check flag to the meta db");
|
|
const auto & slice = val ? kTrue : kFalse;
|
|
GenericDBPut(p->db, p->db.meta, kRpaNeedsFullCheck, slice, errPrefix, p->db.defWriteOpts);
|
|
p->rpaInfo.rpaNeedsFullCheckCachedVal = int(val);
|
|
DebugM("Wrote rpa_needs_full_check = ", val, " to db");
|
|
}
|
|
|
|
bool Storage::isRpaNeedsFullCheck() const
|
|
{
|
|
if (!p->db.meta) return false;
|
|
const int cachedVal = p->rpaInfo.rpaNeedsFullCheckCachedVal.load();
|
|
if (cachedVal > -1) return cachedVal;
|
|
static const QString errPrefix("Error reading rpa_needs_full_check flag from the meta db");
|
|
const int dbVal = /* 0 or 1 */ GenericDBGet<bool>(p->db, p->db.meta, kRpaNeedsFullCheck, true, errPrefix, false,
|
|
p->db.defReadOpts).value_or(false);
|
|
p->rpaInfo.rpaNeedsFullCheckCachedVal = dbVal;
|
|
return dbVal;
|
|
}
|
|
|
|
// public version of above, always latches to true
|
|
void Storage::flagRpaIndexAsPotentiallyInconsistent()
|
|
{
|
|
ExclusiveLockGuard g(p->blocksLock);
|
|
setRpaNeedsFullCheck(true);
|
|
}
|
|
|
|
bool Storage::runRpaSlowCheckIfDBIsPotentiallyInconsistent(BlockHeight configuredStartHeight, BlockHeight tipHeight)
|
|
{
|
|
ExclusiveLockGuard g(p->blocksLock);
|
|
if (isRpaNeedsFullCheck()) {
|
|
try {
|
|
// To avoid infinite consistency-check-loops if there is a gap at the beginning before our configured height
|
|
// we must clamp the DB to the height range we know we need now, before proceeding.
|
|
clampRpaEntries_nolock(nullptr, configuredStartHeight, tipHeight);
|
|
loadCheckRpaDB();
|
|
} catch (const std::exception &e) { Fatal() << "Caught exception: " << e.what(); }
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
|
|
void Storage::saveUtxoCt(rocksdb::WriteBatch &batch)
|
|
{
|
|
static const QString errPrefix("Error writing the utxo count to the meta db");
|
|
const int64_t ct = p->utxoCt.load();
|
|
GenericBatchPut(batch, p->db.meta, kUtxoCount, ct, errPrefix);
|
|
}
|
|
int64_t Storage::readUtxoCtFromDB() const
|
|
{
|
|
static const QString errPrefix("Error reading the utxo count from the meta db");
|
|
return GenericDBGet<int64_t>(p->db, p->db.meta, kUtxoCount, true, errPrefix, false, p->db.defReadOpts).value_or(0LL);
|
|
}
|
|
|
|
|
|
std::optional<TxHash> Storage::hashForTxNum(TxNum n, bool throwIfMissing, bool *wasCached, bool skipCache) const
|
|
{
|
|
std::optional<TxHash> ret;
|
|
if (!skipCache) ret = p->lruNum2Hash.object(n);
|
|
if (ret.has_value()) {
|
|
if (wasCached) *wasCached = true;
|
|
++p->lruCacheStats.num2HashHits;
|
|
return ret;
|
|
} else if (wasCached) *wasCached = false;
|
|
if (!skipCache) ++p->lruCacheStats.num2HashMisses;
|
|
|
|
static const QString kErrMsg ("Error reading TxHash for TxNum %1: %2");
|
|
QString errStr;
|
|
const auto bytes = p->db.txNumsDRA->readRecord(n, &errStr);
|
|
if (bytes.isEmpty()) {
|
|
errStr = kErrMsg.arg(n).arg(errStr);
|
|
if (throwIfMissing)
|
|
throw DatabaseError(errStr);
|
|
Warning() << errStr;
|
|
} else {
|
|
ret.emplace(bytes);
|
|
}
|
|
if (!skipCache && ret.has_value()) {
|
|
// save in cache
|
|
p->lruNum2Hash.insert(n, *ret, p->lruNum2HashSizeCalc());
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
std::optional<unsigned> Storage::heightForTxNum(TxNum n) const
|
|
{
|
|
SharedLockGuard g(p->blkInfoLock);
|
|
return heightForTxNum_nolock(n);
|
|
}
|
|
|
|
std::optional<unsigned> Storage::heightForTxNum_nolock(TxNum n) const
|
|
{
|
|
std::optional<unsigned> ret;
|
|
auto it = p->blkInfosByTxNum.upper_bound(n); // O(logN) search; find the block *AFTER* n, then go back one to find the block in range
|
|
if (it != p->blkInfosByTxNum.begin()) {
|
|
--it;
|
|
const auto & bi = p->blkInfos[it->second];
|
|
if (n >= bi.txNum0 && n < bi.txNum0+bi.nTx)
|
|
ret = it->second;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
std::optional<TxHash> Storage::hashForHeightAndPos(BlockHeight height, uint32_t posInBlock,
|
|
const SharedLockGuard *existingBlocksLock) const
|
|
{
|
|
std::optional<TxHash> ret;
|
|
Span<const uint32_t> singleItem{&posInBlock, size_t{1u}};
|
|
auto vec = hashesForHeightAndPosVec(height, singleItem, existingBlocksLock);
|
|
if (vec.empty()) return ret; // bad height
|
|
ret = std::move(vec.front());
|
|
return ret;
|
|
}
|
|
|
|
std::vector<std::optional<TxHash>> Storage::hashesForHeightAndPosVec(BlockHeight height, Span<const uint32_t> positionsInBlock,
|
|
const SharedLockGuard *existingBlocksLock) const
|
|
{
|
|
std::vector<std::optional<TxHash>> ret;
|
|
if (positionsInBlock.empty()) return ret; // unlikely fast path
|
|
ret.reserve(positionsInBlock.size());
|
|
BlkInfo bi;
|
|
|
|
// Below is to implement optionally locking with: SharedLockGuard(p->blocksLock), if existingBlocksLock is nullptr
|
|
SharedLockGuard maybeLockedByUs;
|
|
if (existingBlocksLock == nullptr) {
|
|
maybeLockedByUs = SharedLockGuard(p->blocksLock);
|
|
} else if (UNLIKELY(existingBlocksLock->mutex() != &p->blocksLock)) {
|
|
Error() << "Internal Error: expected the `existingBlocksLock` to be holding `p->blocksLock` (but it is not) in "
|
|
<< __func__ << ". FIXME!";
|
|
return ret;
|
|
}
|
|
|
|
// At this point p->blocksLock is held for the rest of the function (either by caller or by us).
|
|
// We need to hold p->blocksLock here to get a consistent view (so that data doesn't mutate from beneath us).
|
|
|
|
{
|
|
SharedLockGuard g(p->blkInfoLock);
|
|
if (height >= p->blkInfos.size())
|
|
return ret; // empty vector for bad height
|
|
bi = p->blkInfos[height];
|
|
}
|
|
for (const uint32_t posInBlock : positionsInBlock) {
|
|
if (posInBlock >= bi.nTx)
|
|
ret.emplace_back(std::nullopt); // indicate this position is bad with a nullopt
|
|
else {
|
|
const TxNum txNum = bi.txNum0 + posInBlock;
|
|
ret.push_back(hashForTxNum(txNum));
|
|
}
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
|
|
// NOTE: the returned vector has hashes in bitcoind memory order (little endian -- unlike every other function in this file!)
|
|
std::vector<TxHash> Storage::txHashesForBlockInBitcoindMemoryOrder(BlockHeight height) const
|
|
{
|
|
std::vector<TxHash> ret;
|
|
std::pair<TxNum, size_t> startCount{0,0};
|
|
SharedLockGuard(p->blocksLock); // guarantee a consistent view (so that data doesn't mutate from underneath us)
|
|
{
|
|
// check cache
|
|
auto opt = p->lruHeight2Hashes_BitcoindMemOrder.object(height);
|
|
if (opt.has_value()) {
|
|
// cache hit! return the cached item
|
|
auto & vec = *opt;
|
|
// convert from QVector to std::vector -- TODO: see if we can make the whole call path use QVector to avoid
|
|
// these copies.
|
|
ret.reserve(size_t(vec.size()));
|
|
ret.insert(ret.end(), vec.begin(), vec.end()); // We do it this way because QVector::toStdVector() doesn't reserve() first :/
|
|
++p->lruCacheStats.height2HashesHits;
|
|
return ret;
|
|
}
|
|
}
|
|
++p->lruCacheStats.height2HashesMisses;
|
|
{
|
|
SharedLockGuard g(p->blkInfoLock);
|
|
if (height >= p->blkInfos.size())
|
|
return ret;
|
|
const BlkInfo & bi = p->blkInfos[height];
|
|
startCount = { bi.txNum0, bi.nTx };
|
|
}
|
|
QString err;
|
|
auto vec = p->db.txNumsDRA->readRecords(startCount.first, startCount.second, &err);
|
|
if (vec.size() != startCount.second || !err.isEmpty()) {
|
|
Warning() << "Failed to read " << startCount.second << " txNums for height " << height << ". " << err;
|
|
return ret;
|
|
}
|
|
Util::reverseEachItem(vec); // reverse each hash to make them all be in bitcoind memory order.
|
|
ret.swap(vec);
|
|
{
|
|
// put result in cache
|
|
p->lruHeight2Hashes_BitcoindMemOrder.insert(height,
|
|
#if QT_VERSION < QT_VERSION_CHECK(5, 14, 0)
|
|
QVector<TxHash>::fromStdVector(ret),
|
|
#else
|
|
Util::toVec<QVector<TxHash>>(ret),
|
|
#endif
|
|
p->lruHeight2HashSizeCalc(ret.size()));
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
/// Returns a lambda that can be called to increment the counter. If the counter exceeds maxHistory, lambda will throw.
|
|
/// Used below in getHistory(), listUnspent(), getBalance()
|
|
static auto GetMaxHistoryCtrFunc(const QString &name, const QString &itemName, size_t maxHistory)
|
|
{
|
|
return [name, itemName, maxHistory, ctr = size_t{0u}](size_t incr = 1u) mutable {
|
|
if (UNLIKELY((ctr += incr) > maxHistory)) {
|
|
throw HistoryTooLarge(QString("%1 for %2 exceeds max history %3 with %4 items!")
|
|
.arg(name, itemName).arg(maxHistory).arg(ctr));
|
|
}
|
|
};
|
|
}
|
|
|
|
auto Storage::getHistory(const HashX & hashX, bool conf, bool unconf, BlockHeight fromHeight,
|
|
std::optional<BlockHeight> optToHeight) const -> History
|
|
{
|
|
History ret;
|
|
if (hashX.length() != HashLen)
|
|
return ret;
|
|
auto IncrementCtrAndThrowIfExceedsMaxHistory = GetMaxHistoryCtrFunc("History", QString("scripthash %1").arg(QString(hashX.toHex())),
|
|
options->maxHistory);
|
|
try {
|
|
SharedLockGuard g(p->blocksLock); // makes sure history doesn't mutate from underneath our feet
|
|
if (conf) {
|
|
static const QString err("Error retrieving history for a script hash");
|
|
auto nums_opt = GenericDBGet<TxNumVec>(p->db, p->db.shist, hashX, true, err, false, p->db.defReadOpts);
|
|
if (nums_opt.has_value()) {
|
|
const auto & nums = *nums_opt;
|
|
IncrementCtrAndThrowIfExceedsMaxHistory(nums.size());
|
|
ret.reserve(nums.size());
|
|
// TODO: The below could use some optimization. A batched version of both hashForTxNum and
|
|
// heightForTxNum are low-hanging fruit for optimization. Each call to the below takes a shared lock
|
|
// then releases it, for each item. I imagine batched versions would have significantly less overhead
|
|
// per item, which could add up to huge performance savings on large histories. This is a very
|
|
// low hanging fruit for optimization -- thus I am leaving this comment here so I can remember to come
|
|
// back and optmize the below. /TODO
|
|
for (auto num : nums) {
|
|
const BlockHeight height = heightForTxNum(num).value(); // may throw, same deal
|
|
|
|
// Assumption for this loop: the nums are in order!
|
|
if (optToHeight && height >= *optToHeight) break; // threshold of "to height" reached
|
|
else if (height < fromHeight) continue; // keep looping until we hit a height that at least "from height"
|
|
|
|
const auto hash = hashForTxNum(num).value(); // may throw, but that indicates some database inconsistency. we catch below
|
|
ret.emplace_back(/* HistoryItem: */ hash, int(height));
|
|
}
|
|
}
|
|
}
|
|
if (unconf) {
|
|
auto [mempool, lock] = this->mempool();
|
|
if (auto it = mempool.hashXTxs.find(hashX); it != mempool.hashXTxs.end()) {
|
|
const auto & txvec = it->second;
|
|
IncrementCtrAndThrowIfExceedsMaxHistory(txvec.size());
|
|
ret.reserve(ret.size() + txvec.size());
|
|
for (const auto & tx : txvec)
|
|
ret.emplace_back(/* HistoryItem: */ tx->hash, tx->hasUnconfirmedParents() ? -1 : 0, tx->fee);
|
|
}
|
|
}
|
|
} catch (const std::exception &e) {
|
|
Warning(Log::Magenta) << __func__ << ": " << e.what();
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
auto Storage::getRpaHistory(const Rpa::Prefix &prefix, bool includeConfirmed, bool includeMempool,
|
|
BlockHeight fromHeight, std::optional<BlockHeight> endHeight) const-> History
|
|
{
|
|
History ret;
|
|
auto IncrementCtrAndThrowIfExceedsMaxHistory = GetMaxHistoryCtrFunc("RPA History", QString("prefix '%1'").arg(QString(prefix.toHex())),
|
|
options->rpa.maxHistory);
|
|
double tReadDb = 0., tPfxSearch = 0., tResolveTxIdx = 0., tWaitForLock = 0., tBuildRes = 0.;
|
|
|
|
Tic t0;
|
|
SharedLockGuard g(p->blocksLock); // makes sure history doesn't mutate from underneath our feet
|
|
tWaitForLock += t0.msec<double>();
|
|
|
|
const int rpaStartHeight = getConfiguredRpaStartHeight();
|
|
if (UNLIKELY(rpaStartHeight < 0)) {
|
|
// This should have been caught by the caller. Warn to log here since we don't want to do this filtering of
|
|
// requests here in this asynch-called function since it wastes resources to do it this late in the pipeline.
|
|
Warning() << "getRpaHistory() called but RPA appears to be disabled. FIXME!";
|
|
throw InternalError("RPA is disabled");
|
|
}
|
|
|
|
const auto tipHeight = latestHeight();
|
|
if (UNLIKELY( ! tipHeight)) throw InternalError("No blockchain");
|
|
if (unsigned(rpaStartHeight) > *tipHeight) {
|
|
// Nothing to do! Index not yet enabled! Warn here since likely the admin has misconfigured his server.
|
|
Warning() << "getRpaHistory called but rpa_start_height is " << rpaStartHeight << ", which is greater than the"
|
|
<< " block chain height of " << *tipHeight << ".\n\nIf you wish to enable RPA indexing, set the RPA"
|
|
<< " start height to below the blockchain height using the `rpa_start_height` configuration"
|
|
<< " variable. If, on the other hand, you wish to disable RPA indexing, set `rpa = false` in the"
|
|
<< " configuration file.\n\n";
|
|
return ret;
|
|
}
|
|
|
|
try {
|
|
if (includeConfirmed) {
|
|
// sanitize `fromHeight` and `endHeight`; restrict to range: [rpaStartHeight, tipHeight + 1)
|
|
fromHeight = std::max<unsigned>(rpaStartHeight, fromHeight); // restrict `from` to be >= configured height
|
|
endHeight = std::min(endHeight.value_or(*tipHeight + 1u), *tipHeight + 1u); // define and restrict `end` to be <= tip height + 1
|
|
|
|
// We use an iterator and seek forward each time because this is far faster since our table rows are in order
|
|
// of height (serialized as big endian). Note that the assumption here is that the rpa table contains
|
|
// *only* records of the form: Key = 4-byte big endian height, Value = serialized Rpa::PrefixTable.
|
|
// If this assumption changes, update this code to not use this assumption as an optimization.
|
|
std::unique_ptr<rocksdb::Iterator> iter{p->db->NewIterator(p->db.defReadOpts, p->db.rpa)};
|
|
if (UNLIKELY(!iter)) throw DatabaseError("Unable to obtain an iterator to the rpa db");
|
|
|
|
BlockHeight height = fromHeight;
|
|
size_t blockScansRemaining = std::max(options->rpa.historyBlockLimit, 1u); // use configured limit (default: 60)
|
|
for ( /* */; blockScansRemaining && height < *endHeight; ++height, --blockScansRemaining) {
|
|
Tic t1;
|
|
const RpaDBKey dbKey(height);
|
|
if (height == fromHeight)
|
|
iter->Seek(ToSlice(dbKey));
|
|
else
|
|
iter->Next(); // bump iterator one item... this is the secret sauce to make this fast.
|
|
bool ok{};
|
|
if (UNLIKELY(!iter->Valid() || RpaDBKey::fromBytes(FromSlice(iter->key()), &ok, true) != dbKey || !ok)) {
|
|
// This should never happen -- error to console just in case we have bugs and/or missing data.
|
|
Error() << "Missing RPA PrefixTable for height: " << height << ". This should never happen."
|
|
<< " Report this to situation to the developers.";
|
|
break;
|
|
}
|
|
// Note: This read-only Rpa::PrefixTable is "lazy loaded" and populated only for records we access on-demand
|
|
const auto valueSlice = iter->value(); // NB: slice is invalidated when iter is modified
|
|
const auto prefixTable = Deserialize<Rpa::PrefixTable>(FromSlice(valueSlice)); // Throws on failure to deserialize.
|
|
tReadDb += t1.msec<double>();
|
|
// Update RpaInfo stats
|
|
p->rpaInfo.nReads.fetch_add(1, std::memory_order_relaxed);
|
|
p->rpaInfo.nBytesRead.fetch_add(sizeof(uint32_t) + valueSlice.size(), std::memory_order_relaxed);
|
|
|
|
t1 = Tic();
|
|
const bool needSort = prefix.range().size() > 1u; // if prefix spans multiple rows of table, sort and uniqueify
|
|
auto txIdxVec = prefixTable.searchPrefix(prefix, needSort);
|
|
tPfxSearch += t1.msec<double>();
|
|
if (txIdxVec.empty()) continue; // no match for this prefix at this height, keep going
|
|
|
|
IncrementCtrAndThrowIfExceedsMaxHistory(txIdxVec.size());
|
|
|
|
t1 = Tic();
|
|
const auto vecOfOptHashes = hashesForHeightAndPosVec(height, txIdxVec, &g /* <-- tell callee not to re-lock blocksLock */);
|
|
tResolveTxIdx += t1.msec<double>();
|
|
t1 = Tic();
|
|
for (const auto & optHash : vecOfOptHashes) {
|
|
if (LIKELY(optHash)) ret.emplace_back(*optHash, int(height));
|
|
}
|
|
tBuildRes += t1.msec<double>();
|
|
}
|
|
|
|
// Special behavior: disable mempool append if we didn't reach past tipHeight
|
|
if (includeMempool && height <= *tipHeight)
|
|
includeMempool = false;
|
|
}
|
|
if (includeMempool) {
|
|
auto [mempool, lock] = this->mempool();
|
|
if (LIKELY(mempool.optPrefixTable)) {
|
|
const auto origSize = ret.size();
|
|
const bool needSort = prefix.range().size() > 1u; // if prefix spans multiple rows of mempool table, sort and uniqueify
|
|
Tic t1;
|
|
const auto txHashes = mempool.optPrefixTable->searchPrefix(prefix, needSort /* to get unique hashes */);
|
|
tPfxSearch += t1.msec<double>();
|
|
|
|
IncrementCtrAndThrowIfExceedsMaxHistory(txHashes.size());
|
|
|
|
t1 = Tic();
|
|
for (const auto & txHash : txHashes) {
|
|
if (auto it = mempool.txs.find(txHash); LIKELY(it != mempool.txs.end())) {
|
|
const int height = it->second->hasUnconfirmedParents() ? -1 : 0;
|
|
ret.emplace_back(txHash, height, it->second->fee);
|
|
} else {
|
|
Error() << "Tx: " << Util::ToHexFast(txHash) << " for prefix '" << prefix.toHex() << "'"
|
|
<< " exists in Mempool prefix table but not in Mempool txs! FIXME!";
|
|
}
|
|
}
|
|
// force unconf parent to sort after conf parent txns
|
|
std::sort(ret.begin() + origSize, ret.end(), [](const HistoryItem &a, const HistoryItem &b){
|
|
int ha = std::max(a.height, -1), hb = std::max(b.height, -1);
|
|
if (ha <= 0) ha = 0x7f'ff'ff'fe - ha; // -1 becomes -> 0x7f'ff'ff'ff, 0 becomes -> 0x7f'ff'ff'fe
|
|
if (hb <= 0) hb = 0x7f'ff'ff'fe - hb;
|
|
return std::tie(ha, a.hash) < std::tie(hb, b.hash);
|
|
});
|
|
// uniqueify
|
|
auto last = std::unique(ret.begin() + origSize, ret.end());
|
|
ret.erase(last, ret.end());
|
|
tBuildRes += t1.msec<double>();
|
|
} else {
|
|
// This should never happen for mempool.
|
|
Warning() << "Missing RPA PrefixTable for mempool. This should never happen. Contact the developers to report this.";
|
|
}
|
|
}
|
|
} catch (const std::exception &e) {
|
|
Warning(Log::Magenta) << __func__ << ": " << e.what();
|
|
}
|
|
Debug() << "getRpaHistory returned " << ret.size() << " items"
|
|
<< ", readDb: " << QString::number(tReadDb, 'f', 3) << " msec"
|
|
<< ", pfxSearch: " << QString::number(tPfxSearch, 'f', 3) << " msec"
|
|
<< ", resolveTxIdx: " << QString::number(tResolveTxIdx, 'f', 3) << " msec"
|
|
<< ", waitForLock: " << QString::number(tWaitForLock, 'f', 3) << " msec"
|
|
<< ", buildResults: " << QString::number(tBuildRes, 'f', 3) << " msec"
|
|
<< ", total: " << t0.msecStr() << " msec";
|
|
return ret;
|
|
}
|
|
|
|
static bool ShouldTokenFilter(const Storage::TokenFilterOption tokenFilter, const bitcoin::token::OutputDataPtr & p)
|
|
{
|
|
switch (tokenFilter) {
|
|
case Storage::TokenFilterOption::ExcludeTokens:
|
|
return bool(p);
|
|
case Storage::TokenFilterOption::IncludeTokens:
|
|
return false;
|
|
case Storage::TokenFilterOption::OnlyTokens:
|
|
return !p;
|
|
}
|
|
// not normally reached unless there's a programming error of some sort
|
|
throw InternalError(QString("Invalid TokenFilterOption encountered: %1. This shouldn't happen! FIXME!")
|
|
.arg(int(tokenFilter)));
|
|
}
|
|
|
|
auto Storage::listUnspent(const HashX & hashX, const TokenFilterOption tokenFilter) const -> UnspentItems
|
|
{
|
|
UnspentItems ret;
|
|
if (hashX.length() != HashLen)
|
|
return ret;
|
|
try {
|
|
auto ShouldFilter = [tokenFilter](const bitcoin::token::OutputDataPtr & p) { return ShouldTokenFilter(tokenFilter, p); };
|
|
auto IncrementCtrAndThrowIfExceedsMaxHistory = GetMaxHistoryCtrFunc("Unspent UTXOs",
|
|
QString("scripthash %1").arg(QString(hashX.toHex())),
|
|
options->maxHistory);
|
|
constexpr size_t iota = 10; // we initially reserve this many items in the returned array in order to prevent redundant allocations in the common case.
|
|
std::unordered_set<TXO> mempoolConfirmedSpends;
|
|
mempoolConfirmedSpends.reserve(iota);
|
|
ret.reserve(iota);
|
|
{
|
|
// take shared lock (ensure history doesn't mutate from underneath our feet)
|
|
SharedLockGuard g(p->blocksLock);
|
|
const TxNum veryHighTxNum = getTxNum() + 100000000; // pick an absurdly high TxNum that is 100 million past current. This is a fudge so sorting works ok for unconfirmed tx's so that they appear at the end.
|
|
{
|
|
// grab mempool utxos for scripthash -- we do mempool first so as to build the "mempoolConfirmedSpends" set as we iterate.
|
|
auto [mempool, lock] = this->mempool(); // shared lock
|
|
if (auto it = mempool.hashXTxs.find(hashX); it != mempool.hashXTxs.end()) {
|
|
const auto & txvec = it->second;
|
|
for (const auto & tx : txvec) {
|
|
if (!tx) {
|
|
// defensive programming. should never happen
|
|
Warning() << "Cannot find tx for sh " << hashX.toHex() << ". FIXME!!";
|
|
continue;
|
|
}
|
|
if (auto it2 = tx->hashXs.find(hashX); LIKELY(it2 != tx->hashXs.end())) {
|
|
const auto & ioinfo = it2->second;
|
|
// make sure to put any confirmed spends we see now in the "mempool confirmed spends" set
|
|
// so we know not to include them in the list of utxos from the DB later in this function!
|
|
for (const auto & [txo, txoinfo] : ioinfo.confirmedSpends) {
|
|
mempoolConfirmedSpends.insert(txo);
|
|
}
|
|
|
|
// throw if we would iterate too much below
|
|
IncrementCtrAndThrowIfExceedsMaxHistory(ioinfo.utxo.size());
|
|
|
|
for (const auto ionum : ioinfo.utxo) {
|
|
if (decltype(tx->txos.cbegin()) it3;
|
|
LIKELY( ionum < tx->txos.size() && (it3 = tx->txos.cbegin() + ionum)->isValid() ))
|
|
{
|
|
if (ShouldFilter(it3->tokenDataPtr))
|
|
continue;
|
|
ret.push_back(UnspentItem{
|
|
{ tx->hash, 0 /* always put 0 for height here */, tx->fee }, // base HistoryItem
|
|
ionum, // .tx_pos
|
|
it3->amount, // .value
|
|
TxNum(1) + veryHighTxNum + TxNum(tx->hasUnconfirmedParents() ? 1 : 0), // .txNum (this is fudged for sorting at the end properly)
|
|
it3->tokenDataPtr, // .token_data
|
|
});
|
|
} else {
|
|
// this should never happen!
|
|
Warning() << "Cannot find txo " << ionum << " for sh " << hashX.toHex() << " in tx " << tx->hash.toHex();
|
|
continue;
|
|
}
|
|
}
|
|
} else {
|
|
// defensive programming. should never happen
|
|
Warning() << "Cannot find scripthash " << hashX.toHex() << " in tx 'hashX -> IOInfo' map for tx " << tx->hash.toHex() << ". FIXME!";
|
|
}
|
|
}
|
|
}
|
|
} // release mempool lock
|
|
{ // begin confirmed/db search
|
|
std::unique_ptr<rocksdb::Iterator> iter(p->db->NewIterator(p->db.defReadOpts, p->db.shunspent));
|
|
if (UNLIKELY(!iter)) throw DatabaseError("Unable to obtain an iterator to the shunspent db"); // should never happen
|
|
const rocksdb::Slice prefix = ToSlice(hashX); // points to data in hashX
|
|
|
|
// Search table for all keys that start with hashx's bytes. Note: the loop end-condition is strange.
|
|
// See: https://github.com/facebook/rocksdb/wiki/Prefix-Seek-API-Changes#transition-to-the-new-usage
|
|
rocksdb::Slice key;
|
|
using CTXOVec = std::vector<std::pair<CompactTXO, SHUnspentValue>>;
|
|
CTXOVec ctxoVec;
|
|
constexpr size_t reserveBytes = 256u;
|
|
static_assert(sizeof(CTXOVec::value_type) < reserveBytes);
|
|
ctxoVec.reserve(reserveBytes / sizeof(CTXOVec::value_type)); // rough guess -- pre-allocate ~256 bytes
|
|
// we do it this way as two separate loops in order to avoid the expensive heightForTxNum lookups below in
|
|
// the case where the history is huge.
|
|
for (iter->Seek(prefix); iter->Valid() && (key = iter->key()).starts_with(prefix); iter->Next()) {
|
|
IncrementCtrAndThrowIfExceedsMaxHistory();
|
|
bool ok;
|
|
auto shval = Deserialize<SHUnspentValue>(FromSlice(iter->value()), &ok);
|
|
if (UNLIKELY(!ok || !shval.valid)) {
|
|
auto ctxo = extractCompactTXOFromShunspentKey(key); /* may throw if size is bad, etc */
|
|
throw InternalError(QString("Bad SHUnspentValue in db for ctxo %1, script_hash: %2")
|
|
.arg(ctxo.toString(), QString(hashX.toHex())));
|
|
}
|
|
if (UNLIKELY(!bitcoin::MoneyRange(shval.amount))) {
|
|
auto ctxo = extractCompactTXOFromShunspentKey(key); /* may throw if size is bad, etc */
|
|
throw InternalError(QString("Out-of-range amount in db for ctxo %1, script_hash %2: %3")
|
|
.arg(ctxo.toString(), QString(hashX.toHex())).arg(shval.amount / shval.amount.satoshi()));
|
|
}
|
|
if (ShouldFilter(shval.tokenDataPtr))
|
|
continue;
|
|
auto ctxo = extractCompactTXOFromShunspentKey(key); /* may throw if size is bad, etc */
|
|
ctxoVec.emplace_back(std::move(ctxo), std::move(shval));
|
|
}
|
|
for (auto & [ctxo, shval] : ctxoVec) {
|
|
const auto hash = hashForTxNum(ctxo.txNum()).value(); // may throw, but that indicates some database inconsistency. we catch below
|
|
const auto height = heightForTxNum(ctxo.txNum()).value(); // may throw, same deal
|
|
const TXO txo{ hash, ctxo.N() };
|
|
if (mempoolConfirmedSpends.count(txo))
|
|
// Skip items that are spent in mempool. This fixes a bug in Fulcrum 1.0.2 or earlier where the
|
|
// confirmed spends in the mempool were still appearing in the listunspent utxos.
|
|
continue;
|
|
ret.push_back(UnspentItem{
|
|
{ hash, int(height), {} }, // base HistoryItem
|
|
txo.outN, // .tx_pos
|
|
shval.amount, // .value
|
|
ctxo.txNum(), // .txNum
|
|
std::move(shval.tokenDataPtr), // .token_data
|
|
});
|
|
}
|
|
} // end confirmed/db search
|
|
} // release blocks lock
|
|
std::sort(ret.begin(), ret.end());
|
|
if (const auto sz = ret.size(), cap = ret.capacity(); cap - sz > iota && sz > 0 && double(cap)/double(sz) > 1.20)
|
|
// we only do this if we're wasting enough space (at least iota, and at least 20% space wasted),
|
|
// otherwise we don't bother since this returned object is fairly ephemeral and for smallish disparities
|
|
// between capacity and size, it's fine.
|
|
ret.shrink_to_fit();
|
|
} catch (const std::exception &e) {
|
|
Warning(Log::Magenta) << __func__ << ": " << e.what();
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
auto Storage::getBalance(const HashX &hashX, TokenFilterOption tokenFilter) const -> std::pair<bitcoin::Amount, bitcoin::Amount>
|
|
{
|
|
std::pair<bitcoin::Amount, bitcoin::Amount> ret;
|
|
if (hashX.length() != HashLen)
|
|
return ret;
|
|
auto ShouldFilter = [tokenFilter](const bitcoin::token::OutputDataPtr & p) { return ShouldTokenFilter(tokenFilter, p); };
|
|
auto IncrementCtrAndThrowIfExceedsMaxHistory = GetMaxHistoryCtrFunc("GetBalance UTXOs",
|
|
QString("scripthash %1").arg(QString(hashX.toHex())),
|
|
options->maxHistory);
|
|
try {
|
|
// take shared lock (ensure history doesn't mutate from underneath our feet)
|
|
SharedLockGuard g(p->blocksLock);
|
|
{
|
|
// confirmed -- read from db using an iterator
|
|
std::unique_ptr<rocksdb::Iterator> iter(p->db->NewIterator(p->db.defReadOpts, p->db.shunspent));
|
|
if (UNLIKELY(!iter)) throw DatabaseError("Unable to obtain an iterator to the shunspent db"); // should never happen
|
|
const rocksdb::Slice prefix = ToSlice(hashX); // points to data in hashX
|
|
|
|
// Search table for all keys that start with hashx's bytes. Note: the loop end-condition is strange.
|
|
// See: https://github.com/facebook/rocksdb/wiki/Prefix-Seek-API-Changes#transition-to-the-new-usage
|
|
rocksdb::Slice key;
|
|
for (iter->Seek(prefix); iter->Valid() && (key = iter->key()).starts_with(prefix); iter->Next()) {
|
|
IncrementCtrAndThrowIfExceedsMaxHistory(); // throw if we are iterating too much
|
|
const CompactTXO ctxo = extractCompactTXOFromShunspentKey(key); // may throw if key has the wrong size, etc
|
|
bool ok;
|
|
const auto & [valid, amount, tokenDataPtr] = Deserialize<SHUnspentValue>(FromSlice(iter->value()), &ok);
|
|
if (UNLIKELY(!ok || !valid))
|
|
throw InternalError(QString("Bad SHUnspentValue in db for ctxo %1 (%2)").arg(ctxo.toString(), QString(hashX.toHex())));
|
|
if (UNLIKELY(!bitcoin::MoneyRange(amount)))
|
|
throw InternalError(QString("Out-of-range amount in db for ctxo %1: %2").arg(ctxo.toString()).arg(amount / amount.satoshi()));
|
|
if ( ! ShouldFilter(tokenDataPtr)) {
|
|
ret.first += amount; // tally the result
|
|
}
|
|
}
|
|
if (UNLIKELY(!bitcoin::MoneyRange(ret.first))) {
|
|
ret.first = bitcoin::Amount::zero();
|
|
throw InternalError(QString("Out-of-range total in db for getBalance on scripthash: %1").arg(QString(hashX.toHex())));
|
|
}
|
|
}
|
|
{
|
|
// unconfirmed -- check mempool
|
|
auto [mempool, lock] = this->mempool(); // shared (read only) lock is held until scope end
|
|
if (auto it = mempool.hashXTxs.find(hashX); it != mempool.hashXTxs.end()) {
|
|
// for all tx's involving scripthash
|
|
bitcoin::Amount utxos, spends;
|
|
for (const auto & tx : it->second) {
|
|
assert(bool(tx));
|
|
auto it2 = tx->hashXs.find(hashX);
|
|
if (UNLIKELY(it2 == tx->hashXs.end())) {
|
|
throw InternalError(QString("scripthash %1 lists tx %2, which then lacks the IOInfo for said hashX! FIXME!")
|
|
.arg(QString(hashX.toHex()), QString(tx->hash.toHex())));
|
|
}
|
|
auto & info = it2->second;
|
|
IncrementCtrAndThrowIfExceedsMaxHistory(info.confirmedSpends.size() + info.utxo.size()); // throw if >maxHistory
|
|
for (const auto & [txo, txoinfo] : info.confirmedSpends) {
|
|
if ( ! ShouldFilter(txoinfo.tokenDataPtr))
|
|
spends += txoinfo.amount;
|
|
}
|
|
for (const auto ionum : info.utxo) {
|
|
if (decltype(tx->txos.cbegin()) it3; UNLIKELY( ionum >= tx->txos.size()
|
|
|| !(it3 = tx->txos.cbegin() + ionum)->isValid()) )
|
|
{
|
|
throw InternalError(QString("scripthash %1 lists tx %2, which then lacks a valid TXO IONum %3 for said hashX! FIXME!")
|
|
.arg(QString(hashX.toHex()), QString(tx->hash.toHex())).arg(ionum));
|
|
} else if ( ! ShouldFilter(it3->tokenDataPtr)) {
|
|
utxos += it3->amount;
|
|
}
|
|
}
|
|
}
|
|
ret.second = utxos - spends; // note this may not be MoneyRange (may be negative), which is ok.
|
|
}
|
|
}
|
|
} catch (const std::exception &e) {
|
|
Warning(Log::Magenta) << __func__ << ": " << e.what();
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
auto Storage::getFirstUse(const HashX & hashX) const -> std::optional<FirstUse>
|
|
{
|
|
static const QString err("Database error retrieving history for a script hash");
|
|
try {
|
|
SharedLockGuard g(p->blocksLock); // makes sure history doesn't mutate from underneath our feet
|
|
|
|
// try confirmed txns from db
|
|
if (const auto optba = GenericDBGet<QByteArray>(p->db, p->db.shist, hashX, true, err, true, p->db.defReadOpts)) {
|
|
// the history is a bunch of CompactTXO TxNums concatenated, grab the first one
|
|
if (size_t(optba->size()) < CompactTXO::compactTxNumSize()) {
|
|
throw DatabaseSerializationError(QString("Scripthash %1 has a db entry in scripthash_history that is too short: %2")
|
|
.arg(QString::fromLatin1(hashX.toHex()), QString::fromLatin1(optba->toHex())));
|
|
}
|
|
const TxNum txNum = CompactTXO::txNumFromCompactBytes(reinterpret_cast<const std::byte *>(optba->constData()));
|
|
// NB: Below opt.value() calls may throw, which is what we want.
|
|
const BlockHeight blockHeight = heightForTxNum(txNum).value(); // may throw
|
|
return FirstUse(hashForTxNum(txNum).value(), /* .txHash */
|
|
blockHeight, /* .height */
|
|
BTC::HashRev(headerForHeight(blockHeight).value()) /* .blockHash */);
|
|
} else {
|
|
// try unconfirmed (mempool)
|
|
auto [mempool, lock] = this->mempool();
|
|
if (auto it = mempool.hashXTxs.find(hashX); it != mempool.hashXTxs.end()) {
|
|
for (const auto & tx : it->second) { // Note: txs are sorted by (hasUnfonfirmedParentTx, hash)
|
|
for (const auto & txoinfo : tx->txos) {
|
|
if (txoinfo.hashX == hashX) {
|
|
// found a mempool tx that sends an output to `hashX`
|
|
static const QByteArray zeroes32(QByteArray::size_type(HashLen), char(0));
|
|
return FirstUse(tx->hash, tx->hasUnconfirmedParents() ? -1 : 0, zeroes32);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} catch (const std::exception &e) {
|
|
Warning(Log::Magenta) << __func__ << ": " << e.what();
|
|
}
|
|
return std::nullopt;
|
|
}
|
|
|
|
std::vector<QByteArray> Storage::merkleCacheHelperFunc(unsigned int start, unsigned int count, QString *err)
|
|
{
|
|
auto vec = headersFromHeight_nolock_nocheck(start, count, err); // despite the name of this function, it does take a small lock internally and is thread-safe. we cannot use the public one as that would potentially cause a deadlock here
|
|
for (auto & ba : vec)
|
|
ba = BTC::Hash(ba);
|
|
return vec;
|
|
}
|
|
|
|
void Storage::updateMerkleCache(unsigned int height)
|
|
{
|
|
if (!p->merkleCache->isInitialized() && height) {
|
|
try {
|
|
p->merkleCache->initialize(height+1); // this may take a few seconds
|
|
} catch (const std::exception & e) {
|
|
Error() << e.what();
|
|
}
|
|
}
|
|
}
|
|
|
|
Merkle::BranchAndRootPair Storage::headerBranchAndRoot(unsigned height, unsigned cp_height)
|
|
{
|
|
// TODO: there is a potential, non-fatal race condition here where the caller tried to verify that cp_height is
|
|
// above chainTip height, but it's possible for a reorg to happen and for that to no longer be valid by the time
|
|
// the call path gets to this point. That's fine -- an exception will be thrown. This is only ever called by
|
|
// code that catches exceptions.
|
|
assert(p->merkleCache);
|
|
return p->merkleCache->branchAndRoot(cp_height+1, height);
|
|
}
|
|
|
|
auto Storage::genesisHash() const -> HeaderHash
|
|
{
|
|
SharedLockGuard g(p->headerVerifierLock);
|
|
return p->genesisHash;
|
|
}
|
|
|
|
// HistoryItem & UnspentItem -- operator< and operator== -- for sort.
|
|
bool Storage::HistoryItem::operator<(const HistoryItem &o) const noexcept {
|
|
return std::tie(height, hash) < std::tie(o.height, o.hash);
|
|
}
|
|
bool Storage::HistoryItem::operator==(const HistoryItem &o) const noexcept {
|
|
return std::tie(height, hash) == std::tie(o.height, o.hash);
|
|
}
|
|
bool Storage::UnspentItem::operator<(const UnspentItem &o) const noexcept {
|
|
return std::tie( txNum, tx_pos, value, tokenDataPtr, height, hash)
|
|
< std::tie(o.txNum, o.tx_pos, o.value, o.tokenDataPtr, o.height, o.hash);
|
|
}
|
|
bool Storage::UnspentItem::operator==(const UnspentItem &o) const noexcept {
|
|
return std::tie( txNum, tx_pos, value, tokenDataPtr, height, hash)
|
|
== std::tie(o.txNum, o.tx_pos, o.value, o.tokenDataPtr, o.height, o.hash);
|
|
}
|
|
|
|
auto Storage::mempool() const -> std::pair<const Mempool &, SharedLockGuard>
|
|
{
|
|
return {p->mempool, SharedLockGuard{p->mempoolLock}};
|
|
}
|
|
auto Storage::mutableMempool() -> std::pair<Mempool &, ExclusiveLockGuard>
|
|
{
|
|
return {p->mempool, ExclusiveLockGuard{p->mempoolLock}};
|
|
}
|
|
|
|
bool Storage::isMaybeRecentlySeenTx(const TxHash &txhash) const
|
|
{
|
|
if (mempool().first.txs.contains(txhash)) return true;
|
|
SharedLockGuard g{p->blocksLock};
|
|
return p->recentBlockTxHashes.contains(txhash);
|
|
}
|
|
|
|
void Storage::refreshMempoolHistogram()
|
|
{
|
|
Tic t0;
|
|
Mempool::FeeHistogramVec hist;
|
|
// shared lock
|
|
{
|
|
auto [mempool, lock] = this->mempool();
|
|
hist = mempool.calcCompactFeeHistogram();
|
|
}
|
|
// lock exclusively to do the final swap
|
|
{
|
|
ExclusiveLockGuard g(p->mempoolLock);
|
|
p->mempoolFeeHistogram.swap(hist);
|
|
}
|
|
if (t0.msec() >= 10) DebugM("Storage::refreshMempoolHistogram took ", t0.msecStr(), " msec");
|
|
}
|
|
|
|
auto Storage::mempoolHistogram() const -> Mempool::FeeHistogramVec
|
|
{
|
|
SharedLockGuard g(p->mempoolLock);
|
|
return p->mempoolFeeHistogram;
|
|
}
|
|
|
|
|
|
auto Storage::getTxHeights(const std::vector<TxHash> &txHashes) const -> TxHeightsResult
|
|
{
|
|
TxHeightsResult ret;
|
|
ret.reserve(txHashes.size());
|
|
|
|
SharedLockGuard g(p->blocksLock);
|
|
auto txNums = p->db.txhash2txnumMgr->findMany(txHashes);
|
|
if (UNLIKELY(txNums.size() != txHashes.size()))
|
|
// this should never happen
|
|
throw InternalError("findMany() returned an unexpected number of elements! FIXME!");
|
|
|
|
// missing txNums need a mempool check; check in mempool
|
|
{
|
|
auto [mempool, lock] = this->mempool(); // mempool lock is ok to take with blocksLock held
|
|
for (size_t i = 0; i < txHashes.size(); ++i) {
|
|
if (!txNums[i] && mempool.txs.count(txHashes[i]))
|
|
txNums[i] = 0; // 0 = mempool tx
|
|
}
|
|
}
|
|
|
|
// next, transform all non-0 valid txNums to a height (below takes blkInfoLock which is ok to take after blocksLock)
|
|
SharedLockGuard g2(p->blkInfoLock);
|
|
for (const auto & txNum : txNums) {
|
|
ret.emplace_back();
|
|
auto &optHeight = ret.back();
|
|
if (txNum)
|
|
optHeight = *txNum ? heightForTxNum_nolock(*txNum) : 0; // transform to txNum -> height .. note that 0 already indicates mempool
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
auto Storage::getTxHeight(const TxHash &h) const -> std::optional<BlockHeight>
|
|
{
|
|
// We could have just called the above function but the below is a bit faster since it calls TxHash2TxNumMgr::find()
|
|
// rather than findMany(), which is slightly faster.
|
|
std::optional<BlockHeight> ret;
|
|
SharedLockGuard g(p->blocksLock);
|
|
const auto optTxNum = p->db.txhash2txnumMgr->find(h);
|
|
if (optTxNum) {
|
|
// resolve txNum -> height; this ends up taking blkInfoLock (shared mode)
|
|
ret = heightForTxNum(*optTxNum);
|
|
} else {
|
|
// check mempool, this ends up taking the mempool lock (shared mode)
|
|
if (mempool().first.txs.count(h)) // lock held until statement end
|
|
ret = 0; // 0 = mempool tx
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
auto Storage::getConfirmedTxBlockHeightAndHeader(const TxHash &h) const -> std::optional<std::pair<BlockHeight, Header>>
|
|
{
|
|
std::optional<std::pair<BlockHeight, Header>> ret;
|
|
SharedLockGuard g(p->blocksLock); // Take the blocksLock to make this operation atomic and guaranteed consistent
|
|
if (const auto optTxNum = p->db.txhash2txnumMgr->find(h))
|
|
// resolve txNum -> height; this ends up taking blkInfoLock (shared mode)
|
|
if (const auto optHeight = heightForTxNum(*optTxNum))
|
|
// resolve blockHeight -> blockHeader; this ends up taking headerVerifierLock (shared mode)
|
|
if (const auto optHeader = headerForHeight(*optHeight))
|
|
ret.emplace(*optHeight, *optHeader);
|
|
return ret;
|
|
}
|
|
|
|
size_t Storage::dumpAllScriptHashes(QIODevice *outDev, unsigned int indent, unsigned int ilvl,
|
|
const DumpProgressFunc &progFunc, size_t progInterval) const
|
|
{
|
|
if (!outDev || !outDev->isWritable())
|
|
return 0;
|
|
SharedLockGuard g{p->blocksLock};
|
|
std::unique_ptr<rocksdb::Iterator> it {p->db->NewIterator(p->db.defReadOpts, p->db.shist)};
|
|
if (!it) return 0;
|
|
|
|
const auto INDENT = [outDev, &ilvl, spaces = QByteArray(int(indent), ' ')] {
|
|
for (size_t i = 0; i < ilvl; ++i)
|
|
outDev->write(spaces);
|
|
};
|
|
const Util::VoidFunc NL = indent ? Util::VoidFunc([outDev, &INDENT] {
|
|
outDev->putChar('\n');
|
|
INDENT();
|
|
}) : Util::VoidFunc([]{});
|
|
size_t ctr = 0;
|
|
progInterval = std::max(size_t(1), progInterval);
|
|
|
|
if (indent) INDENT();
|
|
outDev->putChar('[');
|
|
++ilvl;
|
|
NL();
|
|
if (progFunc) progFunc(0); // 0 = indicate operator began
|
|
qint64 lastWriteCt = 0;
|
|
for (it->SeekToFirst(); it->Valid() && outDev && lastWriteCt > -1; it->Next()) {
|
|
const auto sh = it->key();
|
|
if (sh.size() == HashLen) {
|
|
if (LIKELY(ctr)) {
|
|
outDev->putChar(',');
|
|
NL();
|
|
}
|
|
outDev->putChar('"');
|
|
lastWriteCt = outDev->write(Util::ToHexFast(FromSlice(sh)));
|
|
outDev->putChar('"');
|
|
if (UNLIKELY(!(++ctr % progInterval) && progFunc))
|
|
progFunc(ctr);
|
|
}
|
|
}
|
|
--ilvl;
|
|
if (ctr) NL();
|
|
outDev->putChar(']');
|
|
|
|
if (progFunc && ctr % progInterval)
|
|
progFunc(ctr); // always called for last item to indicate operation ended
|
|
return ctr;
|
|
}
|
|
|
|
auto Storage::calcUTXOSetStats(const DumpProgressFunc & progFunc, size_t progInterval) const -> UTXOSetStats
|
|
{
|
|
UTXOSetStats ret;
|
|
if (!p->db.utxoset || !p->db.shunspent) return ret;
|
|
auto readOpts = p->db.defReadOpts;
|
|
const auto [ss, bheight, bhash] = [&] {
|
|
SharedLockGuard g{p->blocksLock};
|
|
using CSnapshot = const rocksdb::Snapshot;
|
|
auto snap = std::shared_ptr<CSnapshot>(p->db->GetSnapshot(), [this](CSnapshot *ss){ p->db->ReleaseSnapshot(ss); });
|
|
const auto & [height, hash] = latestTip(); // takes a subordinate lock to blocksLock
|
|
return std::tuple(snap, height, hash);
|
|
}();
|
|
readOpts.snapshot = ss.get();
|
|
std::unique_ptr<rocksdb::Iterator> it_utxo {p->db->NewIterator(readOpts, p->db.utxoset)};
|
|
std::unique_ptr<rocksdb::Iterator> it_shu {p->db->NewIterator(readOpts, p->db.shunspent)};
|
|
if (!it_utxo || !it_shu) return ret;
|
|
|
|
ret.block_height = bheight >= 0 ? BlockHeight(bheight) : 0;
|
|
ret.block_hash = bhash;
|
|
|
|
// Handle app shutdown by aborting this operation asap if we get a quit signal
|
|
std::atomic_bool quitting = false;
|
|
QMetaObject::Connection conn;
|
|
Defer d([&conn]{ if (conn && ::app()) { ::app()->disconnect(conn); conn = QMetaObject::Connection{}; } });
|
|
if (auto *a = ::app())
|
|
conn = a->connect(a, &App::requestQuit, this, [&quitting] { quitting = true; }, Qt::DirectConnection);
|
|
|
|
auto UpdateProgress = [&, ctr = size_t{0u}]() mutable {
|
|
if (progInterval && (++ctr % progInterval == 0u) && progFunc) progFunc(ctr);
|
|
return !quitting;
|
|
};
|
|
{
|
|
bitcoin::CHash256 hasher;
|
|
for (it_utxo->SeekToFirst(); it_utxo->Valid(); it_utxo->Next()) {
|
|
auto const k = it_utxo->key();
|
|
auto const v = it_utxo->value();
|
|
hasher.Write(reinterpret_cast<const uint8_t *>(k.data()), k.size());
|
|
hasher.Write(reinterpret_cast<const uint8_t *>(v.data()), v.size());
|
|
++ret.utxo_db_ct;
|
|
ret.utxo_db_size_bytes += k.size() + v.size();
|
|
if (UNLIKELY(!UpdateProgress())) { ret = UTXOSetStats{}; return ret; }
|
|
}
|
|
ret.utxo_db_shasum.resize(HashLen);
|
|
hasher.Finalize(reinterpret_cast<uint8_t *>(ret.utxo_db_shasum.data()));
|
|
}
|
|
|
|
{
|
|
bitcoin::CHash256 hasher;
|
|
for (it_shu->SeekToFirst(); it_shu->Valid(); it_shu->Next()) {
|
|
auto const k = it_shu->key();
|
|
auto const v = it_shu->value();
|
|
hasher.Write(reinterpret_cast<const uint8_t *>(k.data()), k.size());
|
|
hasher.Write(reinterpret_cast<const uint8_t *>(v.data()), v.size());
|
|
++ret.shunspent_db_ct;
|
|
ret.shunspent_db_size_bytes += k.size() + v.size();
|
|
if (UNLIKELY(!UpdateProgress())) { ret = UTXOSetStats{}; return ret; }
|
|
}
|
|
ret.shunspent_db_shasum.resize(HashLen);
|
|
hasher.Finalize(reinterpret_cast<uint8_t *>(ret.shunspent_db_shasum.data()));
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
namespace {
|
|
// specializations of Serialize/Deserialize
|
|
template <> QByteArray Serialize(const Meta &m)
|
|
{
|
|
QByteArray ba;
|
|
{
|
|
QDataStream ds(&ba, QIODevice::WriteOnly|QIODevice::Truncate);
|
|
// we serialize the 'magic' value as a simple scalar as a sort of endian check for the DB
|
|
ds << SerializeScalarNoCopy(m.magic) << m.version << m.chain << m.platformBits << m.coin;
|
|
if (m.isMinimumExtraPlatformInfoVersion()) {
|
|
ds << m.appName << m.appVersion << m.rocksDBVersion << m.buildABI << m.osName << m.cpuArch;
|
|
}
|
|
}
|
|
return ba;
|
|
}
|
|
template <> Meta Deserialize(const QByteArray &ba, bool *ok_ptr)
|
|
{
|
|
bool dummy;
|
|
bool &ok (ok_ptr ? *ok_ptr : dummy);
|
|
ok = false;
|
|
Meta m(Meta::ClearedForUnser);
|
|
{
|
|
QDataStream ds(ba);
|
|
QByteArray magicBytes;
|
|
ds >> magicBytes; // read magic as raw bytes.
|
|
if ((ok = ds.status() == QDataStream::Status::Ok)) {
|
|
m.magic = DeserializeScalar<decltype (m.magic)>(magicBytes, &ok);
|
|
if (ok) {
|
|
ds >> m.version >> m.chain;
|
|
ok = ds.status() == QDataStream::Status::Ok;
|
|
if (ok && !ds.atEnd()) {
|
|
// TODO: make this field non-optional. For now we tolerate it missing since we added this field
|
|
// later and we want to be able to still test on our existing db's.
|
|
ds >> m.platformBits;
|
|
}
|
|
ok = ds.status() == QDataStream::Status::Ok;
|
|
if (ok) {
|
|
if (!ds.atEnd()) {
|
|
// Newer db's will always either have an empty string "", "BCH", or "BTC" here.
|
|
// Read the db value now. Client code gets this value via Storage::getCoin().
|
|
ds >> m.coin;
|
|
|
|
// If version >= 3, and magic ok, and no errors, proceed to read the extra platform info
|
|
// which is new in db's with version 3 or above.
|
|
ok = ds.status() == QDataStream::Status::Ok;
|
|
if (ok && m.isMagicOk() && m.isMinimumExtraPlatformInfoVersion()) {
|
|
ds >> m.appName >> m.appVersion >> m.rocksDBVersion >> m.buildABI >> m.osName >> m.cpuArch;
|
|
}
|
|
} else {
|
|
// Older db's pre-1.3.0 lacked this field -- but now we interpret missing data here as
|
|
// "BCH" (since all older db's were always BCH only).
|
|
m.coin = BTC::coinToName(BTC::Coin::BCH);
|
|
Debug() << "Missing coin info from Meta table, defaulting coin to: \"" << m.coin << "\"";
|
|
}
|
|
}
|
|
ok = ds.status() == QDataStream::Status::Ok;
|
|
}
|
|
}
|
|
}
|
|
return m;
|
|
}
|
|
|
|
template <> QByteArray Serialize(const TXO &txo) { return txo.toBytes(false); }
|
|
template <> TXO Deserialize(const QByteArray &ba, bool *ok) {
|
|
TXO ret = TXO::fromBytes(ba); // requires 34 or 35 byte size
|
|
if (ok) *ok = ret.isValid();
|
|
return ret;
|
|
}
|
|
|
|
template <> QByteArray Serialize(const TXOInfo &inf) { return inf.toBytes(); }
|
|
template <> TXOInfo Deserialize(const QByteArray &ba, bool *ok)
|
|
{
|
|
TXOInfo ret = TXOInfo::fromBytes(ba); // will fail if extra bytes at the end
|
|
if (ok) *ok = ret.isValid();
|
|
return ret;
|
|
}
|
|
|
|
template <> Rpa::PrefixTable Deserialize(const QByteArray &ba, bool *ok) {
|
|
Rpa::PrefixTable ret(ba); // Note: PrefixTable does not keep a copy of `ba`, so it's ok if `ba` is a view into a temporary Slice
|
|
if (ok) *ok = true;
|
|
return ret;
|
|
}
|
|
|
|
// essentially takes a byte copy of the data of BlkInfo; note that we waste some space at the end for legacy compat.
|
|
template <> QByteArray Serialize(const BlkInfo &b) {
|
|
static_assert(sizeof(BlkInfo) >= 12 && sizeof(b.txNum0) == 8 && sizeof(b.nTx) == 4,
|
|
"Serialization of BlkInfo assumes 64-bit txNum0 and 32-bit nTx members");
|
|
QByteArray ret(QByteArray::size_type(sizeof(b)), Qt::Uninitialized);
|
|
auto *cur = ret.data();
|
|
std::memcpy(cur, &b.txNum0, sizeof(b.txNum0));
|
|
cur += sizeof(b.txNum0);
|
|
std::memcpy(cur, &b.nTx, sizeof(b.nTx));
|
|
cur += sizeof(b.nTx);
|
|
// On most platforms, the end is padded with 4 bytes because in previous versions of this code we wrote the raw
|
|
// BlkInfo struct to the byte array (which had padding for alignment). We don't do this anymore for
|
|
// privacy/security reasons but emulate the old behavior and pad with zeroes at the end.
|
|
ptrdiff_t padding = (ret.data() + ret.size()) - cur;
|
|
if (padding > 0) {
|
|
std::memset(cur, 0, padding);
|
|
cur += padding;
|
|
}
|
|
assert(cur == ret.data() + ret.size());
|
|
return ret;
|
|
}
|
|
// will fail if the size doesn't match size of BlkInfo exactly
|
|
template <> BlkInfo Deserialize(const QByteArray &ba, bool *ok) {
|
|
BlkInfo ret;
|
|
if (ba.length() != sizeof(ret)) {
|
|
if (ok) *ok = false;
|
|
} else {
|
|
if (ok) *ok = true;
|
|
auto *cur = ba.constData();
|
|
std::memcpy(reinterpret_cast<std::byte *>(&ret.txNum0), cur, sizeof(ret.txNum0));
|
|
cur += sizeof(ret.txNum0);
|
|
std::memcpy(reinterpret_cast<std::byte *>(&ret.nTx), cur, sizeof(ret.nTx));
|
|
cur += sizeof(ret.nTx);
|
|
assert(cur <= ba.constData() + ba.size());
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
struct UndoInfoSerHeader {
|
|
static constexpr uint16_t defMagic = 0xf12cu, v1Ver = 0x1u, v2Ver = 0x2u, v3Ver = 0x3u;
|
|
static constexpr auto defVer = v3Ver;
|
|
uint16_t magic = defMagic; ///< sanity check
|
|
uint16_t ver = defVer; ///< sanity check
|
|
uint32_t len = 0; ///< the length of the entire buffer, including this struct and all data to follow. A sanity check.
|
|
uint32_t nScriptHashes = 0, nAddUndos = 0, nDelUndos = 0; ///< the number of elements in each of the 3 arrays in question.
|
|
|
|
/* ----------- V1 format (back when we had 2 byte IONums) */
|
|
static constexpr size_t addUndoItemSerSize_V1 = TXO::minSize() + HashLen + CompactTXO::minSize();
|
|
static constexpr size_t delUndoItemSerSize_V1 = TXO::minSize() + TXOInfo::minSerSize();
|
|
|
|
/// computes the total size given the ser size of the blkInfo struct.
|
|
/// Requires that nScriptHashes, nAddUndos, and nDelUndos be already filled-in.
|
|
size_t computeTotalSize_V1() const {
|
|
const auto shSize = nScriptHashes * HashLen;
|
|
const auto addsSize = nAddUndos * addUndoItemSerSize_V1;
|
|
const auto delsSize = nDelUndos * delUndoItemSerSize_V1;
|
|
return sizeof(*this) + sizeof(UndoInfo::height) + HashLen + sizeof(BlkInfo) + shSize + addsSize + delsSize;
|
|
}
|
|
bool isLenSane_V1() const { return size_t(len) == computeTotalSize_V1(); }
|
|
|
|
|
|
/* ----------- V2 format (3-byte IONums) */
|
|
static constexpr size_t addUndoItemSerSize_V2 = TXO::maxSize() + HashLen + CompactTXO::maxSize();
|
|
static constexpr size_t delUndoItemSerSize_V2 = TXO::maxSize() + TXOInfo::minSerSize();
|
|
|
|
/// computes the total size given the ser size of the blkInfo struct. Requires that nScriptHashes, nAddUndos, and nDelUndos be already filled-in.
|
|
size_t computeTotalSize_V2() const {
|
|
const auto shSize = nScriptHashes * HashLen;
|
|
const auto addsSize = nAddUndos * addUndoItemSerSize_V2;
|
|
const auto delsSize = nDelUndos * delUndoItemSerSize_V2;
|
|
return sizeof(*this) + sizeof(UndoInfo::height) + HashLen + sizeof(BlkInfo) + shSize + addsSize + delsSize;
|
|
}
|
|
bool isLenSane_V2() const { return size_t(len) == computeTotalSize_V2(); }
|
|
|
|
/* ----------- V3 format (same as V3 but has dynamically-sized TXOInfo objects >= 50 bytes) */
|
|
/// computes the minimum size given the ser size of the blkInfo struct. Requires that nScriptHashes, nAddUndos, and nDelUndos be already filled-in.
|
|
size_t computeMinimumSize_V3() const { return computeTotalSize_V2(); }
|
|
bool isLenMinimallySane_V3() const { return size_t(len) >= computeMinimumSize_V3(); }
|
|
};
|
|
|
|
static_assert(std::has_unique_object_representations_v<UndoInfoSerHeader>, "This type is serialized as bytes to db");
|
|
|
|
// Deserialize a header from bytes -- no checks are done other than length check.
|
|
template <> UndoInfoSerHeader Deserialize(const QByteArray &ba, bool *ok) {
|
|
UndoInfoSerHeader ret;
|
|
if (ba.length() < int(sizeof(ret))) {
|
|
if (ok) *ok = false;
|
|
} else {
|
|
if (ok) *ok = true;
|
|
std::memcpy(reinterpret_cast<std::byte *>(&ret), ba.constData(), sizeof(ret));
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
// UndoInfo -- serialize to V3 format (fixed 3-byte IONums, dynamically-sized TXOInfo objects)
|
|
template <> QByteArray Serialize(const UndoInfo &u) {
|
|
UndoInfoSerHeader hdr;
|
|
// fill these in now so that hdr.computeTotalSize works
|
|
hdr.nScriptHashes = uint32_t(u.scriptHashes.size());
|
|
hdr.nAddUndos = uint32_t(u.addUndos.size());
|
|
hdr.nDelUndos = uint32_t(u.delUndos.size());
|
|
hdr.len = uint32_t(hdr.computeMinimumSize_V3());
|
|
const size_t offset_of_len = offsetof(UndoInfoSerHeader, len);
|
|
QByteArray ret;
|
|
ret.reserve(int(hdr.len));
|
|
// 1. header
|
|
ret.append(ShallowTmp(&hdr));
|
|
// 2. .height
|
|
ret.append(SerializeScalarNoCopy(u.height));
|
|
// 3. .hash
|
|
const auto chkHashLen = [&ret](const QByteArray & hash) -> bool {
|
|
if (hash.length() != HashLen) {
|
|
Warning() << "hash is not " << HashLen << " bytes. Serialize UndoInfo fail. FIXME!";
|
|
ret.clear();
|
|
return false;
|
|
}
|
|
return true;
|
|
};
|
|
if (!chkHashLen(u.hash)) return ret;
|
|
ret.append(u.hash);
|
|
// 4. .blkInfo
|
|
const QByteArray blkInfoBytes = Serialize(u.blkInfo);
|
|
ret.append(blkInfoBytes);
|
|
// 5. .scriptHashes, 32 bytes each, for all in set
|
|
for (const auto & sh : u.scriptHashes) {
|
|
if (UNLIKELY(!chkHashLen(sh))) return ret;
|
|
ret.append(sh);
|
|
}
|
|
// 6. .addUndos, 76 bytes each * nAddUndos
|
|
for (const auto & [txo, hashX, ctxo] : u.addUndos) {
|
|
if (UNLIKELY(!chkHashLen(hashX))) return ret;
|
|
ret.append(txo.toBytes(true /* force wide (3 byte IONum) */));
|
|
ret.append(hashX);
|
|
ret.append(ctxo.toBytes(true /* force wide (3 byte IONum) */));
|
|
}
|
|
// 7. .delUndos, >=85 bytes each * nDelUndos
|
|
for (const auto & [txo, txoInfo] : u.delUndos) {
|
|
ret.append(txo.toBytes(true));
|
|
if (UNLIKELY(!chkHashLen(txoInfo.hashX))) return ret;
|
|
const QByteArray serinfo = Serialize(txoInfo);
|
|
ret.append(VarInt(uint32_t(serinfo.size())).byteArray(false)); // append size as VarInt (our own internal compact int)
|
|
ret.append(serinfo);
|
|
}
|
|
if (UNLIKELY(ret.length() < QByteArray::size_type(hdr.len))) {
|
|
Warning() << "unexpected length when serializing an UndoInfo object: " << ret.length() << ". FIXME!";
|
|
ret.clear();
|
|
return ret;
|
|
}
|
|
// 8. update length since serializing potential token data for each TXOInfo has variable size
|
|
hdr.len = uint32_t(ret.length());
|
|
std::memcpy(ret.data() + offset_of_len, &hdr.len, sizeof(hdr.len));
|
|
|
|
return ret;
|
|
}
|
|
|
|
// UndoInfo -- note this will fail if the byte array has extra bytes at the end
|
|
template <> UndoInfo Deserialize(const QByteArray &ba, bool *ok) {
|
|
UndoInfo ret;
|
|
const auto setOk = [&ok, &ret] (bool b) { if (ok) *ok = b; if (!b) ret.clear(); };
|
|
const auto chkAssertion = [&setOk] (bool assertion, const char *extra = "") {
|
|
if (UNLIKELY(!assertion)) {
|
|
Warning() << "Deserialize UndoInfo called with an invalid byte array! FIXME! " << extra;
|
|
setOk(false);
|
|
}
|
|
return assertion;
|
|
};
|
|
if (!chkAssertion(ba.size() > int(sizeof(UndoInfoSerHeader)), "Short byte count"))
|
|
return ret;
|
|
|
|
bool myok = false;
|
|
// 1. .header
|
|
const UndoInfoSerHeader hdr = Deserialize<UndoInfoSerHeader>(ba, &myok);;
|
|
if (!chkAssertion(myok && int(hdr.len) == ba.size() && hdr.magic == hdr.defMagic
|
|
&& ( (hdr.ver == hdr.v3Ver && hdr.isLenMinimallySane_V3())
|
|
|| (hdr.ver == hdr.v2Ver && hdr.isLenSane_V2())
|
|
|| (hdr.ver == hdr.v1Ver && hdr.isLenSane_V1()) ),
|
|
"Header sanity check fail"))
|
|
return ret;
|
|
ret.deserVersion = hdr.ver;
|
|
|
|
// for v1 we deserialize 2-byte fixed-size IONums, for v2 3-byte fixed-size IONums
|
|
const bool isV1 = hdr.ver == hdr.v1Ver;
|
|
// for v2 we assume fixed-size TXOInfo objects (this was before token data existed), so we deserialize them
|
|
// as a flat array without any VarInt info as to the size of each
|
|
const bool isV2 = hdr.ver == hdr.v2Ver;
|
|
|
|
// print to debug if encountering V1 vs V2
|
|
DebugM("Deserializing ", isV1 ? "V1" : (isV2 ? "V2" : "V3"), " undo info of length ", hdr.len);
|
|
|
|
const size_t TXOSerSize = isV1 ? TXO::minSize() : TXO::maxSize();
|
|
const size_t CompactTXOSerSize = isV1 ? CompactTXO::minSize() : CompactTXO::maxSize();
|
|
//
|
|
// deserialize V1 data -> fixed 2-byte IONums
|
|
// deserialize V2 or V3 data -> fixed 3-byte IONums
|
|
//
|
|
const char *cur = ba.constData() + sizeof(hdr), *const end = ba.constData() + ba.length();
|
|
// 2. .height
|
|
ret.height = DeserializeScalar<decltype(ret.height)>(ShallowTmp(cur, sizeof(ret.height)), &myok);
|
|
if (!chkAssertion(myok && cur < end))
|
|
return ret;
|
|
cur += sizeof(ret.height);
|
|
// 3. .hash
|
|
ret.hash = DeepCpy(cur, HashLen); // deep copy
|
|
if (!chkAssertion(ret.hash.length() == HashLen && cur < end))
|
|
return ret;
|
|
cur += HashLen;
|
|
// 4. .blkInfo
|
|
ret.blkInfo = Deserialize<BlkInfo>(ShallowTmp(cur, sizeof(BlkInfo)), &myok);
|
|
if (!chkAssertion(myok && cur <= end))
|
|
return ret;
|
|
cur += sizeof(BlkInfo);
|
|
// 5. .scriptHashes, 32 bytes each * hdr->nScriptHashes
|
|
ret.scriptHashes.reserve(hdr.nScriptHashes);
|
|
for (unsigned i = 0; i < hdr.nScriptHashes; ++i) {
|
|
if (!chkAssertion(cur+HashLen <= end)) return ret;
|
|
ret.scriptHashes.insert(DeepCpy(cur, HashLen)); // deep copy
|
|
cur += HashLen;
|
|
}
|
|
// 6. .addUndos, 74 (v1) or 76 (v2) bytes each * nAddUndos
|
|
ret.addUndos.reserve(hdr.nAddUndos);
|
|
for (unsigned i = 0; i < hdr.nAddUndos; ++i) {
|
|
if (!chkAssertion(cur+TXOSerSize <= end)) return ret;
|
|
TXO txo = Deserialize<TXO>(ShallowTmp(cur, TXOSerSize), &myok);
|
|
cur += TXOSerSize;
|
|
if (!chkAssertion(myok && cur+HashLen <= end)) return ret;
|
|
QByteArray hashX = DeepCpy(cur, HashLen); // deep copy
|
|
cur += HashLen;
|
|
if (!chkAssertion(cur+CompactTXOSerSize <= end)) return ret;
|
|
CompactTXO ctxo = Deserialize<CompactTXO>(ShallowTmp(cur, CompactTXOSerSize), &myok);
|
|
cur += CompactTXOSerSize;
|
|
if (!chkAssertion(myok)) return ret;
|
|
ret.addUndos.emplace_back(std::move(txo), std::move(hashX), std::move(ctxo));
|
|
}
|
|
// 7. .delUndos, 84 (v1) or 85 (v2) bytes each * nDelUndos, for v3 the size is dynamic but always >= 85
|
|
ret.delUndos.reserve(hdr.nDelUndos);
|
|
for (unsigned i = 0; i < hdr.nDelUndos; ++i) {
|
|
if (!chkAssertion(cur+TXOSerSize <= end)) return ret;
|
|
TXO txo = Deserialize<TXO>(ShallowTmp(cur, TXOSerSize), &myok);
|
|
cur += TXOSerSize;
|
|
if (!chkAssertion(myok && cur + TXOInfo::minSerSize() <= end)) return ret;
|
|
int txoinfo_size{};
|
|
if (isV1 || isV2) {
|
|
txoinfo_size = int(TXOInfo::minSerSize());
|
|
} else {
|
|
// V3 or above: read the byte size as a VarInt.
|
|
Span sp{cur, size_t(end - cur)};
|
|
try {
|
|
const VarInt vi = VarInt::deserialize(sp); // this may throw
|
|
cur = sp.data(); // span was updated to point past the varint
|
|
txoinfo_size = int(vi.value<uint32_t>()); // this may throw
|
|
} catch (const std::exception &e) {
|
|
chkAssertion(false, e.what());
|
|
return ret;
|
|
}
|
|
}
|
|
if (!chkAssertion(txoinfo_size >= int(TXOInfo::minSerSize()) && cur + txoinfo_size <= end,
|
|
"deser of VarInt size for a TXOInfo returned an error"))
|
|
return ret;
|
|
TXOInfo info = Deserialize<TXOInfo>(ShallowTmp(cur, txoinfo_size), &myok);
|
|
if (!chkAssertion(myok, "deser fail on TXOInfo object")) return ret;
|
|
cur += txoinfo_size;
|
|
ret.delUndos.emplace_back(std::move(txo), std::move(info));
|
|
}
|
|
if (!chkAssertion(cur == end, "cur != end")) return ret;
|
|
setOk(true);
|
|
return ret;
|
|
}
|
|
|
|
template <> QByteArray Serialize(const TxNumVec &v)
|
|
{
|
|
// this serializes a vector of TxNums to a compact representation (6 bytes, eg 48 bits per TxNum), in little endian byte order
|
|
constexpr auto compactSize = CompactTXO::compactTxNumSize(); /* 6 */
|
|
const size_t nBytes = v.size() * compactSize;
|
|
QByteArray ret(QByteArray::size_type(nBytes), Qt::Uninitialized);
|
|
if (UNLIKELY(nBytes != size_t(ret.size()))) {
|
|
throw DatabaseSerializationError(QString("Overflow or other error when attempting to serialize a TxNumVec"
|
|
" of %1 bytes").arg(qulonglong(nBytes)));
|
|
}
|
|
std::byte *cur = reinterpret_cast<std::byte *>(ret.data());
|
|
for (const auto num : v) {
|
|
CompactTXO::txNumToCompactBytes(cur, num);
|
|
cur += compactSize;
|
|
}
|
|
return ret;
|
|
}
|
|
// this deserializes a vector of TxNums from a compact representation (6 bytes, eg 48 bits per TxNum), assuming little endian byte order
|
|
template <> TxNumVec Deserialize(const QByteArray &ba, bool *ok)
|
|
{
|
|
constexpr auto compactSize = CompactTXO::compactTxNumSize(); /* 6 */
|
|
const size_t blen = size_t(ba.length());
|
|
const size_t N = blen / compactSize;
|
|
TxNumVec ret;
|
|
if (N * compactSize != blen) {
|
|
// wrong size, not multiple of 6; bail
|
|
if (ok) *ok = false;
|
|
return ret;
|
|
}
|
|
if (ok) *ok = true;
|
|
auto * cur = reinterpret_cast<const std::byte *>(ba.constData());
|
|
auto * const end = reinterpret_cast<const std::byte *>(ba.constData() + blen);
|
|
ret.reserve(N);
|
|
for ( ; cur < end; cur += compactSize) {
|
|
ret.push_back( CompactTXO::txNumFromCompactBytes(cur) );
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
//template <> QByteArray Serialize(const CompactTXO &c) { return c.toBytes(false); }
|
|
template <> CompactTXO Deserialize(const QByteArray &b, bool *ok) {
|
|
CompactTXO ret = CompactTXO::fromBytes(b);
|
|
if (ok) *ok = ret.isValid();
|
|
return ret;
|
|
}
|
|
|
|
QByteArray Serialize2(const bitcoin::Amount &a, const bitcoin::token::OutputData *ptok) {
|
|
QByteArray ret = SerializeScalar(a / a.satoshi());
|
|
BTC::SerializeTokenDataWithPrefix(ret, ptok); // may be no-op if ptok is nullptr
|
|
return ret;
|
|
}
|
|
template <> SHUnspentValue Deserialize(const QByteArray &ba, bool *pok) {
|
|
int pos = 0;
|
|
bool ok;
|
|
const int64_t amt = DeserializeScalar<int64_t>(ba, &ok, &pos);
|
|
SHUnspentValue ret;
|
|
if (ok) {
|
|
ret.amount = amt * bitcoin::Amount::satoshi();
|
|
try {
|
|
ret.tokenDataPtr = BTC::DeserializeTokenDataWithPrefix(ba, pos);
|
|
} catch (const std::exception &e) {
|
|
throw DatabaseSerializationError(
|
|
QString("Got exception deserializing token data in Storage.cpp::Deserialize(): %1 (from bytes: %2)")
|
|
.arg(e.what(), ba.mid(pos).toHex().constData()));
|
|
}
|
|
}
|
|
if (pok) *pok = ok;
|
|
ret.valid = ok;
|
|
return ret;
|
|
}
|
|
|
|
} // end anon namespace
|
|
|
|
#ifdef ENABLE_TESTS
|
|
#include "Storage/RecordFile.h"
|
|
#include "robin_hood/robin_hood.h"
|
|
namespace {
|
|
|
|
template<size_t NB>
|
|
auto DeduceSmallestTypeForNumBytes() {
|
|
if constexpr (NB == 1) return uint8_t{};
|
|
else if constexpr (NB == 2) return uint16_t{};
|
|
else if constexpr (NB <= 4) return uint32_t{};
|
|
else if constexpr (NB <= 8) return uint64_t{};
|
|
#ifdef __SIZEOF_INT128__
|
|
else if constexpr (NB <= 16) return __uint128_t{};
|
|
#endif
|
|
else throw std::domain_error("too big");
|
|
}
|
|
|
|
enum class MyPos { Beginning, Middle, End};
|
|
|
|
template <size_t NB, MyPos where = MyPos::End>
|
|
ByteView MakeTxHashByteKey(const ByteView &bv) {
|
|
const auto len = bv.size();
|
|
if (UNLIKELY(len != HashLen))
|
|
throw BadArgs(QString("%1... is not %2 bytes").arg(QString(Util::ToHexFast(bv.substr(0, 8).toByteArray(false)))).arg(HashLen));
|
|
static_assert(NB > 0 && NB <= HashLen);
|
|
if constexpr (where == MyPos::End)
|
|
return bv.substr(len - NB, NB);
|
|
else if constexpr (where == MyPos::Middle)
|
|
return bv.substr(len/2 - NB/2, NB);
|
|
else // Beginning
|
|
return bv.substr(0, NB);
|
|
}
|
|
|
|
template <size_t NB, MyPos where = MyPos::End, typename KeyType = decltype(DeduceSmallestTypeForNumBytes<NB>())>
|
|
KeyType MakeTxHashNumericKey(const ByteView &bv) {
|
|
static_assert(NB <= sizeof(KeyType));
|
|
static_assert(std::is_standard_layout_v<KeyType> && std::is_trivial_v<KeyType> && !std::is_floating_point_v<KeyType>);
|
|
KeyType ret{};
|
|
std::memcpy(reinterpret_cast<std::byte *>(&ret), MakeTxHashByteKey<NB, where>(bv).data(), NB);
|
|
return ret;
|
|
}
|
|
|
|
inline constexpr size_t NB = 6;
|
|
inline constexpr MyPos POS = MyPos::End;
|
|
|
|
void findCollisions() {
|
|
Debug::forceEnable = true;
|
|
const QString txnumsFile = std::getenv("TFILE") ? std::getenv("TFILE") : "";
|
|
if (txnumsFile.isEmpty() || !QFile::exists(txnumsFile))
|
|
throw Exception("Please pass the TFILE env var as a path to an existing \"txnum2txhash\" data record file");
|
|
std::unique_ptr<RecordFile> rf;
|
|
rf = std::make_unique<RecordFile>(txnumsFile, HashLen, 0x000012e2); // this may throw
|
|
using KeyType = decltype(DeduceSmallestTypeForNumBytes<NB>());
|
|
using CtrType = decltype(DeduceSmallestTypeForNumBytes<NB <= 2 ? (NB == 1 ? 4 : 2) : 1>());
|
|
const auto nrec = rf->numRecords();
|
|
Log() << "Records: " << nrec;
|
|
robin_hood::unordered_flat_map<KeyType, CtrType> cols;
|
|
Log() << "Reserving table ...";
|
|
size_t nCols = 0, maxCol = 0;
|
|
KeyType maxColVal = 0;
|
|
cols.reserve(std::min<size_t>(nrec, std::numeric_limits<KeyType>::max()));
|
|
constexpr size_t batchSize = 50'000;
|
|
Log() << "Using key bytes: " << NB << ", batchSize: " << batchSize;
|
|
const Tic t0;
|
|
for (size_t i = 0; i < nrec; i += batchSize) {
|
|
if (i && 0 == i % 1'000'000) Debug() << i << "/" << nrec << ", collisions so far: " << nCols << " ...";
|
|
QString err;
|
|
const auto recs = rf->readRecords(i, batchSize, &err);
|
|
for (const auto & rec : recs) {
|
|
const auto key = MakeTxHashNumericKey<NB, POS>(rec);
|
|
const auto val = ++cols[key];
|
|
if (val > 1) {
|
|
if (val > maxCol) {
|
|
maxCol = size_t(val);
|
|
maxColVal = key;
|
|
}
|
|
++nCols;
|
|
//Log() << "Collision (" << unsigned(val) << ") for bytes: " << rec.toHex().constData() << ", hash value: " << hash;
|
|
}
|
|
}
|
|
}
|
|
Log() << "Collisions total: " << nCols << ", max col: " << maxCol << ", most common key bytes: "
|
|
<< QByteArray::fromRawData(reinterpret_cast<const char *>(&maxColVal), NB).toHex()
|
|
<< " elapsed: " << t0.secsStr(2) << " sec";
|
|
}
|
|
const auto b1 = App::registerBench("txcol", findCollisions);
|
|
} // end anon namespace
|
|
#endif
|