Fulcrum/Storage.cpp

#include "BTC.h"
#include "LRUCache.h"
#include "RecordFile.h"
#include "Storage.h"

#include "rocksdb/db.h"
#include "rocksdb/iterator.h"
#include "rocksdb/merge_operator.h"
#include "rocksdb/options.h"
#include "rocksdb/slice.h"

#include <QByteArray>
#include <QDir>

#include <atomic>
#include <cstring> // for memcpy
#include <list>
#include <optional>
#include <shared_mutex>
#include <string>
#include <tuple>
#include <type_traits>
#include <utility>
#include <vector>

DatabaseError::~DatabaseError(){} // weak vtable warning suppression
DatabaseSerializationError::~DatabaseSerializationError() {} // weak vtable warning suppression
DatabaseFormatError::~DatabaseFormatError() {} // weak vtable warning suppression
DatabaseKeyNotFound::~DatabaseKeyNotFound() {} // weak vtable warning suppression

namespace {
    /// Encapsulates the 'meta' db table
    struct Meta {
        uint32_t magic = 0xf33db33f, version = 0x1;
        QString chain; ///< "test", "main", etc
        uint16_t platformBits = sizeof(long)*8U; ///< we save the platform wordsize to the db
    };

    // some database keys we use -- todo: if this grows large, move it elsewhere
    static const rocksdb::Slice kMeta{"meta"}, kNumHeaders{"num_headers"};
    static constexpr size_t MAX_HEADERS = 100000000; // 100 mln max headers for now.

    // serialize/deser -- for basic types we use QDataStream, but we also have specializations at the end of this file
    template <typename Type>
    QByteArray Serialize(const Type & n) {
        QByteArray ba;
        if constexpr (std::is_base_of_v<QByteArray, Type>) {
            ba = n;
        } else {
            QDataStream ds(&ba, QIODevice::WriteOnly|QIODevice::Truncate);
            ds << n;
        }
        return ba;
    }
    template <typename Type>
    Type Deserialize(const QByteArray &ba, bool *ok = nullptr) {
        Type ret{};
        if constexpr (std::is_base_of_v<QByteArray, Type>) {
            ret = ba;
        } else {
            QDataStream ds(ba);
            ds >> ret;
            if (ok)
                *ok = ds.status() == QDataStream::Status::Ok;
        }
        return ret;
    }

    /// Return a shallow, temporary copy of the memory of an object as a QByteArray. This reduces typing of
    /// the boilerplate: "QByteArray::FromRawData(reinterpret_cast...." etc everywhere in this file.
    /// Note: It is unsafe to use this function for anything other than obtaining a weak reference to the memory of an
    /// object as a QByteArray for temporary purposes. The original object must live at least as long as this returned
    /// QByteArray.  Note that even copy-constructing a new QByteArray from this returned QByteArray will lead to
    /// dangling pointers. See: https://doc.qt.io/qt-5/qbytearray.html#fromRawData.
    template <typename Object,
              std::enable_if_t<!std::is_pointer_v<std::remove_cv_t<Object>>, int> = 0>
    QByteArray ShallowTmp(const Object *mem, size_t size = sizeof(Object)) {
        return QByteArray::fromRawData(reinterpret_cast<const char *>(mem), int(size));
    }

    /// Construct a QByteArray from a deep copy of any object's memory area. Slower than ShallowTmp above but 100% safe
    /// to use after the original object expires since the returned QByteArray takes ownership of its private copy of
    /// the memory it allocated.
    template <typename Object,
              std::enable_if_t<!std::is_pointer_v<std::remove_cv_t<Object>>, int> = 0>
    QByteArray DeepCpy(const Object *mem, size_t size = sizeof(Object)) {
        return QByteArray(reinterpret_cast<const char *>(mem), int(size));
    }

    /// Serialize a simple value such as an int directly, without using the space overhead that QDataStream imposes.
    /// This is less safe but is more compact since the bytes of the passed-in value are written directly to the
    /// returned QByteArray, without any encapsulation.  Note that use of this mechanism makes all data in the database
    /// no longer platform-neutral, which is ok. The presumption is users can re-synch their DB if switching
    /// architectures.
    template <typename Scalar,
              std::enable_if_t<std::is_scalar_v<Scalar> && !std::is_pointer_v<Scalar>, int> = 0>
    [[maybe_unused]] QByteArray SerializeScalar (const Scalar & s) { return DeepCpy(&s); }
    template <typename Scalar,
              std::enable_if_t<std::is_scalar_v<Scalar> && !std::is_pointer_v<Scalar>, int> = 0>
    QByteArray SerializeScalarNoCopy (const Scalar &s) { return ShallowTmp(&s);  }
    /// Inverse of above.  Pass in an optional 'pos' pointer if you wish to continue reading raw scalars from the same
    /// QByteArray during subsequent calls to this template function.  *ok, if specified, is set to false if we ran off
    /// the QByteArray's bounds, and a default-constructed value of 'Scalar' is returned.  No other safety checking is
    /// done.  On successful deserialization of the scalar, *pos (if specified) is updated to point just past the
    /// last byte of the successuflly converted item.  On failure, *pos is always set to point past the end of the
    /// QByteArray.
    template <typename Scalar,
              std::enable_if_t<std::is_scalar_v<Scalar> && !std::is_pointer_v<Scalar>, int> = 0>
    Scalar DeserializeScalar(const QByteArray &ba, bool *ok = nullptr, int *pos_out = nullptr) {
        Scalar ret{};
        int dummy = 0;
        int & pos = pos_out ? *pos_out : dummy;
        if (pos >= 0 && pos + int(sizeof(ret)) <= ba.size()) {
            if (ok) *ok = true;
            ret = *reinterpret_cast<const Scalar *>(ba.data() + pos);
            pos += sizeof(ret);
        } else {
            if (ok) *ok = false;
            pos = ba.size();
        }
        return ret;
    }

    // specializations
    template <> QByteArray Serialize(const Meta &);
    template <> Meta Deserialize(const QByteArray &, bool *);
    template <> QByteArray Serialize(const TXO &);
    template <> TXO Deserialize(const QByteArray &, bool *);
    template <> QByteArray Serialize(const TXOInfo &);
    template <> TXOInfo Deserialize(const QByteArray &, bool *);
    // TxNumVec
    using TxNumVec = std::vector<TxNum>;
    // this serializes a vector of TxNums to a compact representation (6 bytes, eg 48 bits per TxNum), in little endian byte order
    template <> QByteArray Serialize(const TxNumVec &);
    // this deserializes a vector of TxNums from a compact representation (6 bytes, eg 48 bits per TxNum), assuming little endian byte order
    template <> TxNumVec Deserialize(const QByteArray &, bool *);

    // CompactTXO -- not currently used since we prefer toBytes() directly (TODO: remove if we end up never using this)
    template <> QByteArray Serialize(const CompactTXO &);
    template <> CompactTXO Deserialize(const QByteArray &, bool *);

    /// NOTE: The slice should live as long as the returned QByteArray does.  The QByteArray is a weak pointer into the slice!
    inline QByteArray FromSlice(const rocksdb::Slice &s) { return ShallowTmp(s.data(), s.size()); }

    /// Generic conversion from any type we operate on to a rocksdb::Slice. Note that the type in question should have
    /// a conversion function written (eg Serialize) if it is anything other than a QByteArray or a scalar.
    template<bool safeScalar=false, typename Thing>
    auto ToSlice(const Thing &thing) {
        if constexpr (std::is_base_of_v<rocksdb::Slice, Thing>) {
            // same type, no-op, return ref to thing (const Slice &)
            return static_cast<const rocksdb::Slice &>(thing);
        } else if constexpr (std::is_base_of_v<QByteArray, Thing>) {
            // QByteArray conversion, return reference to data in QByteArray
            return rocksdb::Slice(thing.constData(), size_t(thing.size()));
        } else if constexpr (!safeScalar && std::is_scalar_v<Thing> && !std::is_pointer_v<Thing>) {
            return rocksdb::Slice(reinterpret_cast<const char *>(&thing), sizeof(thing)); // returned slice points to raw scalar memory itself
        } else {
            // the purpose of this holder is to keep the temporary QByteArray alive for as long as the slice itself is alive
            struct BagOfHolding {
                QByteArray bytes;
                rocksdb::Slice slice;
                operator const rocksdb::Slice &() const { return slice; }
            } h { Serialize(thing), ToSlice(h.bytes) };
            return h; // this holder type "acts like" a Slice due to its operator const Slice &()
        }
    };

    /// DB read/write helpers
    /// NOTE: these may throw DatabaseError
    /// If missingOk=false, then the returned optional is guaranteed to have a value if this function returns without throwing.
    /// If missingOk=true, then if there was no other database error and the key was not found, the returned optional !has_value()
    ///
    /// Template arg "safeScalar", if true, will deserialize scalar int, float, etc data using the Deserialize<>
    /// function (uses QDataStream, is platform neutral, but is slightly slower).  If false, we will use the
    /// DeserializeScalar<> fast function for scalars such as ints. It's important to read from the DB in the same
    /// 'safeScalar' mode as was written!
    template <typename RetType, bool safeScalar = false, typename KeyType>
    std::optional<RetType> GenericDBGet(rocksdb::DB *db, const KeyType & keyIn, bool missingOk = false,
                                        const QString & errorMsgPrefix = QString(),  ///< used to specify a custom error message in the thrown exception
                                        bool acceptExtraBytesAtEndOfData = false,
                                        const rocksdb::ReadOptions & ropts = rocksdb::ReadOptions()) ///< if true, we are ok with extra unparsed bytes in data. otherwise we throw. (this check is only done for !safeScalar mode on basic types)
    {
        rocksdb::PinnableSlice datum;
        std::optional<RetType> ret;
        if (UNLIKELY(!db)) throw InternalError("GenericDBGet was passed a null pointer!");
        const auto status = db->Get(ropts, db->DefaultColumnFamily(), ToSlice<safeScalar>(keyIn), &datum);
        if (status.IsNotFound()) {
            if (missingOk)
                return ret; // optional will not has_value() to indicate missing key
            throw DatabaseKeyNotFound(QString("%1: %2")
                                      .arg(!errorMsgPrefix.isEmpty() ? errorMsgPrefix : "Key not found in db")
                                      .arg(QString::fromStdString(status.ToString())));
        } else if (!status.ok()) {
            throw DatabaseError(QString("%1: %2")
                                .arg(!errorMsgPrefix.isEmpty() ? errorMsgPrefix : "Error reading a key from the db")
                                .arg(QString::fromStdString(status.ToString())));
        } else {
            // ok status
            if constexpr (std::is_base_of_v<QByteArray, std::remove_cv_t<RetType> >) {
                static_assert (!safeScalar, "safeScalar=true mode is not supported for QByteArrays (it only is useful for scalar types)" );
                // special compile-time case for QByteArray subclasses -- return a deep copy of the data bytes directly.
                // TODO: figure out a way to do this without the 1 extra copy! (PinnableSlice -> ret).
                ret.emplace( reinterpret_cast<const char *>(datum.data()), QByteArray::size_type(datum.size()) );
            } else if constexpr (!safeScalar && std::is_scalar_v<RetType> && !std::is_pointer_v<RetType>) {
                if (!acceptExtraBytesAtEndOfData && datum.size() > sizeof(RetType)) {
                    // reject extra stuff at end of data stream
                    throw DatabaseFormatError(QString("%1: Extra bytes at the end of data")
                                              .arg(!errorMsgPrefix.isEmpty() ? errorMsgPrefix : "Database format error"));
                }
                bool ok;
                ret.emplace( DeserializeScalar<RetType>(FromSlice(datum), &ok) );
                if (!ok) {
                    throw DatabaseSerializationError(
                                QString("%1: Key was retrieved ok, but data could not be deserialized as a scalar '%2'")
                                .arg(!errorMsgPrefix.isEmpty() ? errorMsgPrefix : "Error deserializing a scalar from db")
                                .arg(typeid (RetType).name()));
                }
            } else {
                if (UNLIKELY(acceptExtraBytesAtEndOfData))
                    Debug() << "Warning:  Caller misuse of function '" << __FUNCTION__
                            << "'. 'acceptExtraBytesAtEndOfData=true' is ignored when deserializing using QDataStream.";
                bool ok;
                ret.emplace( Deserialize<RetType>(FromSlice(datum), &ok) );
                if (!ok) {
                    throw DatabaseSerializationError(
                                QString("%1: Key was retrieved ok, but data could not be deserialized")
                                .arg(!errorMsgPrefix.isEmpty() ? errorMsgPrefix : "Error deserializing an object from db"));
                }
            }
        }
        return ret;
    }

    /// Conveneience for above with the missingOk flag set to false. Will always throw or return a real value.
    template <typename RetType, bool safeScalar = false, typename KeyType>
    RetType GenericDBGetFailIfMissing(rocksdb::DB * db, const KeyType &k, const QString &errMsgPrefix = QString(), bool extraDataOk = false,
                                      const rocksdb::ReadOptions & ropts = rocksdb::ReadOptions())
    {
        return GenericDBGet<RetType, safeScalar>(db, k, false, errMsgPrefix, extraDataOk, ropts).value();
    }

    /// Throws on all errors. Otherwise writes to db.
    template <bool safeScalar = false, typename KeyType, typename ValueType>
    void GenericDBPut
                (rocksdb::DB *db, const KeyType & key, const ValueType & value,
                 const QString & errorMsgPrefix = QString(),  ///< used to specify a custom error message in the thrown exception
                 const rocksdb::WriteOptions & opts = rocksdb::WriteOptions())
    {
        auto st = db->Put(opts, ToSlice<safeScalar>(key), ToSlice<safeScalar>(value));
        if (!st.ok())
            throw DatabaseError(QString("%1: %2")
                                .arg(!errorMsgPrefix.isEmpty() ? errorMsgPrefix : "Error writing object to db")
                                .arg(QString::fromStdString(st.ToString())));
    }
    /// Throws on all errors. Otherwise deletes a key from db. It is not an error to delete a non-existing key.
    template <bool safeScalar = false, typename KeyType>
    void GenericDBDelete
                (rocksdb::DB *db, const KeyType & key,
                 const QString & errorMsgPrefix = QString(),  ///< used to specify a custom error message in the thrown exception
                 const rocksdb::WriteOptions & opts = rocksdb::WriteOptions())
    {
        auto st = db->Delete(opts, ToSlice<safeScalar>(key));
        if (!st.ok())
            throw DatabaseError(QString("%1: %2")
                                .arg(!errorMsgPrefix.isEmpty() ? errorMsgPrefix : "Error deleting a key from db")
                                .arg(QString::fromStdString(st.ToString())));
    }

    //// A helper data structs -- written to the blkinfo table. This helps localize a txnum to a specific position in
    /// a block.  The table is keyed off of block_height(uint32_t) -> serialized BlkInfo (raw bytes)
    struct BlkInfo {
        TxNum txNum0 = 0;
        unsigned nTx = 0;
        BlkInfo() = default;
        BlkInfo(const BlkInfo &) = default;
        [[maybe_unused]] BlkInfo (TxNum txn, unsigned ntx) : txNum0(txn), nTx(ntx) {}
        [[maybe_unused]] bool operator==(const BlkInfo &o) const { return txNum0 == o.txNum0 && nTx == o.nTx; }
        [[maybe_unused]] bool operator<(const BlkInfo &o) const { return txNum0 == o.txNum0 ? nTx < o.nTx : txNum0 < o.txNum0; }
        BlkInfo &operator=(const BlkInfo &) = default;
    };
    // serializes as raw bytes from struct
    template <> QByteArray Serialize(const BlkInfo &);
    // deserializes as raw bytes from struct
    template <> BlkInfo Deserialize(const QByteArray &, bool *);

    /// Block rewind/undo information. One of these is kept around in the db for the last 10 blocks.
    /// It basically stores a record of all the UTXO's added and removed, as well as the set of
    /// scripthashes.
    struct UndoInfo {
        using ScriptHashSet = std::unordered_set<HashX, HashHasher>;
        using UTXOAddUndo = std::tuple<TXO, HashX, CompactTXO>;
        using UTXODelUndo = std::tuple<TXO, TXOInfo>;

        BlockHeight height = 0; ///< we save a copy of this infomation as a sanity check
        BlockHash hash; ///< we save a copy of this information as a sanity check. (bytes are in "reversed", bitcoind ToHex()-style memory order)
        BlkInfo blkInfo; ///< we save a copy of this from the global value for convenience and as a sanity check.

        // below is the actual critical undo information
        ScriptHashSet scriptHashes;
        std::vector<UTXOAddUndo> addUndos;
        std::vector<UTXODelUndo> delUndos;

        [[maybe_unused]] QString toDebugString() const;

        [[maybe_unused]] bool operator==(const UndoInfo &) const; // for debug ser/deser

        bool isValid() const { return hash.size() == HashLen; } ///< cheap, imperfect check for validity
        void clear() { height = 0; hash.clear(); blkInfo = BlkInfo(); scriptHashes.clear(); addUndos.clear(); delUndos.clear(); }
    };

    QString UndoInfo::toDebugString() const {
        QString ret;
        QTextStream ts(&ret);
        ts  << "<Undo info for height: " << height << " addUndos: " << addUndos.size() << " delUndos: " << delUndos.size()
            << " scriptHashes: " << scriptHashes.size() << " nTx: " << blkInfo.nTx << " txNum0: " << blkInfo.txNum0
            << " hash: " << hash.toHex() << ">";
        return ret;
    }

    bool UndoInfo::operator==(const UndoInfo &o) const {
        return height == o.height && hash == o.hash && blkInfo == o.blkInfo && scriptHashes == o.scriptHashes
                && addUndos == o.addUndos && delUndos == o.delUndos;
    }

    // serialize as raw bytes mostly (no QDataStream)
    template <> QByteArray Serialize(const UndoInfo &);
    // serialize from raw bytes mostly (no QDataStream)
    template <> UndoInfo Deserialize(const QByteArray &, bool *);


    /// Associative merge operator used for scripthash history concatenation
    /// TODO: this needs to be made more efficient by implementing the real MergeOperator interface and combining
    /// appends efficiently to reduce allocations.  Right now it's called for each append.
    class ConcatOperator : public rocksdb::AssociativeMergeOperator {
    public:
        ~ConcatOperator() override;

        mutable std::atomic<unsigned> merges = 0;

        // Gives the client a way to express the read -> modify -> write semantics
        // key:           (IN) The key that's associated with this merge operation.
        // existing_value:(IN) null indicates the key does not exist before this op
        // value:         (IN) the value to update/merge the existing_value with
        // new_value:    (OUT) Client is responsible for filling the merge result
        // here. The string that new_value is pointing to will be empty.
        // logger:        (IN) Client could use this to log errors during merge.
        //
        // Return true on success.
        // All values passed in will be client-specific values. So if this method
        // returns false, it is because client specified bad data or there was
        // internal corruption. The client should assume that this will be treated
        // as an error by the library.
        bool Merge(const rocksdb::Slice& key, const rocksdb::Slice* existing_value,
                   const rocksdb::Slice& value, std::string* new_value,
                   rocksdb::Logger* logger) const override;
        const char* Name() const override { return "ConcatOperator"; /* NOTE: this must be the same for the same db each time it is opened! */ }
    };

    ConcatOperator::~ConcatOperator() {} // weak vtable warning prevention

    bool ConcatOperator::Merge(const rocksdb::Slice& key, const rocksdb::Slice* existing_value,
                               const rocksdb::Slice& value, std::string* new_value, rocksdb::Logger* logger) const
    {
        (void)key; (void)logger;
        ++merges;
        new_value->resize( (existing_value ? existing_value->size() : 0) + value.size() );
        char *cur = new_value->data();
        if (existing_value) {
            std::memcpy(cur, existing_value->data(), existing_value->size());
            cur += existing_value->size();
        }
        std::memcpy(cur, value.data(), value.size());
        return true;
    }

}


struct Storage::Pvt
{
    const int blockHeaderSize = BTC::GetBlockHeaderSize();

    Meta meta;
    Lock metaLock;

    std::atomic<std::underlying_type_t<SaveItem>> pendingSaves{0};

    struct RocksDBs {
        const rocksdb::ReadOptions defReadOpts; ///< avoid creating this each time
        const rocksdb::WriteOptions defWriteOpts; ///< avoid creating this each time

        rocksdb::Options opts, shistOpts;

        std::shared_ptr<ConcatOperator> concatOperator;

        std::unique_ptr<rocksdb::DB> meta, headers, blkinfo, utxoset,
                                     shist, shunspent, // scripthash_history and scripthash_unspent
                                     undo; // undo (reorg rewind)
    } db;

    std::unique_ptr<RecordFile> txNumsFile;

    BTC::HeaderVerifier headerVerifier;
    mutable RWLock headerVerifierLock;

    std::atomic<TxNum> txNumNext{0};

    std::vector<BlkInfo> blkInfos;
    std::map<TxNum, unsigned> blkInfosByTxNum; ///< ordered map of TxNum0 for a block -> index into above blkInfo array
    RWLock blkInfoLock; ///< locks blkInfos and blkInfosByTxNum

    std::atomic<int64_t> utxoCt = 0;

    /// TODO: Tune the cahce size. With the below settings it is approx. 50MB-60MB when totally filled
    /// (each entry is ~40 bytes + overhead).  This cache is anticipated to see heavy use for get_history, so we may
    /// wish to make it larger.
    static constexpr size_t nCacheMax = 1000000, nCacheElasticity = 250000;
    LRU::Cache<true, TxNum, TxHash> lruNum2Hash{nCacheMax, nCacheElasticity};
};

Storage::Storage(const std::shared_ptr<Options> & options)
    : Mgr(nullptr), options(options), p(new Pvt)
{
    setObjectName("Storage");
    _thread.setObjectName(objectName());
}

Storage::~Storage() { Debug("%s", __FUNCTION__); cleanup(); }

void Storage::startup()
{
    {   // open all db's ...

        rocksdb::Options & opts(p->db.opts), &shistOpts(p->db.shistOpts);
        // Optimize RocksDB. This is the easiest way to get RocksDB to perform well
        opts.IncreaseParallelism(int(Util::getNPhysicalProcessors()));
        opts.OptimizeLevelStyleCompaction();
        // create the DB if it's not already present
        opts.create_if_missing = true;
        opts.error_if_exists = false;
        //opts.max_open_files = 50; ///< testing -- seems this affects memory usage see: https://github.com/facebook/rocksdb/issues/4112
        opts.keep_log_file_num = 5; // ??
        opts.compression = rocksdb::CompressionType::kNoCompression; // for now we test without compression. TODO: characterize what is fastest and best..
        shistOpts = opts; // copy what we just did
        shistOpts.merge_operator = p->db.concatOperator = std::make_shared<ConcatOperator>(); // this set of options uses the concat merge operator (we use this to append to history entries in the db)

        using DBInfoTup = std::tuple<QString, std::unique_ptr<rocksdb::DB> &, const rocksdb::Options &>;
        const std::list<DBInfoTup> dbs2open = {
            { "meta", p->db.meta, opts },
            { "headers", p->db.headers, opts },
            { "blkinfo" , p->db.blkinfo , opts },
            { "utxoset", p->db.utxoset, opts },
            { "scripthash_history", p->db.shist, shistOpts },
            { "scripthash_unspent", p->db.shunspent, opts },
            { "undo", p->db.undo, opts },
        };
        const auto OpenDB = [this](const DBInfoTup &tup) {
            auto & [name, uptr, opts] = tup;
            rocksdb::DB *db = nullptr;
            rocksdb::Status s;
            // try and open database
            const QString path = options->datadir + QDir::separator() + name;
            s = rocksdb::DB::Open( opts, path.toStdString(), &db);
            if (!s.ok() || !db)
                throw DatabaseError(QString("Error opening %1 database: %2 (path: %3)")
                                    .arg(name).arg(QString::fromStdString(s.ToString())).arg(path));
            uptr.reset(db);
        };

        // open all db's defined above
        for (auto & tup : dbs2open)
            OpenDB(tup);

    }  // /open db's

    // load/check meta
    {
        Meta m_db;
        static const QString errMsg{"Incompatible database format -- delete the datadir and resynch. RocksDB error"};
        if (auto opt = GenericDBGet<Meta>(p->db.meta.get(), kMeta, true, errMsg);
                opt.has_value())
        {
            m_db = opt.value();
            if (m_db.magic != p->meta.magic || m_db.version != p->meta.version || m_db.platformBits != p->meta.platformBits) {
                throw DatabaseFormatError(errMsg);
            }
            p->meta = m_db;
            Debug () << "Read meta from db ok";
        } else {
            // ok, did not exist .. write a new one to db
            saveMeta_impl();
        }
    }

    Log() << "Loading database ...";
    // load headers -- may throw
    loadCheckHeadersInDB();
    // check txnums
    loadCheckTxNumsFileAndBlkInfo();
    // count utxos -- note this depends on "blkInfos" being filled in so it much be called after loadCheckTxNumsFileAndBlkInfo()
    loadCheckUTXOsInDB();

    start(); // starts our thread
}

void Storage::cleanup()
{
    stop(); // joins our thread

    // TODO: unsaved/"dirty state" detection here -- and forced save, if needed.
}


auto Storage::stats() const -> Stats
{
    // TODO ...
    QVariantMap ret;
    auto & c = p->db.concatOperator;
    ret["merge calls"] = c ? c->merges.load() : QVariant();
    return ret;
}

// Keep returned LockGuard in scope while you use the HeaderVerifier
auto Storage::headerVerifier() -> std::pair<BTC::HeaderVerifier &, ExclusiveLockGuard>
{
    return std::pair<BTC::HeaderVerifier &, ExclusiveLockGuard>( p->headerVerifier, p->headerVerifierLock );
}
auto Storage::headerVerifier() const -> std::pair<const BTC::HeaderVerifier &, SharedLockGuard>
{
    return std::pair<const BTC::HeaderVerifier &, SharedLockGuard>( p->headerVerifier, p->headerVerifierLock );
}



QString Storage::getChain() const
{
    LockGuard l(p->metaLock);
    return p->meta.chain;
}

void Storage::setChain(const QString &chain)
{
    {
        LockGuard l(p->metaLock);
        p->meta.chain = chain;
    }
    save(SaveItem::Meta);
}

/// returns the "next" TxNum
TxNum Storage::getTxNum() const { return p->txNumNext.load(); }

auto Storage::latestTip() const -> std::pair<int, HeaderHash> {
    std::pair<int, HeaderHash> ret = headerVerifier().first.lastHeaderProcessed(); // ok; lock stays locked until statement end.

    if (ret.second.isEmpty() || ret.first < 0) {
        ret.first = -1;
        ret.second.clear();
    } else {
        // .ret now has the actual header but we want the hash
        ret.second = BTC::HashRev(ret.second);
    }
    return ret;
}

void Storage::save(SaveSpec typed_spec)
{
    using IntType = decltype(p->pendingSaves.load());
    // enqueue save on event loop if not previously enqueued (we know it was previously enqueued if the p->pendingSaves
    // atomic variable is not 0).
    if (const auto spec = IntType(typed_spec); ! p->pendingSaves.fetch_or(spec))
    {
        QTimer::singleShot(0, this, [this]{save_impl();});
    }
}

void Storage::save_impl(SaveSpec override)
{
    if (const auto flags = SaveSpec(p->pendingSaves.exchange(0))|override; flags) { // atomic clear of flags, grab prev val
        try {
            if (flags & SaveItem::Meta) { // Meta
                LockGuard l(p->metaLock);
                saveMeta_impl();
            }
        } catch (const std::exception & e) {
            Fatal() << e.what(); // will abort app...
        }
    }
}

void Storage::saveMeta_impl()
{
    if (!p->db.meta) return;
    if (auto status = p->db.meta->Put(p->db.defWriteOpts, kMeta, ToSlice(Serialize(p->meta))); !status.ok()) {
        throw DatabaseError("Failed to write meta to db");
    }

    Debug() << "Wrote new metadata to db";
}

void Storage::appendHeader(const Header &h, unsigned int height)
{
    const auto targetHeight = GenericDBGet<uint32_t, true>(p->db.headers.get(), kNumHeaders, true, // missing ok
                                                           "Error reading header count from database").value_or(0);
    if (UNLIKELY(height != targetHeight))
        throw InternalError(QString("Bad use of appendHeader -- expected height %1, got height %2").arg(targetHeight).arg(height));
    rocksdb::WriteBatch batch;
    if (auto stat = batch.Put(ToSlice(uint32_t(height)), ToSlice(h)); !stat.ok())
        throw DatabaseError(QString("Error writing header %1: %2").arg(height).arg(QString::fromStdString(stat.ToString())));
    if (auto stat = batch.Put(kNumHeaders, ToSlice<true>(uint32_t(height+1))); !stat.ok())
        throw DatabaseError(QString("Error writing header size key: %1").arg(QString::fromStdString(stat.ToString())));
    if (auto stat = p->db.headers->Write(p->db.defWriteOpts, &batch); !stat.ok())
        throw DatabaseError(QString("Error writing header %1: %2").arg(height).arg(QString::fromStdString(stat.ToString())));
}

auto Storage::headerForHeight(unsigned height, QString *err) -> std::optional<Header>
{
    std::optional<Header> ret;
    if (int(height) <= latestTip().first) {
        static const QString errMsg("Failed to retrieve header from db");
        try {
            ret.emplace(
                GenericDBGetFailIfMissing<QByteArray>(
                    p->db.headers.get(), uint32_t(height), errMsg, false, p->db.defReadOpts));
            if (UNLIKELY(ret.value().size() != p->blockHeaderSize)) {
                ret.reset();
                throw DatabaseSerializationError("Bad header read from db. Wrong size!"); // jumps to below catch
            }
        } catch (const std::exception &e) {
            if (err) *err = e.what();
        }
    }
    return ret;
}

/// Convenient batched alias for above. Returns a set of headers starting at height. May return < count if not
/// all headers were found. Thead safe.
auto Storage::headersFromHeight(unsigned height, unsigned count, QString *err) -> std::vector<Header>
{
    std::vector<Header> ret;
    int num = std::min(1 + latestTip().first - int(height), int(count));
    if (num > 0) {
        if (err) *err = "";
        ret.reserve(unsigned(num));
        for (int i = 0; i < num; ++i) {
            auto opt = headerForHeight(height + unsigned(i), err);
            if (!opt.has_value())
                break;
            ret.emplace_back(std::move(opt.value()));
        }
    } else if (err) *err = "No headers in the specified range";
    ret.shrink_to_fit();
    return ret;
}


void Storage::loadCheckHeadersInDB()
{
    FatalAssert(!!p->db.headers) << __FUNCTION__ << ": Headers db is not open";

    Log() << "Verifying headers ...";
    uint32_t num = 0;
    const auto t0 = Util::getTimeNS();
    {
        auto * const db = p->db.headers.get();

        num = GenericDBGet<uint32_t, true>(db, kNumHeaders, true, "Error reading header count from database").value_or(0); // missing ok
        if (num > MAX_HEADERS)
            throw DatabaseFormatError(QString("Header count (%1) in database exceeds MAX_HEADERS! This is likely due to"
                                              " a database format mistmatch. Delete the datadir and resynch it.")
                                      .arg(num));
        // verify headers: hashPrevBlock must match what we actually read from db
        if (num) {
            auto [verif, lock] = headerVerifier();
            Debug() << "Verifying " << num << " " << Util::Pluralize("header", num) << " ...";

            const QString errMsg("Error retrieving header from db");
            QString err;
            // read db
            for (uint32_t i = 0; i < num; ++i) {
                // guaranteed to return a value or throw
                const auto bytes = GenericDBGetFailIfMissing<QByteArray>(db, uint32_t(i), errMsg, false, p->db.defReadOpts);
                if (UNLIKELY(bytes.size() != p->blockHeaderSize))
                    throw DatabaseFormatError(QString("Error reading header %1, wrong size: %2").arg(i).arg(bytes.size()));
                if (!verif(bytes, &err))
                    throw DatabaseError(QString("%1. Possible databaase corruption. Delete the datadir and resynch.").arg(err));
            }
        }
    }
    if (num) {
        const auto elapsed = Util::getTimeNS();

        Debug() << "Read & verified " << num << " " << Util::Pluralize("header", num) << " from db in " << QString::number((elapsed-t0)/1e6, 'f', 3) << " msec";
    }
}

void Storage::loadCheckTxNumsFileAndBlkInfo()
{
    // may throw.
    p->txNumsFile = std::make_unique<RecordFile>(options->datadir + QDir::separator() + "txnum2txhash", HashLen, 0x000012e2);
    p->txNumNext = p->txNumsFile->numRecords();
    Debug() << "Read TxNumNext from file: " << p->txNumNext.load();
    TxNum ct = 0;
    if (const int height = latestTip().first; height >= 0)
    {
        p->blkInfos.reserve(std::min(size_t(height+1), MAX_HEADERS));
        Log() << "Checking tx counts ...";
        for (int i = 0; i <= height; ++i) {
            static const QString errMsg("Failed to read a blkInfo from db, the database may be corrupted");
            const auto blkInfo = GenericDBGetFailIfMissing<BlkInfo>(p->db.blkinfo.get(), uint32_t(i), errMsg, false, p->db.defReadOpts);
            if (blkInfo.txNum0 != ct)
                throw DatabaseFormatError(QString("BlkInfo for height %1 does not match computed txNum of %2."
                                                  "\n\nThe database may be corrupted. Delete the datadir and resynch it.\n")
                                          .arg(i).arg(ct));
            ct += blkInfo.nTx;
            p->blkInfos.emplace_back(blkInfo);
            p->blkInfosByTxNum[blkInfo.txNum0] = unsigned(p->blkInfos.size()-1);
        }
        Log() << ct << " total transactions";
    }
    if (ct != p->txNumNext) {
        throw DatabaseFormatError(QString("BlkInfo txNums do not add up to expected value of %1 != %2."
                                          "\n\nThe database may be corrupted. Delete the datadir and resynch it.\n")
                                  .arg(ct).arg(p->txNumNext.load()));
    }
}

// NOTE: this must be called *after* loadCheckTxNumsFileAndBlkInfo(), because it needs a valid p->txNumNext
void Storage::loadCheckUTXOsInDB()
{
    FatalAssert(!!p->db.utxoset) << __FUNCTION__ << ": Utxo set db is not open";

    Log() << "Verifying utxo set ...";

    const auto t0 = Util::getTimeNS();
    {
        const int currentHeight = latestTip().first;

        std::unique_ptr<rocksdb::Iterator> iter(p->db.utxoset->NewIterator(p->db.defReadOpts));
        if (!iter) throw DatabaseError("Unable to obtain an iterator to the utxo set db");
        iter->SeekToFirst();
        p->utxoCt = 0;
        for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
            // TODO: the below checks may be too slow. See about removing them and just counting the iter.
            const auto txo = Deserialize<TXO>(FromSlice(iter->key()));
            if (!txo.isValid()) {
                throw DatabaseSerializationError("Read an invalid txo from the utxo set database."
                                                 " This may be due to a database format mismatch."
                                                 "\n\nDelete the datadir and resynch to bitcoind.\n");
            }
            auto info = Deserialize<TXOInfo>(FromSlice(iter->value()));
            if (!info.isValid())
                throw DatabaseSerializationError(QString("Txo %1 has invalid metadata in the db."
                                                        " This may be due to a database format mismatch."
                                                        "\n\nDelete the datadir and resynch to bitcoind.\n")
                                                 .arg(txo.toString()));
            // uncomment this to do a deep test: TODO: Make this configurable from the CLI -- this last check is very slow.
            //const CompactTXO ctxo = CompactTXO(info.txNum, txo.prevoutN);
            //const QByteArray shuKey = info.hashX + ctxo.toBytes();
            //static const QString errPrefix("Error reading scripthash_unspent");
            if (bool fail1 = false, fail2 = false/*, fail3 = false*/;
                    (fail1 = (info.confirmedHeight.has_value() && int(info.confirmedHeight.value()) > currentHeight))
                    || (fail2 = info.txNum >= p->txNumNext)
                    /*|| (fail3 = !GenericDBGet<QByteArray>(p->db.shunspent.get(), shuKey, true, errPrefix, false, p->db.defReadOpts).has_value())*/) {
                // TODO: reorg? Inconsisent db?  FIXME
                QString msg;
                {
                    QTextStream ts(&msg);
                    ts << "Inconsistent database: txo " << txo.toString() << " at height: "
                       << info.confirmedHeight.value();
                    if (fail1) {
                        ts << " > current height: " << currentHeight << ".";
                    } else if (fail2) {
                        ts << ". TxNum: " << info.txNum << " >= " << p->txNumNext << ".";
                    } /*else if (fail3) {
                        ts << ". Failed to find ctxo " << ctxo.toString() << " in scripthash_unspent db.";
                    }*/
                    ts << "\n\nThe database has been corrupted. Please delete the datadir and resynch to bitcoind.\n";
                }
                throw DatabaseError(msg);
            }
            if (0 == ++p->utxoCt % 100000) {
                *(0 == p->utxoCt % 2500000 ? std::make_unique<Log>() : std::make_unique<Debug>()) << "Verified " << p->utxoCt << " utxos ...";
            }
        }
        const auto ct = utxoSetSize();
        if (ct)
            Log() << "UTXO set: "  << ct << Util::Pluralize(" utxo", ct)
                  << ", " << QString::number(utxoSetSizeMiB(), 'f', 3) << " MiB";
    }
    const auto elapsed = Util::getTimeNS();
    Debug() << "Read txos from db in " << QString::number((elapsed-t0)/1e6, 'f', 3) << " msec";
}


/// Thread-safe. Immediately save a UTXO to the db. May throw on database error.
void Storage::utxoAddToDB(const TXO &txo, const TXOInfo &info, const CompactTXO &ctxo)
{
    assert(bool(p->db.utxoset));
    if (txo.isValid()) {
        static const QString errMsgPrefix("Failed to add a utxo to the utxo db");
        GenericDBPut(p->db.utxoset.get(), txo, info, errMsgPrefix, p->db.defWriteOpts); // may throw on failure

        {
            // Update the scripthash unspent. This is a very simple table which we scan by hashX prefix using
            // an iterator in listUnspent.  Each entry's key is prefixed with the HashX bytes but suffixed with the
            // serialized CompactTXO bytes. Each entry has no data for the value.
            static const QString errMsgPrefix("Failed to add an entry to the scripthash_unspent db");
            static const rocksdb::Slice empty;
            const QByteArray key = info.hashX + ctxo.toBytes();

            GenericDBPut(p->db.shunspent.get(), key, empty, errMsgPrefix, p->db.defWriteOpts); // may throw, which is what we want
        }
        ++p->utxoCt;
    }
}
/// Thread-safe. Query db for a UTXO, and return it if found.  May throw on database error.
std::optional<TXOInfo> Storage::utxoGetFromDB(const TXO &txo, bool throwIfMissing)
{
    assert(bool(p->db.utxoset));
    static const QString errMsgPrefix("Failed to read a utxo from the utxo db");
    return GenericDBGet<TXOInfo>(p->db.utxoset.get(), txo, !throwIfMissing, errMsgPrefix, false, p->db.defReadOpts);
}
/// Delete a Utxo from the db. Will throw only on database error (but not if it was missing).
void Storage::utxoDeleteFromDB(const TXO &txo, const HashX &hashX, const CompactTXO &ctxo)
{
    assert(bool(p->db.utxoset));
    {
        // delete from utxoset db
        auto stat = p->db.utxoset->Delete(rocksdb::WriteOptions(), ToSlice(Serialize(txo)));
        if (!stat.ok()) {
            throw DatabaseError(QString("Failed to delete from the utxo db (%1:%2): %3")
                                .arg(QString(txo.prevoutHash.toHex())).arg(txo.prevoutN).arg(QString::fromStdString(stat.ToString())));
        }
    }
    {
        // delete from scripthash_unspent db
        const QByteArray key = hashX + ctxo.toBytes();
        if (auto stat = p->db.shunspent->Delete(p->db.defWriteOpts, ToSlice(key)); !stat.ok()) {
            throw DatabaseError(QString("Failed to delete a from the scripthash_unspent db (%1:%2): %3")
                                .arg(QString(txo.prevoutHash.toHex())).arg(txo.prevoutN).arg(QString::fromStdString(stat.ToString())));
        }
    }
    --p->utxoCt;
}

int64_t Storage::utxoSetSize() const { return p->utxoCt; }
double Storage::utxoSetSizeMiB() const {
    constexpr int64_t elemSize = TXO::serSize() + TXOInfo::serSize();
    return (utxoSetSize()*elemSize) / 1e6;
}

QString Storage::addBlock(PreProcessedBlockPtr ppb, bool saveUndo, unsigned nReserve)
{
    assert(bool(ppb) && bool(p));

    std::unique_ptr<UndoInfo> undo;

    if (saveUndo) {
        undo.reset(new UndoInfo);
        undo->height = ppb->height;
    }

    QString errRet;

    // take all locks now.. since this is a Big Deal. TODO: add more locks here?
    std::scoped_lock guard(p->headerVerifierLock, p->blkInfoLock);

    const auto verifUndo = p->headerVerifier; // keep a copy for undo purposes in case this fails

    try {
        const auto blockTxNum0 = p->txNumNext.load();

        // Verify header chain makes sense (by checking hashes, using the shared header verifier)
        QByteArray rawHeader;
        {
            QString errMsg;
            if (!p->headerVerifier(ppb->header, &errMsg) ) {
                // XXX possible reorg point. FIXME TODO
                // reorg here? TODO: deal with this better.
                throw Exception(errMsg);
            }
            // save raw header back to our buffer
            rawHeader = p->headerVerifier.lastHeaderProcessed().second;
        }

        {  // add txnum -> txhash association to the TxNumsFile...
            auto batch = p->txNumsFile->beginBatchAppend(); // may throw if io error in c'tor here.
            QString errStr;
            for (const auto & txInfo : ppb->txInfos) {
                if (!batch.append(txInfo.hash, &errStr)) // does not throw here, but we do.
                    throw InternalError(QString("Batch append for txNums failed: %1.\n\n"
                                                "Database is now likely corrupted. Please delete the datadir and resynch.\n")
                                        .arg(errStr));
            }
            // <-- The batch d'tor may close the app on error here with Fatal() if a low-level file error occurs now
            //     on header update (see: RecordFile.cpp, ~BatchAppendContext()).
        }

        p->txNumNext += ppb->txInfos.size(); // update internal counter

        if (p->txNumNext != p->txNumsFile->numRecords())
            throw InternalError("TxNum file and internal txNumNext counter disagree! FIXME!");


        constexpr bool debugPrt = false;

        // update utxoSet
        {
            // reserve space in undo, if in saveUndo mode
            if (undo) {
                undo->addUndos.reserve(ppb->outputs.size());
                undo->delUndos.reserve(ppb->inputs.size());
            }

            // add outputs

            for (const auto & [hashX, ag] : ppb->hashXAggregated) {
                for (const auto oidx : ag.outs) {
                    const auto & out = ppb->outputs[oidx];
                    if (out.spentInInputIndex.has_value()) {
                        if constexpr (debugPrt)
                            Debug() << "Skipping output #: " << oidx << " for " << ppb->txInfos[out.txIdx].hash.toHex() << " (was spent in same block tx: " << ppb->txInfos[ppb->inputs[out.spentInInputIndex.value()].txIdx].hash.toHex() << ")";
                        continue;
                    }
                    const TxHash & hash = ppb->txInfos[out.txIdx].hash;
                    TXOInfo info;
                    info.hashX = hashX;
                    info.amount = out.amount;
                    info.confirmedHeight = ppb->height;
                    info.txNum = blockTxNum0 + out.txIdx;
                    const TXO txo{ hash, out.outN };
                    const CompactTXO ctxo(blockTxNum0+out.txIdx, out.outN);
                    utxoAddToDB(txo, info, ctxo); // add to db
                    if (undo) { // save undo info if we are in saveUndo mode
                        undo->addUndos.emplace_back(txo, info.hashX, ctxo);
                    }
                    if constexpr (debugPrt)
                        Debug() << "Added txo: " << txo.toString()
                                << " (txid: " << hash.toHex() << " height: " << ppb->height << ") "
                                << " amount: " << info.amount.ToString() << " for HashX: " << info.hashX.toHex();
                }
            }

            // add spends (process inputs)
            std::unordered_set<HashX, HashHasher> newHashXInputsResolved;
            unsigned inum = 0;
            for (auto & in : ppb->inputs) {
                const TXO txo{in.prevoutHash, in.prevoutN};
                if (!inum) {
                    // coinbase.. skip
                } else if (in.parentTxOutIdx.has_value()) {
                    // was an input that was spent in this block so it's ok to skip.. we never added it to utxo set
                    if constexpr (debugPrt)
                        Debug() << "Skipping input " << txo.toString() << ", spent in this block (output # " << in.parentTxOutIdx.value() << ")";
                } else if (const auto opt = utxoGetFromDB(txo); opt.has_value()) {
                    const auto & info = opt.value();
                    if (info.confirmedHeight.has_value() && info.confirmedHeight.value() != ppb->height) {
                        // was a prevout from a previos block.. so the ppb didn't have it in the 'involving hashx' set..
                        // mark the spend as having involved this hashX for this ppb now.
                        auto & ag = ppb->hashXAggregated[info.hashX];
                        ag.ins.emplace_back(inum);
                        newHashXInputsResolved.insert(info.hashX);
                        // mark its txidx
                        if (auto & vec = ag.txNumsInvolvingHashX; vec.empty() || vec.back() != in.txIdx)
                            vec.emplace_back(in.txIdx);

                    }
                    if constexpr (debugPrt) {
                        const auto dbgTxIdHex = ppb->txHashForInputIdx(inum).toHex();
                        Debug() << "Spent " << txo.toString() << " amount: " << info.amount.ToString()
                                << " in txid: "  << dbgTxIdHex << " height: " << ppb->height
                                << " input number: " << ppb->numForInputIdx(inum).value_or(0xffff)
                                << " HashX: " << info.hashX.toHex();
                    }
                    // delete from db
                    utxoDeleteFromDB(txo, info.hashX, CompactTXO(info.txNum, txo.prevoutN)); // delete from db
                    if (undo) { // save undo info, if we are in saveUndo mode
                        undo->delUndos.emplace_back(txo, info);
                    }
                } else {
                    QString s;
                    {
                        const auto dbgTxIdHex = ppb->txHashForInputIdx(inum).toHex();
                        QTextStream ts(&s);
                        ts << "Failed to spend: " << in.prevoutHash.toHex() << ":" << in.prevoutN << " (spending txid: " << dbgTxIdHex << ")";
                    }
                    throw Exception(s);
                }
                ++inum;
            }

            // sort and shrink_to_fit new hashX inputs added
            for (const auto & hashX : newHashXInputsResolved) {
                auto & ag = ppb->hashXAggregated[hashX];
                std::sort(ag.ins.begin(), ag.ins.end()); // make sure they are sorted
                std::sort(ag.txNumsInvolvingHashX.begin(), ag.txNumsInvolvingHashX.end());
                auto last = std::unique(ag.txNumsInvolvingHashX.begin(), ag.txNumsInvolvingHashX.end());
                ag.txNumsInvolvingHashX.erase(last, ag.txNumsInvolvingHashX.end());
                ag.ins.shrink_to_fit();
                ag.txNumsInvolvingHashX.shrink_to_fit();
            }

            if constexpr (debugPrt)
                Debug() << "utxoset size: " << utxoSetSize() << " block: " << ppb->height;
        }

        {
            // now.. update the txNumsInvolvingHashX to be offset from txNum0 for this block, and save history to db table
            // history is hashX -> TxNumVec (serialized) as a serities of 6-bytes txNums in blockchain order as they appeared.
            rocksdb::WriteBatch batch;
            for (auto & [hashX, ag] : ppb->hashXAggregated) {
                for (auto & txNum : ag.txNumsInvolvingHashX) {
                    txNum += blockTxNum0; // transform local txIdx to -> txNum (global mapping)
                }
                // save scripthash history for this hashX, by appending to existing history. Note that this uses
                // the 'ConcatOperator' class we defined in this file, which requires rocksdb be compiled with RTTI.
                if (auto st = batch.Merge(ToSlice(hashX), ToSlice(Serialize(ag.txNumsInvolvingHashX))); !st.ok())
                    throw DatabaseError(QString("batch merge fail for hashX %1, block height %2: %3")
                                        .arg(QString(hashX.toHex())).arg(ppb->height).arg(QString::fromStdString(st.ToString())));
            }
            if (auto st = p->db.shist->Write(p->db.defWriteOpts, &batch) ; !st.ok())
                throw DatabaseError(QString("batch merge fail for block height %1: %2")
                                    .arg(ppb->height).arg(QString::fromStdString(st.ToString())));
        }


        {
            // update BlkInfo
            if (nReserve) {
                if (const auto size = p->blkInfos.size(); size + 1 > p->blkInfos.capacity())
                    p->blkInfos.reserve(size + nReserve); // reserve space for new blkinfos in 1 go to save on copying
            }

            p->blkInfos.emplace_back(
                blockTxNum0, // .txNum0
                unsigned(ppb->txInfos.size())
            );

            const auto & blkInfo = p->blkInfos.back();

            p->blkInfosByTxNum[blkInfo.txNum0] = unsigned(p->blkInfos.size()-1);

            // save BlkInfo to db
            static const QString blkInfoErrMsg("Error writing BlkInfo to db");
            GenericDBPut(p->db.blkinfo.get(), uint32_t(ppb->height), blkInfo, blkInfoErrMsg, p->db.defWriteOpts);

            if (undo) {
                // save blkInfo to undo information, if in saveUndo mode
                undo->blkInfo = p->blkInfos.back();
            }
        }

        // save the last of the undo info, if in saveUndo mode
        if (undo) {
            const auto t0 = Util::getTimeNS();
            undo->hash = BTC::HashRev(rawHeader);
            undo->scriptHashes = Util::keySet<decltype (undo->scriptHashes)>(ppb->hashXAggregated);
            static const QString errPrefix("Error saving undo info to undo db");

            GenericDBPut(p->db.undo.get(), uint32_t(ppb->height), *undo, errPrefix, p->db.defWriteOpts); // save undo to db

            if constexpr (debugPrt) {
                // testing undo ser/deser
                Debug() << "Undo info 1: " << undo->toDebugString();
                QByteArray ba = Serialize(*undo);
                Debug() << "Undo info 1 serSize: " << ba.length();
                bool ok;
                auto undo2 = Deserialize<UndoInfo>(ba, &ok);
                ba.fill('z'); // ensure no shallow copies of buffer exist in deserialized object. if they do below tests will fail
                FatalAssert(ok && undo2.isValid()) << "Deser of undo info failed!";
                Debug() << "Undo info 2: " << undo2.toDebugString();
                Debug() << "Undo info 1 == undo info 2: " << (*undo == undo2);
            } else {
                const auto elapsedms = (Util::getTimeNS() - t0)/1e6;
                const size_t nTx = undo->blkInfo.nTx, nSH = undo->scriptHashes.size();
                Debug() << "Saved undo for block " << undo->height << ", "
                        << nTx << " " << Util::Pluralize("transaction", nTx)
                        << " involving " << nSH << " " << Util::Pluralize("scripthash", nSH)
                        << ", in " << QString::number(elapsedms, 'f', 2) << " msec.";
            }
        }
        {
            // Unconditionally expire old undos >10 blocks ago to keep the db tidy. We need to do this for each block
            // because we do not know what old undos may be lurking in the db.
            // TODO: On startup, have the db scan and find the earliest undo and only take this branch if we know an old
            // undo exists for a certain height!
            static const QString errPrefix("Error deleting old/stale undo info from undo db");
            if (const auto expireUndoHeight = int(ppb->height) - int(BTC::maxReorgDepth); expireUndoHeight >= 0)
                GenericDBDelete(p->db.undo.get(), uint32_t(expireUndoHeight), errPrefix, p->db.defWriteOpts);
        }

        appendHeader(rawHeader, ppb->height);

    } catch (const std::exception & e) {
        errRet = e.what();
        p->headerVerifier = verifUndo; // undo header verifier state
    }

    return errRet;
}

std::optional<TxHash> Storage::hashForTxNum(TxNum n, bool throwIfMissing, bool *wasCached, bool skipCache) const
{
    std::optional<TxHash> ret;
    if (!skipCache) ret = p->lruNum2Hash.tryGet(n);
    if (ret.has_value()) {
        if (wasCached) *wasCached = true;
        return ret;
    } else if (wasCached) *wasCached = false;

    static const QString kErrMsg ("Error reading TxHash for TxNum %1: %2");
    QString errStr;
    const auto bytes = p->txNumsFile->readRecord(n, &errStr);
    if (bytes.isEmpty()) {
        errStr = kErrMsg.arg(n).arg(errStr);
        if (throwIfMissing)
            throw DatabaseError(errStr);
        Warning() << errStr;
    } else {
        ret.emplace(bytes);
    }
    if (!skipCache && ret.has_value()) {
        // save in cache
        p->lruNum2Hash.insert(n, ret.value());
    }
    return ret;
}

std::optional<unsigned> Storage::heightForTxNum(TxNum n) const
{
    SharedLockGuard g(p->blkInfoLock);
    std::optional<unsigned> ret;
    auto it = p->blkInfosByTxNum.upper_bound(n);  // O(logN) search; find the block *AFTER* n, then go backw on to find the block in range
    if (it != p->blkInfosByTxNum.begin()) {
        --it;
        const auto & bi = p->blkInfos[it->second];
        if (n >= bi.txNum0 && n < bi.txNum0+bi.nTx)
            ret = it->second;
    }
    return ret;
}


auto Storage::getHistory(const HashX & hashX) const -> History
{
    History ret;
    try {
        static const QString err("Error retrieving history for a script hash");
        auto nums_opt = GenericDBGet<TxNumVec>(p->db.shist.get(), hashX, true, err, false, p->db.defReadOpts);
        if (nums_opt.has_value()) {
            auto & nums = nums_opt.value();
            ret.reserve(nums.size());
            for (auto num : nums) {
                auto hash = hashForTxNum(num).value(); // may throw, but that indicates some database inconsistency. we catch below
                auto height = heightForTxNum(num).value(); // may throw, same deal
                ret.emplace_back(HistoryItem{hash, height});
            }
        }
    } catch (const std::exception &e) {
        Warning(Log::Magenta) << __func__ << ": " << e.what();
    }
    return ret;
}

auto Storage::listUnspent(const HashX & hashX) const -> UnspentItems
{
    UnspentItems ret;
    try {
        std::unique_ptr<rocksdb::Iterator> iter(p->db.shunspent->NewIterator(p->db.defReadOpts));
        const rocksdb::Slice prefix = ToSlice(hashX); // points to data in hashX

        // Search table for all keys that start with hashx's bytes. Note: the loop end-condition is strange.
        // See: https://github.com/facebook/rocksdb/wiki/Prefix-Seek-API-Changes#transition-to-the-new-usage
        rocksdb::Slice key;
        for (iter->Seek(prefix); iter->Valid() && (key = iter->key()).starts_with(prefix); iter->Next()) {
            if (key.size() != HashLen + CompactTXO::serSize())
                // should never happen, indicates db corruption
                throw InternalError("Key size for hashx is invalid");
            const CompactTXO ctxo = CompactTXO::fromBytes(key.data() + HashLen, CompactTXO::serSize());
            if (!ctxo.isValid())
                // should never happen, indicates db corruption
                throw InternalError("Deserialized CompactTXO is invalid");
            static const QString err("Error retrieving the utxo for an unspent item");
            auto hash = hashForTxNum(ctxo.txNum()).value(); // may throw, but that indicates some database inconsistency. we catch below
            auto height = heightForTxNum(ctxo.txNum()).value(); // may throw, same deal
            auto info = GenericDBGetFailIfMissing<TXOInfo>(p->db.utxoset.get(), TXO{ hash, ctxo.N()}, err, false, p->db.defReadOpts); // may throw -- indicates db inconsistency
            ret.emplace_back(UnspentItem{
                { hash, height }, // base HistoryItem
                ctxo.N(),  // .tx_pos
                info.amount, // .value
                info.txNum, // .txNum
            });
        }
        std::sort(ret.begin(), ret.end());
        ret.shrink_to_fit();
    } catch (const std::exception &e) {
        Warning(Log::Magenta) << __func__ << ": " << e.what();
    }
    return ret;
}

// HistoryItem & UnspentItem -- operator< and operator== -- for sort.
bool Storage::HistoryItem::operator<(const HistoryItem &o) const noexcept {
    return height == o.height ? hash < o.hash : height < o.height;
}
bool Storage::HistoryItem::operator==(const HistoryItem &o) const noexcept {
    return height == o.height && hash == o.hash;
}
bool Storage::UnspentItem::operator<(const UnspentItem &o) const noexcept {
    if (txNum == o.txNum) { // order by txNum
        if (tx_pos == o.tx_pos) { // then by tx_pos
            // next by tx_hash, height (this branch shouldn't normally be reached with real blockchain data since
            // txNum:tx_pos defines an UnspentItem completed...
            if (HistoryItem::operator<(o))
                return true;
            else if (HistoryItem::operator==(o))
                return value < o.value;
            return false;
        }
        return tx_pos < o.tx_pos;
    }
    return txNum < o.txNum;
}
bool Storage::UnspentItem::operator==(const UnspentItem &o) const noexcept {
    return txNum == o.txNum && tx_pos == o.tx_pos && value == o.value && HistoryItem::operator==(o);
}

namespace {
    // specializations of Serialize/Deserialize
    template <> QByteArray Serialize(const Meta &m)
    {
        QByteArray ba;
        {
            QDataStream ds(&ba, QIODevice::WriteOnly|QIODevice::Truncate);
            // we serialize the 'magic' value as a simple scalar as a sort of endian check for the DB
            ds << SerializeScalarNoCopy(m.magic) << m.version << m.chain << m.platformBits;
        }
        return ba;
    }
    template <> Meta Deserialize(const QByteArray &ba, bool *ok_ptr)
    {
        bool dummy;
        bool &ok (ok_ptr ? *ok_ptr : dummy);
        ok = false;
        Meta m{0, 0, {}};
        {
            QDataStream ds(ba);
            QByteArray magicBytes;
            ds >> magicBytes; // read magic as raw bytes.
            if ((ok = ds.status() == QDataStream::Status::Ok)) {
                m.magic = DeserializeScalar<decltype (m.magic)>(magicBytes, &ok);
                if (ok) {
                    ds >> m.version >> m.chain;
                    if (!ds.atEnd()) {
                        // TODO: make this field non-optional. For now we tolerate it missing since we added this field
                        // later and we want to be able to still test on our existing db's.
                        ds >> m.platformBits;
                    }
                    ok = ds.status() == QDataStream::Status::Ok;
                }
            }
        }
        return m;
    }

    template <> QByteArray Serialize (const TXO &txo) { return txo.toBytes(); }
    template <> TXO Deserialize(const QByteArray &ba, bool *ok) {
        TXO ret = TXO::fromBytes(ba); // requires exact size, fails if extra bytes at the end
        if (ok) *ok = ret.isValid();
        return ret;
    }

    template <> QByteArray Serialize(const TXOInfo &inf) { return inf.toBytes(); }
    template <> TXOInfo Deserialize(const QByteArray &ba, bool *ok)
    {
        TXOInfo ret = TXOInfo::fromBytes(ba); // will fail if extra bytes at the end
        if (ok) *ok = ret.isValid();
        return ret;
    }

    // deep copy, raw bytes
    template <> QByteArray Serialize(const BlkInfo &b) { return DeepCpy(&b); }
    // will fail if extra bytes at the end
    template <> BlkInfo Deserialize(const QByteArray &ba, bool *ok) {
        BlkInfo ret;
        if (ba.length() != sizeof(ret)) {
            if (ok) *ok = false;
        } else {
            if (ok) *ok = true;
            ret = *reinterpret_cast<const BlkInfo *>(ba.constData());
        }
        return ret;
    }

    struct UndoInfoSerHeader {
        static constexpr uint16_t defMagic = 0xf12c, defVer = 0x1;
        uint16_t magic = defMagic; ///< sanity check
        uint16_t ver = defVer; ///< sanity check
        uint32_t len = 0; ///< the length of the entire buffer, including this struct and all data to follow. A sanity check.
        uint32_t nScriptHashes = 0, nAddUndos = 0, nDelUndos = 0; ///< the number of elements in each of the 3 arrays in question.

        static constexpr size_t addUndoItemSerSize = TXO::serSize() + HashLen + CompactTXO::serSize();
        static constexpr size_t delUndoItemSerSize = TXO::serSize() + TXOInfo::serSize();

        /// computes the total size given the ser size of the blkInfo struct. Requires that nScriptHashes, nAddUndos, and nDelUndos be already filled-in.
        size_t computeTotalSize() const {
            const auto shSize = nScriptHashes * HashLen;
            const auto addsSize = nAddUndos * addUndoItemSerSize;
            const auto delsSize = nDelUndos * delUndoItemSerSize;
            return sizeof(*this) + sizeof(UndoInfo::height) + HashLen + sizeof(BlkInfo) + shSize + addsSize + delsSize;
        }
        bool isLenSane() const { return size_t(len) == computeTotalSize(); }
    };

    // UndoInfo
    template <> QByteArray Serialize(const UndoInfo &u) {
        UndoInfoSerHeader hdr;
        // fill these in now so that hdr.computeTotalSize works
        hdr.nScriptHashes = uint32_t(u.scriptHashes.size());
        hdr.nAddUndos = uint32_t(u.addUndos.size());
        hdr.nDelUndos = uint32_t(u.delUndos.size());
        hdr.len = uint32_t(hdr.computeTotalSize());
        QByteArray ret;
        ret.reserve(int(hdr.len));
        // 1. header
        ret.append(ShallowTmp(&hdr));
        // 2. .height
        ret.append(SerializeScalarNoCopy(u.height));
        // 3. .hash
        const auto chkHashLen = [&ret](const QByteArray & hash) -> bool {
            if (hash.length() != HashLen) {
                Warning() << "hash is not " << HashLen << " bytes. Serialize UndoInfo fail. FIXME!";
                ret.clear();
                return false;
            }
            return true;
        };
        if (!chkHashLen(u.hash)) return ret;
        ret.append(u.hash);
        // 4. .blkInfo
        const QByteArray blkInfoBytes = Serialize(u.blkInfo);
        ret.append(blkInfoBytes);
        // 5. .scriptHashes, 32 bytes each, for all in set
        for (const auto & sh : u.scriptHashes) {
            if (UNLIKELY(!chkHashLen(sh))) return ret;
            ret.append(sh);
        }
        // 6. .addUndos, 64 bytes each * nAddUndos
        for (const auto & [txo, hashX, ctxo] : u.addUndos) {
            if (UNLIKELY(!chkHashLen(hashX))) return ret;
            ret.append(Serialize(txo));
            ret.append(hashX);
            ret.append(Serialize(ctxo));
        }
        // 7. .delUndos, 50 bytes each * nDelUndos
        for (const auto & [txo, txoInfo] : u.delUndos) {
            ret.append(Serialize(txo));
            if (UNLIKELY(!chkHashLen(txoInfo.hashX))) return ret;
            ret.append(Serialize(txoInfo));
        }
        assert(ret.length() == int(hdr.len));
        return ret;
    }

    // UndoInfo -- note this will fail if the byte array has extra bytes at the end
    template <> UndoInfo Deserialize(const QByteArray &ba, bool *ok) {
        UndoInfo ret;
        const auto setOk = [&ok, &ret] (bool b) { if (ok) *ok = b; if (!b) ret.clear(); };
        const auto chkAssertion = [&setOk] (bool assertion, const char *extra = "") {
            if (UNLIKELY(!assertion)) {
                Warning() << "Deserialize UndoInfo called with an invalid byte array! FIXME! " << extra;
                setOk(false);
            }
            return assertion;
        };
        if (!chkAssertion(ba.size() > int(sizeof(UndoInfoSerHeader)), "Short byte count"))
            return ret;

        // 1. .header
        const UndoInfoSerHeader *hdr = reinterpret_cast<decltype (hdr)>(ba.data());
        if (!chkAssertion(int(hdr->len) == ba.size() && hdr->magic == hdr->defMagic && hdr->ver == hdr->defVer
                          && hdr->isLenSane(), "Header sanity check fail"))
            return ret;

        const char *cur = ba.data() + sizeof(*hdr), *const end = ba.data() + ba.length();
        bool myok = false;
        // 2. .height
        ret.height = DeserializeScalar<decltype(ret.height)>(ShallowTmp(cur, sizeof(ret.height)), &myok);
        if (!chkAssertion(myok && cur < end))
            return ret;
        cur += sizeof(ret.height);
        // 3. .hash
        ret.hash = DeepCpy(cur, HashLen); // deep copy
        if (!chkAssertion(ret.hash.length() == HashLen && cur < end))
            return ret;
        cur += HashLen;
        // 4. .blkInfo
        ret.blkInfo = Deserialize<BlkInfo>(ShallowTmp(cur, sizeof(BlkInfo)), &myok);
        if (!chkAssertion(myok && cur <= end))
            return ret;
        cur += sizeof(BlkInfo);
        // 5. .scriptHashes, 32 bytes each * hdr->nScriptHashes
        ret.scriptHashes.reserve(hdr->nScriptHashes);
        for (unsigned i = 0; i < hdr->nScriptHashes; ++i) {
            if (!chkAssertion(cur+HashLen <= end)) return ret;
            ret.scriptHashes.insert(DeepCpy(cur, HashLen)); // deep copy
            cur += HashLen;
        }
        // 6. .addUndos, 64 bytes each * nAddUndos
        ret.addUndos.reserve(hdr->nAddUndos);
        for (unsigned i = 0; i < hdr->nAddUndos; ++i) {
            if (!chkAssertion(cur+TXO::serSize() <= end)) return ret;
            TXO txo = Deserialize<TXO>(ShallowTmp(cur, TXO::serSize()), &myok);
            cur += TXO::serSize();
            if (!chkAssertion(myok && cur+HashLen <= end)) return ret;
            QByteArray hashX = DeepCpy(cur, HashLen); // deep copy
            cur += HashLen;
            if (!chkAssertion(cur+CompactTXO::serSize() <= end)) return ret;
            CompactTXO ctxo = Deserialize<CompactTXO>(ShallowTmp(cur, CompactTXO::serSize()), &myok);
            cur += CompactTXO::serSize();
            if (!chkAssertion(myok)) return ret;
            ret.addUndos.emplace_back(std::move(txo), std::move(hashX), std::move(ctxo));
        }
        // 7. .delUndos, 50 bytes each * nDelUndos
        ret.delUndos.reserve(hdr->nDelUndos);
        for (unsigned i = 0; i < hdr->nDelUndos; ++i) {
            if (!chkAssertion(cur+TXO::serSize() <= end)) return ret;
            TXO txo = Deserialize<TXO>(ShallowTmp(cur, TXO::serSize()), &myok);
            cur += TXO::serSize();
            if (!chkAssertion(myok && cur+TXOInfo::serSize() <= end)) return ret;
            TXOInfo info = Deserialize<TXOInfo>(ShallowTmp(cur, TXOInfo::serSize()), &myok);
            cur += TXOInfo::serSize();
            if (!chkAssertion(myok)) return ret;
            ret.delUndos.emplace_back(std::move(txo), std::move(info));
        }
        chkAssertion(cur == end, "cur != end");
        setOk(true);
        return ret;
    }

    template <> QByteArray Serialize(const TxNumVec &v)
    {
        // this serializes a vector of TxNums to a compact representation (6 bytes, eg 48 bits per TxNum), in little endian byte order
        QByteArray ret(int(v.size()*6), Qt::Uninitialized);
        uint8_t *cur = reinterpret_cast<uint8_t *>(ret.data());
        for (const auto num : v) {
            CompactTXO::txNumToCompactBytes(cur, num);
            cur += 6;
        }
        return ret;
    }
    // this deserializes a vector of TxNums from a compact representation (6 bytes, eg 48 bits per TxNum), assuming little endian byte order
    template <> TxNumVec Deserialize (const QByteArray &ba, bool *ok)
    {
        const size_t blen = size_t(ba.length());
        const size_t N = blen / 6;
        TxNumVec ret;
        if (N * 6 != blen) {
            // wrong size, not multiple of 6; bail
            if (ok) *ok = false;
            return ret;
        }
        if (ok) *ok = true;
        const uint8_t *cur = reinterpret_cast<const uint8_t *>(ba.begin()), *end = reinterpret_cast<const uint8_t *>(ba.end());
        ret.reserve(N);
        for ( ; cur < end; cur += 6) {
            ret.emplace_back( CompactTXO::txNumFromCompactBytes(cur) );
        }
        return ret;
    }

    template <> QByteArray Serialize(const CompactTXO &c) { return c.toBytes(); }
    template <> CompactTXO Deserialize(const QByteArray &b, bool *ok) {
        CompactTXO ret = CompactTXO::fromBytes(b);
        if (ok) *ok = ret.isValid();
        return ret;
    }
}