mirror of
https://github.com/ElementsProject/elements.git
synced 2026-08-15 12:51:00 +02:00
Merge 92fee79dab into merged_master (Bitcoin PR #19806)
Looks like James hardcoded a couple hashes in hard-to-find places, which caused a unit test to fail. I just changed the hash (which covers UTXO data which naturally will be different for us on account of our differently formatted CTxOuts). TODO: determine how Assumeutxo interacts with the fact that we don't save out nonces in our normal UTXO serialization. Probably we will need to remove the nonce from the CCoinStats serialization to avoid having inconsistent hashes across nodes, since these hashes are now checked in assumeutxo?
This commit is contained in:
Andrew Poelstra
commit
61a241c5ca
17 changed files with 715 additions and 30 deletions
|
|
@ -21,6 +21,7 @@
|
|||
#include <logging.h>
|
||||
#include <logging/timer.h>
|
||||
#include <mainchainrpc.h>
|
||||
#include <node/coinstats.h>
|
||||
#include <node/ui_interface.h>
|
||||
#include <optional.h>
|
||||
#include <pegins.h>
|
||||
|
|
@ -5500,7 +5501,8 @@ double GuessVerificationProgress(const CBlockIndex* pindex, int64_t blockInterva
|
|||
|
||||
Optional<uint256> ChainstateManager::SnapshotBlockhash() const {
|
||||
LOCK(::cs_main);
|
||||
if (m_active_chainstate != nullptr) {
|
||||
if (m_active_chainstate != nullptr &&
|
||||
!m_active_chainstate->m_from_snapshot_blockhash.IsNull()) {
|
||||
// If a snapshot chainstate exists, it will always be our active.
|
||||
return m_active_chainstate->m_from_snapshot_blockhash;
|
||||
}
|
||||
|
|
@ -5545,6 +5547,295 @@ CChainState& ChainstateManager::InitializeChainstate(CTxMemPool& mempool, const
|
|||
return *to_modify;
|
||||
}
|
||||
|
||||
const AssumeutxoData* ExpectedAssumeutxo(
|
||||
const int height, const CChainParams& chainparams)
|
||||
{
|
||||
const MapAssumeutxo& valid_assumeutxos_map = chainparams.Assumeutxo();
|
||||
const auto assumeutxo_found = valid_assumeutxos_map.find(height);
|
||||
|
||||
if (assumeutxo_found != valid_assumeutxos_map.end()) {
|
||||
return &assumeutxo_found->second;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
bool ChainstateManager::ActivateSnapshot(
|
||||
CAutoFile& coins_file,
|
||||
const SnapshotMetadata& metadata,
|
||||
bool in_memory)
|
||||
{
|
||||
uint256 base_blockhash = metadata.m_base_blockhash;
|
||||
|
||||
if (this->SnapshotBlockhash()) {
|
||||
LogPrintf("[snapshot] can't activate a snapshot-based chainstate more than once\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
int64_t current_coinsdb_cache_size{0};
|
||||
int64_t current_coinstip_cache_size{0};
|
||||
|
||||
// Cache percentages to allocate to each chainstate.
|
||||
//
|
||||
// These particular percentages don't matter so much since they will only be
|
||||
// relevant during snapshot activation; caches are rebalanced at the conclusion of
|
||||
// this function. We want to give (essentially) all available cache capacity to the
|
||||
// snapshot to aid the bulk load later in this function.
|
||||
static constexpr double IBD_CACHE_PERC = 0.01;
|
||||
static constexpr double SNAPSHOT_CACHE_PERC = 0.99;
|
||||
|
||||
{
|
||||
LOCK(::cs_main);
|
||||
// Resize the coins caches to ensure we're not exceeding memory limits.
|
||||
//
|
||||
// Allocate the majority of the cache to the incoming snapshot chainstate, since
|
||||
// (optimistically) getting to its tip will be the top priority. We'll need to call
|
||||
// `MaybeRebalanceCaches()` once we're done with this function to ensure
|
||||
// the right allocation (including the possibility that no snapshot was activated
|
||||
// and that we should restore the active chainstate caches to their original size).
|
||||
//
|
||||
current_coinsdb_cache_size = this->ActiveChainstate().m_coinsdb_cache_size_bytes;
|
||||
current_coinstip_cache_size = this->ActiveChainstate().m_coinstip_cache_size_bytes;
|
||||
|
||||
// Temporarily resize the active coins cache to make room for the newly-created
|
||||
// snapshot chain.
|
||||
this->ActiveChainstate().ResizeCoinsCaches(
|
||||
static_cast<size_t>(current_coinstip_cache_size * IBD_CACHE_PERC),
|
||||
static_cast<size_t>(current_coinsdb_cache_size * IBD_CACHE_PERC));
|
||||
}
|
||||
|
||||
auto snapshot_chainstate = WITH_LOCK(::cs_main, return MakeUnique<CChainState>(
|
||||
this->ActiveChainstate().m_mempool, m_blockman, base_blockhash));
|
||||
|
||||
{
|
||||
LOCK(::cs_main);
|
||||
snapshot_chainstate->InitCoinsDB(
|
||||
static_cast<size_t>(current_coinsdb_cache_size * SNAPSHOT_CACHE_PERC),
|
||||
in_memory, false, "chainstate");
|
||||
snapshot_chainstate->InitCoinsCache(
|
||||
static_cast<size_t>(current_coinstip_cache_size * SNAPSHOT_CACHE_PERC));
|
||||
}
|
||||
|
||||
const bool snapshot_ok = this->PopulateAndValidateSnapshot(
|
||||
*snapshot_chainstate, coins_file, metadata);
|
||||
|
||||
if (!snapshot_ok) {
|
||||
WITH_LOCK(::cs_main, this->MaybeRebalanceCaches());
|
||||
return false;
|
||||
}
|
||||
|
||||
{
|
||||
LOCK(::cs_main);
|
||||
assert(!m_snapshot_chainstate);
|
||||
m_snapshot_chainstate.swap(snapshot_chainstate);
|
||||
const bool chaintip_loaded = m_snapshot_chainstate->LoadChainTip(::Params());
|
||||
assert(chaintip_loaded);
|
||||
|
||||
m_active_chainstate = m_snapshot_chainstate.get();
|
||||
|
||||
LogPrintf("[snapshot] successfully activated snapshot %s\n", base_blockhash.ToString());
|
||||
LogPrintf("[snapshot] (%.2f MB)\n",
|
||||
m_snapshot_chainstate->CoinsTip().DynamicMemoryUsage() / (1000 * 1000));
|
||||
|
||||
this->MaybeRebalanceCaches();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ChainstateManager::PopulateAndValidateSnapshot(
|
||||
CChainState& snapshot_chainstate,
|
||||
CAutoFile& coins_file,
|
||||
const SnapshotMetadata& metadata)
|
||||
{
|
||||
// It's okay to release cs_main before we're done using `coins_cache` because we know
|
||||
// that nothing else will be referencing the newly created snapshot_chainstate yet.
|
||||
CCoinsViewCache& coins_cache = *WITH_LOCK(::cs_main, return &snapshot_chainstate.CoinsTip());
|
||||
|
||||
uint256 base_blockhash = metadata.m_base_blockhash;
|
||||
|
||||
COutPoint outpoint;
|
||||
Coin coin;
|
||||
const uint64_t coins_count = metadata.m_coins_count;
|
||||
uint64_t coins_left = metadata.m_coins_count;
|
||||
|
||||
LogPrintf("[snapshot] loading coins from snapshot %s\n", base_blockhash.ToString());
|
||||
int64_t flush_now{0};
|
||||
int64_t coins_processed{0};
|
||||
|
||||
while (coins_left > 0) {
|
||||
try {
|
||||
coins_file >> outpoint;
|
||||
} catch (const std::ios_base::failure&) {
|
||||
LogPrintf("[snapshot] bad snapshot - no coins left after deserializing %d coins\n",
|
||||
coins_count - coins_left);
|
||||
return false;
|
||||
}
|
||||
coins_file >> coin;
|
||||
coins_cache.EmplaceCoinInternalDANGER(std::move(outpoint), std::move(coin));
|
||||
|
||||
--coins_left;
|
||||
++coins_processed;
|
||||
|
||||
if (coins_processed % 1000000 == 0) {
|
||||
LogPrintf("[snapshot] %d coins loaded (%.2f%%, %.2f MB)\n",
|
||||
coins_processed,
|
||||
static_cast<float>(coins_processed) * 100 / static_cast<float>(coins_count),
|
||||
coins_cache.DynamicMemoryUsage() / (1000 * 1000));
|
||||
}
|
||||
|
||||
// Batch write and flush (if we need to) every so often.
|
||||
//
|
||||
// If our average Coin size is roughly 41 bytes, checking every 120,000 coins
|
||||
// means <5MB of memory imprecision.
|
||||
if (coins_processed % 120000 == 0) {
|
||||
if (ShutdownRequested()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto snapshot_cache_state = WITH_LOCK(::cs_main,
|
||||
return snapshot_chainstate.GetCoinsCacheSizeState(&snapshot_chainstate.m_mempool));
|
||||
|
||||
if (snapshot_cache_state >=
|
||||
CoinsCacheSizeState::CRITICAL) {
|
||||
LogPrintf("[snapshot] flushing coins cache (%.2f MB)... ", /* Continued */
|
||||
coins_cache.DynamicMemoryUsage() / (1000 * 1000));
|
||||
flush_now = GetTimeMillis();
|
||||
|
||||
// This is a hack - we don't know what the actual best block is, but that
|
||||
// doesn't matter for the purposes of flushing the cache here. We'll set this
|
||||
// to its correct value (`base_blockhash`) below after the coins are loaded.
|
||||
coins_cache.SetBestBlock(GetRandHash());
|
||||
|
||||
coins_cache.Flush();
|
||||
LogPrintf("done (%.2fms)\n", GetTimeMillis() - flush_now);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Important that we set this. This and the coins_cache accesses above are
|
||||
// sort of a layer violation, but either we reach into the innards of
|
||||
// CCoinsViewCache here or we have to invert some of the CChainState to
|
||||
// embed them in a snapshot-activation-specific CCoinsViewCache bulk load
|
||||
// method.
|
||||
coins_cache.SetBestBlock(base_blockhash);
|
||||
|
||||
bool out_of_coins{false};
|
||||
try {
|
||||
coins_file >> outpoint;
|
||||
} catch (const std::ios_base::failure&) {
|
||||
// We expect an exception since we should be out of coins.
|
||||
out_of_coins = true;
|
||||
}
|
||||
if (!out_of_coins) {
|
||||
LogPrintf("[snapshot] bad snapshot - coins left over after deserializing %d coins\n",
|
||||
coins_count);
|
||||
return false;
|
||||
}
|
||||
|
||||
LogPrintf("[snapshot] loaded %d (%.2f MB) coins from snapshot %s\n",
|
||||
coins_count,
|
||||
coins_cache.DynamicMemoryUsage() / (1000 * 1000),
|
||||
base_blockhash.ToString());
|
||||
|
||||
LogPrintf("[snapshot] flushing snapshot chainstate to disk\n");
|
||||
// No need to acquire cs_main since this chainstate isn't being used yet.
|
||||
coins_cache.Flush(); // TODO: if #17487 is merged, add erase=false here for better performance.
|
||||
|
||||
assert(coins_cache.GetBestBlock() == base_blockhash);
|
||||
|
||||
CCoinsStats stats;
|
||||
auto breakpoint_fnc = [] { /* TODO insert breakpoint here? */ };
|
||||
|
||||
// As above, okay to immediately release cs_main here since no other context knows
|
||||
// about the snapshot_chainstate.
|
||||
CCoinsViewDB* snapshot_coinsdb = WITH_LOCK(::cs_main, return &snapshot_chainstate.CoinsDB());
|
||||
|
||||
if (!GetUTXOStats(snapshot_coinsdb, stats, CoinStatsHashType::HASH_SERIALIZED, breakpoint_fnc)) {
|
||||
LogPrintf("[snapshot] failed to generate coins stats\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Ensure that the base blockhash appears in the known chain of valid headers. We're willing to
|
||||
// wait a bit here because the snapshot may have been loaded on startup, before we've
|
||||
// received headers from the network.
|
||||
|
||||
int max_secs_to_wait_for_headers = 60 * 10;
|
||||
CBlockIndex* snapshot_start_block = nullptr;
|
||||
|
||||
while (max_secs_to_wait_for_headers > 0) {
|
||||
snapshot_start_block = WITH_LOCK(::cs_main,
|
||||
return m_blockman.LookupBlockIndex(base_blockhash));
|
||||
--max_secs_to_wait_for_headers;
|
||||
|
||||
if (!snapshot_start_block) {
|
||||
std::this_thread::sleep_for(std::chrono::seconds(1));
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (snapshot_start_block == nullptr) {
|
||||
LogPrintf("[snapshot] timed out waiting for snapshot start blockheader %s\n",
|
||||
base_blockhash.ToString());
|
||||
return false;
|
||||
}
|
||||
|
||||
// Assert that the deserialized chainstate contents match the expected assumeutxo value.
|
||||
|
||||
int base_height = snapshot_start_block->nHeight;
|
||||
auto maybe_au_data = ExpectedAssumeutxo(base_height, ::Params());
|
||||
|
||||
if (!maybe_au_data) {
|
||||
LogPrintf("[snapshot] assumeutxo height in snapshot metadata not recognized " /* Continued */
|
||||
"(%d) - refusing to load snapshot\n", base_height);
|
||||
return false;
|
||||
}
|
||||
|
||||
const AssumeutxoData& au_data = *maybe_au_data;
|
||||
|
||||
if (stats.hashSerialized != au_data.hash_serialized) {
|
||||
LogPrintf("[snapshot] bad snapshot content hash: expected %s, got %s\n",
|
||||
au_data.hash_serialized.ToString(), stats.hashSerialized.ToString());
|
||||
return false;
|
||||
}
|
||||
|
||||
snapshot_chainstate.m_chain.SetTip(snapshot_start_block);
|
||||
|
||||
// The remainder of this function requires modifying data protected by cs_main.
|
||||
LOCK(::cs_main);
|
||||
|
||||
// Fake various pieces of CBlockIndex state:
|
||||
//
|
||||
// - nChainTx: so that we accurately report IBD-to-tip progress
|
||||
// - nTx: so that LoadBlockIndex() loads assumed-valid CBlockIndex entries
|
||||
// (among other things)
|
||||
// - nStatus & BLOCK_OPT_WITNESS: so that RewindBlockIndex() doesn't zealously
|
||||
// unwind the assumed-valid chain.
|
||||
//
|
||||
CBlockIndex* index = nullptr;
|
||||
for (int i = 0; i <= snapshot_chainstate.m_chain.Height(); ++i) {
|
||||
index = snapshot_chainstate.m_chain[i];
|
||||
|
||||
if (!index->nTx) {
|
||||
index->nTx = 1;
|
||||
}
|
||||
index->nChainTx = index->pprev ? index->pprev->nChainTx + index->nTx : 1;
|
||||
|
||||
// We need to fake this flag so that CChainState::RewindBlockIndex()
|
||||
// won't try to rewind the entire assumed-valid chain on startup.
|
||||
if (index->pprev && ::IsWitnessEnabled(index->pprev, ::Params().GetConsensus())) {
|
||||
index->nStatus |= BLOCK_OPT_WITNESS;
|
||||
}
|
||||
}
|
||||
|
||||
assert(index);
|
||||
index->nChainTx = metadata.m_nchaintx;
|
||||
snapshot_chainstate.setBlockIndexCandidates.insert(snapshot_start_block);
|
||||
|
||||
LogPrintf("[snapshot] validated snapshot (%.2f MB)\n",
|
||||
coins_cache.DynamicMemoryUsage() / (1000 * 1000));
|
||||
return true;
|
||||
}
|
||||
|
||||
CChainState& ChainstateManager::ActiveChainstate() const
|
||||
{
|
||||
LOCK(::cs_main);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue