Merge 66c08e741d into merged_master (Bitcoin PR bitcoin/bitcoin#24865)

src/wallet/rpc/backup.cpp

@@ -96,6 +96,22 @@ static void RescanWallet(CWallet& wallet, const WalletRescanReserver& reserver,
     }
 }
 
+static void EnsureBlockDataFromTime(const CWallet& wallet, int64_t timestamp)
+{
+    auto& chain{wallet.chain()};
+    if (!chain.havePruned()) {
+        return;
+    }
+
+    int height{0};
+    const bool found{chain.findFirstBlockWithTimeAndHeight(timestamp - TIMESTAMP_WINDOW, 0, FoundBlock().height(height))};
+
+    uint256 tip_hash{WITH_LOCK(wallet.cs_wallet, return wallet.GetLastBlockHash())};
+    if (found && !chain.hasBlocks(tip_hash, height)) {
+        throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Pruned blocks from height %d required to import keys. Use RPC call getblockchaininfo to determine your pruned height.", height));
+    }
+}
+
 RPCHelpMan importprivkey()
 {
     return RPCHelpMan{"importprivkey",
@@ -507,13 +523,6 @@ RPCHelpMan importwallet()
 
     EnsureLegacyScriptPubKeyMan(*pwallet, true);
 
-    if (pwallet->chain().havePruned()) {
-        // Exit early and print an error.
-        // If a block is pruned after this check, we will import the key(s),
-        // but fail the rescan with a generic error.
-        throw JSONRPCError(RPC_WALLET_ERROR, "Importing wallets is disabled when blocks are pruned");
-    }
-
     WalletRescanReserver reserver(*pwallet);
     if (!reserver.reserve()) {
         throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait.");
@@ -568,15 +577,18 @@ RPCHelpMan importwallet()
                         fLabel = true;
                     }
                 }
+                nTimeBegin = std::min(nTimeBegin, nTime);
                 keys.push_back(std::make_tuple(key, nTime, fLabel, strLabel));
             } else if(IsHex(vstr[0])) {
                 std::vector<unsigned char> vData(ParseHex(vstr[0]));
                 CScript script = CScript(vData.begin(), vData.end());
                 int64_t birth_time = ParseISO8601DateTime(vstr[1]);
+                if (birth_time > 0) nTimeBegin = std::min(nTimeBegin, birth_time);
                 scripts.push_back(std::pair<CScript, int64_t>(script, birth_time));
             }
         }
         file.close();
+        EnsureBlockDataFromTime(*pwallet, nTimeBegin);
         // We now know whether we are importing private keys, so we can error if private keys are disabled
         if (keys.size() > 0 && pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) {
             pwallet->chain().showProgress("", 100, false); // hide progress dialog in GUI
@@ -605,8 +617,6 @@ RPCHelpMan importwallet()
 
             if (has_label)
                 pwallet->SetAddressBook(PKHash(keyid), label, "receive");
-
-            nTimeBegin = std::min(nTimeBegin, time);
             progress++;
         }
         for (const auto& script_pair : scripts) {
@@ -619,9 +629,6 @@ RPCHelpMan importwallet()
                 fGood = false;
                 continue;
             }
-            if (time > 0) {
-                nTimeBegin = std::min(nTimeBegin, time);
-            }
 
             progress++;
         }

test/functional/feature_pruning.py

@@ -10,8 +10,11 @@ This test takes 30 mins or more (up to 2 hours)
 """
 import os
 
-from test_framework.blocktools import create_coinbase
-from test_framework.messages import CBlock
+from test_framework.blocktools import (
+    MIN_BLOCKS_TO_KEEP,
+    create_block,
+    create_coinbase,
+)
 from test_framework.script import (
     CScript,
     OP_NOP,
@@ -48,21 +51,7 @@ def mine_large_blocks(node, n):
     previousblockhash = int(best_block["hash"], 16)
 
     for _ in range(n):
-        # Build the coinbase transaction (with large scriptPubKey)
-        coinbase_tx = create_coinbase(height)
-        coinbase_tx.vin[0].nSequence = 2 ** 32 - 1
-        coinbase_tx.vout[0].scriptPubKey = big_script
-        coinbase_tx.rehash()
-
-        # Build the block
-        block = CBlock()
-        block.nVersion = best_block["version"]
-        block.hashPrevBlock = previousblockhash
-        block.nTime = mine_large_blocks.nTime
-        block.nBits = int('207fffff', 16)
-        block.nNonce = 0
-        block.vtx = [coinbase_tx]
-        block.hashMerkleRoot = block.calc_merkle_root()
+        block = create_block(hashprev=previousblockhash, ntime=mine_large_blocks.nTime, coinbase=create_coinbase(height, script_pubkey=big_script))
         block.solve()
 
         # Submit to the node
@@ -345,7 +334,7 @@ class PruneTest(BitcoinTestFramework):
         assert has_block(2), "blk00002.dat is still there, should be pruned by now"
 
         # advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat)
-        self.generate(node, 288, sync_fun=self.no_op)
+        self.generate(node, MIN_BLOCKS_TO_KEEP, sync_fun=self.no_op)
         prune(1000)
         assert not has_block(2), "blk00002.dat is still there, should be pruned by now"
         assert not has_block(3), "blk00003.dat is still there, should be pruned by now"

test/functional/test_framework/blocktools.py

@@ -63,6 +63,7 @@ WITNESS_COMMITMENT_HEADER = b"\xaa\x21\xa9\xed"
 
 NORMAL_GBT_REQUEST_PARAMS = {"rules": ["segwit"]}
 VERSIONBITS_LAST_OLD_BLOCK_VERSION = 4
+MIN_BLOCKS_TO_KEEP = 288
 
 # Assumes a BIP34 valid commitment exists
 def get_coinbase_height(coinbase):
@@ -138,7 +139,7 @@ def script_BIP34_coinbase_height(height):
     return CScript([CScriptNum(height)])
 
 
-def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0, nValue=50):
+def create_coinbase(height, pubkey=None, *, script_pubkey=None, extra_output_script=None, fees=0, nValue=50):
     """Create a coinbase transaction.
 
     If pubkey is passed in, the coinbase output will be a P2PK output;
@@ -157,6 +158,8 @@ def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0, nValu
     coinbaseoutput.nValue = CTxOutValue(value)
     if pubkey is not None:
         coinbaseoutput.scriptPubKey = key_to_p2pk_script(pubkey)
+    elif script_pubkey is not None:
+        coinbaseoutput.scriptPubKey = script_pubkey
     else:
         coinbaseoutput.scriptPubKey = CScript([OP_TRUE])
     coinbase.vout = [coinbaseoutput]

test/functional/test_runner.py

@@ -88,6 +88,7 @@ EXTENDED_SCRIPTS = [
     'feature_index_prune.py',
     'feature_trim_headers.py',
     'wallet_elements_regression_fundrawtransaction.py --legacy-wallet', # ELEMENTS FIXME
+    'wallet_pruning.py --legacy-wallet',
 ]
 
 BASE_SCRIPTS = [

test/functional/wallet_pruning.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+"""Test wallet import on pruned node."""
+import os
+
+from test_framework.util import assert_equal, assert_raises_rpc_error
+from test_framework.blocktools import (
+    COINBASE_MATURITY,
+    create_block
+)
+from test_framework.blocktools import create_coinbase
+from test_framework.test_framework import BitcoinTestFramework
+
+from test_framework.script import (
+    CScript,
+    OP_RETURN,
+    OP_TRUE,
+)
+
+class WalletPruningTest(BitcoinTestFramework):
+    def add_options(self, parser):
+        self.add_wallet_options(parser, descriptors=False)
+
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.num_nodes = 2
+        self.wallet_names = []
+        self.extra_args = [
+            [], # node dedicated to mining
+            ['-prune=550'], # node dedicated to testing pruning
+        ]
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_wallet()
+        self.skip_if_no_bdb()
+
+    def mine_large_blocks(self, node, n):
+        # Get the block parameters for the first block
+        best_block = node.getblock(node.getbestblockhash())
+        height = int(best_block["height"]) + 1
+        self.nTime = max(self.nTime, int(best_block["time"])) + 1
+        previousblockhash = int(best_block["hash"], 16)
+        big_script = CScript([OP_RETURN] + [OP_TRUE] * 950000)
+        for _ in range(n):
+            block = create_block(hashprev=previousblockhash, ntime=self.nTime, coinbase=create_coinbase(height, script_pubkey=big_script))
+            block.solve()
+
+            # Submit to the node
+            node.submitblock(block.serialize().hex())
+
+            previousblockhash = block.sha256
+            height += 1
+
+            # Simulate 10 minutes of work time per block
+            # Important for matching a timestamp with a block +- some window
+            self.nTime += 600
+            for n in self.nodes:
+                if n.running:
+                    n.setmocktime(self.nTime) # Update node's time to accept future blocks
+        self.sync_all()
+
+    def test_wallet_import_pruned(self, wallet_name):
+        self.log.info("Make sure we can import wallet when pruned and required blocks are still available")
+
+        wallet_file = wallet_name + ".dat"
+        wallet_birthheight = self.get_birthheight(wallet_file)
+
+        # Verify that the block at wallet's birthheight is available at the pruned node
+        self.nodes[1].getblock(self.nodes[1].getblockhash(wallet_birthheight))
+
+        # Import wallet into pruned node
+        self.nodes[1].createwallet(wallet_name="wallet_pruned", descriptors=False, load_on_startup=True)
+        self.nodes[1].importwallet(os.path.join(self.nodes[0].datadir, wallet_file))
+
+        # Make sure that prune node's wallet correctly accounts for balances
+        assert_equal(self.nodes[1].getbalance(), self.nodes[0].getbalance())
+
+        self.log.info("- Done")
+
+    def test_wallet_import_pruned_with_missing_blocks(self, wallet_name):
+        self.log.info("Make sure we cannot import wallet when pruned and required blocks are not available")
+
+        wallet_file = wallet_name + ".dat"
+        wallet_birthheight = self.get_birthheight(wallet_file)
+
+        # Verify that the block at wallet's birthheight is not available at the pruned node
+        assert_raises_rpc_error(-1, "Block not available (pruned data)", self.nodes[1].getblock, self.nodes[1].getblockhash(wallet_birthheight))
+
+        # Make sure wallet cannot be imported because of missing blocks
+        # This will try to rescan blocks `TIMESTAMP_WINDOW` (2h) before the wallet birthheight.
+        # There are 6 blocks an hour, so 11 blocks (excluding birthheight).
+        assert_raises_rpc_error(-4, f"Pruned blocks from height {wallet_birthheight - 11} required to import keys. Use RPC call getblockchaininfo to determine your pruned height.", self.nodes[1].importwallet, os.path.join(self.nodes[0].datadir, wallet_file))
+        self.log.info("- Done")
+
+    def get_birthheight(self, wallet_file):
+        """Gets birthheight of a wallet on node0"""
+        with open(os.path.join(self.nodes[0].datadir, wallet_file), 'r', encoding="utf8") as f:
+            for line in f:
+                if line.startswith('# * Best block at time of backup'):
+                    wallet_birthheight = int(line.split(' ')[9])
+                    return wallet_birthheight
+
+    def has_block(self, block_index):
+        """Checks if the pruned node has the specific blk0000*.dat file"""
+        return os.path.isfile(os.path.join(self.nodes[1].datadir, self.chain, "blocks", f"blk{block_index:05}.dat"))
+
+    def create_wallet(self, wallet_name, *, unload=False):
+        """Creates and dumps a wallet on the non-pruned node0 to be later import by the pruned node"""
+        self.nodes[0].createwallet(wallet_name=wallet_name, descriptors=False, load_on_startup=True)
+        self.nodes[0].dumpwallet(os.path.join(self.nodes[0].datadir, wallet_name + ".dat"))
+        if (unload):
+            self.nodes[0].unloadwallet(wallet_name)
+
+    def run_test(self):
+        self.nTime = 0
+        self.log.info("Warning! This test requires ~1.3GB of disk space")
+
+        self.log.info("Generating a long chain of blocks...")
+
+        # A blk*.dat file is 128MB
+        # Generate 250 light blocks
+        self.generate(self.nodes[0], 250, sync_fun=self.no_op)
+        # Generate 50MB worth of large blocks in the blk00000.dat file
+        self.mine_large_blocks(self.nodes[0], 50)
+
+        # Create a wallet which birth's block is in the blk00000.dat file
+        wallet_birthheight_1 = "wallet_birthheight_1"
+        assert_equal(self.has_block(1), False)
+        self.create_wallet(wallet_birthheight_1, unload=True)
+
+        # Generate enough large blocks to reach pruning disk limit
+        # Not pruning yet because we are still below PruneAfterHeight
+        self.mine_large_blocks(self.nodes[0], 600)
+        self.log.info("- Long chain created")
+
+        # Create a wallet with birth height > wallet_birthheight_1
+        wallet_birthheight_2 = "wallet_birthheight_2"
+        self.create_wallet(wallet_birthheight_2)
+
+        # Fund wallet to later verify that importwallet correctly accounts for balances
+        self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 1, self.nodes[0].getnewaddress(), sync_fun=self.no_op)
+
+        # We've reached pruning storage & height limit but
+        # pruning doesn't run until another chunk (blk*.dat file) is allocated.
+        # That's why we are generating another 5 large blocks
+        self.mine_large_blocks(self.nodes[0], 5)
+
+        # blk00000.dat file is now pruned from node1
+        assert_equal(self.has_block(0), False)
+
+        self.test_wallet_import_pruned(wallet_birthheight_2)
+        self.test_wallet_import_pruned_with_missing_blocks(wallet_birthheight_1)
+
+if __name__ == '__main__':
+    WalletPruningTest().main()