MERGE-FIX: Update compatibility functional tests

I should have split this into one commit copying the entire folder and
one where I fix the ELEMENTS-based things. However, grepping the diff
for "elements" case-insensitive makes you catch all differences.
The next commit will contain the more subtle differences.

test/bitcoin_functional/functional/README.md

@@ -20,9 +20,13 @@ don't have test cases for.
 - Where possible, try to adhere to [PEP-8 guidelines](https://www.python.org/dev/peps/pep-0008/)
 - Use a python linter like flake8 before submitting PRs to catch common style
   nits (eg trailing whitespace, unused imports, etc)
+- The oldest supported Python version is specified in [doc/dependencies.md](/doc/dependencies.md).
+  Consider using [pyenv](https://github.com/pyenv/pyenv), which checks [.python-version](/.python-version),
+  to prevent accidentally introducing modern syntax from an unsupported Python version.
+  The Travis linter also checks this, but [possibly not in all cases](https://github.com/bitcoin/bitcoin/pull/14884#discussion_r239585126).
 - See [the python lint script](/test/lint/lint-python.sh) that checks for violations that
   could lead to bugs and issues in the test code.
-- Avoid wildcard imports where possible
+- Avoid wildcard imports
 - Use a module-level docstring to describe what the test is testing, and how it
   is testing it.
 - When subclassing the BitcoinTestFramwork, place overrides for the
@@ -39,6 +43,7 @@ don't have test cases for.
     - `mining` for tests for mining features, eg `mining_prioritisetransaction.py`
     - `p2p` for tests that explicitly test the p2p interface, eg `p2p_disconnect_ban.py`
     - `rpc` for tests for individual RPC methods or features, eg `rpc_listtransactions.py`
+    - `tool` for tests for tools, eg `tool_wallet.py`
     - `wallet` for tests for wallet features, eg `wallet_keypool.py`
 - use an underscore to separate words
     - exception: for tests for specific RPCs or command line options which don't include underscores, name the test after the exact RPC or argument name, eg `rpc_decodescript.py`, not `rpc_decode_script.py`
@@ -118,3 +123,36 @@ Helpers for script.py
 
 #### [test_framework/blocktools.py](test_framework/blocktools.py)
 Helper functions for creating blocks and transactions.
+
+### Benchmarking with perf
+
+An easy way to profile node performance during functional tests is provided
+for Linux platforms using `perf`.
+
+Perf will sample the running node and will generate profile data in the node's
+datadir. The profile data can then be presented using `perf report` or a graphical
+tool like [hotspot](https://github.com/KDAB/hotspot).
+
+There are two ways of invoking perf: one is to use the `--perf` flag when
+running tests, which will profile each node during the entire test run: perf
+begins to profile when the node starts and ends when it shuts down. The other
+way is the use the `profile_with_perf` context manager, e.g.
+
+```python
+with node.profile_with_perf("send-big-msgs"):
+    # Perform activity on the node you're interested in profiling, e.g.:
+    for _ in range(10000):
+        node.p2p.send_message(some_large_message)
+```
+
+To see useful textual output, run
+
+```sh
+perf report -i /path/to/datadir/send-big-msgs.perf.data.xxxx --stdio | c++filt | less
+```
+
+#### See also:
+
+- [Installing perf](https://askubuntu.com/q/50145)
+- [Perf examples](http://www.brendangregg.com/perf.html)
+- [Hotspot](https://github.com/KDAB/hotspot): a GUI for perf output analysis

test/bitcoin_functional/functional/combine_logs.py

@@ -2,7 +2,9 @@
 """Combine logs from multiple bitcoin nodes as well as the test_framework log.
 
 This streams the combined log output to stdout. Use combine_logs.py > outputfile
-to write to an outputfile."""
+to write to an outputfile.
+
+If no argument is provided, the most recent test directory will be used."""
 
 import argparse
 from collections import defaultdict, namedtuple
@@ -11,6 +13,13 @@ import itertools
 import os
 import re
 import sys
+import tempfile
+
+# N.B.: don't import any local modules here - this script must remain executable
+# without the parent module installed.
+
+# Should match same symbol in `test_framework.test_framework`.
+TMPDIR_PREFIX = "bitcoin_func_test_"
 
 # Matches on the date format at the start of the log event
 TIMESTAMP_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{6})?Z")
@@ -19,22 +28,30 @@ LogEvent = namedtuple('LogEvent', ['timestamp', 'source', 'event'])
 
 def main():
     """Main function. Parses args, reads the log files and renders them as text or html."""
-
-    parser = argparse.ArgumentParser(usage='%(prog)s [options] <test temporary directory>', description=__doc__)
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument(
+        'testdir', nargs='?', default='',
+        help=('temporary test directory to combine logs from. '
+              'Defaults to the most recent'))
     parser.add_argument('-c', '--color', dest='color', action='store_true', help='outputs the combined log with events colored by source (requires posix terminal colors. Use less -r for viewing)')
     parser.add_argument('--html', dest='html', action='store_true', help='outputs the combined log as html. Requires jinja2. pip install jinja2')
-    args, unknown_args = parser.parse_known_args()
+    args = parser.parse_args()
 
     if args.html and args.color:
         print("Only one out of --color or --html should be specified")
         sys.exit(1)
 
-    # There should only be one unknown argument - the path of the temporary test directory
-    if len(unknown_args) != 1:
-        print("Unexpected arguments" + str(unknown_args))
+    testdir = args.testdir or find_latest_test_dir()
+
+    if not testdir:
+        print("No test directories found")
         sys.exit(1)
 
-    log_events = read_logs(unknown_args[0])
+    if not args.testdir:
+        print("Opening latest test directory: {}".format(testdir), file=sys.stderr)
+
+    log_events = read_logs(testdir)
 
     print_logs(log_events, color=args.color, html=args.html)
 
@@ -53,6 +70,29 @@ def read_logs(tmp_dir):
 
     return heapq.merge(*[get_log_events(source, f) for source, f in files])
 
+
+def find_latest_test_dir():
+    """Returns the latest tmpfile test directory prefix."""
+    tmpdir = tempfile.gettempdir()
+
+    def join_tmp(basename):
+        return os.path.join(tmpdir, basename)
+
+    def is_valid_test_tmpdir(basename):
+        fullpath = join_tmp(basename)
+        return (
+            os.path.isdir(fullpath)
+            and basename.startswith(TMPDIR_PREFIX)
+            and os.access(fullpath, os.R_OK)
+        )
+
+    testdir_paths = [
+        join_tmp(name) for name in os.listdir(tmpdir) if is_valid_test_tmpdir(name)
+    ]
+
+    return max(testdir_paths, key=os.path.getmtime) if testdir_paths else None
+
+
 def get_log_events(source, logfile):
     """Generator function that returns individual log events.
 

test/bitcoin_functional/functional/data/invalid_txs.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+# Copyright (c) 2015-2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""
+Templates for constructing various sorts of invalid transactions.
+
+These templates (or an iterator over all of them) can be reused in different
+contexts to test using a number of invalid transaction types.
+
+Hopefully this makes it easier to get coverage of a full variety of tx
+validation checks through different interfaces (AcceptBlock, AcceptToMemPool,
+etc.) without repeating ourselves.
+
+Invalid tx cases not covered here can be found by running:
+
+    $ diff \
+      <(grep -IREho "bad-txns[a-zA-Z-]+" src | sort -u) \
+      <(grep -IEho "bad-txns[a-zA-Z-]+" test/functional/data/invalid_txs.py | sort -u)
+
+"""
+import abc
+
+from test_framework.messages import CTransaction, CTxIn, CTxOut, COutPoint
+from test_framework import script as sc
+from test_framework.blocktools import create_tx_with_script, MAX_BLOCK_SIGOPS
+
+basic_p2sh = sc.CScript([sc.OP_HASH160, sc.hash160(sc.CScript([sc.OP_0])), sc.OP_EQUAL])
+
+
+class BadTxTemplate:
+    """Allows simple construction of a certain kind of invalid tx. Base class to be subclassed."""
+    __metaclass__ = abc.ABCMeta
+
+    # The expected error code given by bitcoind upon submission of the tx.
+    reject_reason = ""
+
+    # Only specified if it differs from mempool acceptance error.
+    block_reject_reason = ""
+
+    # Do we expect to be disconnected after submitting this tx?
+    expect_disconnect = False
+
+    # Is this tx considered valid when included in a block, but not for acceptance into
+    # the mempool (i.e. does it violate policy but not consensus)?
+    valid_in_block = False
+
+    def __init__(self, *, spend_tx=None, spend_block=None):
+        self.spend_tx = spend_block.vtx[0] if spend_block else spend_tx
+        self.spend_avail = sum(o.nValue for o in self.spend_tx.vout)
+        self.valid_txin = CTxIn(COutPoint(self.spend_tx.sha256, 0), b"", 0xffffffff)
+
+    @abc.abstractmethod
+    def get_tx(self, *args, **kwargs):
+        """Return a CTransaction that is invalid per the subclass."""
+        pass
+
+
+class OutputMissing(BadTxTemplate):
+    reject_reason = "bad-txns-vout-empty"
+    expect_disconnect = False
+
+    def get_tx(self):
+        tx = CTransaction()
+        tx.vin.append(self.valid_txin)
+        tx.calc_sha256()
+        return tx
+
+
+class InputMissing(BadTxTemplate):
+    reject_reason = "bad-txns-vin-empty"
+    expect_disconnect = False
+
+    def get_tx(self):
+        tx = CTransaction()
+        tx.vout.append(CTxOut(0, sc.CScript([sc.OP_TRUE] * 100)))
+        tx.calc_sha256()
+        return tx
+
+
+class SizeTooSmall(BadTxTemplate):
+    reject_reason = "tx-size-small"
+    expect_disconnect = False
+    valid_in_block = True
+
+    def get_tx(self):
+        tx = CTransaction()
+        tx.vin.append(self.valid_txin)
+        tx.vout.append(CTxOut(0, sc.CScript([sc.OP_TRUE])))
+        tx.calc_sha256()
+        return tx
+
+
+class BadInputOutpointIndex(BadTxTemplate):
+    # Won't be rejected - nonexistent outpoint index is treated as an orphan since the coins
+    # database can't distinguish between spent outpoints and outpoints which never existed.
+    reject_reason = None
+    expect_disconnect = False
+
+    def get_tx(self):
+        num_indices = len(self.spend_tx.vin)
+        bad_idx = num_indices + 100
+
+        tx = CTransaction()
+        tx.vin.append(CTxIn(COutPoint(self.spend_tx.sha256, bad_idx), b"", 0xffffffff))
+        tx.vout.append(CTxOut(0, basic_p2sh))
+        tx.calc_sha256()
+        return tx
+
+
+class DuplicateInput(BadTxTemplate):
+    reject_reason = 'bad-txns-inputs-duplicate'
+    expect_disconnect = True
+
+    def get_tx(self):
+        tx = CTransaction()
+        tx.vin.append(self.valid_txin)
+        tx.vin.append(self.valid_txin)
+        tx.vout.append(CTxOut(1, basic_p2sh))
+        tx.calc_sha256()
+        return tx
+
+
+class NonexistentInput(BadTxTemplate):
+    reject_reason = None  # Added as an orphan tx.
+    expect_disconnect = False
+
+    def get_tx(self):
+        tx = CTransaction()
+        tx.vin.append(CTxIn(COutPoint(self.spend_tx.sha256 + 1, 0), b"", 0xffffffff))
+        tx.vin.append(self.valid_txin)
+        tx.vout.append(CTxOut(1, basic_p2sh))
+        tx.calc_sha256()
+        return tx
+
+
+class SpendTooMuch(BadTxTemplate):
+    reject_reason = 'bad-txns-in-belowout'
+    expect_disconnect = True
+
+    def get_tx(self):
+        return create_tx_with_script(
+            self.spend_tx, 0, script_pub_key=basic_p2sh, amount=(self.spend_avail + 1))
+
+
+class SpendNegative(BadTxTemplate):
+    reject_reason = 'bad-txns-vout-negative'
+    expect_disconnect = True
+
+    def get_tx(self):
+        return create_tx_with_script(self.spend_tx, 0, amount=-1)
+
+
+class InvalidOPIFConstruction(BadTxTemplate):
+    reject_reason = "mandatory-script-verify-flag-failed (Invalid OP_IF construction)"
+    expect_disconnect = True
+    valid_in_block = True
+
+    def get_tx(self):
+        return create_tx_with_script(
+            self.spend_tx, 0, script_sig=b'\x64' * 35,
+            amount=(self.spend_avail // 2))
+
+
+class TooManySigops(BadTxTemplate):
+    reject_reason = "bad-txns-too-many-sigops"
+    block_reject_reason = "bad-blk-sigops, out-of-bounds SigOpCount"
+    expect_disconnect = False
+
+    def get_tx(self):
+        lotsa_checksigs = sc.CScript([sc.OP_CHECKSIG] * (MAX_BLOCK_SIGOPS))
+        return create_tx_with_script(
+            self.spend_tx, 0,
+            script_pub_key=lotsa_checksigs,
+            amount=1)
+
+
+def iter_all_templates():
+    """Iterate through all bad transaction template types."""
+    return BadTxTemplate.__subclasses__()

test/bitcoin_functional/functional/example_test.py

@@ -13,7 +13,7 @@ is testing and *how* it's being tested
 # libraries then local imports).
 from collections import defaultdict
 
-# Avoid wildcard * imports if possible
+# Avoid wildcard * imports
 from test_framework.blocktools import (create_block, create_coinbase)
 from test_framework.messages import CInv
 from test_framework.mininode import (

test/bitcoin_functional/functional/feature_assumevalid.py

@@ -180,7 +180,7 @@ class AssumeValidTest(BitcoinTestFramework):
         for i in range(2202):
             p2p1.send_message(msg_block(self.blocks[i]))
         # Syncing 2200 blocks can take a while on slow systems. Give it plenty of time to sync.
-        p2p1.sync_with_ping(120)
+        p2p1.sync_with_ping(150)
         assert_equal(self.nodes[1].getblock(self.nodes[1].getbestblockhash())['height'], 2202)
 
         # Send blocks to node2. Block 102 will be rejected.

test/bitcoin_functional/functional/feature_block.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2015-2018 The Bitcoin Core developers
+# Copyright (c) 2015-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test block processing."""
@@ -7,7 +7,13 @@ import copy
 import struct
 import time
 
-from test_framework.blocktools import create_block, create_coinbase, create_tx_with_script, get_legacy_sigopcount_block
+from test_framework.blocktools import (
+    create_block,
+    create_coinbase,
+    create_tx_with_script,
+    get_legacy_sigopcount_block,
+    MAX_BLOCK_SIGOPS,
+)
 from test_framework.key import CECKey
 from test_framework.messages import (
     CBlock,
@@ -45,8 +51,7 @@ from test_framework.script import (
 )
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal
-
-MAX_BLOCK_SIGOPS = 20000
+from data import invalid_txs
 
 #  Use this class for tests that require behavior other than normal "mininode" behavior.
 #  For now, it is used to serialize a bloated varint (b64).
@@ -95,16 +100,21 @@ class FullBlockTest(BitcoinTestFramework):
         self.save_spendable_output()
         self.sync_blocks([b0])
 
+        # These constants chosen specifically to trigger an immature coinbase spend
+        # at a certain time below.
+        NUM_BUFFER_BLOCKS_TO_GENERATE = 99
+        NUM_OUTPUTS_TO_COLLECT = 33
+
         # Allow the block to mature
         blocks = []
-        for i in range(99):
-            blocks.append(self.next_block(5000 + i))
+        for i in range(NUM_BUFFER_BLOCKS_TO_GENERATE):
+            blocks.append(self.next_block("maturitybuffer.{}".format(i)))
             self.save_spendable_output()
         self.sync_blocks(blocks)
 
         # collect spendable outputs now to avoid cluttering the code later on
         out = []
-        for i in range(33):
+        for i in range(NUM_OUTPUTS_TO_COLLECT):
             out.append(self.get_spendable_output())
 
         # Start by building a couple of blocks on top (which output is spent is
@@ -116,7 +126,48 @@ class FullBlockTest(BitcoinTestFramework):
         b2 = self.next_block(2, spend=out[1])
         self.save_spendable_output()
 
-        self.sync_blocks([b1, b2])
+        self.sync_blocks([b1, b2], timeout=4)
+
+        # Select a txn with an output eligible for spending. This won't actually be spent,
+        # since we're testing submission of a series of blocks with invalid txns.
+        attempt_spend_tx = out[2]
+
+        # Submit blocks for rejection, each of which contains a single transaction
+        # (aside from coinbase) which should be considered invalid.
+        for TxTemplate in invalid_txs.iter_all_templates():
+            template = TxTemplate(spend_tx=attempt_spend_tx)
+
+            if template.valid_in_block:
+                continue
+
+            self.log.info("Reject block with invalid tx: %s", TxTemplate.__name__)
+            blockname = "for_invalid.%s" % TxTemplate.__name__
+            badblock = self.next_block(blockname)
+            badtx = template.get_tx()
+            if TxTemplate != invalid_txs.InputMissing:
+                self.sign_tx(badtx, attempt_spend_tx)
+            else:
+                # Segwit is active in regtest at this point, so to deserialize a
+                # transaction without any inputs correctly, we set the outputs
+                # to an empty list. This is a hack, as the serialization of an
+                # empty list of outputs is deserialized as flags==0 and thus
+                # deserialization of the outputs is skipped.
+                # A policy check requires "loose" txs to be of a minimum size,
+                # so vtx is not set to be empty in the TxTemplate class and we
+                # only apply the workaround where txs are not "loose", i.e. in
+                # blocks.
+                #
+                # The workaround has the purpose that both sides calculate
+                # the same tx hash in the merkle tree
+                badtx.vout = []
+            badtx.rehash()
+            badblock = self.update_block(blockname, [badtx])
+            self.sync_blocks(
+                [badblock], success=False,
+                reject_reason=(template.block_reject_reason or template.reject_reason),
+                reconnect=True, timeout=2)
+
+            self.move_tip(2)
 
         # Fork like this:
         #
@@ -579,14 +630,14 @@ class FullBlockTest(BitcoinTestFramework):
         while b47.sha256 < target:
             b47.nNonce += 1
             b47.rehash()
-        self.sync_blocks([b47], False, request_block=False)
+        self.sync_blocks([b47], False, force_send=True, reject_reason='high-hash')
 
         self.log.info("Reject a block with a timestamp >2 hours in the future")
         self.move_tip(44)
         b48 = self.next_block(48, solve=False)
         b48.nTime = int(time.time()) + 60 * 60 * 3
         b48.solve()
-        self.sync_blocks([b48], False, request_block=False)
+        self.sync_blocks([b48], False, force_send=True, reject_reason='time-too-new')
 
         self.log.info("Reject a block with invalid merkle hash")
         self.move_tip(44)
@@ -600,7 +651,7 @@ class FullBlockTest(BitcoinTestFramework):
         b50 = self.next_block(50)
         b50.nBits = b50.nBits - 1
         b50.solve()
-        self.sync_blocks([b50], False, request_block=False, reconnect=True)
+        self.sync_blocks([b50], False, force_send=True, reject_reason='bad-diffbits', reconnect=True)
 
         self.log.info("Reject a block with two coinbase transactions")
         self.move_tip(44)
@@ -630,7 +681,7 @@ class FullBlockTest(BitcoinTestFramework):
         b54 = self.next_block(54, spend=out[15])
         b54.nTime = b35.nTime - 1
         b54.solve()
-        self.sync_blocks([b54], False, request_block=False)
+        self.sync_blocks([b54], False, force_send=True, reject_reason='time-too-old')
 
         # valid timestamp
         self.move_tip(53)
@@ -874,7 +925,7 @@ class FullBlockTest(BitcoinTestFramework):
         #                                                                                    \-> b67 (20)
         #
         #
-        self.log.info("Reject a block with a transaction double spending a transaction creted in the same block")
+        self.log.info("Reject a block with a transaction double spending a transaction created in the same block")
         self.move_tip(65)
         b67 = self.next_block(67)
         tx1 = self.create_and_sign_transaction(out[20], out[20].vout[0].nValue)
@@ -1078,11 +1129,11 @@ class FullBlockTest(BitcoinTestFramework):
 
         self.move_tip(77)
         b80 = self.next_block(80, spend=out[25])
-        self.sync_blocks([b80], False, request_block=False)
+        self.sync_blocks([b80], False, force_send=True)
         self.save_spendable_output()
 
         b81 = self.next_block(81, spend=out[26])
-        self.sync_blocks([b81], False, request_block=False)  # other chain is same length
+        self.sync_blocks([b81], False, force_send=True)  # other chain is same length
         self.save_spendable_output()
 
         b82 = self.next_block(82, spend=out[27])
@@ -1169,7 +1220,7 @@ class FullBlockTest(BitcoinTestFramework):
         blocks = []
         spend = out[32]
         for i in range(89, LARGE_REORG_SIZE + 89):
-            b = self.next_block(i, spend)
+            b = self.next_block(i, spend, version=4)
             tx = CTransaction()
             script_length = MAX_BLOCK_BASE_SIZE - len(b.serialize()) - 69
             script_output = CScript([b'\x00' * script_length])
@@ -1181,26 +1232,38 @@ class FullBlockTest(BitcoinTestFramework):
             self.save_spendable_output()
             spend = self.get_spendable_output()
 
-        self.sync_blocks(blocks, True, timeout=180)
+        self.sync_blocks(blocks, True, timeout=480)
         chain1_tip = i
 
         # now create alt chain of same length
         self.move_tip(88)
         blocks2 = []
         for i in range(89, LARGE_REORG_SIZE + 89):
-            blocks2.append(self.next_block("alt" + str(i)))
-        self.sync_blocks(blocks2, False, request_block=False)
+            blocks2.append(self.next_block("alt" + str(i), version=4))
+        self.sync_blocks(blocks2, False, force_send=True)
 
         # extend alt chain to trigger re-org
-        block = self.next_block("alt" + str(chain1_tip + 1))
-        self.sync_blocks([block], True, timeout=180)
+        block = self.next_block("alt" + str(chain1_tip + 1), version=4)
+        self.sync_blocks([block], True, timeout=480)
 
         # ... and re-org back to the first chain
         self.move_tip(chain1_tip)
-        block = self.next_block(chain1_tip + 1)
-        self.sync_blocks([block], False, request_block=False)
-        block = self.next_block(chain1_tip + 2)
-        self.sync_blocks([block], True, timeout=180)
+        block = self.next_block(chain1_tip + 1, version=4)
+        self.sync_blocks([block], False, force_send=True)
+        block = self.next_block(chain1_tip + 2, version=4)
+        self.sync_blocks([block], True, timeout=480)
+
+        self.log.info("Reject a block with an invalid block header version")
+        b_v1 = self.next_block('b_v1', version=1)
+        self.sync_blocks([b_v1], success=False, force_send=True, reject_reason='bad-version(0x00000001)')
+
+        self.move_tip(chain1_tip + 2)
+        b_cb34 = self.next_block('b_cb34', version=4)
+        b_cb34.vtx[0].vin[0].scriptSig = b_cb34.vtx[0].vin[0].scriptSig[:-1]
+        b_cb34.vtx[0].rehash()
+        b_cb34.hashMerkleRoot = b_cb34.calc_merkle_root()
+        b_cb34.solve()
+        self.sync_blocks([b_cb34], success=False, reject_reason='bad-cb-height', reconnect=True)
 
     # Helper methods
     ################
@@ -1229,7 +1292,7 @@ class FullBlockTest(BitcoinTestFramework):
         tx.rehash()
         return tx
 
-    def next_block(self, number, spend=None, additional_coinbase_value=0, script=CScript([OP_TRUE]), solve=True):
+    def next_block(self, number, spend=None, additional_coinbase_value=0, script=CScript([OP_TRUE]), solve=True, *, version=1):
         if self.tip is None:
             base_block_hash = self.genesis_hash
             block_time = int(time.time()) + 1
@@ -1242,11 +1305,11 @@ class FullBlockTest(BitcoinTestFramework):
         coinbase.vout[0].nValue += additional_coinbase_value
         coinbase.rehash()
         if spend is None:
-            block = create_block(base_block_hash, coinbase, block_time)
+            block = create_block(base_block_hash, coinbase, block_time, version=version)
         else:
             coinbase.vout[0].nValue += spend.vout[0].nValue - 1  # all but one satoshi to fees
             coinbase.rehash()
-            block = create_block(base_block_hash, coinbase, block_time)
+            block = create_block(base_block_hash, coinbase, block_time, version=version)
             tx = self.create_tx(spend, 0, 1, script)  # spend 1 satoshi
             self.sign_tx(tx, spend)
             self.add_transactions_to_block(block, [tx])
@@ -1288,7 +1351,7 @@ class FullBlockTest(BitcoinTestFramework):
         self.blocks[block_number] = block
         return block
 
-    def bootstrap_p2p(self):
+    def bootstrap_p2p(self, timeout=10):
         """Add a P2P connection to the node.
 
         Helper to connect and wait for version handshake."""
@@ -1299,24 +1362,25 @@ class FullBlockTest(BitcoinTestFramework):
         # an INV for the next block and receive two getheaders - one for the
         # IBD and one for the INV. We'd respond to both and could get
         # unexpectedly disconnected if the DoS score for that error is 50.
-        self.nodes[0].p2p.wait_for_getheaders(timeout=5)
+        self.nodes[0].p2p.wait_for_getheaders(timeout=timeout)
 
-    def reconnect_p2p(self):
+    def reconnect_p2p(self, timeout=60):
         """Tear down and bootstrap the P2P connection to the node.
 
         The node gets disconnected several times in this test. This helper
         method reconnects the p2p and restarts the network thread."""
         self.nodes[0].disconnect_p2ps()
-        self.bootstrap_p2p()
+        self.bootstrap_p2p(timeout=timeout)
 
-    def sync_blocks(self, blocks, success=True, reject_reason=None, request_block=True, reconnect=False, timeout=60):
+    def sync_blocks(self, blocks, success=True, reject_reason=None, force_send=False, reconnect=False, timeout=60):
         """Sends blocks to test node. Syncs and verifies that tip has advanced to most recent block.
 
         Call with success = False if the tip shouldn't advance to the most recent block."""
-        self.nodes[0].p2p.send_blocks_and_test(blocks, self.nodes[0], success=success, reject_reason=reject_reason, request_block=request_block, timeout=timeout, expect_disconnect=reconnect)
+        self.nodes[0].p2p.send_blocks_and_test(blocks, self.nodes[0], success=success, reject_reason=reject_reason, force_send=force_send, timeout=timeout, expect_disconnect=reconnect)
 
         if reconnect:
-            self.reconnect_p2p()
+            self.reconnect_p2p(timeout=timeout)
+
 
 if __name__ == '__main__':
     FullBlockTest().main()

test/bitcoin_functional/functional/feature_blocksdir.py

@@ -18,6 +18,8 @@ class BlocksdirTest(BitcoinTestFramework):
 
     def run_test(self):
         self.stop_node(0)
+        assert os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest", "blocks"))
+        assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "blocks"))
         shutil.rmtree(self.nodes[0].datadir)
         initialize_datadir(self.options.tmpdir, 0)
         self.log.info("Starting with nonexistent blocksdir ...")

test/bitcoin_functional/functional/feature_config_args.py

@@ -33,6 +33,19 @@ class ConfArgsTest(BitcoinTestFramework):
             conf.write('server=1\nrpcuser=someuser\nrpcpassword=some#pass')
         self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 3, using # in rpcpassword can be ambiguous and should be avoided')
 
+        with open(inc_conf_file_path, 'w', encoding='utf-8') as conf:
+            conf.write('server=1\nrpcuser=someuser\nmain.rpcpassword=some#pass')
+        self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 3, using # in rpcpassword can be ambiguous and should be avoided')
+
+        with open(inc_conf_file_path, 'w', encoding='utf-8') as conf:
+            conf.write('server=1\nrpcuser=someuser\n[main]\nrpcpassword=some#pass')
+        self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 4, using # in rpcpassword can be ambiguous and should be avoided')
+
+        with open(inc_conf_file_path, 'w', encoding='utf-8') as conf:
+            conf.write('testnot.datadir=1\n[testnet]\n')
+        self.restart_node(0)
+        self.nodes[0].stop_node(expected_stderr='Warning: Section [testnet] is not recognized.' + os.linesep + 'Warning: Section [testnot] is not recognized.')
+
         with open(inc_conf_file_path, 'w', encoding='utf-8') as conf:
             conf.write('')  # clear
 

test/bitcoin_functional/functional/feature_dbcrash.py

@@ -46,7 +46,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework):
         self.num_nodes = 4
         self.setup_clean_chain = False
         # Need a bit of extra time for the nodes to start up for this test
-        self.rpc_timewait = 90
+        self.rpc_timeout = 90
 
         # Set -maxmempool=0 to turn off mempool memory sharing with dbcache
         # Set -rpcservertimeout=900 to reduce socket disconnects in this

test/bitcoin_functional/functional/feature_filelock.py

@@ -23,8 +23,7 @@ class FilelockTest(BitcoinTestFramework):
         self.log.info("Using datadir {}".format(datadir))
 
         self.log.info("Check that we can't start a second bitcoind instance using the same datadir")
-        # ELEMENTS:
-        expected_msg = "Error: Cannot obtain a lock on data directory {}. Elements Core is probably already running.".format(datadir)
+        expected_msg = "Error: Cannot obtain a lock on data directory {}. Bitcoin Core is probably already running.".format(datadir)
         self.nodes[1].assert_start_raises_init_error(extra_args=['-datadir={}'.format(self.nodes[0].datadir), '-noserver'], expected_msg=expected_msg)
 
         if self.is_wallet_compiled():

test/bitcoin_functional/functional/feature_pruning.py

@@ -29,7 +29,7 @@ class PruneTest(BitcoinTestFramework):
     def set_test_params(self):
         self.setup_clean_chain = True
         self.num_nodes = 6
-        self.rpc_timewait = 900
+        self.rpc_timeout = 900
 
         # Create nodes 0 and 1 to mine.
         # Create node 2 to test pruning.
@@ -191,6 +191,8 @@ class PruneTest(BitcoinTestFramework):
     def reorg_back(self):
         # Verify that a block on the old main chain fork has been pruned away
         assert_raises_rpc_error(-1, "Block not available (pruned data)", self.nodes[2].getblock, self.forkhash)
+        with self.nodes[2].assert_debug_log(expected_msgs=['block verification stopping at height', '(pruning, no data)']):
+            self.nodes[2].verifychain(checklevel=4, nblocks=0)
         self.log.info("Will need to redownload block %d" % self.forkheight)
 
         # Verify that we have enough history to reorg back to the fork point
@@ -249,7 +251,7 @@ class PruneTest(BitcoinTestFramework):
                 return index
 
         def prune(index, expected_ret=None):
-            ret = node.pruneblockchain(height(index))
+            ret = node.pruneblockchain(height=height(index))
             # Check the return value. When use_timestamp is True, just check
             # that the return value is less than or equal to the expected
             # value, because when more than one block is generated per second,
@@ -318,7 +320,7 @@ class PruneTest(BitcoinTestFramework):
         if has_block(3):
             raise AssertionError("blk00003.dat is still there, should be pruned by now")
 
-        # stop node, start back up with auto-prune at 550MB, make sure still runs
+        # stop node, start back up with auto-prune at 550 MiB, make sure still runs
         self.stop_node(node_number)
         self.start_node(node_number, extra_args=["-prune=550"])
 

test/bitcoin_functional/functional/feature_segwit.py

@@ -38,6 +38,8 @@ def find_spendable_utxo(node, min_value):
 
     raise AssertionError("Unspent output equal or higher than %s not found" % min_value)
 
+txs_mined = {} # txindex from txid to blockhash
+
 class SegWitTest(BitcoinTestFramework):
     def set_test_params(self):
         self.setup_clean_chain = True
@@ -90,7 +92,7 @@ class SegWitTest(BitcoinTestFramework):
 
         self.log.info("Verify sigops are counted in GBT with pre-BIP141 rules before the fork")
         txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1)
-        tmpl = self.nodes[0].getblocktemplate({})
+        tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']})
         assert(tmpl['sizelimit'] == 1000000)
         assert('weightlimit' not in tmpl)
         assert(tmpl['sigoplimit'] == 20000)
@@ -153,10 +155,10 @@ class SegWitTest(BitcoinTestFramework):
 
         self.log.info("Verify previous witness txs skipped for mining can now be mined")
         assert_equal(len(self.nodes[2].getrawmempool()), 4)
-        block = self.nodes[2].generate(1)  # block 432 (first block with new rules; 432 = 144 * 3)
+        blockhash = self.nodes[2].generate(1)[0]  # block 432 (first block with new rules; 432 = 144 * 3)
         sync_blocks(self.nodes)
         assert_equal(len(self.nodes[2].getrawmempool()), 0)
-        segwit_tx_list = self.nodes[2].getblock(block[0])["tx"]
+        segwit_tx_list = self.nodes[2].getblock(blockhash)["tx"]
         assert_equal(len(segwit_tx_list), 5)
 
         self.log.info("Verify default node can't accept txs with missing witness")
@@ -170,15 +172,16 @@ class SegWitTest(BitcoinTestFramework):
         self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V1][0], False, witness_script(True, self.pubkey[0]))
 
         self.log.info("Verify block and transaction serialization rpcs return differing serializations depending on rpc serialization flag")
-        assert(self.nodes[2].getblock(block[0], False) != self.nodes[0].getblock(block[0], False))
-        assert(self.nodes[1].getblock(block[0], False) == self.nodes[2].getblock(block[0], False))
-        for i in range(len(segwit_tx_list)):
-            tx = FromHex(CTransaction(), self.nodes[2].gettransaction(segwit_tx_list[i])["hex"])
-            assert(self.nodes[2].getrawtransaction(segwit_tx_list[i]) != self.nodes[0].getrawtransaction(segwit_tx_list[i]))
-            assert(self.nodes[1].getrawtransaction(segwit_tx_list[i], 0) == self.nodes[2].getrawtransaction(segwit_tx_list[i]))
-            assert(self.nodes[0].getrawtransaction(segwit_tx_list[i]) != self.nodes[2].gettransaction(segwit_tx_list[i])["hex"])
-            assert(self.nodes[1].getrawtransaction(segwit_tx_list[i]) == self.nodes[2].gettransaction(segwit_tx_list[i])["hex"])
-            assert(self.nodes[0].getrawtransaction(segwit_tx_list[i]) == bytes_to_hex_str(tx.serialize_without_witness()))
+        assert(self.nodes[2].getblock(blockhash, False) != self.nodes[0].getblock(blockhash, False))
+        assert(self.nodes[1].getblock(blockhash, False) == self.nodes[2].getblock(blockhash, False))
+
+        for tx_id in segwit_tx_list:
+            tx = FromHex(CTransaction(), self.nodes[2].gettransaction(tx_id)["hex"])
+            assert(self.nodes[2].getrawtransaction(tx_id, False, blockhash) != self.nodes[0].getrawtransaction(tx_id, False, blockhash))
+            assert(self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].getrawtransaction(tx_id, False, blockhash))
+            assert(self.nodes[0].getrawtransaction(tx_id, False, blockhash) != self.nodes[2].gettransaction(tx_id)["hex"])
+            assert(self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].gettransaction(tx_id)["hex"])
+            assert(self.nodes[0].getrawtransaction(tx_id, False, blockhash) == bytes_to_hex_str(tx.serialize_without_witness()))
 
         self.log.info("Verify witness txs without witness data are invalid after the fork")
         self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch) (code 64)', wit_ids[NODE_2][WIT_V0][2], sign=False)
@@ -232,15 +235,7 @@ class SegWitTest(BitcoinTestFramework):
         assert(tx.wit.is_null())
         assert(txid3 in self.nodes[0].getrawmempool())
 
-        # Now try calling getblocktemplate() without segwit support.
-        template = self.nodes[0].getblocktemplate()
-
-        # Check that tx1 is the only transaction of the 3 in the template.
-        template_txids = [t['txid'] for t in template['transactions']]
-        assert(txid2 not in template_txids and txid3 not in template_txids)
-        assert(txid1 in template_txids)
-
-        # Check that running with segwit support results in all 3 being included.
+        # Check that getblocktemplate includes all transactions.
         template = self.nodes[0].getblocktemplate({"rules": ["segwit"]})
         template_txids = [t['txid'] for t in template['transactions']]
         assert(txid1 in template_txids)
@@ -542,7 +537,7 @@ class SegWitTest(BitcoinTestFramework):
         tx.rehash()
         signresults = self.nodes[0].signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize_without_witness()))['hex']
         txid = self.nodes[0].sendrawtransaction(signresults, True)
-        self.nodes[0].generate(1)
+        txs_mined[txid] = self.nodes[0].generate(1)[0]
         sync_blocks(self.nodes)
         watchcount = 0
         spendcount = 0
@@ -585,7 +580,7 @@ class SegWitTest(BitcoinTestFramework):
         tx = CTransaction()
         for i in txids:
             txtmp = CTransaction()
-            txraw = self.nodes[0].getrawtransaction(i)
+            txraw = self.nodes[0].getrawtransaction(i, 0, txs_mined[i])
             f = BytesIO(hex_str_to_bytes(txraw))
             txtmp.deserialize(f)
             for j in range(len(txtmp.vout)):

test/bitcoin_functional/functional/feature_shutdown.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test bitcoind shutdown."""
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal, get_rpc_proxy, wait_until
+from threading import Thread
+
+def test_long_call(node):
+    block = node.waitfornewblock()
+    assert_equal(block['height'], 0)
+
+class ShutdownTest(BitcoinTestFramework):
+
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.num_nodes = 1
+
+    def run_test(self):
+        node = get_rpc_proxy(self.nodes[0].url, 1, timeout=600, coveragedir=self.nodes[0].coverage_dir)
+        # Force connection establishment by executing a dummy command.
+        node.getblockcount()
+        Thread(target=test_long_call, args=(node,)).start()
+        # Wait until the server is executing the above `waitfornewblock`.
+        wait_until(lambda: len(self.nodes[0].getrpcinfo()['active_commands']) == 2)
+        # Wait 1 second after requesting shutdown but not before the `stop` call
+        # finishes. This is to ensure event loop waits for current connections
+        # to close.
+        self.stop_node(0, wait=1000)
+
+if __name__ == '__main__':
+    ShutdownTest().main()

test/bitcoin_functional/functional/interface_http.py

@@ -31,13 +31,13 @@ class HTTPBasicsTest (BitcoinTestFramework):
         conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
         out1 = conn.getresponse().read()
         assert(b'"error":null' in out1)
-        assert(conn.sock!=None) #according to http/1.1 connection must still be open!
+        assert(conn.sock is not None) #according to http/1.1 connection must still be open!
 
         #send 2nd request without closing connection
         conn.request('POST', '/', '{"method": "getchaintips"}', headers)
         out1 = conn.getresponse().read()
         assert(b'"error":null' in out1) #must also response with a correct json-rpc message
-        assert(conn.sock!=None) #according to http/1.1 connection must still be open!
+        assert(conn.sock is not None) #according to http/1.1 connection must still be open!
         conn.close()
 
         #same should be if we add keep-alive because this should be the std. behaviour
@@ -48,13 +48,13 @@ class HTTPBasicsTest (BitcoinTestFramework):
         conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
         out1 = conn.getresponse().read()
         assert(b'"error":null' in out1)
-        assert(conn.sock!=None) #according to http/1.1 connection must still be open!
+        assert(conn.sock is not None) #according to http/1.1 connection must still be open!
 
         #send 2nd request without closing connection
         conn.request('POST', '/', '{"method": "getchaintips"}', headers)
         out1 = conn.getresponse().read()
         assert(b'"error":null' in out1) #must also response with a correct json-rpc message
-        assert(conn.sock!=None) #according to http/1.1 connection must still be open!
+        assert(conn.sock is not None) #according to http/1.1 connection must still be open!
         conn.close()
 
         #now do the same with "Connection: close"
@@ -65,7 +65,7 @@ class HTTPBasicsTest (BitcoinTestFramework):
         conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
         out1 = conn.getresponse().read()
         assert(b'"error":null' in out1)
-        assert(conn.sock==None) #now the connection must be closed after the response
+        assert(conn.sock is None) #now the connection must be closed after the response
 
         #node1 (2nd node) is running with disabled keep-alive option
         urlNode1 = urllib.parse.urlparse(self.nodes[1].url)
@@ -88,7 +88,7 @@ class HTTPBasicsTest (BitcoinTestFramework):
         conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
         out1 = conn.getresponse().read()
         assert(b'"error":null' in out1)
-        assert(conn.sock!=None) #connection must be closed because bitcoind should use keep-alive by default
+        assert(conn.sock is not None) #connection must be closed because bitcoind should use keep-alive by default
 
         # Check excessive request size
         conn = http.client.HTTPConnection(urlNode2.hostname, urlNode2.port)

test/bitcoin_functional/functional/interface_rest.py

@@ -22,6 +22,8 @@ from test_framework.util import (
     hex_str_to_bytes,
 )
 
+from test_framework.messages import BLOCK_HEADER_SIZE
+
 class ReqType(Enum):
     JSON = 1
     BIN = 2
@@ -88,15 +90,17 @@ class RESTTest (BitcoinTestFramework):
 
         txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1)
         self.sync_all()
-        self.nodes[1].generatetoaddress(1, not_related_address)
-        self.sync_all()
-        bb_hash = self.nodes[0].getbestblockhash()
 
-        assert_equal(self.nodes[1].getbalance(), Decimal("0.1"))
-
-        self.log.info("Load the transaction using the /tx URI")
+        self.log.info("Test the /tx URI")
 
         json_obj = self.test_rest_request("/tx/{}".format(txid))
+        assert_equal(json_obj['txid'], txid)
+
+        # Check hex format response
+        hex_response = self.test_rest_request("/tx/{}".format(txid), req_type=ReqType.HEX, ret_type=RetType.OBJ)
+        assert_greater_than_or_equal(int(hex_response.getheader('content-length')),
+                                     json_obj['size']*2)
+
         spent = (json_obj['vin'][0]['txid'], json_obj['vin'][0]['vout'])  # get the vin to later check for utxo (should be spent by then)
         # get n of 0.1 outpoint
         n, = filter_output_indices_by_value(json_obj['vout'], Decimal('0.1'))
@@ -104,9 +108,14 @@ class RESTTest (BitcoinTestFramework):
 
         self.log.info("Query an unspent TXO using the /getutxos URI")
 
-        json_obj = self.test_rest_request("/getutxos/{}-{}".format(*spending))
+        self.nodes[1].generatetoaddress(1, not_related_address)
+        self.sync_all()
+        bb_hash = self.nodes[0].getbestblockhash()
+
+        assert_equal(self.nodes[1].getbalance(), Decimal("0.1"))
 
         # Check chainTip response
+        json_obj = self.test_rest_request("/getutxos/{}-{}".format(*spending))
         assert_equal(json_obj['chaintipHash'], bb_hash)
 
         # Make sure there is one utxo
@@ -198,35 +207,62 @@ class RESTTest (BitcoinTestFramework):
         self.nodes[0].generate(1)  # generate block to not affect upcoming tests
         self.sync_all()
 
-        self.log.info("Test the /block and /headers URIs")
+        self.log.info("Test the /block, /blockhashbyheight and /headers URIs")
         bb_hash = self.nodes[0].getbestblockhash()
 
+        # Check result if block does not exists
+        assert_equal(self.test_rest_request('/headers/1/0000000000000000000000000000000000000000000000000000000000000000'), [])
+        self.test_rest_request('/block/0000000000000000000000000000000000000000000000000000000000000000', status=404, ret_type=RetType.OBJ)
+
+        # Check result if block is not in the active chain
+        self.nodes[0].invalidateblock(bb_hash)
+        assert_equal(self.test_rest_request('/headers/1/{}'.format(bb_hash)), [])
+        self.test_rest_request('/block/{}'.format(bb_hash))
+        self.nodes[0].reconsiderblock(bb_hash)
+
         # Check binary format
         response = self.test_rest_request("/block/{}".format(bb_hash), req_type=ReqType.BIN, ret_type=RetType.OBJ)
-        assert_greater_than(int(response.getheader('content-length')), 80)
+        assert_greater_than(int(response.getheader('content-length')), BLOCK_HEADER_SIZE)
         response_bytes = response.read()
 
         # Compare with block header
         response_header = self.test_rest_request("/headers/1/{}".format(bb_hash), req_type=ReqType.BIN, ret_type=RetType.OBJ)
-        assert_equal(int(response_header.getheader('content-length')), 80)
+        assert_equal(int(response_header.getheader('content-length')), BLOCK_HEADER_SIZE)
         response_header_bytes = response_header.read()
-        assert_equal(response_bytes[:80], response_header_bytes)
+        assert_equal(response_bytes[:BLOCK_HEADER_SIZE], response_header_bytes)
 
         # Check block hex format
         response_hex = self.test_rest_request("/block/{}".format(bb_hash), req_type=ReqType.HEX, ret_type=RetType.OBJ)
-        assert_greater_than(int(response_hex.getheader('content-length')), 160)
+        assert_greater_than(int(response_hex.getheader('content-length')), BLOCK_HEADER_SIZE*2)
         response_hex_bytes = response_hex.read().strip(b'\n')
         assert_equal(binascii.hexlify(response_bytes), response_hex_bytes)
 
         # Compare with hex block header
         response_header_hex = self.test_rest_request("/headers/1/{}".format(bb_hash), req_type=ReqType.HEX, ret_type=RetType.OBJ)
-        assert_greater_than(int(response_header_hex.getheader('content-length')), 160)
-        response_header_hex_bytes = response_header_hex.read(160)
-        assert_equal(binascii.hexlify(response_bytes[:80]), response_header_hex_bytes)
+        assert_greater_than(int(response_header_hex.getheader('content-length')), BLOCK_HEADER_SIZE*2)
+        response_header_hex_bytes = response_header_hex.read(BLOCK_HEADER_SIZE*2)
+        assert_equal(binascii.hexlify(response_bytes[:BLOCK_HEADER_SIZE]), response_header_hex_bytes)
 
         # Check json format
         block_json_obj = self.test_rest_request("/block/{}".format(bb_hash))
         assert_equal(block_json_obj['hash'], bb_hash)
+        assert_equal(self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']))['blockhash'], bb_hash)
+
+        # Check hex/bin format
+        resp_hex = self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']), req_type=ReqType.HEX, ret_type=RetType.OBJ)
+        assert_equal(resp_hex.read().decode('utf-8').rstrip(), bb_hash)
+        resp_bytes = self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']), req_type=ReqType.BIN, ret_type=RetType.BYTES)
+        blockhash = binascii.hexlify(resp_bytes[::-1]).decode('utf-8')
+        assert_equal(blockhash, bb_hash)
+
+        # Check invalid blockhashbyheight requests
+        resp = self.test_rest_request("/blockhashbyheight/abc", ret_type=RetType.OBJ, status=400)
+        assert_equal(resp.read().decode('utf-8').rstrip(), "Invalid height: abc")
+        resp = self.test_rest_request("/blockhashbyheight/1000000", ret_type=RetType.OBJ, status=404)
+        assert_equal(resp.read().decode('utf-8').rstrip(), "Block height out of range")
+        resp = self.test_rest_request("/blockhashbyheight/-1", ret_type=RetType.OBJ, status=400)
+        assert_equal(resp.read().decode('utf-8').rstrip(), "Invalid height: -1")
+        self.test_rest_request("/blockhashbyheight/", ret_type=RetType.OBJ, status=400)
 
         # Compare with json block header
         json_obj = self.test_rest_request("/headers/1/{}".format(bb_hash))
@@ -244,17 +280,6 @@ class RESTTest (BitcoinTestFramework):
         json_obj = self.test_rest_request("/headers/5/{}".format(bb_hash))
         assert_equal(len(json_obj), 5)  # now we should have 5 header objects
 
-        self.log.info("Test the /tx URI")
-
-        tx_hash = block_json_obj['tx'][0]['txid']
-        json_obj = self.test_rest_request("/tx/{}".format(tx_hash))
-        assert_equal(json_obj['txid'], tx_hash)
-
-        # Check hex format response
-        hex_response = self.test_rest_request("/tx/{}".format(tx_hash), req_type=ReqType.HEX, ret_type=RetType.OBJ)
-        assert_greater_than_or_equal(int(hex_response.getheader('content-length')),
-                                     json_obj['size']*2)
-
         self.log.info("Test tx inclusion in the /mempool and /block URIs")
 
         # Make 3 tx and mine them on node 1

test/bitcoin_functional/functional/interface_rpc.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Tests some generic aspects of the RPC interface."""
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal, assert_greater_than_or_equal
+
+class RPCInterfaceTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 1
+        self.setup_clean_chain = True
+
+    def test_getrpcinfo(self):
+        self.log.info("Testing getrpcinfo...")
+
+        info = self.nodes[0].getrpcinfo()
+        assert_equal(len(info['active_commands']), 1)
+
+        command = info['active_commands'][0]
+        assert_equal(command['method'], 'getrpcinfo')
+        assert_greater_than_or_equal(command['duration'], 0)
+
+    def test_batch_request(self):
+        self.log.info("Testing basic JSON-RPC batch request...")
+
+        results = self.nodes[0].batch([
+            # A basic request that will work fine.
+            {"method": "getblockcount", "id": 1},
+            # Request that will fail.  The whole batch request should still
+            # work fine.
+            {"method": "invalidmethod", "id": 2},
+            # Another call that should succeed.
+            {"method": "getbestblockhash", "id": 3},
+        ])
+
+        result_by_id = {}
+        for res in results:
+            result_by_id[res["id"]] = res
+
+        assert_equal(result_by_id[1]['error'], None)
+        assert_equal(result_by_id[1]['result'], 0)
+
+        assert_equal(result_by_id[2]['error']['code'], -32601)
+        assert_equal(result_by_id[2]['result'], None)
+
+        assert_equal(result_by_id[3]['error'], None)
+        assert result_by_id[3]['result'] is not None
+
+    def run_test(self):
+        self.test_getrpcinfo()
+        self.test_batch_request()
+
+
+if __name__ == '__main__':
+    RPCInterfaceTest().main()

test/bitcoin_functional/functional/mempool_accept.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
-# Copyright (c) 2017 The Bitcoin Core developers
+# Copyright (c) 2017-2018 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test mempool acceptance of raw transactions."""
 
 from io import BytesIO
+import math
+
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.messages import (
     BIP125_SEQUENCE_NUMBER,
@@ -56,6 +58,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework):
         self.mempool_size = 0
         wait_until(lambda: node.getblockcount() == 200)
         assert_equal(node.getmempoolinfo()['size'], self.mempool_size)
+        coins = node.listunspent()
 
         self.log.info('Should not accept garbage to testmempoolaccept')
         assert_raises_rpc_error(-3, 'Expected type array, got string', lambda: node.testmempoolaccept(rawtxs='ff00baar'))
@@ -63,13 +66,14 @@ class MempoolAcceptanceTest(BitcoinTestFramework):
         assert_raises_rpc_error(-22, 'TX decode failed', lambda: node.testmempoolaccept(rawtxs=['ff00baar']))
 
         self.log.info('A transaction already in the blockchain')
-        coin = node.listunspent()[0]  # Pick a random coin(base) to spend
+        coin = coins.pop()  # Pick a random coin(base) to spend
         raw_tx_in_block = node.signrawtransactionwithwallet(node.createrawtransaction(
             inputs=[{'txid': coin['txid'], 'vout': coin['vout']}],
             outputs=[{node.getnewaddress(): 0.3}, {node.getnewaddress(): 49}],
         ))['hex']
         txid_in_block = node.sendrawtransaction(hexstring=raw_tx_in_block, allowhighfees=True)
         node.generate(1)
+        self.mempool_size = 0
         self.check_mempool_result(
             result_expected=[{'txid': txid_in_block, 'allowed': False, 'reject-reason': '18: txn-already-known'}],
             rawtxs=[raw_tx_in_block],
@@ -89,9 +93,25 @@ class MempoolAcceptanceTest(BitcoinTestFramework):
             rawtxs=[raw_tx_0],
         )
 
+        self.log.info('A final transaction not in the mempool')
+        coin = coins.pop()  # Pick a random coin(base) to spend
+        raw_tx_final = node.signrawtransactionwithwallet(node.createrawtransaction(
+            inputs=[{'txid': coin['txid'], 'vout': coin['vout'], "sequence": 0xffffffff}],  # SEQUENCE_FINAL
+            outputs=[{node.getnewaddress(): 0.025}],
+            locktime=node.getblockcount() + 2000,  # Can be anything
+        ))['hex']
+        tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_final)))
+        self.check_mempool_result(
+            result_expected=[{'txid': tx.rehash(), 'allowed': True}],
+            rawtxs=[bytes_to_hex_str(tx.serialize())],
+            allowhighfees=True,
+        )
+        node.sendrawtransaction(hexstring=raw_tx_final, allowhighfees=True)
+        self.mempool_size += 1
+
         self.log.info('A transaction in the mempool')
         node.sendrawtransaction(hexstring=raw_tx_0)
-        self.mempool_size = 1
+        self.mempool_size += 1
         self.check_mempool_result(
             result_expected=[{'txid': txid_0, 'allowed': False, 'reject-reason': '18: txn-already-in-mempool'}],
             rawtxs=[raw_tx_0],
@@ -181,7 +201,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework):
 
         self.log.info('A really large transaction')
         tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference)))
-        tx.vin = [tx.vin[0]] * (MAX_BLOCK_BASE_SIZE // len(tx.vin[0].serialize()))
+        tx.vin = [tx.vin[0]] * math.ceil(MAX_BLOCK_BASE_SIZE / len(tx.vin[0].serialize()))
         self.check_mempool_result(
             result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-oversize'}],
             rawtxs=[bytes_to_hex_str(tx.serialize())],

test/bitcoin_functional/functional/mempool_persist.py

@@ -42,6 +42,7 @@ import time
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal, assert_raises_rpc_error, wait_until
 
+
 class MempoolPersistTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 3
@@ -60,7 +61,7 @@ class MempoolPersistTest(BitcoinTestFramework):
 
         self.log.debug("Send 5 transactions from node2 (to its own address)")
         for i in range(5):
-            self.nodes[2].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("10"))
+            last_txid = self.nodes[2].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("10"))
         node2_balance = self.nodes[2].getbalance()
         self.sync_all()
 
@@ -68,6 +69,13 @@ class MempoolPersistTest(BitcoinTestFramework):
         assert_equal(len(self.nodes[0].getrawmempool()), 5)
         assert_equal(len(self.nodes[1].getrawmempool()), 5)
 
+        self.log.debug("Prioritize a transaction on node0")
+        fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees']
+        assert_equal(fees['base'], fees['modified'])
+        self.nodes[0].prioritisetransaction(txid=last_txid, fee_delta=1000)
+        fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees']
+        assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified'])
+
         self.log.debug("Stop-start the nodes. Verify that node0 has the transactions in its mempool and node1 does not. Verify that node2 calculates its balance correctly after loading wallet transactions.")
         self.stop_nodes()
         # Give this node a head-start, so we can be "extra-sure" that it didn't load anything later
@@ -81,6 +89,10 @@ class MempoolPersistTest(BitcoinTestFramework):
         # The others have loaded their mempool. If node_1 loaded anything, we'd probably notice by now:
         assert_equal(len(self.nodes[1].getrawmempool()), 0)
 
+        self.log.debug('Verify prioritization is loaded correctly')
+        fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees']
+        assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified'])
+
         # Verify accounting of mempool transactions after restart is correct
         self.nodes[2].syncwithvalidationinterfacequeue()  # Flush mempool to wallet
         assert_equal(node2_balance, self.nodes[2].getbalance())

test/bitcoin_functional/functional/mining_basic.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2014-2018 The Bitcoin Core developers
+# Copyright (c) 2014-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test mining RPCs
@@ -11,10 +11,14 @@
 import copy
 from decimal import Decimal
 
-from test_framework.blocktools import create_coinbase
+from test_framework.blocktools import (
+    create_coinbase,
+    TIME_GENESIS_BLOCK,
+)
 from test_framework.messages import (
     CBlock,
     CBlockHeader,
+    BLOCK_HEADER_SIZE
 )
 from test_framework.mininode import (
     P2PDataStore,
@@ -24,21 +28,37 @@ from test_framework.util import (
     assert_equal,
     assert_raises_rpc_error,
     bytes_to_hex_str as b2x,
+    connect_nodes_bi,
 )
+from test_framework.script import CScriptNum
 
 
 def assert_template(node, block, expect, rehash=True):
     if rehash:
         block.hashMerkleRoot = block.calc_merkle_root()
-    rsp = node.getblocktemplate({'data': b2x(block.serialize()), 'mode': 'proposal'})
+    rsp = node.getblocktemplate(template_request={'data': b2x(block.serialize()), 'mode': 'proposal', 'rules': ['segwit']})
     assert_equal(rsp, expect)
 
+
 class MiningTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 2
-        self.setup_clean_chain = False
+        self.setup_clean_chain = True
+
+    def mine_chain(self):
+        self.log.info('Create some old blocks')
+        for t in range(TIME_GENESIS_BLOCK, TIME_GENESIS_BLOCK + 200 * 600, 600):
+            self.nodes[0].setmocktime(t)
+            self.nodes[0].generate(1)
+        mining_info = self.nodes[0].getmininginfo()
+        assert_equal(mining_info['blocks'], 200)
+        assert_equal(mining_info['currentblocktx'], 0)
+        assert_equal(mining_info['currentblockweight'], 4000)
+        self.restart_node(0)
+        connect_nodes_bi(self.nodes, 0, 1)
 
     def run_test(self):
+        self.mine_chain()
         node = self.nodes[0]
 
         def assert_submitblock(block, result_str_1, result_str_2=None):
@@ -51,24 +71,32 @@ class MiningTest(BitcoinTestFramework):
         mining_info = node.getmininginfo()
         assert_equal(mining_info['blocks'], 200)
         assert_equal(mining_info['chain'], 'regtest')
-        assert_equal(mining_info['currentblocktx'], 0)
-        assert_equal(mining_info['currentblockweight'], 0)
+        assert 'currentblocktx' not in mining_info
+        assert 'currentblockweight' not in mining_info
         assert_equal(mining_info['difficulty'], Decimal('4.656542373906925E-10'))
         assert_equal(mining_info['networkhashps'], Decimal('0.003333333333333334'))
         assert_equal(mining_info['pooledtx'], 0)
 
         # Mine a block to leave initial block download
         node.generatetoaddress(1, node.get_deterministic_priv_key().address)
-        tmpl = node.getblocktemplate()
+        tmpl = node.getblocktemplate({'rules': ['segwit']})
         self.log.info("getblocktemplate: Test capability advertised")
         assert 'proposal' in tmpl['capabilities']
         assert 'coinbasetxn' not in tmpl
 
-        coinbase_tx = create_coinbase(height=int(tmpl["height"]) + 1)
+        next_height = int(tmpl["height"])
+        coinbase_tx = create_coinbase(height=next_height)
         # sequence numbers must not be max for nLockTime to have effect
         coinbase_tx.vin[0].nSequence = 2 ** 32 - 2
         coinbase_tx.rehash()
 
+        # round-trip the encoded bip34 block height commitment
+        assert_equal(CScriptNum.decode(coinbase_tx.vin[0].scriptSig), next_height)
+        # round-trip negative and multi-byte CScriptNums to catch python regression
+        assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(1500))), 1500)
+        assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(-1500))), -1500)
+        assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(-1))), -1)
+
         block = CBlock()
         block.nVersion = tmpl["version"]
         block.hashPrevBlock = int(tmpl["previousblockhash"], 16)
@@ -77,6 +105,9 @@ class MiningTest(BitcoinTestFramework):
         block.nNonce = 0
         block.vtx = [coinbase_tx]
 
+        self.log.info("getblocktemplate: segwit rule must be set")
+        assert_raises_rpc_error(-8, "getblocktemplate must be called with the segwit rule set", node.getblocktemplate)
+
         self.log.info("getblocktemplate: Test valid block")
         assert_template(node, block, None)
 
@@ -93,7 +124,7 @@ class MiningTest(BitcoinTestFramework):
         assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, b2x(bad_block.serialize()))
 
         self.log.info("getblocktemplate: Test truncated final transaction")
-        assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(block.serialize()[:-1]), 'mode': 'proposal'})
+        assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(block.serialize()[:-1]), 'mode': 'proposal', 'rules': ['segwit']})
 
         self.log.info("getblocktemplate: Test duplicate transaction")
         bad_block = copy.deepcopy(block)
@@ -119,11 +150,10 @@ class MiningTest(BitcoinTestFramework):
 
         self.log.info("getblocktemplate: Test bad tx count")
         # The tx count is immediately after the block header
-        TX_COUNT_OFFSET = 80
         bad_block_sn = bytearray(block.serialize())
-        assert_equal(bad_block_sn[TX_COUNT_OFFSET], 1)
-        bad_block_sn[TX_COUNT_OFFSET] += 1
-        assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(bad_block_sn), 'mode': 'proposal'})
+        assert_equal(bad_block_sn[BLOCK_HEADER_SIZE], 1)
+        bad_block_sn[BLOCK_HEADER_SIZE] += 1
+        assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(bad_block_sn), 'mode': 'proposal', 'rules': ['segwit']})
 
         self.log.info("getblocktemplate: Test bad bits")
         bad_block = copy.deepcopy(block)
@@ -152,9 +182,9 @@ class MiningTest(BitcoinTestFramework):
         assert_submitblock(bad_block, 'prev-blk-not-found', 'prev-blk-not-found')
 
         self.log.info('submitheader tests')
-        assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='xx' * 80))
-        assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='ff' * 78))
-        assert_raises_rpc_error(-25, 'Must submit previous header', lambda: node.submitheader(hexdata='ff' * 80))
+        assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='xx' * BLOCK_HEADER_SIZE))
+        assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='ff' * (BLOCK_HEADER_SIZE-2)))
+        assert_raises_rpc_error(-25, 'Must submit previous header', lambda: node.submitheader(hexdata=b2x(super(CBlock, bad_block).serialize())))
 
         block.nTime += 1
         block.solve()

test/bitcoin_functional/functional/mining_getblocktemplate_longpoll.py

@@ -15,14 +15,14 @@ class LongpollThread(threading.Thread):
     def __init__(self, node):
         threading.Thread.__init__(self)
         # query current longpollid
-        template = node.getblocktemplate()
+        template = node.getblocktemplate({'rules': ['segwit']})
         self.longpollid = template['longpollid']
         # create a new connection to the node, we can't use the same
         # connection from two threads
         self.node = get_rpc_proxy(node.url, 1, timeout=600, coveragedir=node.coverage_dir)
 
     def run(self):
-        self.node.getblocktemplate({'longpollid':self.longpollid})
+        self.node.getblocktemplate({'longpollid': self.longpollid, 'rules': ['segwit']})
 
 class GetBlockTemplateLPTest(BitcoinTestFramework):
     def set_test_params(self):
@@ -34,10 +34,10 @@ class GetBlockTemplateLPTest(BitcoinTestFramework):
     def run_test(self):
         self.log.info("Warning: this test will take about 70 seconds in the best case. Be patient.")
         self.nodes[0].generate(10)
-        template = self.nodes[0].getblocktemplate()
+        template = self.nodes[0].getblocktemplate({'rules': ['segwit']})
         longpollid = template['longpollid']
         # longpollid should not change between successive invocations if nothing else happens
-        template2 = self.nodes[0].getblocktemplate()
+        template2 = self.nodes[0].getblocktemplate({'rules': ['segwit']})
         assert(template2['longpollid'] == longpollid)
 
         # Test 1: test that the longpolling wait if we do nothing

test/bitcoin_functional/functional/mining_prioritisetransaction.py

@@ -84,7 +84,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework):
                 high_fee_tx = x
 
         # Something high-fee should have been mined!
-        assert(high_fee_tx != None)
+        assert(high_fee_tx is not None)
 
         # Add a prioritisation before a tx is in the mempool (de-prioritising a
         # high-fee transaction so that it's now low fee).
@@ -142,10 +142,10 @@ class PrioritiseTransactionTest(BitcoinTestFramework):
         # getblocktemplate to (eventually) return a new block.
         mock_time = int(time.time())
         self.nodes[0].setmocktime(mock_time)
-        template = self.nodes[0].getblocktemplate()
+        template = self.nodes[0].getblocktemplate({'rules': ['segwit']})
         self.nodes[0].prioritisetransaction(txid=tx_id, fee_delta=-int(self.relayfee*COIN))
         self.nodes[0].setmocktime(mock_time+10)
-        new_template = self.nodes[0].getblocktemplate()
+        new_template = self.nodes[0].getblocktemplate({'rules': ['segwit']})
 
         assert(template != new_template)
 

test/bitcoin_functional/functional/p2p_disconnect_ban.py

@@ -22,7 +22,7 @@ class DisconnectBanTest(BitcoinTestFramework):
 
         self.log.info("setban: successfully ban single IP address")
         assert_equal(len(self.nodes[1].getpeerinfo()), 2)  # node1 should have 2 connections to node0 at this point
-        self.nodes[1].setban("127.0.0.1", "add")
+        self.nodes[1].setban(subnet="127.0.0.1", command="add")
         wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 0, timeout=10)
         assert_equal(len(self.nodes[1].getpeerinfo()), 0)  # all nodes must be disconnected at this point
         assert_equal(len(self.nodes[1].listbanned()), 1)

test/bitcoin_functional/functional/p2p_invalid_locator.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2015-2017 The Bitcoin Core developers
+# Copyright (c) 2015-2018 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test node responses to invalid locators.

test/bitcoin_functional/functional/p2p_invalid_messages.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python3
-# Copyright (c) 2015-2018 The Bitcoin Core developers
+# Copyright (c) 2015-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test node responses to invalid network messages."""
+import os
 import struct
 
 from test_framework import messages
@@ -15,7 +16,7 @@ class msg_unrecognized:
 
     command = b'badmsg'
 
-    def __init__(self, str_data):
+    def __init__(self, *, str_data):
         self.str_data = str_data.encode() if not isinstance(str_data, bytes) else str_data
 
     def serialize(self):
@@ -25,19 +26,14 @@ class msg_unrecognized:
         return "{}(data={})".format(self.command, self.str_data)
 
 
-class msg_nametoolong(msg_unrecognized):
-
-    command = b'thisnameiswayyyyyyyyytoolong'
-
-
 class InvalidMessagesTest(BitcoinTestFramework):
-
     def set_test_params(self):
         self.num_nodes = 1
         self.setup_clean_chain = True
 
     def run_test(self):
         """
+         . Test msg header
         0. Send a bunch of large (4MB) messages of an unrecognized type. Check to see
            that it isn't an effective DoS against the node.
 
@@ -45,10 +41,12 @@ class InvalidMessagesTest(BitcoinTestFramework):
 
         2. Send a few messages with an incorrect data size in the header, ensure the
            messages are ignored.
-
-        3. Send an unrecognized message with a command name longer than 12 characters.
-
         """
+        self.test_magic_bytes()
+        self.test_checksum()
+        self.test_size()
+        self.test_command()
+
         node = self.nodes[0]
         self.node = node
         node.add_p2p_connection(P2PDataStore())
@@ -63,10 +61,13 @@ class InvalidMessagesTest(BitcoinTestFramework):
         # Send as large a message as is valid, ensure we aren't disconnected but
         # also can't exhaust resources.
         #
-        msg_at_size = msg_unrecognized("b" * valid_data_limit)
+        msg_at_size = msg_unrecognized(str_data="b" * valid_data_limit)
         assert len(msg_at_size.serialize()) == msg_limit
 
-        with node.assert_memory_usage_stable(perc_increase_allowed=0.03):
+        increase_allowed = 0.5
+        if [s for s in os.environ.get("BITCOIN_CONFIG", "").split(" ") if "--with-sanitizers" in s and "address" in s]:
+            increase_allowed = 3.5
+        with node.assert_memory_usage_stable(increase_allowed=increase_allowed):
             self.log.info(
                 "Sending a bunch of large, junk messages to test "
                 "memory exhaustion. May take a bit...")
@@ -82,7 +83,7 @@ class InvalidMessagesTest(BitcoinTestFramework):
 
             # Peer 1, despite serving up a bunch of nonsense, should still be connected.
             self.log.info("Waiting for node to drop junk messages.")
-            node.p2p.sync_with_ping(timeout=30)
+            node.p2p.sync_with_ping(timeout=120)
             assert node.p2p.is_connected
 
         #
@@ -90,10 +91,10 @@ class InvalidMessagesTest(BitcoinTestFramework):
         #
         # Send an oversized message, ensure we're disconnected.
         #
-        msg_over_size = msg_unrecognized("b" * (valid_data_limit + 1))
+        msg_over_size = msg_unrecognized(str_data="b" * (valid_data_limit + 1))
         assert len(msg_over_size.serialize()) == (msg_limit + 1)
 
-        with node.assert_debug_log(["Oversized message from peer=0, disconnecting"]):
+        with node.assert_debug_log(["Oversized message from peer=4, disconnecting"]):
             # An unknown message type (or *any* message type) over
             # MAX_PROTOCOL_MESSAGE_LENGTH should result in a disconnect.
             node.p2p.send_message(msg_over_size)
@@ -109,7 +110,7 @@ class InvalidMessagesTest(BitcoinTestFramework):
         # Send messages with an incorrect data size in the header.
         #
         actual_size = 100
-        msg = msg_unrecognized("b" * actual_size)
+        msg = msg_unrecognized(str_data="b" * actual_size)
 
         # TODO: handle larger-than cases. I haven't been able to pin down what behavior to expect.
         for wrong_size in (2, 77, 78, 79):
@@ -136,18 +137,59 @@ class InvalidMessagesTest(BitcoinTestFramework):
             node.disconnect_p2ps()
             node.add_p2p_connection(P2PDataStore())
 
-        #
-        # 3.
-        #
-        # Send a message with a too-long command name.
-        #
-        node.p2p.send_message(msg_nametoolong("foobar"))
-        node.p2p.wait_for_disconnect(timeout=4)
-
         # Node is still up.
         conn = node.add_p2p_connection(P2PDataStore())
         conn.sync_with_ping()
 
+    def test_magic_bytes(self):
+        conn = self.nodes[0].add_p2p_connection(P2PDataStore())
+        conn._on_data = lambda: None  # Need to ignore all incoming messages from now, since they come with "invalid" magic bytes
+        conn.magic_bytes = b'\x00\x11\x22\x32'
+        with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: INVALID MESSAGESTART ping']):
+            conn.send_message(messages.msg_ping(nonce=0xff))
+            conn.wait_for_disconnect(timeout=1)
+            self.nodes[0].disconnect_p2ps()
+
+    def test_checksum(self):
+        conn = self.nodes[0].add_p2p_connection(P2PDataStore())
+        with self.nodes[0].assert_debug_log(['ProcessMessages(badmsg, 2 bytes): CHECKSUM ERROR expected 78df0a04 was ffffffff']):
+            msg = conn.build_message(msg_unrecognized(str_data="d"))
+            cut_len = (
+                4 +  # magic
+                12 +  # command
+                4  #len
+            )
+            # modify checksum
+            msg = msg[:cut_len] + b'\xff' * 4 + msg[cut_len + 4:]
+            self.nodes[0].p2p.send_raw_message(msg)
+            conn.sync_with_ping(timeout=1)
+            self.nodes[0].disconnect_p2ps()
+
+    def test_size(self):
+        conn = self.nodes[0].add_p2p_connection(P2PDataStore())
+        with self.nodes[0].assert_debug_log(['']):
+            msg = conn.build_message(msg_unrecognized(str_data="d"))
+            cut_len = (
+                4 +  # magic
+                12  # command
+            )
+            # modify len to MAX_SIZE + 1
+            msg = msg[:cut_len] + struct.pack("<I", 0x02000000 + 1) + msg[cut_len + 4:]
+            self.nodes[0].p2p.send_raw_message(msg)
+            conn.wait_for_disconnect(timeout=1)
+            self.nodes[0].disconnect_p2ps()
+
+    def test_command(self):
+        conn = self.nodes[0].add_p2p_connection(P2PDataStore())
+        with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: ERRORS IN HEADER']):
+            msg = msg_unrecognized(str_data="d")
+            msg.command = b'\xff' * 12
+            msg = conn.build_message(msg)
+            # Modify command
+            msg = msg[:7] + b'\x00' + msg[7 + 1:]
+            self.nodes[0].p2p.send_raw_message(msg)
+            conn.sync_with_ping(timeout=1)
+            self.nodes[0].disconnect_p2ps()
 
     def _tweak_msg_data_size(self, message, wrong_size):
         """
@@ -170,6 +212,5 @@ class InvalidMessagesTest(BitcoinTestFramework):
         return raw_msg_with_wrong_size
 
 
-
 if __name__ == '__main__':
     InvalidMessagesTest().main()

test/bitcoin_functional/functional/p2p_invalid_tx.py

@@ -5,7 +5,7 @@
 """Test node responses to invalid transactions.
 
 In this test we connect to one node over p2p, and test tx requests."""
-from test_framework.blocktools import create_block, create_coinbase, create_tx_with_script
+from test_framework.blocktools import create_block, create_coinbase
 from test_framework.messages import (
     COIN,
     COutPoint,
@@ -19,6 +19,7 @@ from test_framework.util import (
     assert_equal,
     wait_until,
 )
+from data import invalid_txs
 
 
 class InvalidTxRequestTest(BitcoinTestFramework):
@@ -63,12 +64,21 @@ class InvalidTxRequestTest(BitcoinTestFramework):
         self.log.info("Mature the block.")
         self.nodes[0].generatetoaddress(100, self.nodes[0].get_deterministic_priv_key().address)
 
-        # b'\x64' is OP_NOTIF
-        # Transaction will be rejected with code 16 (REJECT_INVALID)
-        # and we get disconnected immediately
-        self.log.info('Test a transaction that is rejected')
-        tx1 = create_tx_with_script(block1.vtx[0], 0, script_sig=b'\x64' * 35, amount=50 * COIN - 12000)
-        node.p2p.send_txs_and_test([tx1], node, success=False, expect_disconnect=True)
+        # Iterate through a list of known invalid transaction types, ensuring each is
+        # rejected. Some are consensus invalid and some just violate policy.
+        for BadTxTemplate in invalid_txs.iter_all_templates():
+            self.log.info("Testing invalid transaction: %s", BadTxTemplate.__name__)
+            template = BadTxTemplate(spend_block=block1)
+            tx = template.get_tx()
+            node.p2p.send_txs_and_test(
+                [tx], node, success=False,
+                expect_disconnect=template.expect_disconnect,
+                reject_reason=template.reject_reason,
+            )
+
+            if template.expect_disconnect:
+                self.log.info("Reconnecting to peer")
+                self.reconnect_p2p()
 
         # Make two p2p connections to provide the node with orphans
         # * p2ps[0] will send valid orphan txs (one with low fee)

test/bitcoin_functional/functional/p2p_segwit.py

@@ -545,31 +545,13 @@ class SegWitTest(BitcoinTestFramework):
 
     @subtest
     def test_getblocktemplate_before_lockin(self):
-        # Node0 is segwit aware, node2 is not.
-        for node in [self.nodes[0], self.nodes[2]]:
-            gbt_results = node.getblocktemplate()
-            block_version = gbt_results['version']
-            # If we're not indicating segwit support, we will still be
-            # signalling for segwit activation.
-            assert_equal((block_version & (1 << VB_WITNESS_BIT) != 0), node == self.nodes[0])
-            # If we don't specify the segwit rule, then we won't get a default
-            # commitment.
-            assert('default_witness_commitment' not in gbt_results)
-
-        # Workaround:
-        # Can either change the tip, or change the mempool and wait 5 seconds
-        # to trigger a recomputation of getblocktemplate.
         txid = int(self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1), 16)
-        # Using mocktime lets us avoid sleep()
-        sync_mempools(self.nodes)
-        self.nodes[0].setmocktime(int(time.time()) + 10)
-        self.nodes[2].setmocktime(int(time.time()) + 10)
 
         for node in [self.nodes[0], self.nodes[2]]:
             gbt_results = node.getblocktemplate({"rules": ["segwit"]})
             block_version = gbt_results['version']
             if node == self.nodes[2]:
-                # If this is a non-segwit node, we should still not get a witness
+                # If this is a non-segwit node, we should not get a witness
                 # commitment, nor a version bit signalling segwit.
                 assert_equal(block_version & (1 << VB_WITNESS_BIT), 0)
                 assert('default_witness_commitment' not in gbt_results)
@@ -586,10 +568,6 @@ class SegWitTest(BitcoinTestFramework):
                 script = get_witness_script(witness_root, 0)
                 assert_equal(witness_commitment, bytes_to_hex_str(script))
 
-        # undo mocktime
-        self.nodes[0].setmocktime(0)
-        self.nodes[2].setmocktime(0)
-
     @subtest
     def advance_to_segwit_lockin(self):
         """Mine enough blocks to lock in segwit, but don't activate."""
@@ -777,7 +755,7 @@ class SegWitTest(BitcoinTestFramework):
         spend_tx.vin[0].scriptSig = CScript([p2wsh_pubkey, b'a'])
         spend_tx.rehash()
         with self.nodes[0].assert_debug_log(
-                expected_msgs=('Not relaying invalid transaction {}'.format(spend_tx.hash), 'was not accepted: mandatory-script-verify-flag-failed (Script evaluated without error but finished with a false/empty top stack element)')):
+                expected_msgs=(spend_tx.hash, 'was not accepted: mandatory-script-verify-flag-failed (Script evaluated without error but finished with a false/empty top stack element)')):
             test_transaction_acceptance(self.nodes[0], self.test_node, spend_tx, with_witness=False, accepted=False)
 
         # Now put the witness script in the witness, should succeed after

test/bitcoin_functional/functional/p2p_sendheaders.py

@@ -490,7 +490,7 @@ class SendHeadersTest(BitcoinTestFramework):
 
         # Now announce a header that forks the last two blocks
         tip = blocks[0].sha256
-        height -= 1
+        height -= 2
         blocks = []
 
         # Create extra blocks for later

test/bitcoin_functional/functional/p2p_timeouts.py

@@ -14,11 +14,11 @@
 - Wait 1 second
 - Assert that we're connected
 - Send a ping to no_verack_node and no_version_node
-- Wait 30 seconds
+- Wait 1 second
 - Assert that we're still connected
 - Send a ping to no_verack_node and no_version_node
-- Wait 31 seconds
-- Assert that we're no longer connected (timeout to receive version/verack is 60 seconds)
+- Wait 2 seconds
+- Assert that we're no longer connected (timeout to receive version/verack is 3 seconds)
 """
 
 from time import sleep
@@ -36,6 +36,8 @@ class TimeoutsTest(BitcoinTestFramework):
     def set_test_params(self):
         self.setup_clean_chain = True
         self.num_nodes = 1
+        # set timeout to receive version/verack to 3 seconds
+        self.extra_args = [["-peertimeout=3"]]
 
     def run_test(self):
         # Setup the p2p connections
@@ -52,7 +54,7 @@ class TimeoutsTest(BitcoinTestFramework):
         no_verack_node.send_message(msg_ping())
         no_version_node.send_message(msg_ping())
 
-        sleep(30)
+        sleep(1)
 
         assert "version" in no_verack_node.last_message
 
@@ -63,11 +65,21 @@ class TimeoutsTest(BitcoinTestFramework):
         no_verack_node.send_message(msg_ping())
         no_version_node.send_message(msg_ping())
 
-        sleep(31)
+        expected_timeout_logs = [
+            "version handshake timeout from 0",
+            "socket no message in first 3 seconds, 1 0 from 1",
+            "socket no message in first 3 seconds, 0 0 from 2",
+        ]
 
-        assert not no_verack_node.is_connected
-        assert not no_version_node.is_connected
-        assert not no_send_node.is_connected
+        with self.nodes[0].assert_debug_log(expected_msgs=expected_timeout_logs):
+            sleep(3)
+            # By now, we waited a total of 5 seconds. Off-by-two for two
+            # reasons:
+            #  * The internal precision is one second
+            #  * Account for network delay
+            assert not no_verack_node.is_connected
+            assert not no_version_node.is_connected
+            assert not no_send_node.is_connected
 
 if __name__ == '__main__':
     TimeoutsTest().main()

test/bitcoin_functional/functional/rpc_bind.py

@@ -48,9 +48,12 @@ class RPCBindTest(BitcoinTestFramework):
         at a non-localhost IP.
         '''
         self.log.info("Allow IP test for %s:%d" % (rpchost, rpcport))
-        base_args = ['-disablewallet', '-nolisten'] + ['-rpcallowip='+x for x in allow_ips]
+        node_args = \
+            ['-disablewallet', '-nolisten'] + \
+            ['-rpcallowip='+x for x in allow_ips] + \
+            ['-rpcbind='+addr for addr in ['127.0.0.1', "%s:%d" % (rpchost, rpcport)]] # Bind to localhost as well so start_nodes doesn't hang
         self.nodes[0].rpchost = None
-        self.start_nodes([base_args])
+        self.start_nodes([node_args])
         # connect to node through non-loopback interface
         node = get_rpc_proxy(rpc_url(self.nodes[0].datadir, 0, "%s:%d" % (rpchost, rpcport)), 0, coveragedir=self.options.coveragedir)
         node.getnetworkinfo()
@@ -67,7 +70,7 @@ class RPCBindTest(BitcoinTestFramework):
 
         self.log.info("Check for ipv6")
         have_ipv6 = test_ipv6_local()
-        if not have_ipv6 and not self.options.run_ipv4:
+        if not have_ipv6 and not (self.options.run_ipv4 or self.options.run_nonloopback):
             raise SkipTest("This test requires ipv6 support.")
 
         self.log.info("Check for non-loopback interface")
@@ -101,9 +104,9 @@ class RPCBindTest(BitcoinTestFramework):
             # check default without rpcallowip (IPv4 and IPv6 localhost)
             self.run_bind_test(None, '127.0.0.1', [],
                 [('127.0.0.1', self.defaultport), ('::1', self.defaultport)])
-            # check default with rpcallowip (IPv6 any)
+            # check default with rpcallowip (IPv4 and IPv6 localhost)
             self.run_bind_test(['127.0.0.1'], '127.0.0.1', [],
-                [('::0', self.defaultport)])
+                [('127.0.0.1', self.defaultport), ('::1', self.defaultport)])
             # check only IPv6 localhost (explicit)
             self.run_bind_test(['[::1]'], '[::1]', ['[::1]'],
                 [('::1', self.defaultport)])

test/bitcoin_functional/functional/rpc_blockchain.py

@@ -133,7 +133,7 @@ class BlockchainTest(BitcoinTestFramework):
         assert_raises_rpc_error(-8, "Block is not in main chain", self.nodes[0].getchaintxstats, blockhash=blockhash)
         self.nodes[0].reconsiderblock(blockhash)
 
-        chaintxstats = self.nodes[0].getchaintxstats(1)
+        chaintxstats = self.nodes[0].getchaintxstats(nblocks=1)
         # 200 txs plus genesis tx
         assert_equal(chaintxstats['txcount'], 201)
         # tx rate should be 1 per 10 minutes, or 1/600
@@ -211,7 +211,7 @@ class BlockchainTest(BitcoinTestFramework):
 
         besthash = node.getbestblockhash()
         secondbesthash = node.getblockhash(199)
-        header = node.getblockheader(besthash)
+        header = node.getblockheader(blockhash=besthash)
 
         assert_equal(header['hash'], besthash)
         assert_equal(header['height'], 200)
@@ -287,7 +287,7 @@ class BlockchainTest(BitcoinTestFramework):
 
         def assert_waitforheight(height, timeout=2):
             assert_equal(
-                node.waitforblockheight(height, timeout)['height'],
+                node.waitforblockheight(height=height, timeout=timeout)['height'],
                 current_height)
 
         assert_waitforheight(0)

test/bitcoin_functional/functional/rpc_deriveaddresses.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test the deriveaddresses rpc call."""
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.descriptors import descsum_create
+from test_framework.util import assert_equal, assert_raises_rpc_error
+
+class DeriveaddressesTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 1
+        self.supports_cli = 1
+
+    def run_test(self):
+        assert_raises_rpc_error(-5, "Invalid descriptor", self.nodes[0].deriveaddresses, "a")
+
+        descriptor = "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)#t6wfjs64"
+        address = "bcrt1qjqmxmkpmxt80xz4y3746zgt0q3u3ferr34acd5"
+        assert_equal(self.nodes[0].deriveaddresses(descriptor), [address])
+
+        descriptor = descriptor[:-9]
+        assert_raises_rpc_error(-5, "Invalid descriptor", self.nodes[0].deriveaddresses, descriptor)
+
+        descriptor_pubkey = "wpkh(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/0)#s9ga3alw"
+        address = "bcrt1qjqmxmkpmxt80xz4y3746zgt0q3u3ferr34acd5"
+        assert_equal(self.nodes[0].deriveaddresses(descriptor_pubkey), [address])
+
+        ranged_descriptor = "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)#kft60nuy"
+        assert_equal(self.nodes[0].deriveaddresses(ranged_descriptor, 0, 2), [address, "bcrt1qhku5rq7jz8ulufe2y6fkcpnlvpsta7rq4442dy", "bcrt1qpgptk2gvshyl0s9lqshsmx932l9ccsv265tvaq"])
+
+        assert_raises_rpc_error(-8, "Range should not be specified for an un-ranged descriptor", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)"), 0, 2)
+
+        assert_raises_rpc_error(-8, "Range must be specified for a ranged descriptor", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"))
+
+        assert_raises_rpc_error(-8, "Missing range end parameter", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), 0)
+
+        assert_raises_rpc_error(-8, "Range end should be equal to or greater than begin", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), 2, 0)
+
+        assert_raises_rpc_error(-8, "Range should be greater or equal than 0", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), -1, 0)
+
+        combo_descriptor = descsum_create("combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)")
+        assert_equal(self.nodes[0].deriveaddresses(combo_descriptor), ["mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", "mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", address, "2NDvEwGfpEqJWfybzpKPHF2XH3jwoQV3D7x"])
+
+        hardened_without_privkey_descriptor = descsum_create("wpkh(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1'/1/0)")
+        assert_raises_rpc_error(-5, "Cannot derive script without private keys", self.nodes[0].deriveaddresses, hardened_without_privkey_descriptor)
+
+        bare_multisig_descriptor = descsum_create("multi(1,tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/0,tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/1)")
+        assert_raises_rpc_error(-5, "Descriptor does not have a corresponding address", self.nodes[0].deriveaddresses, bare_multisig_descriptor)
+
+if __name__ == '__main__':
+    DeriveaddressesTest().main()

test/bitcoin_functional/functional/rpc_fundrawtransaction.py

@@ -32,7 +32,7 @@ class RawTransactionsTest(BitcoinTestFramework):
     def skip_test_if_missing_module(self):
         self.skip_if_no_wallet()
 
-    def setup_network(self, split=False):
+    def setup_network(self):
         self.setup_nodes()
 
         connect_nodes_bi(self.nodes, 0, 1)

test/bitcoin_functional/functional/rpc_invalidateblock.py

@@ -4,10 +4,15 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test the invalidateblock RPC."""
 
-import time
-
 from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import assert_equal, connect_nodes_bi, sync_blocks
+from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE
+from test_framework.util import (
+    assert_equal,
+    connect_nodes_bi,
+    sync_blocks,
+    wait_until,
+)
+
 
 class InvalidateTest(BitcoinTestFramework):
     def set_test_params(self):
@@ -21,46 +26,66 @@ class InvalidateTest(BitcoinTestFramework):
         self.log.info("Make sure we repopulate setBlockIndexCandidates after InvalidateBlock:")
         self.log.info("Mine 4 blocks on Node 0")
         self.nodes[0].generatetoaddress(4, self.nodes[0].get_deterministic_priv_key().address)
-        assert(self.nodes[0].getblockcount() == 4)
-        besthash = self.nodes[0].getbestblockhash()
+        assert_equal(self.nodes[0].getblockcount(), 4)
+        besthash_n0 = self.nodes[0].getbestblockhash()
 
         self.log.info("Mine competing 6 blocks on Node 1")
         self.nodes[1].generatetoaddress(6, self.nodes[1].get_deterministic_priv_key().address)
-        assert(self.nodes[1].getblockcount() == 6)
+        assert_equal(self.nodes[1].getblockcount(), 6)
 
         self.log.info("Connect nodes to force a reorg")
-        connect_nodes_bi(self.nodes,0,1)
+        connect_nodes_bi(self.nodes, 0, 1)
         sync_blocks(self.nodes[0:2])
-        assert(self.nodes[0].getblockcount() == 6)
+        assert_equal(self.nodes[0].getblockcount(), 6)
         badhash = self.nodes[1].getblockhash(2)
 
         self.log.info("Invalidate block 2 on node 0 and verify we reorg to node 0's original chain")
         self.nodes[0].invalidateblock(badhash)
-        newheight = self.nodes[0].getblockcount()
-        newhash = self.nodes[0].getbestblockhash()
-        if (newheight != 4 or newhash != besthash):
-            raise AssertionError("Wrong tip for node0, hash %s, height %d"%(newhash,newheight))
+        assert_equal(self.nodes[0].getblockcount(), 4)
+        assert_equal(self.nodes[0].getbestblockhash(), besthash_n0)
 
         self.log.info("Make sure we won't reorg to a lower work chain:")
-        connect_nodes_bi(self.nodes,1,2)
+        connect_nodes_bi(self.nodes, 1, 2)
         self.log.info("Sync node 2 to node 1 so both have 6 blocks")
         sync_blocks(self.nodes[1:3])
-        assert(self.nodes[2].getblockcount() == 6)
+        assert_equal(self.nodes[2].getblockcount(), 6)
         self.log.info("Invalidate block 5 on node 1 so its tip is now at 4")
         self.nodes[1].invalidateblock(self.nodes[1].getblockhash(5))
-        assert(self.nodes[1].getblockcount() == 4)
+        assert_equal(self.nodes[1].getblockcount(), 4)
         self.log.info("Invalidate block 3 on node 2, so its tip is now 2")
         self.nodes[2].invalidateblock(self.nodes[2].getblockhash(3))
-        assert(self.nodes[2].getblockcount() == 2)
+        assert_equal(self.nodes[2].getblockcount(), 2)
         self.log.info("..and then mine a block")
         self.nodes[2].generatetoaddress(1, self.nodes[2].get_deterministic_priv_key().address)
         self.log.info("Verify all nodes are at the right height")
-        time.sleep(5)
-        assert_equal(self.nodes[2].getblockcount(), 3)
-        assert_equal(self.nodes[0].getblockcount(), 4)
-        node1height = self.nodes[1].getblockcount()
-        if node1height < 4:
-            raise AssertionError("Node 1 reorged to a lower height: %d"%node1height)
+        wait_until(lambda: self.nodes[2].getblockcount() == 3, timeout=5)
+        wait_until(lambda: self.nodes[0].getblockcount() == 4, timeout=5)
+        wait_until(lambda: self.nodes[1].getblockcount() == 4, timeout=5)
+
+        self.log.info("Verify that we reconsider all ancestors as well")
+        blocks = self.nodes[1].generatetoaddress(10, ADDRESS_BCRT1_UNSPENDABLE)
+        assert_equal(self.nodes[1].getbestblockhash(), blocks[-1])
+        # Invalidate the two blocks at the tip
+        self.nodes[1].invalidateblock(blocks[-1])
+        self.nodes[1].invalidateblock(blocks[-2])
+        assert_equal(self.nodes[1].getbestblockhash(), blocks[-3])
+        # Reconsider only the previous tip
+        self.nodes[1].reconsiderblock(blocks[-1])
+        # Should be back at the tip by now
+        assert_equal(self.nodes[1].getbestblockhash(), blocks[-1])
+
+        self.log.info("Verify that we reconsider all descendants")
+        blocks = self.nodes[1].generatetoaddress(10, ADDRESS_BCRT1_UNSPENDABLE)
+        assert_equal(self.nodes[1].getbestblockhash(), blocks[-1])
+        # Invalidate the two blocks at the tip
+        self.nodes[1].invalidateblock(blocks[-2])
+        self.nodes[1].invalidateblock(blocks[-4])
+        assert_equal(self.nodes[1].getbestblockhash(), blocks[-5])
+        # Reconsider only the previous tip
+        self.nodes[1].reconsiderblock(blocks[-4])
+        # Should be back at the tip by now
+        assert_equal(self.nodes[1].getbestblockhash(), blocks[-1])
+
 
 if __name__ == '__main__':
     InvalidateTest().main()

test/bitcoin_functional/functional/rpc_net.py

@@ -67,19 +67,19 @@ class NetTest(BitcoinTestFramework):
 
         peer_info_after_ping = self.nodes[0].getpeerinfo()
         for before, after in zip(peer_info, peer_info_after_ping):
-            assert_greater_than_or_equal(after['bytesrecv_per_msg']['pong'], before['bytesrecv_per_msg']['pong'] + 32)
-            assert_greater_than_or_equal(after['bytessent_per_msg']['ping'], before['bytessent_per_msg']['ping'] + 32)
+            assert_greater_than_or_equal(after['bytesrecv_per_msg'].get('pong', 0), before['bytesrecv_per_msg'].get('pong', 0) + 32)
+            assert_greater_than_or_equal(after['bytessent_per_msg'].get('ping', 0), before['bytessent_per_msg'].get('ping', 0) + 32)
 
     def _test_getnetworkinginfo(self):
         assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True)
         assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2)
 
-        self.nodes[0].setnetworkactive(False)
+        self.nodes[0].setnetworkactive(state=False)
         assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], False)
         # Wait a bit for all sockets to close
         wait_until(lambda: self.nodes[0].getnetworkinfo()['connections'] == 0, timeout=3)
 
-        self.nodes[0].setnetworkactive(True)
+        self.nodes[0].setnetworkactive(state=True)
         connect_nodes_bi(self.nodes, 0, 1)
         assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True)
         assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2)
@@ -88,7 +88,7 @@ class NetTest(BitcoinTestFramework):
         assert_equal(self.nodes[0].getaddednodeinfo(), [])
         # add a node (node2) to node0
         ip_port = "127.0.0.1:{}".format(p2p_port(2))
-        self.nodes[0].addnode(ip_port, 'add')
+        self.nodes[0].addnode(node=ip_port, command='add')
         # check that the node has indeed been added
         added_nodes = self.nodes[0].getaddednodeinfo(ip_port)
         assert_equal(len(added_nodes), 1)

test/bitcoin_functional/functional/rpc_psbt.py

@@ -5,8 +5,9 @@
 """Test the Partially Signed Transaction RPCs.
 """
 
+from decimal import Decimal
 from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import assert_equal, assert_raises_rpc_error, find_output, disconnect_nodes, connect_nodes_bi, sync_blocks
+from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes_bi, disconnect_nodes, find_output, sync_blocks
 
 import json
 import os
@@ -158,11 +159,11 @@ class PSBTTest(BitcoinTestFramework):
         node1_addr = self.nodes[1].getnewaddress()
         node2_addr = self.nodes[2].getnewaddress()
         txid1 = self.nodes[0].sendtoaddress(node1_addr, 13)
-        txid2 =self.nodes[0].sendtoaddress(node2_addr, 13)
-        self.nodes[0].generate(6)
+        txid2 = self.nodes[0].sendtoaddress(node2_addr, 13)
+        blockhash = self.nodes[0].generate(6)[0]
         self.sync_all()
-        vout1 = find_output(self.nodes[1], txid1, 13)
-        vout2 = find_output(self.nodes[2], txid2, 13)
+        vout1 = find_output(self.nodes[1], txid1, 13, blockhash=blockhash)
+        vout2 = find_output(self.nodes[2], txid2, 13, blockhash=blockhash)
 
         # Create a psbt spending outputs from nodes 1 and 2
         psbt_orig = self.nodes[0].createpsbt([{"txid":txid1,  "vout":vout1}, {"txid":txid2, "vout":vout2}], {self.nodes[0].getnewaddress():25.999})
@@ -210,6 +211,10 @@ class PSBTTest(BitcoinTestFramework):
             assert tx_in["sequence"] > MAX_BIP125_RBF_SEQUENCE
         assert_equal(decoded_psbt["tx"]["locktime"], 0)
 
+        # Make sure change address wallet does not have P2SH innerscript access to results in success
+        # when attempting BnB coin selection
+        self.nodes[0].walletcreatefundedpsbt([], [{self.nodes[2].getnewaddress():unspent["amount"]+1}], block_height+2, {"changeAddress":self.nodes[1].getnewaddress()}, False)
+
         # Regression test for 14473 (mishandling of already-signed witness transaction):
         psbtx_info = self.nodes[0].walletcreatefundedpsbt([{"txid":unspent["txid"], "vout":unspent["vout"]}], [{self.nodes[2].getnewaddress():unspent["amount"]+1}])
         complete_psbt = self.nodes[0].walletprocesspsbt(psbtx_info["psbt"])
@@ -263,6 +268,9 @@ class PSBTTest(BitcoinTestFramework):
             combined = self.nodes[2].combinepsbt(combiner['combine'])
             assert_equal(combined, combiner['result'])
 
+        # Empty combiner test
+        assert_raises_rpc_error(-8, "Parameter 'txs' cannot be empty", self.nodes[0].combinepsbt, [])
+
         # Finalizer test
         for finalizer in finalizers:
             finalized = self.nodes[2].finalizepsbt(finalizer['finalize'], False)['psbt']
@@ -284,5 +292,75 @@ class PSBTTest(BitcoinTestFramework):
         psbt = self.nodes[1].walletcreatefundedpsbt([], [{p2pkh : 1}], 0, {"includeWatching" : True}, True)
         self.nodes[0].decodepsbt(psbt['psbt'])
 
+        # Test decoding error: invalid base64
+        assert_raises_rpc_error(-22, "TX decode failed invalid base64", self.nodes[0].decodepsbt, ";definitely not base64;")
+
+        # Send to all types of addresses
+        addr1 = self.nodes[1].getnewaddress("", "bech32")
+        txid1 = self.nodes[0].sendtoaddress(addr1, 11)
+        vout1 = find_output(self.nodes[0], txid1, 11)
+        addr2 = self.nodes[1].getnewaddress("", "legacy")
+        txid2 = self.nodes[0].sendtoaddress(addr2, 11)
+        vout2 = find_output(self.nodes[0], txid2, 11)
+        addr3 = self.nodes[1].getnewaddress("", "p2sh-segwit")
+        txid3 = self.nodes[0].sendtoaddress(addr3, 11)
+        vout3 = find_output(self.nodes[0], txid3, 11)
+        self.sync_all()
+
+        # Update a PSBT with UTXOs from the node
+        # Bech32 inputs should be filled with witness UTXO. Other inputs should not be filled because they are non-witness
+        psbt = self.nodes[1].createpsbt([{"txid":txid1, "vout":vout1},{"txid":txid2, "vout":vout2},{"txid":txid3, "vout":vout3}], {self.nodes[0].getnewaddress():32.999})
+        decoded = self.nodes[1].decodepsbt(psbt)
+        assert "witness_utxo" not in decoded['inputs'][0] and "non_witness_utxo" not in decoded['inputs'][0]
+        assert "witness_utxo" not in decoded['inputs'][1] and "non_witness_utxo" not in decoded['inputs'][1]
+        assert "witness_utxo" not in decoded['inputs'][2] and "non_witness_utxo" not in decoded['inputs'][2]
+        updated = self.nodes[1].utxoupdatepsbt(psbt)
+        decoded = self.nodes[1].decodepsbt(updated)
+        assert "witness_utxo" in decoded['inputs'][0] and "non_witness_utxo" not in decoded['inputs'][0]
+        assert "witness_utxo" not in decoded['inputs'][1] and "non_witness_utxo" not in decoded['inputs'][1]
+        assert "witness_utxo" not in decoded['inputs'][2] and "non_witness_utxo" not in decoded['inputs'][2]
+
+        # Two PSBTs with a common input should not be joinable
+        psbt1 = self.nodes[1].createpsbt([{"txid":txid1, "vout":vout1}], {self.nodes[0].getnewaddress():Decimal('10.999')})
+        assert_raises_rpc_error(-8, "exists in multiple PSBTs", self.nodes[1].joinpsbts, [psbt1, updated])
+
+        # Join two distinct PSBTs
+        addr4 = self.nodes[1].getnewaddress("", "p2sh-segwit")
+        txid4 = self.nodes[0].sendtoaddress(addr4, 5)
+        vout4 = find_output(self.nodes[0], txid4, 5)
+        self.nodes[0].generate(6)
+        self.sync_all()
+        psbt2 = self.nodes[1].createpsbt([{"txid":txid4, "vout":vout4}], {self.nodes[0].getnewaddress():Decimal('4.999')})
+        psbt2 = self.nodes[1].walletprocesspsbt(psbt2)['psbt']
+        psbt2_decoded = self.nodes[0].decodepsbt(psbt2)
+        assert "final_scriptwitness" in psbt2_decoded['inputs'][0] and "final_scriptSig" in psbt2_decoded['inputs'][0]
+        joined = self.nodes[0].joinpsbts([psbt, psbt2])
+        joined_decoded = self.nodes[0].decodepsbt(joined)
+        assert len(joined_decoded['inputs']) == 4 and len(joined_decoded['outputs']) == 2 and "final_scriptwitness" not in joined_decoded['inputs'][3] and "final_scriptSig" not in joined_decoded['inputs'][3]
+
+        # Newly created PSBT needs UTXOs and updating
+        addr = self.nodes[1].getnewaddress("", "p2sh-segwit")
+        txid = self.nodes[0].sendtoaddress(addr, 7)
+        addrinfo = self.nodes[1].getaddressinfo(addr)
+        blockhash = self.nodes[0].generate(6)[0]
+        self.sync_all()
+        vout = find_output(self.nodes[0], txid, 7, blockhash=blockhash)
+        psbt = self.nodes[1].createpsbt([{"txid":txid, "vout":vout}], {self.nodes[0].getnewaddress("", "p2sh-segwit"):Decimal('6.999')})
+        analyzed = self.nodes[0].analyzepsbt(psbt)
+        assert not analyzed['inputs'][0]['has_utxo'] and not analyzed['inputs'][0]['is_final'] and analyzed['inputs'][0]['next'] == 'updater' and analyzed['next'] == 'updater'
+
+        # After update with wallet, only needs signing
+        updated = self.nodes[1].walletprocesspsbt(psbt, False, 'ALL', True)['psbt']
+        analyzed = self.nodes[0].analyzepsbt(updated)
+        assert analyzed['inputs'][0]['has_utxo'] and not analyzed['inputs'][0]['is_final'] and analyzed['inputs'][0]['next'] == 'signer' and analyzed['next'] == 'signer' and analyzed['inputs'][0]['missing']['signatures'][0] == addrinfo['embedded']['witness_program']
+
+        # Check fee and size things
+        assert analyzed['fee'] == Decimal('0.001') and analyzed['estimated_vsize'] == 134 and analyzed['estimated_feerate'] == '0.00746268 BTC/kB'
+
+        # After signing and finalizing, needs extracting
+        signed = self.nodes[1].walletprocesspsbt(updated)['psbt']
+        analyzed = self.nodes[0].analyzepsbt(signed)
+        assert analyzed['inputs'][0]['has_utxo'] and analyzed['inputs'][0]['is_final'] and analyzed['next'] == 'extractor'
+
 if __name__ == '__main__':
     PSBTTest().main()

test/bitcoin_functional/functional/rpc_rawtransaction.py

@@ -42,12 +42,12 @@ class RawTransactionsTest(BitcoinTestFramework):
     def set_test_params(self):
         self.setup_clean_chain = True
         self.num_nodes = 3
-        self.extra_args = [["-addresstype=legacy"], ["-addresstype=legacy"], ["-addresstype=legacy"]]
+        self.extra_args = [["-addresstype=legacy", "-txindex"], ["-addresstype=legacy", "-txindex"], ["-addresstype=legacy", "-txindex"]]
 
     def skip_test_if_missing_module(self):
         self.skip_if_no_wallet()
 
-    def setup_network(self, split=False):
+    def setup_network(self):
         super().setup_network()
         connect_nodes_bi(self.nodes, 0, 2)
 
@@ -74,8 +74,7 @@ class RawTransactionsTest(BitcoinTestFramework):
         assert_raises_rpc_error(-1, "createrawtransaction", self.nodes[0].createrawtransaction, [])
 
         # Test `createrawtransaction` invalid extra parameters
-        # ELEMENTS:
-        #assert_raises_rpc_error(-1, "createrawtransaction", self.nodes[0].createrawtransaction, [], {}, 0, False, 'foo')
+        assert_raises_rpc_error(-1, "createrawtransaction", self.nodes[0].createrawtransaction, [], {}, 0, False, 'foo')
 
         # Test `createrawtransaction` invalid `inputs`
         txid = '1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000'
@@ -101,6 +100,8 @@ class RawTransactionsTest(BitcoinTestFramework):
         assert_raises_rpc_error(-3, "Amount out of range", self.nodes[0].createrawtransaction, [], {address: -1})
         assert_raises_rpc_error(-8, "Invalid parameter, duplicated address: %s" % address, self.nodes[0].createrawtransaction, [], multidict([(address, 1), (address, 1)]))
         assert_raises_rpc_error(-8, "Invalid parameter, duplicated address: %s" % address, self.nodes[0].createrawtransaction, [], [{address: 1}, {address: 1}])
+        assert_raises_rpc_error(-8, "Invalid parameter, duplicate key: data", self.nodes[0].createrawtransaction, [], [{"data": 'aa'}, {"data": "bb"}])
+        assert_raises_rpc_error(-8, "Invalid parameter, duplicate key: data", self.nodes[0].createrawtransaction, [], multidict([("data", 'aa'), ("data", "bb")]))
         assert_raises_rpc_error(-8, "Invalid parameter, key-value pair must contain exactly one key", self.nodes[0].createrawtransaction, [], [{'a': 1, 'b': 2}])
         assert_raises_rpc_error(-8, "Invalid parameter, key-value pair not an object as expected", self.nodes[0].createrawtransaction, [], [['key-value pair1'], ['2']])
 
@@ -128,19 +129,12 @@ class RawTransactionsTest(BitcoinTestFramework):
             bytes_to_hex_str(tx.serialize()),
             self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}, {address2: 99}]),
         )
-        # Two data outputs
-        tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=multidict([('data', '99'), ('data', '99')])))))
-        assert_equal(len(tx.vout), 2)
-        assert_equal(
-            bytes_to_hex_str(tx.serialize()),
-            self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{'data': '99'}, {'data': '99'}]),
-        )
         # Multiple mixed outputs
-        tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=multidict([(address, 99), ('data', '99'), ('data', '99')])))))
+        tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=multidict([(address, 99), (address2, 99), ('data', '99')])))))
         assert_equal(len(tx.vout), 3)
         assert_equal(
             bytes_to_hex_str(tx.serialize()),
-            self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}, {'data': '99'}, {'data': '99'}]),
+            self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}, {address2: 99}, {'data': '99'}]),
         )
 
         for type in ["bech32", "p2sh-segwit", "legacy"]:

test/bitcoin_functional/functional/rpc_scantxoutset.py

@@ -10,6 +10,9 @@ from decimal import Decimal
 import shutil
 import os
 
+def descriptors(out):
+    return sorted(u['desc'] for u in out['unspents'])
+
 class ScantxoutsetTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 1
@@ -93,5 +96,10 @@ class ScantxoutsetTest(BitcoinTestFramework):
         assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1499}])['total_amount'], Decimal("12.288"))
         assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1500}])['total_amount'], Decimal("28.672"))
 
+        # Test the reported descriptors for a few matches
+        assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0h/0'/*)", "range": 1499}])), ["pkh([0c5f9a1e/0'/0'/0]026dbd8b2315f296d36e6b6920b1579ca75569464875c7ebe869b536a7d9503c8c)#dzxw429x", "pkh([0c5f9a1e/0'/0'/1]033e6f25d76c00bedb3a8993c7d5739ee806397f0529b1b31dda31ef890f19a60c)#43rvceed"])
+        assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)"])), ["pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)#cxmct4w8"])
+        assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1500}])), ['pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)#cxmct4w8', 'pkh([0c5f9a1e/1/1/1500]03832901c250025da2aebae2bfb38d5c703a57ab66ad477f9c578bfbcd78abca6f)#vchwd07g', 'pkh([0c5f9a1e/1/1/1]030d820fc9e8211c4169be8530efbc632775d8286167afd178caaf1089b77daba7)#z2t3ypsa'])
+
 if __name__ == '__main__':
     ScantxoutsetTest().main()

test/bitcoin_functional/functional/rpc_signrawtransaction.py

@@ -5,14 +5,17 @@
 """Test transaction signing using the signrawtransaction* RPCs."""
 
 from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import assert_equal, assert_raises_rpc_error
+from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str, hex_str_to_bytes
+from test_framework.messages import sha256
+from test_framework.script import CScript, OP_0
 
+from decimal import Decimal
 
 class SignRawTransactionsTest(BitcoinTestFramework):
     def set_test_params(self):
         self.setup_clean_chain = True
-        self.num_nodes = 1
-        self.extra_args = [["-deprecatedrpc=signrawtransaction"]]
+        self.num_nodes = 2
+        self.extra_args = [["-deprecatedrpc=signrawtransaction"], []]
 
     def skip_test_if_missing_module(self):
         self.skip_if_no_wallet()
@@ -143,9 +146,33 @@ class SignRawTransactionsTest(BitcoinTestFramework):
         assert_equal(rawTxSigned['errors'][1]['witness'], ["304402203609e17b84f6a7d30c80bfa610b5b4542f32a8a0d5447a12fb1366d7f01cc44a0220573a954c4518331561406f90300e8f3358f51928d43c212a8caed02de67eebee01", "025476c2e83188368da1ff3e292e7acafcdb3566bb0ad253f62fc70f07aeee6357"])
         assert not rawTxSigned['errors'][0]['witness']
 
+    def witness_script_test(self):
+        # Now test signing transaction to P2SH-P2WSH addresses without wallet
+        # Create a new P2SH-P2WSH 1-of-1 multisig address:
+        embedded_address = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress())
+        embedded_privkey = self.nodes[1].dumpprivkey(embedded_address["address"])
+        p2sh_p2wsh_address = self.nodes[1].addmultisigaddress(1, [embedded_address["pubkey"]], "", "p2sh-segwit")
+        # send transaction to P2SH-P2WSH 1-of-1 multisig address
+        self.nodes[0].generate(101)
+        self.nodes[0].sendtoaddress(p2sh_p2wsh_address["address"], 49.999)
+        self.nodes[0].generate(1)
+        self.sync_all()
+        # Find the UTXO for the transaction node[1] should have received, check witnessScript matches
+        unspent_output = self.nodes[1].listunspent(0, 999999, [p2sh_p2wsh_address["address"]])[0]
+        assert_equal(unspent_output["witnessScript"], p2sh_p2wsh_address["redeemScript"])
+        p2sh_redeemScript = CScript([OP_0, sha256(hex_str_to_bytes(p2sh_p2wsh_address["redeemScript"]))])
+        assert_equal(unspent_output["redeemScript"], bytes_to_hex_str(p2sh_redeemScript))
+        # Now create and sign a transaction spending that output on node[0], which doesn't know the scripts or keys
+        spending_tx = self.nodes[0].createrawtransaction([unspent_output], {self.nodes[1].getnewaddress(): Decimal("49.998")})
+        spending_tx_signed = self.nodes[0].signrawtransactionwithkey(spending_tx, [embedded_privkey], [unspent_output])
+        # Check the signing completed successfully
+        assert 'complete' in spending_tx_signed
+        assert_equal(spending_tx_signed['complete'], True)
+
     def run_test(self):
         self.successful_signing_test()
         self.script_verification_error_test()
+        self.witness_script_test()
         self.test_with_lock_outputs()
 
 

test/bitcoin_functional/functional/rpc_zmq.py

@@ -1,37 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2018 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-"""Test for the ZMQ RPC methods."""
-
-from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import assert_equal
-
-
-class RPCZMQTest(BitcoinTestFramework):
-
-    address = "tcp://127.0.0.1:28332"
-
-    def set_test_params(self):
-        self.num_nodes = 1
-        self.setup_clean_chain = True
-
-    def skip_test_if_missing_module(self):
-        self.skip_if_no_py3_zmq()
-        self.skip_if_no_bitcoind_zmq()
-
-    def run_test(self):
-        self._test_getzmqnotifications()
-
-    def _test_getzmqnotifications(self):
-        self.restart_node(0, extra_args=[])
-        assert_equal(self.nodes[0].getzmqnotifications(), [])
-
-        self.restart_node(0, extra_args=["-zmqpubhashtx=%s" % self.address])
-        assert_equal(self.nodes[0].getzmqnotifications(), [
-            {"type": "pubhashtx", "address": self.address},
-        ])
-
-
-if __name__ == '__main__':
-    RPCZMQTest().main()

test/bitcoin_functional/functional/test_framework/blocktools.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2015-2018 The Bitcoin Core developers
+# Copyright (c) 2015-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Utilities for manipulating blocks and transactions."""
@@ -41,12 +41,19 @@ from .script import (
 from .util import assert_equal
 from io import BytesIO
 
+MAX_BLOCK_SIGOPS = 20000
+
+# Genesis block time (regtest)
+TIME_GENESIS_BLOCK = 1296688602
+
 # From BIP141
 WITNESS_COMMITMENT_HEADER = b"\xaa\x21\xa9\xed"
 
-def create_block(hashprev, coinbase, ntime=None):
+
+def create_block(hashprev, coinbase, ntime=None, *, version=1):
     """Create a block (with regtest difficulty)."""
     block = CBlock()
+    block.nVersion = version
     if ntime is None:
         import time
         block.nTime = int(time.time() + 600)

test/bitcoin_functional/functional/test_framework/descriptors.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+# Copyright (c) 2019 Pieter Wuille
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Utility functions related to output descriptors"""
+
+INPUT_CHARSET = "0123456789()[],'/*abcdefgh@:$%{}IJKLMNOPQRSTUVWXYZ&+-.;<=>?!^_|~ijklmnopqrstuvwxyzABCDEFGH`#\"\\ "
+CHECKSUM_CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l"
+GENERATOR = [0xf5dee51989, 0xa9fdca3312, 0x1bab10e32d, 0x3706b1677a, 0x644d626ffd]
+
+def descsum_polymod(symbols):
+    """Internal function that computes the descriptor checksum."""
+    chk = 1
+    for value in symbols:
+        top = chk >> 35
+        chk = (chk & 0x7ffffffff) << 5 ^ value
+        for i in range(5):
+            chk ^= GENERATOR[i] if ((top >> i) & 1) else 0
+    return chk
+
+def descsum_expand(s):
+    """Internal function that does the character to symbol expansion"""
+    groups = []
+    symbols = []
+    for c in s:
+        if not c in INPUT_CHARSET:
+            return None
+        v = INPUT_CHARSET.find(c)
+        symbols.append(v & 31)
+        groups.append(v >> 5)
+        if len(groups) == 3:
+            symbols.append(groups[0] * 9 + groups[1] * 3 + groups[2])
+            groups = []
+    if len(groups) == 1:
+        symbols.append(groups[0])
+    elif len(groups) == 2:
+        symbols.append(groups[0] * 3 + groups[1])
+    return symbols
+
+def descsum_create(s):
+    """Add a checksum to a descriptor without"""
+    symbols = descsum_expand(s) + [0, 0, 0, 0, 0, 0, 0, 0]
+    checksum = descsum_polymod(symbols) ^ 1
+    return s + '#' + ''.join(CHECKSUM_CHARSET[(checksum >> (5 * (7 - i))) & 31] for i in range(8))
+
+def descsum_check(s, require=True):
+    """Verify that the checksum is correct in a descriptor"""
+    if not '#' in s:
+        return not require
+    if s[-9] != '#':
+        return False
+    if not all(x in CHECKSUM_CHARSET for x in s[-8:]):
+        return False
+    symbols = descsum_expand(s[:-9]) + [CHECKSUM_CHARSET.find(x) for x in s[-8:]]
+    return descsum_polymod(symbols) == 1

test/bitcoin_functional/functional/test_framework/messages.py

@@ -28,7 +28,7 @@ import struct
 import time
 
 from test_framework.siphash import siphash256
-from test_framework.util import hex_str_to_bytes, bytes_to_hex_str
+from test_framework.util import hex_str_to_bytes, bytes_to_hex_str, assert_equal
 
 MIN_VERSION_SUPPORTED = 60001
 MY_VERSION = 70014  # past bip-31 for ping/pong
@@ -450,6 +450,8 @@ class CTransaction:
         if flags != 0:
             self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))]
             self.wit.deserialize(f)
+        else:
+            self.wit = CTxWitness()
         self.nLockTime = struct.unpack("<I", f.read(4))[0]
         self.sha256 = None
         self.hash = None
@@ -589,6 +591,8 @@ class CBlockHeader:
             % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
                time.ctime(self.nTime), self.nBits, self.nNonce)
 
+BLOCK_HEADER_SIZE = len(CBlockHeader().serialize())
+assert_equal(BLOCK_HEADER_SIZE, 80)
 
 class CBlock(CBlockHeader):
     __slots__ = ("vtx",)
@@ -764,7 +768,7 @@ class HeaderAndShortIDs:
         self.prefilled_txn = []
         self.use_witness = False
 
-        if p2pheaders_and_shortids != None:
+        if p2pheaders_and_shortids is not None:
             self.header = p2pheaders_and_shortids.header
             self.nonce = p2pheaders_and_shortids.nonce
             self.shortids = p2pheaders_and_shortids.shortids
@@ -822,7 +826,7 @@ class BlockTransactionsRequest:
 
     def __init__(self, blockhash=0, indexes = None):
         self.blockhash = blockhash
-        self.indexes = indexes if indexes != None else []
+        self.indexes = indexes if indexes is not None else []
 
     def deserialize(self, f):
         self.blockhash = deser_uint256(f)
@@ -863,7 +867,7 @@ class BlockTransactions:
 
     def __init__(self, blockhash=0, transactions = None):
         self.blockhash = blockhash
-        self.transactions = transactions if transactions != None else []
+        self.transactions = transactions if transactions is not None else []
 
     def deserialize(self, f):
         self.blockhash = deser_uint256(f)
@@ -1052,7 +1056,7 @@ class msg_getdata:
     command = b"getdata"
 
     def __init__(self, inv=None):
-        self.inv = inv if inv != None else []
+        self.inv = inv if inv is not None else []
 
     def deserialize(self, f):
         self.inv = deser_vector(f, CInv)

test/bitcoin_functional/functional/test_framework/mininode.py

@@ -118,7 +118,7 @@ class P2PConnection(asyncio.Protocol):
         # The initial message to send after the connection was made:
         self.on_connection_send_msg = None
         self.recvbuf = b""
-        self.network = net
+        self.magic_bytes = MAGIC_BYTES[net]
         logger.debug('Connecting to Bitcoin Node: %s:%d' % (self.dstaddr, self.dstport))
 
         loop = NetworkThread.network_event_loop
@@ -170,7 +170,7 @@ class P2PConnection(asyncio.Protocol):
             while True:
                 if len(self.recvbuf) < 4:
                     return
-                if self.recvbuf[:4] != MAGIC_BYTES[self.network]:
+                if self.recvbuf[:4] != self.magic_bytes:
                     raise ValueError("got garbage %s" % repr(self.recvbuf))
                 if len(self.recvbuf) < 4 + 12 + 4 + 4:
                     return
@@ -232,7 +232,7 @@ class P2PConnection(asyncio.Protocol):
         """Build a serialized P2P message"""
         command = message.command
         data = message.serialize()
-        tmsg = MAGIC_BYTES[self.network]
+        tmsg = self.magic_bytes
         tmsg += command
         tmsg += b"\x00" * (12 - len(command))
         tmsg += struct.pack("<I", len(data))
@@ -511,14 +511,14 @@ class P2PDataStore(P2PInterface):
         if response is not None:
             self.send_message(response)
 
-    def send_blocks_and_test(self, blocks, node, *, success=True, request_block=True, reject_reason=None, expect_disconnect=False, timeout=60):
+    def send_blocks_and_test(self, blocks, node, *, success=True, force_send=False, reject_reason=None, expect_disconnect=False, timeout=60):
         """Send blocks to test node and test whether the tip advances.
 
          - add all blocks to our block_store
          - send a headers message for the final block
          - the on_getheaders handler will ensure that any getheaders are responded to
-         - if request_block is True: wait for getdata for each of the blocks. The on_getdata handler will
-           ensure that any getdata messages are responded to
+         - if force_send is False: wait for getdata for each of the blocks. The on_getdata handler will
+           ensure that any getdata messages are responded to. Otherwise send the full block unsolicited.
          - if success is True: assert that the node's tip advances to the most recent block
          - if success is False: assert that the node's tip doesn't advance
          - if reject_reason is set: assert that the correct reject message is logged"""
@@ -530,9 +530,11 @@ class P2PDataStore(P2PInterface):
 
         reject_reason = [reject_reason] if reject_reason else []
         with node.assert_debug_log(expected_msgs=reject_reason):
-            self.send_message(msg_headers([CBlockHeader(blocks[-1])]))
-
-            if request_block:
+            if force_send:
+                for b in blocks:
+                    self.send_message(msg_block(block=b))
+            else:
+                self.send_message(msg_headers([CBlockHeader(blocks[-1])]))
                 wait_until(lambda: blocks[-1].sha256 in self.getdata_requests, timeout=timeout, lock=mininode_lock)
 
             if expect_disconnect:

test/bitcoin_functional/functional/test_framework/script.py

@@ -385,6 +385,22 @@ class CScriptNum:
             r[-1] |= 0x80
         return bytes([len(r)]) + r
 
+    @staticmethod
+    def decode(vch):
+        result = 0
+        # We assume valid push_size and minimal encoding
+        value = vch[1:]
+        if len(value) == 0:
+            return result
+        for i, byte in enumerate(value):
+            result |= int(byte) << 8*i
+        if value[-1] >= 0x80:
+            # Mask for all but the highest result bit
+            num_mask = (2**(len(value)*8) - 1) >> 1
+            result &= num_mask
+            result *= -1
+        return result
+
 
 class CScript(bytes):
     """Serialized script
@@ -434,6 +450,10 @@ class CScript(bytes):
         # join makes no sense for a CScript()
         raise NotImplementedError
 
+    # Python 3.4 compatibility
+    def hex(self):
+        return hexlify(self).decode('ascii')
+
     def __new__(cls, value=b''):
         if isinstance(value, bytes) or isinstance(value, bytearray):
             return super(CScript, cls).__new__(cls, value)

test/bitcoin_functional/functional/test_framework/test_framework.py

@@ -43,6 +43,8 @@ TEST_EXIT_PASSED = 0
 TEST_EXIT_FAILED = 1
 TEST_EXIT_SKIPPED = 77
 
+TMPDIR_PREFIX = "bitcoin_func_test_"
+
 
 class SkipTest(Exception):
     """This exception is raised to skip a test"""
@@ -93,7 +95,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
         self.nodes = []
         self.network_thread = None
         self.mocktime = 0
-        self.rpc_timewait = 60  # Wait for up to 60 seconds for the RPC server to respond
+        self.rpc_timeout = 60  # Wait for up to 60 seconds for the RPC server to respond
         self.supports_cli = False
         self.bind_to_localhost_only = True
         self.set_test_params()
@@ -127,6 +129,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
                             help="Attach a python debugger if test fails")
         parser.add_argument("--usecli", dest="usecli", default=False, action="store_true",
                             help="use bitcoin-cli instead of RPC for all commands")
+        parser.add_argument("--perf", dest="perf", default=False, action="store_true",
+                            help="profile running nodes with perf for the duration of the test")
         self.add_options(parser)
         self.options = parser.parse_args()
 
@@ -138,6 +142,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
 
         config = configparser.ConfigParser()
         config.read_file(open(self.options.configfile))
+        self.config = config
+
         self.options.bitcoind = os.getenv("BITCOIND", default=config["environment"]["BUILDDIR"] + '/src/elementsd' + config["environment"]["EXEEXT"])
         self.options.bitcoincli = os.getenv("BITCOINCLI", default=config["environment"]["BUILDDIR"] + '/src/elements-cli' + config["environment"]["EXEEXT"])
 
@@ -152,7 +158,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
             self.options.tmpdir = os.path.abspath(self.options.tmpdir)
             os.makedirs(self.options.tmpdir, exist_ok=False)
         else:
-            self.options.tmpdir = tempfile.mkdtemp(prefix="test")
+            self.options.tmpdir = tempfile.mkdtemp(prefix=TMPDIR_PREFIX)
         self._start_logging()
 
         self.log.debug('Setting up network thread')
@@ -200,11 +206,20 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
                 node.cleanup_on_exit = False
             self.log.info("Note: bitcoinds were not stopped and may still be running")
 
-        if not self.options.nocleanup and not self.options.noshutdown and success != TestStatus.FAILED:
+        should_clean_up = (
+            not self.options.nocleanup and
+            not self.options.noshutdown and
+            success != TestStatus.FAILED and
+            not self.options.perf
+        )
+        if should_clean_up:
             self.log.info("Cleaning up {} on exit".format(self.options.tmpdir))
             cleanup_tree_on_exit = True
+        elif self.options.perf:
+            self.log.warning("Not cleaning up dir {} due to perf data".format(self.options.tmpdir))
+            cleanup_tree_on_exit = False
         else:
-            self.log.warning("Not cleaning up dir %s" % self.options.tmpdir)
+            self.log.warning("Not cleaning up dir {}".format(self.options.tmpdir))
             cleanup_tree_on_exit = False
 
         if success == TestStatus.PASSED:
@@ -280,7 +295,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
     # Public helper methods. These can be accessed by the subclass test scripts.
 
     def add_nodes(self, num_nodes, extra_args=None, *, rpchost=None, binary=None):
-        """Instantiate TestNode objects"""
+        """Instantiate TestNode objects.
+
+        Should only be called once after the nodes have been specified in
+        set_test_params()."""
         if self.bind_to_localhost_only:
             extra_confs = [["bind=127.0.0.1"]] * num_nodes
         else:
@@ -293,7 +311,21 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
         assert_equal(len(extra_args), num_nodes)
         assert_equal(len(binary), num_nodes)
         for i in range(num_nodes):
-            self.nodes.append(TestNode(i, get_datadir_path(self.options.tmpdir, i), rpchost=rpchost, timewait=self.rpc_timewait, bitcoind=binary[i], bitcoin_cli=self.options.bitcoincli, mocktime=self.mocktime, coverage_dir=self.options.coveragedir, extra_conf=extra_confs[i], extra_args=extra_args[i], use_cli=self.options.usecli))
+            self.nodes.append(TestNode(
+                i,
+                get_datadir_path(self.options.tmpdir, i),
+                rpchost=rpchost,
+                timewait=self.rpc_timeout,
+                bitcoind=binary[i],
+                bitcoin_cli=self.options.bitcoincli,
+                mocktime=self.mocktime,
+                coverage_dir=self.options.coveragedir,
+                cwd=self.options.tmpdir,
+                extra_conf=extra_confs[i],
+                extra_args=extra_args[i],
+                use_cli=self.options.usecli,
+                start_perf=self.options.perf,
+            ))
 
     def start_node(self, i, *args, **kwargs):
         """Start a bitcoind"""
@@ -326,16 +358,16 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
             for node in self.nodes:
                 coverage.write_all_rpc_commands(self.options.coveragedir, node.rpc)
 
-    def stop_node(self, i, expected_stderr=''):
+    def stop_node(self, i, expected_stderr='', wait=0):
         """Stop a bitcoind test node"""
-        self.nodes[i].stop_node(expected_stderr)
+        self.nodes[i].stop_node(expected_stderr, wait=wait)
         self.nodes[i].wait_until_stopped()
 
-    def stop_nodes(self):
+    def stop_nodes(self, wait=0):
         """Stop multiple bitcoind test nodes"""
         for node in self.nodes:
             # Issue RPC to stop nodes
-            node.stop_node()
+            node.stop_node(wait=wait)
 
         for node in self.nodes:
             # Wait for nodes to stop
@@ -372,21 +404,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
             sync_blocks(group)
             sync_mempools(group)
 
-    def enable_mocktime(self):
-        """Enable mocktime for the script.
-
-        mocktime may be needed for scripts that use the cached version of the
-        blockchain.  If the cached version of the blockchain is used without
-        mocktime then the mempools will not sync due to IBD.
-
-        For backward compatibility of the python scripts with previous
-        versions of the cache, this helper function sets mocktime to Jan 1,
-        2014 + (201 * 10 * 60)"""
-        self.mocktime = 1388534400 + (201 * 10 * 60)
-
-    def disable_mocktime(self):
-        self.mocktime = 0
-
     # Private helper methods. These should not be accessed by the subclass test scripts.
 
     def _start_logging(self):
@@ -444,7 +461,19 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
                 args = [self.options.bitcoind, "-datadir=" + datadir, '-disablewallet']
                 if i > 0:
                     args.append("-connect=127.0.0.1:" + str(p2p_port(0)))
-                self.nodes.append(TestNode(i, get_datadir_path(self.options.cachedir, i), extra_conf=["bind=127.0.0.1"], extra_args=[], rpchost=None, timewait=self.rpc_timewait, bitcoind=self.options.bitcoind, bitcoin_cli=self.options.bitcoincli, mocktime=self.mocktime, coverage_dir=None))
+                self.nodes.append(TestNode(
+                    i,
+                    get_datadir_path(self.options.cachedir, i),
+                    extra_conf=["bind=127.0.0.1"],
+                    extra_args=[],
+                    rpchost=None,
+                    timewait=self.rpc_timeout,
+                    bitcoind=self.options.bitcoind,
+                    bitcoin_cli=self.options.bitcoincli,
+                    mocktime=self.mocktime,
+                    coverage_dir=None,
+                    cwd=self.options.tmpdir,
+                ))
                 self.nodes[i].args = args
                 self.start_node(i)
 
@@ -452,6 +481,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
             for node in self.nodes:
                 node.wait_for_rpc_connection()
 
+            # For backward compatibility of the python scripts with previous
+            # versions of the cache, set mocktime to Jan 1,
+            # 2014 + (201 * 10 * 60)"""
+            self.mocktime = 1388534400 + (201 * 10 * 60)
+
             # Create a 200-block-long chain; each of the 4 first nodes
             # gets 25 mature blocks and 25 immature.
             # Note: To preserve compatibility with older versions of
@@ -459,7 +493,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
             #
             # blocks are created with timestamps 10 minutes apart
             # starting from 2010 minutes in the past
-            self.enable_mocktime()
             block_time = self.mocktime - (201 * 10 * 60)
             for i in range(2):
                 for peer in range(4):
@@ -473,7 +506,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
             # Shut them down, and clean up cache directories:
             self.stop_nodes()
             self.nodes = []
-            self.disable_mocktime()
+            self.mocktime = 0
 
             def cache_path(n, *paths):
                 return os.path.join(get_datadir_path(self.options.cachedir, n), "regtest", *paths)

test/bitcoin_functional/functional/test_framework/test_node.py

@@ -18,6 +18,8 @@ import tempfile
 import time
 import urllib.parse
 import collections
+import shlex
+import sys
 
 from .authproxy import JSONRPCException
 from .util import (
@@ -59,7 +61,13 @@ class TestNode():
     To make things easier for the test writer, any unrecognised messages will
     be dispatched to the RPC connection."""
 
-    def __init__(self, i, datadir, *, rpchost, timewait, bitcoind, bitcoin_cli, mocktime, coverage_dir, extra_conf=None, extra_args=None, use_cli=False):
+    def __init__(self, i, datadir, *, rpchost, timewait, bitcoind, bitcoin_cli, mocktime, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False):
+        """
+        Kwargs:
+            start_perf (bool): If True, begin profiling the node with `perf` as soon as
+                the node starts.
+        """
+
         self.index = i
         self.datadir = datadir
         self.stdout_dir = os.path.join(self.datadir, "stdout")
@@ -68,7 +76,8 @@ class TestNode():
         self.rpc_timeout = timewait
         self.binary = bitcoind
         self.coverage_dir = coverage_dir
-        if extra_conf != None:
+        self.cwd = cwd
+        if extra_conf is not None:
             append_config(datadir, extra_conf)
         # Most callers will just need to add extra args to the standard list below.
         # For those callers that need more flexibility, they can just set the args property directly.
@@ -87,6 +96,7 @@ class TestNode():
 
         self.cli = TestNodeCLI(bitcoin_cli, self.datadir)
         self.use_cli = use_cli
+        self.start_perf = start_perf
 
         self.running = False
         self.process = None
@@ -95,6 +105,8 @@ class TestNode():
         self.url = None
         self.log = logging.getLogger('TestFramework.node%d' % i)
         self.cleanup_on_exit = True # Whether to kill the node when this object goes away
+        # Cache perf subprocesses here by their data output filename.
+        self.perf_subprocesses = {}
 
         self.p2ps = []
 
@@ -115,7 +127,7 @@ class TestNode():
         ]
         return PRIV_KEYS[self.index]
 
-    def get_mem_rss(self):
+    def get_mem_rss_kilobytes(self):
         """Get the memory usage (RSS) per `ps`.
 
         Returns None if `ps` is unavailable.
@@ -160,7 +172,7 @@ class TestNode():
             assert self.rpc_connected and self.rpc is not None, self._node_msg("Error: no RPC connection")
             return getattr(self.rpc, name)
 
-    def start(self, extra_args=None, *, stdout=None, stderr=None, **kwargs):
+    def start(self, extra_args=None, *, cwd=None, stdout=None, stderr=None, **kwargs):
         """Start the node."""
         if extra_args is None:
             extra_args = self.extra_args
@@ -173,6 +185,9 @@ class TestNode():
         self.stderr = stderr
         self.stdout = stdout
 
+        if cwd is None:
+            cwd = self.cwd
+
         # Delete any existing cookie file -- if such a file exists (eg due to
         # unclean shutdown), it will get overwritten anyway by bitcoind, and
         # potentially interfere with our attempt to authenticate
@@ -181,11 +196,14 @@ class TestNode():
         # add environment variable LIBC_FATAL_STDERR_=1 so that libc errors are written to stderr and not the terminal
         subp_env = dict(os.environ, LIBC_FATAL_STDERR_="1")
 
-        self.process = subprocess.Popen(self.args + extra_args, env=subp_env, stdout=stdout, stderr=stderr, **kwargs)
+        self.process = subprocess.Popen(self.args + extra_args, env=subp_env, stdout=stdout, stderr=stderr, cwd=cwd, **kwargs)
 
         self.running = True
         self.log.debug("bitcoind started, waiting for RPC to come up")
 
+        if self.start_perf:
+            self._start_perf()
+
     def wait_for_rpc_connection(self):
         """Sets up an RPC connection to the bitcoind process. Returns False if unable to connect."""
         # Poll at a rate of four times per second
@@ -195,12 +213,15 @@ class TestNode():
                 raise FailedToStartError(self._node_msg(
                     'bitcoind exited with status {} during initialization'.format(self.process.returncode)))
             try:
-                self.rpc = get_rpc_proxy(rpc_url(self.datadir, self.index, self.rpchost), self.index, timeout=self.rpc_timeout, coveragedir=self.coverage_dir)
-                self.rpc.getblockcount()
+                rpc = get_rpc_proxy(rpc_url(self.datadir, self.index, self.rpchost), self.index, timeout=self.rpc_timeout, coveragedir=self.coverage_dir)
+                rpc.getblockcount()
                 # If the call to getblockcount() succeeds then the RPC connection is up
+                self.log.debug("RPC successfully started")
+                if self.use_cli:
+                    return
+                self.rpc = rpc
                 self.rpc_connected = True
                 self.url = self.rpc.url
-                self.log.debug("RPC successfully started")
                 return
             except IOError as e:
                 if e.errno != errno.ECONNREFUSED:  # Port not yet open?
@@ -228,16 +249,20 @@ class TestNode():
             wallet_path = "wallet/{}".format(urllib.parse.quote(wallet_name))
             return self.rpc / wallet_path
 
-    def stop_node(self, expected_stderr=''):
+    def stop_node(self, expected_stderr='', wait=0):
         """Stop the node."""
         if not self.running:
             return
         self.log.debug("Stopping node")
         try:
-            self.stop()
+            self.stop(wait=wait)
         except http.client.CannotSendRequest:
             self.log.exception("Unable to stop node.")
 
+        # If there are any running perf processes, stop them.
+        for profile_name in tuple(self.perf_subprocesses.keys()):
+            self._stop_perf(profile_name)
+
         # Check that stderr is as expected
         self.stderr.seek(0)
         stderr = self.stderr.read().decode('utf-8').strip()
@@ -291,15 +316,19 @@ class TestNode():
                     self._raise_assertion_error('Expected message "{}" does not partially match log:\n\n{}\n\n'.format(expected_msg, print_log))
 
     @contextlib.contextmanager
-    def assert_memory_usage_stable(self, perc_increase_allowed=0.03):
+    def assert_memory_usage_stable(self, *, increase_allowed=0.03):
         """Context manager that allows the user to assert that a node's memory usage (RSS)
         hasn't increased beyond some threshold percentage.
+
+        Args:
+            increase_allowed (float): the fractional increase in memory allowed until failure;
+                e.g. `0.12` for up to 12% increase allowed.
         """
-        before_memory_usage = self.get_mem_rss()
+        before_memory_usage = self.get_mem_rss_kilobytes()
 
         yield
 
-        after_memory_usage = self.get_mem_rss()
+        after_memory_usage = self.get_mem_rss_kilobytes()
 
         if not (before_memory_usage and after_memory_usage):
             self.log.warning("Unable to detect memory usage (RSS) - skipping memory check.")
@@ -307,12 +336,90 @@ class TestNode():
 
         perc_increase_memory_usage = (after_memory_usage / before_memory_usage) - 1
 
-        if perc_increase_memory_usage > perc_increase_allowed:
+        if perc_increase_memory_usage > increase_allowed:
             self._raise_assertion_error(
                 "Memory usage increased over threshold of {:.3f}% from {} to {} ({:.3f}%)".format(
-                    perc_increase_allowed * 100, before_memory_usage, after_memory_usage,
+                    increase_allowed * 100, before_memory_usage, after_memory_usage,
                     perc_increase_memory_usage * 100))
 
+    @contextlib.contextmanager
+    def profile_with_perf(self, profile_name):
+        """
+        Context manager that allows easy profiling of node activity using `perf`.
+
+        See `test/functional/README.md` for details on perf usage.
+
+        Args:
+            profile_name (str): This string will be appended to the
+                profile data filename generated by perf.
+        """
+        subp = self._start_perf(profile_name)
+
+        yield
+
+        if subp:
+            self._stop_perf(profile_name)
+
+    def _start_perf(self, profile_name=None):
+        """Start a perf process to profile this node.
+
+        Returns the subprocess running perf."""
+        subp = None
+
+        def test_success(cmd):
+            return subprocess.call(
+                # shell=True required for pipe use below
+                cmd, shell=True,
+                stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) == 0
+
+        if not sys.platform.startswith('linux'):
+            self.log.warning("Can't profile with perf; only availabe on Linux platforms")
+            return None
+
+        if not test_success('which perf'):
+            self.log.warning("Can't profile with perf; must install perf-tools")
+            return None
+
+        if not test_success('readelf -S {} | grep .debug_str'.format(shlex.quote(self.binary))):
+            self.log.warning(
+                "perf output won't be very useful without debug symbols compiled into bitcoind")
+
+        output_path = tempfile.NamedTemporaryFile(
+            dir=self.datadir,
+            prefix="{}.perf.data.".format(profile_name or 'test'),
+            delete=False,
+        ).name
+
+        cmd = [
+            'perf', 'record',
+            '-g',                     # Record the callgraph.
+            '--call-graph', 'dwarf',  # Compatibility for gcc's --fomit-frame-pointer.
+            '-F', '101',              # Sampling frequency in Hz.
+            '-p', str(self.process.pid),
+            '-o', output_path,
+        ]
+        subp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        self.perf_subprocesses[profile_name] = subp
+
+        return subp
+
+    def _stop_perf(self, profile_name):
+        """Stop (and pop) a perf subprocess."""
+        subp = self.perf_subprocesses.pop(profile_name)
+        output_path = subp.args[subp.args.index('-o') + 1]
+
+        subp.terminate()
+        subp.wait(timeout=10)
+
+        stderr = subp.stderr.read().decode()
+        if 'Consider tweaking /proc/sys/kernel/perf_event_paranoid' in stderr:
+            self.log.warning(
+                "perf couldn't collect data! Try "
+                "'sudo sysctl -w kernel.perf_event_paranoid=-1'")
+        else:
+            report_cmd = "perf report -i {}".format(output_path)
+            self.log.info("See perf output by running '{}'".format(report_cmd))
+
     def assert_start_raises_init_error(self, extra_args=None, expected_msg=None, match=ErrorMatch.FULL_TEXT, *args, **kwargs):
         """Attempt to start the node and expect it to raise an error.
 
@@ -398,6 +505,14 @@ class TestNodeCLIAttr:
     def get_request(self, *args, **kwargs):
         return lambda: self(*args, **kwargs)
 
+def arg_to_cli(arg):
+    if isinstance(arg, bool):
+        return str(arg).lower()
+    elif isinstance(arg, dict) or isinstance(arg, list):
+        return json.dumps(arg)
+    else:
+        return str(arg)
+
 class TestNodeCLI():
     """Interface to bitcoin-cli for an individual node"""
 
@@ -429,8 +544,8 @@ class TestNodeCLI():
 
     def send_cli(self, command=None, *args, **kwargs):
         """Run bitcoin-cli command. Deserializes returned string as python object."""
-        pos_args = [str(arg).lower() if type(arg) is bool else str(arg) for arg in args]
-        named_args = [str(key) + "=" + str(value) for (key, value) in kwargs.items()]
+        pos_args = [arg_to_cli(arg) for arg in args]
+        named_args = [str(key) + "=" + arg_to_cli(value) for (key, value) in kwargs.items()]
         assert not (pos_args and named_args), "Cannot use positional arguments and named arguments in the same bitcoin-cli call"
         p_args = [self.binary, "-datadir=" + self.datadir] + self.options
         if named_args:

test/bitcoin_functional/functional/test_framework/util.py

@@ -346,7 +346,7 @@ def get_auth_cookie(datadir):
                 if line.startswith("rpcpassword="):
                     assert password is None  # Ensure that there is only one rpcpassword line
                     password = line.split("=")[1].strip("\n")
-    if os.path.isfile(os.path.join(datadir, "regtest", ".cookie")):
+    if os.path.isfile(os.path.join(datadir, "regtest", ".cookie")) and os.access(os.path.join(datadir, "regtest", ".cookie"), os.R_OK):
         with open(os.path.join(datadir, "regtest", ".cookie"), 'r', encoding="ascii") as f:
             userpass = f.read()
             split_userpass = userpass.split(':')
@@ -430,12 +430,12 @@ def sync_mempools(rpc_connections, *, wait=1, timeout=60, flush_scheduler=True):
 # Transaction/Block functions
 #############################
 
-def find_output(node, txid, amount):
+def find_output(node, txid, amount, *, blockhash=None):
     """
     Return index to output of txid with value amount
     Raises exception if there is none.
     """
-    txdata = node.getrawtransaction(txid, 1)
+    txdata = node.getrawtransaction(txid, 1, blockhash)
     for i in range(len(txdata["vout"])):
         if txdata["vout"][i]["value"] == amount:
             return i

test/bitcoin_functional/functional/test_framework/wallet_util.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Useful util functions for testing the wallet"""
+from collections import namedtuple
+
+from test_framework.address import (
+    key_to_p2pkh,
+    key_to_p2sh_p2wpkh,
+    key_to_p2wpkh,
+    script_to_p2sh,
+    script_to_p2sh_p2wsh,
+    script_to_p2wsh,
+)
+from test_framework.script import (
+    CScript,
+    OP_0,
+    OP_2,
+    OP_3,
+    OP_CHECKMULTISIG,
+    OP_CHECKSIG,
+    OP_DUP,
+    OP_EQUAL,
+    OP_EQUALVERIFY,
+    OP_HASH160,
+    hash160,
+    sha256,
+)
+from test_framework.util import hex_str_to_bytes
+
+Key = namedtuple('Key', ['privkey',
+                         'pubkey',
+                         'p2pkh_script',
+                         'p2pkh_addr',
+                         'p2wpkh_script',
+                         'p2wpkh_addr',
+                         'p2sh_p2wpkh_script',
+                         'p2sh_p2wpkh_redeem_script',
+                         'p2sh_p2wpkh_addr'])
+
+Multisig = namedtuple('Multisig', ['privkeys',
+                                   'pubkeys',
+                                   'p2sh_script',
+                                   'p2sh_addr',
+                                   'redeem_script',
+                                   'p2wsh_script',
+                                   'p2wsh_addr',
+                                   'p2sh_p2wsh_script',
+                                   'p2sh_p2wsh_addr'])
+
+def get_key(node):
+    """Generate a fresh key on node
+
+    Returns a named tuple of privkey, pubkey and all address and scripts."""
+    addr = node.getnewaddress()
+    pubkey = node.getaddressinfo(addr)['pubkey']
+    pkh = hash160(hex_str_to_bytes(pubkey))
+    return Key(privkey=node.dumpprivkey(addr),
+               pubkey=pubkey,
+               p2pkh_script=CScript([OP_DUP, OP_HASH160, pkh, OP_EQUALVERIFY, OP_CHECKSIG]).hex(),
+               p2pkh_addr=key_to_p2pkh(pubkey),
+               p2wpkh_script=CScript([OP_0, pkh]).hex(),
+               p2wpkh_addr=key_to_p2wpkh(pubkey),
+               p2sh_p2wpkh_script=CScript([OP_HASH160, hash160(CScript([OP_0, pkh])), OP_EQUAL]).hex(),
+               p2sh_p2wpkh_redeem_script=CScript([OP_0, pkh]).hex(),
+               p2sh_p2wpkh_addr=key_to_p2sh_p2wpkh(pubkey))
+
+def get_multisig(node):
+    """Generate a fresh 2-of-3 multisig on node
+
+    Returns a named tuple of privkeys, pubkeys and all address and scripts."""
+    addrs = []
+    pubkeys = []
+    for _ in range(3):
+        addr = node.getaddressinfo(node.getnewaddress())
+        addrs.append(addr['address'])
+        pubkeys.append(addr['pubkey'])
+    script_code = CScript([OP_2] + [hex_str_to_bytes(pubkey) for pubkey in pubkeys] + [OP_3, OP_CHECKMULTISIG])
+    witness_script = CScript([OP_0, sha256(script_code)])
+    return Multisig(privkeys=[node.dumpprivkey(addr) for addr in addrs],
+                    pubkeys=pubkeys,
+                    p2sh_script=CScript([OP_HASH160, hash160(script_code), OP_EQUAL]).hex(),
+                    p2sh_addr=script_to_p2sh(script_code),
+                    redeem_script=script_code.hex(),
+                    p2wsh_script=witness_script.hex(),
+                    p2wsh_addr=script_to_p2wsh(script_code),
+                    p2sh_p2wsh_script=CScript([OP_HASH160, witness_script, OP_EQUAL]).hex(),
+                    p2sh_p2wsh_addr=script_to_p2sh_p2wsh(script_code))
+
+def test_address(node, address, **kwargs):
+    """Get address info for `address` and test whether the returned values are as expected."""
+    addr_info = node.getaddressinfo(address)
+    for key, value in kwargs.items():
+        if value is None:
+            if key in addr_info.keys():
+                raise AssertionError("key {} unexpectedly returned in getaddressinfo.".format(key))
+        elif addr_info[key] != value:
+            raise AssertionError("key {} value {} did not match expected value {}".format(key, addr_info[key], value))

test/bitcoin_functional/functional/test_runner.py

@@ -7,8 +7,6 @@
 This module calls down into individual test cases via subprocess. It will
 forward all unrecognized arguments onto the individual test scripts.
 
-Functional tests are disabled on Windows by default. Use --force to run them anyway.
-
 For a description of arguments recognized by test scripts, see
 `test/functional/test_framework/test_framework.py:BitcoinTestFramework.main`.
 
@@ -109,6 +107,7 @@ BASE_SCRIPTS = [
     #'interface_bitcoin_cli.py',
     'mempool_resurrect.py',
     'wallet_txn_doublespend.py --mineblock',
+    'tool_wallet.py',
     'wallet_txn_clone.py',
     'wallet_txn_clone.py --segwit',
     'rpc_getchaintips.py',
@@ -118,9 +117,10 @@ BASE_SCRIPTS = [
     'mempool_persist.py',
     'wallet_multiwallet.py',
     'wallet_multiwallet.py --usecli',
-    'wallet_disableprivatekeys.py',
-    'wallet_disableprivatekeys.py --usecli',
+    'wallet_createwallet.py',
+    'wallet_createwallet.py --usecli',
     'interface_http.py',
+    'interface_rpc.py',
     'rpc_psbt.py',
     # ELEMENTS:
     #'rpc_users.py',
@@ -155,6 +155,7 @@ BASE_SCRIPTS = [
     'wallet_importprunedfunds.py',
     'p2p_leak_tx.py',
     'rpc_signmessage.py',
+    'wallet_balance.py',
     'feature_nulldummy.py',
     'mempool_accept.py',
     'wallet_import_rescan.py',
@@ -175,12 +176,16 @@ BASE_SCRIPTS = [
     'wallet_fallbackfee.py',
     'feature_minchainwork.py',
     'rpc_getblockstats.py',
+    'wallet_create_tx.py',
     'p2p_fingerprint.py',
     'feature_uacomment.py',
+    'wallet_coinbase_category.py',
     'feature_filelock.py',
     'p2p_unrequested_blocks.py',
     # ELEMENTS:
     #'feature_includeconf.py',
+    'rpc_deriveaddresses.py',
+    'rpc_deriveaddresses.py --usecli',
     'rpc_scantxoutset.py',
     'feature_logging.py',
     'p2p_node_network_limited.py',
@@ -189,6 +194,7 @@ BASE_SCRIPTS = [
     #'feature_config_args.py',
     'rpc_help.py',
     'feature_help.py',
+    'feature_shutdown.py',
     # Don't append tests at the end to avoid merge conflicts
     # Put them in a random line within the section that fits their approximate run-time
 ]
@@ -223,7 +229,6 @@ def main():
     parser.add_argument('--ci', action='store_true', help='Run checks and code that are usually only enabled in a continuous integration environment')
     parser.add_argument('--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.')
     parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests')
-    parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).')
     parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit')
     parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.')
     parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.')
@@ -251,22 +256,12 @@ def main():
     # Create base test directory
     tmpdir = "%s/test_runner_₿_🏃_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
 
-    # If we fixed the command-line and filename encoding issue on Windows, these two lines could be removed
-    if config["environment"]["EXEEXT"] == ".exe":
-        tmpdir = "%s/test_runner_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
-
     os.makedirs(tmpdir)
 
     logging.debug("Temporary test directory at %s" % tmpdir)
 
     enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND")
 
-    if config["environment"]["EXEEXT"] == ".exe" and not args.force:
-        # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
-        # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
-        print("Tests currently disabled on Windows by default. Use --force option to enable")
-        sys.exit(0)
-
     if not enable_bitcoind:
         print("No functional tests to run.")
         print("Rerun ./configure with --with-daemon and then make")
@@ -277,7 +272,7 @@ def main():
     if tests:
         # Individual tests have been specified. Run specified tests that exist
         # in the ALL_SCRIPTS list. Accept the name with or without .py extension.
-        tests = [re.sub("\.py$", "", test) + ".py" for test in tests]
+        tests = [test + ".py" if ".py" not in test else test for test in tests]
         for test in tests:
             if test in ALL_SCRIPTS:
                 test_list.append(test)
@@ -565,7 +560,7 @@ class TestResult():
 def check_script_prefixes():
     """Check that test scripts start with one of the allowed name prefixes."""
 
-    good_prefixes_re = re.compile("(example|feature|interface|mempool|mining|p2p|rpc|wallet)_")
+    good_prefixes_re = re.compile("(example|feature|interface|mempool|mining|p2p|rpc|wallet|tool)_")
     bad_script_names = [script for script in ALL_SCRIPTS if good_prefixes_re.match(script) is None]
 
     if bad_script_names:

test/bitcoin_functional/functional/tool_wallet.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test bitcoin-wallet."""
+import subprocess
+import textwrap
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal
+
+class ToolWalletTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 1
+        self.setup_clean_chain = True
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_wallet()
+
+    def bitcoin_wallet_process(self, *args):
+        binary = self.config["environment"]["BUILDDIR"] + '/src/bitcoin-wallet' + self.config["environment"]["EXEEXT"]
+        args = ['-datadir={}'.format(self.nodes[0].datadir), '-regtest'] + list(args)
+        return subprocess.Popen([binary] + args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+
+    def assert_raises_tool_error(self, error, *args):
+        p = self.bitcoin_wallet_process(*args)
+        stdout, stderr = p.communicate()
+        assert_equal(p.poll(), 1)
+        assert_equal(stdout, '')
+        assert_equal(stderr.strip(), error)
+
+    def assert_tool_output(self, output, *args):
+        p = self.bitcoin_wallet_process(*args)
+        stdout, stderr = p.communicate()
+        assert_equal(p.poll(), 0)
+        assert_equal(stderr, '')
+        assert_equal(stdout, output)
+
+    def run_test(self):
+
+        self.assert_raises_tool_error('Invalid command: foo', 'foo')
+        # `bitcoin-wallet help` is an error. Use `bitcoin-wallet -help`
+        self.assert_raises_tool_error('Invalid command: help', 'help')
+        self.assert_raises_tool_error('Error: two methods provided (info and create). Only one method should be provided.', 'info', 'create')
+        self.assert_raises_tool_error('Error parsing command line arguments: Invalid parameter -foo', '-foo')
+        self.assert_raises_tool_error('Error loading wallet.dat. Is wallet being used by other process?', '-wallet=wallet.dat', 'info')
+        self.assert_raises_tool_error('Error: no wallet file at nonexistent.dat', '-wallet=nonexistent.dat', 'info')
+
+        # stop the node to close the wallet to call info command
+        self.stop_node(0)
+
+        out = textwrap.dedent('''\
+            Wallet info
+            ===========
+            Encrypted: no
+            HD (hd seed available): yes
+            Keypool Size: 2
+            Transactions: 0
+            Address Book: 3
+        ''')
+        self.assert_tool_output(out, '-wallet=wallet.dat', 'info')
+
+        # mutate the wallet to check the info command output changes accordingly
+        self.start_node(0)
+        self.nodes[0].generate(1)
+        self.stop_node(0)
+
+        out = textwrap.dedent('''\
+            Wallet info
+            ===========
+            Encrypted: no
+            HD (hd seed available): yes
+            Keypool Size: 2
+            Transactions: 1
+            Address Book: 3
+        ''')
+        self.assert_tool_output(out, '-wallet=wallet.dat', 'info')
+
+        out = textwrap.dedent('''\
+            Topping up keypool...
+            Wallet info
+            ===========
+            Encrypted: no
+            HD (hd seed available): yes
+            Keypool Size: 2000
+            Transactions: 0
+            Address Book: 0
+        ''')
+        self.assert_tool_output(out, '-wallet=foo', 'create')
+
+        self.start_node(0, ['-wallet=foo'])
+        out = self.nodes[0].getwalletinfo()
+        self.stop_node(0)
+
+        assert_equal(0, out['txcount'])
+        assert_equal(1000, out['keypoolsize'])
+        assert_equal(1000, out['keypoolsize_hd_internal'])
+        assert_equal(True, 'hdseedid' in out)
+
+if __name__ == '__main__':
+    ToolWalletTest().main()

test/bitcoin_functional/functional/wallet_abandonconflict.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2014-2018 The Bitcoin Core developers
+# Copyright (c) 2014-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test the abandontransaction RPC.
@@ -13,7 +13,15 @@
 from decimal import Decimal
 
 from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes, disconnect_nodes, sync_blocks, sync_mempools
+from test_framework.util import (
+    assert_equal,
+    assert_raises_rpc_error,
+    connect_nodes,
+    disconnect_nodes,
+    sync_blocks,
+    sync_mempools,
+)
+
 
 class AbandonConflictTest(BitcoinTestFramework):
     def set_test_params(self):
@@ -40,21 +48,21 @@ class AbandonConflictTest(BitcoinTestFramework):
 
         sync_blocks(self.nodes)
         newbalance = self.nodes[0].getbalance()
-        assert(balance - newbalance < Decimal("0.001")) #no more than fees lost
+        assert balance - newbalance < Decimal("0.001")  #no more than fees lost
         balance = newbalance
 
         # Disconnect nodes so node0's transactions don't get into node1's mempool
         disconnect_nodes(self.nodes[0], 1)
 
         # Identify the 10btc outputs
-        nA = next(i for i, vout in enumerate(self.nodes[0].getrawtransaction(txA, 1)["vout"]) if vout["value"] == Decimal("10"))
-        nB = next(i for i, vout in enumerate(self.nodes[0].getrawtransaction(txB, 1)["vout"]) if vout["value"] == Decimal("10"))
-        nC = next(i for i, vout in enumerate(self.nodes[0].getrawtransaction(txC, 1)["vout"]) if vout["value"] == Decimal("10"))
+        nA = next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(txA)["details"] if tx_out["amount"] == Decimal("10"))
+        nB = next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(txB)["details"] if tx_out["amount"] == Decimal("10"))
+        nC = next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(txC)["details"] if tx_out["amount"] == Decimal("10"))
 
-        inputs =[]
+        inputs = []
         # spend 10btc outputs from txA and txB
-        inputs.append({"txid":txA, "vout":nA})
-        inputs.append({"txid":txB, "vout":nB})
+        inputs.append({"txid": txA, "vout": nA})
+        inputs.append({"txid": txB, "vout": nB})
         outputs = {}
 
         outputs[self.nodes[0].getnewaddress()] = Decimal("14.99998")
@@ -63,12 +71,12 @@ class AbandonConflictTest(BitcoinTestFramework):
         txAB1 = self.nodes[0].sendrawtransaction(signed["hex"])
 
         # Identify the 14.99998btc output
-        nAB = next(i for i, vout in enumerate(self.nodes[0].getrawtransaction(txAB1, 1)["vout"]) if vout["value"] == Decimal("14.99998"))
+        nAB = next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(txAB1)["details"] if tx_out["amount"] == Decimal("14.99998"))
 
         #Create a child tx spending AB1 and C
         inputs = []
-        inputs.append({"txid":txAB1, "vout":nAB})
-        inputs.append({"txid":txC, "vout":nC})
+        inputs.append({"txid": txAB1, "vout": nAB})
+        inputs.append({"txid": txC, "vout": nC})
         outputs = {}
         outputs[self.nodes[0].getnewaddress()] = Decimal("24.9996")
         signed2 = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs, outputs))
@@ -76,8 +84,8 @@ class AbandonConflictTest(BitcoinTestFramework):
 
         # Create a child tx spending ABC2
         signed3_change = Decimal("24.999")
-        inputs = [ {"txid":txABC2, "vout":0} ]
-        outputs = { self.nodes[0].getnewaddress(): signed3_change }
+        inputs = [{"txid": txABC2, "vout": 0}]
+        outputs = {self.nodes[0].getnewaddress(): signed3_change}
         signed3 = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs, outputs))
         # note tx is never directly referenced, only abandoned as a child of the above
         self.nodes[0].sendrawtransaction(signed3["hex"])
@@ -105,7 +113,7 @@ class AbandonConflictTest(BitcoinTestFramework):
         unconfbalance = self.nodes[0].getunconfirmedbalance() + self.nodes[0].getbalance()
         assert_equal(unconfbalance, newbalance)
         # Also shouldn't show up in listunspent
-        assert(not txABC2 in [utxo["txid"] for utxo in self.nodes[0].listunspent(0)])
+        assert not txABC2 in [utxo["txid"] for utxo in self.nodes[0].listunspent(0)]
         balance = newbalance
 
         # Abandon original transaction and verify inputs are available again
@@ -145,8 +153,8 @@ class AbandonConflictTest(BitcoinTestFramework):
 
         # Create a double spend of AB1 by spending again from only A's 10 output
         # Mine double spend from node 1
-        inputs =[]
-        inputs.append({"txid":txA, "vout":nA})
+        inputs = []
+        inputs.append({"txid": txA, "vout": nA})
         outputs = {}
         outputs[self.nodes[1].getnewaddress()] = Decimal("9.9999")
         tx = self.nodes[0].createrawtransaction(inputs, outputs)
@@ -172,5 +180,6 @@ class AbandonConflictTest(BitcoinTestFramework):
         self.log.info("conflicted has not resumed causing its inputs to be seen as spent.  See Issue #7315")
         self.log.info(str(balance) + " -> " + str(newbalance) + " ?")
 
+
 if __name__ == '__main__':
     AbandonConflictTest().main()

test/bitcoin_functional/functional/wallet_address_types.py

@@ -54,6 +54,10 @@ from decimal import Decimal
 import itertools
 
 from test_framework.test_framework import BitcoinTestFramework
+from test_framework.descriptors import (
+    descsum_create,
+    descsum_check,
+)
 from test_framework.util import (
     assert_equal,
     assert_greater_than,
@@ -99,6 +103,8 @@ class AddressTypeTest(BitcoinTestFramework):
         """Run sanity checks on an address."""
         info = self.nodes[node].getaddressinfo(address)
         assert(self.nodes[node].validateaddress(address)['isvalid'])
+        assert_equal(info.get('solvable'), True)
+
         if not multisig and typ == 'legacy':
             # P2PKH
             assert(not info['isscript'])
@@ -146,6 +152,54 @@ class AddressTypeTest(BitcoinTestFramework):
             # Unknown type
             assert(False)
 
+    def test_desc(self, node, address, multisig, typ, utxo):
+        """Run sanity checks on a descriptor reported by getaddressinfo."""
+        info = self.nodes[node].getaddressinfo(address)
+        assert('desc' in info)
+        assert_equal(info['desc'], utxo['desc'])
+        assert(self.nodes[node].validateaddress(address)['isvalid'])
+
+        # Use a ridiculously roundabout way to find the key origin info through
+        # the PSBT logic. However, this does test consistency between the PSBT reported
+        # fingerprints/paths and the descriptor logic.
+        psbt = self.nodes[node].createpsbt([{'txid':utxo['txid'], 'vout':utxo['vout']}],[{address:0.00010000}])
+        psbt = self.nodes[node].walletprocesspsbt(psbt, False, "ALL", True)
+        decode = self.nodes[node].decodepsbt(psbt['psbt'])
+        key_descs = {}
+        for deriv in decode['inputs'][0]['bip32_derivs']:
+            assert_equal(len(deriv['master_fingerprint']), 8)
+            assert_equal(deriv['path'][0], 'm')
+            key_descs[deriv['pubkey']] = '[' + deriv['master_fingerprint'] + deriv['path'][1:] + ']' + deriv['pubkey']
+
+        # Verify the descriptor checksum against the Python implementation
+        assert(descsum_check(info['desc']))
+        # Verify that stripping the checksum and recreating it using Python roundtrips
+        assert(info['desc'] == descsum_create(info['desc'][:-9]))
+        # Verify that stripping the checksum and feeding it to getdescriptorinfo roundtrips
+        assert(info['desc'] == self.nodes[0].getdescriptorinfo(info['desc'][:-9])['descriptor'])
+
+        if not multisig and typ == 'legacy':
+            # P2PKH
+            assert_equal(info['desc'], descsum_create("pkh(%s)" % key_descs[info['pubkey']]))
+        elif not multisig and typ == 'p2sh-segwit':
+            # P2SH-P2WPKH
+            assert_equal(info['desc'], descsum_create("sh(wpkh(%s))" % key_descs[info['pubkey']]))
+        elif not multisig and typ == 'bech32':
+            # P2WPKH
+            assert_equal(info['desc'], descsum_create("wpkh(%s)" % key_descs[info['pubkey']]))
+        elif typ == 'legacy':
+            # P2SH-multisig
+            assert_equal(info['desc'], descsum_create("sh(multi(2,%s,%s))" % (key_descs[info['pubkeys'][0]], key_descs[info['pubkeys'][1]])))
+        elif typ == 'p2sh-segwit':
+            # P2SH-P2WSH-multisig
+            assert_equal(info['desc'], descsum_create("sh(wsh(multi(2,%s,%s)))" % (key_descs[info['embedded']['pubkeys'][0]], key_descs[info['embedded']['pubkeys'][1]])))
+        elif typ == 'bech32':
+            # P2WSH-multisig
+            assert_equal(info['desc'], descsum_create("wsh(multi(2,%s,%s))" % (key_descs[info['pubkeys'][0]], key_descs[info['pubkeys'][1]])))
+        else:
+            # Unknown type
+            assert(False)
+
     def test_change_output_type(self, node_sender, destinations, expected_type):
         txid = self.nodes[node_sender].sendmany(dummy="", amounts=dict.fromkeys(destinations, 0.001))
         raw_tx = self.nodes[node_sender].getrawtransaction(txid)
@@ -198,6 +252,7 @@ class AddressTypeTest(BitcoinTestFramework):
             self.log.debug("Old balances are {}".format(old_balances))
             to_send = (old_balances[from_node] / 101).quantize(Decimal("0.00000001"))
             sends = {}
+            addresses = {}
 
             self.log.debug("Prepare sends")
             for n, to_node in enumerate(range(from_node, from_node + 4)):
@@ -228,6 +283,7 @@ class AddressTypeTest(BitcoinTestFramework):
 
                 # Output entry
                 sends[address] = to_send * 10 * (1 + n)
+                addresses[to_node] = (address, typ)
 
             self.log.debug("Sending: {}".format(sends))
             self.nodes[from_node].sendmany("", sends)
@@ -244,6 +300,17 @@ class AddressTypeTest(BitcoinTestFramework):
             self.nodes[5].generate(1)
             sync_blocks(self.nodes)
 
+            # Verify that the receiving wallet contains a UTXO with the expected address, and expected descriptor
+            for n, to_node in enumerate(range(from_node, from_node + 4)):
+                to_node %= 4
+                found = False
+                for utxo in self.nodes[to_node].listunspent():
+                    if utxo['address'] == addresses[to_node][0]:
+                        found = True
+                        self.test_desc(to_node, addresses[to_node][0], multisig, addresses[to_node][1], utxo)
+                        break
+                assert found
+
             new_balances = self.get_balances()
             self.log.debug("Check new balances: {}".format(new_balances))
             # We don't know what fee was set, so we can only check bounds on the balance of the sending node

test/bitcoin_functional/functional/wallet_backup.py

@@ -48,7 +48,7 @@ class WalletBackupTest(BitcoinTestFramework):
     def skip_test_if_missing_module(self):
         self.skip_if_no_wallet()
 
-    def setup_network(self, split=False):
+    def setup_network(self):
         self.setup_nodes()
         connect_nodes(self.nodes[0], 3)
         connect_nodes(self.nodes[1], 3)

test/bitcoin_functional/functional/wallet_balance.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test the wallet balance RPC methods."""
+from decimal import Decimal
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    assert_equal,
+    assert_raises_rpc_error,
+)
+
+RANDOM_COINBASE_ADDRESS = 'mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ'
+
+def create_transactions(node, address, amt, fees):
+    # Create and sign raw transactions from node to address for amt.
+    # Creates a transaction for each fee and returns an array
+    # of the raw transactions.
+    utxos = node.listunspent(0)
+
+    # Create transactions
+    inputs = []
+    ins_total = 0
+    for utxo in utxos:
+        inputs.append({"txid": utxo["txid"], "vout": utxo["vout"]})
+        ins_total += utxo['amount']
+        if ins_total > amt:
+            break
+
+    txs = []
+    for fee in fees:
+        outputs = {address: amt, node.getrawchangeaddress(): ins_total - amt - fee}
+        raw_tx = node.createrawtransaction(inputs, outputs, 0, True)
+        raw_tx = node.signrawtransactionwithwallet(raw_tx)
+        txs.append(raw_tx)
+
+    return txs
+
+class WalletTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 2
+        self.setup_clean_chain = True
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_wallet()
+
+    def run_test(self):
+        # Check that nodes don't own any UTXOs
+        assert_equal(len(self.nodes[0].listunspent()), 0)
+        assert_equal(len(self.nodes[1].listunspent()), 0)
+
+        self.log.info("Mining one block for each node")
+
+        self.nodes[0].generate(1)
+        self.sync_all()
+        self.nodes[1].generate(1)
+        self.nodes[1].generatetoaddress(100, RANDOM_COINBASE_ADDRESS)
+        self.sync_all()
+
+        assert_equal(self.nodes[0].getbalance(), 50)
+        assert_equal(self.nodes[1].getbalance(), 50)
+
+        self.log.info("Test getbalance with different arguments")
+        assert_equal(self.nodes[0].getbalance("*"), 50)
+        assert_equal(self.nodes[0].getbalance("*", 1), 50)
+        assert_equal(self.nodes[0].getbalance("*", 1, True), 50)
+        assert_equal(self.nodes[0].getbalance(minconf=1), 50)
+
+        # Send 40 BTC from 0 to 1 and 60 BTC from 1 to 0.
+        txs = create_transactions(self.nodes[0], self.nodes[1].getnewaddress(), 40, [Decimal('0.01')])
+        self.nodes[0].sendrawtransaction(txs[0]['hex'])
+        self.nodes[1].sendrawtransaction(txs[0]['hex'])  # sending on both nodes is faster than waiting for propagation
+
+        self.sync_all()
+        txs = create_transactions(self.nodes[1], self.nodes[0].getnewaddress(), 60, [Decimal('0.01'), Decimal('0.02')])
+        self.nodes[1].sendrawtransaction(txs[0]['hex'])
+        self.nodes[0].sendrawtransaction(txs[0]['hex'])  # sending on both nodes is faster than waiting for propagation
+        self.sync_all()
+
+        # First argument of getbalance must be set to "*"
+        assert_raises_rpc_error(-32, "dummy first argument must be excluded or set to \"*\"", self.nodes[1].getbalance, "")
+
+        self.log.info("Test getbalance and getunconfirmedbalance with unconfirmed inputs")
+
+        # getbalance without any arguments includes unconfirmed transactions, but not untrusted transactions
+        assert_equal(self.nodes[0].getbalance(), Decimal('9.99'))  # change from node 0's send
+        assert_equal(self.nodes[1].getbalance(), Decimal('29.99'))  # change from node 1's send
+        # Same with minconf=0
+        assert_equal(self.nodes[0].getbalance(minconf=0), Decimal('9.99'))
+        assert_equal(self.nodes[1].getbalance(minconf=0), Decimal('29.99'))
+        # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago
+        # TODO: fix getbalance tracking of coin spentness depth
+        assert_equal(self.nodes[0].getbalance(minconf=1), Decimal('0'))
+        assert_equal(self.nodes[1].getbalance(minconf=1), Decimal('0'))
+        # getunconfirmedbalance
+        assert_equal(self.nodes[0].getunconfirmedbalance(), Decimal('60'))  # output of node 1's spend
+        assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0'))  # Doesn't include output of node 0's send since it was spent
+
+        # Node 1 bumps the transaction fee and resends
+        self.nodes[1].sendrawtransaction(txs[1]['hex'])
+        self.sync_all()
+
+        self.log.info("Test getbalance and getunconfirmedbalance with conflicted unconfirmed inputs")
+
+        assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], Decimal('60'))  # output of node 1's send
+        assert_equal(self.nodes[0].getunconfirmedbalance(), Decimal('60'))
+        assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('0'))  # Doesn't include output of node 0's send since it was spent
+        assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0'))
+
+        self.nodes[1].generatetoaddress(1, RANDOM_COINBASE_ADDRESS)
+        self.sync_all()
+
+        # balances are correct after the transactions are confirmed
+        assert_equal(self.nodes[0].getbalance(), Decimal('69.99'))  # node 1's send plus change from node 0's send
+        assert_equal(self.nodes[1].getbalance(), Decimal('29.98'))  # change from node 0's send
+
+        # Send total balance away from node 1
+        txs = create_transactions(self.nodes[1], self.nodes[0].getnewaddress(), Decimal('29.97'), [Decimal('0.01')])
+        self.nodes[1].sendrawtransaction(txs[0]['hex'])
+        self.nodes[1].generatetoaddress(2, RANDOM_COINBASE_ADDRESS)
+        self.sync_all()
+
+        # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago
+        # TODO: fix getbalance tracking of coin spentness depth
+        # getbalance with minconf=3 should still show the old balance
+        assert_equal(self.nodes[1].getbalance(minconf=3), Decimal('0'))
+
+        # getbalance with minconf=2 will show the new balance.
+        assert_equal(self.nodes[1].getbalance(minconf=2), Decimal('0'))
+
+if __name__ == '__main__':
+    WalletTest().main()

test/bitcoin_functional/functional/wallet_basic.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2014-2018 The Bitcoin Core developers
+# Copyright (c) 2014-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test the wallet."""
@@ -19,6 +19,7 @@ from test_framework.util import (
     wait_until,
 )
 
+
 class WalletTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 4
@@ -67,15 +68,6 @@ class WalletTest(BitcoinTestFramework):
         assert_equal(self.nodes[1].getbalance(), 50)
         assert_equal(self.nodes[2].getbalance(), 0)
 
-        # Check getbalance with different arguments
-        assert_equal(self.nodes[0].getbalance("*"), 50)
-        assert_equal(self.nodes[0].getbalance("*", 1), 50)
-        assert_equal(self.nodes[0].getbalance("*", 1, True), 50)
-        assert_equal(self.nodes[0].getbalance(minconf=1), 50)
-
-        # first argument of getbalance must be excluded or set to "*"
-        assert_raises_rpc_error(-32, "dummy first argument must be excluded or set to \"*\"", self.nodes[0].getbalance, "")
-
         # Check that only first and second nodes have UTXOs
         utxos = self.nodes[0].listunspent()
         assert_equal(len(utxos), 1)
@@ -203,7 +195,7 @@ class WalletTest(BitcoinTestFramework):
         txid = self.nodes[2].sendtoaddress(address, 10, "", "", False)
         self.nodes[2].generate(1)
         self.sync_all([self.nodes[0:3]])
-        node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), Decimal('84'), fee_per_byte, self.get_vsize(self.nodes[2].getrawtransaction(txid)))
+        node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), Decimal('84'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex']))
         assert_equal(self.nodes[0].getbalance(), Decimal('10'))
 
         # Send 10 BTC with subtract fee from amount
@@ -212,14 +204,14 @@ class WalletTest(BitcoinTestFramework):
         self.sync_all([self.nodes[0:3]])
         node_2_bal -= Decimal('10')
         assert_equal(self.nodes[2].getbalance(), node_2_bal)
-        node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), Decimal('20'), fee_per_byte, self.get_vsize(self.nodes[2].getrawtransaction(txid)))
+        node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), Decimal('20'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex']))
 
         # Sendmany 10 BTC
         txid = self.nodes[2].sendmany('', {address: 10}, 0, "", [])
         self.nodes[2].generate(1)
         self.sync_all([self.nodes[0:3]])
         node_0_bal += Decimal('10')
-        node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), node_2_bal - Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].getrawtransaction(txid)))
+        node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), node_2_bal - Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex']))
         assert_equal(self.nodes[0].getbalance(), node_0_bal)
 
         # Sendmany 10 BTC with subtract fee from amount
@@ -228,7 +220,7 @@ class WalletTest(BitcoinTestFramework):
         self.sync_all([self.nodes[0:3]])
         node_2_bal -= Decimal('10')
         assert_equal(self.nodes[2].getbalance(), node_2_bal)
-        node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), node_0_bal + Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].getrawtransaction(txid)))
+        node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), node_0_bal + Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex']))
 
         # Test ResendWalletTransactions:
         # Create a couple of transactions, then start up a fourth
@@ -246,11 +238,7 @@ class WalletTest(BitcoinTestFramework):
         assert_equal(set(relayed), {txid1, txid2})
         sync_mempools(self.nodes)
 
-        assert(txid1 in self.nodes[3].getrawmempool())
-
-        # Exercise balance rpcs
-        assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], 1)
-        assert_equal(self.nodes[0].getunconfirmedbalance(), 1)
+        assert txid1 in self.nodes[3].getrawmempool()
 
         # check if we can list zero value tx as available coins
         # 1. create raw_tx
@@ -277,7 +265,7 @@ class WalletTest(BitcoinTestFramework):
             if uTx['txid'] == zero_value_txid:
                 found = True
                 assert_equal(uTx['amount'], Decimal('0'))
-        assert(found)
+        assert found
 
         # do some -walletbroadcast tests
         self.stop_nodes()
@@ -337,12 +325,38 @@ class WalletTest(BitcoinTestFramework):
         tx_obj = self.nodes[0].gettransaction(txid)
         assert_equal(tx_obj['amount'], Decimal('-0.0001'))
 
+        # General checks for errors from incorrect inputs
         # This will raise an exception because the amount type is wrong
         assert_raises_rpc_error(-3, "Invalid amount", self.nodes[0].sendtoaddress, self.nodes[2].getnewaddress(), "1f-4")
 
         # This will raise an exception since generate does not accept a string
         assert_raises_rpc_error(-1, "not an integer", self.nodes[0].generate, "2")
 
+        # This will raise an exception for the invalid private key format
+        assert_raises_rpc_error(-5, "Invalid private key encoding", self.nodes[0].importprivkey, "invalid")
+
+        # This will raise an exception for importing an address with the PS2H flag
+        temp_address = self.nodes[1].getnewaddress()
+        assert_raises_rpc_error(-5, "Cannot use the p2sh flag with an address - use a script instead", self.nodes[0].importaddress, temp_address, "label", False, True)
+
+        # This will raise an exception for attempting to dump the private key of an address you do not own
+        assert_raises_rpc_error(-3, "Address does not refer to a key", self.nodes[0].dumpprivkey, temp_address)
+
+        # This will raise an exception for attempting to get the private key of an invalid Bitcoin address
+        assert_raises_rpc_error(-5, "Invalid Bitcoin address", self.nodes[0].dumpprivkey, "invalid")
+
+        # This will raise an exception for attempting to set a label for an invalid Bitcoin address
+        assert_raises_rpc_error(-5, "Invalid Bitcoin address", self.nodes[0].setlabel, "invalid address", "label")
+
+        # This will raise an exception for importing an invalid address
+        assert_raises_rpc_error(-5, "Invalid Bitcoin address or script", self.nodes[0].importaddress, "invalid")
+
+        # This will raise an exception for attempting to import a pubkey that isn't in hex
+        assert_raises_rpc_error(-5, "Pubkey must be a hex string", self.nodes[0].importpubkey, "not hex")
+
+        # This will raise an exception for importing an invalid pubkey
+        assert_raises_rpc_error(-5, "Pubkey is not a valid public key", self.nodes[0].importpubkey, "5361746f736869204e616b616d6f746f")
+
         # Import address and private key to check correct behavior of spendable unspents
         # 1. Send some coins to generate new UTXO
         address_to_import = self.nodes[2].getnewaddress()
@@ -354,7 +368,7 @@ class WalletTest(BitcoinTestFramework):
         self.nodes[1].importaddress(address_to_import)
 
         # 3. Validate that the imported address is watch-only on node1
-        assert(self.nodes[1].getaddressinfo(address_to_import)["iswatchonly"])
+        assert self.nodes[1].getaddressinfo(address_to_import)["iswatchonly"]
 
         # 4. Check that the unspents after import are not spendable
         assert_array_result(self.nodes[1].listunspent(),
@@ -396,7 +410,7 @@ class WalletTest(BitcoinTestFramework):
                 addr = self.nodes[0].getnewaddress()
                 self.nodes[0].setlabel(addr, label)
                 assert_equal(self.nodes[0].getaddressinfo(addr)['label'], label)
-                assert(label in self.nodes[0].listlabels())
+                assert label in self.nodes[0].listlabels()
         self.nodes[0].rpc.ensure_ascii = True  # restore to default
 
         # maintenance tests
@@ -455,8 +469,8 @@ class WalletTest(BitcoinTestFramework):
         # Without walletrejectlongchains, we will still generate a txid
         # The tx will be stored in the wallet but not accepted to the mempool
         extra_txid = self.nodes[0].sendtoaddress(sending_addr, Decimal('0.0001'))
-        assert(extra_txid not in self.nodes[0].getrawmempool())
-        assert(extra_txid in [tx["txid"] for tx in self.nodes[0].listtransactions()])
+        assert extra_txid not in self.nodes[0].getrawmempool()
+        assert extra_txid in [tx["txid"] for tx in self.nodes[0].listtransactions()]
         self.nodes[0].abandontransaction(extra_txid)
         total_txs = len(self.nodes[0].listtransactions("*", 99999))
 
@@ -493,7 +507,7 @@ class WalletTest(BitcoinTestFramework):
         self.nodes[0].generate(1)
         destination = self.nodes[1].getnewaddress()
         txid = self.nodes[0].sendtoaddress(destination, 0.123)
-        tx = self.nodes[0].decoderawtransaction(self.nodes[0].getrawtransaction(txid))
+        tx = self.nodes[0].decoderawtransaction(self.nodes[0].gettransaction(txid)['hex'])
         output_addresses = [vout['scriptPubKey']['addresses'][0] for vout in tx["vout"]]
         assert len(output_addresses) > 1
         for address in output_addresses:
@@ -504,5 +518,6 @@ class WalletTest(BitcoinTestFramework):
         self.nodes[0].setlabel(change, 'foobar')
         assert_equal(self.nodes[0].getaddressinfo(change)['ischange'], False)
 
+
 if __name__ == '__main__':
     WalletTest().main()

test/bitcoin_functional/functional/wallet_coinbase_category.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+# Copyright (c) 2014-2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test coinbase transactions return the correct categories.
+
+Tests listtransactions, listsinceblock, and gettransaction.
+"""
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    assert_array_result
+)
+
+class CoinbaseCategoryTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 1
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_wallet()
+
+    def assert_category(self, category, address, txid, skip):
+        assert_array_result(self.nodes[0].listtransactions(skip=skip),
+                            {"address": address},
+                            {"category": category})
+        assert_array_result(self.nodes[0].listsinceblock()["transactions"],
+                            {"address": address},
+                            {"category": category})
+        assert_array_result(self.nodes[0].gettransaction(txid)["details"],
+                            {"address": address},
+                            {"category": category})
+
+    def run_test(self):
+        # Generate one block to an address
+        address = self.nodes[0].getnewaddress()
+        self.nodes[0].generatetoaddress(1, address)
+        hash = self.nodes[0].getbestblockhash()
+        txid = self.nodes[0].getblock(hash)["tx"][0]
+
+        # Coinbase transaction is immature after 1 confirmation
+        self.assert_category("immature", address, txid, 0)
+
+        # Mine another 99 blocks on top
+        self.nodes[0].generate(99)
+        # Coinbase transaction is still immature after 100 confirmations
+        self.assert_category("immature", address, txid, 99)
+
+        # Mine one more block
+        self.nodes[0].generate(1)
+        # Coinbase transaction is now matured, so category is "generate"
+        self.assert_category("generate", address, txid, 100)
+
+        # Orphan block that paid to address
+        self.nodes[0].invalidateblock(hash)
+        # Coinbase transaction is now orphaned
+        self.assert_category("orphan", address, txid, 100)
+
+if __name__ == '__main__':
+    CoinbaseCategoryTest().main()

test/bitcoin_functional/functional/wallet_create_tx.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    assert_equal,
+)
+
+
+class CreateTxWalletTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = False
+        self.num_nodes = 1
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_wallet()
+
+    def run_test(self):
+        self.log.info('Check that we have some (old) blocks and that anti-fee-sniping is disabled')
+        assert_equal(self.nodes[0].getblockchaininfo()['blocks'], 200)
+        txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1)
+        tx = self.nodes[0].decoderawtransaction(self.nodes[0].gettransaction(txid)['hex'])
+        assert_equal(tx['locktime'], 0)
+
+        self.log.info('Check that anti-fee-sniping is enabled when we mine a recent block')
+        self.nodes[0].generate(1)
+        txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1)
+        tx = self.nodes[0].decoderawtransaction(self.nodes[0].gettransaction(txid)['hex'])
+        assert 0 < tx['locktime'] <= 201
+
+
+if __name__ == '__main__':
+    CreateTxWalletTest().main()

test/bitcoin_functional/functional/wallet_createwallet.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test createwallet arguments.
+"""
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    assert_equal,
+    assert_raises_rpc_error,
+)
+
+class CreateWalletTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = False
+        self.num_nodes = 1
+        self.supports_cli = True
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_wallet()
+
+    def run_test(self):
+        node = self.nodes[0]
+        node.generate(1) # Leave IBD for sethdseed
+
+        self.nodes[0].createwallet(wallet_name='w0')
+        w0 = node.get_wallet_rpc('w0')
+        address1 = w0.getnewaddress()
+
+        self.log.info("Test disableprivatekeys creation.")
+        self.nodes[0].createwallet(wallet_name='w1', disable_private_keys=True)
+        w1 = node.get_wallet_rpc('w1')
+        assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w1.getnewaddress)
+        assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w1.getrawchangeaddress)
+        w1.importpubkey(w0.getaddressinfo(address1)['pubkey'])
+
+        self.log.info('Test that private keys cannot be imported')
+        addr = w0.getnewaddress('', 'legacy')
+        privkey = w0.dumpprivkey(addr)
+        assert_raises_rpc_error(-4, 'Cannot import private keys to a wallet with private keys disabled', w1.importprivkey, privkey)
+        result = w1.importmulti([{'scriptPubKey': {'address': addr}, 'timestamp': 'now', 'keys': [privkey]}])
+        assert(not result[0]['success'])
+        assert('warning' not in result[0])
+        assert_equal(result[0]['error']['code'], -4)
+        assert_equal(result[0]['error']['message'], 'Cannot import private keys to a wallet with private keys disabled')
+
+        self.log.info("Test blank creation with private keys disabled.")
+        self.nodes[0].createwallet(wallet_name='w2', disable_private_keys=True, blank=True)
+        w2 = node.get_wallet_rpc('w2')
+        assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w2.getnewaddress)
+        assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w2.getrawchangeaddress)
+        w2.importpubkey(w0.getaddressinfo(address1)['pubkey'])
+
+        self.log.info("Test blank creation with private keys enabled.")
+        self.nodes[0].createwallet(wallet_name='w3', disable_private_keys=False, blank=True)
+        w3 = node.get_wallet_rpc('w3')
+        assert_equal(w3.getwalletinfo()['keypoolsize'], 0)
+        assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getnewaddress)
+        assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getrawchangeaddress)
+        # Import private key
+        w3.importprivkey(w0.dumpprivkey(address1))
+        # Imported private keys are currently ignored by the keypool
+        assert_equal(w3.getwalletinfo()['keypoolsize'], 0)
+        assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getnewaddress)
+        # Set the seed
+        w3.sethdseed()
+        assert_equal(w3.getwalletinfo()['keypoolsize'], 1)
+        w3.getnewaddress()
+        w3.getrawchangeaddress()
+
+        self.log.info("Test blank creation with privkeys enabled and then encryption")
+        self.nodes[0].createwallet(wallet_name='w4', disable_private_keys=False, blank=True)
+        w4 = node.get_wallet_rpc('w4')
+        assert_equal(w4.getwalletinfo()['keypoolsize'], 0)
+        assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getnewaddress)
+        assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getrawchangeaddress)
+        # Encrypt the wallet. Nothing should change about the keypool
+        w4.encryptwallet('pass')
+        assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getnewaddress)
+        assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getrawchangeaddress)
+        # Now set a seed and it should work. Wallet should also be encrypted
+        w4.walletpassphrase('pass', 2)
+        w4.sethdseed()
+        w4.getnewaddress()
+        w4.getrawchangeaddress()
+
+        self.log.info("Test blank creation with privkeys disabled and then encryption")
+        self.nodes[0].createwallet(wallet_name='w5', disable_private_keys=True, blank=True)
+        w5 = node.get_wallet_rpc('w5')
+        assert_equal(w5.getwalletinfo()['keypoolsize'], 0)
+        assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w5.getnewaddress)
+        assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w5.getrawchangeaddress)
+        # Encrypt the wallet
+        w5.encryptwallet('pass')
+        assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w5.getnewaddress)
+        assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w5.getrawchangeaddress)
+
+if __name__ == '__main__':
+    CreateWalletTest().main()

test/bitcoin_functional/functional/wallet_disableprivatekeys.py

@@ -1,35 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2018 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-"""Test disable-privatekeys mode.
-"""
-
-from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import (
-    assert_raises_rpc_error,
-)
-
-
-class DisablePrivateKeysTest(BitcoinTestFramework):
-    def set_test_params(self):
-        self.setup_clean_chain = False
-        self.num_nodes = 1
-        self.supports_cli = True
-
-    def skip_test_if_missing_module(self):
-        self.skip_if_no_wallet()
-
-    def run_test(self):
-        node = self.nodes[0]
-        self.log.info("Test disableprivatekeys creation.")
-        self.nodes[0].createwallet('w1', True)
-        self.nodes[0].createwallet('w2')
-        w1 = node.get_wallet_rpc('w1')
-        w2 = node.get_wallet_rpc('w2')
-        assert_raises_rpc_error(-4,"Error: Private keys are disabled for this wallet", w1.getnewaddress)
-        assert_raises_rpc_error(-4,"Error: Private keys are disabled for this wallet", w1.getrawchangeaddress)
-        w1.importpubkey(w2.getaddressinfo(w2.getnewaddress())['pubkey'])
-
-if __name__ == '__main__':
-    DisablePrivateKeysTest().main()

test/bitcoin_functional/functional/wallet_dump.py

@@ -94,7 +94,7 @@ class WalletDumpTest(BitcoinTestFramework):
     def skip_test_if_missing_module(self):
         self.skip_if_no_wallet()
 
-    def setup_network(self, split=False):
+    def setup_network(self):
         self.add_nodes(self.num_nodes, extra_args=self.extra_args)
         self.start_nodes()
 

test/bitcoin_functional/functional/wallet_encryption.py

@@ -31,12 +31,18 @@ class WalletEncryptionTest(BitcoinTestFramework):
         privkey = self.nodes[0].dumpprivkey(address)
         assert_equal(privkey[:1], "c")
         assert_equal(len(privkey), 52)
+        assert_raises_rpc_error(-15, "Error: running with an unencrypted wallet, but walletpassphrase was called", self.nodes[0].walletpassphrase, 'ff', 1)
+        assert_raises_rpc_error(-15, "Error: running with an unencrypted wallet, but walletpassphrasechange was called.", self.nodes[0].walletpassphrasechange, 'ff', 'ff')
 
         # Encrypt the wallet
+        assert_raises_rpc_error(-8, "passphrase can not be empty", self.nodes[0].encryptwallet, '')
         self.nodes[0].encryptwallet(passphrase)
 
         # Test that the wallet is encrypted
         assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address)
+        assert_raises_rpc_error(-15, "Error: running with an encrypted wallet, but encryptwallet was called.", self.nodes[0].encryptwallet, 'ff')
+        assert_raises_rpc_error(-8, "passphrase can not be empty", self.nodes[0].walletpassphrase, '', 1)
+        assert_raises_rpc_error(-8, "passphrase can not be empty", self.nodes[0].walletpassphrasechange, '', 'ff')
 
         # Check that walletpassphrase works
         self.nodes[0].walletpassphrase(passphrase, 2)

test/bitcoin_functional/functional/wallet_groups.py

@@ -21,7 +21,7 @@ class WalletGroupTest(BitcoinTestFramework):
         self.setup_clean_chain = True
         self.num_nodes = 3
         self.extra_args = [[], [], ['-avoidpartialspends']]
-        self.rpc_timewait = 120
+        self.rpc_timeout = 120
 
     def skip_test_if_missing_module(self):
         self.skip_if_no_wallet()

test/bitcoin_functional/functional/wallet_hd.py

@@ -27,7 +27,6 @@ class WalletHDTest(BitcoinTestFramework):
     def run_test(self):
         # Make sure we use hd, keep masterkeyid
         masterkeyid = self.nodes[1].getwalletinfo()['hdseedid']
-        assert_equal(masterkeyid, self.nodes[1].getwalletinfo()['hdmasterkeyid'])
         assert_equal(len(masterkeyid), 40)
 
         # create an internal key
@@ -53,7 +52,6 @@ class WalletHDTest(BitcoinTestFramework):
             hd_info = self.nodes[1].getaddressinfo(hd_add)
             assert_equal(hd_info["hdkeypath"], "m/0'/0'/"+str(i)+"'")
             assert_equal(hd_info["hdseedid"], masterkeyid)
-            assert_equal(hd_info["hdmasterkeyid"], masterkeyid)
             self.nodes[0].sendtoaddress(hd_add, 1)
             self.nodes[0].generate(1)
         self.nodes[0].sendtoaddress(non_hd_add, 1)
@@ -83,7 +81,6 @@ class WalletHDTest(BitcoinTestFramework):
             hd_info_2 = self.nodes[1].getaddressinfo(hd_add_2)
             assert_equal(hd_info_2["hdkeypath"], "m/0'/0'/"+str(i)+"'")
             assert_equal(hd_info_2["hdseedid"], masterkeyid)
-            assert_equal(hd_info_2["hdmasterkeyid"], masterkeyid)
         assert_equal(hd_add, hd_add_2)
         connect_nodes_bi(self.nodes, 0, 1)
         self.sync_all()

test/bitcoin_functional/functional/wallet_import_rescan.py

@@ -46,11 +46,11 @@ class Variant(collections.namedtuple("Variant", "call data rescan prune")):
 
         if self.call == Call.single:
             if self.data == Data.address:
-                response = self.try_rpc(self.node.importaddress, address=self.address["address"], rescan=rescan)
+                response = self.try_rpc(self.node.importaddress, address=self.address["address"], label=self.label, rescan=rescan)
             elif self.data == Data.pub:
-                response = self.try_rpc(self.node.importpubkey, pubkey=self.address["pubkey"], rescan=rescan)
+                response = self.try_rpc(self.node.importpubkey, pubkey=self.address["pubkey"], label=self.label, rescan=rescan)
             elif self.data == Data.priv:
-                response = self.try_rpc(self.node.importprivkey, privkey=self.key, rescan=rescan)
+                response = self.try_rpc(self.node.importprivkey, privkey=self.key, label=self.label, rescan=rescan)
             assert_equal(response, None)
 
         elif self.call in (Call.multiaddress, Call.multiscript):
@@ -61,18 +61,32 @@ class Variant(collections.namedtuple("Variant", "call data rescan prune")):
                 "timestamp": timestamp + TIMESTAMP_WINDOW + (1 if self.rescan == Rescan.late_timestamp else 0),
                 "pubkeys": [self.address["pubkey"]] if self.data == Data.pub else [],
                 "keys": [self.key] if self.data == Data.priv else [],
+                "label": self.label,
                 "watchonly": self.data != Data.priv
             }], {"rescan": self.rescan in (Rescan.yes, Rescan.late_timestamp)})
             assert_equal(response, [{"success": True}])
 
     def check(self, txid=None, amount=None, confirmations=None):
-        """Verify that listreceivedbyaddress returns expected values."""
+        """Verify that listtransactions/listreceivedbyaddress return expected values."""
+
+        txs = self.node.listtransactions(label=self.label, count=10000, include_watchonly=True)
+        assert_equal(len(txs), self.expected_txs)
 
         addresses = self.node.listreceivedbyaddress(minconf=0, include_watchonly=True, address_filter=self.address['address'])
         if self.expected_txs:
             assert_equal(len(addresses[0]["txids"]), self.expected_txs)
 
         if txid is not None:
+            tx, = [tx for tx in txs if tx["txid"] == txid]
+            assert_equal(tx["label"], self.label)
+            assert_equal(tx["address"], self.address["address"])
+            assert_equal(tx["amount"], amount)
+            assert_equal(tx["category"], "receive")
+            assert_equal(tx["label"], self.label)
+            assert_equal(tx["txid"], txid)
+            assert_equal(tx["confirmations"], confirmations)
+            assert_equal("trusted" not in tx, True)
+
             address, = [ad for ad in addresses if txid in ad["txids"]]
             assert_equal(address["address"], self.address["address"])
             assert_equal(address["amount"], self.expected_balance)
@@ -134,7 +148,8 @@ class ImportRescanTest(BitcoinTestFramework):
         # Create one transaction on node 0 with a unique amount for
         # each possible type of wallet import RPC.
         for i, variant in enumerate(IMPORT_VARIANTS):
-            variant.address = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress())
+            variant.label = "label {} {}".format(i, variant)
+            variant.address = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress(variant.label))
             variant.key = self.nodes[1].dumpprivkey(variant.address["address"])
             variant.initial_amount = 1 - (i + 1) / 64
             variant.initial_txid = self.nodes[0].sendtoaddress(variant.address["address"], variant.initial_amount)

test/bitcoin_functional/functional/wallet_import_with_label.py

@@ -11,7 +11,7 @@ with and without a label.
 """
 
 from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import assert_equal
+from test_framework.wallet_util import test_address
 
 
 class ImportWithLabel(BitcoinTestFramework):
@@ -32,11 +32,11 @@ class ImportWithLabel(BitcoinTestFramework):
         address = self.nodes[0].getnewaddress()
         label = "Test Label"
         self.nodes[1].importaddress(address, label)
-        address_assert = self.nodes[1].getaddressinfo(address)
-
-        assert_equal(address_assert["iswatchonly"], True)
-        assert_equal(address_assert["ismine"], False)
-        assert_equal(address_assert["label"], label)
+        test_address(self.nodes[1],
+                     address,
+                     iswatchonly=True,
+                     ismine=False,
+                     label=label)
 
         self.log.info(
             "Import the watch-only address's private key without a "
@@ -45,7 +45,9 @@ class ImportWithLabel(BitcoinTestFramework):
         priv_key = self.nodes[0].dumpprivkey(address)
         self.nodes[1].importprivkey(priv_key)
 
-        assert_equal(label, self.nodes[1].getaddressinfo(address)["label"])
+        test_address(self.nodes[1],
+                     address,
+                     label=label)
 
         self.log.info(
             "Test importaddress without label and importprivkey with label."
@@ -53,11 +55,11 @@ class ImportWithLabel(BitcoinTestFramework):
         self.log.info("Import a watch-only address without a label.")
         address2 = self.nodes[0].getnewaddress()
         self.nodes[1].importaddress(address2)
-        address_assert2 = self.nodes[1].getaddressinfo(address2)
-
-        assert_equal(address_assert2["iswatchonly"], True)
-        assert_equal(address_assert2["ismine"], False)
-        assert_equal(address_assert2["label"], "")
+        test_address(self.nodes[1],
+                     address2,
+                     iswatchonly=True,
+                     ismine=False,
+                     label="")
 
         self.log.info(
             "Import the watch-only address's private key with a "
@@ -67,18 +69,20 @@ class ImportWithLabel(BitcoinTestFramework):
         label2 = "Test Label 2"
         self.nodes[1].importprivkey(priv_key2, label2)
 
-        assert_equal(label2, self.nodes[1].getaddressinfo(address2)["label"])
+        test_address(self.nodes[1],
+                     address2,
+                     label=label2)
 
         self.log.info("Test importaddress with label and importprivkey with label.")
         self.log.info("Import a watch-only address with a label.")
         address3 = self.nodes[0].getnewaddress()
         label3_addr = "Test Label 3 for importaddress"
         self.nodes[1].importaddress(address3, label3_addr)
-        address_assert3 = self.nodes[1].getaddressinfo(address3)
-
-        assert_equal(address_assert3["iswatchonly"], True)
-        assert_equal(address_assert3["ismine"], False)
-        assert_equal(address_assert3["label"], label3_addr)
+        test_address(self.nodes[1],
+                     address3,
+                     iswatchonly=True,
+                     ismine=False,
+                     label=label3_addr)
 
         self.log.info(
             "Import the watch-only address's private key with a "
@@ -88,7 +92,9 @@ class ImportWithLabel(BitcoinTestFramework):
         label3_priv = "Test Label 3 for importprivkey"
         self.nodes[1].importprivkey(priv_key3, label3_priv)
 
-        assert_equal(label3_priv, self.nodes[1].getaddressinfo(address3)["label"])
+        test_address(self.nodes[1],
+                     address3,
+                     label=label3_priv)
 
         self.log.info(
             "Test importprivkey won't label new dests with the same "
@@ -98,15 +104,12 @@ class ImportWithLabel(BitcoinTestFramework):
         address4 = self.nodes[0].getnewaddress()
         label4_addr = "Test Label 4 for importaddress"
         self.nodes[1].importaddress(address4, label4_addr)
-        address_assert4 = self.nodes[1].getaddressinfo(address4)
-
-        assert_equal(address_assert4["iswatchonly"], True)
-        assert_equal(address_assert4["ismine"], False)
-        assert_equal(address_assert4["label"], label4_addr)
-
-        self.log.info("Asserts address has no embedded field with dests.")
-
-        assert_equal(address_assert4.get("embedded"), None)
+        test_address(self.nodes[1],
+                     address4,
+                     iswatchonly=True,
+                     ismine=False,
+                     label=label4_addr,
+                     embedded=None)
 
         self.log.info(
             "Import the watch-only address's private key without a "
@@ -116,16 +119,14 @@ class ImportWithLabel(BitcoinTestFramework):
         )
         priv_key4 = self.nodes[0].dumpprivkey(address4)
         self.nodes[1].importprivkey(priv_key4)
-        address_assert4 = self.nodes[1].getaddressinfo(address4)
+        embedded_addr = self.nodes[1].getaddressinfo(address4)['embedded']['address']
 
-        assert address_assert4.get("embedded")
-
-        bcaddress_assert = self.nodes[1].getaddressinfo(
-            address_assert4["embedded"]["address"]
-        )
-
-        assert_equal(address_assert4["label"], label4_addr)
-        assert_equal(bcaddress_assert["label"], "")
+        test_address(self.nodes[1],
+                     embedded_addr,
+                     label="")
+        test_address(self.nodes[1],
+                     address4,
+                     label=label4_addr)
 
         self.stop_nodes()
 

test/bitcoin_functional/functional/wallet_importprunedfunds.py

@@ -81,7 +81,7 @@ class ImportPrunedFundsTest(BitcoinTestFramework):
 
         # Import with affiliated address with no rescan
         self.nodes[1].importaddress(address=address2, rescan=False)
-        self.nodes[1].importprunedfunds(rawtxn2, proof2)
+        self.nodes[1].importprunedfunds(rawtransaction=rawtxn2, txoutproof=proof2)
         assert [tx for tx in self.nodes[1].listtransactions(include_watchonly=True) if tx['txid'] == txnid2]
 
         # Import with private key with no rescan

test/bitcoin_functional/functional/wallet_keypool.py

@@ -21,7 +21,6 @@ class KeyPoolTest(BitcoinTestFramework):
         addr_before_encrypting = nodes[0].getnewaddress()
         addr_before_encrypting_data = nodes[0].getaddressinfo(addr_before_encrypting)
         wallet_info_old = nodes[0].getwalletinfo()
-        assert_equal(wallet_info_old['hdseedid'], wallet_info_old['hdmasterkeyid'])
         assert(addr_before_encrypting_data['hdseedid'] == wallet_info_old['hdseedid'])
 
         # Encrypt wallet and wait to terminate
@@ -30,7 +29,6 @@ class KeyPoolTest(BitcoinTestFramework):
         addr = nodes[0].getnewaddress()
         addr_data = nodes[0].getaddressinfo(addr)
         wallet_info = nodes[0].getwalletinfo()
-        assert_equal(wallet_info['hdseedid'], wallet_info['hdmasterkeyid'])
         assert(addr_before_encrypting_data['hdseedid'] != wallet_info['hdseedid'])
         assert(addr_data['hdseedid'] == wallet_info['hdseedid'])
         assert_raises_rpc_error(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress)

test/bitcoin_functional/functional/wallet_keypool_topup.py

@@ -24,8 +24,8 @@ from test_framework.util import (
 class KeypoolRestoreTest(BitcoinTestFramework):
     def set_test_params(self):
         self.setup_clean_chain = True
-        self.num_nodes = 2
-        self.extra_args = [[], ['-keypool=100']]
+        self.num_nodes = 4
+        self.extra_args = [[], ['-keypool=100'], ['-keypool=100'], ['-keypool=100']]
 
     def skip_test_if_missing_module(self):
         self.skip_if_no_wallet()
@@ -40,32 +40,47 @@ class KeypoolRestoreTest(BitcoinTestFramework):
         shutil.copyfile(wallet_path, wallet_backup_path)
         self.start_node(1, self.extra_args[1])
         connect_nodes_bi(self.nodes, 0, 1)
+        connect_nodes_bi(self.nodes, 0, 2)
+        connect_nodes_bi(self.nodes, 0, 3)
 
-        self.log.info("Generate keys for wallet")
-        for _ in range(90):
-            addr_oldpool = self.nodes[1].getnewaddress()
-        for _ in range(20):
-            addr_extpool = self.nodes[1].getnewaddress()
+        for i, output_type in enumerate(["legacy", "p2sh-segwit", "bech32"]):
 
-        self.log.info("Send funds to wallet")
-        self.nodes[0].sendtoaddress(addr_oldpool, 10)
-        self.nodes[0].generate(1)
-        self.nodes[0].sendtoaddress(addr_extpool, 5)
-        self.nodes[0].generate(1)
-        sync_blocks(self.nodes)
+            self.log.info("Generate keys for wallet with address type: {}".format(output_type))
+            idx = i+1
+            for _ in range(90):
+                addr_oldpool = self.nodes[idx].getnewaddress(address_type=output_type)
+            for _ in range(20):
+                addr_extpool = self.nodes[idx].getnewaddress(address_type=output_type)
 
-        self.log.info("Restart node with wallet backup")
-        self.stop_node(1)
-        shutil.copyfile(wallet_backup_path, wallet_path)
-        self.start_node(1, self.extra_args[1])
-        connect_nodes_bi(self.nodes, 0, 1)
-        self.sync_all()
+            # Make sure we're creating the outputs we expect
+            address_details = self.nodes[idx].validateaddress(addr_extpool)
+            if i == 0:
+                assert(not address_details["isscript"] and not address_details["iswitness"])
+            elif i == 1:
+                assert(address_details["isscript"] and not address_details["iswitness"])
+            else:
+                assert(not address_details["isscript"] and address_details["iswitness"])
 
-        self.log.info("Verify keypool is restored and balance is correct")
-        assert_equal(self.nodes[1].getbalance(), 15)
-        assert_equal(self.nodes[1].listtransactions()[0]['category'], "receive")
-        # Check that we have marked all keys up to the used keypool key as used
-        assert_equal(self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress())['hdkeypath'], "m/0'/0'/110'")
+
+            self.log.info("Send funds to wallet")
+            self.nodes[0].sendtoaddress(addr_oldpool, 10)
+            self.nodes[0].generate(1)
+            self.nodes[0].sendtoaddress(addr_extpool, 5)
+            self.nodes[0].generate(1)
+            sync_blocks(self.nodes)
+
+            self.log.info("Restart node with wallet backup")
+            self.stop_node(idx)
+            shutil.copyfile(wallet_backup_path, wallet_path)
+            self.start_node(idx, self.extra_args[idx])
+            connect_nodes_bi(self.nodes, 0, idx)
+            self.sync_all()
+
+            self.log.info("Verify keypool is restored and balance is correct")
+            assert_equal(self.nodes[idx].getbalance(), 15)
+            assert_equal(self.nodes[idx].listtransactions()[0]['category'], "receive")
+            # Check that we have marked all keys up to the used keypool key as used
+            assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress())['hdkeypath'], "m/0'/0'/110'")
 
 
 if __name__ == '__main__':

test/bitcoin_functional/functional/wallet_listtransactions.py

@@ -25,12 +25,13 @@ def tx_from_hex(hexstring):
 class ListTransactionsTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 2
-        self.enable_mocktime()
 
     def skip_test_if_missing_module(self):
         self.skip_if_no_wallet()
 
     def run_test(self):
+        self.nodes[0].generate(1)  # Get out of IBD
+        self.sync_all()
         # Simple send, 0 to 1:
         txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1)
         self.sync_all()
@@ -97,9 +98,10 @@ class ListTransactionsTest(BitcoinTestFramework):
         txid = self.nodes[1].sendtoaddress(multisig["address"], 0.1)
         self.nodes[1].generate(1)
         self.sync_all()
-        assert not [tx for tx in self.nodes[0].listtransactions(dummy="*", count=100, skip=0, include_watchonly=False) if "label" in tx and tx["label"] == "watchonly"]
-        txs = [tx for tx in self.nodes[0].listtransactions(dummy="*", count=100, skip=0, include_watchonly=True) if "label" in tx and tx['label'] == 'watchonly']
-        assert_array_result(txs, {"category": "receive", "amount": Decimal("0.1")}, {"txid": txid})
+        assert len(self.nodes[0].listtransactions(label="watchonly", count=100, include_watchonly=False)) == 0
+        assert_array_result(self.nodes[0].listtransactions(label="watchonly", count=100, include_watchonly=True),
+                            {"category": "receive", "amount": Decimal("0.1")},
+                            {"txid": txid, "label": "watchonly"})
 
         self.run_rbf_opt_in_test()
 

test/bitcoin_functional/functional/wallet_multiwallet.py

@@ -223,6 +223,9 @@ class MultiWalletTest(BitcoinTestFramework):
         # Fail to load duplicate wallets
         assert_raises_rpc_error(-4, 'Wallet file verification failed: Error loading wallet w1. Duplicate -wallet filename specified.', self.nodes[0].loadwallet, wallet_names[0])
 
+        # Fail to load duplicate wallets by different ways (directory and filepath)
+        assert_raises_rpc_error(-4, "Wallet file verification failed: Error loading wallet wallet.dat. Duplicate -wallet filename specified.", self.nodes[0].loadwallet, 'wallet.dat')
+
         # Fail to load if one wallet is a copy of another
         assert_raises_rpc_error(-1, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy')
 
@@ -312,6 +315,14 @@ class MultiWalletTest(BitcoinTestFramework):
             self.nodes[0].loadwallet(wallet_name)
             assert_equal(rpc.getaddressinfo(addr)['ismine'], True)
 
+        # Test .walletlock file is closed
+        self.start_node(1)
+        wallet = os.path.join(self.options.tmpdir, 'my_wallet')
+        self.nodes[0].createwallet(wallet)
+        assert_raises_rpc_error(-4, "Error initializing wallet database environment", self.nodes[1].loadwallet, wallet)
+        self.nodes[0].unloadwallet(wallet)
+        self.nodes[1].loadwallet(wallet)
+
 
 if __name__ == '__main__':
     MultiWalletTest().main()

test/bitcoin_functional/functional/wallet_txn_clone.py

@@ -4,13 +4,16 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test the wallet accounts properly when there are cloned transactions with malleated scriptsigs."""
 
+import io
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_equal,
+    bytes_to_hex_str as b2x,
     connect_nodes,
     disconnect_nodes,
     sync_blocks,
 )
+from test_framework.messages import CTransaction, COIN
 
 class TxnMallTest(BitcoinTestFramework):
     def set_test_params(self):
@@ -65,26 +68,21 @@ class TxnMallTest(BitcoinTestFramework):
 
         # Construct a clone of tx1, to be malleated
         rawtx1 = self.nodes[0].getrawtransaction(txid1, 1)
-        clone_inputs = [{"txid": rawtx1["vin"][0]["txid"], "vout": rawtx1["vin"][0]["vout"]}]
+        clone_inputs = [{"txid": rawtx1["vin"][0]["txid"], "vout": rawtx1["vin"][0]["vout"], "sequence": rawtx1["vin"][0]["sequence"]}]
         clone_outputs = {rawtx1["vout"][0]["scriptPubKey"]["addresses"][0]: rawtx1["vout"][0]["value"],
                          rawtx1["vout"][1]["scriptPubKey"]["addresses"][0]: rawtx1["vout"][1]["value"]}
         clone_locktime = rawtx1["locktime"]
         clone_raw = self.nodes[0].createrawtransaction(clone_inputs, clone_outputs, clone_locktime)
 
         # createrawtransaction randomizes the order of its outputs, so swap them if necessary.
-        # output 0 is at version+#inputs+input+sigstub+sequence+#outputs
-        # 40 BTC serialized is 00286bee00000000
-        pos0 = 2 * (4 + 1 + 36 + 1 + 4 + 1)
-        hex40 = "00286bee00000000"
-        output_len = 16 + 2 + 2 * int("0x" + clone_raw[pos0 + 16:pos0 + 16 + 2], 0)
-        if (rawtx1["vout"][0]["value"] == 40 and clone_raw[pos0:pos0 + 16] != hex40 or rawtx1["vout"][0]["value"] != 40 and clone_raw[pos0:pos0 + 16] == hex40):
-            output0 = clone_raw[pos0:pos0 + output_len]
-            output1 = clone_raw[pos0 + output_len:pos0 + 2 * output_len]
-            clone_raw = clone_raw[:pos0] + output1 + output0 + clone_raw[pos0 + 2 * output_len:]
+        clone_tx = CTransaction()
+        clone_tx.deserialize(io.BytesIO(bytes.fromhex(clone_raw)))
+        if (rawtx1["vout"][0]["value"] == 40 and clone_tx.vout[0].nValue != 40*COIN or rawtx1["vout"][0]["value"] != 40 and clone_tx.vout[0].nValue == 40*COIN):
+            (clone_tx.vout[0], clone_tx.vout[1]) = (clone_tx.vout[1], clone_tx.vout[0])
 
         # Use a different signature hash type to sign.  This creates an equivalent but malleated clone.
         # Don't send the clone anywhere yet
-        tx1_clone = self.nodes[0].signrawtransactionwithwallet(clone_raw, None, "ALL|ANYONECANPAY")
+        tx1_clone = self.nodes[0].signrawtransactionwithwallet(b2x(clone_tx.serialize()), None, "ALL|ANYONECANPAY")
         assert_equal(tx1_clone["complete"], True)
 
         # Have node0 mine a block, if requested: