diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index a60c7811b..8cc3d43d0 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -53,26 +53,40 @@ cd specter-desktop python3 -m cryptoadvance.specter server --config DevelopmentConfig ``` -## Howto run the tests -Run the tests (still very limited): +## How to run the tests +_TODO: Need more thorough tests!_ +Set up the dependencies: ```sh pip3 install -r test_requirements.txt pip3 install -e . +``` -# needs a bitcoind on your path +If you have a local bitcoind already installed: +``` +# Run all the tests pytest +``` -# needs a working docker-setup (but not bitcoind) -# prerequsisite: -# docker pull registry.gitlab.com/cryptoadvance/specter-desktop/python-bitcoind:v0.20.1 +OR run against bitcoind in Docker: +``` +# Pull the bitcoind image if you haven't already: +docker pull registry.gitlab.com/cryptoadvance/specter-desktop/python-bitcoind:v0.20.1 + +# Run all the tests against the docker bitcoind image pytest --docker +``` -# Run all the tests in a specific test-file -pytest tests/test_specter +Running specific test subsets: +``` +# Run all the tests in a specific test file +pytest tests/test_specter.py # Run all tests in a specific file matching "Manager" -pytest tests/test_specter -k Manager +pytest tests/test_specter.py -k Manager + +# Run a specific test +pytest tests/test_specter.py::test_specter ``` Check the cypress-section on how to run cypress-frontend-tests. @@ -139,9 +153,10 @@ If Someone could figure out a better way to do that avoiding this strange this . Developing against a bitcoind-API makes most sense with the [Regtest Mode](https://bitcoin.org/en/developer-examples#regtest-mode). Depending on preferences and usecases, there are three major ways on how this dependency can be fullfilled: * Easiest way via Docker * The unittests on Travis-CI are using a script which is installing and compiling bitcoind -* bitcoind is manually started (out of scope for this document) +* Manually run local bitcoind in Regtest -In order to make the "docker-way" even easier, there is a python-script which detects a running-docker-bitcoind and/or is booting one up. Use it like this: +### Automatically mine and deposit test coins +In order to make the "docker-way" even easier, there is a python-script which detects a running-docker-bitcoind and/or is boots one up. Use it like this: ``` python3 -m cryptoadvance.specter bitcoind @@ -159,6 +174,42 @@ After that, you can configure the bitcoin-core-connection in specter-desktop lik * Host: localhost * Port: 18443 +### Manually mine and deposit test coins +If you're not using the integrated Docker method above, start your local bitcoind in regtest mode: +``` +bitcoind -regtest -fallbackfee=0.0001 +``` + +In another terminal initialize a default wallet to mine to: +``` +bitcoin-cli -regtest createwallet satoshiswallet +``` + +Get a new address to deposit newly mined coins: +``` +bitcoin-cli -regtest getnewaddress +``` + +Mine coins to the new address +``` +bitcoin-cli -regtest generatetoaddress 101
+``` + +Create a wallet in Specter and send test coins to a receive addr for the new wallet +``` +bitcoin-cli -regtest sendtoaddress
+``` + +Mine the next block when you want a pending tx to be confirmed +``` +bitcoin-cli -regtest generatetoaddress 1
+``` + +Cleanup: Stop your local regtest instance +``` +bitcoin-cli -regtest stop +``` + ## IDE-specific Configuration (might be outdated) ### Visual Studio Code diff --git a/src/cryptoadvance/specter/bitcoind.py b/src/cryptoadvance/specter/bitcoind.py index 344861745..add87c96b 100644 --- a/src/cryptoadvance/specter/bitcoind.py +++ b/src/cryptoadvance/specter/bitcoind.py @@ -83,7 +83,9 @@ class BitcoindController: def __init__(self, rpcport=18443): self.rpcconn = Btcd_conn(rpcport=rpcport) - def start_bitcoind(self, cleanup_at_exit=False, cleanup_hard=False, datadir=None): + def start_bitcoind( + self, cleanup_at_exit=False, cleanup_hard=False, datadir=None, extra_args=[] + ): """starts bitcoind with a specific rpcport=18543 by default. That's not the standard in order to make pytest running while developing locally against a different regtest-instance @@ -94,7 +96,10 @@ class BitcoindController: logger.debug("Starting bitcoind") self._start_bitcoind( - cleanup_at_exit, cleanup_hard=cleanup_hard, datadir=datadir + cleanup_at_exit, + cleanup_hard=cleanup_hard, + datadir=datadir, + extra_args=extra_args, ) self.wait_for_bitcoind(self.rpcconn) @@ -111,7 +116,9 @@ class BitcoindController: """ wrapper for convenience """ return self.rpcconn.get_rpc() - def _start_bitcoind(self, cleanup_at_exit, cleanup_hard=False): + def _start_bitcoind( + self, cleanup_at_exit, cleanup_hard=False, datadir=None, extra_args=[] + ): raise Exception("This should not be used in the baseclass!") def check_existing(self): @@ -185,7 +192,12 @@ class BitcoindController: @classmethod def construct_bitcoind_cmd( - cls, rpcconn, run_docker=True, datadir=None, bitcoind_path="bitcoind" + cls, + rpcconn, + run_docker=True, + datadir=None, + bitcoind_path="bitcoind", + extra_args=[], ): """ returns a bitcoind-command to run bitcoind """ btcd_cmd = "{} ".format(bitcoind_path) @@ -203,6 +215,8 @@ class BitcoindController: if datadir == None: datadir = tempfile.mkdtemp(prefix="bitcoind_datadir") btcd_cmd += " -datadir={} ".format(datadir) + if extra_args: + btcd_cmd += " {}".format(" ".join(extra_args)) logger.debug("constructed bitcoind-command: %s", btcd_cmd) return btcd_cmd @@ -215,14 +229,18 @@ class BitcoindPlainController(BitcoindController): self.bitcoind_path = bitcoind_path self.rpcconn.ipaddress = "localhost" - def _start_bitcoind(self, cleanup_at_exit=True, cleanup_hard=False, datadir=None): + def _start_bitcoind( + self, cleanup_at_exit=True, cleanup_hard=False, datadir=None, extra_args=[] + ): if datadir == None: datadir = tempfile.mkdtemp(prefix="specter_btc_regtest_plain_datadir_") + bitcoind_cmd = self.construct_bitcoind_cmd( self.rpcconn, run_docker=False, datadir=datadir, bitcoind_path=self.bitcoind_path, + extra_args=extra_args, ) logger.debug("About to execute: {}".format(bitcoind_cmd)) # exec will prevent creating a child-process and will make bitcoind_proc.terminate() work as expected @@ -231,40 +249,38 @@ class BitcoindPlainController(BitcoindController): "Running bitcoind-process with pid {}".format(self.bitcoind_proc.pid) ) - def cleanup_bitcoind(*args): - timeout = 50 # in secs - if cleanup_hard: - self.bitcoind_proc.kill() # might be usefull for e.g. testing. We can't wait for so long - logger.info( - f"Killed bitcoind with pid {self.bitcoind_proc.pid}, Removing {datadir}" - ) - shutil.rmtree(datadir, ignore_errors=True) - else: - self.bitcoind_proc.terminate() # might take a bit longer than kill but it'll preserve block-height - logger.info( - f"Terminated bitcoind with pid {self.bitcoind_proc.pid}, waiting for termination (timeout {timeout} secs)..." - ) - # self.bitcoind_proc.wait() # doesn't have a timeout - procs = psutil.Process().children() - for p in procs: - p.terminate() - _, alive = psutil.wait_procs(procs, timeout=timeout) - for p in alive: - logger.info("bitcoind did not terminated in time, killing!") - p.kill() - if cleanup_at_exit: logger.debug("Register function cleanup_bitcoind for SIGINT and SIGTERM") # atexit.register(cleanup_bitcoind) # This is for CTRL-C --> SIGINT - signal.signal(signal.SIGINT, cleanup_bitcoind) + signal.signal(signal.SIGINT, self.cleanup_bitcoind) # This is for kill $pid --> SIGTERM - signal.signal(signal.SIGTERM, cleanup_bitcoind) + signal.signal(signal.SIGTERM, self.cleanup_bitcoind) + + def cleanup_bitcoind(self, cleanup_hard=None, datadir=None): + timeout = 50 # in secs + if cleanup_hard: + self.bitcoind_proc.kill() # might be usefull for e.g. testing. We can't wait for so long + logger.info( + f"Killed bitcoind with pid {self.bitcoind_proc.pid}, Removing {datadir}" + ) + shutil.rmtree(datadir, ignore_errors=True) + else: + self.bitcoind_proc.terminate() # might take a bit longer than kill but it'll preserve block-height + logger.info( + f"Terminated bitcoind with pid {self.bitcoind_proc.pid}, waiting for termination (timeout {timeout} secs)..." + ) + # self.bitcoind_proc.wait() # doesn't have a timeout + procs = psutil.Process().children() + for p in procs: + p.terminate() + _, alive = psutil.wait_procs(procs, timeout=timeout) + for p in alive: + logger.info("bitcoind did not terminated in time, killing!") + p.kill() def stop_bitcoind(self): - # not necessary as the cleanup_bitcoind() will do it automatically! - # ToDo: Implement it nevertheless - pass + self.cleanup_bitcoind() def check_existing(self): """other then in docker, we won't check on the "instance-level". This will return true if if a @@ -290,11 +306,13 @@ class BitcoindDockerController(BitcoindController): btcd_container.stop() btcd_container.remove() - def _start_bitcoind(self, cleanup_at_exit, cleanup_hard=False, datadir=None): + def _start_bitcoind( + self, cleanup_at_exit, cleanup_hard=False, datadir=None, extra_args=[] + ): if datadir != None: # ignored pass - bitcoind_path = self.construct_bitcoind_cmd(self.rpcconn) + bitcoind_path = self.construct_bitcoind_cmd(self.rpcconn, extra_args=extra_args) dclient = docker.from_env() logger.debug("Running (in docker): {}".format(bitcoind_path)) ports = { diff --git a/src/cryptoadvance/specter/devices/bitcoin_core.py b/src/cryptoadvance/specter/devices/bitcoin_core.py index 9520c22fe..2a4341a03 100644 --- a/src/cryptoadvance/specter/devices/bitcoin_core.py +++ b/src/cryptoadvance/specter/devices/bitcoin_core.py @@ -144,7 +144,7 @@ class BitcoinCore(Device): def create_psbts(self, base64_psbt, wallet): return {"core": base64_psbt} - def sign_psbt(self, base64_psbt, wallet, file_password): + def sign_psbt(self, base64_psbt, wallet, file_password=None): # Load the wallet if not loaded self._load_wallet(wallet.manager) rpc = wallet.manager.rpc.wallet( @@ -161,6 +161,19 @@ class BitcoinCore(Device): rpc.walletlock() return signed_psbt + def sign_raw_tx(self, raw_tx, wallet, file_password=None): + # Load the wallet if not loaded + self._load_wallet(wallet.manager) + rpc = wallet.manager.rpc.wallet( + os.path.join(wallet.manager.rpc_path + "_hotstorage", self.alias) + ) + if file_password: + rpc.walletpassphrase(file_password, 60) + signed_tx = rpc.signrawtransactionwithwallet(raw_tx) + if file_password: + rpc.walletlock() + return signed_tx + def delete( self, wallet_manager, bitcoin_datadir=get_default_datadir(), chain="main" ): diff --git a/src/cryptoadvance/specter/server_endpoints/wallets.py b/src/cryptoadvance/specter/server_endpoints/wallets.py index 26a3f7681..a87189dcb 100644 --- a/src/cryptoadvance/specter/server_endpoints/wallets.py +++ b/src/cryptoadvance/specter/server_endpoints/wallets.py @@ -474,6 +474,12 @@ def history(wallet_alias): if action == "freezeutxo": wallet.toggle_freeze_utxo(request.form.getlist("selected_utxo")) tx_list_type = "utxo" + elif action == "abandon_tx": + txid = request.form["txid"] + try: + wallet.abandontransaction(txid) + except SpecterError as e: + flash(str(e), "error") # update balances in the wallet app.specter.check_blockheight() @@ -1046,6 +1052,10 @@ def decoderawtx(wallet_alias): if "blockhash" in tx and "blockheight" not in tx: tx["blockheight"] = wallet.rpc.getblockheader(tx["blockhash"])["height"] ##################### Remove until here after dropping Core v0.19 support ##################### + + if tx["confirmations"] == 0: + tx["is_purged"] = wallet.is_tx_purged(txid) + return { "success": True, "tx": tx, diff --git a/src/cryptoadvance/specter/templates/includes/tx-data.html b/src/cryptoadvance/specter/templates/includes/tx-data.html index f80f2563d..6b0937b56 100644 --- a/src/cryptoadvance/specter/templates/includes/tx-data.html +++ b/src/cryptoadvance/specter/templates/includes/tx-data.html @@ -5,12 +5,12 @@ text-align: left; } .tx_info { - padding: 1em; - border: 1px solid #506072; - border-radius: 4px; - text-align: center; - margin-bottom: 2em; - } + padding: 1em; + border: 1px solid #506072; + border-radius: 4px; + text-align: center; + margin-bottom: 2em; + }

Transaction details


@@ -76,7 +76,26 @@ let walletName = jsonResponse.walletName; let tx = jsonResponse.tx; let walletLink = `{{ url_for('wallets_endpoint.wallet', wallet_alias='WALLET_ALIAS') }}`.replace("WALLET_ALIAS", self.wallet); - let rawtxHTML = ` + let rawtxHTML = ""; + if(tx.is_purged) { + rawtxHTML += ` +
+

This transaction (aka "tx") could not be found in your node's mempool!

+

If the bitcoin network is very busy, nodes will start purging the pending txs with the lowest fees. Txs older than two weeks are also purged.

+

A purged tx will never complete. Abandoning this tx will clear it from your wallet, making those funds spendable once again.

+

But first verify that other nodes have also purged your tx! Enter the tx id into a public block explorer (warning: slight privacy leak). If the tx cannot be found, it is safe to abandon this tx.

+ +

+
+ + + + +
+
+ `; + } + rawtxHTML += ` diff --git a/src/cryptoadvance/specter/wallet.py b/src/cryptoadvance/specter/wallet.py index 09e036259..c3360d1ea 100644 --- a/src/cryptoadvance/specter/wallet.py +++ b/src/cryptoadvance/specter/wallet.py @@ -258,7 +258,7 @@ class Wallet: obj = self.rpc.listsinceblock() txs = obj["transactions"] last_block = obj["lastblock"] - addresses = [tx["address"] for tx in txs] + addresses = [tx["address"] for tx in txs if "address" in tx] # remove duplicates addresses = list(dict.fromkeys(addresses)) max_recv = self.address_index - 1 @@ -713,6 +713,30 @@ class Wallet: except Exception as e: logger.warning("Could not get transaction {}, error: {}".format(txid, e)) + def is_tx_purged(self, txid): + # Is tx unconfirmed and no longer in the mempool? + try: + tx = self.rpc.gettransaction(txid) + + # Do this quick test first to avoid the costlier rpc call + if tx["confirmations"] > 0: + return False + + return txid not in self.rpc.getrawmempool() + except Exception as e: + logger.warning("Could not check is_tx_purged {}, error: {}".format(txid, e)) + + def abandontransaction(self, txid): + # Sanity checks: tx must be unconfirmed and cannot be in the mempool + tx = self.rpc.gettransaction(txid) + if tx["confirmations"] != 0: + raise SpecterError("Cannot abandon a transaction that has a confirmation.") + elif txid in self.rpc.getrawmempool(): + raise SpecterError( + "Cannot abandon a transaction that is still in the mempool." + ) + self.rpc.abandontransaction(txid) + def rescanutxo(self, explorer=None, requests_session=None, only_tor=False): delete_file(self._transactions.path) self.fetch_transactions() diff --git a/tests/bitcoin_core/README.md b/tests/bitcoin_core/README.md new file mode 100644 index 000000000..1cd572fd6 --- /dev/null +++ b/tests/bitcoin_core/README.md @@ -0,0 +1,10 @@ +These files have been directly copied from bitcoin core's `test` directory in order to test against more complicated node conditions. + +* messages.py: + * Copied as-is. There are quite a number of classes and utility functions in here and it was more straightforward to leave them all intact. + * `siphash` and `util` imports edited to relative imports so they'll work within the context of Specter's test runner. +* siphash.py: + * Copied as-is due to its simplicity. +* util.py: + * Heavily stripped down to its bare minimum. + * Edits to `create_lots_of_big_transactions` to be compatible with Specter's test suite node handling. diff --git a/tests/bitcoin_core/test/functional/test_framework/messages.py b/tests/bitcoin_core/test/functional/test_framework/messages.py new file mode 100644 index 000000000..4fc0b4266 --- /dev/null +++ b/tests/bitcoin_core/test/functional/test_framework/messages.py @@ -0,0 +1,1789 @@ +#!/usr/bin/env python3 + +# Specter note: Black formatting fails on this file. Disable: +# fmt: off + +# Copyright (c) 2010 ArtForz -- public domain half-a-node +# Copyright (c) 2012 Jeff Garzik +# Copyright (c) 2010-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Bitcoin test framework primitive and message structures + +CBlock, CTransaction, CBlockHeader, CTxIn, CTxOut, etc....: + data structures that should map to corresponding structures in + bitcoin/primitives + +msg_block, msg_tx, msg_headers, etc.: + data structures that represent network messages + +ser_*, deser_*: functions that handle serialization/deserialization. + +Classes use __slots__ to ensure extraneous attributes aren't accidentally added +by tests, compromising their intended effect. +""" +from codecs import encode +import copy +import hashlib +from io import BytesIO +import math +import random +import socket +import struct +import time + +from .siphash import siphash256 +from .util import hex_str_to_bytes, assert_equal + +MAX_LOCATOR_SZ = 101 +MAX_BLOCK_BASE_SIZE = 1000000 +MAX_BLOOM_FILTER_SIZE = 36000 +MAX_BLOOM_HASH_FUNCS = 50 + +COIN = 100000000 # 1 btc in satoshis +MAX_MONEY = 21000000 * COIN + +BIP125_SEQUENCE_NUMBER = 0xfffffffd # Sequence number that is BIP 125 opt-in and BIP 68-opt-out + +MAX_PROTOCOL_MESSAGE_LENGTH = 4000000 # Maximum length of incoming protocol messages +MAX_HEADERS_RESULTS = 2000 # Number of headers sent in one getheaders result +MAX_INV_SIZE = 50000 # Maximum number of entries in an 'inv' protocol message + +NODE_NETWORK = (1 << 0) +NODE_BLOOM = (1 << 2) +NODE_WITNESS = (1 << 3) +NODE_COMPACT_FILTERS = (1 << 6) +NODE_NETWORK_LIMITED = (1 << 10) + +MSG_TX = 1 +MSG_BLOCK = 2 +MSG_FILTERED_BLOCK = 3 +MSG_CMPCT_BLOCK = 4 +MSG_WTX = 5 +MSG_WITNESS_FLAG = 1 << 30 +MSG_TYPE_MASK = 0xffffffff >> 2 +MSG_WITNESS_TX = MSG_TX | MSG_WITNESS_FLAG + +FILTER_TYPE_BASIC = 0 + +WITNESS_SCALE_FACTOR = 4 + +# Serialization/deserialization tools +def sha256(s): + return hashlib.new('sha256', s).digest() + +def hash256(s): + return sha256(sha256(s)) + +def ser_compact_size(l): + r = b"" + if l < 253: + r = struct.pack("B", l) + elif l < 0x10000: + r = struct.pack(">= 32 + return rs + + +def uint256_from_str(s): + r = 0 + t = struct.unpack("> 24) & 0xFF + v = (c & 0xFFFFFF) << (8 * (nbytes - 3)) + return v + + +# deser_function_name: Allow for an alternate deserialization function on the +# entries in the vector. +def deser_vector(f, c, deser_function_name=None): + nit = deser_compact_size(f) + r = [] + for _ in range(nit): + t = c() + if deser_function_name: + getattr(t, deser_function_name)(f) + else: + t.deserialize(f) + r.append(t) + return r + + +# ser_function_name: Allow for an alternate serialization function on the +# entries in the vector (we use this for serializing the vector of transactions +# for a witness block). +def ser_vector(l, ser_function_name=None): + r = ser_compact_size(len(l)) + for i in l: + if ser_function_name: + r += getattr(i, ser_function_name)() + else: + r += i.serialize() + return r + + +def deser_uint256_vector(f): + nit = deser_compact_size(f) + r = [] + for _ in range(nit): + t = deser_uint256(f) + r.append(t) + return r + + +def ser_uint256_vector(l): + r = ser_compact_size(len(l)) + for i in l: + r += ser_uint256(i) + return r + + +def deser_string_vector(f): + nit = deser_compact_size(f) + r = [] + for _ in range(nit): + t = deser_string(f) + r.append(t) + return r + + +def ser_string_vector(l): + r = ser_compact_size(len(l)) + for sv in l: + r += ser_string(sv) + return r + + +# Deserialize from a hex string representation (eg from RPC) +def FromHex(obj, hex_string): + obj.deserialize(BytesIO(hex_str_to_bytes(hex_string))) + return obj + +# Convert a binary-serializable object to hex (eg for submission via RPC) +def ToHex(obj): + return obj.serialize().hex() + +# Objects that map to bitcoind objects, which can be serialized/deserialized + + +class CAddress: + __slots__ = ("net", "ip", "nServices", "port", "time") + + # see https://github.com/bitcoin/bips/blob/master/bip-0155.mediawiki + NET_IPV4 = 1 + + ADDRV2_NET_NAME = { + NET_IPV4: "IPv4" + } + + ADDRV2_ADDRESS_LENGTH = { + NET_IPV4: 4 + } + + def __init__(self): + self.time = 0 + self.nServices = 1 + self.net = self.NET_IPV4 + self.ip = "0.0.0.0" + self.port = 0 + + def deserialize(self, f, *, with_time=True): + """Deserialize from addrv1 format (pre-BIP155)""" + if with_time: + # VERSION messages serialize CAddress objects without time + self.time = struct.unpack("H", f.read(2))[0] + + def serialize(self, *, with_time=True): + """Serialize in addrv1 format (pre-BIP155)""" + assert self.net == self.NET_IPV4 + r = b"" + if with_time: + # VERSION messages serialize CAddress objects without time + r += struct.pack("H", self.port) + return r + + def deserialize_v2(self, f): + """Deserialize from addrv2 format (BIP155)""" + self.time = struct.unpack("H", f.read(2))[0] + + def serialize_v2(self): + """Serialize in addrv2 format (BIP155)""" + assert self.net == self.NET_IPV4 + r = b"" + r += struct.pack("H", self.port) + return r + + def __repr__(self): + return ("CAddress(nServices=%i net=%s addr=%s port=%i)" + % (self.nServices, self.ADDRV2_NET_NAME[self.net], self.ip, self.port)) + + +class CInv: + __slots__ = ("hash", "type") + + typemap = { + 0: "Error", + MSG_TX: "TX", + MSG_BLOCK: "Block", + MSG_TX | MSG_WITNESS_FLAG: "WitnessTx", + MSG_BLOCK | MSG_WITNESS_FLAG: "WitnessBlock", + MSG_FILTERED_BLOCK: "filtered Block", + MSG_CMPCT_BLOCK: "CompactBlock", + MSG_WTX: "WTX", + } + + def __init__(self, t=0, h=0): + self.type = t + self.hash = h + + def deserialize(self, f): + self.type = struct.unpack(" 21000000 * COIN: + return False + return True + + # Calculate the virtual transaction size using witness and non-witness + # serialization size (does NOT use sigops). + def get_vsize(self): + with_witness_size = len(self.serialize_with_witness()) + without_witness_size = len(self.serialize_without_witness()) + return math.ceil(((WITNESS_SCALE_FACTOR - 1) * without_witness_size + with_witness_size) / WITNESS_SCALE_FACTOR) + + def __repr__(self): + return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \ + % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime) + + +class CBlockHeader: + __slots__ = ("hash", "hashMerkleRoot", "hashPrevBlock", "nBits", "nNonce", + "nTime", "nVersion", "sha256") + + def __init__(self, header=None): + if header is None: + self.set_null() + else: + self.nVersion = header.nVersion + self.hashPrevBlock = header.hashPrevBlock + self.hashMerkleRoot = header.hashMerkleRoot + self.nTime = header.nTime + self.nBits = header.nBits + self.nNonce = header.nNonce + self.sha256 = header.sha256 + self.hash = header.hash + self.calc_sha256() + + def set_null(self): + self.nVersion = 1 + self.hashPrevBlock = 0 + self.hashMerkleRoot = 0 + self.nTime = 0 + self.nBits = 0 + self.nNonce = 0 + self.sha256 = None + self.hash = None + + def deserialize(self, f): + self.nVersion = struct.unpack(" 1: + newhashes = [] + for i in range(0, len(hashes), 2): + i2 = min(i+1, len(hashes)-1) + newhashes.append(hash256(hashes[i] + hashes[i2])) + hashes = newhashes + return uint256_from_str(hashes[0]) + + def calc_merkle_root(self): + hashes = [] + for tx in self.vtx: + tx.calc_sha256() + hashes.append(ser_uint256(tx.sha256)) + return self.get_merkle_root(hashes) + + def calc_witness_merkle_root(self): + # For witness root purposes, the hash of the + # coinbase, with witness, is defined to be 0...0 + hashes = [ser_uint256(0)] + + for tx in self.vtx[1:]: + # Calculate the hashes with witness data + hashes.append(ser_uint256(tx.calc_sha256(True))) + + return self.get_merkle_root(hashes) + + def is_valid(self): + self.calc_sha256() + target = uint256_from_compact(self.nBits) + if self.sha256 > target: + return False + for tx in self.vtx: + if not tx.is_valid(): + return False + if self.calc_merkle_root() != self.hashMerkleRoot: + return False + return True + + def solve(self): + self.rehash() + target = uint256_from_compact(self.nBits) + while self.sha256 > target: + self.nNonce += 1 + self.rehash() + + def __repr__(self): + return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" \ + % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, + time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx)) + + +class PrefilledTransaction: + __slots__ = ("index", "tx") + + def __init__(self, index=0, tx = None): + self.index = index + self.tx = tx + + def deserialize(self, f): + self.index = deser_compact_size(f) + self.tx = CTransaction() + self.tx.deserialize(f) + + def serialize(self, with_witness=True): + r = b"" + r += ser_compact_size(self.index) + if with_witness: + r += self.tx.serialize_with_witness() + else: + r += self.tx.serialize_without_witness() + return r + + def serialize_without_witness(self): + return self.serialize(with_witness=False) + + def serialize_with_witness(self): + return self.serialize(with_witness=True) + + def __repr__(self): + return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx)) + + +# This is what we send on the wire, in a cmpctblock message. +class P2PHeaderAndShortIDs: + __slots__ = ("header", "nonce", "prefilled_txn", "prefilled_txn_length", + "shortids", "shortids_length") + + def __init__(self): + self.header = CBlockHeader() + self.nonce = 0 + self.shortids_length = 0 + self.shortids = [] + self.prefilled_txn_length = 0 + self.prefilled_txn = [] + + def deserialize(self, f): + self.header.deserialize(f) + self.nonce = struct.unpack("= 70001: + # Relay field is optional for version 70001 onwards + try: + self.relay = struct.unpack(" +class msg_headers: + __slots__ = ("headers",) + msgtype = b"headers" + + def __init__(self, headers=None): + self.headers = headers if headers is not None else [] + + def deserialize(self, f): + # comment in bitcoind indicates these should be deserialized as blocks + blocks = deser_vector(f, CBlock) + for x in blocks: + self.headers.append(CBlockHeader(x)) + + def serialize(self): + blocks = [CBlock(x) for x in self.headers] + return ser_vector(blocks) + + def __repr__(self): + return "msg_headers(headers=%s)" % repr(self.headers) + + +class msg_merkleblock: + __slots__ = ("merkleblock",) + msgtype = b"merkleblock" + + def __init__(self, merkleblock=None): + if merkleblock is None: + self.merkleblock = CMerkleBlock() + else: + self.merkleblock = merkleblock + + def deserialize(self, f): + self.merkleblock.deserialize(f) + + def serialize(self): + return self.merkleblock.serialize() + + def __repr__(self): + return "msg_merkleblock(merkleblock=%s)" % (repr(self.merkleblock)) + + +class msg_filterload: + __slots__ = ("data", "nHashFuncs", "nTweak", "nFlags") + msgtype = b"filterload" + + def __init__(self, data=b'00', nHashFuncs=0, nTweak=0, nFlags=0): + self.data = data + self.nHashFuncs = nHashFuncs + self.nTweak = nTweak + self.nFlags = nFlags + + def deserialize(self, f): + self.data = deser_string(f) + self.nHashFuncs = struct.unpack("> (64 - b) | (n & ((1 << (64 - b)) - 1)) << b + + +def siphash_round(v0, v1, v2, v3): + v0 = (v0 + v1) & ((1 << 64) - 1) + v1 = rotl64(v1, 13) + v1 ^= v0 + v0 = rotl64(v0, 32) + v2 = (v2 + v3) & ((1 << 64) - 1) + v3 = rotl64(v3, 16) + v3 ^= v2 + v0 = (v0 + v3) & ((1 << 64) - 1) + v3 = rotl64(v3, 21) + v3 ^= v0 + v2 = (v2 + v1) & ((1 << 64) - 1) + v1 = rotl64(v1, 17) + v1 ^= v2 + v2 = rotl64(v2, 32) + return (v0, v1, v2, v3) + + +def siphash256(k0, k1, h): + n0 = h & ((1 << 64) - 1) + n1 = (h >> 64) & ((1 << 64) - 1) + n2 = (h >> 128) & ((1 << 64) - 1) + n3 = (h >> 192) & ((1 << 64) - 1) + v0 = 0x736F6D6570736575 ^ k0 + v1 = 0x646F72616E646F6D ^ k1 + v2 = 0x6C7967656E657261 ^ k0 + v3 = 0x7465646279746573 ^ k1 ^ n0 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n0 + v3 ^= n1 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n1 + v3 ^= n2 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n2 + v3 ^= n3 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n3 + v3 ^= 0x2000000000000000 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= 0x2000000000000000 + v2 ^= 0xFF + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + return v0 ^ v1 ^ v2 ^ v3 diff --git a/tests/bitcoin_core/test/functional/test_framework/util.py b/tests/bitcoin_core/test/functional/test_framework/util.py new file mode 100644 index 000000000..9e6be0d0d --- /dev/null +++ b/tests/bitcoin_core/test/functional/test_framework/util.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Helpful routines for regression testing.""" + +""" +HEAVILY trimmed down to bare minimum for Specter test cases +* create_lots_of_big_transactions edited for Specter compatibility +""" + + +from binascii import unhexlify +from decimal import Decimal, ROUND_DOWN +from io import BytesIO + + +def assert_equal(thing1, thing2, *args): + if thing1 != thing2 or any(thing1 != arg for arg in args): + raise AssertionError( + "not(%s)" % " == ".join(str(arg) for arg in (thing1, thing2) + args) + ) + + +def hex_str_to_bytes(hex_str): + return unhexlify(hex_str.encode("ascii")) + + +def satoshi_round(amount): + return Decimal(amount).quantize(Decimal("0.00000001"), rounding=ROUND_DOWN) + + +# Create large OP_RETURN txouts that can be appended to a transaction +# to make it large (helper for constructing large transactions). +def gen_return_txouts(): + # Some pre-processing to create a bunch of OP_RETURN txouts to insert into transactions we create + # So we have big transactions (and therefore can't fit very many into each block) + # create one script_pubkey + script_pubkey = "6a4d0200" # OP_RETURN OP_PUSH2 512 bytes + for _ in range(512): + script_pubkey = script_pubkey + "01" + # concatenate 128 txouts of above script_pubkey which we'll insert before the txout for change + txouts = [] + from .messages import CTxOut + + txout = CTxOut() + txout.nValue = 0 + txout.scriptPubKey = hex_str_to_bytes(script_pubkey) + for _ in range(128): + txouts.append(txout) + return txouts + + +# Create a spend of each passed-in utxo, splicing in "txouts" to each raw +# transaction to make it large. See gen_return_txouts() above. +def create_lots_of_big_transactions(wallet, txouts, utxos, num, fee): + node = wallet.rpc + addr = node.getnewaddress() + txids = [] + from .messages import CTransaction + + for _ in range(num): + t = utxos.pop() + inputs = [{"txid": t["txid"], "vout": t["vout"]}] + outputs = {} + change = t["amount"] - fee + outputs[addr] = float(satoshi_round(change)) + rawtx = node.createrawtransaction(inputs, outputs) + tx = CTransaction() + tx.deserialize(BytesIO(hex_str_to_bytes(rawtx))) + for txout in txouts: + tx.vout.append(txout) + newtx = tx.serialize().hex() + signresult = wallet.devices[0].sign_raw_tx(newtx, wallet) + txid = node.sendrawtransaction(signresult["hex"], 0) + txids.append(txid) + return txids diff --git a/tests/conftest.py b/tests/conftest.py index ba02be227..88b17913a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -44,24 +44,25 @@ def pytest_generate_tests(metafunc): metafunc.parametrize("docker", [False], scope="module") -@pytest.fixture(scope="module") -def bitcoin_regtest(docker, request): +def instantiate_bitcoind_controller(docker, request, rpcport=18543, extra_args=[]): # logging.getLogger().setLevel(logging.DEBUG) requested_version = request.config.getoption("--bitcoind-version") if docker: bitcoind_controller = BitcoindDockerController( - rpcport=18543, docker_tag=requested_version + rpcport=rpcport, docker_tag=requested_version ) else: if os.path.isfile("tests/bitcoin/src/bitcoind"): bitcoind_controller = BitcoindPlainController( - bitcoind_path="tests/bitcoin/src/bitcoind" + bitcoind_path="tests/bitcoin/src/bitcoind", rpcport=rpcport ) # always prefer the self-compiled bitcoind if existing else: - bitcoind_controller = ( - BitcoindPlainController() + bitcoind_controller = BitcoindPlainController( + rpcport=rpcport ) # Alternatively take the one on the path for now - bitcoind_controller.start_bitcoind(cleanup_at_exit=True, cleanup_hard=True) + bitcoind_controller.start_bitcoind( + cleanup_at_exit=True, cleanup_hard=True, extra_args=extra_args + ) running_version = bitcoind_controller.version() requested_version = request.config.getoption("--bitcoind-version") assert ( @@ -72,6 +73,11 @@ def bitcoin_regtest(docker, request): return bitcoind_controller +@pytest.fixture(scope="module") +def bitcoin_regtest(docker, request): + return instantiate_bitcoind_controller(docker, request, extra_args=None) + + @pytest.fixture def empty_data_folder(): # Make sure that this folder never ever gets a reasonable non-testing use-case diff --git a/tests/test_bitcoind.py b/tests/test_bitcoind.py index 3e3e9c319..b96ce6953 100644 --- a/tests/test_bitcoind.py +++ b/tests/test_bitcoind.py @@ -7,6 +7,8 @@ from cryptoadvance.specter.bitcoind import fetch_wallet_addresses_for_mining def test_bitcoinddocker_running(caplog, docker, request): + # TODO: Refactor this to use conftest.instantiate_bitcoind_controller + # to reduce redundant code? caplog.set_level(logging.INFO) caplog.set_level(logging.DEBUG, logger="cryptoadvance.specter") requested_version = request.config.getoption("--bitcoind-version") diff --git a/tests/test_specter.py b/tests/test_specter.py index 0d032d9e8..8e8364f57 100644 --- a/tests/test_specter.py +++ b/tests/test_specter.py @@ -1,7 +1,11 @@ import json, logging, pytest -from cryptoadvance.specter.specter import get_rpc, Specter -from cryptoadvance.specter.helpers import alias +from decimal import Decimal +from cryptoadvance.specter.helpers import alias, generate_mnemonic +from cryptoadvance.specter.key import Key from cryptoadvance.specter.rpc import BitcoinRPC +from cryptoadvance.specter.specter import get_rpc, Specter +from cryptoadvance.specter.specter_error import SpecterError +from cryptoadvance.specter.wallet_manager import WalletManager def test_alias(): @@ -39,3 +43,176 @@ def test_specter(specter_regtest_configured, caplog): # that might only work if your chain is fresh # assert json_return['blocks'] == 100 assert json_return["chain"] == "regtest" + + +def test_abandon_purged_tx( + caplog, docker, request, devices_filled_data_folder, device_manager +): + # Specter should support calling abandontransaction if a pending tx has been purged + # from the mempool. Test starts a new bitcoind with a restricted mempool to make it + # easier to spam the mempool and purge our target tx. + # TODO: Similar test but for maxmempoolexpiry? + + # Copied and adapted from: + # https://github.com/bitcoin/bitcoin/blob/master/test/functional/mempool_limit.py + from bitcoin_core.test.functional.test_framework.util import ( + gen_return_txouts, + satoshi_round, + create_lots_of_big_transactions, + ) + from conftest import instantiate_bitcoind_controller + + caplog.set_level(logging.DEBUG) + + # ==== Specter-specific: do custom setup ==== + # Instantiate a new bitcoind w/limited mempool. Use a different port to not interfere + # with existing instance for other tests. + bitcoind_controller = instantiate_bitcoind_controller( + docker, + request, + rpcport=18998, + extra_args=["-acceptnonstdtxn=1", "-maxmempool=5", "-spendzeroconfchange=0"], + ) + rpcconn = bitcoind_controller.rpcconn + rpc = rpcconn.get_rpc() + assert rpc is not None + assert rpc.ipaddress != None + + # Note: Our utxo creation is simpler than mempool_limit.py's approach since we're + # running in regtest and can just use generatetoaddress(). + + # Instantiate a new Specter instance to talk to this bitcoind + config = { + "rpc": { + "autodetect": False, + "user": rpcconn.rpcuser, + "password": rpcconn.rpcpassword, + "port": rpcconn.rpcport, + "host": rpcconn.ipaddress, + "protocol": "http", + }, + "auth": { + "method": "rpcpasswordaspin", + }, + } + specter = Specter(data_folder=devices_filled_data_folder, config=config) + specter.check() + + specter.check_node_info() + assert specter._info["mempool_info"]["maxmempool"] == 5 * 1000 * 1000 # 5MB + + # Largely copy-and-paste from test_wallet_manager.test_wallet_createpsbt. + # TODO: Make a test fixture in conftest.py that sets up already funded wallets + # for a bitcoin core hot wallet. + wallet_manager = WalletManager( + 200100, + devices_filled_data_folder, + rpc, + "regtest", + device_manager, + ) + + # Create a new device that can sign psbts (Bitcoin Core hot wallet) + device = device_manager.add_device( + name="bitcoin_core_hot_wallet", device_type="bitcoincore", keys=[] + ) + device.setup_device(file_password=None, wallet_manager=wallet_manager) + device.add_hot_wallet_keys( + mnemonic=generate_mnemonic(strength=128), + passphrase="", + paths=["m/49h/0h/0h"], + file_password=None, + wallet_manager=wallet_manager, + testnet=True, + keys_range=[0, 1000], + keys_purposes=[], + ) + + wallet = wallet_manager.create_wallet( + "bitcoincore_test_wallet", 1, "sh-wpkh", [device.keys[0]], [device] + ) + + # Fund the wallet. Going to need a LOT of utxos to play with. + logging.info("Generating utxos to wallet") + address = wallet.getnewaddress() + wallet.rpc.generatetoaddress(91, address) + + # newly minted coins need 100 blocks to get spendable + # let's mine another 100 blocks to get these coins spendable + wallet.rpc.generatetoaddress(101, address) + + # update the wallet data + wallet.get_balance() + + # ==== Begin test from mempool_limit.py ==== + txouts = gen_return_txouts() + relayfee = satoshi_round(rpc.getnetworkinfo()["relayfee"]) + + logging.info("Check that mempoolminfee is minrelytxfee") + assert satoshi_round(rpc.getmempoolinfo()["minrelaytxfee"]) == Decimal("0.00001000") + assert satoshi_round(rpc.getmempoolinfo()["mempoolminfee"]) == Decimal("0.00001000") + + txids = [] + utxos = wallet.rpc.listunspent() + + logging.info("Create a mempool tx that will be evicted") + us0 = utxos.pop() + inputs = [{"txid": us0["txid"], "vout": us0["vout"]}] + outputs = {wallet.getnewaddress(): 0.0001} + tx = wallet.rpc.createrawtransaction(inputs, outputs) + wallet.rpc.settxfee(str(relayfee)) # specifically fund this tx with low fee + txF = wallet.rpc.fundrawtransaction(tx) + wallet.rpc.settxfee(0) # return to automatic fee selection + txFS = device.sign_raw_tx(txF["hex"], wallet) + txid = wallet.rpc.sendrawtransaction(txFS["hex"]) + + # ==== Specter-specific: can't abandon a valid pending tx ==== + try: + wallet.abandontransaction(txid) + except SpecterError as e: + assert "Cannot abandon" in str(e) + + # ==== Resume test from mempool_limit.py ==== + # Spam the mempool with big transactions! + relayfee = satoshi_round(rpc.getnetworkinfo()["relayfee"]) + base_fee = float(relayfee) * 100 + for i in range(3): + txids.append([]) + txids[i] = create_lots_of_big_transactions( + wallet, txouts, utxos[30 * i : 30 * i + 30], 30, (i + 1) * base_fee + ) + + logging.info("The tx should be evicted by now") + assert txid not in wallet.rpc.getrawmempool() + txdata = wallet.rpc.gettransaction(txid) + assert txdata["confirmations"] == 0 # confirmation should still be 0 + + # ==== Specter-specific: Verify purge and abandon ==== + assert wallet.is_tx_purged(txid) + wallet.abandontransaction(txid) + + # tx will still be in the wallet but marked "abandoned" + txdata = wallet.rpc.gettransaction(txid) + for detail in txdata["details"]: + if detail["category"] == "send": + assert detail["abandoned"] + + # Can we now spend those same inputs? + outputs = {wallet.getnewaddress(): 0.0001} + tx = wallet.rpc.createrawtransaction(inputs, outputs) + + # Fund this tx with a high enough fee + relayfee = satoshi_round(rpc.getnetworkinfo()["relayfee"]) + wallet.rpc.settxfee(str(relayfee * Decimal("3.0"))) + + txF = wallet.rpc.fundrawtransaction(tx) + wallet.rpc.settxfee(0) # return to automatic fee selection + txFS = device.sign_raw_tx(txF["hex"], wallet) + txid = wallet.rpc.sendrawtransaction(txFS["hex"]) + + # Should have been accepted by the mempool + assert txid in wallet.rpc.getrawmempool() + assert wallet.get_balance()["untrusted_pending"] == 0.0001 + + # Clean up + bitcoind_controller.stop_bitcoind() diff --git a/tests/test_wallet_manager.py b/tests/test_wallet_manager.py index c9af9b2e6..6c02d5d14 100644 --- a/tests/test_wallet_manager.py +++ b/tests/test_wallet_manager.py @@ -1,4 +1,5 @@ import json, os +from conftest import instantiate_bitcoind_controller from cryptoadvance.specter.rpc import RpcError from cryptoadvance.specter.specter_error import SpecterError from cryptoadvance.specter.wallet import Wallet @@ -6,11 +7,16 @@ from cryptoadvance.specter.key import Key from cryptoadvance.specter.wallet_manager import WalletManager -def test_WalletManager(bitcoin_regtest, devices_filled_data_folder, device_manager): +def test_WalletManager(docker, request, devices_filled_data_folder, device_manager): + # Instantiate a fresh bitcoind instance to isolate this test. + bitcoind_controller = instantiate_bitcoind_controller( + docker, request, rpcport=18998 + ) + wm = WalletManager( 200100, devices_filled_data_folder, - bitcoin_regtest.get_rpc(), + bitcoind_controller.rpcconn.get_rpc(), "regtest", device_manager, ) @@ -71,12 +77,20 @@ def test_WalletManager(bitcoin_regtest, devices_filled_data_folder, device_manag assert not os.path.exists(wallet_fullpath) assert len(wm.wallets) == 1 + # cleanup + bitcoind_controller.stop_bitcoind() + + +def test_wallet_createpsbt(docker, request, devices_filled_data_folder, device_manager): + # Instantiate a fresh bitcoind instance to isolate this test. + bitcoind_controller = instantiate_bitcoind_controller( + docker, request, rpcport=18998 + ) -def test_wallet_createpsbt(bitcoin_regtest, devices_filled_data_folder, device_manager): wm = WalletManager( 200100, devices_filled_data_folder, - bitcoin_regtest.get_rpc(), + bitcoind_controller.rpcconn.get_rpc(), "regtest", device_manager, ) @@ -166,6 +180,9 @@ def test_wallet_createpsbt(bitcoin_regtest, devices_filled_data_folder, device_m assert len(wallet.rpc.listlockunspent()) == 0 assert wallet.full_available_balance == wallet.fullbalance + # cleanup + bitcoind_controller.stop_bitcoind() + def test_wallet_sortedmulti( bitcoin_regtest, devices_filled_data_folder, device_manager
Transaction id:TxID: