From 16fc83f47d119b4966944a7d707ef604874b2afa Mon Sep 17 00:00:00 2001 From: openoms <43343391+openoms@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:57:42 +0200 Subject: [PATCH] chore: migrate the watch-only Bitcoin Core wallet to use descriptors (#177) * chore: update wallet creation to use watch-only-descriptor-wallet * remove duplicate version check * fix: update release URL in install script and correct typo in wallet import message * ci: test amd64 image descriptor wallet migration with Bats * ci: boot amd64 image tests with OVMF pflash * fix: gate descriptor wallet migration on Bitcoin Core v30 Detect the connected Bitcoin Core version over RPC before migrating. Keep wallet.dat for v29.x or when the version cannot be determined. Add regression coverage for both migration and compatibility paths. * ci: make amd64 image tests independent of guest apt Inject a pinned bats-core checkout into the temporary VM instead of installing Bats through the guest package repositories. Remove the duplicate pull request trigger and simplify artifact lookup. * ci: harden image build run * docs(FAQ): how automatic migration works --- .github/workflows/amd64-image-test.yml | 65 ++++++ FAQ.md | 70 ++++++- README.md | 2 +- ci/amd64/test.amd64-image-bats.sh | 138 +++++++++++++ scripts/_functions.bitcoincore.sh | 156 +++++++++++--- scripts/_functions.sh | 4 +- scripts/install.bitcoincore.sh | 14 +- scripts/install.joinmarket.sh | 3 +- scripts/menu.wallet.sh | 2 +- scripts/standalone/_functions.standalone.sh | 6 +- test/README.md | 34 ++++ test/bats/descriptor-wallet.bats | 214 ++++++++++++++++++++ test/run-bats-local.sh | 11 + 13 files changed, 675 insertions(+), 44 deletions(-) create mode 100644 .github/workflows/amd64-image-test.yml create mode 100755 ci/amd64/test.amd64-image-bats.sh create mode 100644 test/README.md create mode 100644 test/bats/descriptor-wallet.bats create mode 100755 test/run-bats-local.sh diff --git a/.github/workflows/amd64-image-test.yml b/.github/workflows/amd64-image-test.yml new file mode 100644 index 0000000..bc403df --- /dev/null +++ b/.github/workflows/amd64-image-test.yml @@ -0,0 +1,65 @@ +name: amd64-image-test + +concurrency: + group: amd64-image-test-${{ github.event.workflow_run.id || inputs.run_id || github.run_id }} + cancel-in-progress: true + +on: + workflow_run: + workflows: ["amd64-image-build"] + types: [completed] + workflow_dispatch: + inputs: + run_id: + description: "amd64-image-build workflow run ID to test" + required: true + type: string + +permissions: + actions: read + contents: read + +jobs: + bats-image-test: + name: Run Bats against amd64 image artifact + runs-on: ubuntu-22.04 + timeout-minutes: 240 + if: ${{ github.event_name != 'workflow_run' || (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_repository.full_name == github.repository) }} + + steps: + - uses: actions/checkout@v4 + with: + repository: ${{ github.event.workflow_run.head_repository.full_name || github.repository }} + ref: ${{ github.event.workflow_run.head_sha || github.ref }} + + - name: Check out bats-core + uses: actions/checkout@v4 + with: + repository: bats-core/bats-core + # bats-core v1.12.0 + ref: 713504bc0224a19b3d7c7958c18dc07f64f54b44 + path: .bats-core + persist-credentials: false + + - name: Download amd64 image artifact + uses: actions/download-artifact@v4 + with: + github-token: ${{ github.token }} + run-id: ${{ github.event.workflow_run.id || inputs.run_id }} + pattern: joininbox-amd64-image-* + path: artifacts + merge-multiple: true + + - name: Verify and decompress image + shell: bash + run: | + set -euo pipefail + + cd artifacts + sha256sum -c joininbox-amd64-debian.qcow2.gz.sha256 + gzip -dk joininbox-amd64-debian.qcow2.gz + sha256sum -c joininbox-amd64-debian.qcow2.sha256 + + - name: Run image Bats tests + timeout-minutes: 30 + run: ci/amd64/test.amd64-image-bats.sh "${GITHUB_WORKSPACE}/artifacts/joininbox-amd64-debian.qcow2" diff --git a/FAQ.md b/FAQ.md index ae14865..db608dd 100644 --- a/FAQ.md +++ b/FAQ.md @@ -30,6 +30,7 @@ - [Wallet recovery](#wallet-recovery) - [on JoininBox](#on-joininbox) - [on the remote node](#on-the-remote-node) +- [Migrating from legacy wallet.dat to descriptor wallet](#migrating-from-legacy-walletdat-to-descriptor-wallet) - [USB SSD recommendation](#usb-ssd-recommendation) - [Pruned node notes](#pruned-node-notes) - [External drive](#external-drive) @@ -195,7 +196,7 @@ Use `ssh` with `torsocks` on the desktop (needs Tor installed): server=1 daemon=1 disablewallet=0 -main.wallet=wallet.dat +main.wallet=watch-only-descriptor-wallet # Connection settings rpcuser=REDACTED @@ -428,7 +429,7 @@ JoinMarket docs: * https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/master/docs/USAGE.md#recover ### on JoininBox -* Connect the remote bitcoind with `CONFIG` -> `CONNECT` menu so it checks if the connection is successful. It will also set the remote watch-only wallet in bitcoind to "joininbox" so will need to rescan that after recovering an old wallet with previously used addresses. +* Connect the remote bitcoind with `CONFIG` -> `CONNECT` menu so it checks if the connection is successful. It will also set the remote watch-only-descriptor-wallet in bitcoind to "joininbox" so will need to rescan that after recovering an old wallet with previously used addresses. * When using the CLI and connecting to the remote node over Tor, you will need to use the script with the torsocks prefix like: `torsocks python3 wallet-tool.py --recoversync -g 20 ~/.joinmarket/wallets/wallet.jmdat` @@ -437,7 +438,7 @@ JoinMarket docs: * Use the menu option `WALLET` -> `RESCAN` or follow manually * the wallet defined as `rpc_wallet =` -in the joinmarket.cfg is the wallet which is used as watch only in the remote bitcoind. +in the joinmarket.cfg is the wallet which is used as a watch-only-descriptor-wallet in the remote bitcoind. You need to run rescanblockchain on that wallet in bitcoind after importing the joinmarket wallet. * The wallet is set in the joinmarket.cfg (by default called `joininbox` should show up when you run: `bitcoin-cli listwallets` @@ -449,6 +450,65 @@ Rescanning from the first SegWit block is sufficient for the default SegWit wall * Monitor progress (on a RaspiBlitz): `sudo tail -fn 100 /mnt/hdd/app-storage/bitcoin/debug.log` Once the rescan is finished you balances should appear in the `INFO` menu (`wallet-tool.py`) + +## Migrating from legacy wallet.dat to descriptor wallet + +Starting with the 0.9.0 version, JoininBox uses Bitcoin Core's descriptor wallets (`watch-only-descriptor-wallet`) instead of the legacy `wallet.dat`. This change provides better compatibility with modern Bitcoin Core versions (v26+) and aligns with Bitcoin Core's default wallet format. + +### Why this change? + +Bitcoin Core has deprecated BDB (Berkeley DB) wallets in favor of descriptor wallets. The new descriptor wallets: +- Are the default in Bitcoin Core v26+ +- Don't require the `deprecatedrpc=create_bdb` configuration +- Have better performance and features +- Are actively maintained and improved + +### How automatic migration works + +When `rpc_wallet_file` is still set to `wallet.dat`, JoininBox queries the connected Bitcoin Core node using RPC before changing the configuration: + +- Bitcoin Core v30.0 and newer: JoininBox atomically changes `rpc_wallet_file` to `watch-only-descriptor-wallet`, creates or loads that descriptor wallet, and displays the migration notice when the old `wallet.dat` is present. +- Bitcoin Core v29.x and earlier: JoininBox keeps using `wallet.dat` and does not start automatic migration. +- Version unavailable: JoininBox leaves `wallet.dat` configured rather than migrating without confirming compatibility. + +The migration does not rename, modify, or delete the old `wallet.dat`. It changes which Bitcoin Core wallet JoinMarket uses for watch-only address imports and transaction history. + +### Migration steps for existing users + +If you're upgrading from a previous version of JoininBox that used `wallet.dat`, follow these steps: + +1. **The migration notice will appear automatically on Bitcoin Core v30.0 and newer** + When you first use any wallet-related function after updating, JoininBox checks the connected Bitcoin Core version. On v30.0 and newer it switches the configured RPC wallet to `watch-only-descriptor-wallet`, detects the old `wallet.dat`, and displays a migration notice. Bitcoin Core v29.x and earlier continue using `wallet.dat` without starting the automatic migration. + +2. **Open each JoinMarket wallet** + Go to `WALLET` -> `DISPLAY` and open each of your JoinMarket wallets (`.jmdat` files) at least once. This imports the addresses into the new `watch-only-descriptor-wallet` in Bitcoin Core. + +3. **Run a blockchain rescan** + After opening all your wallets, go to `WALLET` -> `RESCAN` and enter a blockheight: + - Use `481824` (first SegWit block) for wallets created after August 2017 + - Can use a later blockheight if you know when your wallet had its first deposit + +4. **Wait for the rescan to complete** + The rescan can take several hours depending on the blockheight and your hardware. Monitor progress with: + ```bash + # On standalone JoininBox: + sudo tail -f /home/bitcoin/.bitcoin/debug.log + + # On RaspiBlitz: + sudo tail -f /mnt/hdd/bitcoin/debug.log + ``` + +5. **Verify your balances** + Once the rescan completes, check your wallet balances with `WALLET` -> `DISPLAY`. + +### Notes + +- The old `wallet.dat` is not deleted and remains in Bitcoin Core +- Automatic migration only runs when the connected Bitcoin Core version is v30.0 or newer +- You only need to perform this migration once +- The migration notice will not appear again after you acknowledge it +- If you have issues, you can reset the migration flag by removing `walletMigrationDone=true` from `/home/joinmarket/joinin.conf` + ## USB SSD recommendation **JoininBox operates on the minimum viable hardware under the assumption that the seed (and passphrase) of the wallets used is safely backed up and can be recovered fully** * The above warning is especially true for SDcard as they fail often, use a good quality one. @@ -519,7 +579,7 @@ Alternatively to a pruned node there could be a larger >400 GB storage connected # -rw------- 1 bitcoin bitcoin 1521305 Mar 21 10:38 peers.dat # -rw-r--r-- 1 bitcoin bitcoin 7 Mar 21 10:08 settings.json # drwx------ 34 bitcoin bitcoin 4096 Dec 7 23:39 specter - # drwx------ 2 bitcoin bitcoin 4096 Mar 21 10:38 wallet.dat + # drwx------ 2 bitcoin bitcoin 4096 Mar 21 10:38 watch-only-descriptor-wallet installMainnet ... # # OK - the bitcoind.service is now enabled @@ -528,7 +588,7 @@ Alternatively to a pruned node there could be a larger >400 GB storage connected # # # Monitor the bitcoind with: sudo tail -f /home/bitcoin/.bitcoin/mainnet/debug.log # - # # Create wallet.dat ... + # # Create watch-only-descriptor-wallet ... # error code: -28 # error message: # Loading block index... diff --git a/README.md b/README.md index a9f254f..f9e7ef4 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ A minimalistic, security focused linux environment for JoinMarket with a termina * Start a pruned node from https://pruned.host4coins.net/blocks * JoininBox is part of the RaspiBlitz SERVICES -**The addresses, transactions and balances of JoinMarket can be seen in the watch-only wallet of the connected node.** +**The addresses, transactions and balances of JoinMarket can be seen in the watch-only-descriptor-wallet of the connected node.** * use your own or a trusted node * to protect privacy in case of physical access use disk encryption diff --git a/ci/amd64/test.amd64-image-bats.sh b/ci/amd64/test.amd64-image-bats.sh new file mode 100755 index 0000000..5313656 --- /dev/null +++ b/ci/amd64/test.amd64-image-bats.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +set -euo pipefail + +image="${1:-${GITHUB_WORKSPACE:-$(pwd)}/ci/amd64/builds/joininbox-amd64-debian-qemu/joininbox-amd64-debian.qcow2}" +ssh_port="${SSH_PORT:-2222}" +ssh_password="${SSH_PASSWORD:-joininbox}" +qemu_pid_file="${RUNNER_TEMP:-/tmp}/joininbox-qemu.pid" +bats_core_dir="${BATS_CORE_DIR:-${GITHUB_WORKSPACE:-$(pwd)}/.bats-core}" + +if [ ! -f "${image}" ]; then + echo "Missing image: ${image}" >&2 + exit 1 +fi + +if [ ! -x "${bats_core_dir}/bin/bats" ]; then + echo "Missing bats-core checkout: ${bats_core_dir}" >&2 + exit 1 +fi + +sudo apt-get update +sudo apt-get install -y ovmf qemu-system-x86 sshpass + +ssh_opts=( + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o ConnectTimeout=5 + -p "${ssh_port}" +) + +ovmf_code="${OVMF_CODE:-${OVMF_BIOS:-}}" +ovmf_vars_template="${OVMF_VARS:-}" +ovmf_vars="${RUNNER_TEMP:-/tmp}/joininbox-ovmf-vars.fd" +qemu_firmware_args=() + +if [ -z "${ovmf_code}" ]; then + for candidate in \ + /usr/share/OVMF/OVMF_CODE_4M.fd \ + /usr/share/OVMF/OVMF_CODE_4M.secboot.fd \ + /usr/share/OVMF/OVMF_CODE_4M.ms.fd \ + /usr/share/OVMF/OVMF_CODE.fd \ + /usr/share/OVMF/OVMF.fd \ + /usr/share/ovmf/OVMF_CODE_4M.fd \ + /usr/share/ovmf/OVMF_CODE_4M.secboot.fd \ + /usr/share/ovmf/OVMF_CODE_4M.ms.fd \ + /usr/share/ovmf/OVMF_CODE.fd \ + /usr/share/ovmf/OVMF.fd \ + OVMF.fd; do + if [ -f "${candidate}" ]; then + ovmf_code="${candidate}" + break + fi + done +fi + +if [ -z "${ovmf_code}" ] || [ ! -f "${ovmf_code}" ]; then + echo "No OVMF firmware found. Set OVMF_CODE or OVMF_BIOS to the firmware path." >&2 + find /usr/share/OVMF /usr/share/ovmf -maxdepth 1 -type f -name '*.fd' -print 2>/dev/null || true + exit 1 +fi + +case "${ovmf_code##*/}" in + *CODE*) + if [ -z "${ovmf_vars_template}" ]; then + for candidate in \ + "${ovmf_code/CODE/VARS}" \ + /usr/share/OVMF/OVMF_VARS_4M.fd \ + /usr/share/OVMF/OVMF_VARS.fd \ + /usr/share/ovmf/OVMF_VARS_4M.fd \ + /usr/share/ovmf/OVMF_VARS.fd; do + if [ -f "${candidate}" ]; then + ovmf_vars_template="${candidate}" + break + fi + done + fi + + if [ -z "${ovmf_vars_template}" ] || [ ! -f "${ovmf_vars_template}" ]; then + echo "No OVMF VARS template found for ${ovmf_code}. Set OVMF_VARS to the template path." >&2 + find /usr/share/OVMF /usr/share/ovmf -maxdepth 1 -type f -name '*.fd' -print 2>/dev/null || true + exit 1 + fi + + cp "${ovmf_vars_template}" "${ovmf_vars}" + qemu_firmware_args=( + -drive "if=pflash,format=raw,readonly=on,file=${ovmf_code}" + -drive "if=pflash,format=raw,file=${ovmf_vars}" + ) + ;; + *) + qemu_firmware_args=(-bios "${ovmf_code}") + ;; +esac + +cleanup() { + if [ -f "${qemu_pid_file}" ]; then + qemu_pid="$(cat "${qemu_pid_file}")" + if kill -0 "${qemu_pid}" 2>/dev/null; then + kill "${qemu_pid}" 2>/dev/null || true + timeout 30s tail --pid="${qemu_pid}" -f /dev/null 2>/dev/null || + kill -9 "${qemu_pid}" 2>/dev/null || + true + fi + fi +} +trap cleanup EXIT + +rm -f "${qemu_pid_file}" + +qemu-system-x86_64 \ + -m 2048 \ + -smp 2 \ + "${qemu_firmware_args[@]}" \ + -drive "file=${image},format=qcow2" \ + -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" \ + -device e1000,netdev=net0 \ + -display none \ + -snapshot \ + -pidfile "${qemu_pid_file}" \ + -daemonize + +echo "Waiting for SSH in the booted image" +for attempt in {1..120}; do + if sshpass -p "${ssh_password}" ssh "${ssh_opts[@]}" joinmarket@127.0.0.1 "true" 2>/dev/null; then + break + fi + if [ "${attempt}" -eq 120 ]; then + echo "Timed out waiting for SSH" >&2 + exit 1 + fi + sleep 5 +done + +tar -C "${bats_core_dir}" -cf - . | + sshpass -p "${ssh_password}" ssh "${ssh_opts[@]}" joinmarket@127.0.0.1 \ + "mkdir -p /tmp/bats-core && tar -C /tmp/bats-core -xf -" + +sshpass -p "${ssh_password}" ssh "${ssh_opts[@]}" joinmarket@127.0.0.1 \ + "PATH=/tmp/bats-core/bin:\$PATH /home/joinmarket/joininbox/test/run-bats-local.sh" diff --git a/scripts/_functions.bitcoincore.sh b/scripts/_functions.bitcoincore.sh index e4f1786..4c1988c 100755 --- a/scripts/_functions.bitcoincore.sh +++ b/scripts/_functions.bitcoincore.sh @@ -247,8 +247,8 @@ setJMconfigToSignet() { sed -i "s/^rpc_password =.*/rpc_password = $RPCPWSIGNET/g" $JMcfgPath echo "# rpc_password = $RPCPWSIGNET" # rpc_wallet_file - sed -i "s/^rpc_wallet_file =.*/rpc_wallet_file = wallet.dat/g" $JMcfgPath - echo "# using the bitcoind wallet: wallet.dat" + sed -i "s/^rpc_wallet_file =.*/rpc_wallet_file = watch-only-descriptor-wallet/g" $JMcfgPath + echo "# using the bitcoind watch-only-descriptor-wallet" # rpc_host sed -i "s/^rpc_host =.*/rpc_host = 127.0.0.1/g" $JMcfgPath echo "# rpc_host = 127.0.0.1" @@ -321,38 +321,79 @@ function getRPC { fi } +# getConnectedBitcoinCoreVersion - read the numeric version over node RPC +function getConnectedBitcoinCoreVersion { + local tor="" + if [ "$(echo "$rpc_host" | grep -c .onion)" -gt 0 ]; then + tor="torsocks" + fi + $tor curl -sS --data-binary \ + '{"jsonrpc": "1.0", "id":"get_bitcoin_core_version", "method": "getnetworkinfo", "params": []}' \ + "http://$rpc_user:$rpc_pass@$rpc_host:$rpc_port/" 2>/dev/null | + jq -r '.result.version // empty' 2>/dev/null +} + +# migrateLegacyRPCWalletConfig - switch the persisted JoinMarket RPC wallet +# from the legacy wallet.dat name to the descriptor wallet used by JoininBox +# when the connected Bitcoin Core version is v30.0 or newer +function migrateLegacyRPCWalletConfig { + if [ "$rpc_wallet" != "wallet.dat" ]; then + return 0 + fi + + local bitcoinCoreVersion + bitcoinCoreVersion=$(getConnectedBitcoinCoreVersion) + if ! [[ "$bitcoinCoreVersion" =~ ^[0-9]+$ ]]; then + echo "# Could not determine the connected Bitcoin Core version; keeping wallet.dat" + return 0 + fi + # Bitcoin Core's numeric version is 290200 for v29.2 and 300000 for v30.0. + if [ "$bitcoinCoreVersion" -lt 300000 ]; then + echo "# Connected Bitcoin Core is v29.x or earlier; keeping wallet.dat" + return 0 + fi + + echo "# Migrating the configured Bitcoin Core wallet from wallet.dat to watch-only-descriptor-wallet" + local migrationConfigOutput + if ! migrationConfigOutput=$(mktemp "${JMcfgPath}.XXXXXX"); then + echo "# Failed to create a temporary descriptor wallet configuration" >&2 + return 1 + fi + if ! sed \ + "s/^rpc_wallet_file =.*/rpc_wallet_file = watch-only-descriptor-wallet/g" \ + "$JMcfgPath" >"$migrationConfigOutput"; then + rm -f "$migrationConfigOutput" + echo "# Failed to prepare the descriptor wallet configuration" >&2 + return 1 + fi + if ! mv "$migrationConfigOutput" "$JMcfgPath"; then + rm -f "$migrationConfigOutput" + echo "# Failed to update the descriptor wallet configuration" >&2 + return 1 + fi + getRPC + if [ "$rpc_wallet" != "watch-only-descriptor-wallet" ]; then + echo "# Failed to select the descriptor wallet configuration" >&2 + return 1 + fi +} + # checkRPCwallet function checkRPCwallet { getRPC + migrateLegacyRPCWalletConfig || return 1 if [ $# -eq 0 ]; then rpc_wallet=$rpc_wallet else rpc_wallet=$1 fi - echo "# Check 'deprecatedrpc=create_bdb' in bitcoin.conf" - source ${joininConfPath} - if [ $runningEnv = standalone ]; then - bitcoinConfPath="/home/bitcoin/.bitcoin/bitcoin.conf" - elif [ $runningEnv = raspiblitz ]; then - if [ -f "/mnt/hdd/raspiblitz.conf" ]; then - bitcoinConfPath="/mnt/hdd/bitcoin/bitcoin.conf" - else - bitcoinConfPath="/mnt/hdd/app-data/bitcoin/bitcoin.conf" - fi - fi - if ! sudo grep -c "deprecatedrpc=create_bdb" "$bitcoinConfPath"; then - echo "# Place 'deprecatedrpc=create_bdb' in bitcoin.conf" - echo "deprecatedrpc=create_bdb" | sudo tee -a "$bitcoinConfPath" - echo "# Restarting bitcoind" - sudo systemctl restart bitcoind - fi echo "# Making sure the set $rpc_wallet wallet is present in bitcoind" trap 'rm -f "$connectionOutput"' EXIT connectionOutput=$(mktemp -p /dev/shm/) walletFound=$(customRPC "# Check wallet" "listwallets" 2>$connectionOutput | grep -c "$rpc_wallet") if [ $walletFound -eq 0 ]; then - echo "# Setting a watch only wallet in Bitcoin Core named $rpc_wallet" + echo "# Setting a watch-only-descriptor-wallet in Bitcoin Core named $rpc_wallet" tor="" if [ $(echo $rpc_host | grep -c .onion) -gt 0 ]; then tor="torsocks" @@ -361,8 +402,8 @@ function checkRPCwallet { fi #TODO rewrite customRPC to support multiple params $tor curl -sS --data-binary \ - '{"jsonrpc": "1.0", "id":"# Create the bitcoind wallet", "method": "createwallet", "params": {"wallet_name":"'"$rpc_wallet"'","descriptors":false}}' \ - http://$rpc_user:$rpc_pass@$rpc_host:$rpc_port/wallet/$rpc_wallet | jq . + '{"jsonrpc": "1.0", "id":"# Create the bitcoind wallet", "method": "createwallet", "params": {"wallet_name":"'"$rpc_wallet"'","descriptors":true,"disable_private_keys":true}}' \ + http://$rpc_user:$rpc_pass@$rpc_host:$rpc_port/ | jq . echo walletFound=$(customRPC "# Check wallet" "listwallets" 2>$connectionOutput | grep -c "$rpc_wallet") if [ $walletFound -eq 0 ]; then @@ -374,6 +415,75 @@ function checkRPCwallet { echo fi echo "# The wallet: $rpc_wallet is present and loaded in the connected bitcoind" + + # Check for wallet migration from legacy wallet.dat + checkWalletMigration +} + +# checkWalletMigration - detects legacy wallet.dat and guides user through migration +# This function checks if: +# 1. The old wallet.dat exists in Bitcoin Core +# 2. The new watch-only-descriptor-wallet is being used +# 3. Migration has not been completed yet +# If migration is needed, it prompts the user to rescan after opening their JM wallets +function checkWalletMigration { + # Skip if migration was already completed + if grep -q "walletMigrationDone=true" "${joininConfPath}" 2>/dev/null; then + return 0 + fi + + # Skip if we're not using the new descriptor wallet + if [ "$rpc_wallet" != "watch-only-descriptor-wallet" ]; then + return 0 + fi + + # RPC settings are already available from parent checkRPCwallet function + tor="" + if [ "$(echo "$rpc_host" | grep -c .onion)" -gt 0 ]; then + tor="torsocks" + fi + + # Check if old wallet.dat exists in bitcoind (try to load it to see if it exists) + # First check listwalletdir for wallet.dat + oldWalletExists=$($tor curl -sS --data-binary \ + '{"jsonrpc": "1.0", "id":"check_old_wallet", "method": "listwalletdir", "params": []}' \ + "http://$rpc_user:$rpc_pass@$rpc_host:$rpc_port/" 2>/dev/null | jq -r '.result.wallets[].name' 2>/dev/null | grep -c "^wallet.dat$") + + if [ "$oldWalletExists" -gt 0 ]; then + echo + echo "########################################################################" + echo "# WALLET MIGRATION NOTICE" + echo "########################################################################" + echo + echo "# A legacy wallet.dat was detected in Bitcoin Core." + echo "# JoininBox now uses descriptor wallets (watch-only-descriptor-wallet)" + echo "# for better compatibility with modern Bitcoin Core versions." + echo + echo "# To complete the migration and see your transaction history:" + echo + echo "# 1. Open each of your JoinMarket wallets once using:" + echo "# WALLET -> DISPLAY" + echo "# This imports the addresses into the new descriptor wallet." + echo + echo "# 2. After opening all wallets, run a blockchain rescan:" + echo "# WALLET -> RESCAN" + echo "# Use blockheight 481824 (first SegWit block) or later if you know when your wallet had its first deposit." + echo + echo "# The rescan may take several hours depending on wallet age." + echo "# You can monitor progress in the Bitcoin Core debug.log" + echo + echo "########################################################################" + echo + echo "# Press ENTER to continue..." + read -r + + # Mark migration notice as shown (user can still run rescan manually) + if ! grep -q "walletMigrationDone=" "${joininConfPath}" 2>/dev/null; then + echo "walletMigrationDone=true" >>"${joininConfPath}" + else + sed -i "s/^walletMigrationDone=.*/walletMigrationDone=true/g" "${joininConfPath}" + fi + fi } # customRPC - sends a custom RPC command @@ -435,7 +545,7 @@ function connectLocalNode() { elif [ "${network}" = testnet ]; then rpc_port="18332" fi - rpc_wallet="wallet.dat" + rpc_wallet="watch-only-descriptor-wallet" if [ $runningEnv = raspiblitz ]; then if [ -f "/mnt/hdd/raspiblitz.conf" ]; then rpc_user=$(sudo cat /mnt/hdd/bitcoin/bitcoin.conf | grep rpcuser | cut -c 9-) diff --git a/scripts/_functions.sh b/scripts/_functions.sh index c215143..318634c 100755 --- a/scripts/_functions.sh +++ b/scripts/_functions.sh @@ -282,8 +282,8 @@ function generateJMconfig() { fi sed -i "s/^rpc_port =.*/rpc_port = $RPCPORT/g" $JMcfgPath echo "# rpc_port = $RPCPORT" - sed -i "s/^rpc_wallet_file =.*/rpc_wallet_file = wallet.dat/g" $JMcfgPath - echo "# using the bitcoind wallet: wallet.dat" + sed -i "s/^rpc_wallet_file =.*/rpc_wallet_file = watch-only-descriptor-wallet/g" $JMcfgPath + echo "# using the bitcoind watch-only-descriptor-wallet" # set joinin.conf value /home/joinmarket/set.value.sh set network mainnet ${joininConfPath} fi diff --git a/scripts/install.bitcoincore.sh b/scripts/install.bitcoincore.sh index b29da0c..170f058 100755 --- a/scripts/install.bitcoincore.sh +++ b/scripts/install.bitcoincore.sh @@ -5,13 +5,13 @@ source /home/joinmarket/_functions.sh # check connectedRemoteNode var in joinin.conf if ! grep -Eq "^connectedRemoteNode=" $joininConfPath; then - echo "connectedRemoteNode=off" >> $joininConfPath + echo "connectedRemoteNode=off" >>$joininConfPath fi if [ "$1" = "signetOn" ]; then installBitcoinCore installSignet - if [ "$connectedRemoteNode" = "on" ];then + if [ "$connectedRemoteNode" = "on" ]; then backupJMconf fi generateJMconfig @@ -24,16 +24,16 @@ if [ "$1" = "signetOn" ]; then bitcoinUser="joinmarket" cliPath="/home/joinmarket/bitcoin/" fi - if [ ! -f /home/${bitcoinUser}/.bitcoin/signet/wallets/wallet.dat/wallet.dat ];then - echo "# Create wallet.dat for signet ..." + if [ ! -d /home/${bitcoinUser}/.bitcoin/signet/wallets/watch-only-descriptor-wallet ]; then + echo "# Create watch-only-descriptor-wallet for signet ..." sleep 10 - sudo -u ${bitcoinUser} ${cliPath}/bitcoin-cli -signet -named createwallet wallet_name=wallet.dat descriptors=false + sudo -u ${bitcoinUser} ${cliPath}/bitcoin-cli -signet -named createwallet wallet_name=watch-only-descriptor-wallet descriptors=true disable_private_keys=true fi elif [ "$1" = "signetOff" ]; then removeSignetdService - isSignet=$(grep -c "network = signet" < $JMcfgPath) - if [ $isSignet -gt 0 ];then + isSignet=$(grep -c "network = signet" <$JMcfgPath) + if [ $isSignet -gt 0 ]; then echo "# Removing the joinmarket.cfg with signet settings" rm -f $JMcfgPath else diff --git a/scripts/install.joinmarket.sh b/scripts/install.joinmarket.sh index a54e35f..6cb3e81 100755 --- a/scripts/install.joinmarket.sh +++ b/scripts/install.joinmarket.sh @@ -120,9 +120,8 @@ range_argument install "install" "config" "update" "testPR" "commit" : "${version:=${testedJMversion:-$testedJMcommit}}" # Only check GitHub releases if version looks like a tag (starts with 'v') if [[ "${version}" == v* ]]; then - curl -s "https://github.com/JoinMarket-Org/joinmarket-clientserver/release/tag/${version}" | grep -q "\"message\": \"Version not found\"" && error_msg "'There is no: https://github.com/JoinMarket-Org/joinmarket-clientserver/release/tag/${version}'" + curl -s "https://github.com/JoinMarket-Org/joinmarket-clientserver/releases/tag/${version}" | grep -q "\"message\": \"Version not found\"" && error_msg "'There is no: https://github.com/JoinMarket-Org/joinmarket-clientserver/releases/tag/${version}'" fi - : "${qtgui:=false}" range_argument qtgui "0" "1" "false" "true" diff --git a/scripts/menu.wallet.sh b/scripts/menu.wallet.sh index 08e919f..ea2801f 100755 --- a/scripts/menu.wallet.sh +++ b/scripts/menu.wallet.sh @@ -194,7 +194,7 @@ Enter the new gap limit to be used" 16 60 2> "$gaplimit" echo /home/joinmarket/start.script.sh wallet-tool "$(cat $wallet)"|grep mixdepth|sed -n '1~2p'|awk '{print $3}' echo - echo "Import the master public keys to Specter Desktop or Electrum to create watch only wallets." + echo "Import the master public keys to Specter Desktop or Electrum to create watch-only wallets." echo echo "Press ENTER to return to the menu..." read key diff --git a/scripts/standalone/_functions.standalone.sh b/scripts/standalone/_functions.standalone.sh index a4108cb..d03ed66 100755 --- a/scripts/standalone/_functions.standalone.sh +++ b/scripts/standalone/_functions.standalone.sh @@ -290,9 +290,9 @@ WantedBy=multi-user.target echo "# Monitor the bitcoind with: sudo tail -f /home/bitcoin/.bitcoin/mainnet/debug.log" echo - if [ ! -f /home/bitcoin/.bitcoin/mainnet/wallets/wallet.dat/wallet.dat ]; then - echo "# Create wallet.dat ..." + if [ ! -d /home/bitcoin/.bitcoin/wallets/watch-only-descriptor-wallet ]; then + echo "# Create watch-only-descriptor-wallet ..." sleep 10 - sudo -u bitcoin /usr/local/bin/bitcoin-cli -named createwallet wallet_name=wallet.dat descriptors=false + sudo -u bitcoin /usr/local/bin/bitcoin-cli -named createwallet wallet_name=watch-only-descriptor-wallet descriptors=true disable_private_keys=true fi } diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..7196ac1 --- /dev/null +++ b/test/README.md @@ -0,0 +1,34 @@ +# JoininBox integration tests + +Run the local Bats suite with: + +```bash +test/run-bats-local.sh +``` + +The descriptor wallet tests require: + +- `bats` +- `bitcoind` +- `bitcoin-cli` +- `curl` +- `jq` + +The suite starts its own temporary `bitcoind -regtest` datadir and does not use +mainnet, signet, or any existing Bitcoin Core state. + +The `amd64-image-test` workflow downloads a previously built +`joininbox-amd64-image-*` artifact, verifies the compressed and raw checksums, +decompresses a runner-local qcow2 copy, boots it with QEMU in snapshot mode, +copies a pinned `bats-core` checkout into that temporary VM session, and runs +the same suite from the JoininBox checkout inside the image. This avoids +depending on the guest's configured APT repositories just to install test +tooling. + +The test workflow has two entry points: + +- `workflow_run`: runs after a successful `amd64-image-build` once this workflow + exists on the repository default branch. +- `workflow_dispatch`: reruns against a specific build artifact by providing the + `amd64-image-build` workflow run ID, as long as the artifact is still retained + by GitHub Actions. diff --git a/test/bats/descriptor-wallet.bats b/test/bats/descriptor-wallet.bats new file mode 100644 index 0000000..8cb1d9b --- /dev/null +++ b/test/bats/descriptor-wallet.bats @@ -0,0 +1,214 @@ +#!/usr/bin/env bats + +root_dir="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" +PATH="/home/joinmarket/bitcoin:/usr/local/bin:$PATH" + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + skip "$1 is required" + fi +} + +setup() { + require_command bitcoind + require_command bitcoin-cli + require_command curl + require_command jq + + rpc_user="joininbox" + rpc_pass="joininbox" + rpc_port="$((20000 + (RANDOM % 20000)))" + p2p_port="$((40000 + (RANDOM % 20000)))" + bitcoin_datadir="${BATS_TEST_TMPDIR}/bitcoin" + joinmarket_cfg="${BATS_TEST_TMPDIR}/joinmarket.cfg" + joinin_conf="${BATS_TEST_TMPDIR}/joinin.conf" + + mkdir -p "$bitcoin_datadir" + + bitcoind \ + -regtest \ + -datadir="$bitcoin_datadir" \ + -server \ + -daemonwait \ + -rpcuser="$rpc_user" \ + -rpcpassword="$rpc_pass" \ + -rpcport="$rpc_port" \ + -port="$p2p_port" \ + -fallbackfee=0.0001 + + cat >"$joinmarket_cfg" <"$joinin_conf" +} + +teardown() { + if [ -n "${bitcoin_datadir:-}" ] && [ -d "$bitcoin_datadir" ]; then + bitcoin-cli \ + -regtest \ + -datadir="$bitcoin_datadir" \ + -rpcuser="$rpc_user" \ + -rpcpassword="$rpc_pass" \ + -rpcport="$rpc_port" \ + stop >/dev/null 2>&1 || true + fi +} + +load_joininbox_bitcoin_functions() { + # shellcheck source=scripts/_functions.bitcoincore.sh + # shellcheck disable=SC1091 + source "$root_dir/scripts/_functions.bitcoincore.sh" + # shellcheck disable=SC2034 + JMcfgPath="$joinmarket_cfg" + # shellcheck disable=SC2034 + joininConfPath="$joinin_conf" + + mktemp() { + if [ "${1:-}" = "-p" ] && [ "${2:-}" = "/dev/shm/" ]; then + command mktemp "${BATS_TEST_TMPDIR}/joininbox.XXXXXX" + else + command mktemp "$@" + fi + } +} + +wallet_info() { + bitcoin-cli \ + -regtest \ + -datadir="$bitcoin_datadir" \ + -rpcuser="$rpc_user" \ + -rpcpassword="$rpc_pass" \ + -rpcport="$rpc_port" \ + -rpcwallet=watch-only-descriptor-wallet \ + getwalletinfo +} + +check_wallet_migration_with_enter() { + printf "\n" | checkWalletMigration +} + +check_rpc_wallet_with_enter() { + printf "\n" | checkRPCwallet +} + +@test "checkRPCwallet creates the configured descriptor watch-only wallet" { + load_joininbox_bitcoin_functions + + run checkRPCwallet + + [ "$status" -eq 0 ] + [[ "$output" == *"The wallet: watch-only-descriptor-wallet is present and loaded"* ]] + + run wallet_info + [ "$status" -eq 0 ] + [ "$(jq -r '.descriptors' <<<"$output")" = "true" ] + [ "$(jq -r '.private_keys_enabled' <<<"$output")" = "false" ] + run grep -q "walletMigrationDone" "$joinin_conf" + [ "$status" -ne 0 ] +} + +@test "customRPC uses the descriptor wallet RPC endpoint" { + bitcoin-cli \ + -regtest \ + -datadir="$bitcoin_datadir" \ + -rpcuser="$rpc_user" \ + -rpcpassword="$rpc_pass" \ + -rpcport="$rpc_port" \ + -named createwallet \ + wallet_name=watch-only-descriptor-wallet \ + descriptors=true \ + disable_private_keys=true >/dev/null + + load_joininbox_bitcoin_functions + + run customRPC "# Wallet info" "getwalletinfo" "" + + [ "$status" -eq 0 ] + [[ "$output" == *'"walletname": "watch-only-descriptor-wallet"'* ]] + [[ "$output" == *'"descriptors": true'* ]] + [[ "$output" == *'"private_keys_enabled": false'* ]] +} + +@test "checkRPCwallet migrates a persisted wallet.dat configuration on Bitcoin Core v30 or later" { + bitcoin-cli \ + -regtest \ + -datadir="$bitcoin_datadir" \ + -rpcuser="$rpc_user" \ + -rpcpassword="$rpc_pass" \ + -rpcport="$rpc_port" \ + -named createwallet \ + wallet_name=wallet.dat \ + descriptors=true \ + disable_private_keys=true >/dev/null + sed \ + "s/^rpc_wallet_file =.*/rpc_wallet_file = wallet.dat/" \ + "$joinmarket_cfg" >"${joinmarket_cfg}.legacy" + mv "${joinmarket_cfg}.legacy" "$joinmarket_cfg" + + load_joininbox_bitcoin_functions + + run check_rpc_wallet_with_enter + + [ "$status" -eq 0 ] + [[ "$output" == *"Migrating the configured Bitcoin Core wallet"* ]] + [[ "$output" == *"WALLET MIGRATION NOTICE"* ]] + grep -q "^rpc_wallet_file = watch-only-descriptor-wallet$" "$joinmarket_cfg" + grep -q "^walletMigrationDone=true$" "$joinin_conf" + + run wallet_info + [ "$status" -eq 0 ] + [ "$(jq -r '.descriptors' <<<"$output")" = "true" ] + [ "$(jq -r '.private_keys_enabled' <<<"$output")" = "false" ] +} + +@test "migrateLegacyRPCWalletConfig keeps wallet.dat on Bitcoin Core v29.2" { + sed \ + "s/^rpc_wallet_file =.*/rpc_wallet_file = wallet.dat/" \ + "$joinmarket_cfg" >"${joinmarket_cfg}.legacy" + mv "${joinmarket_cfg}.legacy" "$joinmarket_cfg" + + load_joininbox_bitcoin_functions + getConnectedBitcoinCoreVersion() { + echo 290200 + } + getRPC >/dev/null + + run migrateLegacyRPCWalletConfig + + [ "$status" -eq 0 ] + [[ "$output" == *"v29.x or earlier; keeping wallet.dat"* ]] + grep -q "^rpc_wallet_file = wallet.dat$" "$joinmarket_cfg" +} + +@test "checkWalletMigration shows the notice once when wallet.dat exists" { + bitcoin-cli \ + -regtest \ + -datadir="$bitcoin_datadir" \ + -rpcuser="$rpc_user" \ + -rpcpassword="$rpc_pass" \ + -rpcport="$rpc_port" \ + -named createwallet \ + wallet_name=wallet.dat \ + descriptors=true \ + disable_private_keys=true >/dev/null + + load_joininbox_bitcoin_functions + # shellcheck disable=SC2034 + rpc_host="127.0.0.1" + # shellcheck disable=SC2034 + rpc_wallet="watch-only-descriptor-wallet" + + run check_wallet_migration_with_enter + + [ "$status" -eq 0 ] + [[ "$output" == *"WALLET MIGRATION NOTICE"* ]] + grep -q "^walletMigrationDone=true$" "$joinin_conf" + + run checkWalletMigration + [ "$status" -eq 0 ] + [ "$output" = "" ] +} diff --git a/test/run-bats-local.sh b/test/run-bats-local.sh new file mode 100755 index 0000000..ee7320b --- /dev/null +++ b/test/run-bats-local.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +if ! command -v bats >/dev/null 2>&1; then + echo "bats is required. Install bats-core, then rerun this script." >&2 + exit 127 +fi + +bats test/bats