Update documentation for the upcoming release

Bumps [log](https://github.com/rust-lang/log) from 0.4.29 to 0.4.33.
- [Release notes](https://github.com/rust-lang/log/releases)
- [Changelog](https://github.com/rust-lang/log/blob/master/CHANGELOG.md)
- [Commits](https://github.com/rust-lang/log/compare/0.4.29...0.4.33)

---
updated-dependencies:
- dependency-name: log
  dependency-version: 0.4.33
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/workflows/rust.yml

@@ -16,7 +16,7 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest]
+        os: [ubuntu-latest]
         build-args:
           [
             --locked --no-default-features,
@@ -27,7 +27,7 @@ jobs:
             build-args: --locked --features metrics_process
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v6
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: rustfmt, clippy
@@ -35,6 +35,9 @@ jobs:
       - name: Install Rust
         run: rustup component add rustfmt clippy
 
+      - name: Install other dependencies
+        run: sudo apt install build-essential libclang-dev
+
       - name: Format
         run: cargo fmt --all -- --check
 
@@ -52,7 +55,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
       - name: Build
         run: docker build -f Dockerfile.ci . --rm -t electrs:tests
       - name: Test

CONTRIBUTING.md

@@ -27,13 +27,13 @@ The codebase is maintained using the "contributor workflow" where everyone
 without exception contributes patch proposals using "pull requests". This
 facilitates social contribution, easy testing and peer review.
 
-To contribute a patch, the workflow is a as follows:
+To contribute a patch, the workflow is as follows:
 
 1. Fork repository
 2. Create topic branch
 3. Commit patches
 
-Please keep commits should atomic and diffs easy to read. For this reason
+Please keep commits atomic and diffs easy to read. For this reason
 do not mix any formatting fixes or code moves with actual code changes.
 Further, each commit, individually, should compile and pass tests, in order to
 ensure git bisect and other automated tools function properly.
@@ -84,8 +84,8 @@ grammar fixes.
 
 ### Formatting
 
-The repository currently does use `rustfmt` for all the formatting needs. Running the automated
+The repository currently uses `rustfmt` for all the formatting needs. Running the automated
 script mentioned above would format all your code for you :)
 
 ## Ending Notes
-Get cracking, have fun, and do ask for help when in doubt!
+Get cracking, have fun, and do ask for help when in doubt!

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "electrs"
-version = "0.10.8"
+version = "0.11.1"
 authors = ["Roman Zeyde <me@romanzey.de>"]
 description = "An efficient re-implementation of Electrum Server in Rust"
 license = "MIT"
@@ -10,7 +10,7 @@ keywords = ["bitcoin", "electrum", "server", "index", "database"]
 documentation = "https://docs.rs/electrs/"
 readme = "README.md"
 edition = "2021"
-rust-version = "1.63.0"
+rust-version = "1.85.0"
 build = "build.rs"
 
 [features]
@@ -23,35 +23,25 @@ spec = "internal/config_specification.toml"
 
 [dependencies]
 anyhow = "1.0"
-bitcoin = { version = "0.32.5", features = ["serde", "rand-std"] }
-bitcoin_slices = { version = "0.9", features = ["bitcoin", "sha2"] }
-bitcoincore-rpc = { version = "0.19.0" }
+bindex = { version = "0.1.2", default-features = false, features = ["bitcoin_serde"] }
 configure_me = "0.4"
 crossbeam-channel = "0.5"
 dirs-next = "2.0"
 env_logger = "0.10"
+jsonrpc = { version = "0.20.1", default-features = false, features = ["simple_http"]}
 log = "0.4"
-parking_lot = "0.12"
-prometheus = { version = "0.13", optional = true }
-rayon = "1.9"
-serde = "1.0"
-serde_derive = "1.0, <=1.0.171"  # avoid precompiled binaries (https://github.com/serde-rs/serde/issues/2538)
+prometheus = { version = "0.14", optional = true }
+rayon = "1.12"
+serde = "1.0.184"
+serde_derive = "1.0.184"
 serde_json = "1.0"
 tiny_http = { version = "0.12", optional = true }
 
 [target.'cfg(windows)'.dependencies]
-ctrlc = "=3.4.2"
+ctrlc = "=3.5.2"
 
 [target.'cfg(not(windows))'.dependencies]
-signal-hook = "0.3"
-
-[dependencies.electrs-rocksdb]
-version = "0.19.0-e3"
-
-default-features = false
-# ZSTD is used for data compression
-# Snappy is only for checking old DB
-features = ["zstd", "snappy"]
+signal-hook = "0.4"
 
 [build-dependencies]
 configure_me_codegen = { version = "0.4.8", default-features = false }
@@ -59,7 +49,4 @@ configure_me_codegen = { version = "0.4.8", default-features = false }
 [dev-dependencies]
 bitcoin-test-data = "0.2.0"
 hex_lit = "0.1.1"
-tempfile = "3.14"
-
-[profile.release]
-lto = true
+tempfile = "3.25"

Dockerfile

@@ -2,13 +2,13 @@
 # The maintainers of electrs are not deeply familiar with Docker, so you should DYOR.
 # If you are not familiar with Docker either it's probably be safer to NOT use it.
 
-FROM debian:bookworm-slim AS base
+FROM debian:trixie-slim AS base
 RUN apt-get update -qqy
 RUN apt-get install -qqy librocksdb-dev curl
 
 ### Electrum Rust Server ###
 FROM base AS electrs-build
-RUN apt-get install -qqy cargo clang cmake
+RUN apt-get install -qqy cargo build-essential libclang-dev
 
 # Install electrs
 WORKDIR /build/electrs

Dockerfile.ci

@@ -2,13 +2,13 @@
 # The maintainers of electrs are not deeply familiar with Docker, so you should DYOR.
 # If you are not familiar with Docker either it's probably be safer to NOT use it.
 
-FROM debian:bookworm-slim as base
+FROM debian:trixie-slim AS base
 RUN apt-get update -qqy
-RUN apt-get install -qqy librocksdb-dev wget
+RUN apt-get install -qqy librocksdb-dev libsqlite3-dev libevent-dev libboost-dev wget
 
 ### Electrum Rust Server ###
-FROM base as electrs-build
-RUN apt-get install -qqy cargo clang cmake
+FROM base AS electrs-build
+RUN apt-get install -qqy cargo build-essential libclang-dev
 
 # Install electrs
 WORKDIR /build/electrs
@@ -17,18 +17,17 @@ ENV ROCKSDB_INCLUDE_DIR=/usr/include
 ENV ROCKSDB_LIB_DIR=/usr/lib
 RUN cargo install --locked --path .
 
-### Bitcoin Core ###
-FROM base as bitcoin-build
+### Bitcoin Core (custom build for /blockpart/ REST endpoint) ###
+FROM base AS bitcoin-build
 # Download
 WORKDIR /build/bitcoin
-ARG ARCH=x86_64
-ARG BITCOIND_VERSION=28.0
-RUN wget -q https://bitcoincore.org/bin/bitcoin-core-$BITCOIND_VERSION/bitcoin-$BITCOIND_VERSION-$ARCH-linux-gnu.tar.gz
-RUN tar xvf bitcoin-$BITCOIND_VERSION-$ARCH-linux-gnu.tar.gz
-RUN mv -v bitcoin-$BITCOIND_VERSION/bin/bitcoind .
-RUN mv -v bitcoin-$BITCOIND_VERSION/bin/bitcoin-cli .
 
-FROM base as result
+RUN wget -q https://bitcoincore.org/bin/bitcoin-core-31.1/bitcoin-31.1-x86_64-linux-gnu.tar.gz
+RUN tar xvf bitcoin-31.1-x86_64-linux-gnu.tar.gz
+RUN mv -v bitcoin-31.1/bin/bitcoind .
+RUN mv -v bitcoin-31.1/bin/bitcoin-cli .
+
+FROM base AS result
 # Copy the binaries
 COPY --from=electrs-build /root/.cargo/bin/electrs /usr/bin/electrs
 COPY --from=bitcoin-build /build/bitcoin/bitcoind /build/bitcoin/bitcoin-cli /usr/bin/
@@ -37,7 +36,7 @@ RUN bitcoind -version && bitcoin-cli -version
 ### Electrum ###
 # Clone latest Electrum wallet and a few test tools
 WORKDIR /build/
-RUN apt-get install -qqy git libsecp256k1-1 python3-cryptography python3-setuptools python3-venv python3-pip jq curl
+RUN apt-get install -qqy git libsecp256k1-2 python3-cryptography python3-setuptools python3-venv python3-pip jq curl
 RUN git clone --recurse-submodules https://github.com/spesmilo/electrum/ && cd electrum/ && git log -1
 RUN python3 -m venv --system-site-packages venv && \
     ELECTRUM_ECC_DONT_COMPILE=1 venv/bin/pip install -e electrum/ && \

README.md

@@ -3,7 +3,7 @@
 # Electrum Server in Rust
 
 [![CI](https://github.com/romanz/electrs/actions/workflows/rust.yml/badge.svg)](https://github.com/romanz/electrs/actions)
-[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com)
+[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](https://github.com/romanz/electrs/compare)
 [![crates.io](https://img.shields.io/crates/v/electrs.svg)](https://crates.io/crates/electrs)
 [![gitter.im](https://badges.gitter.im/romanz/electrs.svg)](https://gitter.im/romanz/electrs)
 
@@ -41,13 +41,13 @@ which is better optimized for public usage at the cost of consuming *significant
 
 ## Features
 
- * Supports Electrum protocol [v1.4](https://electrumx-spesmilo.readthedocs.io/en/latest/protocol.html)
+ * Supports Electrum protocol [v1.4](https://electrum-protocol.readthedocs.io/)
  * Maintains an index over transaction inputs and outputs, allowing fast balance queries
  * Fast synchronization of the Bitcoin blockchain (~6.5 hours for ~504GB @ August 2023) using HDD storage.
  * Low index storage overhead (~10%), relying on a local full node for transaction retrieval
  * Efficient mempool tracker (allowing better fee [estimation](https://github.com/spesmilo/electrum/blob/59c1d03f018026ac301c4e74facfc64da8ae4708/RELEASE-NOTES#L34-L46))
  * Low CPU & memory usage (after initial indexing)
- * [`txindex`](https://github.com/bitcoinbook/bitcoinbook/blob/develop/ch03.asciidoc#txindex) is not required for the Bitcoin node
+ * [`txindex`](https://github.com/bitcoinbook/bitcoinbook/blob/develop/ch03_bitcoin-core.adoc#txindex) is not required for the Bitcoin node
  * Uses a single [RocksDB](https://github.com/spacejam/rust-rocksdb) database, for better consistency and crash recovery
 
 ## Altcoins
@@ -69,4 +69,4 @@ All contributions to this project are welcome. Please refer to the [Contributing
 
 [Our logo](logo/) is generously provided by [Dominik Průša](https://github.com/DominoPrusa) under the MIT license.
 Based on the [Electrum logo](https://github.com/spesmilo/electrum/blob/master/LICENCE)
-and the [Rust language logo](https://www.rust-lang.org/policies/media-guide).
+and the [Rust language logo](https://rustfoundation.org/policy/rust-trademark-policy).

RELEASE-NOTES.md

@@ -1,3 +1,29 @@
+# 0.11.1 (Feb 22 2026)
+
+* Update `transaction.id_from_pos` response for API compliance.
+* Expose configurable magic on non-signet.
+* Update dependencies (`bitcoin`, `ctrlc`, `log`, `serde_json`, `signal-hook`, `tempfile`).
+
+# 0.11.0 (Nov 16 2025)
+
+* Update MSRV to 1.85
+* Support Debian 13 (trixie) / Ubuntu 25.04 (plucky)
+* Update RocksDB crate to `rust-rocksdb` 0.36
+  - [Dynamic linking with RocksDB](doc/install.md) now requires [RocksDB 9.10.0](https://packages.debian.org/trixie/librocksdb-dev) (instead of 7.8.3).
+  - Old electrs versions won't be able to read the new RocksDB database format (requiring reindex).
+* Support `blockchain.transaction.broadcast_package` Electrum RPC.
+* Update dependencies (`bitcoin`, `ctrlc`, `log`, `prometheus`, `rayon`, `serde`, `serde_json`, `tempfile`, `tempfile`).
+
+# 0.10.10 (Jul 19 2025)
+
+* Update documentation links and typos.
+* Update dependencies (`bitcoin`, `crossbeam-channel`, `log`, `prometheus`, `signal-hook`, `tempfile`).
+
+# 0.10.9 (Feb 01 2025)
+
+* Update `bitcoin_slices` to 0.10.0
+* Allow concurrent DB background operations for SSDs
+
 # 0.10.8 (Dec 30 2024)
 
 *  Separate blocks reading & index writing into scoped threads
@@ -7,7 +33,6 @@
 *  Run CI only on `master` branch and PRs
 *  Update dependencies (`bitcoin`, `bitcoin_slices`, `crossbeam-channel`, `tempfile`)
 
-
 # 0.10.7 (Nov 05 2024)
 
 * Support testnet4
@@ -245,7 +270,7 @@ Full list of changes:
 
 * Fix JSONRPC errors' handling (#398, #390)
 * Optimize Dockerfile (#387, #388, #392)
-* Fix signet default port (https://github.com/romanz/electrs/b53178c140e575b0527a70ead566d50c7fe6cb1f)
+* Fix signet default port (https://github.com/romanz/electrs/commit/b53178c140e575b0527a70ead566d50c7fe6cb1f)
 
 # 0.8.9 (19 Mar 2021)
 

TODO.md

@@ -5,7 +5,7 @@
 
 # Rust
 
-* Use [bytes](https://carllerche.github.io/bytes/bytes/index.html) instead of `Vec<u8>` when possible
+* Use [bytes](https://docs.rs/bytes/latest/bytes/) instead of `Vec<u8>` when possible
 * Use generators instead of vectors
 * Use proper HTTP parser for JSONRPC replies over persistent connection
 

contrib/history.py

@@ -42,6 +42,12 @@ def main():
     elif args.network == 'testnet':
         port = 60001
         from pycoin.symbols.xtn import network
+    elif args.network == 'testnet4':
+        port = 40001
+        from pycoin.symbols.xtn import network
+    elif args.network == 'signet':
+        port = 60601
+        from pycoin.symbols.xtn import network
     elif args.network == 'mainnet':
         port = 50001
         from pycoin.symbols.btc import network

doc/config.md

@@ -5,11 +5,12 @@ If you use automated systems, refer to their documentation first!
 
 ### Bitcoind configuration
 
-Pruning must be turned **off** for `electrs` to work.
+REST API must be **enabled** for block indexing and transaction lookup.
+
+Pruning must be **disabled** for `electrs` to work.
+
 `txindex` is allowed but unnecessary for `electrs`.
 However, you might still need it if you run other services (e.g.`eclair`).
-The option `maxconnections` (if used) should be set to 12 or more for bitcoind to accept inbound p2p connections.
-Note that setting `maxuploadtarget` may cause p2p-based sync to fail - so consider using `-whitelist=download@127.0.0.1` to disable the limit for local p2p connections.
 
 The highly recommended way of authenticating `electrs` is using cookie file.
 It's the most [secure](https://github.com/Kixunil/security_writings/blob/master/cookie_files.md) and robust method.
@@ -21,7 +22,7 @@ You can skip it if you're running both daemons under the same user and with the
 Example command for running `bitcoind` (assuming same user, default dirs):
 
 ```bash
-$ bitcoind -server=1 -txindex=0 -prune=0
+$ bitcoind -server=1 -rest=1 -txindex=0 -prune=0
 ```
 ### Electrs configuration
 
@@ -139,7 +140,7 @@ HiddenServiceVersion 3
 HiddenServicePort 50001 127.0.0.1:50001
 ```
 
-If you use [the *beta* Debian repository](binaries.md#cnative-os-packages),
+If you use [the *beta* Debian repository](binaries.md#native-os-packages),
 it is cleaner to install `tor-hs-patch-config` using `apt` and then placing the configuration into a file inside `/etc/tor/hidden-services.d`.
 
 Restart the service:
@@ -162,7 +163,7 @@ For more details, see http://docs.electrum.org/en/latest/tor.html.
 
 ### Sample Systemd Unit File
 
-If you use [the *beta* Debian repository](binaries.md#cnative-os-packages), you should skip this section,
+If you use [the *beta* Debian repository](binaries.md#native-os-packages), you should skip this section,
 as the appropriate systemd unit file is installed automatically.
 
 You may wish to have systemd manage electrs so that it's "always on".

doc/config_example.toml

@@ -14,9 +14,6 @@ cookie_file = "/var/run/bitcoin-mainnet/cookie"
 # The listening RPC address of bitcoind, port is usually 8332
 daemon_rpc_addr = "127.0.0.1:8332"
 
-# The listening P2P address of bitcoind, port is usually 8333
-daemon_p2p_addr = "127.0.0.1:8333"
-
 # Directory where the index should be stored. It should have at least 70GB of free space.
 db_dir = "/some/fast/storage/with/big/size"
 

doc/cookie_deprecation.md

@@ -21,7 +21,7 @@ The option was confusing:
 
 If you're installing `electrs` for the first time, just don't use `cookie`.
 If you're updating, reconsider the motivation above.
-If you used copying script, just use `cookie_file` to get the cookie directly.
+If you used a copying script, just use `cookie_file` to get the cookie directly.
 If you also used `BindsTo`, we recommend removing it.
 If you used fixed username and password because you didn't know about cookie or did it before `cookie_file` was implemented, reconsider using cookie authentication.
 If you really have to use fixed username and password, specify them using `auth` option (`username:password` like before) and remove the `cookie` option.

doc/install.md

@@ -1,21 +1,14 @@
 ## Quickstart
 
-<details>
-<summary>Building from source on an Ubuntu 21.10 VM:</summary>
+Building from source on an Ubuntu 25.04 / Debian 13:
 
 ```bash
 $ sudo apt update
-$ sudo apt install -y clang cmake build-essential git cargo
-$ git clone https://github.com/romanz/electrs
-$ cd electrs
-$ cargo build --locked --release
-$ ./target/release/electrs --version  # should print the latest version
+$ sudo apt install -y build-essential libclang-dev git cargo
+$ git clone https://github.com/romanz/electrs && cd electrs
+$ cargo run --locked --release -- --version  # build and print the latest version
 ```
 
-</details>
-
-[![asciicast](https://asciinema.org/a/XKznxilP4O7lCZiVZ9vZNd5vx.svg)](https://asciinema.org/a/XKznxilP4O7lCZiVZ9vZNd5vx?speed=3)
-
 ## Manual installation from source
 
 **See below for automated/binary installation options.**
@@ -25,14 +18,14 @@ $ ./target/release/electrs --version  # should print the latest version
 Note for Raspberry Pi 4 owners: the old versions of OS/toolchains produce broken binaries.
 Make sure to use latest OS! (see #226)
 
-Install [recent Rust](https://rustup.rs/) (1.63.0+, `apt install cargo` is preferred for Debian 12),
-[latest Bitcoin Core](https://bitcoincore.org/en/download/) (0.21+)
+Install [recent Rust](https://rustup.rs/) (1.85.0+, `apt install cargo` is preferred for Debian 13),
+[latest Bitcoin Core](https://bitcoincore.org/en/download/) (31.0+)
 and [latest Electrum wallet](https://electrum.org/#download) (4.0+).
 
 Also, install the following packages (on Debian or Ubuntu):
 ```bash
 $ sudo apt update
-$ sudo apt install clang cmake build-essential  # for building 'rust-rocksdb'
+$ sudo apt install build-essential libclang-dev  # for building 'rust-rocksdb'
 ```
 
 There are two ways to compile `electrs`: by statically linking to `librocksdb` or dynamically linking.
@@ -52,18 +45,18 @@ The advantages of dynamic linking:
 * Cross compilation is more reliable
 * If another application is also using `rocksdb`, you don't store it on disk and in RAM twice
 
-If you decided to use dynamic linking, you will also need to install the library ([7.8.3 release](https://github.com/facebook/rocksdb/releases/tag/v7.8.3) is required).
-On [Debian 12 (bookworm)](https://packages.debian.org/bookworm/librocksdb-dev) and [Ubuntu 23.04 (lunar)](https://packages.ubuntu.com/lunar/librocksdb-dev):
+If you decided to use dynamic linking, you will also need to install the library ([9.10.0 release](https://github.com/facebook/rocksdb/releases/tag/v9.10.0) is required).
+On [Debian 13 (trixie)](https://packages.debian.org/bookworm/librocksdb-dev) and [Ubuntu 25.04 (plucky)](https://packages.ubuntu.com/plucky/librocksdb-dev):
 
 ```bash
-$ sudo apt install librocksdb-dev=7.8.3-2
+$ sudo apt install librocksdb-dev=9.10.0-1
 ```
 
 For other versions of Debian or Ubuntu, you can build librocksdb and install inside `/usr/local` directory using following command.
 
 ```bash
 $ sudo apt install -y libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
-$ git clone -b v7.8.3 --depth 1 https://github.com/facebook/rocksdb && cd rocksdb
+$ git clone -b v9.10.0 --depth 1 https://github.com/facebook/rocksdb && cd rocksdb
 $ make shared_lib -j $(nproc) && sudo make install-shared
 $ cd .. && rm -r rocksdb
 ```
@@ -149,7 +142,7 @@ sudo chroot debootstrap-buster /bin/bash
 apt install curl
 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
 source "$HOME/.cargo/env"
-apt install clang cmake build-essential
+apt install build-essential libclang-dev
 
 # install target specific cross compiler (armhf/gnueabihf)
 apt install gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf libc6-dev-armhf-cross

doc/schema.md

@@ -1,53 +1 @@
-# Index Schema
-
-The index is stored at a single RocksDB database using the following column families.
-Most of the data is stored in key-only DB rows (i.e. having empty values).
-
-## Transaction outputs' index (`funding`)
-
-Allows efficiently finding all funding transactions for a specific address:
-
-|  Script Hash Prefix  | Confirmed Block Height |
-| -------------------- | ---------------------- |
-| `SHA256(script)[:8]` | `height as u32`        |
-
-## Transaction inputs' index (`spending`)
-
-Allows efficiently finding spending transaction of a specific output:
-
-| Previous Outpoint Prefix | Confirmed Block Height |
-| ------------------------ | ---------------------- |
-| `txid[:8] as u64 + vout` | `height as u32`        |
-
-
-## Transaction ID index (`txid`)
-
-In order to save storage space, we map the 8-byte transaction ID prefix to its confirmed block height:
-
-| Txid Prefix | Confirmed height |
-| ----------- | ---------------- |
-| `txid[:8]`  | `height as u32`  |
-
-Note that this mapping allows us to use `getrawtransaction` RPC to retrieve actual transaction data from without `-txindex` enabled
-(by explicitly specifying the [blockhash](https://github.com/bitcoin/bitcoin/commit/497d0e014cc79d46531d570e74e4aeae72db602d)).
-
-## Headers (`headers`)
-
-For faster loading, we store all block headers in RocksDB:
-
-|    Serialized header    |
-| ----------------------- |
-| `header as BlockHeader` |
-
-In addition, we also store the chain tip:
-
-| Key |   |           Value          |
-| --- | - | ------------------------ |
-| `T` |   | `blockhash as BlockHash` |
-
-## Configuration (`config`)
-
-| Key |   |            Value            |
-| --- | - | --------------------------- |
-| `C` |   | `serialized config as JSON` |
-
+See https://github.com/romanz/bindex-rs for more details.

doc/upgrading.md

@@ -1,7 +1,24 @@
+### Important changes from versions older than 0.12.0
+
+In 0.12.0 we have changed the RocksDB index format to optimize electrs performance.
+
+We also use new bitcoind REST API endpoints (instead of P2P protocol)
+
+* [#32540 (index: fetch spent transaction outputs by blockhash)](https://github.com/bitcoin/bitcoin/pull/32540) → released in 30.0
+* [#33657 (rest: allow reading partial block data from storage)](https://github.com/bitcoin/bitcoin/pull/33657) → released in 31.0
+
+Upgrading checklist:
+
+* Make sure you upgrade at a time when you don't need to use electrs for a while.
+  Because of reindex electrs will be unable to serve your requests for a few hours.
+  (The exact time depends on your hardware.)
+* Make sure you have at least 120 GB of free space (for reindexing mainnet chain).
+
+
 ### Important changes from versions older than 0.9.3
 
 * If you use `verbose` (or `-v` argument), switch to `log_filters` (or `RUST_LOG` environment variable).
-  Please note that it allows to set per-module filters, but module naming is considered unstable.
+  Please note that it allows setting per-module filters, but module naming is considered unstable.
   If you have used `-vv` (the value suggested in the documentation), switch to `--log-filters INFO`:
 
 
@@ -11,7 +28,7 @@
 |WARN      |                  1|Things that could indicate serious problems                           |
 |INFO      |                  2|Various significant events and suggestions                            |
 |DEBUG     |                  3|Details that could be useful when debugging - only use when debugging!|
-|TRACE     |                  4|**Very** detailed information - only use when debugging!              |  
+|TRACE     |                  4|**Very** detailed information - only use when debugging!              |
 
 
 ### Important changes from versions older than 0.9.0
@@ -22,7 +39,7 @@ Some guides were suggesting trace log level and we started to trace much more in
 
 Upgrading checklist:
 
-* Make sure you upgrade at time when you don't need to use electrs for a while.
+* Make sure you upgrade at a time when you don't need to use electrs for a while.
   Because of reindex electrs will be unable to serve your requests for a few hours.
   (The exact time depends on your hardware.)
   If you wish to check the database without reindexing run electrs with `--no-auto-reindex`.
@@ -38,7 +55,7 @@ Upgrading checklist:
 * If you use `verbose = 4` (or `-vvvv` argument) lower it down to `2` (`-vv`) for production use.
   Keeping it would waste resources because we utilize it more now.
 * **After reindexing**, if you did **not** delete `mainnet` subdirectory within `db_dir` check that `electrs` works as expected and then *delete whole `mainnet` subdirectory*.
-* If you are using our Dockerfile, please make sure to re-map the DB volume (see [the section above](docker.md#docker-based-installation-from-source)).
+* If you are using our Dockerfile, please make sure to re-map the DB volume (see [the section above](install.md#docker-based-installation-from-source)).
 
 ### Important changes from version older than 0.8.8
 
@@ -69,8 +86,8 @@ sudo apt update
 sudo apt upgrade
 ```
 
-Similarly for other distributions - use their respective commands.  
-If a new version of `electrs` is not yet in the package system, try wait a few days or contact the maintainers of the packages if it's been a long time.
+Similarly for other distributions - use their respective commands.
+If a new version of `electrs` is not yet in the package system, try waiting a few days or contact the maintainers of the packages if it has been a long time.
 
 ### Upgrading manual installation
 
@@ -85,6 +102,6 @@ If a new version of `electrs` is not yet in the package system, try wait a few d
    If you don't remember which linking you used, you probably used static.
    This step will take a few tens of minutes (but dynamic linking is a bit faster), go grab a coffee.
    Also remember that you need enough free RAM, the build will die otherwise
-7. If you've previously copied `electrs` into `/usr/local/bin` run: sudo `cp target/release/electrs /usr/local/bin`
+7. If you've previously copied `electrs` into `/usr/local/bin` run: `sudo cp target/release/electrs /usr/local/bin`
    If you've previously installed `electrs` using `cargo install`: `cargo install --locked --path . -f`
 8. If you've manually configured systemd service: `sudo systemctl restart electrs`

doc/usage.md

@@ -1,77 +1,63 @@
 ## Quickstart
 
-<details>
-<summary>Assuming Bitcoin Core 0.21+ is installed on the same machine (with the standard configuration at `~/.bitcoin/bitcoin.conf`):</summary>
+Assuming Bitcoin Core 31+ is installed on the same machine (with the standard configuration at `~/.bitcoin/bitcoin.conf`):
 
 ```bash
-$ bitcoind -server=1 -prune=0 &
+$ bitcoind -server=1 -rest=1 -prune=0 &
 $ # ... wait until the chain is synced (e.g. using `bitcoin-cli getblockchaininfo`)
-$ electrs --log-filters=INFO --db-dir ./db --daemon-dir ~/.bitcoin --network bitcoin
+$ RUST_LOG=INFO electrs  --network bitcoin --db-dir ./db --daemon-dir ~/.bitcoin
 ```
 
-</details>
-
-[![asciicast](https://asciinema.org/a/zRNZp5HsBDi5rAlGWU7470Pzl.svg)](https://asciinema.org/a/zRNZp5HsBDi5rAlGWU7470Pzl?speed=3)
-
 ## Usage
 
-First index sync should take ~6.5 hours for ~504GB @ August 2023 (on a dual core Intel CPU @ 3.3 GHz, 8 GB RAM, 1TB WD Blue HDD):
+First index sync should take ~2 hours for ~800GB `.bitcoin/blocks/` @ July 2026 (on a 6-core CPU, 32 GB RAM, 2TB NVMe):
 ```bash
-$ du -ch ~/.bitcoin/blocks/blk*.dat | tail -n1
-336G  total
+$ du -ch ~/.bitcoin/blocks/*.dat | tail -n1
+802G  total
 
-$ ./target/release/electrs --network bitcoin --db-dir ./db --daemon-dir /home/user/.bitcoin
-Starting electrs 0.10.0 on x86_64 linux with Config { network: Bitcoin, db_path: "./db/bitcoin", daemon_dir: "/home/user/.bitcoin", daemon_auth: CookieFile("/home/user/.bitcoin/.cookie"), daemon_rpc_addr: 127.0.0.1:8332, daemon_p2p_addr: 127.0.0.1:8333, electrum_rpc_addr: 127.0.0.1:50001, monitoring_addr: 127.0.0.1:4224, wait_duration: 10s, jsonrpc_timeout: 15s, index_batch_size: 10, index_lookup_limit: None, reindex_last_blocks: 0, auto_reindex: true, ignore_mempool: false, sync_once: false, skip_block_download_wait: false, disable_electrum_rpc: false, server_banner: "Welcome to electrs 0.10.0 (Electrum Rust Server)!", signet_magic: f9beb4d9, args: [] }
-[2023-08-16T19:17:11.193Z INFO  electrs::metrics::metrics_impl] serving Prometheus metrics on 127.0.0.1:4224
-[2023-08-16T19:17:11.193Z INFO  electrs::server] serving Electrum RPC on 127.0.0.1:50001
-[2023-08-16T19:17:12.355Z INFO  electrs::db] "./db/bitcoin": 0 SST files, 0 GB, 0 Grows
-[2023-08-16T19:17:12.446Z INFO  electrs::index] indexing 2000 blocks: [1..2000]
-[2023-08-16T19:17:12.866Z INFO  electrs::chain] chain updated: tip=00000000dfd5d65c9d8561b4b8f60a63018fe3933ecb131fb37f905f87da951a, height=2000
-[2023-08-16T19:17:12.879Z INFO  electrs::index] indexing 2000 blocks: [2001..4000]
-[2023-08-16T19:17:13.227Z INFO  electrs::chain] chain updated: tip=00000000922e2aa9e84a474350a3555f49f06061fd49df50a9352f156692a842, height=4000
-[2023-08-16T19:17:13.238Z INFO  electrs::index] indexing 2000 blocks: [4001..6000]
-[2023-08-16T19:17:13.587Z INFO  electrs::chain] chain updated: tip=00000000dbbb79792303bdd1c6c4d7ab9c21bba0667213c2eca955e11230c5a5, height=6000
-[2023-08-16T19:17:13.598Z INFO  electrs::index] indexing 2000 blocks: [6001..8000]
-[2023-08-16T19:17:13.950Z INFO  electrs::chain] chain updated: tip=0000000094fbacdffec05aea9847000522a258c269ae37a74a818afb96fc27d9, height=8000
-[2023-08-16T19:17:13.961Z INFO  electrs::index] indexing 2000 blocks: [8001..10000]
-<...>
-[2023-08-17T00:13:16.443Z INFO  electrs::index] indexing 2000 blocks: [798001..800000]
-[2023-08-17T00:14:58.310Z INFO  electrs::chain] chain updated: tip=00000000000000000002a7c4c1e48d76c5a37902165a270156b7a8d72728a054, height=800000
-[2023-08-17T00:14:58.325Z INFO  electrs::index] indexing 2000 blocks: [800001..802000]
-[2023-08-17T00:16:36.425Z INFO  electrs::chain] chain updated: tip=0000000000000000000311b41f1d611f977b024b947568c1dd760704360f148a, height=802000
-[2023-08-17T00:16:36.437Z INFO  electrs::index] indexing 1534 blocks: [802001..803534]
-[2023-08-17T00:17:51.338Z INFO  electrs::chain] chain updated: tip=00000000000000000003c0cd1b62ed8bb502e24bcbfeee16e81d6ea33d026263, height=803534
-[2023-08-17T00:18:00.592Z INFO  electrs::db] starting config compaction
-[2023-08-17T00:18:00.778Z INFO  electrs::db] starting headers compaction
-[2023-08-17T00:18:00.870Z INFO  electrs::db] starting txid compaction
-[2023-08-17T00:33:34.370Z INFO  electrs::db] starting funding compaction
-[2023-08-17T01:03:56.784Z INFO  electrs::db] starting spending compaction
-[2023-08-17T01:35:05.983Z INFO  electrs::db] finished full compaction
-[2023-08-17T01:36:23.300Z INFO  electrs::index] indexing 5 blocks: [803535..803539]
-[2023-08-17T01:36:23.646Z INFO  electrs::chain] chain updated: tip=000000000000000000006a3aaddd4b643607b33e000f1200d35005c330ecfa88, height=803539
-[2023-08-17T01:41:26.009Z INFO  electrs::index] indexing 1 blocks: [803540..803540]
-[2023-08-17T01:41:26.143Z INFO  electrs::chain] chain updated: tip=00000000000000000003266d31db92629b64241eef7ce708244f6d6283b080b4, height=803540
-[2023-08-17T01:42:42.999Z INFO  electrs::index] indexing 1 blocks: [803541..803541]
-[2023-08-17T01:42:43.153Z INFO  electrs::chain] chain updated: tip=00000000000000000000884a77c8b8ad2fb0c25510a3251bf5ef57f0db275146, height=803541
+$ RUST_LOG=INFO electrs --network bitcoin --db-dir ./db --daemon-dir ~/.bitcoin
+Starting electrs 0.12.0 on x86_64 linux with Config { network: Bitcoin, db_dir: "./db", ... }
+[2026-07-18T16:37:24.024Z INFO  electrs::metrics::metrics_impl] serving Prometheus metrics on 127.0.0.1:4224
+[2026-07-18T16:37:24.025Z INFO  electrs::server] serving Electrum RPC on 127.0.0.1:50001
+[2026-07-18T16:37:24.025Z INFO  bindex::chain] index: Config { db_path: "./db/bitcoin", url: "http://127.0.0.1:8332" }
+[2026-07-18T16:37:24.054Z INFO  bindex::db] CF config: 0 files, 0.000000 MBs
+[2026-07-18T16:37:24.054Z INFO  bindex::db] CF headers: 0 files, 0.000000 MBs
+[2026-07-18T16:37:24.054Z INFO  bindex::db] CF txpos: 0 files, 0.000000 MBs
+[2026-07-18T16:37:24.054Z INFO  bindex::db] CF txid: 0 files, 0.000000 MBs
+[2026-07-18T16:37:24.054Z INFO  bindex::db] CF script_hash: 0 files, 0.000000 MBs
+[2026-07-18T16:37:24.054Z INFO  bindex::db] CF spending: 0 files, 0.000000 MBs
+[2026-07-18T16:37:24.054Z INFO  bindex::db] CF funding: 0 files, 0.000000 MBs
+[2026-07-18T16:37:24.129Z INFO  bindex::chain] block=00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09 height=1000: indexed 1001 blocks, 0.226[MB], dt = 0.047[s]: 0.047 [ms/block], 0.000 [MB/block], 4.837 [MB/s]
+[2026-07-18T16:37:24.196Z INFO  bindex::chain] block=00000000dfd5d65c9d8561b4b8f60a63018fe3933ecb131fb37f905f87da951a height=2000: indexed 1000 blocks, 0.234[MB], dt = 0.043[s]: 0.043 [ms/block], 0.000 [MB/block], 5.376 [MB/s]
+[2026-07-18T16:37:24.263Z INFO  bindex::chain] block=000000004a81b9aa469b11649996ecb0a452c16d1181e72f9f980850a1c5ecce height=3000: indexed 1000 blocks, 0.230[MB], dt = 0.043[s]: 0.043 [ms/block], 0.000 [MB/block], 5.298 [MB/s]
+[2026-07-18T16:37:24.331Z INFO  bindex::chain] block=00000000922e2aa9e84a474350a3555f49f06061fd49df50a9352f156692a842 height=4000: indexed 1000 blocks, 0.234[MB], dt = 0.044[s]: 0.044 [ms/block], 0.000 [MB/block], 5.366 [MB/s]
+[2026-07-18T16:37:24.398Z INFO  bindex::chain] block=000000004d78d2a8a93a1d20a24d721268690bebd2b51f7e80657d57e226eef9 height=5000: indexed 1000 blocks, 0.229[MB], dt = 0.043[s]: 0.043 [ms/block], 0.000 [MB/block], 5.349 [MB/s]
+...
+[2026-07-18T18:02:21.114Z INFO  bindex::chain] block=0000000000000000000055222909c19ed96cd371013861337214a3c39a63d828 height=955000: indexed 1000 blocks, 1842.985[MB], dt = 9.457[s]: 9.457 [ms/block], 1.843 [MB/block], 194.886 [MB/s]
+[2026-07-18T18:02:33.592Z INFO  bindex::chain] block=000000000000000000012d9ea47c8b3282ae1a7792d54b8b4bcc0536c1883191 height=956000: indexed 1000 blocks, 1846.173[MB], dt = 9.522[s]: 9.522 [ms/block], 1.846 [MB/block], 193.879 [MB/s]
+[2026-07-18T18:02:45.872Z INFO  bindex::chain] block=00000000000000000000e898fc9d01b0729d8830f2068248debbff575a3b0990 height=957000: indexed 1000 blocks, 1855.303[MB], dt = 9.295[s]: 9.295 [ms/block], 1.855 [MB/block], 199.600 [MB/s]
+[2026-07-18T18:02:58.221Z INFO  bindex::chain] block=000000000000000000022c6a6d538a4372b0eeadb4ea9eb2edd4e32ec780084b height=958000: indexed 1000 blocks, 1844.747[MB], dt = 9.376[s]: 9.376 [ms/block], 1.845 [MB/block], 196.759 [MB/s]
+[2026-07-18T18:03:05.539Z INFO  bindex::chain] block=000000000000000000020934afa0945d959250e9a4598d39c53fa280639b096a height=958594: indexed 594 blocks, 1115.055[MB], dt = 5.357[s]: 9.018 [ms/block], 1.877 [MB/block], 208.155 [MB/s]
+[2026-07-18T18:03:05.666Z INFO  bindex::db] started auto compactions
 ```
 You can specify options via command-line parameters, environment variables or using config files.
 See the documentation above.
 
-Note that the final DB size should be ~10% of the `blk*.dat` files, but it may increase to ~20% at the end of the initial sync (just before the [full compaction is invoked](https://github.com/facebook/rocksdb/wiki/Manual-Compaction)).
+Note that the final DB size should be ~7% of the `blocks/*.dat` files, but it may increase to ~14% at the end of the initial sync (just before the [full compaction is invoked](https://github.com/facebook/rocksdb/wiki/Manual-Compaction)).
 
 It should take roughly 18 hours to sync and compact the index on an ODROID-HC1 with 8 CPU cores @ 2GHz, 2GB RAM, and an SSD using the command above.
 
 The index database is stored here:
 ```bash
-$ du db/
-42G db/mainnet/
+$ du -h db/bitcoin/
+56G	db/bitcoin/
 ```
 
 See [extra configuration suggestions](config.md#extra-configuration-suggestions) that you might want to consider.
 
 ## Electrum client
 
-If you happen to use the Electrum client from [the *beta* Debian repository](binaries.md#cnative-os-packages), it's pre-configured out-of-the-box already
+If you happen to use the Electrum client from [the *beta* Debian repository](binaries.md#native-os-packages), it's pre-configured out-of-the-box already
 Read below otherwise.
 
 There's a prepared script for launching `electrum` in such way to connect only to the local `electrs` instance to protect your privacy.
@@ -98,11 +84,11 @@ You can invoke any supported RPC using `netcat`, for example:
 
 ```
 $ echo '{"jsonrpc": "2.0", "method": "server.version", "params": ["", "1.4"], "id": 0}' | netcat 127.0.0.1 50001
-{"id":0,"jsonrpc":"2.0","result":["electrs 0.9.0","1.4"]}
+{"id":0,"jsonrpc":"2.0","result":["electrs 0.12.0","1.4"]}
 ```
 
-For more complex tasks, you may need to convert addresses to 
-[script hashes](https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-basics.html#script-hashes) - see 
+For more complex tasks, you may need to convert addresses to
+[script hashes](https://electrum-protocol.readthedocs.io/en/latest/protocol-basics.html#script-hashes) - see
 [contrib/history.py](https://github.com/romanz/electrs/blob/master/contrib/history.py) for getting an address balance and history:
 
 ```

examples/tx_collisions.rs

@@ -1,31 +0,0 @@
-use anyhow::{Context, Result};
-use electrs_rocksdb::{ColumnFamilyDescriptor, IteratorMode, Options, DB};
-
-fn main() -> Result<()> {
-    let path = std::env::args().nth(1).context("missing DB path")?;
-    let cf_names = DB::list_cf(&Options::default(), &path)?;
-    let cfs: Vec<_> = cf_names
-        .iter()
-        .map(|name| ColumnFamilyDescriptor::new(name, Options::default()))
-        .collect();
-    let db = DB::open_cf_descriptors(&Options::default(), &path, cfs)?;
-    let cf = db.cf_handle("txid").context("missing column family")?;
-
-    let mut state: Option<(u64, u32)> = None;
-    for row in db.iterator_cf(cf, IteratorMode::Start) {
-        let (curr, _value) = row?;
-        let curr_prefix = u64::from_le_bytes(curr[..8].try_into()?);
-        let curr_height = u32::from_le_bytes(curr[8..].try_into()?);
-
-        if let Some((prev_prefix, prev_height)) = state {
-            if prev_prefix == curr_prefix {
-                eprintln!(
-                    "prefix={:x} heights: {} {}",
-                    curr_prefix, prev_height, curr_height
-                );
-            };
-        }
-        state = Some((curr_prefix, curr_height));
-    }
-    Ok(())
-}

internal/config_specification.toml

@@ -34,6 +34,12 @@ name = "db_log_dir"
 type = "std::path::PathBuf"
 doc = "Directory to store index database internal log (default: same as specified by `db_dir`)"
 
+[[param]]
+name = "db_parallelism"
+type = "u8"
+doc = "Max threads to use for DB background operations (flushes and compactions)"
+default = "1"
+
 [[param]]
 name = "daemon_dir"
 type = "std::path::PathBuf"
@@ -57,7 +63,7 @@ doc = "JSONRPC authentication cookie file (default: ~/.bitcoin/.cookie)"
 [[param]]
 name = "network"
 type = "crate::config::BitcoinNetwork"
-convert_into = "::bitcoin::Network"
+convert_into = "crate::bitcoin::Network"
 doc = "Select Bitcoin network type ('bitcoin', 'testnet', 'testnet4', 'regtest' or 'signet')"
 default = "Default::default()"
 
@@ -70,10 +76,6 @@ doc = "Electrum server JSONRPC 'addr:port' to listen on (default: '127.0.0.1:500
 name = "daemon_rpc_addr"
 type = "crate::config::ResolvAddr"
 doc = "Bitcoin daemon JSONRPC 'addr:port' to connect (default: 127.0.0.1:8332 for mainnet, 127.0.0.1:18332 for testnet, 127.0.0.1:18443 for regtest and 127.0.0.1:18554 for signet)"
-[[param]]
-name = "daemon_p2p_addr"
-type = "crate::config::ResolvAddr"
-doc = "Bitcoin daemon p2p 'addr:port' to connect (default: 127.0.0.1:8333 for mainnet, 127.0.0.1:18333 for testnet, 127.0.0.1:18444 for regtest and 127.0.0.1:38333 for signet)"
 
 [[param]]
 name = "monitoring_addr"
@@ -92,12 +94,6 @@ type = "u64"
 doc = "Duration to wait until bitcoind JSON-RPC timeouts (must be greater than wait_duration_secs)."
 default = "15"
 
-[[param]]
-name = "index_batch_size"
-type = "usize"
-doc = "Number of blocks to get in a single p2p protocol request from bitcoind"
-default = "10"
-
 [[switch]]
 name = "ignore_mempool"
 doc = "Don't sync mempool - queries will show only confirmed transactions."
@@ -140,8 +136,3 @@ default = "concat!(\"Welcome to electrs \", env!(\"CARGO_PKG_VERSION\"), \" (Ele
 name = "log_filters"
 type = "String"
 doc = "Logging filters, overriding `RUST_LOG` environment variable (see https://docs.rs/env_logger/ for details)"
-
-[[param]]
-name = "signet_magic"
-type = "String"
-doc = "network magic for custom signet network in hex format, as found in Bitcoin Core logs (signet only)"

server.sh

@@ -9,7 +9,7 @@ NETWORK=$1
 shift
 
 DB=${DB-./db}
-export RUST_LOG=${RUST_LOG-electrs=INFO}
+export RUST_LOG=${RUST_LOG-INFO}
 target/release/electrs --network $NETWORK --db-dir $DB --daemon-dir $HOME/.bitcoin $*
 
 # use SIGINT to quit

src/cache.rs

@@ -1,43 +0,0 @@
-use bitcoin::Txid;
-use parking_lot::RwLock;
-
-use std::collections::HashMap;
-use std::sync::Arc;
-
-use crate::metrics::{self, Histogram, Metrics};
-
-pub(crate) struct Cache {
-    txs: Arc<RwLock<HashMap<Txid, Box<[u8]>>>>,
-
-    // stats
-    txs_size: Histogram,
-}
-
-impl Cache {
-    pub fn new(metrics: &Metrics) -> Self {
-        Cache {
-            txs: Default::default(),
-            txs_size: metrics.histogram_vec(
-                "cache_txs_size",
-                "Cached transactions' size (in bytes)",
-                "type",
-                metrics::default_size_buckets(),
-            ),
-        }
-    }
-
-    pub fn add_tx(&self, txid: Txid, f: impl FnOnce() -> Box<[u8]>) {
-        self.txs.write().entry(txid).or_insert_with(|| {
-            let tx = f();
-            self.txs_size.observe("serialized", tx.len() as f64);
-            tx
-        });
-    }
-
-    pub fn get_tx<F, T>(&self, txid: &Txid, f: F) -> Option<T>
-    where
-        F: FnOnce(&[u8]) -> T,
-    {
-        self.txs.read().get(txid).map(|tx_bytes| f(tx_bytes))
-    }
-}

src/chain.rs

@@ -1,261 +0,0 @@
-use std::collections::HashMap;
-
-use bitcoin::blockdata::block::Header as BlockHeader;
-use bitcoin::{BlockHash, Network};
-
-/// A new header found, to be added to the chain at specific height
-pub(crate) struct NewHeader {
-    header: BlockHeader,
-    hash: BlockHash,
-    height: usize,
-}
-
-impl NewHeader {
-    pub(crate) fn from((header, height): (BlockHeader, usize)) -> Self {
-        Self {
-            header,
-            hash: header.block_hash(),
-            height,
-        }
-    }
-
-    pub(crate) fn height(&self) -> usize {
-        self.height
-    }
-
-    pub(crate) fn hash(&self) -> BlockHash {
-        self.hash
-    }
-}
-
-/// Current blockchain headers' list
-pub struct Chain {
-    headers: Vec<(BlockHash, BlockHeader)>,
-    heights: HashMap<BlockHash, usize>,
-}
-
-impl Chain {
-    // create an empty chain
-    pub fn new(network: Network) -> Self {
-        let genesis = bitcoin::blockdata::constants::genesis_block(network);
-        let genesis_hash = genesis.block_hash();
-        Self {
-            headers: vec![(genesis_hash, genesis.header)],
-            heights: std::iter::once((genesis_hash, 0)).collect(), // genesis header @ zero height
-        }
-    }
-
-    pub(crate) fn drop_last_headers(&mut self, n: usize) {
-        if n == 0 {
-            return;
-        }
-        let new_height = self.height().saturating_sub(n);
-        self.update(vec![NewHeader::from((
-            self.headers[new_height].1,
-            new_height,
-        ))])
-    }
-
-    /// Load the chain from a collection of headers, up to the given tip
-    pub(crate) fn load(&mut self, headers: impl Iterator<Item = BlockHeader>, tip: BlockHash) {
-        let genesis_hash = self.headers[0].0;
-
-        let header_map: HashMap<BlockHash, BlockHeader> =
-            headers.map(|h| (h.block_hash(), h)).collect();
-        let mut blockhash = tip;
-        let mut new_headers: Vec<&BlockHeader> = Vec::with_capacity(header_map.len());
-        while blockhash != genesis_hash {
-            let header = match header_map.get(&blockhash) {
-                Some(header) => header,
-                None => panic!("missing header {} while loading from DB", blockhash),
-            };
-            blockhash = header.prev_blockhash;
-            new_headers.push(header);
-        }
-        info!("loading {} headers, tip={}", new_headers.len(), tip);
-        let new_headers = new_headers.into_iter().rev().copied(); // order by height
-        self.update(new_headers.zip(1..).map(NewHeader::from).collect())
-    }
-
-    /// Get the block hash at specified height (if exists)
-    pub(crate) fn get_block_hash(&self, height: usize) -> Option<BlockHash> {
-        self.headers.get(height).map(|(hash, _header)| *hash)
-    }
-
-    /// Get the block header at specified height (if exists)
-    pub(crate) fn get_block_header(&self, height: usize) -> Option<&BlockHeader> {
-        self.headers.get(height).map(|(_hash, header)| header)
-    }
-
-    /// Get the block height given the specified hash (if exists)
-    pub(crate) fn get_block_height(&self, blockhash: &BlockHash) -> Option<usize> {
-        self.heights.get(blockhash).copied()
-    }
-
-    /// Update the chain with a list of new headers (possibly a reorg)
-    pub(crate) fn update(&mut self, headers: Vec<NewHeader>) {
-        if let Some(first_height) = headers.first().map(|h| h.height) {
-            for (hash, _header) in self.headers.drain(first_height..) {
-                assert!(self.heights.remove(&hash).is_some());
-            }
-            for (h, height) in headers.into_iter().zip(first_height..) {
-                assert_eq!(h.height, height);
-                assert_eq!(h.hash, h.header.block_hash());
-                assert!(self.heights.insert(h.hash, h.height).is_none());
-                self.headers.push((h.hash, h.header));
-            }
-            info!(
-                "chain updated: tip={}, height={}",
-                self.headers.last().unwrap().0,
-                self.headers.len() - 1
-            );
-        }
-    }
-
-    /// Best block hash
-    pub(crate) fn tip(&self) -> BlockHash {
-        self.headers.last().expect("empty chain").0
-    }
-
-    /// Number of blocks (excluding genesis block)
-    pub(crate) fn height(&self) -> usize {
-        self.headers.len() - 1
-    }
-
-    /// List of block hashes for efficient fork detection and block/header sync
-    /// see https://en.bitcoin.it/wiki/Protocol_documentation#getblocks
-    pub(crate) fn locator(&self) -> Vec<BlockHash> {
-        let mut result = vec![];
-        let mut index = self.headers.len() - 1;
-        let mut step = 1;
-        loop {
-            if result.len() >= 10 {
-                step *= 2;
-            }
-            result.push(self.headers[index].0);
-            if index == 0 {
-                break;
-            }
-            index = index.saturating_sub(step);
-        }
-        result
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::{Chain, NewHeader};
-    use bitcoin::blockdata::block::Header as BlockHeader;
-    use bitcoin::consensus::deserialize;
-    use bitcoin::Network::Regtest;
-    use hex_lit::hex;
-
-    #[test]
-    fn test_genesis() {
-        let regtest = Chain::new(Regtest);
-        assert_eq!(regtest.height(), 0);
-        assert_eq!(
-            regtest.tip(),
-            "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206"
-                .parse()
-                .unwrap()
-        );
-    }
-
-    #[test]
-    fn test_updates() {
-        let byte_headers = [
-hex!("0000002006226e46111a0b59caaf126043eb5bbf28c34f3a5e332a1fc7b2b73cf188910f1d14d3c7ff12d6adf494ebbcfba69baa915a066358b68a2b8c37126f74de396b1d61cc60ffff7f2000000000"),
-hex!("00000020d700ae5d3c705702e0a5d9ababd22ded079f8a63b880b1866321d6bfcb028c3fc816efcf0e84ccafa1dda26be337f58d41b438170c357cda33a68af5550590bc1e61cc60ffff7f2004000000"),
-hex!("00000020d13731bc59bc0989e06a5e7cab9843a4e17ad65c7ca47cd77f50dfd24f1f55793f7f342526aca9adb6ce8f33d8a07662c97d29d83b9e18117fb3eceecb2ab99b1e61cc60ffff7f2001000000"),
-hex!("00000020a603def3e1255cadfb6df072946327c58b344f9bfb133e8e3e280d1c2d55b31c731a68f70219472864a7cb010cd53dc7e0f67e57f7d08b97e5e092b0c3942ad51f61cc60ffff7f2001000000"),
-hex!("0000002041dd202b3b2edcdd3c8582117376347d48ff79ff97c95e5ac814820462012e785142dc360975b982ca43eecd14b4ba6f019041819d4fc5936255d7a2c45a96651f61cc60ffff7f2000000000"),
-hex!("0000002072e297a2d6b633c44f3c9b1a340d06f3ce4e6bcd79ebd4c4ff1c249a77e1e37c59c7be1ca0964452e1735c0d2740f0d98a11445a6140c36b55770b5c0bcf801f1f61cc60ffff7f2000000000"),
-hex!("000000200c9eb5889a8e924d1c4e8e79a716514579e41114ef37d72295df8869d6718e4ac5840f28de43ff25c7b9200aaf7873b20587c92827eaa61943484ca828bdd2e11f61cc60ffff7f2000000000"),
-hex!("000000205873f322b333933e656b07881bb399dae61a6c0fa74188b5fb0e3dd71c9e2442f9e2f433f54466900407cf6a9f676913dd54aad977f7b05afcd6dcd81e98ee752061cc60ffff7f2004000000"),
-hex!("00000020fd1120713506267f1dba2e1856ca1d4490077d261cde8d3e182677880df0d856bf94cfa5e189c85462813751ab4059643759ed319a81e0617113758f8adf67bc2061cc60ffff7f2000000000"),
-hex!("000000200030d7f9c11ef35b89a0eefb9a5e449909339b5e7854d99804ea8d6a49bf900a0304d2e55fe0b6415949cff9bca0f88c0717884a5e5797509f89f856af93624a2061cc60ffff7f2002000000"),
-        ];
-        let headers: Vec<BlockHeader> = byte_headers
-            .iter()
-            .map(|byte_header| deserialize(byte_header).unwrap())
-            .collect();
-
-        for chunk_size in 1..headers.len() {
-            let mut regtest = Chain::new(Regtest);
-            let mut height = 0;
-            let mut tip = regtest.tip();
-            for chunk in headers.chunks(chunk_size) {
-                let mut update = vec![];
-                for header in chunk {
-                    height += 1;
-                    tip = header.block_hash();
-                    update.push(NewHeader::from((*header, height)))
-                }
-                regtest.update(update);
-                assert_eq!(regtest.tip(), tip);
-                assert_eq!(regtest.height(), height);
-            }
-            assert_eq!(regtest.tip(), headers.last().unwrap().block_hash());
-            assert_eq!(regtest.height(), headers.len());
-        }
-
-        // test loading from a list of headers and tip
-        let mut regtest = Chain::new(Regtest);
-        regtest.load(
-            headers.iter().copied(),
-            headers.last().unwrap().block_hash(),
-        );
-        assert_eq!(regtest.height(), headers.len());
-
-        // test getters
-        for (header, height) in headers.iter().zip(1usize..) {
-            assert_eq!(regtest.get_block_header(height), Some(header));
-            assert_eq!(regtest.get_block_hash(height), Some(header.block_hash()));
-            assert_eq!(regtest.get_block_height(&header.block_hash()), Some(height));
-        }
-
-        // test chain shortening
-        for i in (0..=headers.len()).rev() {
-            let hash = regtest.get_block_hash(i).unwrap();
-            assert_eq!(regtest.get_block_height(&hash), Some(i));
-            assert_eq!(regtest.height(), i);
-            assert_eq!(regtest.tip(), hash);
-            regtest.drop_last_headers(1);
-        }
-        assert_eq!(regtest.height(), 0);
-        assert_eq!(
-            regtest.tip(),
-            "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206"
-                .parse()
-                .unwrap()
-        );
-
-        regtest.drop_last_headers(1);
-        assert_eq!(regtest.height(), 0);
-        assert_eq!(
-            regtest.tip(),
-            "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206"
-                .parse()
-                .unwrap()
-        );
-
-        // test reorg
-        let mut regtest = Chain::new(Regtest);
-        regtest.load(
-            headers.iter().copied(),
-            headers.last().unwrap().block_hash(),
-        );
-        let height = regtest.height();
-
-        let new_header: BlockHeader = deserialize(&hex!("000000200030d7f9c11ef35b89a0eefb9a5e449909339b5e7854d99804ea8d6a49bf900a0304d2e55fe0b6415949cff9bca0f88c0717884a5e5797509f89f856af93624a7a6bcc60ffff7f2000000000")).unwrap();
-        regtest.update(vec![NewHeader::from((new_header, height))]);
-        assert_eq!(regtest.height(), height);
-        assert_eq!(
-            regtest.tip(),
-            "0e16637fe0700a7c52e9a6eaa58bd6ac7202652103be8f778680c66f51ad2e9b"
-                .parse()
-                .unwrap()
-        );
-    }
-}

src/config.rs

@@ -1,6 +1,5 @@
-use bitcoin::p2p::Magic;
-use bitcoin::Network;
-use bitcoincore_rpc::Auth;
+use crate::bitcoin::Network;
+use crate::types::Auth;
 use dirs_next::home_dir;
 
 use std::ffi::{OsStr, OsString};
@@ -17,6 +16,7 @@ pub const ELECTRS_VERSION: &str = env!("CARGO_PKG_VERSION");
 const DEFAULT_SERVER_ADDRESS: [u8; 4] = [127, 0, 0, 1]; // by default, serve on IPv4 localhost
 
 mod internal {
+    #![allow(unused_attributes, unused_imports, clippy::enum_variant_names)]
     include!(concat!(env!("OUT_DIR"), "/configure_me_config.rs"));
 }
 
@@ -121,20 +121,20 @@ impl From<BitcoinNetwork> for Network {
 }
 
 /// Parsed and post-processed configuration
+#[allow(dead_code)]
 #[derive(Debug)]
 pub struct Config {
     // See below for the documentation of each field:
     pub network: Network,
-    pub db_path: PathBuf,
+    pub db_dir: PathBuf,
     pub db_log_dir: Option<PathBuf>,
+    pub db_parallelism: u8,
     pub daemon_auth: SensitiveAuth,
     pub daemon_rpc_addr: SocketAddr,
-    pub daemon_p2p_addr: SocketAddr,
     pub electrum_rpc_addr: SocketAddr,
     pub monitoring_addr: SocketAddr,
     pub wait_duration: Duration,
     pub jsonrpc_timeout: Duration,
-    pub index_batch_size: usize,
     pub index_lookup_limit: Option<usize>,
     pub reindex_last_blocks: usize,
     pub auto_reindex: bool,
@@ -143,7 +143,6 @@ pub struct Config {
     pub skip_block_download_wait: bool,
     pub disable_electrum_rpc: bool,
     pub server_banner: String,
-    pub signet_magic: Magic,
 }
 
 pub struct SensitiveAuth(pub Auth);
@@ -196,37 +195,12 @@ impl Config {
             internal::prelude::Config::including_optional_config_files(default_config_files())
                 .unwrap_or_exit();
 
-        fn unsupported_network(network: Network) -> ! {
-            eprintln!("Error: unsupported network: {}", network);
-            std::process::exit(1);
-        }
-
-        let db_subdir = match config.network {
-            Network::Bitcoin => "bitcoin",
-            Network::Testnet => "testnet",
-            Network::Testnet4 => "testnet4",
-            Network::Regtest => "regtest",
-            Network::Signet => "signet",
-            unsupported => unsupported_network(unsupported),
-        };
-
-        config.db_dir.push(db_subdir);
-
         let default_daemon_rpc_port = match config.network {
             Network::Bitcoin => 8332,
             Network::Testnet => 18332,
             Network::Testnet4 => 48332,
             Network::Regtest => 18443,
             Network::Signet => 38332,
-            unsupported => unsupported_network(unsupported),
-        };
-        let default_daemon_p2p_port = match config.network {
-            Network::Bitcoin => 8333,
-            Network::Testnet => 18333,
-            Network::Testnet4 => 48333,
-            Network::Regtest => 18444,
-            Network::Signet => 38333,
-            unsupported => unsupported_network(unsupported),
         };
         let default_electrum_port = match config.network {
             Network::Bitcoin => 50001,
@@ -234,7 +208,6 @@ impl Config {
             Network::Testnet4 => 40001,
             Network::Regtest => 60401,
             Network::Signet => 60601,
-            unsupported => unsupported_network(unsupported),
         };
         let default_monitoring_port = match config.network {
             Network::Bitcoin => 4224,
@@ -242,32 +215,12 @@ impl Config {
             Network::Testnet4 => 44224,
             Network::Regtest => 24224,
             Network::Signet => 34224,
-            unsupported => unsupported_network(unsupported),
-        };
-
-        let magic = match (config.network, config.signet_magic) {
-            (Network::Signet, Some(magic)) => magic.parse().unwrap_or_else(|error| {
-                eprintln!(
-                    "Error: signet magic '{}' is not a valid hex string: {}",
-                    magic, error
-                );
-                std::process::exit(1);
-            }),
-            (network, None) => network.magic(),
-            (_, Some(_)) => {
-                eprintln!("Error: signet magic only available on signet");
-                std::process::exit(1);
-            }
         };
 
         let daemon_rpc_addr: SocketAddr = config.daemon_rpc_addr.map_or(
             (DEFAULT_SERVER_ADDRESS, default_daemon_rpc_port).into(),
             ResolvAddr::resolve_or_exit,
         );
-        let daemon_p2p_addr: SocketAddr = config.daemon_p2p_addr.map_or(
-            (DEFAULT_SERVER_ADDRESS, default_daemon_p2p_port).into(),
-            ResolvAddr::resolve_or_exit,
-        );
         let electrum_rpc_addr: SocketAddr = config.electrum_rpc_addr.map_or(
             (DEFAULT_SERVER_ADDRESS, default_electrum_port).into(),
             ResolvAddr::resolve_or_exit,
@@ -290,7 +243,6 @@ impl Config {
             Network::Testnet4 => config.daemon_dir.push("testnet4"),
             Network::Regtest => config.daemon_dir.push("regtest"),
             Network::Signet => config.daemon_dir.push("signet"),
-            unsupported => unsupported_network(unsupported),
         }
 
         let mut deprecated_options_used = false;
@@ -352,16 +304,15 @@ impl Config {
 
         let config = Config {
             network: config.network,
-            db_path: config.db_dir,
+            db_dir: config.db_dir,
             db_log_dir: config.db_log_dir,
+            db_parallelism: config.db_parallelism,
             daemon_auth,
             daemon_rpc_addr,
-            daemon_p2p_addr,
             electrum_rpc_addr,
             monitoring_addr,
             wait_duration: Duration::from_secs(config.wait_duration_secs),
             jsonrpc_timeout: Duration::from_secs(config.jsonrpc_timeout_secs),
-            index_batch_size: config.index_batch_size,
             index_lookup_limit,
             reindex_last_blocks: config.reindex_last_blocks,
             auto_reindex: config.auto_reindex,
@@ -370,7 +321,6 @@ impl Config {
             skip_block_download_wait: config.skip_block_download_wait,
             disable_electrum_rpc: config.disable_electrum_rpc,
             server_banner: config.server_banner,
-            signet_magic: magic,
         };
         eprintln!(
             "Starting electrs {} on {} {} with {:?}",
@@ -394,9 +344,6 @@ mod tests {
 
     #[test]
     fn test_auth_debug() {
-        let auth = Auth::None;
-        assert_eq!(format!("{:?}", SensitiveAuth(auth)), "None");
-
         let auth = Auth::CookieFile(Path::new("/foo/bar/.cookie").to_path_buf());
         assert_eq!(
             format!("{:?}", SensitiveAuth(auth)),

src/daemon.rs

@@ -1,10 +1,10 @@
 use anyhow::{Context, Result};
 
-use bitcoin::{consensus::deserialize, hashes::hex::FromHex};
-use bitcoin::{Amount, BlockHash, Transaction, Txid};
-use bitcoincore_rpc::{json, jsonrpc, Auth, Client, RpcApi};
-use crossbeam_channel::Receiver;
-use parking_lot::Mutex;
+use crate::bitcoin::{
+    self, consensus::deserialize, consensus::encode::serialize_hex, hashes::hex::FromHex, Amount,
+    BlockHash, Transaction, Txid,
+};
+use crate::types::Auth;
 use serde::Serialize;
 use serde_json::{json, value::RawValue, Value};
 
@@ -12,52 +12,62 @@ use std::fs::File;
 use std::io::Read;
 use std::path::Path;
 
-use crate::{
-    chain::{Chain, NewHeader},
-    config::Config,
-    metrics::Metrics,
-    p2p::Connection,
-    signals::ExitFlag,
-    types::SerBlock,
-};
+use crate::{config::Config, signals::ExitFlag};
 
 enum PollResult {
     Done(Result<()>),
     Retry,
 }
 
-fn rpc_poll(client: &mut Client, skip_block_download_wait: bool) -> PollResult {
-    match client.get_blockchain_info() {
-        Ok(info) => {
-            if skip_block_download_wait {
-                // bitcoind RPC is available, don't wait for block download to finish
-                return PollResult::Done(Ok(()));
-            }
-            let left_blocks = info.headers - info.blocks;
-            if info.initial_block_download || left_blocks > 0 {
-                info!(
-                    "waiting for {} blocks to download{}",
-                    left_blocks,
-                    if info.initial_block_download {
-                        " (IBD)"
-                    } else {
-                        ""
-                    }
-                );
-                return PollResult::Retry;
-            }
-            PollResult::Done(Ok(()))
-        }
-        Err(err) => {
-            if let Some(e) = extract_bitcoind_error(&err) {
-                if e.code == -28 {
-                    debug!("waiting for RPC warmup: {}", e.message);
-                    return PollResult::Retry;
-                }
-            }
-            PollResult::Done(Err(err).context("daemon not available"))
-        }
-    }
+#[derive(Deserialize)]
+pub struct GetMempoolEntryResultFees {
+    /// Transaction fee in BTC
+    #[serde(with = "bitcoin::amount::serde::as_btc")]
+    pub base: Amount,
+}
+
+#[derive(Deserialize)]
+pub struct GetMempoolEntryResult {
+    /// Virtual transaction size as defined in BIP 141. This is different from actual serialized
+    /// size for witness transactions as witness data is discounted.
+    #[serde(alias = "size")]
+    pub vsize: u64,
+    /// Fee information
+    pub fees: GetMempoolEntryResultFees,
+    /// Unconfirmed transactions used as inputs for this transaction
+    pub depends: Vec<bitcoin::Txid>,
+}
+
+#[derive(Deserialize)]
+pub struct GetBlockchainInfoResult {
+    pub blocks: u64,
+    pub headers: u64,
+    pub initialblockdownload: bool,
+    pub pruned: bool,
+}
+
+#[derive(Deserialize)]
+pub struct EstimateSmartFeeResult {
+    /// Estimate fee rate in BTC/kB.
+    #[serde(
+        default,
+        rename = "feerate",
+        skip_serializing_if = "Option::is_none",
+        with = "bitcoin::amount::serde::as_btc::opt"
+    )]
+    pub fee_rate: Option<Amount>,
+}
+
+#[derive(Deserialize)]
+pub struct GetNetworkInfoResult {
+    #[serde(rename = "relayfee", with = "bitcoin::amount::serde::as_btc")]
+    pub relay_fee: Amount,
+}
+
+#[derive(Deserialize)]
+pub struct GetMempoolInfoResult {
+    /// True if the mempool is fully loaded
+    pub loaded: Option<bool>,
 }
 
 fn read_cookie(path: &Path) -> Result<(String, String)> {
@@ -78,44 +88,38 @@ fn read_cookie(path: &Path) -> Result<(String, String)> {
     Ok((parts[0].to_owned(), parts[1].to_owned()))
 }
 
-fn rpc_connect(config: &Config) -> Result<Client> {
+fn jsonrpc_client(config: &Config) -> Result<jsonrpc::Client> {
     let rpc_url = format!("http://{}", config.daemon_rpc_addr);
-    // Allow `wait_for_new_block` to take a bit longer before timing out.
+    // Allow RPC calls to take longer before timing out.
     // See https://github.com/romanz/electrs/issues/495 for more details.
     let builder = jsonrpc::simple_http::SimpleHttpTransport::builder()
         .url(&rpc_url)?
         .timeout(config.jsonrpc_timeout);
     let builder = match config.daemon_auth.get_auth() {
-        Auth::None => builder,
         Auth::UserPass(user, pass) => builder.auth(user, Some(pass)),
         Auth::CookieFile(path) => {
             let (user, pass) = read_cookie(&path)?;
             builder.auth(user, Some(pass))
         }
     };
-    Ok(Client::from_jsonrpc(jsonrpc::Client::with_transport(
-        builder.build(),
-    )))
+    Ok(jsonrpc::Client::with_transport(builder.build()))
 }
 
 pub struct Daemon {
-    p2p: Mutex<Connection>,
-    rpc: Client,
+    client: jsonrpc::Client,
 }
 
 impl Daemon {
-    pub(crate) fn connect(
-        config: &Config,
-        exit_flag: &ExitFlag,
-        metrics: &Metrics,
-    ) -> Result<Self> {
-        let mut rpc = rpc_connect(config)?;
+    pub(crate) fn connect(config: &Config, exit_flag: &ExitFlag) -> Result<Self> {
+        let daemon = Daemon {
+            client: jsonrpc_client(config)?,
+        };
 
         loop {
             exit_flag
                 .poll()
                 .context("bitcoin RPC polling interrupted")?;
-            match rpc_poll(&mut rpc, config.skip_block_download_wait) {
+            match daemon.rpc_poll(config.skip_block_download_wait) {
                 PollResult::Done(result) => {
                     result.context("bitcoind RPC polling failed")?;
                     break; // on success, finish polling
@@ -126,34 +130,85 @@ impl Daemon {
             }
         }
 
-        let network_info = rpc.get_network_info()?;
-        if network_info.version < 21_00_00 {
-            bail!("electrs requires bitcoind 0.21+");
-        }
-        if !network_info.network_active {
-            bail!("electrs requires active bitcoind p2p network");
-        }
-        let info = rpc.get_blockchain_info()?;
-        if info.pruned {
-            bail!("electrs requires non-pruned bitcoind node");
-        }
+        Ok(daemon)
+    }
 
-        let p2p = Mutex::new(Connection::connect(
-            config.network,
-            config.daemon_p2p_addr,
-            metrics,
-            config.signet_magic,
-        )?);
-        Ok(Self { p2p, rpc })
+    fn call<T: for<'a> serde::de::Deserialize<'a>>(
+        &self,
+        cmd: &str,
+        args: &[serde_json::Value],
+    ) -> Result<T, jsonrpc::Error> {
+        let raw = serde_json::value::to_raw_value(args)?;
+        let req = self.client.build_request(cmd, Some(&*raw));
+        let resp = self.client.send_request(req)?;
+        resp.result()
+    }
+
+    fn batch_request<T>(&self, name: &str, items: &[T]) -> Result<Vec<Option<jsonrpc::Response>>>
+    where
+        T: Serialize,
+    {
+        debug!("calling {} on {} items", name, items.len());
+        let args: Vec<Box<RawValue>> = items
+            .iter()
+            .map(|item| jsonrpc::try_arg([item]).context("failed to serialize into JSON"))
+            .collect::<Result<Vec<_>>>()?;
+        let reqs: Vec<jsonrpc::Request> = args
+            .iter()
+            .map(|arg| self.client.build_request(name, Some(arg)))
+            .collect();
+        match self.client.send_batch(&reqs) {
+            Ok(values) => {
+                assert_eq!(items.len(), values.len());
+                Ok(values)
+            }
+            Err(err) => bail!("batch {} request failed: {}", name, err),
+        }
+    }
+
+    fn rpc_poll(&self, skip_block_download_wait: bool) -> PollResult {
+        match self.call::<GetBlockchainInfoResult>("getblockchaininfo", &[]) {
+            Ok(info) => {
+                if info.pruned {
+                    return PollResult::Done(Err(anyhow!(
+                        "electrs requires non-pruned bitcoind node"
+                    )));
+                }
+                if skip_block_download_wait {
+                    // bitcoind RPC is available, don't wait for block download to finish
+                    return PollResult::Done(Ok(()));
+                }
+                let left_blocks = info.headers - info.blocks;
+                if info.initialblockdownload || left_blocks > 0 {
+                    info!(
+                        "waiting for {} blocks to download{}",
+                        left_blocks,
+                        if info.initialblockdownload {
+                            " (IBD)"
+                        } else {
+                            ""
+                        }
+                    );
+                    return PollResult::Retry;
+                }
+                PollResult::Done(Ok(()))
+            }
+            Err(err) => {
+                if let Some(e) = extract_bitcoind_error(&err) {
+                    if e.code == -28 {
+                        debug!("waiting for RPC warmup: {}", e.message);
+                        return PollResult::Retry;
+                    }
+                }
+                PollResult::Done(Err(err).context("daemon not available"))
+            }
+        }
     }
 
     pub(crate) fn estimate_fee(&self, nblocks: u16) -> Result<Option<Amount>> {
-        let res = self.rpc.estimate_smart_fee(nblocks, None);
-        if let Err(bitcoincore_rpc::Error::JsonRpc(jsonrpc::Error::Rpc(RpcError {
-            code: -32603,
-            ..
-        }))) = res
-        {
+        let res =
+            self.call::<EstimateSmartFeeResult>("estimatesmartfee", &[json!(nblocks), Value::Null]);
+        if let Err(jsonrpc::Error::Rpc(jsonrpc::error::RpcError { code: -32603, .. })) = res {
             return Ok(None); // don't fail when fee estimation is disabled (e.g. with `-blocksonly=1`)
         }
         Ok(res.context("failed to estimate fee")?.fee_rate)
@@ -161,84 +216,64 @@ impl Daemon {
 
     pub(crate) fn get_relay_fee(&self) -> Result<Amount> {
         Ok(self
-            .rpc
-            .get_network_info()
+            .call::<GetNetworkInfoResult>("getnetworkinfo", &[])
             .context("failed to get relay fee")?
             .relay_fee)
     }
 
     pub(crate) fn broadcast(&self, tx: &Transaction) -> Result<Txid> {
-        self.rpc
-            .send_raw_transaction(tx)
+        self.call("sendrawtransaction", &[json!(serialize_hex(tx))])
             .context("failed to broadcast transaction")
     }
 
-    pub(crate) fn get_transaction_info(
-        &self,
-        txid: &Txid,
-        blockhash: Option<BlockHash>,
-    ) -> Result<Value> {
-        // No need to parse the resulting JSON, just return it as-is to the client.
-        self.rpc
-            .call(
-                "getrawtransaction",
-                &[json!(txid), json!(true), json!(blockhash)],
-            )
-            .context("failed to get transaction info")
-    }
-
-    pub(crate) fn get_transaction_hex(
-        &self,
-        txid: &Txid,
-        blockhash: Option<BlockHash>,
-    ) -> Result<Value> {
-        use bitcoin::consensus::serde::{hex::Lower, Hex, With};
-
-        let tx = self.get_transaction(txid, blockhash)?;
-        #[derive(serde::Serialize)]
-        #[serde(transparent)]
-        struct TxAsHex(#[serde(with = "With::<Hex<Lower>>")] Transaction);
-        serde_json::to_value(TxAsHex(tx)).map_err(Into::into)
+    pub(crate) fn submitpackage(&self, txs: &[Transaction]) -> Result<Value> {
+        let package: Vec<String> = txs.iter().map(serialize_hex).collect();
+        self.call("submitpackage", &[json!(package)])
+            .context("failed to submitpackage package")
     }
 
     pub(crate) fn get_transaction(
         &self,
         txid: &Txid,
         blockhash: Option<BlockHash>,
-    ) -> Result<Transaction> {
-        self.rpc
-            .get_raw_transaction(txid, blockhash.as_ref())
-            .context("failed to get transaction")
+        verbose: bool,
+    ) -> Result<Value> {
+        self.call(
+            "getrawtransaction",
+            &[json!(txid), json!(verbose), json!(blockhash)],
+        )
+        .context("failed to get transaction")
     }
 
     pub(crate) fn get_block_txids(&self, blockhash: BlockHash) -> Result<Vec<Txid>> {
-        Ok(self
-            .rpc
-            .get_block_info(&blockhash)
-            .context("failed to get block txids")?
-            .tx)
+        #[derive(Clone, PartialEq, Debug, Deserialize, Serialize)]
+        struct GetBlockResult {
+            pub tx: Vec<bitcoin::Txid>,
+        }
+        let res: GetBlockResult = self
+            .call("getblock", &[json!(blockhash), json!(1)])
+            .context("failed to get block txids")?;
+        Ok(res.tx)
     }
 
-    pub(crate) fn get_mempool_info(&self) -> Result<json::GetMempoolInfoResult> {
-        self.rpc
-            .get_mempool_info()
+    pub(crate) fn get_mempool_info(&self) -> Result<GetMempoolInfoResult> {
+        self.call("getmempoolinfo", &[])
             .context("failed to get mempool info")
     }
 
     pub(crate) fn get_mempool_txids(&self) -> Result<Vec<Txid>> {
-        self.rpc
-            .get_raw_mempool()
+        self.call("getrawmempool", &[])
             .context("failed to get mempool txids")
     }
 
     pub(crate) fn get_mempool_entries(
         &self,
         txids: &[Txid],
-    ) -> Result<Vec<Option<json::GetMempoolEntryResult>>> {
-        let results = batch_request(self.rpc.get_jsonrpc_client(), "getmempoolentry", txids)?;
+    ) -> Result<Vec<Option<GetMempoolEntryResult>>> {
+        let results = self.batch_request("getmempoolentry", txids)?;
         Ok(results
             .into_iter()
-            .map(|r| match r?.result::<json::GetMempoolEntryResult>() {
+            .map(|r| match r?.result::<GetMempoolEntryResult>() {
                 Ok(entry) => Some(entry),
                 Err(err) => {
                     debug!("failed to get mempool entry: {}", err); // probably due to RBF
@@ -252,7 +287,7 @@ impl Daemon {
         &self,
         txids: &[Txid],
     ) -> Result<Vec<Option<Transaction>>> {
-        let results = batch_request(self.rpc.get_jsonrpc_client(), "getrawtransaction", txids)?;
+        let results = self.batch_request("getrawtransaction", txids)?;
         Ok(results
             .into_iter()
             .map(|r| -> Option<Transaction> {
@@ -280,58 +315,11 @@ impl Daemon {
             })
             .collect())
     }
-
-    pub(crate) fn get_new_headers(&self, chain: &Chain) -> Result<Vec<NewHeader>> {
-        self.p2p.lock().get_new_headers(chain)
-    }
-
-    pub(crate) fn for_blocks<B, F>(&self, blockhashes: B, func: F) -> Result<()>
-    where
-        B: IntoIterator<Item = BlockHash>,
-        F: FnMut(BlockHash, SerBlock),
-    {
-        self.p2p.lock().for_blocks(blockhashes, func)
-    }
-
-    pub(crate) fn new_block_notification(&self) -> Receiver<()> {
-        self.p2p.lock().new_block_notification()
-    }
 }
 
-pub(crate) type RpcError = bitcoincore_rpc::jsonrpc::error::RpcError;
-
-pub(crate) fn extract_bitcoind_error(err: &bitcoincore_rpc::Error) -> Option<&RpcError> {
-    use bitcoincore_rpc::{
-        jsonrpc::error::Error::Rpc as ServerError, Error::JsonRpc as JsonRpcError,
-    };
+pub(crate) fn extract_bitcoind_error(err: &jsonrpc::Error) -> Option<&jsonrpc::error::RpcError> {
     match err {
-        JsonRpcError(ServerError(e)) => Some(e),
+        jsonrpc::error::Error::Rpc(e) => Some(e),
         _ => None,
     }
 }
-
-fn batch_request<T>(
-    client: &jsonrpc::Client,
-    name: &str,
-    items: &[T],
-) -> Result<Vec<Option<jsonrpc::Response>>>
-where
-    T: Serialize,
-{
-    debug!("calling {} on {} items", name, items.len());
-    let args: Vec<Box<RawValue>> = items
-        .iter()
-        .map(|item| jsonrpc::try_arg([item]).context("failed to serialize into JSON"))
-        .collect::<Result<Vec<_>>>()?;
-    let reqs: Vec<jsonrpc::Request> = args
-        .iter()
-        .map(|arg| client.build_request(name, Some(arg)))
-        .collect();
-    match client.send_batch(&reqs) {
-        Ok(values) => {
-            assert_eq!(items.len(), values.len());
-            Ok(values)
-        }
-        Err(err) => bail!("batch {} request failed: {}", name, err),
-    }
-}

src/db.rs

@@ -1,538 +0,0 @@
-use anyhow::{Context, Result};
-use electrs_rocksdb as rocksdb;
-
-use std::path::Path;
-use std::sync::atomic::{AtomicBool, Ordering};
-
-use crate::types::{HashPrefix, SerializedHashPrefixRow, SerializedHeaderRow};
-
-#[derive(Default)]
-pub(crate) struct WriteBatch {
-    pub(crate) tip_row: [u8; 32],
-    pub(crate) header_rows: Vec<SerializedHeaderRow>,
-    pub(crate) funding_rows: Vec<SerializedHashPrefixRow>,
-    pub(crate) spending_rows: Vec<SerializedHashPrefixRow>,
-    pub(crate) txid_rows: Vec<SerializedHashPrefixRow>,
-}
-
-impl WriteBatch {
-    pub(crate) fn sort(&mut self) {
-        self.header_rows.sort_unstable();
-        self.funding_rows.sort_unstable();
-        self.spending_rows.sort_unstable();
-        self.txid_rows.sort_unstable();
-    }
-}
-
-/// RocksDB wrapper for index storage
-pub struct DBStore {
-    db: rocksdb::DB,
-    bulk_import: AtomicBool,
-}
-
-const CONFIG_CF: &str = "config";
-const HEADERS_CF: &str = "headers";
-const TXID_CF: &str = "txid";
-const FUNDING_CF: &str = "funding";
-const SPENDING_CF: &str = "spending";
-
-const COLUMN_FAMILIES: &[&str] = &[CONFIG_CF, HEADERS_CF, TXID_CF, FUNDING_CF, SPENDING_CF];
-
-const CONFIG_KEY: &str = "C";
-const TIP_KEY: &[u8] = b"T";
-
-// Taken from https://github.com/facebook/rocksdb/blob/master/include/rocksdb/db.h#L654-L689
-const DB_PROPERTIES: &[&str] = &[
-    "rocksdb.num-immutable-mem-table",
-    "rocksdb.mem-table-flush-pending",
-    "rocksdb.compaction-pending",
-    "rocksdb.background-errors",
-    "rocksdb.cur-size-active-mem-table",
-    "rocksdb.cur-size-all-mem-tables",
-    "rocksdb.size-all-mem-tables",
-    "rocksdb.num-entries-active-mem-table",
-    "rocksdb.num-entries-imm-mem-tables",
-    "rocksdb.num-deletes-active-mem-table",
-    "rocksdb.num-deletes-imm-mem-tables",
-    "rocksdb.estimate-num-keys",
-    "rocksdb.estimate-table-readers-mem",
-    "rocksdb.is-file-deletions-enabled",
-    "rocksdb.num-snapshots",
-    "rocksdb.oldest-snapshot-time",
-    "rocksdb.num-live-versions",
-    "rocksdb.current-super-version-number",
-    "rocksdb.estimate-live-data-size",
-    "rocksdb.min-log-number-to-keep",
-    "rocksdb.min-obsolete-sst-number-to-keep",
-    "rocksdb.total-sst-files-size",
-    "rocksdb.live-sst-files-size",
-    "rocksdb.base-level",
-    "rocksdb.estimate-pending-compaction-bytes",
-    "rocksdb.num-running-compactions",
-    "rocksdb.num-running-flushes",
-    "rocksdb.actual-delayed-write-rate",
-    "rocksdb.is-write-stopped",
-    "rocksdb.estimate-oldest-key-time",
-    "rocksdb.block-cache-capacity",
-    "rocksdb.block-cache-usage",
-    "rocksdb.block-cache-pinned-usage",
-];
-
-#[derive(Debug, Deserialize, Serialize)]
-struct Config {
-    compacted: bool,
-    format: u64,
-}
-
-const CURRENT_FORMAT: u64 = 0;
-
-impl Default for Config {
-    fn default() -> Self {
-        Config {
-            compacted: false,
-            format: CURRENT_FORMAT,
-        }
-    }
-}
-
-fn default_opts() -> rocksdb::Options {
-    let mut block_opts = rocksdb::BlockBasedOptions::default();
-    block_opts.set_checksum_type(rocksdb::ChecksumType::CRC32c);
-
-    let mut opts = rocksdb::Options::default();
-    opts.set_keep_log_file_num(10);
-    opts.set_max_open_files(16);
-    opts.set_compaction_style(rocksdb::DBCompactionStyle::Level);
-    opts.set_compression_type(rocksdb::DBCompressionType::Zstd);
-    opts.set_target_file_size_base(256 << 20);
-    opts.set_write_buffer_size(256 << 20);
-    opts.set_disable_auto_compactions(true); // for initial bulk load
-    opts.set_advise_random_on_open(false); // bulk load uses sequential I/O
-    opts.set_prefix_extractor(rocksdb::SliceTransform::create_fixed_prefix(8));
-    opts.set_block_based_table_factory(&block_opts);
-    opts
-}
-
-impl DBStore {
-    fn create_cf_descriptors() -> Vec<rocksdb::ColumnFamilyDescriptor> {
-        COLUMN_FAMILIES
-            .iter()
-            .map(|&name| rocksdb::ColumnFamilyDescriptor::new(name, default_opts()))
-            .collect()
-    }
-
-    fn open_internal(path: &Path, log_dir: Option<&Path>) -> Result<Self> {
-        let mut db_opts = default_opts();
-        db_opts.create_if_missing(true);
-        db_opts.create_missing_column_families(true);
-        if let Some(d) = log_dir {
-            db_opts.set_db_log_dir(d);
-        }
-
-        let db = rocksdb::DB::open_cf_descriptors(&db_opts, path, Self::create_cf_descriptors())
-            .with_context(|| format!("failed to open DB: {}", path.display()))?;
-        let live_files = db.live_files()?;
-        info!(
-            "{:?}: {} SST files, {} GB, {} Grows",
-            path,
-            live_files.len(),
-            live_files.iter().map(|f| f.size).sum::<usize>() as f64 / 1e9,
-            live_files.iter().map(|f| f.num_entries).sum::<u64>() as f64 / 1e9
-        );
-        let store = DBStore {
-            db,
-            bulk_import: AtomicBool::new(true),
-        };
-        Ok(store)
-    }
-
-    fn is_legacy_format(&self) -> bool {
-        // In legacy DB format, all data was stored in a single (default) column family.
-        self.db
-            .iterator(rocksdb::IteratorMode::Start)
-            .next()
-            .is_some()
-    }
-
-    /// Opens a new RocksDB at the specified location.
-    pub fn open(path: &Path, log_dir: Option<&Path>, auto_reindex: bool) -> Result<Self> {
-        let mut store = Self::open_internal(path, log_dir)?;
-        let config = store.get_config();
-        debug!("DB {:?}", config);
-        let mut config = config.unwrap_or_default(); // use default config when DB is empty
-
-        let reindex_cause = if store.is_legacy_format() {
-            Some("legacy format".to_owned())
-        } else if config.format != CURRENT_FORMAT {
-            Some(format!(
-                "unsupported format {} != {}",
-                config.format, CURRENT_FORMAT
-            ))
-        } else {
-            None
-        };
-        if let Some(cause) = reindex_cause {
-            if !auto_reindex {
-                bail!("re-index required due to {}", cause);
-            }
-            warn!(
-                "Database needs to be re-indexed due to {}, going to delete {}",
-                cause,
-                path.display()
-            );
-            // close DB before deletion
-            drop(store);
-            rocksdb::DB::destroy(&default_opts(), path).with_context(|| {
-                format!(
-                    "re-index required but the old database ({}) can not be deleted",
-                    path.display()
-                )
-            })?;
-            store = Self::open_internal(path, log_dir)?;
-            config = Config::default(); // re-init config after dropping DB
-        }
-        if config.compacted {
-            store.start_compactions();
-        }
-        store.set_config(config);
-        Ok(store)
-    }
-
-    fn config_cf(&self) -> &rocksdb::ColumnFamily {
-        self.db.cf_handle(CONFIG_CF).expect("missing CONFIG_CF")
-    }
-
-    fn funding_cf(&self) -> &rocksdb::ColumnFamily {
-        self.db.cf_handle(FUNDING_CF).expect("missing FUNDING_CF")
-    }
-
-    fn spending_cf(&self) -> &rocksdb::ColumnFamily {
-        self.db.cf_handle(SPENDING_CF).expect("missing SPENDING_CF")
-    }
-
-    fn txid_cf(&self) -> &rocksdb::ColumnFamily {
-        self.db.cf_handle(TXID_CF).expect("missing TXID_CF")
-    }
-
-    fn headers_cf(&self) -> &rocksdb::ColumnFamily {
-        self.db.cf_handle(HEADERS_CF).expect("missing HEADERS_CF")
-    }
-
-    pub(crate) fn iter_funding(
-        &self,
-        prefix: HashPrefix,
-    ) -> impl Iterator<Item = SerializedHashPrefixRow> + '_ {
-        self.iter_prefix_cf(self.funding_cf(), prefix)
-    }
-
-    pub(crate) fn iter_spending(
-        &self,
-        prefix: HashPrefix,
-    ) -> impl Iterator<Item = SerializedHashPrefixRow> + '_ {
-        self.iter_prefix_cf(self.spending_cf(), prefix)
-    }
-
-    pub(crate) fn iter_txid(
-        &self,
-        prefix: HashPrefix,
-    ) -> impl Iterator<Item = SerializedHashPrefixRow> + '_ {
-        self.iter_prefix_cf(self.txid_cf(), prefix)
-    }
-
-    fn iter_cf<const N: usize>(
-        &self,
-        cf: &rocksdb::ColumnFamily,
-        readopts: rocksdb::ReadOptions,
-        prefix: Option<HashPrefix>,
-    ) -> impl Iterator<Item = [u8; N]> + '_ {
-        DBIterator::new(self.db.raw_iterator_cf_opt(cf, readopts), prefix)
-    }
-
-    fn iter_prefix_cf(
-        &self,
-        cf: &rocksdb::ColumnFamily,
-        prefix: HashPrefix,
-    ) -> impl Iterator<Item = SerializedHashPrefixRow> + '_ {
-        let mut opts = rocksdb::ReadOptions::default();
-        opts.set_prefix_same_as_start(true); // requires .set_prefix_extractor() above.
-        self.iter_cf(cf, opts, Some(prefix))
-    }
-
-    pub(crate) fn iter_headers(&self) -> impl Iterator<Item = SerializedHeaderRow> + '_ {
-        let mut opts = rocksdb::ReadOptions::default();
-        opts.fill_cache(false);
-        self.iter_cf(self.headers_cf(), opts, None)
-    }
-
-    pub(crate) fn get_tip(&self) -> Option<Vec<u8>> {
-        self.db
-            .get_cf(self.headers_cf(), TIP_KEY)
-            .expect("get_tip failed")
-    }
-
-    pub(crate) fn write(&self, batch: &WriteBatch) {
-        let mut db_batch = rocksdb::WriteBatch::default();
-        for key in &batch.funding_rows {
-            db_batch.put_cf(self.funding_cf(), key, b"");
-        }
-        for key in &batch.spending_rows {
-            db_batch.put_cf(self.spending_cf(), key, b"");
-        }
-        for key in &batch.txid_rows {
-            db_batch.put_cf(self.txid_cf(), key, b"");
-        }
-        for key in &batch.header_rows {
-            db_batch.put_cf(self.headers_cf(), key, b"");
-        }
-        db_batch.put_cf(self.headers_cf(), TIP_KEY, batch.tip_row);
-
-        let mut opts = rocksdb::WriteOptions::new();
-        let bulk_import = self.bulk_import.load(Ordering::Relaxed);
-        opts.set_sync(!bulk_import);
-        opts.disable_wal(bulk_import);
-        self.db.write_opt(db_batch, &opts).unwrap();
-    }
-
-    pub(crate) fn flush(&self) {
-        debug!("flushing DB column families");
-        let mut config = self.get_config().unwrap_or_default();
-        for name in COLUMN_FAMILIES {
-            let cf = self.db.cf_handle(name).expect("missing CF");
-            self.db.flush_cf(cf).expect("CF flush failed");
-        }
-        if !config.compacted {
-            for name in COLUMN_FAMILIES {
-                info!("starting {} compaction", name);
-                let cf = self.db.cf_handle(name).expect("missing CF");
-                self.db.compact_range_cf(cf, None::<&[u8]>, None::<&[u8]>);
-            }
-            config.compacted = true;
-            self.set_config(config);
-            info!("finished full compaction");
-            self.start_compactions();
-        }
-        if log_enabled!(log::Level::Trace) {
-            let stats = self
-                .db
-                .property_value("rocksdb.dbstats")
-                .expect("failed to get property")
-                .expect("missing property");
-            trace!("RocksDB stats: {}", stats);
-        }
-    }
-
-    pub(crate) fn get_properties(
-        &self,
-    ) -> impl Iterator<Item = (&'static str, &'static str, u64)> + '_ {
-        COLUMN_FAMILIES.iter().flat_map(move |cf_name| {
-            let cf = self.db.cf_handle(cf_name).expect("missing CF");
-            DB_PROPERTIES.iter().filter_map(move |property_name| {
-                let value = self
-                    .db
-                    .property_int_value_cf(cf, *property_name)
-                    .expect("failed to get property");
-                Some((*cf_name, *property_name, value?))
-            })
-        })
-    }
-
-    fn start_compactions(&self) {
-        self.bulk_import.store(false, Ordering::Relaxed);
-        for name in COLUMN_FAMILIES {
-            let cf = self.db.cf_handle(name).expect("missing CF");
-            self.db
-                .set_options_cf(cf, &[("disable_auto_compactions", "false")])
-                .expect("failed to start auto-compactions");
-        }
-        debug!("auto-compactions enabled");
-    }
-
-    fn set_config(&self, config: Config) {
-        let mut opts = rocksdb::WriteOptions::default();
-        opts.set_sync(true);
-        opts.disable_wal(false);
-        let value = serde_json::to_vec(&config).expect("failed to serialize config");
-        self.db
-            .put_cf_opt(self.config_cf(), CONFIG_KEY, value, &opts)
-            .expect("DB::put failed");
-    }
-
-    fn get_config(&self) -> Option<Config> {
-        self.db
-            .get_cf(self.config_cf(), CONFIG_KEY)
-            .expect("DB::get failed")
-            .map(|value| serde_json::from_slice(&value).expect("failed to deserialize Config"))
-    }
-}
-
-struct DBIterator<'a, const N: usize> {
-    raw: rocksdb::DBRawIterator<'a>,
-    prefix: Option<HashPrefix>,
-    done: bool,
-}
-
-impl<'a, const N: usize> DBIterator<'a, N> {
-    fn new(mut raw: rocksdb::DBRawIterator<'a>, prefix: Option<HashPrefix>) -> Self {
-        match prefix {
-            Some(key) => raw.seek(key),
-            None => raw.seek_to_first(),
-        };
-        Self {
-            raw,
-            prefix,
-            done: false,
-        }
-    }
-}
-
-impl<const N: usize> Iterator for DBIterator<'_, N> {
-    type Item = [u8; N];
-
-    fn next(&mut self) -> Option<Self::Item> {
-        while !self.done {
-            let key = match self.raw.key() {
-                Some(key) => key,
-                None => {
-                    self.raw.status().expect("DB scan failed");
-                    break; // end of scan
-                }
-            };
-            let prefix_match = match self.prefix {
-                Some(key_prefix) => key.starts_with(&key_prefix),
-                None => true,
-            };
-            if !prefix_match {
-                break; // prefix mismatch
-            }
-            let result: Option<[u8; N]> = key.try_into().ok();
-            self.raw.next();
-            match result {
-                Some(value) => return Some(value),
-                None => continue, // skip keys with size != N
-            }
-        }
-        self.done = true;
-        None
-    }
-}
-
-impl Drop for DBStore {
-    fn drop(&mut self) {
-        info!("closing DB at {}", self.db.path().display());
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::{rocksdb, DBStore, WriteBatch, CURRENT_FORMAT};
-    use std::ffi::{OsStr, OsString};
-    use std::path::Path;
-
-    #[test]
-    fn test_reindex_new_format() {
-        let dir = tempfile::tempdir().unwrap();
-        {
-            let store = DBStore::open(dir.path(), None, false).unwrap();
-            let mut config = store.get_config().unwrap();
-            config.format += 1;
-            store.set_config(config);
-        };
-        assert_eq!(
-            DBStore::open(dir.path(), None, false)
-                .err()
-                .unwrap()
-                .to_string(),
-            format!(
-                "re-index required due to unsupported format {} != {}",
-                CURRENT_FORMAT + 1,
-                CURRENT_FORMAT
-            )
-        );
-        {
-            let store = DBStore::open(dir.path(), None, true).unwrap();
-            store.flush();
-            let config = store.get_config().unwrap();
-            assert_eq!(config.format, CURRENT_FORMAT);
-            assert!(!store.is_legacy_format());
-        }
-    }
-
-    #[test]
-    fn test_reindex_legacy_format() {
-        let dir = tempfile::tempdir().unwrap();
-        {
-            let mut db_opts = rocksdb::Options::default();
-            db_opts.create_if_missing(true);
-            let db = rocksdb::DB::open(&db_opts, dir.path()).unwrap();
-            db.put(b"F", b"").unwrap(); // insert legacy DB compaction marker (in 'default' column family)
-        };
-        assert_eq!(
-            DBStore::open(dir.path(), None, false)
-                .err()
-                .unwrap()
-                .to_string(),
-            format!("re-index required due to legacy format",)
-        );
-        {
-            let store = DBStore::open(dir.path(), None, true).unwrap();
-            store.flush();
-            let config = store.get_config().unwrap();
-            assert_eq!(config.format, CURRENT_FORMAT);
-        }
-    }
-
-    #[test]
-    fn test_db_prefix_scan() {
-        let dir = tempfile::tempdir().unwrap();
-        let store = DBStore::open(dir.path(), None, true).unwrap();
-
-        let items = [
-            *b"ab          ",
-            *b"abcdefgh    ",
-            *b"abcdefghj   ",
-            *b"abcdefghjk  ",
-            *b"abcdefghxyz ",
-            *b"abcdefgi    ",
-            *b"b           ",
-            *b"c           ",
-        ];
-
-        store.write(&WriteBatch {
-            txid_rows: items.to_vec(),
-            ..Default::default()
-        });
-
-        let rows = store.iter_txid(*b"abcdefgh");
-        assert_eq!(rows.collect::<Vec<_>>(), items[1..5]);
-    }
-
-    #[test]
-    fn test_db_log_in_same_dir() {
-        let dir1 = tempfile::tempdir().unwrap();
-        let _store = DBStore::open(dir1.path(), None, true).unwrap();
-
-        // LOG file is created in dir1
-        let dir_files = list_log_files(dir1.path());
-        assert_eq!(dir_files, vec![OsStr::new("LOG")]);
-
-        let dir2 = tempfile::tempdir().unwrap();
-        let dir3 = tempfile::tempdir().unwrap();
-        let _store = DBStore::open(dir2.path(), Some(dir3.path()), true).unwrap();
-
-        // *_LOG file is not created in dir2, but in dir3
-        let dir_files = list_log_files(dir2.path());
-        assert_eq!(dir_files, Vec::<OsString>::new());
-
-        let dir_files = list_log_files(dir3.path());
-        assert_eq!(dir_files.len(), 1);
-        assert!(dir_files[0].to_str().unwrap().ends_with("_LOG"));
-    }
-
-    fn list_log_files(path: &Path) -> Vec<OsString> {
-        path.read_dir()
-            .unwrap()
-            .map(|e| e.unwrap().file_name())
-            .filter(|e| e.to_str().unwrap().contains("LOG"))
-            .collect()
-    }
-}

src/electrum.rs

@@ -1,11 +1,11 @@
-use anyhow::{bail, Context, Result};
-use bitcoin::{
+use crate::bitcoin::{
     consensus::{deserialize, encode::serialize_hex},
     hashes::hex::FromHex,
     hex::DisplayHex,
-    BlockHash, Txid,
+    BlockHash, Transaction, Txid,
 };
-use crossbeam_channel::Receiver;
+use anyhow::{bail, Context, Result};
+use bindex::ScriptHash;
 use rayon::prelude::*;
 use serde_derive::Deserialize;
 use serde_json::{self, json, Value};
@@ -13,18 +13,17 @@ use serde_json::{self, json, Value};
 use std::collections::{hash_map::Entry, HashMap};
 use std::fmt;
 use std::iter::FromIterator;
+use std::net::SocketAddr;
 use std::str::FromStr;
 
 use crate::{
-    cache::Cache,
     config::{Config, ELECTRS_VERSION},
-    daemon::{self, extract_bitcoind_error, Daemon},
+    daemon::{extract_bitcoind_error, Daemon},
     merkle::Proof,
     metrics::{self, Histogram, Metrics},
     signals::Signal,
     status::ScriptHashStatus,
     tracker::Tracker,
-    types::ScriptHash,
 };
 
 const PROTOCOL_VERSION: &str = "1.4";
@@ -78,6 +77,29 @@ impl From<&TxGetArgs> for (Txid, bool) {
     }
 }
 
+#[derive(Deserialize)]
+#[serde(untagged)]
+enum BroadcastArgs {
+    Package((Vec<String>,)),
+    PackageVerbose((Vec<String>, bool)),
+}
+
+impl BroadcastArgs {
+    fn txs(&self) -> &[String] {
+        match self {
+            BroadcastArgs::Package((txs,)) => txs,
+            BroadcastArgs::PackageVerbose((txs, _)) => txs,
+        }
+    }
+
+    fn verbose(&self) -> bool {
+        match self {
+            BroadcastArgs::Package(_) => false,
+            BroadcastArgs::PackageVerbose((_, verbose)) => *verbose,
+        }
+    }
+}
+
 enum StandardError {
     ParseError,
     InvalidRequest,
@@ -90,7 +112,7 @@ enum RpcError {
     Standard(StandardError),
     // Electrum-specific errors
     BadRequest(anyhow::Error),
-    DaemonError(daemon::RpcError),
+    DaemonError(jsonrpc::error::RpcError),
     UnavailableIndex,
 }
 
@@ -122,12 +144,11 @@ impl RpcError {
 /// Electrum RPC handler
 pub struct Rpc {
     tracker: Tracker,
-    cache: Cache,
     rpc_duration: Histogram,
     daemon: Daemon,
     signal: Signal,
     banner: String,
-    port: u16,
+    addr: SocketAddr,
 }
 
 impl Rpc {
@@ -142,16 +163,14 @@ impl Rpc {
 
         let tracker = Tracker::new(config, metrics)?;
         let signal = Signal::new();
-        let daemon = Daemon::connect(config, signal.exit_flag(), tracker.metrics())?;
-        let cache = Cache::new(tracker.metrics());
+        let daemon = Daemon::connect(config, signal.exit_flag())?;
         Ok(Self {
             tracker,
-            cache,
             rpc_duration,
             daemon,
             signal,
             banner: config.server_banner.clone(),
-            port: config.electrum_rpc_addr.port(),
+            addr: config.electrum_rpc_addr,
         })
     }
 
@@ -159,24 +178,17 @@ impl Rpc {
         &self.signal
     }
 
-    pub fn new_block_notification(&self) -> Receiver<()> {
-        self.daemon.new_block_notification()
-    }
-
     pub fn sync(&mut self) -> Result<bool> {
         self.tracker.sync(&self.daemon, self.signal.exit_flag())
     }
 
     pub fn update_client(&self, client: &mut Client) -> Result<Vec<String>> {
-        let chain = self.tracker.chain();
+        let headers = self.tracker.headers();
         let mut notifications = client
             .scripthashes
             .par_iter_mut()
             .filter_map(|(scripthash, status)| -> Option<Result<Value>> {
-                match self
-                    .tracker
-                    .update_scripthash_status(status, &self.daemon, &self.cache)
-                {
+                match self.tracker.update_scripthash_status(status) {
                     Ok(true) => Some(Ok(notification(
                         "blockchain.scripthash.subscribe",
                         &[json!(scripthash), json!(status.statushash())],
@@ -189,11 +201,11 @@ impl Rpc {
             .context("failed to update status")?;
 
         if let Some(old_tip) = client.tip {
-            let new_tip = self.tracker.chain().tip();
-            if old_tip != new_tip {
-                client.tip = Some(new_tip);
-                let height = chain.height();
-                let header = chain.get_block_header(height).unwrap();
+            let new_tip = headers.tip().unwrap();
+            if old_tip != new_tip.hash() {
+                client.tip = Some(new_tip.hash());
+                let height = headers.tip_height().unwrap();
+                let header = new_tip.header();
                 notifications.push(notification(
                     "blockchain.headers.subscribe",
                     &[json!({"hex": serialize_hex(&header), "height": height})],
@@ -204,34 +216,36 @@ impl Rpc {
     }
 
     fn headers_subscribe(&self, client: &mut Client) -> Result<Value> {
-        let chain = self.tracker.chain();
-        client.tip = Some(chain.tip());
-        let height = chain.height();
-        let header = chain.get_block_header(height).unwrap();
-        Ok(json!({"hex": serialize_hex(header), "height": height}))
+        let headers = self.tracker.headers();
+        let height = headers.tip_height().unwrap();
+        let header = headers.tip().unwrap();
+        client.tip = Some(header.hash());
+        Ok(json!({"hex": serialize_hex(header.header()), "height": height}))
     }
 
     fn block_header(&self, (height,): (usize,)) -> Result<Value> {
-        let chain = self.tracker.chain();
-        let header = match chain.get_block_header(height) {
+        let header = match self.tracker.headers().iter_headers().nth(height) {
             None => bail!("no header at {}", height),
             Some(header) => header,
         };
-        Ok(json!(serialize_hex(header)))
+        Ok(json!(serialize_hex(header.header())))
     }
 
     fn block_headers(&self, (start_height, count): (usize, usize)) -> Result<Value> {
-        let chain = self.tracker.chain();
+        let headers = self.tracker.headers();
         let max_count = 2016usize;
         // return only the available block headers
         let end_height = std::cmp::min(
-            chain.height() + 1,
+            headers.tip_height().map_or(0, |h| h + 1),
             start_height + std::cmp::min(count, max_count),
         );
         let heights = start_height..end_height;
         let count = heights.len();
-        let hex_headers =
-            heights.filter_map(|height| chain.get_block_header(height).map(serialize_hex));
+        let hex_headers = headers
+            .iter_headers()
+            .skip(start_height)
+            .take(count)
+            .map(|h| serialize_hex(h.header()));
 
         Ok(json!({"count": count, "hex": String::from_iter(hex_headers), "max": max_count}))
     }
@@ -353,52 +367,70 @@ impl Rpc {
 
     fn new_status(&self, scripthash: ScriptHash) -> Result<ScriptHashStatus> {
         let mut status = ScriptHashStatus::new(scripthash);
-        self.tracker
-            .update_scripthash_status(&mut status, &self.daemon, &self.cache)?;
+        self.tracker.update_scripthash_status(&mut status)?;
         Ok(status)
     }
 
     fn transaction_broadcast(&self, (tx_hex,): &(String,)) -> Result<Value> {
-        let tx_bytes = Vec::from_hex(tx_hex).context("non-hex transaction")?;
-        let tx = deserialize(&tx_bytes).context("invalid transaction")?;
-        let txid = self.daemon.broadcast(&tx)?;
+        let txid = self.daemon.broadcast(&tx_from_hex(tx_hex)?)?;
         Ok(json!(txid))
     }
 
+    fn transaction_broadcast_package(&self, args: &BroadcastArgs) -> Result<Value> {
+        let txs: Vec<Transaction> = args
+            .txs()
+            .iter()
+            .map(|s| tx_from_hex(s))
+            .collect::<Result<_>>()?;
+        let response = self.daemon.submitpackage(&txs)?;
+        if args.verbose() {
+            return Ok(response);
+        }
+        let build_result = || -> Option<Value> {
+            let success = response.get("package_msg")? == &json!("success");
+
+            let mut errors = vec![];
+            for tx in response.get("tx-results")?.as_object()?.values() {
+                let tx_obj = tx.as_object()?;
+                if let Some(error) = tx_obj.get("error") {
+                    let txid = tx_obj.get("txid");
+                    errors.push(json!({"error": error, "txid": txid}));
+                }
+            }
+            Some(if errors.is_empty() {
+                json!({"success": success})
+            } else {
+                json!({"success": success, "errors": errors})
+            })
+        };
+        build_result().ok_or(anyhow!("Unexpected `submitpackage` response"))
+    }
+
     fn transaction_get(&self, args: &TxGetArgs) -> Result<Value> {
         let (txid, verbose) = args.into();
         if verbose {
             let blockhash = self
                 .tracker
-                .lookup_transaction(&self.daemon, txid)?
+                .lookup_transaction(txid)?
                 .map(|(blockhash, _tx)| blockhash);
-            return self.daemon.get_transaction_info(&txid, blockhash);
+            return self.daemon.get_transaction(&txid, blockhash, verbose);
         }
-        // if the scripthash was subscribed, tx should be cached
-        if let Some(tx_hex) = self
-            .cache
-            .get_tx(&txid, |tx_bytes| tx_bytes.to_lower_hex_string())
-        {
-            return Ok(json!(tx_hex));
-        }
-        debug!("tx cache miss: txid={}", txid);
         // use internal index to load confirmed transaction
         if let Some(tx_hex) = self
             .tracker
-            .lookup_transaction(&self.daemon, txid)?
+            .lookup_transaction(txid)?
             .map(|(_blockhash, tx)| tx.to_lower_hex_string())
         {
             return Ok(json!(tx_hex));
         }
         // load unconfirmed transaction via RPC
-        Ok(json!(self.daemon.get_transaction_hex(&txid, None)?))
+        Ok(json!(self.daemon.get_transaction(&txid, None, verbose)?))
     }
 
     fn transaction_get_merkle(&self, (txid, height): &(Txid, usize)) -> Result<Value> {
-        let chain = self.tracker.chain();
-        let blockhash = match chain.get_block_hash(*height) {
+        let blockhash = match self.tracker.headers().iter_headers().nth(*height) {
             None => bail!("missing block at {}", height),
-            Some(blockhash) => blockhash,
+            Some(header) => header.hash(),
         };
         let txids = self.daemon.get_block_txids(blockhash)?;
         match txids.iter().position(|current_txid| *current_txid == *txid) {
@@ -418,10 +450,9 @@ impl Rpc {
         &self,
         (height, tx_pos, merkle): (usize, usize, bool),
     ) -> Result<Value> {
-        let chain = self.tracker.chain();
-        let blockhash = match chain.get_block_hash(height) {
+        let blockhash = match self.tracker.headers().iter_headers().nth(height) {
             None => bail!("missing block at {}", height),
-            Some(blockhash) => blockhash,
+            Some(header) => header.hash(),
         };
         let txids = self.daemon.get_block_txids(blockhash)?;
         if tx_pos >= txids.len() {
@@ -430,9 +461,9 @@ impl Rpc {
         let txid: Txid = txids[tx_pos];
         if merkle {
             let proof = Proof::create(&txids, tx_pos);
-            Ok(json!({"tx_id": txid, "merkle": proof.to_hex()}))
+            Ok(json!({"tx_hash": txid, "merkle": proof.to_hex()}))
         } else {
-            Ok(json!({ "tx_id": txid }))
+            Ok(json!({ "tx_hash": txid }))
         }
     }
 
@@ -455,8 +486,12 @@ impl Rpc {
 
     fn features(&self) -> Result<Value> {
         Ok(json!({
-            "genesis_hash": self.tracker.chain().get_block_hash(0),
-            "hosts": { "tcp_port": self.port },
+            "genesis_hash": self.tracker.headers().genesis().unwrap().hash(),
+            "hosts": {
+                self.addr.ip().to_string(): {
+                    "tcp_port": self.addr.port()
+                }
+            },
             "protocol_max": PROTOCOL_VERSION,
             "protocol_min": PROTOCOL_VERSION,
             "pruning": null,
@@ -562,6 +597,9 @@ impl Rpc {
                 Params::ScriptHashSubscribe(args) => self.scripthash_subscribe(client, args),
                 Params::ScriptHashUnsubscribe(args) => self.scripthash_unsubscribe(client, args),
                 Params::TransactionBroadcast(args) => self.transaction_broadcast(args),
+                Params::TransactionBroadcastPackage(args) => {
+                    self.transaction_broadcast_package(args)
+                }
                 Params::TransactionGet(args) => self.transaction_get(args),
                 Params::TransactionGetMerkle(args) => self.transaction_get_merkle(args),
                 Params::TransactionFromPosition(args) => self.transaction_from_pos(*args),
@@ -578,6 +616,7 @@ enum Params {
     BlockHeader((usize,)),
     BlockHeaders((usize, usize)),
     TransactionBroadcast((String,)),
+    TransactionBroadcastPackage(BroadcastArgs),
     Donation,
     EstimateFee((u16,)),
     Features,
@@ -611,6 +650,9 @@ impl Params {
             "blockchain.scripthash.subscribe" => Params::ScriptHashSubscribe(convert(params)?),
             "blockchain.scripthash.unsubscribe" => Params::ScriptHashUnsubscribe(convert(params)?),
             "blockchain.transaction.broadcast" => Params::TransactionBroadcast(convert(params)?),
+            "blockchain.transaction.broadcast_package" => {
+                Params::TransactionBroadcastPackage(convert(params)?)
+            }
             "blockchain.transaction.get" => Params::TransactionGet(convert(params)?),
             "blockchain.transaction.get_merkle" => Params::TransactionGetMerkle(convert(params)?),
             "blockchain.transaction.id_from_pos" => {
@@ -655,7 +697,7 @@ impl Call {
             Err(err) => {
                 warn!("RPC {} failed: {:#}", self.method, err);
                 match err
-                    .downcast_ref::<bitcoincore_rpc::Error>()
+                    .downcast_ref::<jsonrpc::Error>()
                     .and_then(extract_bitcoind_error)
                 {
                     Some(e) => error_msg(&self.id, RpcError::DaemonError(e.clone())),
@@ -763,9 +805,15 @@ fn check_between(version_str: &str, min_str: &str, max_str: &str) -> Result<()>
     Ok(())
 }
 
+fn tx_from_hex(tx_hex: &str) -> Result<Transaction> {
+    let tx_bytes = Vec::from_hex(tx_hex).context("non-hex transaction")?;
+    let tx = deserialize(&tx_bytes).context("invalid transaction")?;
+    Ok(tx)
+}
+
 #[cfg(test)]
 mod tests {
-    use super::{check_between, parse_version, Version};
+    use super::*;
 
     #[test]
     fn test_version() {
@@ -788,4 +836,19 @@ mod tests {
         assert!(check_between("1.4", "1.4.1", "1.5").is_err());
         assert!(check_between("1.4", "1", "1").is_err());
     }
+
+    #[test]
+    fn test_requests() {
+        assert!(matches!(
+            parse_requests("foo"),
+            Err(StandardError::ParseError)
+        ));
+        assert!(matches!(
+            parse_requests(r"{}"),
+            Err(StandardError::InvalidRequest)
+        ));
+        assert!(parse_requests(r#"{"id":1,"method":"name","params":[]}"#).is_ok());
+        assert!(parse_requests(r#"{"id":1,"method":"name","params":[],"unrelated":42}"#).is_ok());
+        assert!(parse_requests(r#" { "id" : 1 , "method" : "name" , "params" : [ ] } "#).is_ok());
+    }
 }

src/index.rs

@@ -1,317 +0,0 @@
-use anyhow::{Context, Result};
-use bitcoin::consensus::{deserialize, Decodable, Encodable};
-use bitcoin::hashes::Hash;
-use bitcoin::{BlockHash, OutPoint, Txid};
-use bitcoin_slices::{bsl, Visit, Visitor};
-use std::ops::ControlFlow;
-use std::thread;
-
-use crate::{
-    chain::{Chain, NewHeader},
-    daemon::Daemon,
-    db::{DBStore, WriteBatch},
-    metrics::{self, Gauge, Histogram, Metrics},
-    signals::ExitFlag,
-    types::{
-        bsl_txid, HashPrefixRow, HeaderRow, ScriptHash, ScriptHashRow, SerBlock, SpendingPrefixRow,
-        TxidRow,
-    },
-};
-
-#[derive(Clone)]
-struct Stats {
-    update_duration: Histogram,
-    update_size: Histogram,
-    height: Gauge,
-    db_properties: Gauge,
-}
-
-impl Stats {
-    fn new(metrics: &Metrics) -> Self {
-        Self {
-            update_duration: metrics.histogram_vec(
-                "index_update_duration",
-                "Index update duration (in seconds)",
-                "step",
-                metrics::default_duration_buckets(),
-            ),
-            update_size: metrics.histogram_vec(
-                "index_update_size",
-                "Index update size (in bytes)",
-                "step",
-                metrics::default_size_buckets(),
-            ),
-            height: metrics.gauge("index_height", "Indexed block height", "type"),
-            db_properties: metrics.gauge("index_db_properties", "Index DB properties", "name"),
-        }
-    }
-
-    fn observe_duration<T>(&self, label: &str, f: impl FnOnce() -> T) -> T {
-        self.update_duration.observe_duration(label, f)
-    }
-
-    fn observe_size<const N: usize>(&self, label: &str, rows: &[[u8; N]]) {
-        self.update_size.observe(label, (rows.len() * N) as f64);
-    }
-
-    fn observe_batch(&self, batch: &WriteBatch) {
-        self.observe_size("write_funding_rows", &batch.funding_rows);
-        self.observe_size("write_spending_rows", &batch.spending_rows);
-        self.observe_size("write_txid_rows", &batch.txid_rows);
-        self.observe_size("write_header_rows", &batch.header_rows);
-        debug!(
-            "writing {} funding and {} spending rows from {} transactions, {} blocks",
-            batch.funding_rows.len(),
-            batch.spending_rows.len(),
-            batch.txid_rows.len(),
-            batch.header_rows.len()
-        );
-    }
-
-    fn observe_chain(&self, chain: &Chain) {
-        self.height.set("tip", chain.height() as f64);
-    }
-
-    fn observe_db(&self, store: &DBStore) {
-        for (cf, name, value) in store.get_properties() {
-            self.db_properties
-                .set(&format!("{}:{}", name, cf), value as f64);
-        }
-    }
-}
-
-/// Confirmed transactions' address index
-pub struct Index {
-    store: DBStore,
-    batch_size: usize,
-    lookup_limit: Option<usize>,
-    chain: Chain,
-    stats: Stats,
-    is_ready: bool,
-    flush_needed: bool,
-}
-
-impl Index {
-    pub(crate) fn load(
-        store: DBStore,
-        mut chain: Chain,
-        metrics: &Metrics,
-        batch_size: usize,
-        lookup_limit: Option<usize>,
-        reindex_last_blocks: usize,
-    ) -> Result<Self> {
-        if let Some(row) = store.get_tip() {
-            let tip = deserialize(&row).expect("invalid tip");
-            let headers = store
-                .iter_headers()
-                .map(|row| HeaderRow::from_db_row(row).header);
-            chain.load(headers, tip);
-            chain.drop_last_headers(reindex_last_blocks);
-        };
-        let stats = Stats::new(metrics);
-        stats.observe_chain(&chain);
-        stats.observe_db(&store);
-        Ok(Index {
-            store,
-            batch_size,
-            lookup_limit,
-            chain,
-            stats,
-            is_ready: false,
-            flush_needed: false,
-        })
-    }
-
-    pub(crate) fn chain(&self) -> &Chain {
-        &self.chain
-    }
-
-    pub(crate) fn limit_result<T>(&self, entries: impl Iterator<Item = T>) -> Result<Vec<T>> {
-        let mut entries = entries.fuse();
-        let result: Vec<T> = match self.lookup_limit {
-            Some(lookup_limit) => entries.by_ref().take(lookup_limit).collect(),
-            None => entries.by_ref().collect(),
-        };
-        if entries.next().is_some() {
-            bail!(">{} index entries, query may take too long", result.len())
-        }
-        Ok(result)
-    }
-
-    pub(crate) fn filter_by_txid(&self, txid: Txid) -> impl Iterator<Item = BlockHash> + '_ {
-        self.store
-            .iter_txid(TxidRow::scan_prefix(txid))
-            .map(|row| HashPrefixRow::from_db_row(row).height())
-            .filter_map(move |height| self.chain.get_block_hash(height))
-    }
-
-    pub(crate) fn filter_by_funding(
-        &self,
-        scripthash: ScriptHash,
-    ) -> impl Iterator<Item = BlockHash> + '_ {
-        self.store
-            .iter_funding(ScriptHashRow::scan_prefix(scripthash))
-            .map(|row| HashPrefixRow::from_db_row(row).height())
-            .filter_map(move |height| self.chain.get_block_hash(height))
-    }
-
-    pub(crate) fn filter_by_spending(
-        &self,
-        outpoint: OutPoint,
-    ) -> impl Iterator<Item = BlockHash> + '_ {
-        self.store
-            .iter_spending(SpendingPrefixRow::scan_prefix(outpoint))
-            .map(|row| HashPrefixRow::from_db_row(row).height())
-            .filter_map(move |height| self.chain.get_block_hash(height))
-    }
-
-    // Return `Ok(true)` when the chain is fully synced and the index is compacted.
-    pub(crate) fn sync(&mut self, daemon: &Daemon, exit_flag: &ExitFlag) -> Result<bool> {
-        let new_headers = self
-            .stats
-            .observe_duration("headers", || daemon.get_new_headers(&self.chain))?;
-        match (new_headers.first(), new_headers.last()) {
-            (Some(first), Some(last)) => {
-                let count = new_headers.len();
-                info!(
-                    "indexing {} blocks: [{}..{}]",
-                    count,
-                    first.height(),
-                    last.height()
-                );
-            }
-            _ => {
-                if self.flush_needed {
-                    self.store.flush(); // full compaction is performed on the first flush call
-                    self.flush_needed = false;
-                }
-                self.is_ready = true;
-                return Ok(true); // no more blocks to index (done for now)
-            }
-        }
-
-        thread::scope(|scope| -> Result<()> {
-            let (tx, rx) = crossbeam_channel::bounded(1);
-
-            let chunks = new_headers.chunks(self.batch_size);
-            let index = &self; // to be moved into reader thread
-            let reader = scope.spawn(move || -> Result<()> {
-                for chunk in chunks {
-                    exit_flag.poll().with_context(|| {
-                        format!(
-                            "indexing interrupted at height: {}",
-                            chunk.first().unwrap().height()
-                        )
-                    })?;
-                    let batch = index.index_blocks(daemon, chunk)?;
-                    tx.send(batch).context("writer disconnected")?;
-                }
-                Ok(()) // `tx` is dropped, to stop the iteration on `rx`
-            });
-
-            let index = &self; // to be moved into writer thread
-            let writer = scope.spawn(move || {
-                let stats = &index.stats;
-                for mut batch in rx {
-                    stats.observe_duration("sort", || batch.sort()); // pre-sort to optimize DB writes
-                    stats.observe_batch(&batch);
-                    stats.observe_duration("write", || index.store.write(&batch));
-                    stats.observe_db(&index.store);
-                }
-            });
-
-            reader.join().expect("reader thread panic")?;
-            writer.join().expect("writer thread panic");
-            Ok(())
-        })?;
-        self.chain.update(new_headers);
-        self.stats.observe_chain(&self.chain);
-        self.flush_needed = true;
-        Ok(false) // sync is not done
-    }
-
-    fn index_blocks(&self, daemon: &Daemon, chunk: &[NewHeader]) -> Result<WriteBatch> {
-        let blockhashes: Vec<BlockHash> = chunk.iter().map(|h| h.hash()).collect();
-        let mut heights = chunk.iter().map(|h| h.height());
-
-        let mut batch = WriteBatch::default();
-
-        daemon.for_blocks(blockhashes, |blockhash, block| {
-            let height = heights.next().expect("unexpected block");
-            self.stats.observe_duration("block", || {
-                index_single_block(blockhash, block, height, &mut batch);
-            });
-            self.stats.height.set("tip", height as f64);
-        })?;
-        let heights: Vec<_> = heights.collect();
-        assert!(
-            heights.is_empty(),
-            "some blocks were not indexed: {:?}",
-            heights
-        );
-        Ok(batch)
-    }
-
-    pub(crate) fn is_ready(&self) -> bool {
-        self.is_ready
-    }
-}
-
-fn index_single_block(
-    block_hash: BlockHash,
-    block: SerBlock,
-    height: usize,
-    batch: &mut WriteBatch,
-) {
-    struct IndexBlockVisitor<'a> {
-        batch: &'a mut WriteBatch,
-        height: usize,
-    }
-
-    impl Visitor for IndexBlockVisitor<'_> {
-        fn visit_transaction(&mut self, tx: &bsl::Transaction) -> ControlFlow<()> {
-            let txid = bsl_txid(tx);
-            self.batch
-                .txid_rows
-                .push(TxidRow::row(txid, self.height).to_db_row());
-            ControlFlow::Continue(())
-        }
-
-        fn visit_tx_out(&mut self, _vout: usize, tx_out: &bsl::TxOut) -> ControlFlow<()> {
-            let script = bitcoin::Script::from_bytes(tx_out.script_pubkey());
-            // skip indexing unspendable outputs
-            if !script.is_op_return() {
-                let row = ScriptHashRow::row(ScriptHash::new(script), self.height);
-                self.batch.funding_rows.push(row.to_db_row());
-            }
-            ControlFlow::Continue(())
-        }
-
-        fn visit_tx_in(&mut self, _vin: usize, tx_in: &bsl::TxIn) -> ControlFlow<()> {
-            let prevout: OutPoint = tx_in.prevout().into();
-            // skip indexing coinbase transactions' input
-            if !prevout.is_null() {
-                let row = SpendingPrefixRow::row(prevout, self.height);
-                self.batch.spending_rows.push(row.to_db_row());
-            }
-            ControlFlow::Continue(())
-        }
-
-        fn visit_block_header(&mut self, header: &bsl::BlockHeader) -> ControlFlow<()> {
-            let header = bitcoin::block::Header::consensus_decode(&mut header.as_ref())
-                .expect("block header was already validated");
-            self.batch
-                .header_rows
-                .push(HeaderRow::new(header).to_db_row());
-            ControlFlow::Continue(())
-        }
-    }
-
-    let mut index_block = IndexBlockVisitor { batch, height };
-    bsl::Block::visit(&block, &mut index_block).expect("core returned invalid block");
-
-    let len = block_hash
-        .consensus_encode(&mut (&mut batch.tip_row as &mut [u8]))
-        .expect("in-memory writers don't error");
-    debug_assert_eq!(len, BlockHash::LEN);
-}

src/lib.rs

@@ -7,17 +7,12 @@ extern crate log;
 #[macro_use]
 extern crate serde_derive;
 
-mod cache;
-mod chain;
 mod config;
 mod daemon;
-mod db;
 mod electrum;
-mod index;
 mod mempool;
 mod merkle;
 mod metrics;
-mod p2p;
 mod server;
 mod signals;
 mod status;
@@ -26,3 +21,6 @@ mod tracker;
 mod types;
 
 pub use server::run;
+
+use bindex::bitcoin;
+use bindex::bitcoin_slices;

src/mempool.rs

@@ -5,15 +5,15 @@ use std::convert::TryFrom;
 use std::iter::FromIterator;
 use std::ops::Bound;
 
-use bitcoin::hashes::Hash;
-use bitcoin::{Amount, OutPoint, Transaction, Txid};
+use crate::bitcoin::hashes::Hash;
+use crate::bitcoin::{Amount, OutPoint, Transaction, Txid};
+use bindex::ScriptHash;
 use serde::ser::{Serialize, SerializeSeq, Serializer};
 
 use crate::{
     daemon::Daemon,
     metrics::{Gauge, Metrics},
     signals::ExitFlag,
-    types::ScriptHash,
 };
 
 pub(crate) struct Entry {
@@ -80,7 +80,7 @@ impl MempoolSyncUpdate {
             );
             let chunk_entries: Vec<Entry> = txids_chunk
                 .iter()
-                .zip(entries.into_iter().zip(txs.into_iter()))
+                .zip(entries.into_iter().zip(txs))
                 .filter_map(|(txid, (entry, tx))| {
                     let entry = match entry {
                         Some(entry) => entry,
@@ -354,7 +354,7 @@ impl Serialize for FeeHistogram {
         S: Serializer,
     {
         let mut seq = serializer.serialize_seq(Some(self.vsize.len()))?;
-        // https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-methods.html#mempool-get-fee-histogram
+        // https://electrum-protocol.readthedocs.io/en/latest/protocol-methods.html#mempool-get-fee-histogram
         let fee_rates =
             (0..FeeHistogram::BINS).map(|i| u64::MAX.checked_shr(i as u32).unwrap_or(0));
         fee_rates
@@ -368,7 +368,7 @@ impl Serialize for FeeHistogram {
 #[cfg(test)]
 mod tests {
     use super::FeeHistogram;
-    use bitcoin::Amount;
+    use crate::bitcoin::Amount;
     use serde_json::json;
 
     #[test]

src/merkle.rs

@@ -1,4 +1,4 @@
-use bitcoin::{hash_types::TxMerkleNode, hashes::Hash, Txid};
+use crate::bitcoin::{hash_types::TxMerkleNode, hashes::Hash, Txid};
 
 pub(crate) struct Proof {
     proof: Vec<TxMerkleNode>,
@@ -54,7 +54,7 @@ impl Proof {
 
 #[cfg(test)]
 mod tests {
-    use bitcoin::{consensus::encode::deserialize, Block, Txid};
+    use crate::bitcoin::{consensus::encode::deserialize, Block, Txid};
     use std::path::Path;
 
     use super::Proof;

src/p2p.rs

@@ -1,411 +0,0 @@
-use anyhow::{Context, Result};
-use bitcoin::blockdata::block::Header as BlockHeader;
-use bitcoin::consensus::Encodable;
-use bitcoin::{
-    consensus::{
-        encode::{self, ReadExt, VarInt},
-        Decodable,
-    },
-    hashes::Hash,
-    io,
-    p2p::{
-        self, address,
-        message::{self, CommandString, NetworkMessage},
-        message_blockdata::{GetHeadersMessage, Inventory},
-        message_network, Magic,
-    },
-    secp256k1::{self, rand::Rng},
-    Block, BlockHash, Network,
-};
-use bitcoin_slices::{bsl, Parse};
-use crossbeam_channel::{bounded, select, Receiver, Sender};
-
-use std::io::Write;
-use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpStream};
-use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
-
-use crate::types::SerBlock;
-use crate::{
-    chain::{Chain, NewHeader},
-    config::ELECTRS_VERSION,
-    metrics::{default_duration_buckets, default_size_buckets, Histogram, Metrics},
-};
-
-enum Request {
-    GetNewHeaders(GetHeadersMessage),
-    GetBlocks(Vec<Inventory>),
-}
-
-impl Request {
-    fn get_new_headers(chain: &Chain) -> Request {
-        Request::GetNewHeaders(GetHeadersMessage::new(
-            chain.locator(),
-            BlockHash::all_zeros(),
-        ))
-    }
-
-    fn get_blocks(blockhashes: &[BlockHash]) -> Request {
-        Request::GetBlocks(
-            blockhashes
-                .iter()
-                .map(|blockhash| Inventory::WitnessBlock(*blockhash))
-                .collect(),
-        )
-    }
-}
-
-pub(crate) struct Connection {
-    req_send: Sender<Request>,
-    blocks_recv: Receiver<SerBlock>,
-    headers_recv: Receiver<Vec<BlockHeader>>,
-    new_block_recv: Receiver<()>,
-
-    blocks_duration: Histogram,
-}
-
-impl Connection {
-    /// Get new block headers (supporting reorgs).
-    /// https://en.bitcoin.it/wiki/Protocol_documentation#getheaders
-    /// Defined as `&mut self` to prevent concurrent invocations (https://github.com/romanz/electrs/pull/526#issuecomment-934685515).
-    pub(crate) fn get_new_headers(&mut self, chain: &Chain) -> Result<Vec<NewHeader>> {
-        self.req_send.send(Request::get_new_headers(chain))?;
-        let headers = self
-            .headers_recv
-            .recv()
-            .context("failed to get new headers")?;
-
-        debug!("got {} new headers", headers.len());
-        let prev_blockhash = match headers.first() {
-            None => return Ok(vec![]),
-            Some(first) => first.prev_blockhash,
-        };
-        let new_heights = match chain.get_block_height(&prev_blockhash) {
-            Some(last_height) => (last_height + 1)..,
-            None => bail!("missing prev_blockhash: {}", prev_blockhash),
-        };
-        Ok(headers
-            .into_iter()
-            .zip(new_heights)
-            .map(NewHeader::from)
-            .collect())
-    }
-
-    /// Request and process the specified blocks (in the specified order).
-    /// See https://en.bitcoin.it/wiki/Protocol_documentation#getblocks for details.
-    /// Defined as `&mut self` to prevent concurrent invocations (https://github.com/romanz/electrs/pull/526#issuecomment-934685515).
-    pub(crate) fn for_blocks<B, F>(&mut self, blockhashes: B, mut func: F) -> Result<()>
-    where
-        B: IntoIterator<Item = BlockHash>,
-        F: FnMut(BlockHash, SerBlock),
-    {
-        self.blocks_duration.observe_duration("total", || {
-            let blockhashes: Vec<BlockHash> = blockhashes.into_iter().collect();
-            if blockhashes.is_empty() {
-                return Ok(());
-            }
-            self.blocks_duration.observe_duration("request", || {
-                debug!("loading {} blocks", blockhashes.len());
-                self.req_send.send(Request::get_blocks(&blockhashes))
-            })?;
-
-            for hash in blockhashes {
-                let block = self.blocks_duration.observe_duration("response", || {
-                    let block = self
-                        .blocks_recv
-                        .recv()
-                        .with_context(|| format!("failed to get block {}", hash))?;
-                    let header = bsl::BlockHeader::parse(&block[..])
-                        .expect("core returned invalid blockheader")
-                        .parsed_owned();
-                    ensure!(
-                        &header.block_hash_sha2()[..] == hash.as_byte_array(),
-                        "got unexpected block"
-                    );
-                    Ok(block)
-                })?;
-                self.blocks_duration
-                    .observe_duration("process", || func(hash, block));
-            }
-            Ok(())
-        })
-    }
-
-    /// Note: only a single receiver will get the notification (https://github.com/romanz/electrs/pull/526#issuecomment-934687415).
-    pub(crate) fn new_block_notification(&self) -> Receiver<()> {
-        self.new_block_recv.clone()
-    }
-
-    pub(crate) fn connect(
-        network: Network,
-        address: SocketAddr,
-        metrics: &Metrics,
-        magic: Magic,
-    ) -> Result<Self> {
-        let recv_conn = TcpStream::connect(address)
-            .with_context(|| format!("{} p2p failed to connect: {:?}", network, address))?;
-        let mut send_conn = recv_conn
-            .try_clone()
-            .context("failed to clone connection")?;
-
-        let (tx_send, tx_recv) = bounded::<NetworkMessage>(1);
-        let (rx_send, rx_recv) = bounded::<RawNetworkMessage>(1);
-
-        let send_duration = metrics.histogram_vec(
-            "p2p_send_duration",
-            "Time spent sending p2p messages (in seconds)",
-            "step",
-            default_duration_buckets(),
-        );
-        let recv_duration = metrics.histogram_vec(
-            "p2p_recv_duration",
-            "Time spent receiving p2p messages (in seconds)",
-            "step",
-            default_duration_buckets(),
-        );
-        let parse_duration = metrics.histogram_vec(
-            "p2p_parse_duration",
-            "Time spent parsing p2p messages (in seconds)",
-            "step",
-            default_duration_buckets(),
-        );
-        let recv_size = metrics.histogram_vec(
-            "p2p_recv_size",
-            "Size of p2p messages read (in bytes)",
-            "message",
-            default_size_buckets(),
-        );
-        let blocks_duration = metrics.histogram_vec(
-            "p2p_blocks_duration",
-            "Time spent getting blocks via p2p protocol (in seconds)",
-            "step",
-            default_duration_buckets(),
-        );
-
-        let mut buffer = vec![];
-        crate::thread::spawn("p2p_send", move || loop {
-            use std::net::Shutdown;
-            let msg = match send_duration.observe_duration("wait", || tx_recv.recv()) {
-                Ok(msg) => msg,
-                Err(_) => {
-                    // p2p_loop is closed, so tx_send is disconnected
-                    debug!("closing p2p_send thread: no more messages to send");
-                    // close the stream reader (p2p_recv thread may block on it)
-                    if let Err(e) = send_conn.shutdown(Shutdown::Read) {
-                        warn!("failed to shutdown p2p connection: {}", e)
-                    }
-                    return Ok(());
-                }
-            };
-            send_duration.observe_duration("send", || {
-                trace!("send: {:?}", msg);
-                let raw_msg = message::RawNetworkMessage::new(magic, msg);
-                buffer.clear();
-                raw_msg
-                    .consensus_encode(&mut buffer)
-                    .expect("in-memory writers don't error");
-                send_conn
-                    .write_all(buffer.as_slice())
-                    .context("p2p failed to send")
-            })?;
-        });
-
-        let mut stream_reader = std::io::BufReader::new(recv_conn);
-        crate::thread::spawn("p2p_recv", move || loop {
-            let start = Instant::now();
-            let raw_msg = RawNetworkMessage::consensus_decode(&mut stream_reader);
-            {
-                let duration = duration_to_seconds(start.elapsed());
-                let label = format!(
-                    "recv_{}",
-                    raw_msg
-                        .as_ref()
-                        .map(|msg| msg.cmd.as_ref())
-                        .unwrap_or("err")
-                );
-                recv_duration.observe(&label, duration);
-            }
-            let raw_msg = match raw_msg {
-                Ok(raw_msg) => {
-                    recv_size.observe(raw_msg.cmd.as_ref(), raw_msg.raw.len() as f64);
-                    if raw_msg.magic != magic {
-                        bail!("unexpected magic {} (instead of {})", raw_msg.magic, magic)
-                    }
-                    raw_msg
-                }
-                Err(encode::Error::Io(e)) if e.kind() == io::ErrorKind::UnexpectedEof => {
-                    debug!("closing p2p_recv thread: connection closed");
-                    return Ok(());
-                }
-                Err(e) => bail!("failed to recv a message from peer: {}", e),
-            };
-
-            recv_duration.observe_duration("wait", || rx_send.send(raw_msg))?;
-        });
-
-        let (req_send, req_recv) = bounded::<Request>(1);
-        let (blocks_send, blocks_recv) = bounded::<SerBlock>(10);
-        let (headers_send, headers_recv) = bounded::<Vec<BlockHeader>>(1);
-        let (new_block_send, new_block_recv) = bounded::<()>(0);
-        let (init_send, init_recv) = bounded::<()>(0);
-
-        tx_send.send(build_version_message())?;
-
-        crate::thread::spawn("p2p_loop", move || loop {
-            select! {
-                recv(rx_recv) -> result => {
-                    let raw_msg = match result {
-                        Ok(raw_msg) => raw_msg,
-                        Err(_) => {  // p2p_recv is closed, so rx_send is disconnected
-                            debug!("closing p2p_loop thread: peer has disconnected");
-                            return Ok(()); // new_block_send is dropped, causing the server to exit
-                        }
-                    };
-
-                    let label = format!("parse_{}", raw_msg.cmd.as_ref());
-                    let msg = match parse_duration.observe_duration(&label, || raw_msg.parse()) {
-                        Ok(msg) => msg,
-                        Err(err) => bail!("failed to parse {err}"),
-                    };
-                    trace!("recv: {:?}", msg);
-
-                    match msg {
-                        ParsedNetworkMessage::Version(version) => {
-                            debug!("peer version: {:?}", version);
-                            tx_send.send(NetworkMessage::Verack)?;
-                        }
-                        ParsedNetworkMessage::Inv(inventory) => {
-                            debug!("peer inventory: {:?}", inventory);
-                            if inventory.iter().any(|inv| matches!(inv, Inventory::Block(_))) {
-                                let _ = new_block_send.try_send(()); // best-effort notification
-                            }
-
-                        },
-                        ParsedNetworkMessage::Ping(nonce) => {
-                            tx_send.send(NetworkMessage::Pong(nonce))?; // connection keep-alive
-                        }
-                        ParsedNetworkMessage::Verack => {
-                            init_send.send(())?; // peer acknowledged our version
-                        }
-                        ParsedNetworkMessage::Block(block) => blocks_send.send(block)?,
-                        ParsedNetworkMessage::Headers(headers) => headers_send.send(headers)?,
-                        ParsedNetworkMessage::Ignored => (),
-                    }
-                }
-                recv(req_recv) -> result => {
-                    let req = match result {
-                        Ok(req) => req,
-                        Err(_) => {  // self is dropped, so req_send is disconnected
-                            debug!("closing p2p_loop thread: no more requests to handle");
-                            return Ok(());
-                        }
-                    };
-                    let msg = match req {
-                        Request::GetNewHeaders(msg) => NetworkMessage::GetHeaders(msg),
-                        Request::GetBlocks(inv) => NetworkMessage::GetData(inv),
-                    };
-                    tx_send.send(msg)?;
-                }
-            }
-        });
-
-        init_recv.recv()?; // wait until `verack` is received
-
-        Ok(Connection {
-            req_send,
-            blocks_recv,
-            headers_recv,
-            new_block_recv,
-            blocks_duration,
-        })
-    }
-}
-
-fn build_version_message() -> NetworkMessage {
-    let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0);
-    let timestamp = SystemTime::now()
-        .duration_since(UNIX_EPOCH)
-        .expect("Time error")
-        .as_secs() as i64;
-
-    let services = p2p::ServiceFlags::NONE;
-
-    NetworkMessage::Version(message_network::VersionMessage {
-        version: p2p::PROTOCOL_VERSION,
-        services,
-        timestamp,
-        receiver: address::Address::new(&addr, services),
-        sender: address::Address::new(&addr, services),
-        nonce: secp256k1::rand::thread_rng().gen(),
-        user_agent: format!("/electrs:{}/", ELECTRS_VERSION),
-        start_height: 0,
-        relay: false,
-    })
-}
-
-struct RawNetworkMessage {
-    magic: Magic,
-    cmd: CommandString,
-    raw: Vec<u8>,
-}
-
-impl RawNetworkMessage {
-    fn parse(self) -> Result<ParsedNetworkMessage> {
-        let mut raw: &[u8] = &self.raw;
-        let payload = match self.cmd.as_ref() {
-            "version" => ParsedNetworkMessage::Version(Decodable::consensus_decode(&mut raw)?),
-            "verack" => ParsedNetworkMessage::Verack,
-            "inv" => ParsedNetworkMessage::Inv(Decodable::consensus_decode(&mut raw)?),
-            "block" => ParsedNetworkMessage::Block(self.raw),
-            "headers" => {
-                let len = VarInt::consensus_decode(&mut raw)?.0;
-                let mut headers = Vec::with_capacity(len as usize);
-                for _ in 0..len {
-                    headers.push(Block::consensus_decode(&mut raw)?.header);
-                }
-                ParsedNetworkMessage::Headers(headers)
-            }
-            "ping" => ParsedNetworkMessage::Ping(Decodable::consensus_decode(&mut raw)?),
-            "pong" => ParsedNetworkMessage::Ignored, // unused
-            "addr" => ParsedNetworkMessage::Ignored, // unused
-            "alert" => ParsedNetworkMessage::Ignored, // https://bitcoin.org/en/alert/2016-11-01-alert-retirement
-            _ => bail!(
-                "unsupported message: command={}, payload={:?}",
-                self.cmd,
-                self.raw
-            ),
-        };
-        Ok(payload)
-    }
-}
-
-#[derive(Debug)]
-enum ParsedNetworkMessage {
-    Version(message_network::VersionMessage),
-    Verack,
-    Inv(Vec<Inventory>),
-    Ping(u64),
-    Headers(Vec<BlockHeader>),
-    Block(SerBlock),
-    Ignored,
-}
-
-impl Decodable for RawNetworkMessage {
-    fn consensus_decode<D: bitcoin::io::Read + ?Sized>(d: &mut D) -> Result<Self, encode::Error> {
-        let magic = Decodable::consensus_decode(d)?;
-        let cmd = Decodable::consensus_decode(d)?;
-
-        let len = u32::consensus_decode(d)?;
-        let _checksum = <[u8; 4]>::consensus_decode(d)?; // assume data is correct
-        let mut raw = vec![0u8; len as usize];
-        d.read_slice(&mut raw)?;
-
-        Ok(RawNetworkMessage { magic, cmd, raw })
-    }
-}
-
-/// `duration_to_seconds` converts Duration to seconds.
-#[inline]
-pub fn duration_to_seconds(d: Duration) -> f64 {
-    let nanos = f64::from(d.subsec_nanos()) / 1e9;
-    d.as_secs() as f64 + nanos
-}

src/server.rs

@@ -85,7 +85,6 @@ fn serve() -> Result<()> {
     );
     let mut rpc = Rpc::new(&config, metrics)?;
 
-    let new_block_rx = rpc.new_block_notification();
     let mut peers = HashMap::<usize, Peer>::new();
     loop {
         // initial sync and compaction may take a few hours
@@ -107,14 +106,6 @@ fn serve() -> Result<()> {
                     result.context("signal channel disconnected")?;
                     rpc.signal().exit_flag().poll().context("RPC server interrupted")?;
                 },
-                // Handle new blocks' notifications
-                recv(new_block_rx) -> result => match result {
-                    Ok(_) => (), // sync and update
-                    Err(_) => {
-                        info!("disconnected from bitcoind");
-                        return Ok(());
-                    }
-                },
                 // Handle Electrum RPC requests
                 recv(server_rx) -> event => {
                     let first = once(event.context("server disconnected")?);

src/status.rs

@@ -1,27 +1,16 @@
-use anyhow::Result;
-use bitcoin::{
-    consensus::serialize,
+use crate::bitcoin::{
+    consensus::Decodable,
     hashes::{sha256, Hash, HashEngine},
     Amount, BlockHash, OutPoint, SignedAmount, Transaction, Txid,
 };
-use bitcoin_slices::{bsl, Visit, Visitor};
-use rayon::prelude::*;
+use anyhow::Result;
+use bindex::{IndexedChain, ScriptHash};
 use serde::ser::{Serialize, Serializer};
 
+use std::collections::{BTreeMap, HashMap, HashSet};
 use std::convert::TryFrom;
-use std::{
-    collections::{BTreeMap, HashMap, HashSet},
-    ops::ControlFlow,
-};
 
-use crate::{
-    cache::Cache,
-    chain::Chain,
-    daemon::Daemon,
-    index::Index,
-    mempool::Mempool,
-    types::{bsl_txid, ScriptHash, SerBlock, StatusHash},
-};
+use crate::{mempool::Mempool, types::StatusHash};
 
 /// Given a scripthash, store relevant inputs and outputs of a specific transaction
 struct TxEntry {
@@ -52,8 +41,8 @@ impl TxEntry {
 }
 
 // Confirmation height of a transaction or its mempool state:
-// https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-history
-// https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-mempool
+// https://electrum-protocol.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-history
+// https://electrum-protocol.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-mempool
 enum Height {
     Confirmed { height: usize },
     Unconfirmed { has_unconfirmed_inputs: bool },
@@ -89,8 +78,8 @@ impl std::fmt::Display for Height {
 }
 
 // A single history entry:
-// https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-history
-// https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-mempool
+// https://electrum-protocol.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-history
+// https://electrum-protocol.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-mempool
 #[derive(Serialize)]
 pub(crate) struct HistoryEntry {
     #[serde(rename = "tx_hash")]
@@ -98,14 +87,14 @@ pub(crate) struct HistoryEntry {
     height: Height,
     #[serde(
         skip_serializing_if = "Option::is_none",
-        with = "bitcoin::amount::serde::as_sat::opt"
+        with = "crate::bitcoin::amount::serde::as_sat::opt"
     )]
     fee: Option<Amount>,
 }
 
 impl HistoryEntry {
     // Hash to compute ScriptHash status, as defined here:
-    // https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-basics.html#status
+    // https://electrum-protocol.readthedocs.io/en/latest/protocol-basics.html#status
     fn hash(&self, engine: &mut sha256::HashEngine) {
         let s = format!("{}:{}:", self.txid, self.height);
         engine.input(s.as_bytes());
@@ -130,34 +119,43 @@ impl HistoryEntry {
     }
 }
 
+/// Make sure blocks are sorted by ascending height.
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
+struct BlockKey(usize, BlockHash);
+
+impl From<bindex::Location<'_>> for BlockKey {
+    fn from(location: bindex::Location<'_>) -> Self {
+        Self(location.block_height(), location.block_hash())
+    }
+}
+
 /// ScriptHash subscription status
 pub struct ScriptHashStatus {
     scripthash: ScriptHash, // specific scripthash to be queried
-    tip: BlockHash,         // used for skipping confirmed entries' sync
-    confirmed: HashMap<BlockHash, Vec<TxEntry>>, // confirmed entries, partitioned per block (may contain stale blocks)
+    confirmed: BTreeMap<BlockKey, Vec<TxEntry>>, // confirmed entries (chronologically ordered), partitioned per block
     mempool: Vec<TxEntry>,                       // unconfirmed entries
     history: Vec<HistoryEntry>,                  // computed from confirmed and mempool entries
     statushash: Option<StatusHash>,              // computed from history
 }
 
 /// Specific scripthash balance
-/// https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-balance
+/// https://electrum-protocol.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-balance
 #[derive(Default, Eq, PartialEq, Serialize)]
 pub(crate) struct Balance {
-    #[serde(with = "bitcoin::amount::serde::as_sat", rename = "confirmed")]
+    #[serde(with = "crate::bitcoin::amount::serde::as_sat", rename = "confirmed")]
     confirmed_balance: Amount,
-    #[serde(with = "bitcoin::amount::serde::as_sat", rename = "unconfirmed")]
+    #[serde(with = "crate::bitcoin::amount::serde::as_sat", rename = "unconfirmed")]
     mempool_delta: SignedAmount,
 }
 
 /// A single unspent transaction output entry
-/// https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-listunspent
+/// https://electrum-protocol.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-listunspent
 #[derive(Serialize)]
 pub(crate) struct UnspentEntry {
     height: usize, // 0 = mempool entry
     tx_hash: Txid,
     tx_pos: u32,
-    #[serde(with = "bitcoin::amount::serde::as_sat")]
+    #[serde(with = "crate::bitcoin::amount::serde::as_sat")]
     value: Amount,
 }
 
@@ -169,25 +167,23 @@ struct Unspent {
 }
 
 impl Unspent {
-    fn build(status: &ScriptHashStatus, chain: &Chain) -> Self {
+    fn build(status: &ScriptHashStatus) -> Self {
         let mut unspent = Unspent::default();
         // First, add all relevant entries' funding outputs to the outpoints' map
         status
-            .confirmed_height_entries(chain)
+            .confirmed_height_entries()
             .for_each(|(height, entries)| entries.iter().for_each(|e| unspent.insert(e, height)));
         // Then, remove spent outpoints from the map
-        status
-            .confirmed_entries(chain)
-            .for_each(|e| unspent.remove(e));
+        status.confirmed_entries().for_each(|e| unspent.remove(e));
 
         unspent.balance.confirmed_balance = unspent.balance();
         // Now, do the same over the mempool (first add funding outputs, and then remove spent ones)
         status.mempool.iter().for_each(|e| unspent.insert(e, 0)); // mempool height = 0
         status.mempool.iter().for_each(|e| unspent.remove(e));
 
-        unspent.balance.mempool_delta = unspent.balance().to_signed().unwrap()
-            - unspent.balance.confirmed_balance.to_signed().unwrap();
-
+        let total_balance = unspent.balance().to_signed().unwrap();
+        let confirmed_balance = unspent.balance.confirmed_balance.to_signed().unwrap();
+        unspent.balance.mempool_delta = total_balance - confirmed_balance;
         unspent
     }
 
@@ -232,8 +228,7 @@ impl ScriptHashStatus {
     pub fn new(scripthash: ScriptHash) -> Self {
         Self {
             scripthash,
-            tip: BlockHash::all_zeros(),
-            confirmed: HashMap::new(),
+            confirmed: BTreeMap::new(),
             mempool: Vec::new(),
             history: Vec::new(),
             statushash: None,
@@ -242,41 +237,33 @@ impl ScriptHashStatus {
 
     /// Iterate through confirmed TxEntries with their corresponding block heights.
     /// Skip entries from stale blocks.
-    fn confirmed_height_entries<'a>(
-        &'a self,
-        chain: &'a Chain,
-    ) -> impl Iterator<Item = (usize, &'a [TxEntry])> + 'a {
+    fn confirmed_height_entries<'a>(&'a self) -> impl Iterator<Item = (usize, &'a [TxEntry])> + 'a {
         self.confirmed
             .iter()
-            .filter_map(move |(blockhash, entries)| {
-                chain
-                    .get_block_height(blockhash)
-                    .map(|height| (height, &entries[..]))
-            })
+            .map(move |(blockid, entries)| (blockid.0, &entries[..]))
     }
 
     /// Iterate through confirmed TxEntries.
-    /// Skip entries from stale blocks.
-    fn confirmed_entries<'a>(&'a self, chain: &'a Chain) -> impl Iterator<Item = &'a TxEntry> + 'a {
-        self.confirmed_height_entries(chain)
+    fn confirmed_entries<'a>(&'a self) -> impl Iterator<Item = &'a TxEntry> + 'a {
+        self.confirmed_height_entries()
             .flat_map(|(_height, entries)| entries)
     }
 
     /// Collect all funded and confirmed outpoints (as a set).
-    fn confirmed_outpoints(&self, chain: &Chain) -> HashSet<OutPoint> {
-        self.confirmed_entries(chain)
+    fn confirmed_outpoints(&self) -> HashSet<OutPoint> {
+        self.confirmed_entries()
             .flat_map(TxEntry::funding_outpoints)
             .collect()
     }
 
     /// Collect unspent transaction entries
-    pub(crate) fn get_unspent(&self, chain: &Chain) -> Vec<UnspentEntry> {
-        Unspent::build(self, chain).into_entries()
+    pub(crate) fn get_unspent(&self) -> Vec<UnspentEntry> {
+        Unspent::build(self).into_entries()
     }
 
     /// Collect unspent transaction balance
-    pub(crate) fn get_balance(&self, chain: &Chain) -> Balance {
-        Unspent::build(self, chain).balance
+    pub(crate) fn get_balance(&self) -> Balance {
+        Unspent::build(self).balance
     }
 
     /// Collect transaction history entries
@@ -285,8 +272,8 @@ impl ScriptHashStatus {
     }
 
     /// Collect all confirmed history entries (in block order).
-    fn get_confirmed_history(&self, chain: &Chain) -> Vec<HistoryEntry> {
-        self.confirmed_height_entries(chain)
+    fn get_confirmed_history(&self) -> Vec<HistoryEntry> {
+        self.confirmed_height_entries()
             .collect::<BTreeMap<usize, &[TxEntry]>>()
             .into_iter()
             .flat_map(|(height, entries)| {
@@ -311,88 +298,59 @@ impl ScriptHashStatus {
             .collect()
     }
 
-    /// Apply `func` only on the new blocks (to be fetched via p2p interface).
-    fn for_new_blocks<B, F>(&self, blockhashes: B, daemon: &Daemon, func: F) -> Result<()>
-    where
-        B: IntoIterator<Item = BlockHash>,
-        F: FnMut(BlockHash, SerBlock),
-    {
-        daemon.for_blocks(
-            blockhashes
-                .into_iter()
-                .filter(|blockhash| !self.confirmed.contains_key(blockhash)),
-            func,
-        )
-    }
-
     /// Get funding and spending entries from new blocks.
     /// Also cache relevant transactions and their merkle proofs.
-    fn sync_confirmed(
-        &self,
-        index: &Index,
-        daemon: &Daemon,
-        cache: &Cache,
-        outpoints: &mut HashSet<OutPoint>,
-    ) -> Result<HashMap<BlockHash, Vec<TxEntry>>> {
-        // Will be updated during the following block scans
-        let mut result = HashMap::<BlockHash, HashMap<usize, TxEntry>>::new();
-
-        let funding_blockhashes = index.limit_result(index.filter_by_funding(self.scripthash))?;
-        self.for_new_blocks(funding_blockhashes, daemon, |blockhash, block| {
-            let block_entries = result.entry(blockhash).or_default(); // the block may already exist
-
-            // extract relevant funding transactions
-            for filtered_outputs in filter_block_txs_outputs(block, self.scripthash) {
-                cache.add_tx(filtered_outputs.txid, move || filtered_outputs.tx_bytes);
-                // store funded outpoints (to check for spending later)
-                outpoints.extend(make_outpoints(
-                    filtered_outputs.txid,
-                    &filtered_outputs.result,
-                ));
-                block_entries
-                    .entry(filtered_outputs.pos) // the transaction may already exist
-                    .or_insert_with(|| TxEntry::new(filtered_outputs.txid))
-                    .outputs = filtered_outputs.result;
+    fn sync_confirmed(&mut self, index: &IndexedChain) -> Result<HashSet<OutPoint>> {
+        let headers = index.headers();
+        let mut latest_header = None;
+        // Drop entries from stale blocks
+        while let Some(entry) = self.confirmed.last_entry() {
+            let BlockKey(height, hash) = entry.key();
+            match headers.get_header(*hash, *height) {
+                Ok(header) => {
+                    latest_header = Some(header);
+                    break;
+                }
+                Err(err) => {
+                    warn!("drop reorged block: {}", err);
+                    entry.remove();
+                    continue;
+                }
             }
-        })?;
-        let spending_blockhashes: HashSet<BlockHash> = outpoints
-            .par_iter() // use rayon for concurrent index lookups
-            .flat_map_iter(|outpoint| index.filter_by_spending(*outpoint))
-            .collect();
-        self.for_new_blocks(spending_blockhashes, daemon, |blockhash, block| {
-            let block_entries = result.entry(blockhash).or_default(); // the block may already exist
+        }
+        // Recompute all funded outpoints
+        let mut outpoints = self.confirmed_outpoints();
+        // Process transactions in chronological order
+        for location in index.locations_by_scripthash(&self.scripthash, latest_header)? {
+            let tx_bytes = index.get_tx_bytes(&location)?;
+            let tx = Transaction::consensus_decode_from_finite_reader(&mut &tx_bytes[..])?;
 
-            // extract relevant spending transactions
-            for filtered_inputs in filter_block_txs_inputs(&block, outpoints) {
-                cache.add_tx(filtered_inputs.txid, move || filtered_inputs.tx_bytes);
-                block_entries
-                    .entry(filtered_inputs.pos) // the transaction may already exist
-                    .or_insert_with(|| TxEntry::new(filtered_inputs.txid))
-                    .spent = filtered_inputs.result;
+            // Check if this transaction has relevant inputs/outputs:
+            let spent = filter_inputs(&tx, &outpoints);
+            let outputs = filter_outputs(&tx, self.scripthash);
+            if spent.is_empty() && outputs.is_empty() {
+                continue;
             }
-        })?;
 
-        Ok(result
-            .into_iter()
-            .map(|(blockhash, entries_map)| {
-                let sorted_entries: Vec<TxEntry> = entries_map
-                    .into_iter()
-                    .collect::<BTreeMap<usize, TxEntry>>() // sort transactions by their position in a block
-                    .into_values() // drop position within block
-                    .collect();
-                (blockhash, sorted_entries)
-            })
-            .collect())
+            // Build new TxEntry and add new outpoints (for next transactions)
+            let mut tx_entry = TxEntry::new(tx.compute_txid());
+            tx_entry.spent = spent;
+            tx_entry.outputs = outputs;
+            outpoints.extend(tx_entry.funding_outpoints());
+
+            // Add new per-block entry (if needed)
+            self.confirmed
+                .entry(BlockKey::from(location))
+                .or_default()
+                .push(tx_entry);
+        }
+
+        Ok(outpoints)
     }
 
     /// Get funding and spending entries from current mempool.
     /// Also cache relevant transactions.
-    fn sync_mempool(
-        &self,
-        mempool: &Mempool,
-        cache: &Cache,
-        outpoints: &mut HashSet<OutPoint>,
-    ) -> Vec<TxEntry> {
+    fn sync_mempool(&self, mempool: &Mempool, mut outpoints: HashSet<OutPoint>) -> Vec<TxEntry> {
         let mut result = HashMap::<Txid, TxEntry>::new();
         // extract relevant funding transactions
         for entry in mempool.filter_by_funding(&self.scripthash) {
@@ -404,40 +362,25 @@ impl ScriptHashStatus {
                 .entry(entry.txid) // the transaction may already exist
                 .or_insert_with(|| TxEntry::new(entry.txid))
                 .outputs = funding_outputs;
-            cache.add_tx(entry.txid, || serialize(&entry.tx).into_boxed_slice());
         }
         for entry in outpoints
             .iter()
             .flat_map(|outpoint| mempool.filter_by_spending(outpoint))
         {
-            let spent_outpoints = filter_inputs(&entry.tx, outpoints);
+            let spent_outpoints = filter_inputs(&entry.tx, &outpoints);
             assert!(!spent_outpoints.is_empty());
             result
                 .entry(entry.txid) // the transaction may already exist
                 .or_insert_with(|| TxEntry::new(entry.txid))
                 .spent = spent_outpoints;
-            cache.add_tx(entry.txid, || serialize(&entry.tx).into_boxed_slice());
         }
         result.into_values().collect()
     }
 
-    /// Sync with currently confirmed txs and mempool, downloading non-cached transactions via p2p protocol.
+    /// Sync with currently confirmed txs and mempool, downloading non-cached transactions via REST API.
     /// After a successful sync, scripthash status is updated.
-    pub(crate) fn sync(
-        &mut self,
-        index: &Index,
-        mempool: &Mempool,
-        daemon: &Daemon,
-        cache: &Cache,
-    ) -> Result<()> {
-        let mut outpoints: HashSet<OutPoint> = self.confirmed_outpoints(index.chain());
-
-        let new_tip = index.chain().tip();
-        if self.tip != new_tip {
-            let update = self.sync_confirmed(index, daemon, cache, &mut outpoints)?;
-            self.confirmed.extend(update); // add new blocks to the map
-            self.tip = new_tip;
-        }
+    pub(crate) fn sync(&mut self, index: &IndexedChain, mempool: &Mempool) -> Result<()> {
+        let outpoints = self.sync_confirmed(index)?;
         if !self.confirmed.is_empty() {
             debug!(
                 "{} transactions from {} blocks",
@@ -445,14 +388,13 @@ impl ScriptHashStatus {
                 self.confirmed.len()
             );
         }
-        self.mempool = self.sync_mempool(mempool, cache, &mut outpoints);
+        self.mempool = self.sync_mempool(mempool, outpoints);
         if !self.mempool.is_empty() {
             debug!("{} mempool transactions", self.mempool.len());
         }
         // update history entries and status hash
         self.history.clear();
-        self.history
-            .extend(self.get_confirmed_history(index.chain()));
+        self.history.extend(self.get_confirmed_history());
         self.history.extend(self.get_mempool_history(mempool));
 
         self.statushash = compute_status_hash(&self.history);
@@ -471,36 +413,28 @@ fn make_outpoints(txid: Txid, outputs: &[TxOutput]) -> impl Iterator<Item = OutP
         .map(move |out| OutPoint::new(txid, out.index))
 }
 
+/// Collect outputs that fund given scripthash.
 fn filter_outputs(tx: &Transaction, scripthash: ScriptHash) -> Vec<TxOutput> {
     let outputs = tx.output.iter().zip(0u32..);
     outputs
-        .filter_map(move |(txo, vout)| {
-            if ScriptHash::new(&txo.script_pubkey) == scripthash {
-                Some(TxOutput {
-                    index: vout,
-                    value: txo.value,
-                })
-            } else {
-                None
-            }
+        .filter(|&(txo, _vout)| ScriptHash::new(&txo.script_pubkey) == scripthash)
+        .map(|(txo, vout)| TxOutput {
+            index: vout,
+            value: txo.value,
         })
         .collect()
 }
 
+/// Collect inputs that spend one of the specified outpoints.
 fn filter_inputs(tx: &Transaction, outpoints: &HashSet<OutPoint>) -> Vec<OutPoint> {
-    tx.input
-        .iter()
-        .filter_map(|txi| {
-            if outpoints.contains(&txi.previous_output) {
-                Some(txi.previous_output)
-            } else {
-                None
-            }
-        })
+    let inputs = tx.input.iter();
+    inputs
+        .filter(|&txi| outpoints.contains(&txi.previous_output))
+        .map(|txi| txi.previous_output)
         .collect()
 }
 
-// See https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-basics.html#status for details
+// See https://electrum-protocol.readthedocs.io/en/latest/protocol-basics.html#status for details
 fn compute_status_hash(history: &[HistoryEntry]) -> Option<StatusHash> {
     if history.is_empty() {
         return None;
@@ -512,114 +446,10 @@ fn compute_status_hash(history: &[HistoryEntry]) -> Option<StatusHash> {
     Some(StatusHash::from_engine(engine))
 }
 
-struct FilteredTx<T> {
-    tx_bytes: Box<[u8]>,
-    txid: Txid,
-    pos: usize,
-    result: Vec<T>,
-}
-
-fn filter_block_txs_outputs(block: SerBlock, scripthash: ScriptHash) -> Vec<FilteredTx<TxOutput>> {
-    struct FindOutputs {
-        scripthash: ScriptHash,
-        result: Vec<FilteredTx<TxOutput>>,
-        buffer: Vec<TxOutput>,
-        pos: usize,
-    }
-    impl Visitor for FindOutputs {
-        // Called after all TxOuts are visited
-        fn visit_transaction(&mut self, tx: &bsl::Transaction) -> ControlFlow<()> {
-            if !self.buffer.is_empty() {
-                self.result.push(FilteredTx::<TxOutput> {
-                    tx_bytes: tx.as_ref().into(),
-                    txid: bsl_txid(tx),
-                    pos: self.pos,
-                    result: std::mem::take(&mut self.buffer), // clear buffer for next tx
-                });
-            }
-            self.pos += 1;
-            ControlFlow::Continue(())
-        }
-        // Keep only relevant outputs
-        fn visit_tx_out(&mut self, vout: usize, tx_out: &bsl::TxOut) -> ControlFlow<()> {
-            let current = ScriptHash::hash(tx_out.script_pubkey());
-            if current == self.scripthash {
-                self.buffer.push(TxOutput {
-                    index: vout as u32,
-                    value: Amount::from_sat(tx_out.value()),
-                })
-            }
-            ControlFlow::Continue(())
-        }
-    }
-    let mut find_outputs = FindOutputs {
-        scripthash,
-        result: vec![],
-        buffer: vec![],
-        pos: 0,
-    };
-
-    bsl::Block::visit(&block, &mut find_outputs).expect("core returned invalid block");
-
-    find_outputs.result
-}
-
-fn filter_block_txs_inputs(
-    block: &SerBlock,
-    outpoints: &HashSet<OutPoint>,
-) -> Vec<FilteredTx<OutPoint>> {
-    struct FindInputs<'a> {
-        outpoints: &'a HashSet<OutPoint>,
-        result: Vec<FilteredTx<OutPoint>>,
-        buffer: Vec<OutPoint>,
-        pos: usize,
-    }
-
-    impl Visitor for FindInputs<'_> {
-        // Called after all TxIns are visited
-        fn visit_transaction(&mut self, tx: &bsl::Transaction) -> ControlFlow<()> {
-            if !self.buffer.is_empty() {
-                self.result.push(FilteredTx::<OutPoint> {
-                    tx_bytes: tx.as_ref().into(),
-                    txid: bsl_txid(tx),
-                    pos: self.pos,
-                    result: std::mem::take(&mut self.buffer), // clear buffer for next tx
-                });
-            }
-            self.pos += 1;
-            ControlFlow::Continue(())
-        }
-        // Keep only relevant outpoints
-        fn visit_tx_in(&mut self, _vin: usize, tx_in: &bsl::TxIn) -> ControlFlow<()> {
-            let current: OutPoint = tx_in.prevout().into();
-            if self.outpoints.contains(&current) {
-                self.buffer.push(current);
-            }
-            ControlFlow::Continue(())
-        }
-    }
-
-    let mut find_inputs = FindInputs {
-        outpoints,
-        result: vec![],
-        buffer: vec![],
-        pos: 0,
-    };
-
-    bsl::Block::visit(block, &mut find_inputs).expect("core returned invalid block");
-
-    find_inputs.result
-}
-
 #[cfg(test)]
 mod tests {
-    use std::{collections::HashSet, str::FromStr};
-
-    use crate::types::ScriptHash;
-
     use super::HistoryEntry;
-    use bitcoin::{Address, Amount};
-    use bitcoin_test_data::blocks::mainnet_702861;
+    use crate::bitcoin::Amount;
     use serde_json::json;
 
     #[test]
@@ -644,42 +474,4 @@ mod tests {
             json!({"tx_hash": "5b75086dafeede555fc8f9a810d8b10df57c46f9f176ccc3dd8d2fa20edd685b", "height": 0, "fee": 123})
         );
     }
-
-    #[test]
-    fn test_find_outputs() {
-        let block = mainnet_702861().to_vec();
-
-        let addr = Address::from_str("1A9MXXG26vZVySrNNytQK1N8bX42ZuJ6Ax")
-            .unwrap()
-            .assume_checked();
-        let scripthash = ScriptHash::new(&addr.script_pubkey());
-
-        let result = &super::filter_block_txs_outputs(block, scripthash)[0];
-        assert_eq!(
-            result.txid.to_string(),
-            "7bcdcb44422da5a99daad47d6ba1c3d6f2e48f961a75e42c4fa75029d4b0ef49"
-        );
-        assert_eq!(result.pos, 8);
-        assert_eq!(result.result[0].index, 0);
-        assert_eq!(result.result[0].value.to_sat(), 709503);
-    }
-
-    #[test]
-    fn test_find_inputs() {
-        let block = mainnet_702861().to_vec();
-        let outpoint = bitcoin::OutPoint::from_str(
-            "cc135e792b37a9c4ffd784f696b1e38bd1197f8e67ae1f96c9f13e4618b91866:3",
-        )
-        .unwrap();
-        let mut outpoints = HashSet::new();
-        outpoints.insert(outpoint);
-
-        let result = &super::filter_block_txs_inputs(&block, &outpoints)[0];
-        assert_eq!(
-            result.txid.to_string(),
-            "7bcdcb44422da5a99daad47d6ba1c3d6f2e48f961a75e42c4fa75029d4b0ef49"
-        );
-        assert_eq!(result.pos, 8);
-        assert_eq!(result.result[0], outpoint);
-    }
 }

src/tracker.rs

@@ -1,77 +1,52 @@
-use std::ops::ControlFlow;
-
+use crate::bitcoin::{BlockHash, Txid};
+use crate::bitcoin_slices::{bsl, EmptyVisitor, Visit};
 use anyhow::{Context, Result};
-use bitcoin::{BlockHash, Txid};
-use bitcoin_slices::{bsl, Error::VisitBreak, Visit, Visitor};
+use bindex::IndexedChain;
 
 use crate::{
-    cache::Cache,
-    chain::Chain,
     config::Config,
     daemon::Daemon,
-    db::DBStore,
-    index::Index,
     mempool::{FeeHistogram, Mempool},
     metrics::Metrics,
     signals::ExitFlag,
     status::{Balance, ScriptHashStatus, UnspentEntry},
-    types::bsl_txid,
 };
 
 /// Electrum protocol subscriptions' tracker
 pub struct Tracker {
-    index: Index,
+    index: IndexedChain,
     mempool: Mempool,
-    metrics: Metrics,
     ignore_mempool: bool,
 }
 
-pub(crate) enum Error {
-    NotReady,
-}
-
 impl Tracker {
     pub fn new(config: &Config, metrics: Metrics) -> Result<Self> {
-        let store = DBStore::open(
-            &config.db_path,
-            config.db_log_dir.as_deref(),
-            config.auto_reindex,
-        )?;
-        let chain = Chain::new(config.network);
+        let url = format!("http://{}", config.daemon_rpc_addr);
+        let index = IndexedChain::open(&config.db_dir, config.network, Some(url))
+            .context("failed to open index")?;
         Ok(Self {
-            index: Index::load(
-                store,
-                chain,
-                &metrics,
-                config.index_batch_size,
-                config.index_lookup_limit,
-                config.reindex_last_blocks,
-            )
-            .context("failed to open index")?,
+            index,
             mempool: Mempool::new(&metrics),
-            metrics,
             ignore_mempool: config.ignore_mempool,
         })
     }
 
-    pub(crate) fn chain(&self) -> &Chain {
-        self.index.chain()
+    pub(crate) fn headers(&self) -> &bindex::Headers {
+        self.index.headers()
     }
 
     pub(crate) fn fees_histogram(&self) -> &FeeHistogram {
         self.mempool.fees_histogram()
     }
 
-    pub(crate) fn metrics(&self) -> &Metrics {
-        &self.metrics
-    }
-
     pub(crate) fn get_unspent(&self, status: &ScriptHashStatus) -> Vec<UnspentEntry> {
-        status.get_unspent(self.index.chain())
+        status.get_unspent()
     }
 
     pub(crate) fn sync(&mut self, daemon: &Daemon, exit_flag: &ExitFlag) -> Result<bool> {
-        let done = self.index.sync(daemon, exit_flag)?;
+        exit_flag.poll()?;
+        let stats = self.index.sync(1000)?;
+        let done = stats.indexed_blocks == 0;
         if done && !self.ignore_mempool {
             self.mempool.sync(daemon, exit_flag);
             // TODO: double check tip - and retry on diff
@@ -79,67 +54,36 @@ impl Tracker {
         Ok(done)
     }
 
-    pub(crate) fn status(&self) -> Result<(), Error> {
-        if self.index.is_ready() {
-            return Ok(());
-        }
-        Err(Error::NotReady)
+    pub(crate) fn status(&self) -> Result<()> {
+        Ok(())
     }
 
-    pub(crate) fn update_scripthash_status(
-        &self,
-        status: &mut ScriptHashStatus,
-        daemon: &Daemon,
-        cache: &Cache,
-    ) -> Result<bool> {
+    pub(crate) fn update_scripthash_status(&self, status: &mut ScriptHashStatus) -> Result<bool> {
         let prev_statushash = status.statushash();
-        status.sync(&self.index, &self.mempool, daemon, cache)?;
+        status.sync(&self.index, &self.mempool)?;
         Ok(prev_statushash != status.statushash())
     }
 
     pub(crate) fn get_balance(&self, status: &ScriptHashStatus) -> Balance {
-        status.get_balance(self.chain())
+        status.get_balance()
     }
 
-    pub(crate) fn lookup_transaction(
-        &self,
-        daemon: &Daemon,
-        txid: Txid,
-    ) -> Result<Option<(BlockHash, Box<[u8]>)>> {
+    pub(crate) fn lookup_transaction(&self, txid: Txid) -> Result<Option<(BlockHash, Box<[u8]>)>> {
         // Note: there are two blocks with coinbase transactions having same txid (see BIP-30)
-        let blockhashes = self.index.filter_by_txid(txid);
-        let mut result = None;
-        daemon.for_blocks(blockhashes, |blockhash, block| {
-            if result.is_some() {
-                return; // keep first matching transaction
+        for loc in self.index.locations_by_txid(&txid)? {
+            let tx_bytes = self.index.get_tx_bytes(&loc)?;
+            if txid == compute_txid(&tx_bytes)? {
+                return Ok(Some((loc.block_hash(), tx_bytes.into_boxed_slice())));
             }
-            let mut visitor = FindTransaction::new(txid);
-            result = match bsl::Block::visit(&block, &mut visitor) {
-                Ok(_) | Err(VisitBreak) => visitor.found.map(|tx| (blockhash, tx)),
-                Err(e) => panic!("core returned invalid block: {:?}", e),
-            };
-        })?;
-        Ok(result)
-    }
-}
-
-pub struct FindTransaction {
-    txid: bitcoin::Txid,
-    found: Option<Box<[u8]>>, // no need to deserialize
-}
-
-impl FindTransaction {
-    pub fn new(txid: bitcoin::Txid) -> Self {
-        Self { txid, found: None }
-    }
-}
-impl Visitor for FindTransaction {
-    fn visit_transaction(&mut self, tx: &bsl::Transaction) -> ControlFlow<()> {
-        if self.txid == bsl_txid(tx) {
-            self.found = Some(tx.as_ref().into());
-            ControlFlow::Break(())
-        } else {
-            ControlFlow::Continue(())
         }
+        Ok(None)
     }
 }
+
+fn compute_txid(tx_bytes: &[u8]) -> Result<Txid> {
+    let mut visit = EmptyVisitor {};
+    let res = bsl::Transaction::visit(tx_bytes, &mut visit)
+        .map_err(|err| anyhow!("invalid transaction: {:?}", err))?;
+    ensure!(res.remaining().is_empty(), "non-empty remaining bytes");
+    Ok(Txid::from_raw_hash(res.parsed().txid()))
+}

src/types.rs

@@ -1,292 +1,13 @@
-use anyhow::Result;
-
-use std::convert::TryFrom;
-
-use bitcoin::blockdata::block::Header as BlockHeader;
-use bitcoin::{
-    consensus::encode::{deserialize, Decodable, Encodable},
-    hashes::{hash_newtype, sha256, Hash},
-    io, OutPoint, Script, Txid,
-};
-use bitcoin_slices::bsl;
-
-macro_rules! impl_consensus_encoding {
-    ($thing:ident, $($field:ident),+) => (
-        impl Encodable for $thing {
-            #[inline]
-            fn consensus_encode<S: io::Write + ?Sized>(
-                &self,
-                s: &mut S,
-            ) -> Result<usize, io::Error> {
-                let mut len = 0;
-                $(len += self.$field.consensus_encode(s)?;)+
-                Ok(len)
-            }
-        }
-
-        impl Decodable for $thing {
-            #[inline]
-            fn consensus_decode<D: io::Read + ?Sized>(
-                d: &mut D,
-            ) -> Result<$thing, bitcoin::consensus::encode::Error> {
-                Ok($thing {
-                    $($field: Decodable::consensus_decode(d)?),+
-                })
-            }
-        }
-    );
-}
-
-pub const HASH_PREFIX_LEN: usize = 8;
-const HEIGHT_SIZE: usize = 4;
-
-pub(crate) type HashPrefix = [u8; HASH_PREFIX_LEN];
-pub(crate) type SerializedHashPrefixRow = [u8; HASH_PREFIX_ROW_SIZE];
-type Height = u32;
-pub(crate) type SerBlock = Vec<u8>;
-
-#[derive(Debug, Serialize, Deserialize, PartialEq)]
-pub(crate) struct HashPrefixRow {
-    prefix: HashPrefix,
-    height: Height, // transaction confirmed height
-}
-
-pub const HASH_PREFIX_ROW_SIZE: usize = HASH_PREFIX_LEN + HEIGHT_SIZE;
-
-impl HashPrefixRow {
-    pub(crate) fn to_db_row(&self) -> SerializedHashPrefixRow {
-        let mut row = [0; HASH_PREFIX_ROW_SIZE];
-        let len = self
-            .consensus_encode(&mut (&mut row as &mut [u8]))
-            .expect("in-memory writers don't error");
-        debug_assert_eq!(len, HASH_PREFIX_ROW_SIZE);
-        row
-    }
-
-    pub(crate) fn from_db_row(row: SerializedHashPrefixRow) -> Self {
-        deserialize(&row).expect("bad HashPrefixRow")
-    }
-
-    pub fn height(&self) -> usize {
-        usize::try_from(self.height).expect("invalid height")
-    }
-}
-
-impl_consensus_encoding!(HashPrefixRow, prefix, height);
+use crate::bitcoin::hashes::{hash_newtype, sha256};
 
 hash_newtype! {
-    /// https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-basics.html#script-hashes
-    #[hash_newtype(backward)]
-    pub struct ScriptHash(sha256::Hash);
-}
-
-impl ScriptHash {
-    pub fn new(script: &Script) -> Self {
-        ScriptHash::hash(script.as_bytes())
-    }
-
-    fn prefix(&self) -> HashPrefix {
-        let mut prefix = HashPrefix::default();
-        prefix.copy_from_slice(&self.0[..HASH_PREFIX_LEN]);
-        prefix
-    }
-}
-
-pub(crate) struct ScriptHashRow;
-
-impl ScriptHashRow {
-    pub(crate) fn scan_prefix(scripthash: ScriptHash) -> HashPrefix {
-        scripthash.0[..HASH_PREFIX_LEN].try_into().unwrap()
-    }
-
-    pub(crate) fn row(scripthash: ScriptHash, height: usize) -> HashPrefixRow {
-        HashPrefixRow {
-            prefix: scripthash.prefix(),
-            height: Height::try_from(height).expect("invalid height"),
-        }
-    }
-}
-
-// ***************************************************************************
-
-hash_newtype! {
-    /// https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-basics.html#status
+    /// https://electrum-protocol.readthedocs.io/en/latest/protocol-basics.html#status
     pub struct StatusHash(sha256::Hash);
 }
 
-// ***************************************************************************
-
-fn spending_prefix(prev: OutPoint) -> HashPrefix {
-    let txid_prefix = HashPrefix::try_from(&prev.txid[..HASH_PREFIX_LEN]).unwrap();
-    let value = u64::from_be_bytes(txid_prefix);
-    let value = value.wrapping_add(prev.vout.into());
-    value.to_be_bytes()
-}
-
-pub(crate) struct SpendingPrefixRow;
-
-impl SpendingPrefixRow {
-    pub(crate) fn scan_prefix(outpoint: OutPoint) -> HashPrefix {
-        spending_prefix(outpoint)
-    }
-
-    pub(crate) fn row(outpoint: OutPoint, height: usize) -> HashPrefixRow {
-        HashPrefixRow {
-            prefix: spending_prefix(outpoint),
-            height: Height::try_from(height).expect("invalid height"),
-        }
-    }
-}
-
-// ***************************************************************************
-
-fn txid_prefix(txid: &Txid) -> HashPrefix {
-    let mut prefix = [0u8; HASH_PREFIX_LEN];
-    prefix.copy_from_slice(&txid[..HASH_PREFIX_LEN]);
-    prefix
-}
-
-pub(crate) struct TxidRow;
-
-impl TxidRow {
-    pub(crate) fn scan_prefix(txid: Txid) -> HashPrefix {
-        txid_prefix(&txid)
-    }
-
-    pub(crate) fn row(txid: Txid, height: usize) -> HashPrefixRow {
-        HashPrefixRow {
-            prefix: txid_prefix(&txid),
-            height: Height::try_from(height).expect("invalid height"),
-        }
-    }
-}
-
-// ***************************************************************************
-
-pub(crate) type SerializedHeaderRow = [u8; HEADER_ROW_SIZE];
-
-#[derive(Debug, Serialize, Deserialize)]
-pub(crate) struct HeaderRow {
-    pub(crate) header: BlockHeader,
-}
-
-pub const HEADER_ROW_SIZE: usize = 80;
-
-impl_consensus_encoding!(HeaderRow, header);
-
-impl HeaderRow {
-    pub(crate) fn new(header: BlockHeader) -> Self {
-        Self { header }
-    }
-
-    pub(crate) fn to_db_row(&self) -> SerializedHeaderRow {
-        let mut row = [0; HEADER_ROW_SIZE];
-        let len = self
-            .consensus_encode(&mut (&mut row as &mut [u8]))
-            .expect("in-memory writers don't error");
-        debug_assert_eq!(len, HEADER_ROW_SIZE);
-        row
-    }
-
-    pub(crate) fn from_db_row(row: SerializedHeaderRow) -> Self {
-        deserialize(&row).expect("bad HeaderRow")
-    }
-}
-
-pub(crate) fn bsl_txid(tx: &bsl::Transaction) -> Txid {
-    bitcoin::Txid::from_slice(tx.txid_sha2().as_slice()).expect("invalid txid")
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::types::{spending_prefix, HashPrefixRow, ScriptHash, ScriptHashRow, TxidRow};
-    use bitcoin::{Address, OutPoint, Txid};
-    use hex_lit::hex;
-    use serde_json::{from_str, json};
-
-    use std::str::FromStr;
-
-    #[test]
-    fn test_scripthash_serde() {
-        let hex = "\"4b3d912c1523ece4615e91bf0d27381ca72169dbf6b1c2ffcc9f92381d4984a3\"";
-        let scripthash: ScriptHash = from_str(hex).unwrap();
-        assert_eq!(format!("\"{}\"", scripthash), hex);
-        assert_eq!(json!(scripthash).to_string(), hex);
-    }
-
-    #[test]
-    fn test_scripthash_row() {
-        let hex = "\"4b3d912c1523ece4615e91bf0d27381ca72169dbf6b1c2ffcc9f92381d4984a3\"";
-        let scripthash: ScriptHash = from_str(hex).unwrap();
-        let row1 = ScriptHashRow::row(scripthash, 123456);
-        let db_row = row1.to_db_row();
-        assert_eq!(db_row, hex!("a384491d38929fcc40e20100"));
-        let row2 = HashPrefixRow::from_db_row(db_row);
-        assert_eq!(row1, row2);
-    }
-
-    #[test]
-    fn test_scripthash() {
-        let addr = Address::from_str("1KVNjD3AAnQ3gTMqoTKcWFeqSFujq9gTBT")
-            .unwrap()
-            .assume_checked();
-        let scripthash = ScriptHash::new(&addr.script_pubkey());
-        assert_eq!(
-            scripthash,
-            "00dfb264221d07712a144bda338e89237d1abd2db4086057573895ea2659766a"
-                .parse()
-                .unwrap()
-        );
-    }
-
-    #[test]
-    fn test_txid1_prefix() {
-        // duplicate txids from BIP-30
-        let hex = "d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599";
-        let txid = Txid::from_str(hex).unwrap();
-
-        let row1 = TxidRow::row(txid, 91812);
-        let row2 = TxidRow::row(txid, 91842);
-
-        assert_eq!(row1.to_db_row(), hex!("9985d82954e10f22a4660100"));
-        assert_eq!(row2.to_db_row(), hex!("9985d82954e10f22c2660100"));
-    }
-
-    #[test]
-    fn test_txid2_prefix() {
-        // duplicate txids from BIP-30
-        let hex = "e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468";
-        let txid = Txid::from_str(hex).unwrap();
-
-        let row1 = TxidRow::row(txid, 91722);
-        let row2 = TxidRow::row(txid, 91880);
-
-        // low-endian encoding => rows should be sorted according to block height
-        assert_eq!(row1.to_db_row(), hex!("68b45f58b674e94e4a660100"));
-        assert_eq!(row2.to_db_row(), hex!("68b45f58b674e94ee8660100"));
-    }
-
-    #[test]
-    fn test_spending_prefix() {
-        let txid = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"
-            .parse()
-            .unwrap();
-
-        assert_eq!(
-            spending_prefix(OutPoint { txid, vout: 0 }),
-            [31, 30, 29, 28, 27, 26, 25, 24]
-        );
-        assert_eq!(
-            spending_prefix(OutPoint { txid, vout: 10 }),
-            [31, 30, 29, 28, 27, 26, 25, 34]
-        );
-        assert_eq!(
-            spending_prefix(OutPoint { txid, vout: 255 }),
-            [31, 30, 29, 28, 27, 26, 26, 23]
-        );
-        assert_eq!(
-            spending_prefix(OutPoint { txid, vout: 256 }),
-            [31, 30, 29, 28, 27, 26, 26, 24]
-        );
-    }
+/// The different authentication methods for the client.
+#[derive(Clone, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)]
+pub enum Auth {
+    UserPass(String, String),
+    CookieFile(std::path::PathBuf),
 }

tests/run.sh

@@ -6,7 +6,7 @@ mkdir -p data/{bitcoin,electrum,electrs}
 
 cleanup() {
   trap - SIGTERM SIGINT
-  set +eo pipefail
+  set +e +o pipefail
   jobs
   for j in `jobs -rp`
   do
@@ -16,6 +16,15 @@ cleanup() {
 }
 trap cleanup SIGINT SIGTERM EXIT
 
+wait_for() {
+  CMD=$1
+  shift
+  TEST_ARGS=$*
+  for _ in `seq 0 9`; do
+    test "$($CMD | jq -c .)" $TEST_ARGS && break || sleep 2
+  done
+}
+
 BTC="bitcoin-cli -regtest -datadir=data/bitcoin"
 ELECTRUM="electrum --regtest"
 EL="$ELECTRUM --wallet=data/electrum/wallet"
@@ -25,7 +34,7 @@ tail_log() {
 }
 
 echo "Starting $(bitcoind -version | head -n1)..."
-bitcoind -regtest -datadir=data/bitcoin -printtoconsole=0 &
+bitcoind -rest -regtest -datadir=data/bitcoin -printtoconsole=0 &
 BITCOIND_PID=$!
 
 $BTC -rpcwait getblockcount > /dev/null
@@ -60,7 +69,7 @@ $EL load_wallet
 echo "Running integration tests:"
 
 echo " * getbalance"
-test "`$EL getbalance | jq -c .`" == '{"confirmed":"550","unmatured":"4950"}'
+wait_for "$EL getbalance" == '{"confirmed":"550","unmatured":"4950"}'
 
 echo " * getunusedaddress"
 NEW_ADDR=`$EL getunusedaddress`
@@ -69,7 +78,7 @@ echo " * payto & broadcast"
 TXID=$($EL broadcast $($EL payto $NEW_ADDR 123 --fee 0.001 --password=''))
 
 echo " * get_tx_status"
-test "`$EL get_tx_status $TXID | jq -c .`" == '{"confirmations":0}'
+wait_for "$EL get_tx_status $TXID" == '{"confirmations":0}'
 
 echo " * getaddresshistory"
 test "`$EL getaddresshistory $NEW_ADDR | jq -c .`" == "[{\"fee\":100000,\"height\":0,\"tx_hash\":\"$TXID\"}]"