diff --git a/Cargo.lock b/Cargo.lock index 70abbb5..7a37539 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,9 +47,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.22.1" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "bech32" @@ -57,23 +57,6 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" -[[package]] -name = "bindex" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "becb30e2688944707fe12c33fc1d6920fc4c83f67224cd81bf2767062b852f9e" -dependencies = [ - "bitcoin", - "bitcoin_slices", - "log", - "rayon", - "rust-rocksdb", - "serde", - "serde_json", - "thiserror 2.0.16", - "ureq", -] - [[package]] name = "bindgen" version = "0.69.5" @@ -96,9 +79,9 @@ dependencies = [ [[package]] name = "bitcoin" -version = "0.32.9" +version = "0.32.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf93e61f2dbc3e3c41234ca26a65e2c0b0975c52e0f069ab9893ebbede584d3" +checksum = "1e499f9fc0407f50fe98af744ab44fa67d409f76b6772e1689ec8485eb0c0f66" dependencies = [ "base58ck", "bech32", @@ -156,12 +139,37 @@ dependencies = [ [[package]] name = "bitcoin_slices" -version = "0.11.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b80fcc031ed36f91c31639a4d97acb1487985fdee34565651b2f52f4346859" +checksum = "7943d4257fdfc85afe2a13d2b539a5213e97bb978329dd79b624acbce73042fb" dependencies = [ "bitcoin", "bitcoin_hashes", + "sha2", +] + +[[package]] +name = "bitcoincore-rpc" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedd23ae0fd321affb4bbbc36126c6f49a32818dc6b979395d24da8c9d4e80ee" +dependencies = [ + "bitcoincore-rpc-json", + "jsonrpc", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "bitcoincore-rpc-json" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8909583c5fab98508e80ef73e5592a651c954993dc6b7739963257d19f0e71a" +dependencies = [ + "bitcoin", + "serde", + "serde_json", ] [[package]] @@ -170,6 +178,15 @@ version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block2" version = "0.6.2" @@ -179,12 +196,6 @@ dependencies = [ "objc2", ] -[[package]] -name = "bytes" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" - [[package]] name = "bzip2-sys" version = "0.1.13+1.0.8" @@ -280,6 +291,15 @@ dependencies = [ "void", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -315,16 +335,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] -name = "ctrlc" -version = "3.5.2" +name = "crypto-common" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctrlc" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73736a89c4aff73035ba2ed2e565061954da00d4970fc9ac25dcc85a2a20d790" dependencies = [ "dispatch2", "nix", "windows-sys 0.61.0", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -369,8 +409,10 @@ name = "electrs" version = "0.11.1" dependencies = [ "anyhow", - "bindex", + "bitcoin", "bitcoin-test-data", + "bitcoin_slices", + "bitcoincore-rpc", "configure_me", "configure_me_codegen", "crossbeam-channel", @@ -378,10 +420,11 @@ dependencies = [ "dirs-next", "env_logger", "hex_lit", - "jsonrpc", "log", + "parking_lot", "prometheus", "rayon", + "rust-rocksdb", "serde", "serde_derive", "serde_json", @@ -431,6 +474,16 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.16" @@ -487,22 +540,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" -[[package]] -name = "http" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" -dependencies = [ - "bytes", - "itoa", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - [[package]] name = "httpdate" version = "1.0.3" @@ -553,11 +590,12 @@ dependencies = [ [[package]] name = "jsonrpc" -version = "0.20.1" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f106cca655869522988b976127096f0aa36e0f1a8ff1f1f425a5c85b61e59391" +checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf" dependencies = [ "base64", + "minreq", "serde", "serde_json", ] @@ -576,9 +614,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.180" +version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" [[package]] name = "libredox" @@ -625,9 +663,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "memchr" @@ -642,10 +680,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] -name = "nix" -version = "0.31.1" +name = "minreq" +version = "2.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225e7cfe711e0ba79a68baeddb2982723e4235247aefce1482f2f16c27865b66" +checksum = "f0d2aaba477837b46ec1289588180fabfccf0c3b1d1a0c6b1866240cd6cd5ce9" +dependencies = [ + "log", + "serde", + "serde_json", +] + +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ "bitflags", "cfg-if", @@ -722,18 +771,21 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bddc33f680b79eaf1e2e56da792c3c2236f86985bbc3a886e8ddee17ae4d3a4" -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - [[package]] name = "pkg-config" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.95" @@ -818,10 +870,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" [[package]] -name = "rayon" -version = "1.12.0" +name = "rand" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" dependencies = [ "either", "rayon-core", @@ -956,6 +1038,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113" dependencies = [ "bitcoin_hashes", + "rand", "secp256k1-sys", "serde", ] @@ -1001,9 +1084,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "itoa", "memchr", @@ -1012,6 +1095,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "1.3.0" @@ -1020,9 +1114,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook" -version = "0.4.4" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a0c28ca5908dbdbcd52e6fdaa00358ab88637f8ab33e1f188dd510eb44b53d" +checksum = "3b57709da74f9ff9f4a27dce9526eec25ca8407c45a7887243b031a58935fb8e" dependencies = [ "libc", "signal-hook-registry", @@ -1056,9 +1150,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.25.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" dependencies = [ "fastrand", "getrandom 0.3.3", @@ -1137,6 +1231,12 @@ dependencies = [ "serde", ] +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + [[package]] name = "unicode-ident" version = "1.0.18" @@ -1149,43 +1249,18 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" -[[package]] -name = "ureq" -version = "3.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39cb1dbab692d82a977c0392ffac19e188bd9186a9f32806f0aaa859d75585a" -dependencies = [ - "base64", - "log", - "percent-encoding", - "ureq-proto", - "utf-8", -] - -[[package]] -name = "ureq-proto" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d81f9efa9df032be5934a46a068815a10a042b494b6a58cb0a1a97bb5467ed6f" -dependencies = [ - "base64", - "http", - "httparse", - "log", -] - -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "vcpkg" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "void" version = "1.0.2" @@ -1335,6 +1410,26 @@ dependencies = [ "bitflags", ] +[[package]] +name = "zerocopy" +version = "0.8.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zmij" version = "1.0.2" diff --git a/Cargo.toml b/Cargo.toml index e19ad2e..8af0480 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,30 +23,46 @@ spec = "internal/config_specification.toml" [dependencies] anyhow = "1.0" -bindex = { version = "0.1.2", default-features = false, features = ["bitcoin_serde"] } +bitcoin = { version = "0.32.8", features = ["serde", "rand-std"] } +bitcoin_slices = { version = "0.10.0", features = ["bitcoin", "sha2"] } +bitcoincore-rpc = { version = "0.19.0" } configure_me = "0.4" crossbeam-channel = "0.5" dirs-next = "2.0" env_logger = "0.10" -jsonrpc = { version = "0.20.1", default-features = false, features = ["simple_http"]} log = "0.4" +parking_lot = "0.12" prometheus = { version = "0.14", optional = true } -rayon = "1.12" +rayon = "1.11" serde = "1.0.184" serde_derive = "1.0.184" serde_json = "1.0" tiny_http = { version = "0.12", optional = true } [target.'cfg(windows)'.dependencies] -ctrlc = "=3.5.2" +ctrlc = "=3.5.1" [target.'cfg(not(windows))'.dependencies] signal-hook = "0.4" +[dependencies.rust-rocksdb] +version = "0.36" + +default-features = false +# ZSTD is used for data compression +# Snappy is only for checking old DB +features = ["zstd", "snappy"] + [build-dependencies] configure_me_codegen = { version = "0.4.8", default-features = false } [dev-dependencies] bitcoin-test-data = "0.2.0" hex_lit = "0.1.1" -tempfile = "3.25" +tempfile = "3.24" + +[profile.release] +lto = true + +[lints.clippy] +uninlined_format_args = "allow" # TODO: https://github.com/romanz/electrs/issues/1199 diff --git a/Dockerfile.ci b/Dockerfile.ci index eadb696..0231276 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -4,7 +4,7 @@ FROM debian:trixie-slim AS base RUN apt-get update -qqy -RUN apt-get install -qqy librocksdb-dev libsqlite3-dev libevent-dev libboost-dev wget +RUN apt-get install -qqy librocksdb-dev wget ### Electrum Rust Server ### FROM base AS electrs-build @@ -17,15 +17,16 @@ ENV ROCKSDB_INCLUDE_DIR=/usr/include ENV ROCKSDB_LIB_DIR=/usr/lib RUN cargo install --locked --path . -### Bitcoin Core (custom build for /blockpart/ REST endpoint) ### +### Bitcoin Core ### FROM base AS bitcoin-build # Download WORKDIR /build/bitcoin - -RUN wget -q https://bitcoincore.org/bin/bitcoin-core-31.1/bitcoin-31.1-x86_64-linux-gnu.tar.gz -RUN tar xvf bitcoin-31.1-x86_64-linux-gnu.tar.gz -RUN mv -v bitcoin-31.1/bin/bitcoind . -RUN mv -v bitcoin-31.1/bin/bitcoin-cli . +ARG ARCH=x86_64 +ARG BITCOIND_VERSION=29.0 +RUN wget -q https://bitcoincore.org/bin/bitcoin-core-$BITCOIND_VERSION/bitcoin-$BITCOIND_VERSION-$ARCH-linux-gnu.tar.gz +RUN tar xvf bitcoin-$BITCOIND_VERSION-$ARCH-linux-gnu.tar.gz +RUN mv -v bitcoin-$BITCOIND_VERSION/bin/bitcoind . +RUN mv -v bitcoin-$BITCOIND_VERSION/bin/bitcoin-cli . FROM base AS result # Copy the binaries diff --git a/doc/config.md b/doc/config.md index b234e72..3784b20 100644 --- a/doc/config.md +++ b/doc/config.md @@ -5,12 +5,11 @@ If you use automated systems, refer to their documentation first! ### Bitcoind configuration -REST API must be **enabled** for block indexing and transaction lookup. - -Pruning must be **disabled** for `electrs` to work. - +Pruning must be turned **off** for `electrs` to work. `txindex` is allowed but unnecessary for `electrs`. However, you might still need it if you run other services (e.g.`eclair`). +The option `maxconnections` (if used) should be set to 12 or more for bitcoind to accept inbound p2p connections. +Note that setting `maxuploadtarget` may cause p2p-based sync to fail - so consider using `-whitelist=download@127.0.0.1` to disable the limit for local p2p connections. The highly recommended way of authenticating `electrs` is using cookie file. It's the most [secure](https://github.com/Kixunil/security_writings/blob/master/cookie_files.md) and robust method. @@ -22,7 +21,7 @@ You can skip it if you're running both daemons under the same user and with the Example command for running `bitcoind` (assuming same user, default dirs): ```bash -$ bitcoind -server=1 -rest=1 -txindex=0 -prune=0 +$ bitcoind -server=1 -txindex=0 -prune=0 ``` ### Electrs configuration @@ -140,7 +139,7 @@ HiddenServiceVersion 3 HiddenServicePort 50001 127.0.0.1:50001 ``` -If you use [the *beta* Debian repository](binaries.md#native-os-packages), +If you use [the *beta* Debian repository](binaries.md#cnative-os-packages), it is cleaner to install `tor-hs-patch-config` using `apt` and then placing the configuration into a file inside `/etc/tor/hidden-services.d`. Restart the service: @@ -163,7 +162,7 @@ For more details, see http://docs.electrum.org/en/latest/tor.html. ### Sample Systemd Unit File -If you use [the *beta* Debian repository](binaries.md#native-os-packages), you should skip this section, +If you use [the *beta* Debian repository](binaries.md#cnative-os-packages), you should skip this section, as the appropriate systemd unit file is installed automatically. You may wish to have systemd manage electrs so that it's "always on". diff --git a/doc/config_example.toml b/doc/config_example.toml index 59cf429..1272ddb 100644 --- a/doc/config_example.toml +++ b/doc/config_example.toml @@ -14,6 +14,9 @@ cookie_file = "/var/run/bitcoin-mainnet/cookie" # The listening RPC address of bitcoind, port is usually 8332 daemon_rpc_addr = "127.0.0.1:8332" +# The listening P2P address of bitcoind, port is usually 8333 +daemon_p2p_addr = "127.0.0.1:8333" + # Directory where the index should be stored. It should have at least 70GB of free space. db_dir = "/some/fast/storage/with/big/size" diff --git a/doc/install.md b/doc/install.md index 3ac8229..392cdf5 100644 --- a/doc/install.md +++ b/doc/install.md @@ -1,14 +1,21 @@ ## Quickstart -Building from source on an Ubuntu 25.04 / Debian 13: +
+Building from source on an Ubuntu 25.04 / Debian 13: ```bash $ sudo apt update $ sudo apt install -y build-essential libclang-dev git cargo -$ git clone https://github.com/romanz/electrs && cd electrs -$ cargo run --locked --release -- --version # build and print the latest version +$ git clone https://github.com/romanz/electrs +$ cd electrs +$ cargo build --locked --release +$ ./target/release/electrs --version # should print the latest version ``` +
+ +[![asciicast](https://asciinema.org/a/XKznxilP4O7lCZiVZ9vZNd5vx.svg)](https://asciinema.org/a/XKznxilP4O7lCZiVZ9vZNd5vx?speed=3) + ## Manual installation from source **See below for automated/binary installation options.** @@ -19,7 +26,7 @@ Note for Raspberry Pi 4 owners: the old versions of OS/toolchains produce broken Make sure to use latest OS! (see #226) Install [recent Rust](https://rustup.rs/) (1.85.0+, `apt install cargo` is preferred for Debian 13), -[latest Bitcoin Core](https://bitcoincore.org/en/download/) (31.0+) +[latest Bitcoin Core](https://bitcoincore.org/en/download/) (0.21+) and [latest Electrum wallet](https://electrum.org/#download) (4.0+). Also, install the following packages (on Debian or Ubuntu): diff --git a/doc/schema.md b/doc/schema.md index 901b775..ab8af29 100644 --- a/doc/schema.md +++ b/doc/schema.md @@ -1 +1,53 @@ -See https://github.com/romanz/bindex-rs for more details. +# Index Schema + +The index is stored at a single RocksDB database using the following column families. +Most of the data is stored in key-only DB rows (i.e. having empty values). + +## Transaction outputs' index (`funding`) + +Allows efficiently finding all funding transactions for a specific address: + +| Script Hash Prefix | Confirmed Block Height | +| -------------------- | ---------------------- | +| `SHA256(script)[:8]` | `height as u32` | + +## Transaction inputs' index (`spending`) + +Allows efficiently finding spending transaction of a specific output: + +| Previous Outpoint Prefix | Confirmed Block Height | +| ------------------------ | ---------------------- | +| `txid[:8] as u64 + vout` | `height as u32` | + + +## Transaction ID index (`txid`) + +In order to save storage space, we map the 8-byte transaction ID prefix to its confirmed block height: + +| Txid Prefix | Confirmed height | +| ----------- | ---------------- | +| `txid[:8]` | `height as u32` | + +Note that this mapping allows us to use `getrawtransaction` RPC to retrieve actual transaction data from without `-txindex` enabled +(by explicitly specifying the [blockhash](https://github.com/bitcoin/bitcoin/commit/497d0e014cc79d46531d570e74e4aeae72db602d)). + +## Headers (`headers`) + +For faster loading, we store all block headers in RocksDB: + +| Serialized header | +| ----------------------- | +| `header as BlockHeader` | + +In addition, we also store the chain tip: + +| Key | | Value | +| --- | - | ------------------------ | +| `T` | | `blockhash as BlockHash` | + +## Configuration (`config`) + +| Key | | Value | +| --- | - | --------------------------- | +| `C` | | `serialized config as JSON` | + diff --git a/doc/upgrading.md b/doc/upgrading.md index fd060da..a52bffd 100644 --- a/doc/upgrading.md +++ b/doc/upgrading.md @@ -1,20 +1,3 @@ -### Important changes from versions older than 0.12.0 - -In 0.12.0 we have changed the RocksDB index format to optimize electrs performance. - -We also use new bitcoind REST API endpoints (instead of P2P protocol) - -* [#32540 (index: fetch spent transaction outputs by blockhash)](https://github.com/bitcoin/bitcoin/pull/32540) → released in 30.0 -* [#33657 (rest: allow reading partial block data from storage)](https://github.com/bitcoin/bitcoin/pull/33657) → released in 31.0 - -Upgrading checklist: - -* Make sure you upgrade at a time when you don't need to use electrs for a while. - Because of reindex electrs will be unable to serve your requests for a few hours. - (The exact time depends on your hardware.) -* Make sure you have at least 120 GB of free space (for reindexing mainnet chain). - - ### Important changes from versions older than 0.9.3 * If you use `verbose` (or `-v` argument), switch to `log_filters` (or `RUST_LOG` environment variable). diff --git a/doc/usage.md b/doc/usage.md index 8ec9ee9..e6da8c7 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -1,63 +1,77 @@ ## Quickstart -Assuming Bitcoin Core 31+ is installed on the same machine (with the standard configuration at `~/.bitcoin/bitcoin.conf`): +
+Assuming Bitcoin Core 0.21+ is installed on the same machine (with the standard configuration at `~/.bitcoin/bitcoin.conf`): ```bash -$ bitcoind -server=1 -rest=1 -prune=0 & +$ bitcoind -server=1 -prune=0 & $ # ... wait until the chain is synced (e.g. using `bitcoin-cli getblockchaininfo`) -$ RUST_LOG=INFO electrs --network bitcoin --db-dir ./db --daemon-dir ~/.bitcoin +$ electrs --log-filters=INFO --db-dir ./db --daemon-dir ~/.bitcoin --network bitcoin ``` +
+ +[![asciicast](https://asciinema.org/a/zRNZp5HsBDi5rAlGWU7470Pzl.svg)](https://asciinema.org/a/zRNZp5HsBDi5rAlGWU7470Pzl?speed=3) + ## Usage -First index sync should take ~2 hours for ~800GB `.bitcoin/blocks/` @ July 2026 (on a 6-core CPU, 32 GB RAM, 2TB NVMe): +First index sync should take ~6.5 hours for ~504GB @ August 2023 (on a dual core Intel CPU @ 3.3 GHz, 8 GB RAM, 1TB WD Blue HDD): ```bash -$ du -ch ~/.bitcoin/blocks/*.dat | tail -n1 -802G total +$ du -ch ~/.bitcoin/blocks/blk*.dat | tail -n1 +336G total -$ RUST_LOG=INFO electrs --network bitcoin --db-dir ./db --daemon-dir ~/.bitcoin -Starting electrs 0.12.0 on x86_64 linux with Config { network: Bitcoin, db_dir: "./db", ... } -[2026-07-18T16:37:24.024Z INFO electrs::metrics::metrics_impl] serving Prometheus metrics on 127.0.0.1:4224 -[2026-07-18T16:37:24.025Z INFO electrs::server] serving Electrum RPC on 127.0.0.1:50001 -[2026-07-18T16:37:24.025Z INFO bindex::chain] index: Config { db_path: "./db/bitcoin", url: "http://127.0.0.1:8332" } -[2026-07-18T16:37:24.054Z INFO bindex::db] CF config: 0 files, 0.000000 MBs -[2026-07-18T16:37:24.054Z INFO bindex::db] CF headers: 0 files, 0.000000 MBs -[2026-07-18T16:37:24.054Z INFO bindex::db] CF txpos: 0 files, 0.000000 MBs -[2026-07-18T16:37:24.054Z INFO bindex::db] CF txid: 0 files, 0.000000 MBs -[2026-07-18T16:37:24.054Z INFO bindex::db] CF script_hash: 0 files, 0.000000 MBs -[2026-07-18T16:37:24.054Z INFO bindex::db] CF spending: 0 files, 0.000000 MBs -[2026-07-18T16:37:24.054Z INFO bindex::db] CF funding: 0 files, 0.000000 MBs -[2026-07-18T16:37:24.129Z INFO bindex::chain] block=00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09 height=1000: indexed 1001 blocks, 0.226[MB], dt = 0.047[s]: 0.047 [ms/block], 0.000 [MB/block], 4.837 [MB/s] -[2026-07-18T16:37:24.196Z INFO bindex::chain] block=00000000dfd5d65c9d8561b4b8f60a63018fe3933ecb131fb37f905f87da951a height=2000: indexed 1000 blocks, 0.234[MB], dt = 0.043[s]: 0.043 [ms/block], 0.000 [MB/block], 5.376 [MB/s] -[2026-07-18T16:37:24.263Z INFO bindex::chain] block=000000004a81b9aa469b11649996ecb0a452c16d1181e72f9f980850a1c5ecce height=3000: indexed 1000 blocks, 0.230[MB], dt = 0.043[s]: 0.043 [ms/block], 0.000 [MB/block], 5.298 [MB/s] -[2026-07-18T16:37:24.331Z INFO bindex::chain] block=00000000922e2aa9e84a474350a3555f49f06061fd49df50a9352f156692a842 height=4000: indexed 1000 blocks, 0.234[MB], dt = 0.044[s]: 0.044 [ms/block], 0.000 [MB/block], 5.366 [MB/s] -[2026-07-18T16:37:24.398Z INFO bindex::chain] block=000000004d78d2a8a93a1d20a24d721268690bebd2b51f7e80657d57e226eef9 height=5000: indexed 1000 blocks, 0.229[MB], dt = 0.043[s]: 0.043 [ms/block], 0.000 [MB/block], 5.349 [MB/s] -... -[2026-07-18T18:02:21.114Z INFO bindex::chain] block=0000000000000000000055222909c19ed96cd371013861337214a3c39a63d828 height=955000: indexed 1000 blocks, 1842.985[MB], dt = 9.457[s]: 9.457 [ms/block], 1.843 [MB/block], 194.886 [MB/s] -[2026-07-18T18:02:33.592Z INFO bindex::chain] block=000000000000000000012d9ea47c8b3282ae1a7792d54b8b4bcc0536c1883191 height=956000: indexed 1000 blocks, 1846.173[MB], dt = 9.522[s]: 9.522 [ms/block], 1.846 [MB/block], 193.879 [MB/s] -[2026-07-18T18:02:45.872Z INFO bindex::chain] block=00000000000000000000e898fc9d01b0729d8830f2068248debbff575a3b0990 height=957000: indexed 1000 blocks, 1855.303[MB], dt = 9.295[s]: 9.295 [ms/block], 1.855 [MB/block], 199.600 [MB/s] -[2026-07-18T18:02:58.221Z INFO bindex::chain] block=000000000000000000022c6a6d538a4372b0eeadb4ea9eb2edd4e32ec780084b height=958000: indexed 1000 blocks, 1844.747[MB], dt = 9.376[s]: 9.376 [ms/block], 1.845 [MB/block], 196.759 [MB/s] -[2026-07-18T18:03:05.539Z INFO bindex::chain] block=000000000000000000020934afa0945d959250e9a4598d39c53fa280639b096a height=958594: indexed 594 blocks, 1115.055[MB], dt = 5.357[s]: 9.018 [ms/block], 1.877 [MB/block], 208.155 [MB/s] -[2026-07-18T18:03:05.666Z INFO bindex::db] started auto compactions +$ ./target/release/electrs --network bitcoin --db-dir ./db --daemon-dir /home/user/.bitcoin +Starting electrs 0.10.0 on x86_64 linux with Config { network: Bitcoin, db_path: "./db/bitcoin", daemon_dir: "/home/user/.bitcoin", daemon_auth: CookieFile("/home/user/.bitcoin/.cookie"), daemon_rpc_addr: 127.0.0.1:8332, daemon_p2p_addr: 127.0.0.1:8333, electrum_rpc_addr: 127.0.0.1:50001, monitoring_addr: 127.0.0.1:4224, wait_duration: 10s, jsonrpc_timeout: 15s, index_batch_size: 10, index_lookup_limit: None, reindex_last_blocks: 0, auto_reindex: true, ignore_mempool: false, sync_once: false, skip_block_download_wait: false, disable_electrum_rpc: false, server_banner: "Welcome to electrs 0.10.0 (Electrum Rust Server)!", magic: f9beb4d9, args: [] } +[2023-08-16T19:17:11.193Z INFO electrs::metrics::metrics_impl] serving Prometheus metrics on 127.0.0.1:4224 +[2023-08-16T19:17:11.193Z INFO electrs::server] serving Electrum RPC on 127.0.0.1:50001 +[2023-08-16T19:17:12.355Z INFO electrs::db] "./db/bitcoin": 0 SST files, 0 GB, 0 Grows +[2023-08-16T19:17:12.446Z INFO electrs::index] indexing 2000 blocks: [1..2000] +[2023-08-16T19:17:12.866Z INFO electrs::chain] chain updated: tip=00000000dfd5d65c9d8561b4b8f60a63018fe3933ecb131fb37f905f87da951a, height=2000 +[2023-08-16T19:17:12.879Z INFO electrs::index] indexing 2000 blocks: [2001..4000] +[2023-08-16T19:17:13.227Z INFO electrs::chain] chain updated: tip=00000000922e2aa9e84a474350a3555f49f06061fd49df50a9352f156692a842, height=4000 +[2023-08-16T19:17:13.238Z INFO electrs::index] indexing 2000 blocks: [4001..6000] +[2023-08-16T19:17:13.587Z INFO electrs::chain] chain updated: tip=00000000dbbb79792303bdd1c6c4d7ab9c21bba0667213c2eca955e11230c5a5, height=6000 +[2023-08-16T19:17:13.598Z INFO electrs::index] indexing 2000 blocks: [6001..8000] +[2023-08-16T19:17:13.950Z INFO electrs::chain] chain updated: tip=0000000094fbacdffec05aea9847000522a258c269ae37a74a818afb96fc27d9, height=8000 +[2023-08-16T19:17:13.961Z INFO electrs::index] indexing 2000 blocks: [8001..10000] +<...> +[2023-08-17T00:13:16.443Z INFO electrs::index] indexing 2000 blocks: [798001..800000] +[2023-08-17T00:14:58.310Z INFO electrs::chain] chain updated: tip=00000000000000000002a7c4c1e48d76c5a37902165a270156b7a8d72728a054, height=800000 +[2023-08-17T00:14:58.325Z INFO electrs::index] indexing 2000 blocks: [800001..802000] +[2023-08-17T00:16:36.425Z INFO electrs::chain] chain updated: tip=0000000000000000000311b41f1d611f977b024b947568c1dd760704360f148a, height=802000 +[2023-08-17T00:16:36.437Z INFO electrs::index] indexing 1534 blocks: [802001..803534] +[2023-08-17T00:17:51.338Z INFO electrs::chain] chain updated: tip=00000000000000000003c0cd1b62ed8bb502e24bcbfeee16e81d6ea33d026263, height=803534 +[2023-08-17T00:18:00.592Z INFO electrs::db] starting config compaction +[2023-08-17T00:18:00.778Z INFO electrs::db] starting headers compaction +[2023-08-17T00:18:00.870Z INFO electrs::db] starting txid compaction +[2023-08-17T00:33:34.370Z INFO electrs::db] starting funding compaction +[2023-08-17T01:03:56.784Z INFO electrs::db] starting spending compaction +[2023-08-17T01:35:05.983Z INFO electrs::db] finished full compaction +[2023-08-17T01:36:23.300Z INFO electrs::index] indexing 5 blocks: [803535..803539] +[2023-08-17T01:36:23.646Z INFO electrs::chain] chain updated: tip=000000000000000000006a3aaddd4b643607b33e000f1200d35005c330ecfa88, height=803539 +[2023-08-17T01:41:26.009Z INFO electrs::index] indexing 1 blocks: [803540..803540] +[2023-08-17T01:41:26.143Z INFO electrs::chain] chain updated: tip=00000000000000000003266d31db92629b64241eef7ce708244f6d6283b080b4, height=803540 +[2023-08-17T01:42:42.999Z INFO electrs::index] indexing 1 blocks: [803541..803541] +[2023-08-17T01:42:43.153Z INFO electrs::chain] chain updated: tip=00000000000000000000884a77c8b8ad2fb0c25510a3251bf5ef57f0db275146, height=803541 ``` You can specify options via command-line parameters, environment variables or using config files. See the documentation above. -Note that the final DB size should be ~7% of the `blocks/*.dat` files, but it may increase to ~14% at the end of the initial sync (just before the [full compaction is invoked](https://github.com/facebook/rocksdb/wiki/Manual-Compaction)). +Note that the final DB size should be ~10% of the `blk*.dat` files, but it may increase to ~20% at the end of the initial sync (just before the [full compaction is invoked](https://github.com/facebook/rocksdb/wiki/Manual-Compaction)). It should take roughly 18 hours to sync and compact the index on an ODROID-HC1 with 8 CPU cores @ 2GHz, 2GB RAM, and an SSD using the command above. The index database is stored here: ```bash -$ du -h db/bitcoin/ -56G db/bitcoin/ +$ du db/ +42G db/mainnet/ ``` See [extra configuration suggestions](config.md#extra-configuration-suggestions) that you might want to consider. ## Electrum client -If you happen to use the Electrum client from [the *beta* Debian repository](binaries.md#native-os-packages), it's pre-configured out-of-the-box already +If you happen to use the Electrum client from [the *beta* Debian repository](binaries.md#cnative-os-packages), it's pre-configured out-of-the-box already Read below otherwise. There's a prepared script for launching `electrum` in such way to connect only to the local `electrs` instance to protect your privacy. @@ -84,7 +98,7 @@ You can invoke any supported RPC using `netcat`, for example: ``` $ echo '{"jsonrpc": "2.0", "method": "server.version", "params": ["", "1.4"], "id": 0}' | netcat 127.0.0.1 50001 -{"id":0,"jsonrpc":"2.0","result":["electrs 0.12.0","1.4"]} +{"id":0,"jsonrpc":"2.0","result":["electrs 0.9.0","1.4"]} ``` For more complex tasks, you may need to convert addresses to diff --git a/examples/tx_collisions.rs b/examples/tx_collisions.rs new file mode 100644 index 0000000..0b7fd58 --- /dev/null +++ b/examples/tx_collisions.rs @@ -0,0 +1,31 @@ +use anyhow::{Context, Result}; +use rust_rocksdb::{ColumnFamilyDescriptor, IteratorMode, Options, DB}; + +fn main() -> Result<()> { + let path = std::env::args().nth(1).context("missing DB path")?; + let cf_names = DB::list_cf(&Options::default(), &path)?; + let cfs: Vec<_> = cf_names + .iter() + .map(|name| ColumnFamilyDescriptor::new(name, Options::default())) + .collect(); + let db = DB::open_cf_descriptors(&Options::default(), &path, cfs)?; + let cf = db.cf_handle("txid").context("missing column family")?; + + let mut state: Option<(u64, u32)> = None; + for row in db.iterator_cf(cf, IteratorMode::Start) { + let (curr, _value) = row?; + let curr_prefix = u64::from_le_bytes(curr[..8].try_into()?); + let curr_height = u32::from_le_bytes(curr[8..].try_into()?); + + if let Some((prev_prefix, prev_height)) = state { + if prev_prefix == curr_prefix { + eprintln!( + "prefix={:x} heights: {} {}", + curr_prefix, prev_height, curr_height + ); + }; + } + state = Some((curr_prefix, curr_height)); + } + Ok(()) +} diff --git a/internal/config_specification.toml b/internal/config_specification.toml index f445f41..f4ae65e 100644 --- a/internal/config_specification.toml +++ b/internal/config_specification.toml @@ -63,7 +63,7 @@ doc = "JSONRPC authentication cookie file (default: ~/.bitcoin/.cookie)" [[param]] name = "network" type = "crate::config::BitcoinNetwork" -convert_into = "crate::bitcoin::Network" +convert_into = "::bitcoin::Network" doc = "Select Bitcoin network type ('bitcoin', 'testnet', 'testnet4', 'regtest' or 'signet')" default = "Default::default()" @@ -76,6 +76,10 @@ doc = "Electrum server JSONRPC 'addr:port' to listen on (default: '127.0.0.1:500 name = "daemon_rpc_addr" type = "crate::config::ResolvAddr" doc = "Bitcoin daemon JSONRPC 'addr:port' to connect (default: 127.0.0.1:8332 for mainnet, 127.0.0.1:18332 for testnet, 127.0.0.1:18443 for regtest and 127.0.0.1:18554 for signet)" +[[param]] +name = "daemon_p2p_addr" +type = "crate::config::ResolvAddr" +doc = "Bitcoin daemon p2p 'addr:port' to connect (default: 127.0.0.1:8333 for mainnet, 127.0.0.1:18333 for testnet, 127.0.0.1:18444 for regtest and 127.0.0.1:38333 for signet)" [[param]] name = "monitoring_addr" @@ -94,6 +98,12 @@ type = "u64" doc = "Duration to wait until bitcoind JSON-RPC timeouts (must be greater than wait_duration_secs)." default = "15" +[[param]] +name = "index_batch_size" +type = "usize" +doc = "Number of blocks to get in a single p2p protocol request from bitcoind" +default = "10" + [[switch]] name = "ignore_mempool" doc = "Don't sync mempool - queries will show only confirmed transactions." @@ -136,3 +146,8 @@ default = "concat!(\"Welcome to electrs \", env!(\"CARGO_PKG_VERSION\"), \" (Ele name = "log_filters" type = "String" doc = "Logging filters, overriding `RUST_LOG` environment variable (see https://docs.rs/env_logger/ for details)" + +[[param]] +name = "magic" +type = "String" +doc = "network magic for custom network in hex format, as found in Bitcoin Core logs" diff --git a/server.sh b/server.sh index 7a7fe5f..c66a4c9 100755 --- a/server.sh +++ b/server.sh @@ -9,7 +9,7 @@ NETWORK=$1 shift DB=${DB-./db} -export RUST_LOG=${RUST_LOG-INFO} +export RUST_LOG=${RUST_LOG-electrs=INFO} target/release/electrs --network $NETWORK --db-dir $DB --daemon-dir $HOME/.bitcoin $* # use SIGINT to quit diff --git a/src/cache.rs b/src/cache.rs new file mode 100644 index 0000000..b09a916 --- /dev/null +++ b/src/cache.rs @@ -0,0 +1,43 @@ +use bitcoin::Txid; +use parking_lot::RwLock; + +use std::collections::HashMap; +use std::sync::Arc; + +use crate::metrics::{self, Histogram, Metrics}; + +pub(crate) struct Cache { + txs: Arc>>>, + + // stats + txs_size: Histogram, +} + +impl Cache { + pub fn new(metrics: &Metrics) -> Self { + Cache { + txs: Default::default(), + txs_size: metrics.histogram_vec( + "cache_txs_size", + "Cached transactions' size (in bytes)", + "type", + metrics::default_size_buckets(), + ), + } + } + + pub fn add_tx(&self, txid: Txid, f: impl FnOnce() -> Box<[u8]>) { + self.txs.write().entry(txid).or_insert_with(|| { + let tx = f(); + self.txs_size.observe("serialized", tx.len() as f64); + tx + }); + } + + pub fn get_tx(&self, txid: &Txid, f: F) -> Option + where + F: FnOnce(&[u8]) -> T, + { + self.txs.read().get(txid).map(|tx_bytes| f(tx_bytes)) + } +} diff --git a/src/chain.rs b/src/chain.rs new file mode 100644 index 0000000..a55e50d --- /dev/null +++ b/src/chain.rs @@ -0,0 +1,261 @@ +use std::collections::HashMap; + +use bitcoin::blockdata::block::Header as BlockHeader; +use bitcoin::{BlockHash, Network}; + +/// A new header found, to be added to the chain at specific height +pub(crate) struct NewHeader { + header: BlockHeader, + hash: BlockHash, + height: usize, +} + +impl NewHeader { + pub(crate) fn from((header, height): (BlockHeader, usize)) -> Self { + Self { + header, + hash: header.block_hash(), + height, + } + } + + pub(crate) fn height(&self) -> usize { + self.height + } + + pub(crate) fn hash(&self) -> BlockHash { + self.hash + } +} + +/// Current blockchain headers' list +pub struct Chain { + headers: Vec<(BlockHash, BlockHeader)>, + heights: HashMap, +} + +impl Chain { + // create an empty chain + pub fn new(network: Network) -> Self { + let genesis = bitcoin::blockdata::constants::genesis_block(network); + let genesis_hash = genesis.block_hash(); + Self { + headers: vec![(genesis_hash, genesis.header)], + heights: std::iter::once((genesis_hash, 0)).collect(), // genesis header @ zero height + } + } + + pub(crate) fn drop_last_headers(&mut self, n: usize) { + if n == 0 { + return; + } + let new_height = self.height().saturating_sub(n); + self.update(vec![NewHeader::from(( + self.headers[new_height].1, + new_height, + ))]); + } + + /// Load the chain from a collection of headers, up to the given tip + pub(crate) fn load(&mut self, headers: impl Iterator, tip: BlockHash) { + let genesis_hash = self.headers[0].0; + + let header_map: HashMap = + headers.map(|h| (h.block_hash(), h)).collect(); + let mut blockhash = tip; + let mut new_headers: Vec<&BlockHeader> = Vec::with_capacity(header_map.len()); + while blockhash != genesis_hash { + let header = match header_map.get(&blockhash) { + Some(header) => header, + None => panic!("missing header {} while loading from DB", blockhash), + }; + blockhash = header.prev_blockhash; + new_headers.push(header); + } + info!("loading {} headers, tip={}", new_headers.len(), tip); + let new_headers = new_headers.into_iter().rev().copied(); // order by height + self.update(new_headers.zip(1..).map(NewHeader::from).collect()) + } + + /// Get the block hash at specified height (if exists) + pub(crate) fn get_block_hash(&self, height: usize) -> Option { + self.headers.get(height).map(|(hash, _header)| *hash) + } + + /// Get the block header at specified height (if exists) + pub(crate) fn get_block_header(&self, height: usize) -> Option<&BlockHeader> { + self.headers.get(height).map(|(_hash, header)| header) + } + + /// Get the block height given the specified hash (if exists) + pub(crate) fn get_block_height(&self, blockhash: &BlockHash) -> Option { + self.heights.get(blockhash).copied() + } + + /// Update the chain with a list of new headers (possibly a reorg) + pub(crate) fn update(&mut self, headers: Vec) { + if let Some(first_height) = headers.first().map(|h| h.height) { + for (hash, _header) in self.headers.drain(first_height..) { + assert!(self.heights.remove(&hash).is_some()); + } + for (h, height) in headers.into_iter().zip(first_height..) { + assert_eq!(h.height, height); + assert_eq!(h.hash, h.header.block_hash()); + assert!(self.heights.insert(h.hash, h.height).is_none()); + self.headers.push((h.hash, h.header)); + } + info!( + "chain updated: tip={}, height={}", + self.headers.last().unwrap().0, + self.headers.len() - 1 + ); + } + } + + /// Best block hash + pub(crate) fn tip(&self) -> BlockHash { + self.headers.last().expect("empty chain").0 + } + + /// Number of blocks (excluding genesis block) + pub(crate) fn height(&self) -> usize { + self.headers.len() - 1 + } + + /// List of block hashes for efficient fork detection and block/header sync + /// see https://en.bitcoin.it/wiki/Protocol_documentation#getblocks + pub(crate) fn locator(&self) -> Vec { + let mut result = vec![]; + let mut index = self.headers.len() - 1; + let mut step = 1; + loop { + if result.len() >= 10 { + step *= 2; + } + result.push(self.headers[index].0); + if index == 0 { + break; + } + index = index.saturating_sub(step); + } + result + } +} + +#[cfg(test)] +mod tests { + use super::{Chain, NewHeader}; + use bitcoin::blockdata::block::Header as BlockHeader; + use bitcoin::consensus::deserialize; + use bitcoin::Network::Regtest; + use hex_lit::hex; + + #[test] + fn test_genesis() { + let regtest = Chain::new(Regtest); + assert_eq!(regtest.height(), 0); + assert_eq!( + regtest.tip(), + "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206" + .parse() + .unwrap() + ); + } + + #[test] + fn test_updates() { + let byte_headers = [ +hex!("0000002006226e46111a0b59caaf126043eb5bbf28c34f3a5e332a1fc7b2b73cf188910f1d14d3c7ff12d6adf494ebbcfba69baa915a066358b68a2b8c37126f74de396b1d61cc60ffff7f2000000000"), +hex!("00000020d700ae5d3c705702e0a5d9ababd22ded079f8a63b880b1866321d6bfcb028c3fc816efcf0e84ccafa1dda26be337f58d41b438170c357cda33a68af5550590bc1e61cc60ffff7f2004000000"), +hex!("00000020d13731bc59bc0989e06a5e7cab9843a4e17ad65c7ca47cd77f50dfd24f1f55793f7f342526aca9adb6ce8f33d8a07662c97d29d83b9e18117fb3eceecb2ab99b1e61cc60ffff7f2001000000"), +hex!("00000020a603def3e1255cadfb6df072946327c58b344f9bfb133e8e3e280d1c2d55b31c731a68f70219472864a7cb010cd53dc7e0f67e57f7d08b97e5e092b0c3942ad51f61cc60ffff7f2001000000"), +hex!("0000002041dd202b3b2edcdd3c8582117376347d48ff79ff97c95e5ac814820462012e785142dc360975b982ca43eecd14b4ba6f019041819d4fc5936255d7a2c45a96651f61cc60ffff7f2000000000"), +hex!("0000002072e297a2d6b633c44f3c9b1a340d06f3ce4e6bcd79ebd4c4ff1c249a77e1e37c59c7be1ca0964452e1735c0d2740f0d98a11445a6140c36b55770b5c0bcf801f1f61cc60ffff7f2000000000"), +hex!("000000200c9eb5889a8e924d1c4e8e79a716514579e41114ef37d72295df8869d6718e4ac5840f28de43ff25c7b9200aaf7873b20587c92827eaa61943484ca828bdd2e11f61cc60ffff7f2000000000"), +hex!("000000205873f322b333933e656b07881bb399dae61a6c0fa74188b5fb0e3dd71c9e2442f9e2f433f54466900407cf6a9f676913dd54aad977f7b05afcd6dcd81e98ee752061cc60ffff7f2004000000"), +hex!("00000020fd1120713506267f1dba2e1856ca1d4490077d261cde8d3e182677880df0d856bf94cfa5e189c85462813751ab4059643759ed319a81e0617113758f8adf67bc2061cc60ffff7f2000000000"), +hex!("000000200030d7f9c11ef35b89a0eefb9a5e449909339b5e7854d99804ea8d6a49bf900a0304d2e55fe0b6415949cff9bca0f88c0717884a5e5797509f89f856af93624a2061cc60ffff7f2002000000"), + ]; + let headers: Vec = byte_headers + .iter() + .map(|byte_header| deserialize(byte_header).unwrap()) + .collect(); + + for chunk_size in 1..headers.len() { + let mut regtest = Chain::new(Regtest); + let mut height = 0; + let mut tip = regtest.tip(); + for chunk in headers.chunks(chunk_size) { + let mut update = vec![]; + for header in chunk { + height += 1; + tip = header.block_hash(); + update.push(NewHeader::from((*header, height))) + } + regtest.update(update); + assert_eq!(regtest.tip(), tip); + assert_eq!(regtest.height(), height); + } + assert_eq!(regtest.tip(), headers.last().unwrap().block_hash()); + assert_eq!(regtest.height(), headers.len()); + } + + // test loading from a list of headers and tip + let mut regtest = Chain::new(Regtest); + regtest.load( + headers.iter().copied(), + headers.last().unwrap().block_hash(), + ); + assert_eq!(regtest.height(), headers.len()); + + // test getters + for (header, height) in headers.iter().zip(1usize..) { + assert_eq!(regtest.get_block_header(height), Some(header)); + assert_eq!(regtest.get_block_hash(height), Some(header.block_hash())); + assert_eq!(regtest.get_block_height(&header.block_hash()), Some(height)); + } + + // test chain shortening + for i in (0..=headers.len()).rev() { + let hash = regtest.get_block_hash(i).unwrap(); + assert_eq!(regtest.get_block_height(&hash), Some(i)); + assert_eq!(regtest.height(), i); + assert_eq!(regtest.tip(), hash); + regtest.drop_last_headers(1); + } + assert_eq!(regtest.height(), 0); + assert_eq!( + regtest.tip(), + "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206" + .parse() + .unwrap() + ); + + regtest.drop_last_headers(1); + assert_eq!(regtest.height(), 0); + assert_eq!( + regtest.tip(), + "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206" + .parse() + .unwrap() + ); + + // test reorg + let mut regtest = Chain::new(Regtest); + regtest.load( + headers.iter().copied(), + headers.last().unwrap().block_hash(), + ); + let height = regtest.height(); + + let new_header: BlockHeader = deserialize(&hex!("000000200030d7f9c11ef35b89a0eefb9a5e449909339b5e7854d99804ea8d6a49bf900a0304d2e55fe0b6415949cff9bca0f88c0717884a5e5797509f89f856af93624a7a6bcc60ffff7f2000000000")).unwrap(); + regtest.update(vec![NewHeader::from((new_header, height))]); + assert_eq!(regtest.height(), height); + assert_eq!( + regtest.tip(), + "0e16637fe0700a7c52e9a6eaa58bd6ac7202652103be8f778680c66f51ad2e9b" + .parse() + .unwrap() + ); + } +} diff --git a/src/config.rs b/src/config.rs index d147408..b844dda 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,5 +1,6 @@ -use crate::bitcoin::Network; -use crate::types::Auth; +use bitcoin::p2p::Magic; +use bitcoin::Network; +use bitcoincore_rpc::Auth; use dirs_next::home_dir; use std::ffi::{OsStr, OsString}; @@ -16,7 +17,7 @@ pub const ELECTRS_VERSION: &str = env!("CARGO_PKG_VERSION"); const DEFAULT_SERVER_ADDRESS: [u8; 4] = [127, 0, 0, 1]; // by default, serve on IPv4 localhost mod internal { - #![allow(unused_attributes, unused_imports, clippy::enum_variant_names)] + #![allow(unused_attributes, unused_imports)] include!(concat!(env!("OUT_DIR"), "/configure_me_config.rs")); } @@ -121,20 +122,21 @@ impl From for Network { } /// Parsed and post-processed configuration -#[allow(dead_code)] #[derive(Debug)] pub struct Config { // See below for the documentation of each field: pub network: Network, - pub db_dir: PathBuf, + pub db_path: PathBuf, pub db_log_dir: Option, pub db_parallelism: u8, pub daemon_auth: SensitiveAuth, pub daemon_rpc_addr: SocketAddr, + pub daemon_p2p_addr: SocketAddr, pub electrum_rpc_addr: SocketAddr, pub monitoring_addr: SocketAddr, pub wait_duration: Duration, pub jsonrpc_timeout: Duration, + pub index_batch_size: usize, pub index_lookup_limit: Option, pub reindex_last_blocks: usize, pub auto_reindex: bool, @@ -143,6 +145,7 @@ pub struct Config { pub skip_block_download_wait: bool, pub disable_electrum_rpc: bool, pub server_banner: String, + pub magic: Magic, } pub struct SensitiveAuth(pub Auth); @@ -195,6 +198,16 @@ impl Config { internal::prelude::Config::including_optional_config_files(default_config_files()) .unwrap_or_exit(); + let db_subdir = match config.network { + Network::Bitcoin => "bitcoin", + Network::Testnet => "testnet", + Network::Testnet4 => "testnet4", + Network::Regtest => "regtest", + Network::Signet => "signet", + }; + + config.db_dir.push(db_subdir); + let default_daemon_rpc_port = match config.network { Network::Bitcoin => 8332, Network::Testnet => 18332, @@ -202,6 +215,13 @@ impl Config { Network::Regtest => 18443, Network::Signet => 38332, }; + let default_daemon_p2p_port = match config.network { + Network::Bitcoin => 8333, + Network::Testnet => 18333, + Network::Testnet4 => 48333, + Network::Regtest => 18444, + Network::Signet => 38333, + }; let default_electrum_port = match config.network { Network::Bitcoin => 50001, Network::Testnet => 60001, @@ -217,10 +237,25 @@ impl Config { Network::Signet => 34224, }; + let magic = match config.magic { + Some(magic_hex) => magic_hex.parse().unwrap_or_else(|error| { + eprintln!( + "Error: magic '{}' is not a valid hex string: {}", + magic_hex, error + ); + std::process::exit(1); + }), + None => config.network.magic(), + }; + let daemon_rpc_addr: SocketAddr = config.daemon_rpc_addr.map_or( (DEFAULT_SERVER_ADDRESS, default_daemon_rpc_port).into(), ResolvAddr::resolve_or_exit, ); + let daemon_p2p_addr: SocketAddr = config.daemon_p2p_addr.map_or( + (DEFAULT_SERVER_ADDRESS, default_daemon_p2p_port).into(), + ResolvAddr::resolve_or_exit, + ); let electrum_rpc_addr: SocketAddr = config.electrum_rpc_addr.map_or( (DEFAULT_SERVER_ADDRESS, default_electrum_port).into(), ResolvAddr::resolve_or_exit, @@ -304,15 +339,17 @@ impl Config { let config = Config { network: config.network, - db_dir: config.db_dir, + db_path: config.db_dir, db_log_dir: config.db_log_dir, db_parallelism: config.db_parallelism, daemon_auth, daemon_rpc_addr, + daemon_p2p_addr, electrum_rpc_addr, monitoring_addr, wait_duration: Duration::from_secs(config.wait_duration_secs), jsonrpc_timeout: Duration::from_secs(config.jsonrpc_timeout_secs), + index_batch_size: config.index_batch_size, index_lookup_limit, reindex_last_blocks: config.reindex_last_blocks, auto_reindex: config.auto_reindex, @@ -321,6 +358,7 @@ impl Config { skip_block_download_wait: config.skip_block_download_wait, disable_electrum_rpc: config.disable_electrum_rpc, server_banner: config.server_banner, + magic, }; eprintln!( "Starting electrs {} on {} {} with {:?}", @@ -344,6 +382,9 @@ mod tests { #[test] fn test_auth_debug() { + let auth = Auth::None; + assert_eq!(format!("{:?}", SensitiveAuth(auth)), "None"); + let auth = Auth::CookieFile(Path::new("/foo/bar/.cookie").to_path_buf()); assert_eq!( format!("{:?}", SensitiveAuth(auth)), diff --git a/src/daemon.rs b/src/daemon.rs index f31337e..40df169 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -1,10 +1,11 @@ use anyhow::{Context, Result}; -use crate::bitcoin::{ - self, consensus::deserialize, consensus::encode::serialize_hex, hashes::hex::FromHex, Amount, - BlockHash, Transaction, Txid, -}; -use crate::types::Auth; +use bitcoin::consensus::encode::serialize_hex; +use bitcoin::{consensus::deserialize, hashes::hex::FromHex}; +use bitcoin::{Amount, BlockHash, Transaction, Txid}; +use bitcoincore_rpc::{json, jsonrpc, Auth, Client, RpcApi}; +use crossbeam_channel::Receiver; +use parking_lot::Mutex; use serde::Serialize; use serde_json::{json, value::RawValue, Value}; @@ -12,62 +13,52 @@ use std::fs::File; use std::io::Read; use std::path::Path; -use crate::{config::Config, signals::ExitFlag}; +use crate::{ + chain::{Chain, NewHeader}, + config::Config, + metrics::Metrics, + p2p::Connection, + signals::ExitFlag, + types::SerBlock, +}; enum PollResult { Done(Result<()>), Retry, } -#[derive(Deserialize)] -pub struct GetMempoolEntryResultFees { - /// Transaction fee in BTC - #[serde(with = "bitcoin::amount::serde::as_btc")] - pub base: Amount, -} - -#[derive(Deserialize)] -pub struct GetMempoolEntryResult { - /// Virtual transaction size as defined in BIP 141. This is different from actual serialized - /// size for witness transactions as witness data is discounted. - #[serde(alias = "size")] - pub vsize: u64, - /// Fee information - pub fees: GetMempoolEntryResultFees, - /// Unconfirmed transactions used as inputs for this transaction - pub depends: Vec, -} - -#[derive(Deserialize)] -pub struct GetBlockchainInfoResult { - pub blocks: u64, - pub headers: u64, - pub initialblockdownload: bool, - pub pruned: bool, -} - -#[derive(Deserialize)] -pub struct EstimateSmartFeeResult { - /// Estimate fee rate in BTC/kB. - #[serde( - default, - rename = "feerate", - skip_serializing_if = "Option::is_none", - with = "bitcoin::amount::serde::as_btc::opt" - )] - pub fee_rate: Option, -} - -#[derive(Deserialize)] -pub struct GetNetworkInfoResult { - #[serde(rename = "relayfee", with = "bitcoin::amount::serde::as_btc")] - pub relay_fee: Amount, -} - -#[derive(Deserialize)] -pub struct GetMempoolInfoResult { - /// True if the mempool is fully loaded - pub loaded: Option, +fn rpc_poll(client: &mut Client, skip_block_download_wait: bool) -> PollResult { + match client.get_blockchain_info() { + Ok(info) => { + if skip_block_download_wait { + // bitcoind RPC is available, don't wait for block download to finish + return PollResult::Done(Ok(())); + } + let left_blocks = info.headers - info.blocks; + if info.initial_block_download || left_blocks > 0 { + info!( + "waiting for {} blocks to download{}", + left_blocks, + if info.initial_block_download { + " (IBD)" + } else { + "" + } + ); + return PollResult::Retry; + } + PollResult::Done(Ok(())) + } + Err(err) => { + if let Some(e) = extract_bitcoind_error(&err) { + if e.code == -28 { + debug!("waiting for RPC warmup: {}", e.message); + return PollResult::Retry; + } + } + PollResult::Done(Err(err).context("daemon not available")) + } + } } fn read_cookie(path: &Path) -> Result<(String, String)> { @@ -88,7 +79,7 @@ fn read_cookie(path: &Path) -> Result<(String, String)> { Ok((parts[0].to_owned(), parts[1].to_owned())) } -fn jsonrpc_client(config: &Config) -> Result { +fn rpc_connect(config: &Config) -> Result { let rpc_url = format!("http://{}", config.daemon_rpc_addr); // Allow RPC calls to take longer before timing out. // See https://github.com/romanz/electrs/issues/495 for more details. @@ -96,30 +87,36 @@ fn jsonrpc_client(config: &Config) -> Result { .url(&rpc_url)? .timeout(config.jsonrpc_timeout); let builder = match config.daemon_auth.get_auth() { + Auth::None => builder, Auth::UserPass(user, pass) => builder.auth(user, Some(pass)), Auth::CookieFile(path) => { let (user, pass) = read_cookie(&path)?; builder.auth(user, Some(pass)) } }; - Ok(jsonrpc::Client::with_transport(builder.build())) + Ok(Client::from_jsonrpc(jsonrpc::Client::with_transport( + builder.build(), + ))) } pub struct Daemon { - client: jsonrpc::Client, + p2p: Mutex, + rpc: Client, } impl Daemon { - pub(crate) fn connect(config: &Config, exit_flag: &ExitFlag) -> Result { - let daemon = Daemon { - client: jsonrpc_client(config)?, - }; + pub(crate) fn connect( + config: &Config, + exit_flag: &ExitFlag, + metrics: &Metrics, + ) -> Result { + let mut rpc = rpc_connect(config)?; loop { exit_flag .poll() .context("bitcoin RPC polling interrupted")?; - match daemon.rpc_poll(config.skip_block_download_wait) { + match rpc_poll(&mut rpc, config.skip_block_download_wait) { PollResult::Done(result) => { result.context("bitcoind RPC polling failed")?; break; // on success, finish polling @@ -130,85 +127,33 @@ impl Daemon { } } - Ok(daemon) - } - - fn call serde::de::Deserialize<'a>>( - &self, - cmd: &str, - args: &[serde_json::Value], - ) -> Result { - let raw = serde_json::value::to_raw_value(args)?; - let req = self.client.build_request(cmd, Some(&*raw)); - let resp = self.client.send_request(req)?; - resp.result() - } - - fn batch_request(&self, name: &str, items: &[T]) -> Result>> - where - T: Serialize, - { - debug!("calling {} on {} items", name, items.len()); - let args: Vec> = items - .iter() - .map(|item| jsonrpc::try_arg([item]).context("failed to serialize into JSON")) - .collect::>>()?; - let reqs: Vec = args - .iter() - .map(|arg| self.client.build_request(name, Some(arg))) - .collect(); - match self.client.send_batch(&reqs) { - Ok(values) => { - assert_eq!(items.len(), values.len()); - Ok(values) - } - Err(err) => bail!("batch {} request failed: {}", name, err), + let network_info = rpc.get_network_info()?; + if network_info.version < 21_00_00 { + bail!("electrs requires bitcoind 0.21+"); } - } - - fn rpc_poll(&self, skip_block_download_wait: bool) -> PollResult { - match self.call::("getblockchaininfo", &[]) { - Ok(info) => { - if info.pruned { - return PollResult::Done(Err(anyhow!( - "electrs requires non-pruned bitcoind node" - ))); - } - if skip_block_download_wait { - // bitcoind RPC is available, don't wait for block download to finish - return PollResult::Done(Ok(())); - } - let left_blocks = info.headers - info.blocks; - if info.initialblockdownload || left_blocks > 0 { - info!( - "waiting for {} blocks to download{}", - left_blocks, - if info.initialblockdownload { - " (IBD)" - } else { - "" - } - ); - return PollResult::Retry; - } - PollResult::Done(Ok(())) - } - Err(err) => { - if let Some(e) = extract_bitcoind_error(&err) { - if e.code == -28 { - debug!("waiting for RPC warmup: {}", e.message); - return PollResult::Retry; - } - } - PollResult::Done(Err(err).context("daemon not available")) - } + if !network_info.network_active { + bail!("electrs requires active bitcoind p2p network"); } + let info = rpc.get_blockchain_info()?; + if info.pruned { + bail!("electrs requires non-pruned bitcoind node"); + } + + let p2p = Mutex::new(Connection::connect( + config.daemon_p2p_addr, + metrics, + config.magic, + )?); + Ok(Self { p2p, rpc }) } pub(crate) fn estimate_fee(&self, nblocks: u16) -> Result> { - let res = - self.call::("estimatesmartfee", &[json!(nblocks), Value::Null]); - if let Err(jsonrpc::Error::Rpc(jsonrpc::error::RpcError { code: -32603, .. })) = res { + let res = self.rpc.estimate_smart_fee(nblocks, None); + if let Err(bitcoincore_rpc::Error::JsonRpc(jsonrpc::Error::Rpc(RpcError { + code: -32603, + .. + }))) = res + { return Ok(None); // don't fail when fee estimation is disabled (e.g. with `-blocksonly=1`) } Ok(res.context("failed to estimate fee")?.fee_rate) @@ -216,64 +161,91 @@ impl Daemon { pub(crate) fn get_relay_fee(&self) -> Result { Ok(self - .call::("getnetworkinfo", &[]) + .rpc + .get_network_info() .context("failed to get relay fee")? .relay_fee) } pub(crate) fn broadcast(&self, tx: &Transaction) -> Result { - self.call("sendrawtransaction", &[json!(serialize_hex(tx))]) + self.rpc + .send_raw_transaction(tx) .context("failed to broadcast transaction") } pub(crate) fn submitpackage(&self, txs: &[Transaction]) -> Result { let package: Vec = txs.iter().map(serialize_hex).collect(); - self.call("submitpackage", &[json!(package)]) + self.rpc + .call("submitpackage", &[json!(package)]) .context("failed to submitpackage package") } + pub(crate) fn get_transaction_info( + &self, + txid: &Txid, + blockhash: Option, + ) -> Result { + // No need to parse the resulting JSON, just return it as-is to the client. + self.rpc + .call( + "getrawtransaction", + &[json!(txid), json!(true), json!(blockhash)], + ) + .context("failed to get transaction info") + } + + pub(crate) fn get_transaction_hex( + &self, + txid: &Txid, + blockhash: Option, + ) -> Result { + use bitcoin::consensus::serde::{hex::Lower, Hex, With}; + + let tx = self.get_transaction(txid, blockhash)?; + #[derive(serde::Serialize)] + #[serde(transparent)] + struct TxAsHex(#[serde(with = "With::>")] Transaction); + serde_json::to_value(TxAsHex(tx)).map_err(Into::into) + } + pub(crate) fn get_transaction( &self, txid: &Txid, blockhash: Option, - verbose: bool, - ) -> Result { - self.call( - "getrawtransaction", - &[json!(txid), json!(verbose), json!(blockhash)], - ) - .context("failed to get transaction") + ) -> Result { + self.rpc + .get_raw_transaction(txid, blockhash.as_ref()) + .context("failed to get transaction") } pub(crate) fn get_block_txids(&self, blockhash: BlockHash) -> Result> { - #[derive(Clone, PartialEq, Debug, Deserialize, Serialize)] - struct GetBlockResult { - pub tx: Vec, - } - let res: GetBlockResult = self - .call("getblock", &[json!(blockhash), json!(1)]) - .context("failed to get block txids")?; - Ok(res.tx) + Ok(self + .rpc + .get_block_info(&blockhash) + .context("failed to get block txids")? + .tx) } - pub(crate) fn get_mempool_info(&self) -> Result { - self.call("getmempoolinfo", &[]) + pub(crate) fn get_mempool_info(&self) -> Result { + self.rpc + .get_mempool_info() .context("failed to get mempool info") } pub(crate) fn get_mempool_txids(&self) -> Result> { - self.call("getrawmempool", &[]) + self.rpc + .get_raw_mempool() .context("failed to get mempool txids") } pub(crate) fn get_mempool_entries( &self, txids: &[Txid], - ) -> Result>> { - let results = self.batch_request("getmempoolentry", txids)?; + ) -> Result>> { + let results = batch_request(self.rpc.get_jsonrpc_client(), "getmempoolentry", txids)?; Ok(results .into_iter() - .map(|r| match r?.result::() { + .map(|r| match r?.result::() { Ok(entry) => Some(entry), Err(err) => { debug!("failed to get mempool entry: {}", err); // probably due to RBF @@ -287,7 +259,7 @@ impl Daemon { &self, txids: &[Txid], ) -> Result>> { - let results = self.batch_request("getrawtransaction", txids)?; + let results = batch_request(self.rpc.get_jsonrpc_client(), "getrawtransaction", txids)?; Ok(results .into_iter() .map(|r| -> Option { @@ -315,11 +287,58 @@ impl Daemon { }) .collect()) } + + pub(crate) fn get_new_headers(&self, chain: &Chain) -> Result> { + self.p2p.lock().get_new_headers(chain) + } + + pub(crate) fn for_blocks(&self, blockhashes: B, func: F) -> Result<()> + where + B: IntoIterator, + F: FnMut(BlockHash, SerBlock), + { + self.p2p.lock().for_blocks(blockhashes, func) + } + + pub(crate) fn new_block_notification(&self) -> Receiver<()> { + self.p2p.lock().new_block_notification() + } } -pub(crate) fn extract_bitcoind_error(err: &jsonrpc::Error) -> Option<&jsonrpc::error::RpcError> { +pub(crate) type RpcError = bitcoincore_rpc::jsonrpc::error::RpcError; + +pub(crate) fn extract_bitcoind_error(err: &bitcoincore_rpc::Error) -> Option<&RpcError> { + use bitcoincore_rpc::{ + jsonrpc::error::Error::Rpc as ServerError, Error::JsonRpc as JsonRpcError, + }; match err { - jsonrpc::error::Error::Rpc(e) => Some(e), + JsonRpcError(ServerError(e)) => Some(e), _ => None, } } + +fn batch_request( + client: &jsonrpc::Client, + name: &str, + items: &[T], +) -> Result>> +where + T: Serialize, +{ + debug!("calling {} on {} items", name, items.len()); + let args: Vec> = items + .iter() + .map(|item| jsonrpc::try_arg([item]).context("failed to serialize into JSON")) + .collect::>>()?; + let reqs: Vec = args + .iter() + .map(|arg| client.build_request(name, Some(arg))) + .collect(); + match client.send_batch(&reqs) { + Ok(values) => { + assert_eq!(items.len(), values.len()); + Ok(values) + } + Err(err) => bail!("batch {} request failed: {}", name, err), + } +} diff --git a/src/db.rs b/src/db.rs new file mode 100644 index 0000000..c6b156b --- /dev/null +++ b/src/db.rs @@ -0,0 +1,554 @@ +use anyhow::{Context, Result}; +use rust_rocksdb as rocksdb; + +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; + +use crate::types::{HashPrefix, SerializedHashPrefixRow, SerializedHeaderRow}; + +#[derive(Default)] +pub(crate) struct WriteBatch { + pub(crate) tip_row: [u8; 32], + pub(crate) header_rows: Vec, + pub(crate) funding_rows: Vec, + pub(crate) spending_rows: Vec, + pub(crate) txid_rows: Vec, +} + +impl WriteBatch { + pub(crate) fn sort(&mut self) { + self.header_rows.sort_unstable(); + self.funding_rows.sort_unstable(); + self.spending_rows.sort_unstable(); + self.txid_rows.sort_unstable(); + } +} + +/// RocksDB wrapper for index storage +pub struct DBStore { + db: rocksdb::DB, + bulk_import: AtomicBool, +} + +const CONFIG_CF: &str = "config"; +const HEADERS_CF: &str = "headers"; +const TXID_CF: &str = "txid"; +const FUNDING_CF: &str = "funding"; +const SPENDING_CF: &str = "spending"; + +const COLUMN_FAMILIES: &[&str] = &[CONFIG_CF, HEADERS_CF, TXID_CF, FUNDING_CF, SPENDING_CF]; + +const CONFIG_KEY: &str = "C"; +const TIP_KEY: &[u8] = b"T"; + +// Taken from https://github.com/facebook/rocksdb/blob/master/include/rocksdb/db.h#L654-L689 +const DB_PROPERTIES: &[&str] = &[ + "rocksdb.num-immutable-mem-table", + "rocksdb.mem-table-flush-pending", + "rocksdb.compaction-pending", + "rocksdb.background-errors", + "rocksdb.cur-size-active-mem-table", + "rocksdb.cur-size-all-mem-tables", + "rocksdb.size-all-mem-tables", + "rocksdb.num-entries-active-mem-table", + "rocksdb.num-entries-imm-mem-tables", + "rocksdb.num-deletes-active-mem-table", + "rocksdb.num-deletes-imm-mem-tables", + "rocksdb.estimate-num-keys", + "rocksdb.estimate-table-readers-mem", + "rocksdb.is-file-deletions-enabled", + "rocksdb.num-snapshots", + "rocksdb.oldest-snapshot-time", + "rocksdb.num-live-versions", + "rocksdb.current-super-version-number", + "rocksdb.estimate-live-data-size", + "rocksdb.min-log-number-to-keep", + "rocksdb.min-obsolete-sst-number-to-keep", + "rocksdb.total-sst-files-size", + "rocksdb.live-sst-files-size", + "rocksdb.base-level", + "rocksdb.estimate-pending-compaction-bytes", + "rocksdb.num-running-compactions", + "rocksdb.num-running-flushes", + "rocksdb.actual-delayed-write-rate", + "rocksdb.is-write-stopped", + "rocksdb.estimate-oldest-key-time", + "rocksdb.block-cache-capacity", + "rocksdb.block-cache-usage", + "rocksdb.block-cache-pinned-usage", +]; + +#[derive(Debug, Deserialize, Serialize)] +struct Config { + compacted: bool, + format: u64, +} + +const CURRENT_FORMAT: u64 = 0; + +impl Default for Config { + fn default() -> Self { + Config { + compacted: false, + format: CURRENT_FORMAT, + } + } +} + +fn default_opts(parallelism: u8) -> rocksdb::Options { + let mut block_opts = rocksdb::BlockBasedOptions::default(); + block_opts.set_checksum_type(rocksdb::ChecksumType::CRC32c); + + let mut opts = rocksdb::Options::default(); + opts.increase_parallelism(parallelism.into()); + opts.set_max_subcompactions(parallelism.into()); + + opts.set_keep_log_file_num(10); + opts.set_max_open_files(16); + opts.set_compaction_style(rocksdb::DBCompactionStyle::Level); + opts.set_compression_type(rocksdb::DBCompressionType::Zstd); + opts.set_target_file_size_base(256 << 20); + opts.set_write_buffer_size(256 << 20); + opts.set_disable_auto_compactions(true); // for initial bulk load + opts.set_advise_random_on_open(false); // bulk load uses sequential I/O + opts.set_prefix_extractor(rocksdb::SliceTransform::create_fixed_prefix(8)); + opts.set_block_based_table_factory(&block_opts); + opts +} + +impl DBStore { + fn create_cf_descriptors(parallelism: u8) -> Vec { + COLUMN_FAMILIES + .iter() + .map(|&name| rocksdb::ColumnFamilyDescriptor::new(name, default_opts(parallelism))) + .collect() + } + + fn open_internal(path: &Path, log_dir: Option<&Path>, parallelism: u8) -> Result { + let mut db_opts = default_opts(parallelism); + db_opts.create_if_missing(true); + db_opts.create_missing_column_families(true); + if let Some(d) = log_dir { + db_opts.set_db_log_dir(d); + } + + let db = rocksdb::DB::open_cf_descriptors( + &db_opts, + path, + Self::create_cf_descriptors(parallelism), + ) + .with_context(|| format!("failed to open DB: {}", path.display()))?; + let live_files = db.live_files()?; + info!( + "{:?}: {} SST files, {} GB, {} Grows", + path, + live_files.len(), + live_files.iter().map(|f| f.size).sum::() as f64 / 1e9, + live_files.iter().map(|f| f.num_entries).sum::() as f64 / 1e9 + ); + let store = DBStore { + db, + bulk_import: AtomicBool::new(true), + }; + Ok(store) + } + + fn is_legacy_format(&self) -> bool { + // In legacy DB format, all data was stored in a single (default) column family. + self.db + .iterator(rocksdb::IteratorMode::Start) + .next() + .is_some() + } + + /// Opens a new RocksDB at the specified location. + pub fn open( + path: &Path, + log_dir: Option<&Path>, + auto_reindex: bool, + parallelism: u8, + ) -> Result { + let mut store = Self::open_internal(path, log_dir, parallelism)?; + let config = store.get_config(); + debug!("DB {:?}", config); + let mut config = config.unwrap_or_default(); // use default config when DB is empty + + let reindex_cause = if store.is_legacy_format() { + Some("legacy format".to_owned()) + } else if config.format != CURRENT_FORMAT { + Some(format!( + "unsupported format {} != {}", + config.format, CURRENT_FORMAT + )) + } else { + None + }; + if let Some(cause) = reindex_cause { + if !auto_reindex { + bail!("re-index required due to {}", cause); + } + warn!( + "Database needs to be re-indexed due to {}, going to delete {}", + cause, + path.display() + ); + // close DB before deletion + drop(store); + rocksdb::DB::destroy(&default_opts(parallelism), path).with_context(|| { + format!( + "re-index required but the old database ({}) can not be deleted", + path.display() + ) + })?; + store = Self::open_internal(path, log_dir, parallelism)?; + config = Config::default(); // re-init config after dropping DB + } + if config.compacted { + store.start_compactions(); + } + store.set_config(config); + Ok(store) + } + + fn config_cf(&self) -> &rocksdb::ColumnFamily { + self.db.cf_handle(CONFIG_CF).expect("missing CONFIG_CF") + } + + fn funding_cf(&self) -> &rocksdb::ColumnFamily { + self.db.cf_handle(FUNDING_CF).expect("missing FUNDING_CF") + } + + fn spending_cf(&self) -> &rocksdb::ColumnFamily { + self.db.cf_handle(SPENDING_CF).expect("missing SPENDING_CF") + } + + fn txid_cf(&self) -> &rocksdb::ColumnFamily { + self.db.cf_handle(TXID_CF).expect("missing TXID_CF") + } + + fn headers_cf(&self) -> &rocksdb::ColumnFamily { + self.db.cf_handle(HEADERS_CF).expect("missing HEADERS_CF") + } + + pub(crate) fn iter_funding( + &self, + prefix: HashPrefix, + ) -> impl Iterator + '_ { + self.iter_prefix_cf(self.funding_cf(), prefix) + } + + pub(crate) fn iter_spending( + &self, + prefix: HashPrefix, + ) -> impl Iterator + '_ { + self.iter_prefix_cf(self.spending_cf(), prefix) + } + + pub(crate) fn iter_txid( + &self, + prefix: HashPrefix, + ) -> impl Iterator + '_ { + self.iter_prefix_cf(self.txid_cf(), prefix) + } + + fn iter_cf( + &self, + cf: &rocksdb::ColumnFamily, + readopts: rocksdb::ReadOptions, + prefix: Option, + ) -> impl Iterator + '_ { + DBIterator::new(self.db.raw_iterator_cf_opt(cf, readopts), prefix) + } + + fn iter_prefix_cf( + &self, + cf: &rocksdb::ColumnFamily, + prefix: HashPrefix, + ) -> impl Iterator + '_ { + let mut opts = rocksdb::ReadOptions::default(); + opts.set_prefix_same_as_start(true); // requires .set_prefix_extractor() above. + self.iter_cf(cf, opts, Some(prefix)) + } + + pub(crate) fn iter_headers(&self) -> impl Iterator + '_ { + let mut opts = rocksdb::ReadOptions::default(); + opts.fill_cache(false); + self.iter_cf(self.headers_cf(), opts, None) + } + + pub(crate) fn get_tip(&self) -> Option> { + self.db + .get_cf(self.headers_cf(), TIP_KEY) + .expect("get_tip failed") + } + + pub(crate) fn write(&self, batch: &WriteBatch) { + let mut db_batch = rocksdb::WriteBatch::default(); + let funding_cf = self.funding_cf(); + for key in &batch.funding_rows { + db_batch.put_cf(funding_cf, key, b""); + } + let spending_cf = self.spending_cf(); + for key in &batch.spending_rows { + db_batch.put_cf(spending_cf, key, b""); + } + let txid_cf = self.txid_cf(); + for key in &batch.txid_rows { + db_batch.put_cf(txid_cf, key, b""); + } + let headers_cf = self.headers_cf(); + for key in &batch.header_rows { + db_batch.put_cf(headers_cf, key, b""); + } + db_batch.put_cf(headers_cf, TIP_KEY, batch.tip_row); + + let mut opts = rocksdb::WriteOptions::new(); + let bulk_import = self.bulk_import.load(Ordering::Relaxed); + opts.set_sync(!bulk_import); + opts.disable_wal(bulk_import); + self.db.write_opt(db_batch, &opts).unwrap(); + } + + pub(crate) fn flush(&self) { + debug!("flushing DB column families"); + let mut config = self.get_config().unwrap_or_default(); + for name in COLUMN_FAMILIES { + let cf = self.db.cf_handle(name).expect("missing CF"); + self.db.flush_cf(cf).expect("CF flush failed"); + } + if !config.compacted { + for name in COLUMN_FAMILIES { + info!("starting {} compaction", name); + let cf = self.db.cf_handle(name).expect("missing CF"); + self.db.compact_range_cf(cf, None::<&[u8]>, None::<&[u8]>); + } + config.compacted = true; + self.set_config(config); + info!("finished full compaction"); + self.start_compactions(); + } + if log_enabled!(log::Level::Trace) { + let stats = self + .db + .property_value("rocksdb.dbstats") + .expect("failed to get property") + .expect("missing property"); + trace!("RocksDB stats: {}", stats); + } + } + + pub(crate) fn get_properties( + &self, + ) -> impl Iterator + '_ { + COLUMN_FAMILIES.iter().flat_map(move |cf_name| { + let cf = self.db.cf_handle(cf_name).expect("missing CF"); + DB_PROPERTIES.iter().filter_map(move |property_name| { + let value = self + .db + .property_int_value_cf(cf, *property_name) + .expect("failed to get property"); + Some((*cf_name, *property_name, value?)) + }) + }) + } + + fn start_compactions(&self) { + self.bulk_import.store(false, Ordering::Relaxed); + for name in COLUMN_FAMILIES { + let cf = self.db.cf_handle(name).expect("missing CF"); + self.db + .set_options_cf(cf, &[("disable_auto_compactions", "false")]) + .expect("failed to start auto-compactions"); + } + debug!("auto-compactions enabled"); + } + + fn set_config(&self, config: Config) { + let mut opts = rocksdb::WriteOptions::default(); + opts.set_sync(true); + opts.disable_wal(false); + let value = serde_json::to_vec(&config).expect("failed to serialize config"); + self.db + .put_cf_opt(self.config_cf(), CONFIG_KEY, value, &opts) + .expect("DB::put failed"); + } + + fn get_config(&self) -> Option { + self.db + .get_cf(self.config_cf(), CONFIG_KEY) + .expect("DB::get failed") + .map(|value| serde_json::from_slice(&value).expect("failed to deserialize Config")) + } +} + +struct DBIterator<'a, const N: usize> { + raw: rocksdb::DBRawIterator<'a>, + prefix: Option, + done: bool, +} + +impl<'a, const N: usize> DBIterator<'a, N> { + fn new(mut raw: rocksdb::DBRawIterator<'a>, prefix: Option) -> Self { + match prefix { + Some(key) => raw.seek(key), + None => raw.seek_to_first(), + }; + Self { + raw, + prefix, + done: false, + } + } +} + +impl Iterator for DBIterator<'_, N> { + type Item = [u8; N]; + + fn next(&mut self) -> Option { + while !self.done { + let key = match self.raw.key() { + Some(key) => key, + None => { + self.raw.status().expect("DB scan failed"); + break; // end of scan + } + }; + let prefix_match = match self.prefix { + Some(key_prefix) => key.starts_with(&key_prefix), + None => true, + }; + if !prefix_match { + break; // prefix mismatch + } + let result: Option<[u8; N]> = key.try_into().ok(); + self.raw.next(); + match result { + Some(value) => return Some(value), + None => continue, // skip keys with size != N + } + } + self.done = true; + None + } +} + +impl Drop for DBStore { + fn drop(&mut self) { + info!("closing DB at {}", self.db.path().display()); + } +} + +#[cfg(test)] +mod tests { + use super::{rocksdb, DBStore, WriteBatch, CURRENT_FORMAT}; + use std::ffi::{OsStr, OsString}; + use std::path::Path; + + #[test] + fn test_reindex_new_format() { + let dir = tempfile::tempdir().unwrap(); + { + let store = DBStore::open(dir.path(), None, false, 1).unwrap(); + let mut config = store.get_config().unwrap(); + config.format += 1; + store.set_config(config); + }; + assert_eq!( + DBStore::open(dir.path(), None, false, 1) + .err() + .unwrap() + .to_string(), + format!( + "re-index required due to unsupported format {} != {}", + CURRENT_FORMAT + 1, + CURRENT_FORMAT + ) + ); + { + let store = DBStore::open(dir.path(), None, true, 1).unwrap(); + store.flush(); + let config = store.get_config().unwrap(); + assert_eq!(config.format, CURRENT_FORMAT); + assert!(!store.is_legacy_format()); + } + } + + #[test] + fn test_reindex_legacy_format() { + let dir = tempfile::tempdir().unwrap(); + { + let mut db_opts = rocksdb::Options::default(); + db_opts.create_if_missing(true); + let db = rocksdb::DB::open(&db_opts, dir.path()).unwrap(); + db.put(b"F", b"").unwrap(); // insert legacy DB compaction marker (in 'default' column family) + }; + assert_eq!( + DBStore::open(dir.path(), None, false, 1) + .err() + .unwrap() + .to_string(), + format!("re-index required due to legacy format",) + ); + { + let store = DBStore::open(dir.path(), None, true, 1).unwrap(); + store.flush(); + let config = store.get_config().unwrap(); + assert_eq!(config.format, CURRENT_FORMAT); + } + } + + #[test] + fn test_db_prefix_scan() { + let dir = tempfile::tempdir().unwrap(); + let store = DBStore::open(dir.path(), None, true, 1).unwrap(); + + let items = [ + *b"ab ", + *b"abcdefgh ", + *b"abcdefghj ", + *b"abcdefghjk ", + *b"abcdefghxyz ", + *b"abcdefgi ", + *b"b ", + *b"c ", + ]; + + store.write(&WriteBatch { + txid_rows: items.to_vec(), + ..Default::default() + }); + + let rows = store.iter_txid(*b"abcdefgh"); + assert_eq!(rows.collect::>(), items[1..5]); + } + + #[test] + fn test_db_log_in_same_dir() { + let dir1 = tempfile::tempdir().unwrap(); + let _store = DBStore::open(dir1.path(), None, true, 1).unwrap(); + + // LOG file is created in dir1 + let dir_files = list_log_files(dir1.path()); + assert_eq!(dir_files, vec![OsStr::new("LOG")]); + + let dir2 = tempfile::tempdir().unwrap(); + let dir3 = tempfile::tempdir().unwrap(); + let _store = DBStore::open(dir2.path(), Some(dir3.path()), true, 1).unwrap(); + + // *_LOG file is not created in dir2, but in dir3 + let dir_files = list_log_files(dir2.path()); + assert_eq!(dir_files, Vec::::new()); + + let dir_files = list_log_files(dir3.path()); + assert_eq!(dir_files.len(), 1); + assert!(dir_files[0].to_str().unwrap().ends_with("_LOG")); + } + + fn list_log_files(path: &Path) -> Vec { + path.read_dir() + .unwrap() + .map(|e| e.unwrap().file_name()) + .filter(|e| e.to_str().unwrap().contains("LOG")) + .collect() + } +} diff --git a/src/electrum.rs b/src/electrum.rs index c03a4f5..fa67b1d 100644 --- a/src/electrum.rs +++ b/src/electrum.rs @@ -1,11 +1,11 @@ -use crate::bitcoin::{ +use anyhow::{bail, Context, Result}; +use bitcoin::{ consensus::{deserialize, encode::serialize_hex}, hashes::hex::FromHex, hex::DisplayHex, BlockHash, Transaction, Txid, }; -use anyhow::{bail, Context, Result}; -use bindex::ScriptHash; +use crossbeam_channel::Receiver; use rayon::prelude::*; use serde_derive::Deserialize; use serde_json::{self, json, Value}; @@ -13,17 +13,18 @@ use serde_json::{self, json, Value}; use std::collections::{hash_map::Entry, HashMap}; use std::fmt; use std::iter::FromIterator; -use std::net::SocketAddr; use std::str::FromStr; use crate::{ + cache::Cache, config::{Config, ELECTRS_VERSION}, - daemon::{extract_bitcoind_error, Daemon}, + daemon::{self, extract_bitcoind_error, Daemon}, merkle::Proof, metrics::{self, Histogram, Metrics}, signals::Signal, status::ScriptHashStatus, tracker::Tracker, + types::ScriptHash, }; const PROTOCOL_VERSION: &str = "1.4"; @@ -112,7 +113,7 @@ enum RpcError { Standard(StandardError), // Electrum-specific errors BadRequest(anyhow::Error), - DaemonError(jsonrpc::error::RpcError), + DaemonError(daemon::RpcError), UnavailableIndex, } @@ -144,11 +145,12 @@ impl RpcError { /// Electrum RPC handler pub struct Rpc { tracker: Tracker, + cache: Cache, rpc_duration: Histogram, daemon: Daemon, signal: Signal, banner: String, - addr: SocketAddr, + port: u16, } impl Rpc { @@ -163,14 +165,16 @@ impl Rpc { let tracker = Tracker::new(config, metrics)?; let signal = Signal::new(); - let daemon = Daemon::connect(config, signal.exit_flag())?; + let daemon = Daemon::connect(config, signal.exit_flag(), tracker.metrics())?; + let cache = Cache::new(tracker.metrics()); Ok(Self { tracker, + cache, rpc_duration, daemon, signal, banner: config.server_banner.clone(), - addr: config.electrum_rpc_addr, + port: config.electrum_rpc_addr.port(), }) } @@ -178,17 +182,24 @@ impl Rpc { &self.signal } + pub fn new_block_notification(&self) -> Receiver<()> { + self.daemon.new_block_notification() + } + pub fn sync(&mut self) -> Result { self.tracker.sync(&self.daemon, self.signal.exit_flag()) } pub fn update_client(&self, client: &mut Client) -> Result> { - let headers = self.tracker.headers(); + let chain = self.tracker.chain(); let mut notifications = client .scripthashes .par_iter_mut() .filter_map(|(scripthash, status)| -> Option> { - match self.tracker.update_scripthash_status(status) { + match self + .tracker + .update_scripthash_status(status, &self.daemon, &self.cache) + { Ok(true) => Some(Ok(notification( "blockchain.scripthash.subscribe", &[json!(scripthash), json!(status.statushash())], @@ -201,11 +212,11 @@ impl Rpc { .context("failed to update status")?; if let Some(old_tip) = client.tip { - let new_tip = headers.tip().unwrap(); - if old_tip != new_tip.hash() { - client.tip = Some(new_tip.hash()); - let height = headers.tip_height().unwrap(); - let header = new_tip.header(); + let new_tip = self.tracker.chain().tip(); + if old_tip != new_tip { + client.tip = Some(new_tip); + let height = chain.height(); + let header = chain.get_block_header(height).unwrap(); notifications.push(notification( "blockchain.headers.subscribe", &[json!({"hex": serialize_hex(&header), "height": height})], @@ -216,36 +227,34 @@ impl Rpc { } fn headers_subscribe(&self, client: &mut Client) -> Result { - let headers = self.tracker.headers(); - let height = headers.tip_height().unwrap(); - let header = headers.tip().unwrap(); - client.tip = Some(header.hash()); - Ok(json!({"hex": serialize_hex(header.header()), "height": height})) + let chain = self.tracker.chain(); + client.tip = Some(chain.tip()); + let height = chain.height(); + let header = chain.get_block_header(height).unwrap(); + Ok(json!({"hex": serialize_hex(header), "height": height})) } fn block_header(&self, (height,): (usize,)) -> Result { - let header = match self.tracker.headers().iter_headers().nth(height) { + let chain = self.tracker.chain(); + let header = match chain.get_block_header(height) { None => bail!("no header at {}", height), Some(header) => header, }; - Ok(json!(serialize_hex(header.header()))) + Ok(json!(serialize_hex(header))) } fn block_headers(&self, (start_height, count): (usize, usize)) -> Result { - let headers = self.tracker.headers(); + let chain = self.tracker.chain(); let max_count = 2016usize; // return only the available block headers let end_height = std::cmp::min( - headers.tip_height().map_or(0, |h| h + 1), + chain.height() + 1, start_height + std::cmp::min(count, max_count), ); let heights = start_height..end_height; let count = heights.len(); - let hex_headers = headers - .iter_headers() - .skip(start_height) - .take(count) - .map(|h| serialize_hex(h.header())); + let hex_headers = + heights.filter_map(|height| chain.get_block_header(height).map(serialize_hex)); Ok(json!({"count": count, "hex": String::from_iter(hex_headers), "max": max_count})) } @@ -367,7 +376,8 @@ impl Rpc { fn new_status(&self, scripthash: ScriptHash) -> Result { let mut status = ScriptHashStatus::new(scripthash); - self.tracker.update_scripthash_status(&mut status)?; + self.tracker + .update_scripthash_status(&mut status, &self.daemon, &self.cache)?; Ok(status) } @@ -411,26 +421,35 @@ impl Rpc { if verbose { let blockhash = self .tracker - .lookup_transaction(txid)? + .lookup_transaction(&self.daemon, txid)? .map(|(blockhash, _tx)| blockhash); - return self.daemon.get_transaction(&txid, blockhash, verbose); + return self.daemon.get_transaction_info(&txid, blockhash); } + // if the scripthash was subscribed, tx should be cached + if let Some(tx_hex) = self + .cache + .get_tx(&txid, |tx_bytes| tx_bytes.to_lower_hex_string()) + { + return Ok(json!(tx_hex)); + } + debug!("tx cache miss: txid={}", txid); // use internal index to load confirmed transaction if let Some(tx_hex) = self .tracker - .lookup_transaction(txid)? + .lookup_transaction(&self.daemon, txid)? .map(|(_blockhash, tx)| tx.to_lower_hex_string()) { return Ok(json!(tx_hex)); } // load unconfirmed transaction via RPC - Ok(json!(self.daemon.get_transaction(&txid, None, verbose)?)) + Ok(json!(self.daemon.get_transaction_hex(&txid, None)?)) } fn transaction_get_merkle(&self, (txid, height): &(Txid, usize)) -> Result { - let blockhash = match self.tracker.headers().iter_headers().nth(*height) { + let chain = self.tracker.chain(); + let blockhash = match chain.get_block_hash(*height) { None => bail!("missing block at {}", height), - Some(header) => header.hash(), + Some(blockhash) => blockhash, }; let txids = self.daemon.get_block_txids(blockhash)?; match txids.iter().position(|current_txid| *current_txid == *txid) { @@ -450,9 +469,10 @@ impl Rpc { &self, (height, tx_pos, merkle): (usize, usize, bool), ) -> Result { - let blockhash = match self.tracker.headers().iter_headers().nth(height) { + let chain = self.tracker.chain(); + let blockhash = match chain.get_block_hash(height) { None => bail!("missing block at {}", height), - Some(header) => header.hash(), + Some(blockhash) => blockhash, }; let txids = self.daemon.get_block_txids(blockhash)?; if tx_pos >= txids.len() { @@ -486,12 +506,8 @@ impl Rpc { fn features(&self) -> Result { Ok(json!({ - "genesis_hash": self.tracker.headers().genesis().unwrap().hash(), - "hosts": { - self.addr.ip().to_string(): { - "tcp_port": self.addr.port() - } - }, + "genesis_hash": self.tracker.chain().get_block_hash(0), + "hosts": { "tcp_port": self.port }, "protocol_max": PROTOCOL_VERSION, "protocol_min": PROTOCOL_VERSION, "pruning": null, @@ -697,7 +713,7 @@ impl Call { Err(err) => { warn!("RPC {} failed: {:#}", self.method, err); match err - .downcast_ref::() + .downcast_ref::() .and_then(extract_bitcoind_error) { Some(e) => error_msg(&self.id, RpcError::DaemonError(e.clone())), diff --git a/src/index.rs b/src/index.rs new file mode 100644 index 0000000..5c30d9c --- /dev/null +++ b/src/index.rs @@ -0,0 +1,323 @@ +use anyhow::{Context, Result}; +use bitcoin::consensus::{deserialize, Decodable, Encodable}; +use bitcoin::hashes::Hash; +use bitcoin::{BlockHash, OutPoint, Txid}; +use bitcoin_slices::{bsl, Visit, Visitor}; +use std::ops::ControlFlow; +use std::thread; + +use crate::{ + chain::{Chain, NewHeader}, + daemon::Daemon, + db::{DBStore, WriteBatch}, + metrics::{self, Gauge, Histogram, Metrics}, + signals::ExitFlag, + types::{ + bsl_txid, HashPrefixRow, HeaderRow, ScriptHash, ScriptHashRow, SerBlock, SpendingPrefixRow, + TxidRow, + }, +}; + +#[derive(Clone)] +struct Stats { + update_duration: Histogram, + update_size: Histogram, + height: Gauge, + db_properties: Gauge, +} + +impl Stats { + fn new(metrics: &Metrics) -> Self { + Self { + update_duration: metrics.histogram_vec( + "index_update_duration", + "Index update duration (in seconds)", + "step", + metrics::default_duration_buckets(), + ), + update_size: metrics.histogram_vec( + "index_update_size", + "Index update size (in bytes)", + "step", + metrics::default_size_buckets(), + ), + height: metrics.gauge("index_height", "Indexed block height", "type"), + db_properties: metrics.gauge("index_db_properties", "Index DB properties", "name"), + } + } + + fn observe_duration(&self, label: &str, f: impl FnOnce() -> T) -> T { + self.update_duration.observe_duration(label, f) + } + + fn observe_size(&self, label: &str, rows: &[[u8; N]]) { + self.update_size.observe(label, (rows.len() * N) as f64); + } + + fn observe_batch(&self, batch: &WriteBatch) { + self.observe_size("write_funding_rows", &batch.funding_rows); + self.observe_size("write_spending_rows", &batch.spending_rows); + self.observe_size("write_txid_rows", &batch.txid_rows); + self.observe_size("write_header_rows", &batch.header_rows); + debug!( + "writing {} funding and {} spending rows from {} transactions, {} blocks", + batch.funding_rows.len(), + batch.spending_rows.len(), + batch.txid_rows.len(), + batch.header_rows.len() + ); + } + + fn observe_chain(&self, chain: &Chain) { + self.height.set("tip", chain.height() as f64); + } + + fn observe_db(&self, store: &DBStore) { + for (cf, name, value) in store.get_properties() { + self.db_properties + .set(&format!("{}:{}", name, cf), value as f64); + } + } +} + +/// Confirmed transactions' address index +pub struct Index { + store: DBStore, + batch_size: usize, + lookup_limit: Option, + chain: Chain, + stats: Stats, + is_ready: bool, + flush_needed: bool, +} + +impl Index { + pub(crate) fn load( + store: DBStore, + mut chain: Chain, + metrics: &Metrics, + batch_size: usize, + lookup_limit: Option, + reindex_last_blocks: usize, + ) -> Result { + if let Some(row) = store.get_tip() { + let tip = deserialize(&row).expect("invalid tip"); + let headers = store + .iter_headers() + .map(|row| HeaderRow::from_db_row(row).header); + chain.load(headers, tip); + chain.drop_last_headers(reindex_last_blocks); + }; + let stats = Stats::new(metrics); + stats.observe_chain(&chain); + stats.observe_db(&store); + Ok(Index { + store, + batch_size, + lookup_limit, + chain, + stats, + is_ready: false, + flush_needed: false, + }) + } + + pub(crate) fn chain(&self) -> &Chain { + &self.chain + } + + pub(crate) fn limit_result(&self, entries: impl Iterator) -> Result> { + let mut entries = entries.fuse(); + let result: Vec = match self.lookup_limit { + Some(lookup_limit) => entries.by_ref().take(lookup_limit).collect(), + None => entries.by_ref().collect(), + }; + if entries.next().is_some() { + bail!(">{} index entries, query may take too long", result.len()) + } + Ok(result) + } + + pub(crate) fn filter_by_txid(&self, txid: Txid) -> impl Iterator + '_ { + self.store + .iter_txid(TxidRow::scan_prefix(txid)) + .map(|row| HashPrefixRow::from_db_row(row).height()) + .filter_map(move |height| self.chain.get_block_hash(height)) + } + + pub(crate) fn filter_by_funding( + &self, + scripthash: ScriptHash, + ) -> impl Iterator + '_ { + self.store + .iter_funding(ScriptHashRow::scan_prefix(scripthash)) + .map(|row| HashPrefixRow::from_db_row(row).height()) + .filter_map(move |height| self.chain.get_block_hash(height)) + } + + pub(crate) fn filter_by_spending( + &self, + outpoint: OutPoint, + ) -> impl Iterator + '_ { + self.store + .iter_spending(SpendingPrefixRow::scan_prefix(outpoint)) + .map(|row| HashPrefixRow::from_db_row(row).height()) + .filter_map(move |height| self.chain.get_block_hash(height)) + } + + // Return `Ok(true)` when the chain is fully synced and the index is compacted. + pub(crate) fn sync(&mut self, daemon: &Daemon, exit_flag: &ExitFlag) -> Result { + let new_headers = self + .stats + .observe_duration("headers", || daemon.get_new_headers(&self.chain))?; + match (new_headers.first(), new_headers.last()) { + (Some(first), Some(last)) => { + let count = new_headers.len(); + info!( + "indexing {} blocks: [{}..{}]", + count, + first.height(), + last.height() + ); + } + _ => { + if self.flush_needed { + self.store.flush(); // full compaction is performed on the first flush call + self.flush_needed = false; + } + self.is_ready = true; + return Ok(true); // no more blocks to index (done for now) + } + } + + thread::scope(|scope| -> Result<()> { + let (tx, rx) = crossbeam_channel::bounded(1); + + let chunks = new_headers.chunks(self.batch_size); + let index = &self; // to be moved into reader thread + let reader = thread::Builder::new() + .name("index_build".into()) + .spawn_scoped(scope, move || -> Result<()> { + for chunk in chunks { + exit_flag.poll().with_context(|| { + format!( + "indexing interrupted at height: {}", + chunk.first().unwrap().height() + ) + })?; + let batch = index.index_blocks(daemon, chunk)?; + tx.send(batch).context("writer disconnected")?; + } + Ok(()) // `tx` is dropped, to stop the iteration on `rx` + }) + .expect("spawn failed"); + + let index = &self; // to be moved into writer thread + let writer = thread::Builder::new() + .name("index_write".into()) + .spawn_scoped(scope, move || { + let stats = &index.stats; + for mut batch in rx { + stats.observe_duration("sort", || batch.sort()); // pre-sort to optimize DB writes + stats.observe_batch(&batch); + stats.observe_duration("write", || index.store.write(&batch)); + stats.observe_db(&index.store); + } + }) + .expect("spawn failed"); + + reader.join().expect("reader thread panic")?; + writer.join().expect("writer thread panic"); + Ok(()) + })?; + self.chain.update(new_headers); + self.stats.observe_chain(&self.chain); + self.flush_needed = true; + Ok(false) // sync is not done + } + + fn index_blocks(&self, daemon: &Daemon, chunk: &[NewHeader]) -> Result { + let blockhashes: Vec = chunk.iter().map(|h| h.hash()).collect(); + let mut heights = chunk.iter().map(|h| h.height()); + + let mut batch = WriteBatch::default(); + + daemon.for_blocks(blockhashes, |blockhash, block| { + let height = heights.next().expect("unexpected block"); + self.stats.observe_duration("block", || { + index_single_block(blockhash, block, height, &mut batch); + }); + self.stats.height.set("tip", height as f64); + })?; + let heights: Vec<_> = heights.collect(); + assert!( + heights.is_empty(), + "some blocks were not indexed: {:?}", + heights + ); + Ok(batch) + } + + pub(crate) fn is_ready(&self) -> bool { + self.is_ready + } +} + +fn index_single_block( + block_hash: BlockHash, + block: SerBlock, + height: usize, + batch: &mut WriteBatch, +) { + struct IndexBlockVisitor<'a> { + batch: &'a mut WriteBatch, + height: usize, + } + + impl Visitor for IndexBlockVisitor<'_> { + fn visit_transaction(&mut self, tx: &bsl::Transaction) -> ControlFlow<()> { + let txid = bsl_txid(tx); + self.batch + .txid_rows + .push(TxidRow::row(txid, self.height).to_db_row()); + ControlFlow::Continue(()) + } + + fn visit_tx_out(&mut self, _vout: usize, tx_out: &bsl::TxOut) -> ControlFlow<()> { + let script = bitcoin::Script::from_bytes(tx_out.script_pubkey()); + // skip indexing unspendable outputs + if !script.is_op_return() { + let row = ScriptHashRow::row(ScriptHash::new(script), self.height); + self.batch.funding_rows.push(row.to_db_row()); + } + ControlFlow::Continue(()) + } + + fn visit_tx_in(&mut self, _vin: usize, tx_in: &bsl::TxIn) -> ControlFlow<()> { + let prevout: OutPoint = tx_in.prevout().into(); + // skip indexing coinbase transactions' input + if !prevout.is_null() { + let row = SpendingPrefixRow::row(prevout, self.height); + self.batch.spending_rows.push(row.to_db_row()); + } + ControlFlow::Continue(()) + } + + fn visit_block_header(&mut self, header: &bsl::BlockHeader) -> ControlFlow<()> { + let header = bitcoin::block::Header::consensus_decode(&mut header.as_ref()) + .expect("block header was already validated"); + self.batch + .header_rows + .push(HeaderRow::new(header).to_db_row()); + ControlFlow::Continue(()) + } + } + + let mut index_block = IndexBlockVisitor { batch, height }; + bsl::Block::visit(&block, &mut index_block).expect("core returned invalid block"); + + let len = block_hash + .consensus_encode(&mut (&mut batch.tip_row as &mut [u8])) + .expect("in-memory writers don't error"); + debug_assert_eq!(len, BlockHash::LEN); +} diff --git a/src/lib.rs b/src/lib.rs index 9b8fad8..1458f06 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,12 +7,17 @@ extern crate log; #[macro_use] extern crate serde_derive; +mod cache; +mod chain; mod config; mod daemon; +mod db; mod electrum; +mod index; mod mempool; mod merkle; mod metrics; +mod p2p; mod server; mod signals; mod status; @@ -21,6 +26,3 @@ mod tracker; mod types; pub use server::run; - -use bindex::bitcoin; -use bindex::bitcoin_slices; diff --git a/src/mempool.rs b/src/mempool.rs index f797a14..aa96fd7 100644 --- a/src/mempool.rs +++ b/src/mempool.rs @@ -5,15 +5,15 @@ use std::convert::TryFrom; use std::iter::FromIterator; use std::ops::Bound; -use crate::bitcoin::hashes::Hash; -use crate::bitcoin::{Amount, OutPoint, Transaction, Txid}; -use bindex::ScriptHash; +use bitcoin::hashes::Hash; +use bitcoin::{Amount, OutPoint, Transaction, Txid}; use serde::ser::{Serialize, SerializeSeq, Serializer}; use crate::{ daemon::Daemon, metrics::{Gauge, Metrics}, signals::ExitFlag, + types::ScriptHash, }; pub(crate) struct Entry { @@ -80,7 +80,7 @@ impl MempoolSyncUpdate { ); let chunk_entries: Vec = txids_chunk .iter() - .zip(entries.into_iter().zip(txs)) + .zip(entries.into_iter().zip(txs.into_iter())) .filter_map(|(txid, (entry, tx))| { let entry = match entry { Some(entry) => entry, @@ -368,7 +368,7 @@ impl Serialize for FeeHistogram { #[cfg(test)] mod tests { use super::FeeHistogram; - use crate::bitcoin::Amount; + use bitcoin::Amount; use serde_json::json; #[test] diff --git a/src/merkle.rs b/src/merkle.rs index e34e8eb..8294280 100644 --- a/src/merkle.rs +++ b/src/merkle.rs @@ -1,4 +1,4 @@ -use crate::bitcoin::{hash_types::TxMerkleNode, hashes::Hash, Txid}; +use bitcoin::{hash_types::TxMerkleNode, hashes::Hash, Txid}; pub(crate) struct Proof { proof: Vec, @@ -54,7 +54,7 @@ impl Proof { #[cfg(test)] mod tests { - use crate::bitcoin::{consensus::encode::deserialize, Block, Txid}; + use bitcoin::{consensus::encode::deserialize, Block, Txid}; use std::path::Path; use super::Proof; diff --git a/src/p2p.rs b/src/p2p.rs new file mode 100644 index 0000000..0787f36 --- /dev/null +++ b/src/p2p.rs @@ -0,0 +1,406 @@ +use anyhow::{Context, Result}; +use bitcoin::blockdata::block::Header as BlockHeader; +use bitcoin::consensus::Encodable; +use bitcoin::{ + consensus::{ + encode::{self, ReadExt, VarInt}, + Decodable, + }, + hashes::Hash, + io, + p2p::{ + self, address, + message::{self, CommandString, NetworkMessage}, + message_blockdata::{GetHeadersMessage, Inventory}, + message_network, Magic, + }, + secp256k1::{self, rand::Rng}, + Block, BlockHash, +}; +use bitcoin_slices::{bsl, Parse}; +use crossbeam_channel::{bounded, select, Receiver, Sender}; + +use std::io::Write; +use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpStream}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use crate::types::SerBlock; +use crate::{ + chain::{Chain, NewHeader}, + config::ELECTRS_VERSION, + metrics::{default_duration_buckets, default_size_buckets, Histogram, Metrics}, +}; + +enum Request { + GetNewHeaders(GetHeadersMessage), + GetBlocks(Vec), +} + +impl Request { + fn get_new_headers(chain: &Chain) -> Request { + Request::GetNewHeaders(GetHeadersMessage::new( + chain.locator(), + BlockHash::all_zeros(), + )) + } + + fn get_blocks(blockhashes: &[BlockHash]) -> Request { + Request::GetBlocks( + blockhashes + .iter() + .map(|blockhash| Inventory::WitnessBlock(*blockhash)) + .collect(), + ) + } +} + +pub(crate) struct Connection { + req_send: Sender, + blocks_recv: Receiver, + headers_recv: Receiver>, + new_block_recv: Receiver<()>, + + blocks_duration: Histogram, +} + +impl Connection { + /// Get new block headers (supporting reorgs). + /// https://en.bitcoin.it/wiki/Protocol_documentation#getheaders + /// Defined as `&mut self` to prevent concurrent invocations (https://github.com/romanz/electrs/pull/526#issuecomment-934685515). + pub(crate) fn get_new_headers(&mut self, chain: &Chain) -> Result> { + self.req_send.send(Request::get_new_headers(chain))?; + let headers = self + .headers_recv + .recv() + .context("failed to get new headers")?; + + debug!("got {} new headers", headers.len()); + let prev_blockhash = match headers.first() { + None => return Ok(vec![]), + Some(first) => first.prev_blockhash, + }; + let new_heights = match chain.get_block_height(&prev_blockhash) { + Some(last_height) => (last_height + 1).., + None => bail!("missing prev_blockhash: {}", prev_blockhash), + }; + Ok(headers + .into_iter() + .zip(new_heights) + .map(NewHeader::from) + .collect()) + } + + /// Request and process the specified blocks (in the specified order). + /// See https://en.bitcoin.it/wiki/Protocol_documentation#getblocks for details. + /// Defined as `&mut self` to prevent concurrent invocations (https://github.com/romanz/electrs/pull/526#issuecomment-934685515). + pub(crate) fn for_blocks(&mut self, blockhashes: B, mut func: F) -> Result<()> + where + B: IntoIterator, + F: FnMut(BlockHash, SerBlock), + { + self.blocks_duration.observe_duration("total", || { + let blockhashes: Vec = blockhashes.into_iter().collect(); + if blockhashes.is_empty() { + return Ok(()); + } + self.blocks_duration.observe_duration("request", || { + debug!("loading {} blocks", blockhashes.len()); + self.req_send.send(Request::get_blocks(&blockhashes)) + })?; + + for hash in blockhashes { + let block = self.blocks_duration.observe_duration("response", || { + let block = self + .blocks_recv + .recv() + .with_context(|| format!("failed to get block {}", hash))?; + let header = bsl::BlockHeader::parse(&block[..]) + .expect("core returned invalid blockheader") + .parsed_owned(); + ensure!( + &header.block_hash_sha2()[..] == hash.as_byte_array(), + "got unexpected block" + ); + Ok(block) + })?; + self.blocks_duration + .observe_duration("process", || func(hash, block)); + } + Ok(()) + }) + } + + /// Note: only a single receiver will get the notification (https://github.com/romanz/electrs/pull/526#issuecomment-934687415). + pub(crate) fn new_block_notification(&self) -> Receiver<()> { + self.new_block_recv.clone() + } + + pub(crate) fn connect(address: SocketAddr, metrics: &Metrics, magic: Magic) -> Result { + let recv_conn = TcpStream::connect(address) + .with_context(|| format!("p2p failed to connect: {:?}", address))?; + let mut send_conn = recv_conn + .try_clone() + .context("failed to clone connection")?; + + let (tx_send, tx_recv) = bounded::(1); + let (rx_send, rx_recv) = bounded::(1); + + let send_duration = metrics.histogram_vec( + "p2p_send_duration", + "Time spent sending p2p messages (in seconds)", + "step", + default_duration_buckets(), + ); + let recv_duration = metrics.histogram_vec( + "p2p_recv_duration", + "Time spent receiving p2p messages (in seconds)", + "step", + default_duration_buckets(), + ); + let parse_duration = metrics.histogram_vec( + "p2p_parse_duration", + "Time spent parsing p2p messages (in seconds)", + "step", + default_duration_buckets(), + ); + let recv_size = metrics.histogram_vec( + "p2p_recv_size", + "Size of p2p messages read (in bytes)", + "message", + default_size_buckets(), + ); + let blocks_duration = metrics.histogram_vec( + "p2p_blocks_duration", + "Time spent getting blocks via p2p protocol (in seconds)", + "step", + default_duration_buckets(), + ); + + let mut buffer = vec![]; + crate::thread::spawn("p2p_send", move || loop { + use std::net::Shutdown; + let msg = match send_duration.observe_duration("wait", || tx_recv.recv()) { + Ok(msg) => msg, + Err(_) => { + // p2p_loop is closed, so tx_send is disconnected + debug!("closing p2p_send thread: no more messages to send"); + // close the stream reader (p2p_recv thread may block on it) + if let Err(e) = send_conn.shutdown(Shutdown::Read) { + warn!("failed to shutdown p2p connection: {}", e) + } + return Ok(()); + } + }; + send_duration.observe_duration("send", || { + trace!("send: {:?}", msg); + let raw_msg = message::RawNetworkMessage::new(magic, msg); + buffer.clear(); + raw_msg + .consensus_encode(&mut buffer) + .expect("in-memory writers don't error"); + send_conn + .write_all(buffer.as_slice()) + .context("p2p failed to send") + })?; + }); + + let mut stream_reader = std::io::BufReader::new(recv_conn); + crate::thread::spawn("p2p_recv", move || loop { + let start = Instant::now(); + let raw_msg = RawNetworkMessage::consensus_decode(&mut stream_reader); + { + let duration = duration_to_seconds(start.elapsed()); + let label = format!( + "recv_{}", + raw_msg + .as_ref() + .map(|msg| msg.cmd.as_ref()) + .unwrap_or("err") + ); + recv_duration.observe(&label, duration); + } + let raw_msg = match raw_msg { + Ok(raw_msg) => { + recv_size.observe(raw_msg.cmd.as_ref(), raw_msg.raw.len() as f64); + if raw_msg.magic != magic { + bail!("unexpected magic {} (instead of {})", raw_msg.magic, magic) + } + raw_msg + } + Err(encode::Error::Io(e)) if e.kind() == io::ErrorKind::UnexpectedEof => { + debug!("closing p2p_recv thread: connection closed"); + return Ok(()); + } + Err(e) => bail!("failed to recv a message from peer: {}", e), + }; + + recv_duration.observe_duration("wait", || rx_send.send(raw_msg))?; + }); + + let (req_send, req_recv) = bounded::(1); + let (blocks_send, blocks_recv) = bounded::(10); + let (headers_send, headers_recv) = bounded::>(1); + let (new_block_send, new_block_recv) = bounded::<()>(0); + let (init_send, init_recv) = bounded::<()>(0); + + tx_send.send(build_version_message())?; + + crate::thread::spawn("p2p_loop", move || loop { + select! { + recv(rx_recv) -> result => { + let raw_msg = match result { + Ok(raw_msg) => raw_msg, + Err(_) => { // p2p_recv is closed, so rx_send is disconnected + debug!("closing p2p_loop thread: peer has disconnected"); + return Ok(()); // new_block_send is dropped, causing the server to exit + } + }; + + let label = format!("parse_{}", raw_msg.cmd.as_ref()); + let msg = match parse_duration.observe_duration(&label, || raw_msg.parse()) { + Ok(msg) => msg, + Err(err) => bail!("failed to parse {err}"), + }; + trace!("recv: {:?}", msg); + + match msg { + ParsedNetworkMessage::Version(version) => { + debug!("peer version: {:?}", version); + tx_send.send(NetworkMessage::Verack)?; + } + ParsedNetworkMessage::Inv(inventory) => { + debug!("peer inventory: {:?}", inventory); + if inventory.iter().any(|inv| matches!(inv, Inventory::Block(_))) { + let _ = new_block_send.try_send(()); // best-effort notification + } + + }, + ParsedNetworkMessage::Ping(nonce) => { + tx_send.send(NetworkMessage::Pong(nonce))?; // connection keep-alive + } + ParsedNetworkMessage::Verack => { + init_send.send(())?; // peer acknowledged our version + } + ParsedNetworkMessage::Block(block) => blocks_send.send(block)?, + ParsedNetworkMessage::Headers(headers) => headers_send.send(headers)?, + ParsedNetworkMessage::Ignored => (), + } + } + recv(req_recv) -> result => { + let req = match result { + Ok(req) => req, + Err(_) => { // self is dropped, so req_send is disconnected + debug!("closing p2p_loop thread: no more requests to handle"); + return Ok(()); + } + }; + let msg = match req { + Request::GetNewHeaders(msg) => NetworkMessage::GetHeaders(msg), + Request::GetBlocks(inv) => NetworkMessage::GetData(inv), + }; + tx_send.send(msg)?; + } + } + }); + + init_recv.recv()?; // wait until `verack` is received + + Ok(Connection { + req_send, + blocks_recv, + headers_recv, + new_block_recv, + blocks_duration, + }) + } +} + +fn build_version_message() -> NetworkMessage { + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0); + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time error") + .as_secs() as i64; + + let services = p2p::ServiceFlags::NONE; + + NetworkMessage::Version(message_network::VersionMessage { + version: p2p::PROTOCOL_VERSION, + services, + timestamp, + receiver: address::Address::new(&addr, services), + sender: address::Address::new(&addr, services), + nonce: secp256k1::rand::thread_rng().gen(), + user_agent: format!("/electrs:{}/", ELECTRS_VERSION), + start_height: 0, + relay: false, + }) +} + +struct RawNetworkMessage { + magic: Magic, + cmd: CommandString, + raw: Vec, +} + +impl RawNetworkMessage { + fn parse(self) -> Result { + let mut raw: &[u8] = &self.raw; + let payload = match self.cmd.as_ref() { + "version" => ParsedNetworkMessage::Version(Decodable::consensus_decode(&mut raw)?), + "verack" => ParsedNetworkMessage::Verack, + "inv" => ParsedNetworkMessage::Inv(Decodable::consensus_decode(&mut raw)?), + "block" => ParsedNetworkMessage::Block(self.raw), + "headers" => { + let len = VarInt::consensus_decode(&mut raw)?.0; + let mut headers = Vec::with_capacity(len as usize); + for _ in 0..len { + headers.push(Block::consensus_decode(&mut raw)?.header); + } + ParsedNetworkMessage::Headers(headers) + } + "ping" => ParsedNetworkMessage::Ping(Decodable::consensus_decode(&mut raw)?), + "pong" => ParsedNetworkMessage::Ignored, // unused + "addr" => ParsedNetworkMessage::Ignored, // unused + "alert" => ParsedNetworkMessage::Ignored, // https://bitcoin.org/en/alert/2016-11-01-alert-retirement + _ => bail!( + "unsupported message: command={}, payload={:?}", + self.cmd, + self.raw + ), + }; + Ok(payload) + } +} + +#[derive(Debug)] +enum ParsedNetworkMessage { + Version(message_network::VersionMessage), + Verack, + Inv(Vec), + Ping(u64), + Headers(Vec), + Block(SerBlock), + Ignored, +} + +impl Decodable for RawNetworkMessage { + fn consensus_decode(d: &mut D) -> Result { + let magic = Decodable::consensus_decode(d)?; + let cmd = Decodable::consensus_decode(d)?; + + let len = u32::consensus_decode(d)?; + let _checksum = <[u8; 4]>::consensus_decode(d)?; // assume data is correct + let mut raw = vec![0u8; len as usize]; + d.read_slice(&mut raw)?; + + Ok(RawNetworkMessage { magic, cmd, raw }) + } +} + +/// `duration_to_seconds` converts Duration to seconds. +#[inline] +pub fn duration_to_seconds(d: Duration) -> f64 { + let nanos = f64::from(d.subsec_nanos()) / 1e9; + d.as_secs() as f64 + nanos +} diff --git a/src/server.rs b/src/server.rs index 221fb5a..f50662f 100644 --- a/src/server.rs +++ b/src/server.rs @@ -85,6 +85,7 @@ fn serve() -> Result<()> { ); let mut rpc = Rpc::new(&config, metrics)?; + let new_block_rx = rpc.new_block_notification(); let mut peers = HashMap::::new(); loop { // initial sync and compaction may take a few hours @@ -106,6 +107,14 @@ fn serve() -> Result<()> { result.context("signal channel disconnected")?; rpc.signal().exit_flag().poll().context("RPC server interrupted")?; }, + // Handle new blocks' notifications + recv(new_block_rx) -> result => match result { + Ok(_) => (), // sync and update + Err(_) => { + info!("disconnected from bitcoind"); + return Ok(()); + } + }, // Handle Electrum RPC requests recv(server_rx) -> event => { let first = once(event.context("server disconnected")?); diff --git a/src/status.rs b/src/status.rs index fe16acf..29b1651 100644 --- a/src/status.rs +++ b/src/status.rs @@ -1,16 +1,27 @@ -use crate::bitcoin::{ - consensus::Decodable, +use anyhow::Result; +use bitcoin::{ + consensus::serialize, hashes::{sha256, Hash, HashEngine}, Amount, BlockHash, OutPoint, SignedAmount, Transaction, Txid, }; -use anyhow::Result; -use bindex::{IndexedChain, ScriptHash}; +use bitcoin_slices::{bsl, Visit, Visitor}; +use rayon::prelude::*; use serde::ser::{Serialize, Serializer}; -use std::collections::{BTreeMap, HashMap, HashSet}; use std::convert::TryFrom; +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + ops::ControlFlow, +}; -use crate::{mempool::Mempool, types::StatusHash}; +use crate::{ + cache::Cache, + chain::Chain, + daemon::Daemon, + index::Index, + mempool::Mempool, + types::{bsl_txid, ScriptHash, SerBlock, StatusHash}, +}; /// Given a scripthash, store relevant inputs and outputs of a specific transaction struct TxEntry { @@ -87,7 +98,7 @@ pub(crate) struct HistoryEntry { height: Height, #[serde( skip_serializing_if = "Option::is_none", - with = "crate::bitcoin::amount::serde::as_sat::opt" + with = "bitcoin::amount::serde::as_sat::opt" )] fee: Option, } @@ -119,20 +130,11 @@ impl HistoryEntry { } } -/// Make sure blocks are sorted by ascending height. -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] -struct BlockKey(usize, BlockHash); - -impl From> for BlockKey { - fn from(location: bindex::Location<'_>) -> Self { - Self(location.block_height(), location.block_hash()) - } -} - /// ScriptHash subscription status pub struct ScriptHashStatus { scripthash: ScriptHash, // specific scripthash to be queried - confirmed: BTreeMap>, // confirmed entries (chronologically ordered), partitioned per block + tip: BlockHash, // used for skipping confirmed entries' sync + confirmed: HashMap>, // confirmed entries, partitioned per block (may contain stale blocks) mempool: Vec, // unconfirmed entries history: Vec, // computed from confirmed and mempool entries statushash: Option, // computed from history @@ -142,9 +144,9 @@ pub struct ScriptHashStatus { /// https://electrum-protocol.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-balance #[derive(Default, Eq, PartialEq, Serialize)] pub(crate) struct Balance { - #[serde(with = "crate::bitcoin::amount::serde::as_sat", rename = "confirmed")] + #[serde(with = "bitcoin::amount::serde::as_sat", rename = "confirmed")] confirmed_balance: Amount, - #[serde(with = "crate::bitcoin::amount::serde::as_sat", rename = "unconfirmed")] + #[serde(with = "bitcoin::amount::serde::as_sat", rename = "unconfirmed")] mempool_delta: SignedAmount, } @@ -155,7 +157,7 @@ pub(crate) struct UnspentEntry { height: usize, // 0 = mempool entry tx_hash: Txid, tx_pos: u32, - #[serde(with = "crate::bitcoin::amount::serde::as_sat")] + #[serde(with = "bitcoin::amount::serde::as_sat")] value: Amount, } @@ -167,23 +169,25 @@ struct Unspent { } impl Unspent { - fn build(status: &ScriptHashStatus) -> Self { + fn build(status: &ScriptHashStatus, chain: &Chain) -> Self { let mut unspent = Unspent::default(); // First, add all relevant entries' funding outputs to the outpoints' map status - .confirmed_height_entries() + .confirmed_height_entries(chain) .for_each(|(height, entries)| entries.iter().for_each(|e| unspent.insert(e, height))); // Then, remove spent outpoints from the map - status.confirmed_entries().for_each(|e| unspent.remove(e)); + status + .confirmed_entries(chain) + .for_each(|e| unspent.remove(e)); unspent.balance.confirmed_balance = unspent.balance(); // Now, do the same over the mempool (first add funding outputs, and then remove spent ones) status.mempool.iter().for_each(|e| unspent.insert(e, 0)); // mempool height = 0 status.mempool.iter().for_each(|e| unspent.remove(e)); - let total_balance = unspent.balance().to_signed().unwrap(); - let confirmed_balance = unspent.balance.confirmed_balance.to_signed().unwrap(); - unspent.balance.mempool_delta = total_balance - confirmed_balance; + unspent.balance.mempool_delta = unspent.balance().to_signed().unwrap() + - unspent.balance.confirmed_balance.to_signed().unwrap(); + unspent } @@ -228,7 +232,8 @@ impl ScriptHashStatus { pub fn new(scripthash: ScriptHash) -> Self { Self { scripthash, - confirmed: BTreeMap::new(), + tip: BlockHash::all_zeros(), + confirmed: HashMap::new(), mempool: Vec::new(), history: Vec::new(), statushash: None, @@ -237,33 +242,41 @@ impl ScriptHashStatus { /// Iterate through confirmed TxEntries with their corresponding block heights. /// Skip entries from stale blocks. - fn confirmed_height_entries<'a>(&'a self) -> impl Iterator + 'a { + fn confirmed_height_entries<'a>( + &'a self, + chain: &'a Chain, + ) -> impl Iterator + 'a { self.confirmed .iter() - .map(move |(blockid, entries)| (blockid.0, &entries[..])) + .filter_map(move |(blockhash, entries)| { + chain + .get_block_height(blockhash) + .map(|height| (height, &entries[..])) + }) } /// Iterate through confirmed TxEntries. - fn confirmed_entries<'a>(&'a self) -> impl Iterator + 'a { - self.confirmed_height_entries() + /// Skip entries from stale blocks. + fn confirmed_entries<'a>(&'a self, chain: &'a Chain) -> impl Iterator + 'a { + self.confirmed_height_entries(chain) .flat_map(|(_height, entries)| entries) } /// Collect all funded and confirmed outpoints (as a set). - fn confirmed_outpoints(&self) -> HashSet { - self.confirmed_entries() + fn confirmed_outpoints(&self, chain: &Chain) -> HashSet { + self.confirmed_entries(chain) .flat_map(TxEntry::funding_outpoints) .collect() } /// Collect unspent transaction entries - pub(crate) fn get_unspent(&self) -> Vec { - Unspent::build(self).into_entries() + pub(crate) fn get_unspent(&self, chain: &Chain) -> Vec { + Unspent::build(self, chain).into_entries() } /// Collect unspent transaction balance - pub(crate) fn get_balance(&self) -> Balance { - Unspent::build(self).balance + pub(crate) fn get_balance(&self, chain: &Chain) -> Balance { + Unspent::build(self, chain).balance } /// Collect transaction history entries @@ -272,8 +285,8 @@ impl ScriptHashStatus { } /// Collect all confirmed history entries (in block order). - fn get_confirmed_history(&self) -> Vec { - self.confirmed_height_entries() + fn get_confirmed_history(&self, chain: &Chain) -> Vec { + self.confirmed_height_entries(chain) .collect::>() .into_iter() .flat_map(|(height, entries)| { @@ -298,59 +311,88 @@ impl ScriptHashStatus { .collect() } + /// Apply `func` only on the new blocks (to be fetched via p2p interface). + fn for_new_blocks(&self, blockhashes: B, daemon: &Daemon, func: F) -> Result<()> + where + B: IntoIterator, + F: FnMut(BlockHash, SerBlock), + { + daemon.for_blocks( + blockhashes + .into_iter() + .filter(|blockhash| !self.confirmed.contains_key(blockhash)), + func, + ) + } + /// Get funding and spending entries from new blocks. /// Also cache relevant transactions and their merkle proofs. - fn sync_confirmed(&mut self, index: &IndexedChain) -> Result> { - let headers = index.headers(); - let mut latest_header = None; - // Drop entries from stale blocks - while let Some(entry) = self.confirmed.last_entry() { - let BlockKey(height, hash) = entry.key(); - match headers.get_header(*hash, *height) { - Ok(header) => { - latest_header = Some(header); - break; - } - Err(err) => { - warn!("drop reorged block: {}", err); - entry.remove(); - continue; - } + fn sync_confirmed( + &self, + index: &Index, + daemon: &Daemon, + cache: &Cache, + outpoints: &mut HashSet, + ) -> Result>> { + // Will be updated during the following block scans + let mut result = HashMap::>::new(); + + let funding_blockhashes = index.limit_result(index.filter_by_funding(self.scripthash))?; + self.for_new_blocks(funding_blockhashes, daemon, |blockhash, block| { + let block_entries = result.entry(blockhash).or_default(); // the block may already exist + + // extract relevant funding transactions + for filtered_outputs in filter_block_txs_outputs(block, self.scripthash) { + cache.add_tx(filtered_outputs.txid, move || filtered_outputs.tx_bytes); + // store funded outpoints (to check for spending later) + outpoints.extend(make_outpoints( + filtered_outputs.txid, + &filtered_outputs.result, + )); + block_entries + .entry(filtered_outputs.pos) // the transaction may already exist + .or_insert_with(|| TxEntry::new(filtered_outputs.txid)) + .outputs = filtered_outputs.result; } - } - // Recompute all funded outpoints - let mut outpoints = self.confirmed_outpoints(); - // Process transactions in chronological order - for location in index.locations_by_scripthash(&self.scripthash, latest_header)? { - let tx_bytes = index.get_tx_bytes(&location)?; - let tx = Transaction::consensus_decode_from_finite_reader(&mut &tx_bytes[..])?; + })?; + let spending_blockhashes: HashSet = outpoints + .par_iter() // use rayon for concurrent index lookups + .flat_map_iter(|outpoint| index.filter_by_spending(*outpoint)) + .collect(); + self.for_new_blocks(spending_blockhashes, daemon, |blockhash, block| { + let block_entries = result.entry(blockhash).or_default(); // the block may already exist - // Check if this transaction has relevant inputs/outputs: - let spent = filter_inputs(&tx, &outpoints); - let outputs = filter_outputs(&tx, self.scripthash); - if spent.is_empty() && outputs.is_empty() { - continue; + // extract relevant spending transactions + for filtered_inputs in filter_block_txs_inputs(&block, outpoints) { + cache.add_tx(filtered_inputs.txid, move || filtered_inputs.tx_bytes); + block_entries + .entry(filtered_inputs.pos) // the transaction may already exist + .or_insert_with(|| TxEntry::new(filtered_inputs.txid)) + .spent = filtered_inputs.result; } + })?; - // Build new TxEntry and add new outpoints (for next transactions) - let mut tx_entry = TxEntry::new(tx.compute_txid()); - tx_entry.spent = spent; - tx_entry.outputs = outputs; - outpoints.extend(tx_entry.funding_outpoints()); - - // Add new per-block entry (if needed) - self.confirmed - .entry(BlockKey::from(location)) - .or_default() - .push(tx_entry); - } - - Ok(outpoints) + Ok(result + .into_iter() + .map(|(blockhash, entries_map)| { + let sorted_entries: Vec = entries_map + .into_iter() + .collect::>() // sort transactions by their position in a block + .into_values() // drop position within block + .collect(); + (blockhash, sorted_entries) + }) + .collect()) } /// Get funding and spending entries from current mempool. /// Also cache relevant transactions. - fn sync_mempool(&self, mempool: &Mempool, mut outpoints: HashSet) -> Vec { + fn sync_mempool( + &self, + mempool: &Mempool, + cache: &Cache, + outpoints: &mut HashSet, + ) -> Vec { let mut result = HashMap::::new(); // extract relevant funding transactions for entry in mempool.filter_by_funding(&self.scripthash) { @@ -362,25 +404,40 @@ impl ScriptHashStatus { .entry(entry.txid) // the transaction may already exist .or_insert_with(|| TxEntry::new(entry.txid)) .outputs = funding_outputs; + cache.add_tx(entry.txid, || serialize(&entry.tx).into_boxed_slice()); } for entry in outpoints .iter() .flat_map(|outpoint| mempool.filter_by_spending(outpoint)) { - let spent_outpoints = filter_inputs(&entry.tx, &outpoints); + let spent_outpoints = filter_inputs(&entry.tx, outpoints); assert!(!spent_outpoints.is_empty()); result .entry(entry.txid) // the transaction may already exist .or_insert_with(|| TxEntry::new(entry.txid)) .spent = spent_outpoints; + cache.add_tx(entry.txid, || serialize(&entry.tx).into_boxed_slice()); } result.into_values().collect() } - /// Sync with currently confirmed txs and mempool, downloading non-cached transactions via REST API. + /// Sync with currently confirmed txs and mempool, downloading non-cached transactions via p2p protocol. /// After a successful sync, scripthash status is updated. - pub(crate) fn sync(&mut self, index: &IndexedChain, mempool: &Mempool) -> Result<()> { - let outpoints = self.sync_confirmed(index)?; + pub(crate) fn sync( + &mut self, + index: &Index, + mempool: &Mempool, + daemon: &Daemon, + cache: &Cache, + ) -> Result<()> { + let mut outpoints: HashSet = self.confirmed_outpoints(index.chain()); + + let new_tip = index.chain().tip(); + if self.tip != new_tip { + let update = self.sync_confirmed(index, daemon, cache, &mut outpoints)?; + self.confirmed.extend(update); // add new blocks to the map + self.tip = new_tip; + } if !self.confirmed.is_empty() { debug!( "{} transactions from {} blocks", @@ -388,13 +445,14 @@ impl ScriptHashStatus { self.confirmed.len() ); } - self.mempool = self.sync_mempool(mempool, outpoints); + self.mempool = self.sync_mempool(mempool, cache, &mut outpoints); if !self.mempool.is_empty() { debug!("{} mempool transactions", self.mempool.len()); } // update history entries and status hash self.history.clear(); - self.history.extend(self.get_confirmed_history()); + self.history + .extend(self.get_confirmed_history(index.chain())); self.history.extend(self.get_mempool_history(mempool)); self.statushash = compute_status_hash(&self.history); @@ -413,24 +471,32 @@ fn make_outpoints(txid: Txid, outputs: &[TxOutput]) -> impl Iterator Vec { let outputs = tx.output.iter().zip(0u32..); outputs - .filter(|&(txo, _vout)| ScriptHash::new(&txo.script_pubkey) == scripthash) - .map(|(txo, vout)| TxOutput { - index: vout, - value: txo.value, + .filter_map(move |(txo, vout)| { + if ScriptHash::new(&txo.script_pubkey) == scripthash { + Some(TxOutput { + index: vout, + value: txo.value, + }) + } else { + None + } }) .collect() } -/// Collect inputs that spend one of the specified outpoints. fn filter_inputs(tx: &Transaction, outpoints: &HashSet) -> Vec { - let inputs = tx.input.iter(); - inputs - .filter(|&txi| outpoints.contains(&txi.previous_output)) - .map(|txi| txi.previous_output) + tx.input + .iter() + .filter_map(|txi| { + if outpoints.contains(&txi.previous_output) { + Some(txi.previous_output) + } else { + None + } + }) .collect() } @@ -446,10 +512,114 @@ fn compute_status_hash(history: &[HistoryEntry]) -> Option { Some(StatusHash::from_engine(engine)) } +struct FilteredTx { + tx_bytes: Box<[u8]>, + txid: Txid, + pos: usize, + result: Vec, +} + +fn filter_block_txs_outputs(block: SerBlock, scripthash: ScriptHash) -> Vec> { + struct FindOutputs { + scripthash: ScriptHash, + result: Vec>, + buffer: Vec, + pos: usize, + } + impl Visitor for FindOutputs { + // Called after all TxOuts are visited + fn visit_transaction(&mut self, tx: &bsl::Transaction) -> ControlFlow<()> { + if !self.buffer.is_empty() { + self.result.push(FilteredTx:: { + tx_bytes: tx.as_ref().into(), + txid: bsl_txid(tx), + pos: self.pos, + result: std::mem::take(&mut self.buffer), // clear buffer for next tx + }); + } + self.pos += 1; + ControlFlow::Continue(()) + } + // Keep only relevant outputs + fn visit_tx_out(&mut self, vout: usize, tx_out: &bsl::TxOut) -> ControlFlow<()> { + let current = ScriptHash::hash(tx_out.script_pubkey()); + if current == self.scripthash { + self.buffer.push(TxOutput { + index: vout as u32, + value: Amount::from_sat(tx_out.value()), + }) + } + ControlFlow::Continue(()) + } + } + let mut find_outputs = FindOutputs { + scripthash, + result: vec![], + buffer: vec![], + pos: 0, + }; + + bsl::Block::visit(&block, &mut find_outputs).expect("core returned invalid block"); + + find_outputs.result +} + +fn filter_block_txs_inputs( + block: &SerBlock, + outpoints: &HashSet, +) -> Vec> { + struct FindInputs<'a> { + outpoints: &'a HashSet, + result: Vec>, + buffer: Vec, + pos: usize, + } + + impl Visitor for FindInputs<'_> { + // Called after all TxIns are visited + fn visit_transaction(&mut self, tx: &bsl::Transaction) -> ControlFlow<()> { + if !self.buffer.is_empty() { + self.result.push(FilteredTx:: { + tx_bytes: tx.as_ref().into(), + txid: bsl_txid(tx), + pos: self.pos, + result: std::mem::take(&mut self.buffer), // clear buffer for next tx + }); + } + self.pos += 1; + ControlFlow::Continue(()) + } + // Keep only relevant outpoints + fn visit_tx_in(&mut self, _vin: usize, tx_in: &bsl::TxIn) -> ControlFlow<()> { + let current: OutPoint = tx_in.prevout().into(); + if self.outpoints.contains(¤t) { + self.buffer.push(current); + } + ControlFlow::Continue(()) + } + } + + let mut find_inputs = FindInputs { + outpoints, + result: vec![], + buffer: vec![], + pos: 0, + }; + + bsl::Block::visit(block, &mut find_inputs).expect("core returned invalid block"); + + find_inputs.result +} + #[cfg(test)] mod tests { + use std::{collections::HashSet, str::FromStr}; + + use crate::types::ScriptHash; + use super::HistoryEntry; - use crate::bitcoin::Amount; + use bitcoin::{Address, Amount}; + use bitcoin_test_data::blocks::mainnet_702861; use serde_json::json; #[test] @@ -474,4 +644,42 @@ mod tests { json!({"tx_hash": "5b75086dafeede555fc8f9a810d8b10df57c46f9f176ccc3dd8d2fa20edd685b", "height": 0, "fee": 123}) ); } + + #[test] + fn test_find_outputs() { + let block = mainnet_702861().to_vec(); + + let addr = Address::from_str("1A9MXXG26vZVySrNNytQK1N8bX42ZuJ6Ax") + .unwrap() + .assume_checked(); + let scripthash = ScriptHash::new(&addr.script_pubkey()); + + let result = &super::filter_block_txs_outputs(block, scripthash)[0]; + assert_eq!( + result.txid.to_string(), + "7bcdcb44422da5a99daad47d6ba1c3d6f2e48f961a75e42c4fa75029d4b0ef49" + ); + assert_eq!(result.pos, 8); + assert_eq!(result.result[0].index, 0); + assert_eq!(result.result[0].value.to_sat(), 709503); + } + + #[test] + fn test_find_inputs() { + let block = mainnet_702861().to_vec(); + let outpoint = bitcoin::OutPoint::from_str( + "cc135e792b37a9c4ffd784f696b1e38bd1197f8e67ae1f96c9f13e4618b91866:3", + ) + .unwrap(); + let mut outpoints = HashSet::new(); + outpoints.insert(outpoint); + + let result = &super::filter_block_txs_inputs(&block, &outpoints)[0]; + assert_eq!( + result.txid.to_string(), + "7bcdcb44422da5a99daad47d6ba1c3d6f2e48f961a75e42c4fa75029d4b0ef49" + ); + assert_eq!(result.pos, 8); + assert_eq!(result.result[0], outpoint); + } } diff --git a/src/tracker.rs b/src/tracker.rs index 0c7f23b..2abcf9d 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -1,52 +1,78 @@ -use crate::bitcoin::{BlockHash, Txid}; -use crate::bitcoin_slices::{bsl, EmptyVisitor, Visit}; +use std::ops::ControlFlow; + use anyhow::{Context, Result}; -use bindex::IndexedChain; +use bitcoin::{BlockHash, Txid}; +use bitcoin_slices::{bsl, Error::VisitBreak, Visit, Visitor}; use crate::{ + cache::Cache, + chain::Chain, config::Config, daemon::Daemon, + db::DBStore, + index::Index, mempool::{FeeHistogram, Mempool}, metrics::Metrics, signals::ExitFlag, status::{Balance, ScriptHashStatus, UnspentEntry}, + types::bsl_txid, }; /// Electrum protocol subscriptions' tracker pub struct Tracker { - index: IndexedChain, + index: Index, mempool: Mempool, + metrics: Metrics, ignore_mempool: bool, } +pub(crate) enum Error { + NotReady, +} + impl Tracker { pub fn new(config: &Config, metrics: Metrics) -> Result { - let url = format!("http://{}", config.daemon_rpc_addr); - let index = IndexedChain::open(&config.db_dir, config.network, Some(url)) - .context("failed to open index")?; + let store = DBStore::open( + &config.db_path, + config.db_log_dir.as_deref(), + config.auto_reindex, + config.db_parallelism, + )?; + let chain = Chain::new(config.network); Ok(Self { - index, + index: Index::load( + store, + chain, + &metrics, + config.index_batch_size, + config.index_lookup_limit, + config.reindex_last_blocks, + ) + .context("failed to open index")?, mempool: Mempool::new(&metrics), + metrics, ignore_mempool: config.ignore_mempool, }) } - pub(crate) fn headers(&self) -> &bindex::Headers { - self.index.headers() + pub(crate) fn chain(&self) -> &Chain { + self.index.chain() } pub(crate) fn fees_histogram(&self) -> &FeeHistogram { self.mempool.fees_histogram() } + pub(crate) fn metrics(&self) -> &Metrics { + &self.metrics + } + pub(crate) fn get_unspent(&self, status: &ScriptHashStatus) -> Vec { - status.get_unspent() + status.get_unspent(self.index.chain()) } pub(crate) fn sync(&mut self, daemon: &Daemon, exit_flag: &ExitFlag) -> Result { - exit_flag.poll()?; - let stats = self.index.sync(1000)?; - let done = stats.indexed_blocks == 0; + let done = self.index.sync(daemon, exit_flag)?; if done && !self.ignore_mempool { self.mempool.sync(daemon, exit_flag); // TODO: double check tip - and retry on diff @@ -54,36 +80,67 @@ impl Tracker { Ok(done) } - pub(crate) fn status(&self) -> Result<()> { - Ok(()) + pub(crate) fn status(&self) -> Result<(), Error> { + if self.index.is_ready() { + return Ok(()); + } + Err(Error::NotReady) } - pub(crate) fn update_scripthash_status(&self, status: &mut ScriptHashStatus) -> Result { + pub(crate) fn update_scripthash_status( + &self, + status: &mut ScriptHashStatus, + daemon: &Daemon, + cache: &Cache, + ) -> Result { let prev_statushash = status.statushash(); - status.sync(&self.index, &self.mempool)?; + status.sync(&self.index, &self.mempool, daemon, cache)?; Ok(prev_statushash != status.statushash()) } pub(crate) fn get_balance(&self, status: &ScriptHashStatus) -> Balance { - status.get_balance() + status.get_balance(self.chain()) } - pub(crate) fn lookup_transaction(&self, txid: Txid) -> Result)>> { + pub(crate) fn lookup_transaction( + &self, + daemon: &Daemon, + txid: Txid, + ) -> Result)>> { // Note: there are two blocks with coinbase transactions having same txid (see BIP-30) - for loc in self.index.locations_by_txid(&txid)? { - let tx_bytes = self.index.get_tx_bytes(&loc)?; - if txid == compute_txid(&tx_bytes)? { - return Ok(Some((loc.block_hash(), tx_bytes.into_boxed_slice()))); + let blockhashes = self.index.filter_by_txid(txid); + let mut result = None; + daemon.for_blocks(blockhashes, |blockhash, block| { + if result.is_some() { + return; // keep first matching transaction } - } - Ok(None) + let mut visitor = FindTransaction::new(txid); + result = match bsl::Block::visit(&block, &mut visitor) { + Ok(_) | Err(VisitBreak) => visitor.found.map(|tx| (blockhash, tx)), + Err(e) => panic!("core returned invalid block: {:?}", e), + }; + })?; + Ok(result) } } -fn compute_txid(tx_bytes: &[u8]) -> Result { - let mut visit = EmptyVisitor {}; - let res = bsl::Transaction::visit(tx_bytes, &mut visit) - .map_err(|err| anyhow!("invalid transaction: {:?}", err))?; - ensure!(res.remaining().is_empty(), "non-empty remaining bytes"); - Ok(Txid::from_raw_hash(res.parsed().txid())) +pub struct FindTransaction { + txid: bitcoin::Txid, + found: Option>, // no need to deserialize +} + +impl FindTransaction { + pub fn new(txid: bitcoin::Txid) -> Self { + Self { txid, found: None } + } +} +impl Visitor for FindTransaction { + fn visit_transaction(&mut self, tx: &bsl::Transaction) -> ControlFlow<()> { + if self.txid == bsl_txid(tx) { + self.found = Some(tx.as_ref().into()); + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + } } diff --git a/src/types.rs b/src/types.rs index 54a3ae7..9bd4be6 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,13 +1,292 @@ -use crate::bitcoin::hashes::{hash_newtype, sha256}; +use anyhow::Result; + +use std::convert::TryFrom; + +use bitcoin::blockdata::block::Header as BlockHeader; +use bitcoin::{ + consensus::encode::{deserialize, Decodable, Encodable}, + hashes::{hash_newtype, sha256, Hash}, + io, OutPoint, Script, Txid, +}; +use bitcoin_slices::bsl; + +macro_rules! impl_consensus_encoding { + ($thing:ident, $($field:ident),+) => ( + impl Encodable for $thing { + #[inline] + fn consensus_encode( + &self, + s: &mut S, + ) -> Result { + let mut len = 0; + $(len += self.$field.consensus_encode(s)?;)+ + Ok(len) + } + } + + impl Decodable for $thing { + #[inline] + fn consensus_decode( + d: &mut D, + ) -> Result<$thing, bitcoin::consensus::encode::Error> { + Ok($thing { + $($field: Decodable::consensus_decode(d)?),+ + }) + } + } + ); +} + +pub const HASH_PREFIX_LEN: usize = 8; +const HEIGHT_SIZE: usize = 4; + +pub(crate) type HashPrefix = [u8; HASH_PREFIX_LEN]; +pub(crate) type SerializedHashPrefixRow = [u8; HASH_PREFIX_ROW_SIZE]; +type Height = u32; +pub(crate) type SerBlock = Vec; + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +pub(crate) struct HashPrefixRow { + prefix: HashPrefix, + height: Height, // transaction confirmed height +} + +pub const HASH_PREFIX_ROW_SIZE: usize = HASH_PREFIX_LEN + HEIGHT_SIZE; + +impl HashPrefixRow { + pub(crate) fn to_db_row(&self) -> SerializedHashPrefixRow { + let mut row = [0; HASH_PREFIX_ROW_SIZE]; + let len = self + .consensus_encode(&mut (&mut row as &mut [u8])) + .expect("in-memory writers don't error"); + debug_assert_eq!(len, HASH_PREFIX_ROW_SIZE); + row + } + + pub(crate) fn from_db_row(row: SerializedHashPrefixRow) -> Self { + deserialize(&row).expect("bad HashPrefixRow") + } + + pub fn height(&self) -> usize { + usize::try_from(self.height).expect("invalid height") + } +} + +impl_consensus_encoding!(HashPrefixRow, prefix, height); + +hash_newtype! { + /// https://electrum-protocol.readthedocs.io/en/latest/protocol-basics.html#script-hashes + #[hash_newtype(backward)] + pub struct ScriptHash(sha256::Hash); +} + +impl ScriptHash { + pub fn new(script: &Script) -> Self { + ScriptHash::hash(script.as_bytes()) + } + + fn prefix(&self) -> HashPrefix { + let mut prefix = HashPrefix::default(); + prefix.copy_from_slice(&self.0[..HASH_PREFIX_LEN]); + prefix + } +} + +pub(crate) struct ScriptHashRow; + +impl ScriptHashRow { + pub(crate) fn scan_prefix(scripthash: ScriptHash) -> HashPrefix { + scripthash.0[..HASH_PREFIX_LEN].try_into().unwrap() + } + + pub(crate) fn row(scripthash: ScriptHash, height: usize) -> HashPrefixRow { + HashPrefixRow { + prefix: scripthash.prefix(), + height: Height::try_from(height).expect("invalid height"), + } + } +} + +// *************************************************************************** hash_newtype! { /// https://electrum-protocol.readthedocs.io/en/latest/protocol-basics.html#status pub struct StatusHash(sha256::Hash); } -/// The different authentication methods for the client. -#[derive(Clone, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] -pub enum Auth { - UserPass(String, String), - CookieFile(std::path::PathBuf), +// *************************************************************************** + +fn spending_prefix(prev: OutPoint) -> HashPrefix { + let txid_prefix = HashPrefix::try_from(&prev.txid[..HASH_PREFIX_LEN]).unwrap(); + let value = u64::from_be_bytes(txid_prefix); + let value = value.wrapping_add(prev.vout.into()); + value.to_be_bytes() +} + +pub(crate) struct SpendingPrefixRow; + +impl SpendingPrefixRow { + pub(crate) fn scan_prefix(outpoint: OutPoint) -> HashPrefix { + spending_prefix(outpoint) + } + + pub(crate) fn row(outpoint: OutPoint, height: usize) -> HashPrefixRow { + HashPrefixRow { + prefix: spending_prefix(outpoint), + height: Height::try_from(height).expect("invalid height"), + } + } +} + +// *************************************************************************** + +fn txid_prefix(txid: &Txid) -> HashPrefix { + let mut prefix = [0u8; HASH_PREFIX_LEN]; + prefix.copy_from_slice(&txid[..HASH_PREFIX_LEN]); + prefix +} + +pub(crate) struct TxidRow; + +impl TxidRow { + pub(crate) fn scan_prefix(txid: Txid) -> HashPrefix { + txid_prefix(&txid) + } + + pub(crate) fn row(txid: Txid, height: usize) -> HashPrefixRow { + HashPrefixRow { + prefix: txid_prefix(&txid), + height: Height::try_from(height).expect("invalid height"), + } + } +} + +// *************************************************************************** + +pub(crate) type SerializedHeaderRow = [u8; HEADER_ROW_SIZE]; + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct HeaderRow { + pub(crate) header: BlockHeader, +} + +pub const HEADER_ROW_SIZE: usize = 80; + +impl_consensus_encoding!(HeaderRow, header); + +impl HeaderRow { + pub(crate) fn new(header: BlockHeader) -> Self { + Self { header } + } + + pub(crate) fn to_db_row(&self) -> SerializedHeaderRow { + let mut row = [0; HEADER_ROW_SIZE]; + let len = self + .consensus_encode(&mut (&mut row as &mut [u8])) + .expect("in-memory writers don't error"); + debug_assert_eq!(len, HEADER_ROW_SIZE); + row + } + + pub(crate) fn from_db_row(row: SerializedHeaderRow) -> Self { + deserialize(&row).expect("bad HeaderRow") + } +} + +pub(crate) fn bsl_txid(tx: &bsl::Transaction) -> Txid { + bitcoin::Txid::from_slice(tx.txid_sha2().as_slice()).expect("invalid txid") +} + +#[cfg(test)] +mod tests { + use crate::types::{spending_prefix, HashPrefixRow, ScriptHash, ScriptHashRow, TxidRow}; + use bitcoin::{Address, OutPoint, Txid}; + use hex_lit::hex; + use serde_json::{from_str, json}; + + use std::str::FromStr; + + #[test] + fn test_scripthash_serde() { + let hex = "\"4b3d912c1523ece4615e91bf0d27381ca72169dbf6b1c2ffcc9f92381d4984a3\""; + let scripthash: ScriptHash = from_str(hex).unwrap(); + assert_eq!(format!("\"{}\"", scripthash), hex); + assert_eq!(json!(scripthash).to_string(), hex); + } + + #[test] + fn test_scripthash_row() { + let hex = "\"4b3d912c1523ece4615e91bf0d27381ca72169dbf6b1c2ffcc9f92381d4984a3\""; + let scripthash: ScriptHash = from_str(hex).unwrap(); + let row1 = ScriptHashRow::row(scripthash, 123456); + let db_row = row1.to_db_row(); + assert_eq!(db_row, hex!("a384491d38929fcc40e20100")); + let row2 = HashPrefixRow::from_db_row(db_row); + assert_eq!(row1, row2); + } + + #[test] + fn test_scripthash() { + let addr = Address::from_str("1KVNjD3AAnQ3gTMqoTKcWFeqSFujq9gTBT") + .unwrap() + .assume_checked(); + let scripthash = ScriptHash::new(&addr.script_pubkey()); + assert_eq!( + scripthash, + "00dfb264221d07712a144bda338e89237d1abd2db4086057573895ea2659766a" + .parse() + .unwrap() + ); + } + + #[test] + fn test_txid1_prefix() { + // duplicate txids from BIP-30 + let hex = "d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599"; + let txid = Txid::from_str(hex).unwrap(); + + let row1 = TxidRow::row(txid, 91812); + let row2 = TxidRow::row(txid, 91842); + + assert_eq!(row1.to_db_row(), hex!("9985d82954e10f22a4660100")); + assert_eq!(row2.to_db_row(), hex!("9985d82954e10f22c2660100")); + } + + #[test] + fn test_txid2_prefix() { + // duplicate txids from BIP-30 + let hex = "e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468"; + let txid = Txid::from_str(hex).unwrap(); + + let row1 = TxidRow::row(txid, 91722); + let row2 = TxidRow::row(txid, 91880); + + // low-endian encoding => rows should be sorted according to block height + assert_eq!(row1.to_db_row(), hex!("68b45f58b674e94e4a660100")); + assert_eq!(row2.to_db_row(), hex!("68b45f58b674e94ee8660100")); + } + + #[test] + fn test_spending_prefix() { + let txid = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + .parse() + .unwrap(); + + assert_eq!( + spending_prefix(OutPoint { txid, vout: 0 }), + [31, 30, 29, 28, 27, 26, 25, 24] + ); + assert_eq!( + spending_prefix(OutPoint { txid, vout: 10 }), + [31, 30, 29, 28, 27, 26, 25, 34] + ); + assert_eq!( + spending_prefix(OutPoint { txid, vout: 255 }), + [31, 30, 29, 28, 27, 26, 26, 23] + ); + assert_eq!( + spending_prefix(OutPoint { txid, vout: 256 }), + [31, 30, 29, 28, 27, 26, 26, 24] + ); + } } diff --git a/tests/run.sh b/tests/run.sh index 3a75f89..1e2c03c 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -34,7 +34,7 @@ tail_log() { } echo "Starting $(bitcoind -version | head -n1)..." -bitcoind -rest -regtest -datadir=data/bitcoin -printtoconsole=0 & +bitcoind -regtest -datadir=data/bitcoin -printtoconsole=0 & BITCOIND_PID=$! $BTC -rpcwait getblockcount > /dev/null