diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 80d87b0..db3a668 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -2,6 +2,8 @@ name: electrs on: push: + branches: + - master pull_request: schedule: - cron: "0 0 * * *" # once a day @@ -10,19 +12,22 @@ jobs: build: name: Build - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: + os: [ubuntu-latest] build-args: [ --locked --no-default-features, - --locked, - --locked --features metrics_process, + --locked ] + include: + - os: ubuntu-latest + build-args: --locked --features metrics_process steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@stable with: components: rustfmt, clippy @@ -30,6 +35,9 @@ jobs: - name: Install Rust run: rustup component add rustfmt clippy + - name: Install other dependencies + run: sudo apt install build-essential libclang-dev + - name: Format run: cargo fmt --all -- --check @@ -47,7 +55,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v6 - name: Build run: docker build -f Dockerfile.ci . --rm -t electrs:tests - name: Test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6d9d1b4..7092386 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,13 +27,13 @@ The codebase is maintained using the "contributor workflow" where everyone without exception contributes patch proposals using "pull requests". This facilitates social contribution, easy testing and peer review. -To contribute a patch, the workflow is a as follows: +To contribute a patch, the workflow is as follows: 1. Fork repository 2. Create topic branch 3. Commit patches -Please keep commits should atomic and diffs easy to read. For this reason +Please keep commits atomic and diffs easy to read. For this reason do not mix any formatting fixes or code moves with actual code changes. Further, each commit, individually, should compile and pass tests, in order to ensure git bisect and other automated tools function properly. @@ -84,8 +84,8 @@ grammar fixes. ### Formatting -The repository currently does use `rustfmt` for all the formatting needs. Running the automated +The repository currently uses `rustfmt` for all the formatting needs. Running the automated script mentioned above would format all your code for you :) ## Ending Notes -Get cracking, have fun, and do ask for help when in doubt! \ No newline at end of file +Get cracking, have fun, and do ask for help when in doubt! diff --git a/Cargo.lock b/Cargo.lock index ffcd0c0..70abbb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,27 +1,27 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] [[package]] name = "anyhow" -version = "1.0.80" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "ascii" @@ -31,9 +31,9 @@ checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" [[package]] name = "autocfg" -version = "1.1.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "base58ck" @@ -47,9 +47,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.13.1" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bech32" @@ -58,30 +58,47 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" [[package]] -name = "bindgen" -version = "0.63.0" +name = "bindex" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36d860121800b2a9a94f9b5604b332d5cffb234ce17609ea479d723dbc9d3885" +checksum = "becb30e2688944707fe12c33fc1d6920fc4c83f67224cd81bf2767062b852f9e" dependencies = [ - "bitflags 1.3.2", + "bitcoin", + "bitcoin_slices", + "log", + "rayon", + "rust-rocksdb", + "serde", + "serde_json", + "thiserror 2.0.16", + "ureq", +] + +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags", "cexpr", "clang-sys", + "itertools", "lazy_static", "lazycell", - "peeking_take_while", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", - "syn 1.0.109", + "syn", ] [[package]] name = "bitcoin" -version = "0.32.2" +version = "0.32.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea507acc1cd80fc084ace38544bbcf7ced7c2aa65b653b102de0ce718df668f6" +checksum = "9cf93e61f2dbc3e3c41234ca26a65e2c0b0975c52e0f069ab9893ebbede584d3" dependencies = [ "base58ck", "bech32", @@ -106,9 +123,9 @@ dependencies = [ [[package]] name = "bitcoin-io" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "340e09e8399c7bd8912f495af6aa58bea0c9214773417ffaa8f6460f93aaee56" +checksum = "0b47c4ab7a93edb0c7198c5535ed9b52b63095f4e9b45279c6736cec4b856baf" [[package]] name = "bitcoin-test-data" @@ -118,9 +135,9 @@ checksum = "0c188654f9dce3bc6ce1bfa9c49777ad514bcad37e421b5f53e9d0ee10603f34" [[package]] name = "bitcoin-units" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb54da0b28892f3c52203a7191534033e051b6f4b52bc15480681b57b7e036f5" +checksum = "5285c8bcaa25876d07f37e3d30c303f2609179716e11d688f51e8f1fe70063e2" dependencies = [ "bitcoin-internals", "serde", @@ -139,95 +156,64 @@ dependencies = [ [[package]] name = "bitcoin_slices" -version = "0.8.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8041a1be831c809ada090db2e3bd1469c65b72321bb2f31d7f56261eefc8321" +checksum = "82b80fcc031ed36f91c31639a4d97acb1487985fdee34565651b2f52f4346859" dependencies = [ "bitcoin", "bitcoin_hashes", - "sha2", -] - -[[package]] -name = "bitcoincore-rpc" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedd23ae0fd321affb4bbbc36126c6f49a32818dc6b979395d24da8c9d4e80ee" -dependencies = [ - "bitcoincore-rpc-json", - "jsonrpc", - "log", - "serde", - "serde_json", -] - -[[package]] -name = "bitcoincore-rpc-json" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8909583c5fab98508e80ef73e5592a651c954993dc6b7739963257d19f0e71a" -dependencies = [ - "bitcoin", - "serde", - "serde_json", ] [[package]] name = "bitflags" -version = "1.3.2" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" [[package]] -name = "bitflags" -version = "2.4.1" +name = "block2" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" dependencies = [ - "generic-array", + "objc2", ] [[package]] -name = "byteorder" -version = "1.5.0" +name = "bytes" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "bzip2-sys" -version = "0.1.11+1.0.8" +version = "0.1.13+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" dependencies = [ "cc", - "libc", "pkg-config", ] [[package]] name = "cargo_toml" -version = "0.15.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1521c5948ab432e084eabee0c9e4e965483f73156eaa0b04fc192e3f61205438" +checksum = "2bfbc36312494041e2cdd5f06697b7e89d4b76f42773a0b5556ac290ff22acc2" dependencies = [ "serde", - "toml 0.7.1", + "toml", ] [[package]] name = "cc" -version = "1.0.83" +version = "1.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "5f4ac86a9e5bc1e2b3449ab9d7d3a6a405e3d1bb28d7b9be8614f55846ae3766" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -245,6 +231,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chunked_transfer" version = "1.5.0" @@ -253,13 +245,12 @@ checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" [[package]] name = "clang-sys" -version = "1.6.1" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", - "libloading", ] [[package]] @@ -268,10 +259,10 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d03c1fbdead926855bdafee8ddf16cd42efb3c75d8cde8c87f8937b99510b39d" dependencies = [ - "parse_arg", + "parse_arg 0.1.6", "serde", "serde_derive", - "toml 0.5.11", + "toml", ] [[package]] @@ -284,79 +275,54 @@ dependencies = [ "fmt2io", "serde", "serde_derive", - "toml 0.5.11", + "toml", "unicode-segmentation", "void", ] -[[package]] -name = "cpufeatures" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" -dependencies = [ - "libc", -] - [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.16" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset", ] [[package]] name = "crossbeam-utils" -version = "0.8.18" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" -dependencies = [ - "cfg-if", -] +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] -name = "crypto-common" -version = "0.1.6" +name = "ctrlc" +version = "3.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", + "dispatch2", + "nix", + "windows-sys 0.61.0", ] [[package]] @@ -381,29 +347,39 @@ dependencies = [ ] [[package]] -name = "either" -version = "1.9.0" +name = "dispatch2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "89a09f22a6c6069a18470eb92d2298acf25463f14256d24778e1230d789a2aec" +dependencies = [ + "bitflags", + "block2", + "libc", + "objc2", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "electrs" -version = "0.10.6" +version = "0.11.1" dependencies = [ "anyhow", - "bitcoin", + "bindex", "bitcoin-test-data", - "bitcoin_slices", - "bitcoincore-rpc", "configure_me", "configure_me_codegen", "crossbeam-channel", + "ctrlc", "dirs-next", - "electrs-rocksdb", "env_logger", "hex_lit", + "jsonrpc", "log", - "parking_lot", "prometheus", "rayon", "serde", @@ -414,31 +390,6 @@ dependencies = [ "tiny_http", ] -[[package]] -name = "electrs-librocksdb-sys" -version = "0.9.0-e3+7.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b34c24ce2a4d41fe320cea3a9b703b50efea6c2dd4a6d5b0cdbb6ee81636932f" -dependencies = [ - "bindgen", - "bzip2-sys", - "cc", - "glob", - "libc", - "libz-sys", - "zstd-sys", -] - -[[package]] -name = "electrs-rocksdb" -version = "0.19.0-e3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f33e74754aaacc3ff9ace8d3278a7f1be1173104d1960ea18f36386f0feec1e" -dependencies = [ - "electrs-librocksdb-sys", - "libc", -] - [[package]] name = "env_logger" version = "0.10.2" @@ -454,19 +405,19 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.8" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "fastrand" -version = "2.0.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fmt2io" @@ -481,43 +432,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] -name = "generic-array" -version = "0.14.7" +name = "getrandom" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ - "typenum", - "version_check", + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] name = "getrandom" -version = "0.2.11" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", "libc", - "wasi", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", ] [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" [[package]] name = "hex" @@ -527,9 +474,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hex-conservative" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1aa273bf451e37ed35ced41c71a5e2a4e29064afb104158f2514bcd71c2c986" +checksum = "fda06d18ac606267c40c04e41b9947729bf8b9efe74bd4e82b61a5f26a510b9f" dependencies = [ "arrayvec", ] @@ -540,6 +487,22 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "httpdate" version = "1.0.3" @@ -548,74 +511,62 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.48.0", -] +checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ "hermit-abi", - "rustix 0.38.31", - "windows-sys 0.48.0", + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", ] [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ + "getrandom 0.3.3", "libc", ] [[package]] name = "jsonrpc" -version = "0.18.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf" +checksum = "f106cca655869522988b976127096f0aa36e0f1a8ff1f1f425a5c85b61e59391" dependencies = [ "base64", - "minreq", "serde", "serde_json", ] [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lazycell" @@ -625,36 +576,25 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.153" +version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" - -[[package]] -name = "libloading" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" -dependencies = [ - "cfg-if", - "winapi", -] +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" [[package]] name = "libredox" -version = "0.0.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.4.1", + "bitflags", "libc", - "redox_syscall", ] [[package]] name = "libz-sys" -version = "1.1.12" +version = "1.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" dependencies = [ "cc", "pkg-config", @@ -663,21 +603,21 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.1.4" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -685,24 +625,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" - -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "minimal-lexical" @@ -711,14 +642,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] -name = "minreq" -version = "2.11.2" +name = "nix" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fdef521c74c2884a4f3570bcdb6d2a77b3c533feb6b27ac2ae72673cc221c64" +checksum = "225e7cfe711e0ba79a68baeddb2982723e4235247aefce1482f2f16c27865b66" dependencies = [ - "log", - "serde", - "serde_json", + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", ] [[package]] @@ -732,25 +664,31 @@ dependencies = [ ] [[package]] -name = "nom8" -version = "0.2.0" +name = "objc2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" +checksum = "b7c2599ce0ec54857b29ce62166b0ed9b4f6f1a70ccc9a71165b6154caca8c05" dependencies = [ - "memchr", + "objc2-encode", ] [[package]] -name = "once_cell" -version = "1.19.0" +name = "objc2-encode" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -758,68 +696,80 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.48.5", + "windows-targets", ] [[package]] name = "parse_arg" -version = "0.1.4" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14248cc8eced350e20122a291613de29e4fa129ba2731818c4cdbb44fccd3e55" +checksum = "32f05bccc8b6036fec4e0c511954e3997987a82acb6a0b50642ecf7c744fe225" +dependencies = [ + "parse_arg 1.0.1", +] [[package]] -name = "peeking_take_while" -version = "0.1.2" +name = "parse_arg" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +checksum = "5bddc33f680b79eaf1e2e56da792c3c2236f86985bbc3a886e8ddee17ae4d3a4" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pkg-config" -version = "0.3.28" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" - -[[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "proc-macro2" -version = "1.0.71" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] [[package]] name = "procfs" -version = "0.14.2" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1de8dacb0873f77e6aefc6d71e044761fcc68060290f5b1089fcdf84626bb69" +checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 1.3.2", - "byteorder", + "bitflags", + "hex", + "procfs-core", + "rustix 0.38.44", +] + +[[package]] +name = "procfs-core" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" +dependencies = [ + "bitflags", "hex", - "lazy_static", - "rustix 0.36.17", ] [[package]] name = "prometheus" -version = "0.13.3" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a" dependencies = [ "cfg-if", "fnv", @@ -829,59 +779,49 @@ dependencies = [ "parking_lot", "procfs", "protobuf", - "thiserror", + "thiserror 2.0.16", ] [[package]] name = "protobuf" -version = "2.28.0" +version = "3.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4" +dependencies = [ + "once_cell", + "protobuf-support", + "thiserror 1.0.69", +] + +[[package]] +name = "protobuf-support" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6" +dependencies = [ + "thiserror 1.0.69", +] [[package]] name = "quote" -version = "1.0.33" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] [[package]] -name = "rand" -version = "0.8.5" +name = "r-efi" +version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" [[package]] name = "rayon" -version = "1.10.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" dependencies = [ "either", "rayon-core", @@ -889,9 +829,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -899,29 +839,29 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" dependencies = [ - "bitflags 1.3.2", + "bitflags", ] [[package]] name = "redox_users" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom", + "getrandom 0.2.16", "libredox", - "thiserror", + "thiserror 1.0.69", ] [[package]] name = "regex" -version = "1.9.6" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -931,9 +871,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.9" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -942,9 +882,34 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rust-librocksdb-sys" +version = "0.32.0+9.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50146b7fadd68926e9dcb902bf0515783aaaf5f73af26949f188aead5ede8cd0" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "glob", + "libc", + "libz-sys", + "zstd-sys", +] + +[[package]] +name = "rust-rocksdb" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf088a714aa3fad699f7dbe06047ca732c09629a2f9b28aa16ca6d3897a4c2f" +dependencies = [ + "libc", + "rust-librocksdb-sys", +] [[package]] name = "rustc-hash" @@ -954,37 +919,30 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.36.17" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "305efbd14fde4139eb501df5f136994bb520b033fa9fbdce287507dc23b8c7ed" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 1.3.2", + "bitflags", "errno", - "io-lifetimes", "libc", - "linux-raw-sys 0.1.4", - "windows-sys 0.45.0", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", ] [[package]] name = "rustix" -version = "0.38.31" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" dependencies = [ - "bitflags 2.4.1", + "bitflags", "errno", "libc", - "linux-raw-sys 0.4.12", - "windows-sys 0.52.0", + "linux-raw-sys 0.11.0", + "windows-sys 0.61.0", ] -[[package]] -name = "ryu" -version = "1.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" - [[package]] name = "scopeguard" version = "1.2.0" @@ -993,74 +951,65 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "secp256k1" -version = "0.29.0" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0cc0f1cf93f4969faf3ea1c7d8a9faed25918d96affa959720823dfe86d4f3" +checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113" dependencies = [ "bitcoin_hashes", - "rand", "secp256k1-sys", "serde", ] [[package]] name = "secp256k1-sys" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1433bd67156263443f14d603720b082dd3121779323fce20cba2aa07b874bc1b" +checksum = "d4387882333d3aa8cb20530a17c69a3752e97837832f34f6dccc760e715001d9" dependencies = [ "cc", ] [[package]] name = "serde" -version = "1.0.171" +version = "1.0.223" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +checksum = "a505d71960adde88e293da5cb5eda57093379f64e61cf77bf0e6a63af07a7bac" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.223" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f57cbd357666aa7b3ac84a90b4ea328f1d4ddb6772b430caa5d9e1309bb9e9" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.171" +version = "1.0.223" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +checksum = "3d428d07faf17e306e699ec1e91996e5a165ba5d6bce5b5155173e91a8a01a56" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn", ] [[package]] name = "serde_json" -version = "1.0.109" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0652c533506ad7a2e353cce269330d6afd8bdfb6d75e0ace5b35aacbd7b9e9" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", - "ryu", + "memchr", "serde", -] - -[[package]] -name = "serde_spanned" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" -dependencies = [ - "serde", -] - -[[package]] -name = "sha2" -version = "0.10.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", + "serde_core", + "zmij", ] [[package]] @@ -1071,9 +1020,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook" -version = "0.3.17" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +checksum = "b2a0c28ca5908dbdbcd52e6fdaa00358ab88637f8ab33e1f188dd510eb44b53d" dependencies = [ "libc", "signal-hook-registry", @@ -1081,35 +1030,24 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" dependencies = [ "libc", ] [[package]] name = "smallvec" -version = "1.13.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" [[package]] name = "syn" -version = "1.0.109" +version = "2.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b7d0a2c048d661a1a59fcd7355baa232f7ed34e0ee4df2eef3c1c1c0d3852d8" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" dependencies = [ "proc-macro2", "quote", @@ -1118,15 +1056,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.12.0" +version = "3.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1" dependencies = [ - "cfg-if", "fastrand", + "getrandom 0.3.3", "once_cell", - "rustix 0.38.31", - "windows-sys 0.59.0", + "rustix 1.1.3", + "windows-sys 0.61.0", ] [[package]] @@ -1140,22 +1078,42 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.55" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e3de26b0965292219b4287ff031fcba86837900fe9cd2b34ea8ad893c0953d2" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" +dependencies = [ + "thiserror-impl 2.0.16", ] [[package]] name = "thiserror-impl" -version = "1.0.55" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "268026685b2be38d7103e9e507c938a1fcb3d7e6eb15e87870b617bf37b6d581" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1179,57 +1137,48 @@ dependencies = [ "serde", ] -[[package]] -name = "toml" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772c1426ab886e7362aedf4abc9c0d1348a979517efedfc25862944d10137af0" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90a238ee2e6ede22fb95350acc78e21dc40da00bb66c0334bde83de4ed89424e" -dependencies = [ - "indexmap", - "nom8", - "serde", - "serde_spanned", - "toml_datetime", -] - -[[package]] -name = "typenum" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" - [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "unicode-segmentation" -version = "1.10.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "ureq" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d39cb1dbab692d82a977c0392ffac19e188bd9186a9f32806f0aaa859d75585a" +dependencies = [ + "base64", + "log", + "percent-encoding", + "ureq-proto", + "utf-8", +] + +[[package]] +name = "ureq-proto" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d81f9efa9df032be5934a46a068815a10a042b494b6a58cb0a1a97bb5467ed6f" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] name = "vcpkg" @@ -1237,12 +1186,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - [[package]] name = "void" version = "1.0.2" @@ -1255,6 +1198,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1273,11 +1225,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "winapi", + "windows-sys 0.59.0", ] [[package]] @@ -1287,31 +1239,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows-sys" -version = "0.45.0" +name = "windows-link" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.6", -] +checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" [[package]] name = "windows-sys" @@ -1319,37 +1250,16 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] -name = "windows-targets" -version = "0.42.2" +name = "windows-sys" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", + "windows-link", ] [[package]] @@ -1358,64 +1268,28 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -1428,72 +1302,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -1501,12 +1327,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" +name = "wit-bindgen-rt" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "zmij" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f4a4e8e9dc5c62d159f04fcdbe07f4c3fb710415aab4754bf11505501e3251d" + +[[package]] +name = "zstd-sys" +version = "2.0.13+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index 54f0df8..e19ad2e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "electrs" -version = "0.10.6" +version = "0.11.1" authors = ["Roman Zeyde "] description = "An efficient re-implementation of Electrum Server in Rust" license = "MIT" @@ -10,6 +10,7 @@ keywords = ["bitcoin", "electrum", "server", "index", "database"] documentation = "https://docs.rs/electrs/" readme = "README.md" edition = "2021" +rust-version = "1.85.0" build = "build.rs" [features] @@ -22,30 +23,25 @@ spec = "internal/config_specification.toml" [dependencies] anyhow = "1.0" -bitcoin = { version = "0.32.2", features = ["serde", "rand-std"] } -bitcoin_slices = { version = "0.8", features = ["bitcoin", "sha2"] } -bitcoincore-rpc = { version = "0.19.0" } +bindex = { version = "0.1.2", default-features = false, features = ["bitcoin_serde"] } configure_me = "0.4" crossbeam-channel = "0.5" dirs-next = "2.0" env_logger = "0.10" +jsonrpc = { version = "0.20.1", default-features = false, features = ["simple_http"]} log = "0.4" -parking_lot = "0.12" -prometheus = { version = "0.13", optional = true } -rayon = "1.9" -serde = "1.0" -serde_derive = "1.0, <=1.0.171" # avoid precompiled binaries (https://github.com/serde-rs/serde/issues/2538) +prometheus = { version = "0.14", optional = true } +rayon = "1.12" +serde = "1.0.184" +serde_derive = "1.0.184" serde_json = "1.0" -signal-hook = "0.3" tiny_http = { version = "0.12", optional = true } -[dependencies.electrs-rocksdb] -version = "0.19.0-e3" +[target.'cfg(windows)'.dependencies] +ctrlc = "=3.5.2" -default-features = false -# ZSTD is used for data compression -# Snappy is only for checking old DB -features = ["zstd", "snappy"] +[target.'cfg(not(windows))'.dependencies] +signal-hook = "0.4" [build-dependencies] configure_me_codegen = { version = "0.4.8", default-features = false } @@ -53,4 +49,4 @@ configure_me_codegen = { version = "0.4.8", default-features = false } [dev-dependencies] bitcoin-test-data = "0.2.0" hex_lit = "0.1.1" -tempfile = "3.12" +tempfile = "3.25" diff --git a/Dockerfile b/Dockerfile index f34830b..0553985 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,13 +2,13 @@ # The maintainers of electrs are not deeply familiar with Docker, so you should DYOR. # If you are not familiar with Docker either it's probably be safer to NOT use it. -FROM debian:bookworm-slim as base +FROM debian:trixie-slim AS base RUN apt-get update -qqy RUN apt-get install -qqy librocksdb-dev curl ### Electrum Rust Server ### -FROM base as electrs-build -RUN apt-get install -qqy cargo clang cmake +FROM base AS electrs-build +RUN apt-get install -qqy cargo build-essential libclang-dev # Install electrs WORKDIR /build/electrs @@ -17,7 +17,7 @@ ENV ROCKSDB_INCLUDE_DIR=/usr/include ENV ROCKSDB_LIB_DIR=/usr/lib RUN cargo install --locked --path . -FROM base as result +FROM base AS result # Copy the binaries COPY --from=electrs-build /root/.cargo/bin/electrs /usr/bin/electrs diff --git a/Dockerfile.ci b/Dockerfile.ci index 04318e3..eadb696 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -2,13 +2,13 @@ # The maintainers of electrs are not deeply familiar with Docker, so you should DYOR. # If you are not familiar with Docker either it's probably be safer to NOT use it. -FROM debian:bookworm-slim as base +FROM debian:trixie-slim AS base RUN apt-get update -qqy -RUN apt-get install -qqy librocksdb-dev wget +RUN apt-get install -qqy librocksdb-dev libsqlite3-dev libevent-dev libboost-dev wget ### Electrum Rust Server ### -FROM base as electrs-build -RUN apt-get install -qqy cargo clang cmake +FROM base AS electrs-build +RUN apt-get install -qqy cargo build-essential libclang-dev # Install electrs WORKDIR /build/electrs @@ -17,18 +17,17 @@ ENV ROCKSDB_INCLUDE_DIR=/usr/include ENV ROCKSDB_LIB_DIR=/usr/lib RUN cargo install --locked --path . -### Bitcoin Core ### -FROM base as bitcoin-build +### Bitcoin Core (custom build for /blockpart/ REST endpoint) ### +FROM base AS bitcoin-build # Download WORKDIR /build/bitcoin -ARG ARCH=x86_64 -ARG BITCOIND_VERSION=27.0 -RUN wget -q https://bitcoincore.org/bin/bitcoin-core-$BITCOIND_VERSION/bitcoin-$BITCOIND_VERSION-$ARCH-linux-gnu.tar.gz -RUN tar xvf bitcoin-$BITCOIND_VERSION-$ARCH-linux-gnu.tar.gz -RUN mv -v bitcoin-$BITCOIND_VERSION/bin/bitcoind . -RUN mv -v bitcoin-$BITCOIND_VERSION/bin/bitcoin-cli . -FROM base as result +RUN wget -q https://bitcoincore.org/bin/bitcoin-core-31.1/bitcoin-31.1-x86_64-linux-gnu.tar.gz +RUN tar xvf bitcoin-31.1-x86_64-linux-gnu.tar.gz +RUN mv -v bitcoin-31.1/bin/bitcoind . +RUN mv -v bitcoin-31.1/bin/bitcoin-cli . + +FROM base AS result # Copy the binaries COPY --from=electrs-build /root/.cargo/bin/electrs /usr/bin/electrs COPY --from=bitcoin-build /build/bitcoin/bitcoind /build/bitcoin/bitcoin-cli /usr/bin/ @@ -37,10 +36,10 @@ RUN bitcoind -version && bitcoin-cli -version ### Electrum ### # Clone latest Electrum wallet and a few test tools WORKDIR /build/ -RUN apt-get install -qqy git libsecp256k1-1 python3-cryptography python3-setuptools python3-venv python3-pip jq curl +RUN apt-get install -qqy git libsecp256k1-2 python3-cryptography python3-setuptools python3-venv python3-pip jq curl RUN git clone --recurse-submodules https://github.com/spesmilo/electrum/ && cd electrum/ && git log -1 RUN python3 -m venv --system-site-packages venv && \ - venv/bin/pip install -e electrum/ && \ + ELECTRUM_ECC_DONT_COMPILE=1 venv/bin/pip install -e electrum/ && \ ln /build/venv/bin/electrum /usr/bin/electrum RUN electrum version --offline diff --git a/README.md b/README.md index 14d8e7d..2f1a222 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # Electrum Server in Rust [![CI](https://github.com/romanz/electrs/actions/workflows/rust.yml/badge.svg)](https://github.com/romanz/electrs/actions) -[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com) +[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](https://github.com/romanz/electrs/compare) [![crates.io](https://img.shields.io/crates/v/electrs.svg)](https://crates.io/crates/electrs) [![gitter.im](https://badges.gitter.im/romanz/electrs.svg)](https://gitter.im/romanz/electrs) @@ -41,13 +41,13 @@ which is better optimized for public usage at the cost of consuming *significant ## Features - * Supports Electrum protocol [v1.4](https://electrumx-spesmilo.readthedocs.io/en/latest/protocol.html) + * Supports Electrum protocol [v1.4](https://electrum-protocol.readthedocs.io/) * Maintains an index over transaction inputs and outputs, allowing fast balance queries * Fast synchronization of the Bitcoin blockchain (~6.5 hours for ~504GB @ August 2023) using HDD storage. * Low index storage overhead (~10%), relying on a local full node for transaction retrieval * Efficient mempool tracker (allowing better fee [estimation](https://github.com/spesmilo/electrum/blob/59c1d03f018026ac301c4e74facfc64da8ae4708/RELEASE-NOTES#L34-L46)) * Low CPU & memory usage (after initial indexing) - * [`txindex`](https://github.com/bitcoinbook/bitcoinbook/blob/develop/ch03.asciidoc#txindex) is not required for the Bitcoin node + * [`txindex`](https://github.com/bitcoinbook/bitcoinbook/blob/develop/ch03_bitcoin-core.adoc#txindex) is not required for the Bitcoin node * Uses a single [RocksDB](https://github.com/spacejam/rust-rocksdb) database, for better consistency and crash recovery ## Altcoins @@ -69,4 +69,4 @@ All contributions to this project are welcome. Please refer to the [Contributing [Our logo](logo/) is generously provided by [Dominik Průša](https://github.com/DominoPrusa) under the MIT license. Based on the [Electrum logo](https://github.com/spesmilo/electrum/blob/master/LICENCE) -and the [Rust language logo](https://www.rust-lang.org/policies/media-guide). +and the [Rust language logo](https://rustfoundation.org/policy/rust-trademark-policy). diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index a57375a..679296f 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,45 @@ +# 0.11.1 (Feb 22 2026) + +* Update `transaction.id_from_pos` response for API compliance. +* Expose configurable magic on non-signet. +* Update dependencies (`bitcoin`, `ctrlc`, `log`, `serde_json`, `signal-hook`, `tempfile`). + +# 0.11.0 (Nov 16 2025) + +* Update MSRV to 1.85 +* Support Debian 13 (trixie) / Ubuntu 25.04 (plucky) +* Update RocksDB crate to `rust-rocksdb` 0.36 + - [Dynamic linking with RocksDB](doc/install.md) now requires [RocksDB 9.10.0](https://packages.debian.org/trixie/librocksdb-dev) (instead of 7.8.3). + - Old electrs versions won't be able to read the new RocksDB database format (requiring reindex). +* Support `blockchain.transaction.broadcast_package` Electrum RPC. +* Update dependencies (`bitcoin`, `ctrlc`, `log`, `prometheus`, `rayon`, `serde`, `serde_json`, `tempfile`, `tempfile`). + +# 0.10.10 (Jul 19 2025) + +* Update documentation links and typos. +* Update dependencies (`bitcoin`, `crossbeam-channel`, `log`, `prometheus`, `signal-hook`, `tempfile`). + +# 0.10.9 (Feb 01 2025) + +* Update `bitcoin_slices` to 0.10.0 +* Allow concurrent DB background operations for SSDs + +# 0.10.8 (Dec 30 2024) + +* Separate blocks reading & index writing into scoped threads +* Set HTTP `Content-Type` header for Prometheus response +* Use `ctrlc` in place of `signal-hook` on Windows +* Don't deserialize transactions if not needed +* Run CI only on `master` branch and PRs +* Update dependencies (`bitcoin`, `bitcoin_slices`, `crossbeam-channel`, `tempfile`) + +# 0.10.7 (Nov 05 2024) + +* Support testnet4 +* Enable LTO in release build +* Don't sync mempool when bitcoind mempool is not yet loaded +* Update dependencies (`bitcoin`, `bitcoin_slices`) + # 0.10.6 (Sep 29 2024) * Update dependencies (`bitcoin`, `configure_me_codegen`, `crossbeam-channel`, `log`) @@ -228,7 +270,7 @@ Full list of changes: * Fix JSONRPC errors' handling (#398, #390) * Optimize Dockerfile (#387, #388, #392) -* Fix signet default port (https://github.com/romanz/electrs/b53178c140e575b0527a70ead566d50c7fe6cb1f) +* Fix signet default port (https://github.com/romanz/electrs/commit/b53178c140e575b0527a70ead566d50c7fe6cb1f) # 0.8.9 (19 Mar 2021) diff --git a/TODO.md b/TODO.md index 9736158..ea11fbc 100644 --- a/TODO.md +++ b/TODO.md @@ -5,7 +5,7 @@ # Rust -* Use [bytes](https://carllerche.github.io/bytes/bytes/index.html) instead of `Vec` when possible +* Use [bytes](https://docs.rs/bytes/latest/bytes/) instead of `Vec` when possible * Use generators instead of vectors * Use proper HTTP parser for JSONRPC replies over persistent connection diff --git a/contrib/history.py b/contrib/history.py index ac4867e..ea40d94 100755 --- a/contrib/history.py +++ b/contrib/history.py @@ -42,6 +42,12 @@ def main(): elif args.network == 'testnet': port = 60001 from pycoin.symbols.xtn import network + elif args.network == 'testnet4': + port = 40001 + from pycoin.symbols.xtn import network + elif args.network == 'signet': + port = 60601 + from pycoin.symbols.xtn import network elif args.network == 'mainnet': port = 50001 from pycoin.symbols.btc import network diff --git a/doc/config.md b/doc/config.md index 3784b20..b234e72 100644 --- a/doc/config.md +++ b/doc/config.md @@ -5,11 +5,12 @@ If you use automated systems, refer to their documentation first! ### Bitcoind configuration -Pruning must be turned **off** for `electrs` to work. +REST API must be **enabled** for block indexing and transaction lookup. + +Pruning must be **disabled** for `electrs` to work. + `txindex` is allowed but unnecessary for `electrs`. However, you might still need it if you run other services (e.g.`eclair`). -The option `maxconnections` (if used) should be set to 12 or more for bitcoind to accept inbound p2p connections. -Note that setting `maxuploadtarget` may cause p2p-based sync to fail - so consider using `-whitelist=download@127.0.0.1` to disable the limit for local p2p connections. The highly recommended way of authenticating `electrs` is using cookie file. It's the most [secure](https://github.com/Kixunil/security_writings/blob/master/cookie_files.md) and robust method. @@ -21,7 +22,7 @@ You can skip it if you're running both daemons under the same user and with the Example command for running `bitcoind` (assuming same user, default dirs): ```bash -$ bitcoind -server=1 -txindex=0 -prune=0 +$ bitcoind -server=1 -rest=1 -txindex=0 -prune=0 ``` ### Electrs configuration @@ -139,7 +140,7 @@ HiddenServiceVersion 3 HiddenServicePort 50001 127.0.0.1:50001 ``` -If you use [the *beta* Debian repository](binaries.md#cnative-os-packages), +If you use [the *beta* Debian repository](binaries.md#native-os-packages), it is cleaner to install `tor-hs-patch-config` using `apt` and then placing the configuration into a file inside `/etc/tor/hidden-services.d`. Restart the service: @@ -162,7 +163,7 @@ For more details, see http://docs.electrum.org/en/latest/tor.html. ### Sample Systemd Unit File -If you use [the *beta* Debian repository](binaries.md#cnative-os-packages), you should skip this section, +If you use [the *beta* Debian repository](binaries.md#native-os-packages), you should skip this section, as the appropriate systemd unit file is installed automatically. You may wish to have systemd manage electrs so that it's "always on". diff --git a/doc/config_example.toml b/doc/config_example.toml index 1272ddb..59cf429 100644 --- a/doc/config_example.toml +++ b/doc/config_example.toml @@ -14,9 +14,6 @@ cookie_file = "/var/run/bitcoin-mainnet/cookie" # The listening RPC address of bitcoind, port is usually 8332 daemon_rpc_addr = "127.0.0.1:8332" -# The listening P2P address of bitcoind, port is usually 8333 -daemon_p2p_addr = "127.0.0.1:8333" - # Directory where the index should be stored. It should have at least 70GB of free space. db_dir = "/some/fast/storage/with/big/size" diff --git a/doc/cookie_deprecation.md b/doc/cookie_deprecation.md index 953dfc0..37bec32 100644 --- a/doc/cookie_deprecation.md +++ b/doc/cookie_deprecation.md @@ -21,7 +21,7 @@ The option was confusing: If you're installing `electrs` for the first time, just don't use `cookie`. If you're updating, reconsider the motivation above. -If you used copying script, just use `cookie_file` to get the cookie directly. +If you used a copying script, just use `cookie_file` to get the cookie directly. If you also used `BindsTo`, we recommend removing it. If you used fixed username and password because you didn't know about cookie or did it before `cookie_file` was implemented, reconsider using cookie authentication. If you really have to use fixed username and password, specify them using `auth` option (`username:password` like before) and remove the `cookie` option. diff --git a/doc/install.md b/doc/install.md index b5d8a65..3ac8229 100644 --- a/doc/install.md +++ b/doc/install.md @@ -1,21 +1,14 @@ ## Quickstart -
-Building from source on an Ubuntu 21.10 VM: +Building from source on an Ubuntu 25.04 / Debian 13: ```bash $ sudo apt update -$ sudo apt install -y clang cmake build-essential git cargo -$ git clone https://github.com/romanz/electrs -$ cd electrs -$ cargo build --locked --release -$ ./target/release/electrs --version # should print the latest version +$ sudo apt install -y build-essential libclang-dev git cargo +$ git clone https://github.com/romanz/electrs && cd electrs +$ cargo run --locked --release -- --version # build and print the latest version ``` -
- -[![asciicast](https://asciinema.org/a/XKznxilP4O7lCZiVZ9vZNd5vx.svg)](https://asciinema.org/a/XKznxilP4O7lCZiVZ9vZNd5vx?speed=3) - ## Manual installation from source **See below for automated/binary installation options.** @@ -25,14 +18,14 @@ $ ./target/release/electrs --version # should print the latest version Note for Raspberry Pi 4 owners: the old versions of OS/toolchains produce broken binaries. Make sure to use latest OS! (see #226) -Install [recent Rust](https://rustup.rs/) (1.63.0+, `apt install cargo` is preferred for Debian 12), -[latest Bitcoin Core](https://bitcoincore.org/en/download/) (0.21+) +Install [recent Rust](https://rustup.rs/) (1.85.0+, `apt install cargo` is preferred for Debian 13), +[latest Bitcoin Core](https://bitcoincore.org/en/download/) (31.0+) and [latest Electrum wallet](https://electrum.org/#download) (4.0+). Also, install the following packages (on Debian or Ubuntu): ```bash $ sudo apt update -$ sudo apt install clang cmake build-essential # for building 'rust-rocksdb' +$ sudo apt install build-essential libclang-dev # for building 'rust-rocksdb' ``` There are two ways to compile `electrs`: by statically linking to `librocksdb` or dynamically linking. @@ -52,18 +45,18 @@ The advantages of dynamic linking: * Cross compilation is more reliable * If another application is also using `rocksdb`, you don't store it on disk and in RAM twice -If you decided to use dynamic linking, you will also need to install the library ([7.8.3 release](https://github.com/facebook/rocksdb/releases/tag/v7.8.3) is required). -On [Debian 12 (bookworm)](https://packages.debian.org/bookworm/librocksdb-dev) and [Ubuntu 23.04 (lunar)](https://packages.ubuntu.com/lunar/librocksdb-dev): +If you decided to use dynamic linking, you will also need to install the library ([9.10.0 release](https://github.com/facebook/rocksdb/releases/tag/v9.10.0) is required). +On [Debian 13 (trixie)](https://packages.debian.org/bookworm/librocksdb-dev) and [Ubuntu 25.04 (plucky)](https://packages.ubuntu.com/plucky/librocksdb-dev): ```bash -$ sudo apt install librocksdb-dev=7.8.3-2 +$ sudo apt install librocksdb-dev=9.10.0-1 ``` For other versions of Debian or Ubuntu, you can build librocksdb and install inside `/usr/local` directory using following command. ```bash $ sudo apt install -y libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev -$ git clone -b v7.8.3 --depth 1 https://github.com/facebook/rocksdb && cd rocksdb +$ git clone -b v9.10.0 --depth 1 https://github.com/facebook/rocksdb && cd rocksdb $ make shared_lib -j $(nproc) && sudo make install-shared $ cd .. && rm -r rocksdb ``` @@ -149,7 +142,7 @@ sudo chroot debootstrap-buster /bin/bash apt install curl curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh source "$HOME/.cargo/env" -apt install clang cmake build-essential +apt install build-essential libclang-dev # install target specific cross compiler (armhf/gnueabihf) apt install gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf libc6-dev-armhf-cross diff --git a/doc/schema.md b/doc/schema.md index ab8af29..901b775 100644 --- a/doc/schema.md +++ b/doc/schema.md @@ -1,53 +1 @@ -# Index Schema - -The index is stored at a single RocksDB database using the following column families. -Most of the data is stored in key-only DB rows (i.e. having empty values). - -## Transaction outputs' index (`funding`) - -Allows efficiently finding all funding transactions for a specific address: - -| Script Hash Prefix | Confirmed Block Height | -| -------------------- | ---------------------- | -| `SHA256(script)[:8]` | `height as u32` | - -## Transaction inputs' index (`spending`) - -Allows efficiently finding spending transaction of a specific output: - -| Previous Outpoint Prefix | Confirmed Block Height | -| ------------------------ | ---------------------- | -| `txid[:8] as u64 + vout` | `height as u32` | - - -## Transaction ID index (`txid`) - -In order to save storage space, we map the 8-byte transaction ID prefix to its confirmed block height: - -| Txid Prefix | Confirmed height | -| ----------- | ---------------- | -| `txid[:8]` | `height as u32` | - -Note that this mapping allows us to use `getrawtransaction` RPC to retrieve actual transaction data from without `-txindex` enabled -(by explicitly specifying the [blockhash](https://github.com/bitcoin/bitcoin/commit/497d0e014cc79d46531d570e74e4aeae72db602d)). - -## Headers (`headers`) - -For faster loading, we store all block headers in RocksDB: - -| Serialized header | -| ----------------------- | -| `header as BlockHeader` | - -In addition, we also store the chain tip: - -| Key | | Value | -| --- | - | ------------------------ | -| `T` | | `blockhash as BlockHash` | - -## Configuration (`config`) - -| Key | | Value | -| --- | - | --------------------------- | -| `C` | | `serialized config as JSON` | - +See https://github.com/romanz/bindex-rs for more details. diff --git a/doc/upgrading.md b/doc/upgrading.md index a1de2a2..fd060da 100644 --- a/doc/upgrading.md +++ b/doc/upgrading.md @@ -1,7 +1,24 @@ +### Important changes from versions older than 0.12.0 + +In 0.12.0 we have changed the RocksDB index format to optimize electrs performance. + +We also use new bitcoind REST API endpoints (instead of P2P protocol) + +* [#32540 (index: fetch spent transaction outputs by blockhash)](https://github.com/bitcoin/bitcoin/pull/32540) → released in 30.0 +* [#33657 (rest: allow reading partial block data from storage)](https://github.com/bitcoin/bitcoin/pull/33657) → released in 31.0 + +Upgrading checklist: + +* Make sure you upgrade at a time when you don't need to use electrs for a while. + Because of reindex electrs will be unable to serve your requests for a few hours. + (The exact time depends on your hardware.) +* Make sure you have at least 120 GB of free space (for reindexing mainnet chain). + + ### Important changes from versions older than 0.9.3 * If you use `verbose` (or `-v` argument), switch to `log_filters` (or `RUST_LOG` environment variable). - Please note that it allows to set per-module filters, but module naming is considered unstable. + Please note that it allows setting per-module filters, but module naming is considered unstable. If you have used `-vv` (the value suggested in the documentation), switch to `--log-filters INFO`: @@ -11,7 +28,7 @@ |WARN | 1|Things that could indicate serious problems | |INFO | 2|Various significant events and suggestions | |DEBUG | 3|Details that could be useful when debugging - only use when debugging!| -|TRACE | 4|**Very** detailed information - only use when debugging! | +|TRACE | 4|**Very** detailed information - only use when debugging! | ### Important changes from versions older than 0.9.0 @@ -22,7 +39,7 @@ Some guides were suggesting trace log level and we started to trace much more in Upgrading checklist: -* Make sure you upgrade at time when you don't need to use electrs for a while. +* Make sure you upgrade at a time when you don't need to use electrs for a while. Because of reindex electrs will be unable to serve your requests for a few hours. (The exact time depends on your hardware.) If you wish to check the database without reindexing run electrs with `--no-auto-reindex`. @@ -38,7 +55,7 @@ Upgrading checklist: * If you use `verbose = 4` (or `-vvvv` argument) lower it down to `2` (`-vv`) for production use. Keeping it would waste resources because we utilize it more now. * **After reindexing**, if you did **not** delete `mainnet` subdirectory within `db_dir` check that `electrs` works as expected and then *delete whole `mainnet` subdirectory*. -* If you are using our Dockerfile, please make sure to re-map the DB volume (see [the section above](docker.md#docker-based-installation-from-source)). +* If you are using our Dockerfile, please make sure to re-map the DB volume (see [the section above](install.md#docker-based-installation-from-source)). ### Important changes from version older than 0.8.8 @@ -69,8 +86,8 @@ sudo apt update sudo apt upgrade ``` -Similarly for other distributions - use their respective commands. -If a new version of `electrs` is not yet in the package system, try wait a few days or contact the maintainers of the packages if it's been a long time. +Similarly for other distributions - use their respective commands. +If a new version of `electrs` is not yet in the package system, try waiting a few days or contact the maintainers of the packages if it has been a long time. ### Upgrading manual installation @@ -85,6 +102,6 @@ If a new version of `electrs` is not yet in the package system, try wait a few d If you don't remember which linking you used, you probably used static. This step will take a few tens of minutes (but dynamic linking is a bit faster), go grab a coffee. Also remember that you need enough free RAM, the build will die otherwise -7. If you've previously copied `electrs` into `/usr/local/bin` run: sudo `cp target/release/electrs /usr/local/bin` +7. If you've previously copied `electrs` into `/usr/local/bin` run: `sudo cp target/release/electrs /usr/local/bin` If you've previously installed `electrs` using `cargo install`: `cargo install --locked --path . -f` 8. If you've manually configured systemd service: `sudo systemctl restart electrs` diff --git a/doc/usage.md b/doc/usage.md index 7e71f88..8ec9ee9 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -1,77 +1,63 @@ ## Quickstart -
-Assuming Bitcoin Core 0.21+ is installed on the same machine (with the standard configuration at `~/.bitcoin/bitcoin.conf`): +Assuming Bitcoin Core 31+ is installed on the same machine (with the standard configuration at `~/.bitcoin/bitcoin.conf`): ```bash -$ bitcoind -server=1 -prune=0 & +$ bitcoind -server=1 -rest=1 -prune=0 & $ # ... wait until the chain is synced (e.g. using `bitcoin-cli getblockchaininfo`) -$ electrs --log-filters=INFO --db-dir ./db --daemon-dir ~/.bitcoin --network bitcoin +$ RUST_LOG=INFO electrs --network bitcoin --db-dir ./db --daemon-dir ~/.bitcoin ``` -
- -[![asciicast](https://asciinema.org/a/zRNZp5HsBDi5rAlGWU7470Pzl.svg)](https://asciinema.org/a/zRNZp5HsBDi5rAlGWU7470Pzl?speed=3) - ## Usage -First index sync should take ~6.5 hours for ~504GB @ August 2023 (on a dual core Intel CPU @ 3.3 GHz, 8 GB RAM, 1TB WD Blue HDD): +First index sync should take ~2 hours for ~800GB `.bitcoin/blocks/` @ July 2026 (on a 6-core CPU, 32 GB RAM, 2TB NVMe): ```bash -$ du -ch ~/.bitcoin/blocks/blk*.dat | tail -n1 -336G total +$ du -ch ~/.bitcoin/blocks/*.dat | tail -n1 +802G total -$ ./target/release/electrs --network bitcoin --db-dir ./db --daemon-dir /home/user/.bitcoin -Starting electrs 0.10.0 on x86_64 linux with Config { network: Bitcoin, db_path: "./db/bitcoin", daemon_dir: "/home/user/.bitcoin", daemon_auth: CookieFile("/home/user/.bitcoin/.cookie"), daemon_rpc_addr: 127.0.0.1:8332, daemon_p2p_addr: 127.0.0.1:8333, electrum_rpc_addr: 127.0.0.1:50001, monitoring_addr: 127.0.0.1:4224, wait_duration: 10s, jsonrpc_timeout: 15s, index_batch_size: 10, index_lookup_limit: None, reindex_last_blocks: 0, auto_reindex: true, ignore_mempool: false, sync_once: false, skip_block_download_wait: false, disable_electrum_rpc: false, server_banner: "Welcome to electrs 0.10.0 (Electrum Rust Server)!", signet_magic: f9beb4d9, args: [] } -[2023-08-16T19:17:11.193Z INFO electrs::metrics::metrics_impl] serving Prometheus metrics on 127.0.0.1:4224 -[2023-08-16T19:17:11.193Z INFO electrs::server] serving Electrum RPC on 127.0.0.1:50001 -[2023-08-16T19:17:12.355Z INFO electrs::db] "./db/bitcoin": 0 SST files, 0 GB, 0 Grows -[2023-08-16T19:17:12.446Z INFO electrs::index] indexing 2000 blocks: [1..2000] -[2023-08-16T19:17:12.866Z INFO electrs::chain] chain updated: tip=00000000dfd5d65c9d8561b4b8f60a63018fe3933ecb131fb37f905f87da951a, height=2000 -[2023-08-16T19:17:12.879Z INFO electrs::index] indexing 2000 blocks: [2001..4000] -[2023-08-16T19:17:13.227Z INFO electrs::chain] chain updated: tip=00000000922e2aa9e84a474350a3555f49f06061fd49df50a9352f156692a842, height=4000 -[2023-08-16T19:17:13.238Z INFO electrs::index] indexing 2000 blocks: [4001..6000] -[2023-08-16T19:17:13.587Z INFO electrs::chain] chain updated: tip=00000000dbbb79792303bdd1c6c4d7ab9c21bba0667213c2eca955e11230c5a5, height=6000 -[2023-08-16T19:17:13.598Z INFO electrs::index] indexing 2000 blocks: [6001..8000] -[2023-08-16T19:17:13.950Z INFO electrs::chain] chain updated: tip=0000000094fbacdffec05aea9847000522a258c269ae37a74a818afb96fc27d9, height=8000 -[2023-08-16T19:17:13.961Z INFO electrs::index] indexing 2000 blocks: [8001..10000] -<...> -[2023-08-17T00:13:16.443Z INFO electrs::index] indexing 2000 blocks: [798001..800000] -[2023-08-17T00:14:58.310Z INFO electrs::chain] chain updated: tip=00000000000000000002a7c4c1e48d76c5a37902165a270156b7a8d72728a054, height=800000 -[2023-08-17T00:14:58.325Z INFO electrs::index] indexing 2000 blocks: [800001..802000] -[2023-08-17T00:16:36.425Z INFO electrs::chain] chain updated: tip=0000000000000000000311b41f1d611f977b024b947568c1dd760704360f148a, height=802000 -[2023-08-17T00:16:36.437Z INFO electrs::index] indexing 1534 blocks: [802001..803534] -[2023-08-17T00:17:51.338Z INFO electrs::chain] chain updated: tip=00000000000000000003c0cd1b62ed8bb502e24bcbfeee16e81d6ea33d026263, height=803534 -[2023-08-17T00:18:00.592Z INFO electrs::db] starting config compaction -[2023-08-17T00:18:00.778Z INFO electrs::db] starting headers compaction -[2023-08-17T00:18:00.870Z INFO electrs::db] starting txid compaction -[2023-08-17T00:33:34.370Z INFO electrs::db] starting funding compaction -[2023-08-17T01:03:56.784Z INFO electrs::db] starting spending compaction -[2023-08-17T01:35:05.983Z INFO electrs::db] finished full compaction -[2023-08-17T01:36:23.300Z INFO electrs::index] indexing 5 blocks: [803535..803539] -[2023-08-17T01:36:23.646Z INFO electrs::chain] chain updated: tip=000000000000000000006a3aaddd4b643607b33e000f1200d35005c330ecfa88, height=803539 -[2023-08-17T01:41:26.009Z INFO electrs::index] indexing 1 blocks: [803540..803540] -[2023-08-17T01:41:26.143Z INFO electrs::chain] chain updated: tip=00000000000000000003266d31db92629b64241eef7ce708244f6d6283b080b4, height=803540 -[2023-08-17T01:42:42.999Z INFO electrs::index] indexing 1 blocks: [803541..803541] -[2023-08-17T01:42:43.153Z INFO electrs::chain] chain updated: tip=00000000000000000000884a77c8b8ad2fb0c25510a3251bf5ef57f0db275146, height=803541 +$ RUST_LOG=INFO electrs --network bitcoin --db-dir ./db --daemon-dir ~/.bitcoin +Starting electrs 0.12.0 on x86_64 linux with Config { network: Bitcoin, db_dir: "./db", ... } +[2026-07-18T16:37:24.024Z INFO electrs::metrics::metrics_impl] serving Prometheus metrics on 127.0.0.1:4224 +[2026-07-18T16:37:24.025Z INFO electrs::server] serving Electrum RPC on 127.0.0.1:50001 +[2026-07-18T16:37:24.025Z INFO bindex::chain] index: Config { db_path: "./db/bitcoin", url: "http://127.0.0.1:8332" } +[2026-07-18T16:37:24.054Z INFO bindex::db] CF config: 0 files, 0.000000 MBs +[2026-07-18T16:37:24.054Z INFO bindex::db] CF headers: 0 files, 0.000000 MBs +[2026-07-18T16:37:24.054Z INFO bindex::db] CF txpos: 0 files, 0.000000 MBs +[2026-07-18T16:37:24.054Z INFO bindex::db] CF txid: 0 files, 0.000000 MBs +[2026-07-18T16:37:24.054Z INFO bindex::db] CF script_hash: 0 files, 0.000000 MBs +[2026-07-18T16:37:24.054Z INFO bindex::db] CF spending: 0 files, 0.000000 MBs +[2026-07-18T16:37:24.054Z INFO bindex::db] CF funding: 0 files, 0.000000 MBs +[2026-07-18T16:37:24.129Z INFO bindex::chain] block=00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09 height=1000: indexed 1001 blocks, 0.226[MB], dt = 0.047[s]: 0.047 [ms/block], 0.000 [MB/block], 4.837 [MB/s] +[2026-07-18T16:37:24.196Z INFO bindex::chain] block=00000000dfd5d65c9d8561b4b8f60a63018fe3933ecb131fb37f905f87da951a height=2000: indexed 1000 blocks, 0.234[MB], dt = 0.043[s]: 0.043 [ms/block], 0.000 [MB/block], 5.376 [MB/s] +[2026-07-18T16:37:24.263Z INFO bindex::chain] block=000000004a81b9aa469b11649996ecb0a452c16d1181e72f9f980850a1c5ecce height=3000: indexed 1000 blocks, 0.230[MB], dt = 0.043[s]: 0.043 [ms/block], 0.000 [MB/block], 5.298 [MB/s] +[2026-07-18T16:37:24.331Z INFO bindex::chain] block=00000000922e2aa9e84a474350a3555f49f06061fd49df50a9352f156692a842 height=4000: indexed 1000 blocks, 0.234[MB], dt = 0.044[s]: 0.044 [ms/block], 0.000 [MB/block], 5.366 [MB/s] +[2026-07-18T16:37:24.398Z INFO bindex::chain] block=000000004d78d2a8a93a1d20a24d721268690bebd2b51f7e80657d57e226eef9 height=5000: indexed 1000 blocks, 0.229[MB], dt = 0.043[s]: 0.043 [ms/block], 0.000 [MB/block], 5.349 [MB/s] +... +[2026-07-18T18:02:21.114Z INFO bindex::chain] block=0000000000000000000055222909c19ed96cd371013861337214a3c39a63d828 height=955000: indexed 1000 blocks, 1842.985[MB], dt = 9.457[s]: 9.457 [ms/block], 1.843 [MB/block], 194.886 [MB/s] +[2026-07-18T18:02:33.592Z INFO bindex::chain] block=000000000000000000012d9ea47c8b3282ae1a7792d54b8b4bcc0536c1883191 height=956000: indexed 1000 blocks, 1846.173[MB], dt = 9.522[s]: 9.522 [ms/block], 1.846 [MB/block], 193.879 [MB/s] +[2026-07-18T18:02:45.872Z INFO bindex::chain] block=00000000000000000000e898fc9d01b0729d8830f2068248debbff575a3b0990 height=957000: indexed 1000 blocks, 1855.303[MB], dt = 9.295[s]: 9.295 [ms/block], 1.855 [MB/block], 199.600 [MB/s] +[2026-07-18T18:02:58.221Z INFO bindex::chain] block=000000000000000000022c6a6d538a4372b0eeadb4ea9eb2edd4e32ec780084b height=958000: indexed 1000 blocks, 1844.747[MB], dt = 9.376[s]: 9.376 [ms/block], 1.845 [MB/block], 196.759 [MB/s] +[2026-07-18T18:03:05.539Z INFO bindex::chain] block=000000000000000000020934afa0945d959250e9a4598d39c53fa280639b096a height=958594: indexed 594 blocks, 1115.055[MB], dt = 5.357[s]: 9.018 [ms/block], 1.877 [MB/block], 208.155 [MB/s] +[2026-07-18T18:03:05.666Z INFO bindex::db] started auto compactions ``` You can specify options via command-line parameters, environment variables or using config files. See the documentation above. -Note that the final DB size should be ~10% of the `blk*.dat` files, but it may increase to ~20% at the end of the initial sync (just before the [full compaction is invoked](https://github.com/facebook/rocksdb/wiki/Manual-Compaction)). +Note that the final DB size should be ~7% of the `blocks/*.dat` files, but it may increase to ~14% at the end of the initial sync (just before the [full compaction is invoked](https://github.com/facebook/rocksdb/wiki/Manual-Compaction)). It should take roughly 18 hours to sync and compact the index on an ODROID-HC1 with 8 CPU cores @ 2GHz, 2GB RAM, and an SSD using the command above. The index database is stored here: ```bash -$ du db/ -42G db/mainnet/ +$ du -h db/bitcoin/ +56G db/bitcoin/ ``` See [extra configuration suggestions](config.md#extra-configuration-suggestions) that you might want to consider. ## Electrum client -If you happen to use the Electrum client from [the *beta* Debian repository](binaries.md#cnative-os-packages), it's pre-configured out-of-the-box already +If you happen to use the Electrum client from [the *beta* Debian repository](binaries.md#native-os-packages), it's pre-configured out-of-the-box already Read below otherwise. There's a prepared script for launching `electrum` in such way to connect only to the local `electrs` instance to protect your privacy. @@ -98,11 +84,11 @@ You can invoke any supported RPC using `netcat`, for example: ``` $ echo '{"jsonrpc": "2.0", "method": "server.version", "params": ["", "1.4"], "id": 0}' | netcat 127.0.0.1 50001 -{"id":0,"jsonrpc":"2.0","result":["electrs 0.9.0","1.4"]} +{"id":0,"jsonrpc":"2.0","result":["electrs 0.12.0","1.4"]} ``` -For more complex tasks, you may need to convert addresses to -[script hashes](https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-basics.html#script-hashes) - see +For more complex tasks, you may need to convert addresses to +[script hashes](https://electrum-protocol.readthedocs.io/en/latest/protocol-basics.html#script-hashes) - see [contrib/history.py](https://github.com/romanz/electrs/blob/master/contrib/history.py) for getting an address balance and history: ``` diff --git a/examples/tx_collisions.rs b/examples/tx_collisions.rs deleted file mode 100644 index 839a508..0000000 --- a/examples/tx_collisions.rs +++ /dev/null @@ -1,31 +0,0 @@ -use anyhow::{Context, Result}; -use electrs_rocksdb::{ColumnFamilyDescriptor, IteratorMode, Options, DB}; - -fn main() -> Result<()> { - let path = std::env::args().nth(1).context("missing DB path")?; - let cf_names = DB::list_cf(&Options::default(), &path)?; - let cfs: Vec<_> = cf_names - .iter() - .map(|name| ColumnFamilyDescriptor::new(name, Options::default())) - .collect(); - let db = DB::open_cf_descriptors(&Options::default(), &path, cfs)?; - let cf = db.cf_handle("txid").context("missing column family")?; - - let mut state: Option<(u64, u32)> = None; - for row in db.iterator_cf(cf, IteratorMode::Start) { - let (curr, _value) = row?; - let curr_prefix = u64::from_le_bytes(curr[..8].try_into()?); - let curr_height = u32::from_le_bytes(curr[8..].try_into()?); - - if let Some((prev_prefix, prev_height)) = state { - if prev_prefix == curr_prefix { - eprintln!( - "prefix={:x} heights: {} {}", - curr_prefix, prev_height, curr_height - ); - }; - } - state = Some((curr_prefix, curr_height)); - } - Ok(()) -} diff --git a/internal/config_specification.toml b/internal/config_specification.toml index 75aad9b..f445f41 100644 --- a/internal/config_specification.toml +++ b/internal/config_specification.toml @@ -34,6 +34,12 @@ name = "db_log_dir" type = "std::path::PathBuf" doc = "Directory to store index database internal log (default: same as specified by `db_dir`)" +[[param]] +name = "db_parallelism" +type = "u8" +doc = "Max threads to use for DB background operations (flushes and compactions)" +default = "1" + [[param]] name = "daemon_dir" type = "std::path::PathBuf" @@ -57,8 +63,8 @@ doc = "JSONRPC authentication cookie file (default: ~/.bitcoin/.cookie)" [[param]] name = "network" type = "crate::config::BitcoinNetwork" -convert_into = "::bitcoin::Network" -doc = "Select Bitcoin network type ('bitcoin', 'testnet', 'regtest' or 'signet')" +convert_into = "crate::bitcoin::Network" +doc = "Select Bitcoin network type ('bitcoin', 'testnet', 'testnet4', 'regtest' or 'signet')" default = "Default::default()" [[param]] @@ -70,10 +76,6 @@ doc = "Electrum server JSONRPC 'addr:port' to listen on (default: '127.0.0.1:500 name = "daemon_rpc_addr" type = "crate::config::ResolvAddr" doc = "Bitcoin daemon JSONRPC 'addr:port' to connect (default: 127.0.0.1:8332 for mainnet, 127.0.0.1:18332 for testnet, 127.0.0.1:18443 for regtest and 127.0.0.1:18554 for signet)" -[[param]] -name = "daemon_p2p_addr" -type = "crate::config::ResolvAddr" -doc = "Bitcoin daemon p2p 'addr:port' to connect (default: 127.0.0.1:8333 for mainnet, 127.0.0.1:18333 for testnet, 127.0.0.1:18444 for regtest and 127.0.0.1:38333 for signet)" [[param]] name = "monitoring_addr" @@ -92,12 +94,6 @@ type = "u64" doc = "Duration to wait until bitcoind JSON-RPC timeouts (must be greater than wait_duration_secs)." default = "15" -[[param]] -name = "index_batch_size" -type = "usize" -doc = "Number of blocks to get in a single p2p protocol request from bitcoind" -default = "10" - [[switch]] name = "ignore_mempool" doc = "Don't sync mempool - queries will show only confirmed transactions." @@ -140,8 +136,3 @@ default = "concat!(\"Welcome to electrs \", env!(\"CARGO_PKG_VERSION\"), \" (Ele name = "log_filters" type = "String" doc = "Logging filters, overriding `RUST_LOG` environment variable (see https://docs.rs/env_logger/ for details)" - -[[param]] -name = "signet_magic" -type = "String" -doc = "network magic for custom signet network in hex format, as found in Bitcoin Core logs (signet only)" diff --git a/rust-toolchain.toml b/rust-toolchain.toml deleted file mode 100644 index 07d72a9..0000000 --- a/rust-toolchain.toml +++ /dev/null @@ -1,3 +0,0 @@ -[toolchain] -channel = "1.63.0" -components = [ "rustfmt" ] diff --git a/server.sh b/server.sh index c66a4c9..7a7fe5f 100755 --- a/server.sh +++ b/server.sh @@ -9,7 +9,7 @@ NETWORK=$1 shift DB=${DB-./db} -export RUST_LOG=${RUST_LOG-electrs=INFO} +export RUST_LOG=${RUST_LOG-INFO} target/release/electrs --network $NETWORK --db-dir $DB --daemon-dir $HOME/.bitcoin $* # use SIGINT to quit diff --git a/src/cache.rs b/src/cache.rs deleted file mode 100644 index c54df1f..0000000 --- a/src/cache.rs +++ /dev/null @@ -1,43 +0,0 @@ -use bitcoin::{Transaction, Txid}; -use parking_lot::RwLock; - -use std::collections::HashMap; -use std::sync::Arc; - -use crate::metrics::{self, Histogram, Metrics}; - -pub(crate) struct Cache { - txs: Arc>>, - - // stats - txs_size: Histogram, -} - -impl Cache { - pub fn new(metrics: &Metrics) -> Self { - Cache { - txs: Default::default(), - txs_size: metrics.histogram_vec( - "cache_txs_size", - "Cached transactions' size (in bytes)", - "type", - metrics::default_size_buckets(), - ), - } - } - - pub fn add_tx(&self, txid: Txid, f: impl FnOnce() -> Transaction) { - self.txs.write().entry(txid).or_insert_with(|| { - let tx = f(); - self.txs_size.observe("serialized", tx.total_size() as f64); - tx - }); - } - - pub fn get_tx(&self, txid: &Txid, f: F) -> Option - where - F: FnOnce(&Transaction) -> T, - { - self.txs.read().get(txid).map(f) - } -} diff --git a/src/chain.rs b/src/chain.rs deleted file mode 100644 index 9f17276..0000000 --- a/src/chain.rs +++ /dev/null @@ -1,261 +0,0 @@ -use std::collections::HashMap; - -use bitcoin::blockdata::block::Header as BlockHeader; -use bitcoin::{BlockHash, Network}; - -/// A new header found, to be added to the chain at specific height -pub(crate) struct NewHeader { - header: BlockHeader, - hash: BlockHash, - height: usize, -} - -impl NewHeader { - pub(crate) fn from((header, height): (BlockHeader, usize)) -> Self { - Self { - header, - hash: header.block_hash(), - height, - } - } - - pub(crate) fn height(&self) -> usize { - self.height - } - - pub(crate) fn hash(&self) -> BlockHash { - self.hash - } -} - -/// Current blockchain headers' list -pub struct Chain { - headers: Vec<(BlockHash, BlockHeader)>, - heights: HashMap, -} - -impl Chain { - // create an empty chain - pub fn new(network: Network) -> Self { - let genesis = bitcoin::blockdata::constants::genesis_block(network); - let genesis_hash = genesis.block_hash(); - Self { - headers: vec![(genesis_hash, genesis.header)], - heights: std::iter::once((genesis_hash, 0)).collect(), // genesis header @ zero height - } - } - - pub(crate) fn drop_last_headers(&mut self, n: usize) { - if n == 0 { - return; - } - let new_height = self.height().saturating_sub(n); - self.update(vec![NewHeader::from(( - self.headers[new_height].1, - new_height, - ))]) - } - - /// Load the chain from a collection of headers, up to the given tip - pub(crate) fn load(&mut self, headers: impl Iterator, tip: BlockHash) { - let genesis_hash = self.headers[0].0; - - let header_map: HashMap = - headers.map(|h| (h.block_hash(), h)).collect(); - let mut blockhash = tip; - let mut new_headers: Vec<&BlockHeader> = Vec::with_capacity(header_map.len()); - while blockhash != genesis_hash { - let header = match header_map.get(&blockhash) { - Some(header) => header, - None => panic!("missing header {} while loading from DB", blockhash), - }; - blockhash = header.prev_blockhash; - new_headers.push(header); - } - info!("loading {} headers, tip={}", new_headers.len(), tip); - let new_headers = new_headers.into_iter().rev().copied(); // order by height - self.update(new_headers.zip(1..).map(NewHeader::from).collect()) - } - - /// Get the block hash at specified height (if exists) - pub(crate) fn get_block_hash(&self, height: usize) -> Option { - self.headers.get(height).map(|(hash, _header)| *hash) - } - - /// Get the block header at specified height (if exists) - pub(crate) fn get_block_header(&self, height: usize) -> Option<&BlockHeader> { - self.headers.get(height).map(|(_hash, header)| header) - } - - /// Get the block height given the specified hash (if exists) - pub(crate) fn get_block_height(&self, blockhash: &BlockHash) -> Option { - self.heights.get(blockhash).copied() - } - - /// Update the chain with a list of new headers (possibly a reorg) - pub(crate) fn update(&mut self, headers: Vec) { - if let Some(first_height) = headers.first().map(|h| h.height) { - for (hash, _header) in self.headers.drain(first_height..) { - assert!(self.heights.remove(&hash).is_some()); - } - for (h, height) in headers.into_iter().zip(first_height..) { - assert_eq!(h.height, height); - assert_eq!(h.hash, h.header.block_hash()); - assert!(self.heights.insert(h.hash, h.height).is_none()); - self.headers.push((h.hash, h.header)); - } - info!( - "chain updated: tip={}, height={}", - self.headers.last().unwrap().0, - self.headers.len() - 1 - ); - } - } - - /// Best block hash - pub(crate) fn tip(&self) -> BlockHash { - self.headers.last().expect("empty chain").0 - } - - /// Number of blocks (excluding genesis block) - pub(crate) fn height(&self) -> usize { - self.headers.len() - 1 - } - - /// List of block hashes for efficient fork detection and block/header sync - /// see https://en.bitcoin.it/wiki/Protocol_documentation#getblocks - pub(crate) fn locator(&self) -> Vec { - let mut result = vec![]; - let mut index = self.headers.len() - 1; - let mut step = 1; - loop { - if result.len() >= 10 { - step *= 2; - } - result.push(self.headers[index].0); - if index == 0 { - break; - } - index = index.saturating_sub(step); - } - result - } -} - -#[cfg(test)] -mod tests { - use super::{Chain, NewHeader}; - use bitcoin::blockdata::block::Header as BlockHeader; - use bitcoin::consensus::deserialize; - use bitcoin::Network::Regtest; - use hex_lit::hex; - - #[test] - fn test_genesis() { - let regtest = Chain::new(Regtest); - assert_eq!(regtest.height(), 0); - assert_eq!( - regtest.tip(), - "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206" - .parse() - .unwrap() - ); - } - - #[test] - fn test_updates() { - let byte_headers = [ -hex!("0000002006226e46111a0b59caaf126043eb5bbf28c34f3a5e332a1fc7b2b73cf188910f1d14d3c7ff12d6adf494ebbcfba69baa915a066358b68a2b8c37126f74de396b1d61cc60ffff7f2000000000"), -hex!("00000020d700ae5d3c705702e0a5d9ababd22ded079f8a63b880b1866321d6bfcb028c3fc816efcf0e84ccafa1dda26be337f58d41b438170c357cda33a68af5550590bc1e61cc60ffff7f2004000000"), -hex!("00000020d13731bc59bc0989e06a5e7cab9843a4e17ad65c7ca47cd77f50dfd24f1f55793f7f342526aca9adb6ce8f33d8a07662c97d29d83b9e18117fb3eceecb2ab99b1e61cc60ffff7f2001000000"), -hex!("00000020a603def3e1255cadfb6df072946327c58b344f9bfb133e8e3e280d1c2d55b31c731a68f70219472864a7cb010cd53dc7e0f67e57f7d08b97e5e092b0c3942ad51f61cc60ffff7f2001000000"), -hex!("0000002041dd202b3b2edcdd3c8582117376347d48ff79ff97c95e5ac814820462012e785142dc360975b982ca43eecd14b4ba6f019041819d4fc5936255d7a2c45a96651f61cc60ffff7f2000000000"), -hex!("0000002072e297a2d6b633c44f3c9b1a340d06f3ce4e6bcd79ebd4c4ff1c249a77e1e37c59c7be1ca0964452e1735c0d2740f0d98a11445a6140c36b55770b5c0bcf801f1f61cc60ffff7f2000000000"), -hex!("000000200c9eb5889a8e924d1c4e8e79a716514579e41114ef37d72295df8869d6718e4ac5840f28de43ff25c7b9200aaf7873b20587c92827eaa61943484ca828bdd2e11f61cc60ffff7f2000000000"), -hex!("000000205873f322b333933e656b07881bb399dae61a6c0fa74188b5fb0e3dd71c9e2442f9e2f433f54466900407cf6a9f676913dd54aad977f7b05afcd6dcd81e98ee752061cc60ffff7f2004000000"), -hex!("00000020fd1120713506267f1dba2e1856ca1d4490077d261cde8d3e182677880df0d856bf94cfa5e189c85462813751ab4059643759ed319a81e0617113758f8adf67bc2061cc60ffff7f2000000000"), -hex!("000000200030d7f9c11ef35b89a0eefb9a5e449909339b5e7854d99804ea8d6a49bf900a0304d2e55fe0b6415949cff9bca0f88c0717884a5e5797509f89f856af93624a2061cc60ffff7f2002000000"), - ]; - let headers: Vec = byte_headers - .iter() - .map(|byte_header| deserialize(byte_header).unwrap()) - .collect(); - - for chunk_size in 1..headers.len() { - let mut regtest = Chain::new(Regtest); - let mut height = 0; - let mut tip = regtest.tip(); - for chunk in headers.chunks(chunk_size) { - let mut update = vec![]; - for header in chunk { - height += 1; - tip = header.block_hash(); - update.push(NewHeader::from((*header, height))) - } - regtest.update(update); - assert_eq!(regtest.tip(), tip); - assert_eq!(regtest.height(), height); - } - assert_eq!(regtest.tip(), headers.last().unwrap().block_hash()); - assert_eq!(regtest.height(), headers.len()); - } - - // test loading from a list of headers and tip - let mut regtest = Chain::new(Regtest); - regtest.load( - headers.iter().copied(), - headers.last().unwrap().block_hash(), - ); - assert_eq!(regtest.height(), headers.len()); - - // test getters - for (header, height) in headers.iter().zip(1usize..) { - assert_eq!(regtest.get_block_header(height), Some(header)); - assert_eq!(regtest.get_block_hash(height), Some(header.block_hash())); - assert_eq!(regtest.get_block_height(&header.block_hash()), Some(height)); - } - - // test chain shortening - for i in (0..=headers.len()).rev() { - let hash = regtest.get_block_hash(i).unwrap(); - assert_eq!(regtest.get_block_height(&hash), Some(i)); - assert_eq!(regtest.height(), i); - assert_eq!(regtest.tip(), hash); - regtest.drop_last_headers(1); - } - assert_eq!(regtest.height(), 0); - assert_eq!( - regtest.tip(), - "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206" - .parse() - .unwrap() - ); - - regtest.drop_last_headers(1); - assert_eq!(regtest.height(), 0); - assert_eq!( - regtest.tip(), - "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206" - .parse() - .unwrap() - ); - - // test reorg - let mut regtest = Chain::new(Regtest); - regtest.load( - headers.iter().copied(), - headers.last().unwrap().block_hash(), - ); - let height = regtest.height(); - - let new_header: BlockHeader = deserialize(&hex!("000000200030d7f9c11ef35b89a0eefb9a5e449909339b5e7854d99804ea8d6a49bf900a0304d2e55fe0b6415949cff9bca0f88c0717884a5e5797509f89f856af93624a7a6bcc60ffff7f2000000000")).unwrap(); - regtest.update(vec![NewHeader::from((new_header, height))]); - assert_eq!(regtest.height(), height); - assert_eq!( - regtest.tip(), - "0e16637fe0700a7c52e9a6eaa58bd6ac7202652103be8f778680c66f51ad2e9b" - .parse() - .unwrap() - ); - } -} diff --git a/src/config.rs b/src/config.rs index d37ff35..d147408 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,6 +1,5 @@ -use bitcoin::p2p::Magic; -use bitcoin::Network; -use bitcoincore_rpc::Auth; +use crate::bitcoin::Network; +use crate::types::Auth; use dirs_next::home_dir; use std::ffi::{OsStr, OsString}; @@ -17,10 +16,7 @@ pub const ELECTRS_VERSION: &str = env!("CARGO_PKG_VERSION"); const DEFAULT_SERVER_ADDRESS: [u8; 4] = [127, 0, 0, 1]; // by default, serve on IPv4 localhost mod internal { - #![allow(clippy::enum_variant_names)] - #![allow(clippy::unnecessary_lazy_evaluations)] - #![allow(clippy::useless_conversion)] - + #![allow(unused_attributes, unused_imports, clippy::enum_variant_names)] include!(concat!(env!("OUT_DIR"), "/configure_me_config.rs")); } @@ -111,7 +107,10 @@ impl FromStr for BitcoinNetwork { impl ::configure_me::parse_arg::ParseArgFromStr for BitcoinNetwork { fn describe_type(mut writer: W) -> fmt::Result { - write!(writer, "either 'bitcoin', 'testnet', 'regtest' or 'signet'") + write!( + writer, + "either 'bitcoin', 'testnet', 'testnet4', 'regtest' or 'signet'" + ) } } @@ -122,21 +121,20 @@ impl From for Network { } /// Parsed and post-processed configuration +#[allow(dead_code)] #[derive(Debug)] pub struct Config { // See below for the documentation of each field: pub network: Network, - pub db_path: PathBuf, + pub db_dir: PathBuf, pub db_log_dir: Option, - pub daemon_dir: PathBuf, + pub db_parallelism: u8, pub daemon_auth: SensitiveAuth, pub daemon_rpc_addr: SocketAddr, - pub daemon_p2p_addr: SocketAddr, pub electrum_rpc_addr: SocketAddr, pub monitoring_addr: SocketAddr, pub wait_duration: Duration, pub jsonrpc_timeout: Duration, - pub index_batch_size: usize, pub index_lookup_limit: Option, pub reindex_last_blocks: usize, pub auto_reindex: bool, @@ -145,7 +143,6 @@ pub struct Config { pub skip_block_download_wait: bool, pub disable_electrum_rpc: bool, pub server_banner: String, - pub signet_magic: Magic, } pub struct SensitiveAuth(pub Auth); @@ -192,79 +189,38 @@ fn default_config_files() -> Vec { impl Config { /// Parses args, env vars, config files and post-processes them pub fn from_args() -> Config { - use internal::ResultExt; + use internal::prelude::ResultExt; let (mut config, _args) = - internal::Config::including_optional_config_files(default_config_files()) + internal::prelude::Config::including_optional_config_files(default_config_files()) .unwrap_or_exit(); - fn unsupported_network(network: Network) -> ! { - eprintln!("Error: unsupported network: {}", network); - std::process::exit(1); - } - - let db_subdir = match config.network { - Network::Bitcoin => "bitcoin", - Network::Testnet => "testnet", - Network::Regtest => "regtest", - Network::Signet => "signet", - unsupported => unsupported_network(unsupported), - }; - - config.db_dir.push(db_subdir); - let default_daemon_rpc_port = match config.network { Network::Bitcoin => 8332, Network::Testnet => 18332, + Network::Testnet4 => 48332, Network::Regtest => 18443, Network::Signet => 38332, - unsupported => unsupported_network(unsupported), - }; - let default_daemon_p2p_port = match config.network { - Network::Bitcoin => 8333, - Network::Testnet => 18333, - Network::Regtest => 18444, - Network::Signet => 38333, - unsupported => unsupported_network(unsupported), }; let default_electrum_port = match config.network { Network::Bitcoin => 50001, Network::Testnet => 60001, + Network::Testnet4 => 40001, Network::Regtest => 60401, Network::Signet => 60601, - unsupported => unsupported_network(unsupported), }; let default_monitoring_port = match config.network { Network::Bitcoin => 4224, Network::Testnet => 14224, + Network::Testnet4 => 44224, Network::Regtest => 24224, Network::Signet => 34224, - unsupported => unsupported_network(unsupported), - }; - - let magic = match (config.network, config.signet_magic) { - (Network::Signet, Some(magic)) => magic.parse().unwrap_or_else(|error| { - eprintln!( - "Error: signet magic '{}' is not a valid hex string: {}", - magic, error - ); - std::process::exit(1); - }), - (network, None) => network.magic(), - (_, Some(_)) => { - eprintln!("Error: signet magic only available on signet"); - std::process::exit(1); - } }; let daemon_rpc_addr: SocketAddr = config.daemon_rpc_addr.map_or( (DEFAULT_SERVER_ADDRESS, default_daemon_rpc_port).into(), ResolvAddr::resolve_or_exit, ); - let daemon_p2p_addr: SocketAddr = config.daemon_p2p_addr.map_or( - (DEFAULT_SERVER_ADDRESS, default_daemon_p2p_port).into(), - ResolvAddr::resolve_or_exit, - ); let electrum_rpc_addr: SocketAddr = config.electrum_rpc_addr.map_or( (DEFAULT_SERVER_ADDRESS, default_electrum_port).into(), ResolvAddr::resolve_or_exit, @@ -284,9 +240,9 @@ impl Config { match config.network { Network::Bitcoin => (), Network::Testnet => config.daemon_dir.push("testnet3"), + Network::Testnet4 => config.daemon_dir.push("testnet4"), Network::Regtest => config.daemon_dir.push("regtest"), Network::Signet => config.daemon_dir.push("signet"), - unsupported => unsupported_network(unsupported), } let mut deprecated_options_used = false; @@ -348,17 +304,15 @@ impl Config { let config = Config { network: config.network, - db_path: config.db_dir, + db_dir: config.db_dir, db_log_dir: config.db_log_dir, - daemon_dir: config.daemon_dir, + db_parallelism: config.db_parallelism, daemon_auth, daemon_rpc_addr, - daemon_p2p_addr, electrum_rpc_addr, monitoring_addr, wait_duration: Duration::from_secs(config.wait_duration_secs), jsonrpc_timeout: Duration::from_secs(config.jsonrpc_timeout_secs), - index_batch_size: config.index_batch_size, index_lookup_limit, reindex_last_blocks: config.reindex_last_blocks, auto_reindex: config.auto_reindex, @@ -367,7 +321,6 @@ impl Config { skip_block_download_wait: config.skip_block_download_wait, disable_electrum_rpc: config.disable_electrum_rpc, server_banner: config.server_banner, - signet_magic: magic, }; eprintln!( "Starting electrs {} on {} {} with {:?}", @@ -391,9 +344,6 @@ mod tests { #[test] fn test_auth_debug() { - let auth = Auth::None; - assert_eq!(format!("{:?}", SensitiveAuth(auth)), "None"); - let auth = Auth::CookieFile(Path::new("/foo/bar/.cookie").to_path_buf()); assert_eq!( format!("{:?}", SensitiveAuth(auth)), diff --git a/src/daemon.rs b/src/daemon.rs index c0207b4..f31337e 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -1,10 +1,10 @@ use anyhow::{Context, Result}; -use bitcoin::{consensus::deserialize, hashes::hex::FromHex}; -use bitcoin::{Amount, BlockHash, Transaction, Txid}; -use bitcoincore_rpc::{json, jsonrpc, Auth, Client, RpcApi}; -use crossbeam_channel::Receiver; -use parking_lot::Mutex; +use crate::bitcoin::{ + self, consensus::deserialize, consensus::encode::serialize_hex, hashes::hex::FromHex, Amount, + BlockHash, Transaction, Txid, +}; +use crate::types::Auth; use serde::Serialize; use serde_json::{json, value::RawValue, Value}; @@ -12,52 +12,62 @@ use std::fs::File; use std::io::Read; use std::path::Path; -use crate::{ - chain::{Chain, NewHeader}, - config::Config, - metrics::Metrics, - p2p::Connection, - signals::ExitFlag, - types::SerBlock, -}; +use crate::{config::Config, signals::ExitFlag}; enum PollResult { Done(Result<()>), Retry, } -fn rpc_poll(client: &mut Client, skip_block_download_wait: bool) -> PollResult { - match client.get_blockchain_info() { - Ok(info) => { - if skip_block_download_wait { - // bitcoind RPC is available, don't wait for block download to finish - return PollResult::Done(Ok(())); - } - let left_blocks = info.headers - info.blocks; - if info.initial_block_download || left_blocks > 0 { - info!( - "waiting for {} blocks to download{}", - left_blocks, - if info.initial_block_download { - " (IBD)" - } else { - "" - } - ); - return PollResult::Retry; - } - PollResult::Done(Ok(())) - } - Err(err) => { - if let Some(e) = extract_bitcoind_error(&err) { - if e.code == -28 { - debug!("waiting for RPC warmup: {}", e.message); - return PollResult::Retry; - } - } - PollResult::Done(Err(err).context("daemon not available")) - } - } +#[derive(Deserialize)] +pub struct GetMempoolEntryResultFees { + /// Transaction fee in BTC + #[serde(with = "bitcoin::amount::serde::as_btc")] + pub base: Amount, +} + +#[derive(Deserialize)] +pub struct GetMempoolEntryResult { + /// Virtual transaction size as defined in BIP 141. This is different from actual serialized + /// size for witness transactions as witness data is discounted. + #[serde(alias = "size")] + pub vsize: u64, + /// Fee information + pub fees: GetMempoolEntryResultFees, + /// Unconfirmed transactions used as inputs for this transaction + pub depends: Vec, +} + +#[derive(Deserialize)] +pub struct GetBlockchainInfoResult { + pub blocks: u64, + pub headers: u64, + pub initialblockdownload: bool, + pub pruned: bool, +} + +#[derive(Deserialize)] +pub struct EstimateSmartFeeResult { + /// Estimate fee rate in BTC/kB. + #[serde( + default, + rename = "feerate", + skip_serializing_if = "Option::is_none", + with = "bitcoin::amount::serde::as_btc::opt" + )] + pub fee_rate: Option, +} + +#[derive(Deserialize)] +pub struct GetNetworkInfoResult { + #[serde(rename = "relayfee", with = "bitcoin::amount::serde::as_btc")] + pub relay_fee: Amount, +} + +#[derive(Deserialize)] +pub struct GetMempoolInfoResult { + /// True if the mempool is fully loaded + pub loaded: Option, } fn read_cookie(path: &Path) -> Result<(String, String)> { @@ -78,44 +88,38 @@ fn read_cookie(path: &Path) -> Result<(String, String)> { Ok((parts[0].to_owned(), parts[1].to_owned())) } -fn rpc_connect(config: &Config) -> Result { +fn jsonrpc_client(config: &Config) -> Result { let rpc_url = format!("http://{}", config.daemon_rpc_addr); - // Allow `wait_for_new_block` to take a bit longer before timing out. + // Allow RPC calls to take longer before timing out. // See https://github.com/romanz/electrs/issues/495 for more details. let builder = jsonrpc::simple_http::SimpleHttpTransport::builder() .url(&rpc_url)? .timeout(config.jsonrpc_timeout); let builder = match config.daemon_auth.get_auth() { - Auth::None => builder, Auth::UserPass(user, pass) => builder.auth(user, Some(pass)), Auth::CookieFile(path) => { let (user, pass) = read_cookie(&path)?; builder.auth(user, Some(pass)) } }; - Ok(Client::from_jsonrpc(jsonrpc::Client::with_transport( - builder.build(), - ))) + Ok(jsonrpc::Client::with_transport(builder.build())) } pub struct Daemon { - p2p: Mutex, - rpc: Client, + client: jsonrpc::Client, } impl Daemon { - pub(crate) fn connect( - config: &Config, - exit_flag: &ExitFlag, - metrics: &Metrics, - ) -> Result { - let mut rpc = rpc_connect(config)?; + pub(crate) fn connect(config: &Config, exit_flag: &ExitFlag) -> Result { + let daemon = Daemon { + client: jsonrpc_client(config)?, + }; loop { exit_flag .poll() .context("bitcoin RPC polling interrupted")?; - match rpc_poll(&mut rpc, config.skip_block_download_wait) { + match daemon.rpc_poll(config.skip_block_download_wait) { PollResult::Done(result) => { result.context("bitcoind RPC polling failed")?; break; // on success, finish polling @@ -126,34 +130,85 @@ impl Daemon { } } - let network_info = rpc.get_network_info()?; - if network_info.version < 21_00_00 { - bail!("electrs requires bitcoind 0.21+"); - } - if !network_info.network_active { - bail!("electrs requires active bitcoind p2p network"); - } - let info = rpc.get_blockchain_info()?; - if info.pruned { - bail!("electrs requires non-pruned bitcoind node"); - } + Ok(daemon) + } - let p2p = Mutex::new(Connection::connect( - config.network, - config.daemon_p2p_addr, - metrics, - config.signet_magic, - )?); - Ok(Self { p2p, rpc }) + fn call serde::de::Deserialize<'a>>( + &self, + cmd: &str, + args: &[serde_json::Value], + ) -> Result { + let raw = serde_json::value::to_raw_value(args)?; + let req = self.client.build_request(cmd, Some(&*raw)); + let resp = self.client.send_request(req)?; + resp.result() + } + + fn batch_request(&self, name: &str, items: &[T]) -> Result>> + where + T: Serialize, + { + debug!("calling {} on {} items", name, items.len()); + let args: Vec> = items + .iter() + .map(|item| jsonrpc::try_arg([item]).context("failed to serialize into JSON")) + .collect::>>()?; + let reqs: Vec = args + .iter() + .map(|arg| self.client.build_request(name, Some(arg))) + .collect(); + match self.client.send_batch(&reqs) { + Ok(values) => { + assert_eq!(items.len(), values.len()); + Ok(values) + } + Err(err) => bail!("batch {} request failed: {}", name, err), + } + } + + fn rpc_poll(&self, skip_block_download_wait: bool) -> PollResult { + match self.call::("getblockchaininfo", &[]) { + Ok(info) => { + if info.pruned { + return PollResult::Done(Err(anyhow!( + "electrs requires non-pruned bitcoind node" + ))); + } + if skip_block_download_wait { + // bitcoind RPC is available, don't wait for block download to finish + return PollResult::Done(Ok(())); + } + let left_blocks = info.headers - info.blocks; + if info.initialblockdownload || left_blocks > 0 { + info!( + "waiting for {} blocks to download{}", + left_blocks, + if info.initialblockdownload { + " (IBD)" + } else { + "" + } + ); + return PollResult::Retry; + } + PollResult::Done(Ok(())) + } + Err(err) => { + if let Some(e) = extract_bitcoind_error(&err) { + if e.code == -28 { + debug!("waiting for RPC warmup: {}", e.message); + return PollResult::Retry; + } + } + PollResult::Done(Err(err).context("daemon not available")) + } + } } pub(crate) fn estimate_fee(&self, nblocks: u16) -> Result> { - let res = self.rpc.estimate_smart_fee(nblocks, None); - if let Err(bitcoincore_rpc::Error::JsonRpc(jsonrpc::Error::Rpc(RpcError { - code: -32603, - .. - }))) = res - { + let res = + self.call::("estimatesmartfee", &[json!(nblocks), Value::Null]); + if let Err(jsonrpc::Error::Rpc(jsonrpc::error::RpcError { code: -32603, .. })) = res { return Ok(None); // don't fail when fee estimation is disabled (e.g. with `-blocksonly=1`) } Ok(res.context("failed to estimate fee")?.fee_rate) @@ -161,78 +216,64 @@ impl Daemon { pub(crate) fn get_relay_fee(&self) -> Result { Ok(self - .rpc - .get_network_info() + .call::("getnetworkinfo", &[]) .context("failed to get relay fee")? .relay_fee) } pub(crate) fn broadcast(&self, tx: &Transaction) -> Result { - self.rpc - .send_raw_transaction(tx) + self.call("sendrawtransaction", &[json!(serialize_hex(tx))]) .context("failed to broadcast transaction") } - pub(crate) fn get_transaction_info( - &self, - txid: &Txid, - blockhash: Option, - ) -> Result { - // No need to parse the resulting JSON, just return it as-is to the client. - self.rpc - .call( - "getrawtransaction", - &[json!(txid), json!(true), json!(blockhash)], - ) - .context("failed to get transaction info") - } - - pub(crate) fn get_transaction_hex( - &self, - txid: &Txid, - blockhash: Option, - ) -> Result { - use bitcoin::consensus::serde::{hex::Lower, Hex, With}; - - let tx = self.get_transaction(txid, blockhash)?; - #[derive(serde::Serialize)] - #[serde(transparent)] - struct TxAsHex(#[serde(with = "With::>")] Transaction); - serde_json::to_value(TxAsHex(tx)).map_err(Into::into) + pub(crate) fn submitpackage(&self, txs: &[Transaction]) -> Result { + let package: Vec = txs.iter().map(serialize_hex).collect(); + self.call("submitpackage", &[json!(package)]) + .context("failed to submitpackage package") } pub(crate) fn get_transaction( &self, txid: &Txid, blockhash: Option, - ) -> Result { - self.rpc - .get_raw_transaction(txid, blockhash.as_ref()) - .context("failed to get transaction") + verbose: bool, + ) -> Result { + self.call( + "getrawtransaction", + &[json!(txid), json!(verbose), json!(blockhash)], + ) + .context("failed to get transaction") } pub(crate) fn get_block_txids(&self, blockhash: BlockHash) -> Result> { - Ok(self - .rpc - .get_block_info(&blockhash) - .context("failed to get block txids")? - .tx) + #[derive(Clone, PartialEq, Debug, Deserialize, Serialize)] + struct GetBlockResult { + pub tx: Vec, + } + let res: GetBlockResult = self + .call("getblock", &[json!(blockhash), json!(1)]) + .context("failed to get block txids")?; + Ok(res.tx) + } + + pub(crate) fn get_mempool_info(&self) -> Result { + self.call("getmempoolinfo", &[]) + .context("failed to get mempool info") } pub(crate) fn get_mempool_txids(&self) -> Result> { - self.rpc - .get_raw_mempool() + self.call("getrawmempool", &[]) .context("failed to get mempool txids") } pub(crate) fn get_mempool_entries( &self, txids: &[Txid], - ) -> Result>> { - let results = batch_request(self.rpc.get_jsonrpc_client(), "getmempoolentry", txids)?; + ) -> Result>> { + let results = self.batch_request("getmempoolentry", txids)?; Ok(results .into_iter() - .map(|r| match r?.result::() { + .map(|r| match r?.result::() { Ok(entry) => Some(entry), Err(err) => { debug!("failed to get mempool entry: {}", err); // probably due to RBF @@ -246,7 +287,7 @@ impl Daemon { &self, txids: &[Txid], ) -> Result>> { - let results = batch_request(self.rpc.get_jsonrpc_client(), "getrawtransaction", txids)?; + let results = self.batch_request("getrawtransaction", txids)?; Ok(results .into_iter() .map(|r| -> Option { @@ -274,58 +315,11 @@ impl Daemon { }) .collect()) } - - pub(crate) fn get_new_headers(&self, chain: &Chain) -> Result> { - self.p2p.lock().get_new_headers(chain) - } - - pub(crate) fn for_blocks(&self, blockhashes: B, func: F) -> Result<()> - where - B: IntoIterator, - F: FnMut(BlockHash, SerBlock), - { - self.p2p.lock().for_blocks(blockhashes, func) - } - - pub(crate) fn new_block_notification(&self) -> Receiver<()> { - self.p2p.lock().new_block_notification() - } } -pub(crate) type RpcError = bitcoincore_rpc::jsonrpc::error::RpcError; - -pub(crate) fn extract_bitcoind_error(err: &bitcoincore_rpc::Error) -> Option<&RpcError> { - use bitcoincore_rpc::{ - jsonrpc::error::Error::Rpc as ServerError, Error::JsonRpc as JsonRpcError, - }; +pub(crate) fn extract_bitcoind_error(err: &jsonrpc::Error) -> Option<&jsonrpc::error::RpcError> { match err { - JsonRpcError(ServerError(e)) => Some(e), + jsonrpc::error::Error::Rpc(e) => Some(e), _ => None, } } - -fn batch_request( - client: &jsonrpc::Client, - name: &str, - items: &[T], -) -> Result>> -where - T: Serialize, -{ - debug!("calling {} on {} items", name, items.len()); - let args: Vec> = items - .iter() - .map(|item| jsonrpc::try_arg([item]).context("failed to serialize into JSON")) - .collect::>>()?; - let reqs: Vec = args - .iter() - .map(|arg| client.build_request(name, Some(arg))) - .collect(); - match client.send_batch(&reqs) { - Ok(values) => { - assert_eq!(items.len(), values.len()); - Ok(values) - } - Err(err) => bail!("batch {} request failed: {}", name, err), - } -} diff --git a/src/db.rs b/src/db.rs deleted file mode 100644 index 07e9f9d..0000000 --- a/src/db.rs +++ /dev/null @@ -1,538 +0,0 @@ -use anyhow::{Context, Result}; -use electrs_rocksdb as rocksdb; - -use std::path::Path; -use std::sync::atomic::{AtomicBool, Ordering}; - -use crate::types::{HashPrefix, SerializedHashPrefixRow, SerializedHeaderRow}; - -#[derive(Default)] -pub(crate) struct WriteBatch { - pub(crate) tip_row: [u8; 32], - pub(crate) header_rows: Vec, - pub(crate) funding_rows: Vec, - pub(crate) spending_rows: Vec, - pub(crate) txid_rows: Vec, -} - -impl WriteBatch { - pub(crate) fn sort(&mut self) { - self.header_rows.sort_unstable(); - self.funding_rows.sort_unstable(); - self.spending_rows.sort_unstable(); - self.txid_rows.sort_unstable(); - } -} - -/// RocksDB wrapper for index storage -pub struct DBStore { - db: rocksdb::DB, - bulk_import: AtomicBool, -} - -const CONFIG_CF: &str = "config"; -const HEADERS_CF: &str = "headers"; -const TXID_CF: &str = "txid"; -const FUNDING_CF: &str = "funding"; -const SPENDING_CF: &str = "spending"; - -const COLUMN_FAMILIES: &[&str] = &[CONFIG_CF, HEADERS_CF, TXID_CF, FUNDING_CF, SPENDING_CF]; - -const CONFIG_KEY: &str = "C"; -const TIP_KEY: &[u8] = b"T"; - -// Taken from https://github.com/facebook/rocksdb/blob/master/include/rocksdb/db.h#L654-L689 -const DB_PROPERTIES: &[&str] = &[ - "rocksdb.num-immutable-mem-table", - "rocksdb.mem-table-flush-pending", - "rocksdb.compaction-pending", - "rocksdb.background-errors", - "rocksdb.cur-size-active-mem-table", - "rocksdb.cur-size-all-mem-tables", - "rocksdb.size-all-mem-tables", - "rocksdb.num-entries-active-mem-table", - "rocksdb.num-entries-imm-mem-tables", - "rocksdb.num-deletes-active-mem-table", - "rocksdb.num-deletes-imm-mem-tables", - "rocksdb.estimate-num-keys", - "rocksdb.estimate-table-readers-mem", - "rocksdb.is-file-deletions-enabled", - "rocksdb.num-snapshots", - "rocksdb.oldest-snapshot-time", - "rocksdb.num-live-versions", - "rocksdb.current-super-version-number", - "rocksdb.estimate-live-data-size", - "rocksdb.min-log-number-to-keep", - "rocksdb.min-obsolete-sst-number-to-keep", - "rocksdb.total-sst-files-size", - "rocksdb.live-sst-files-size", - "rocksdb.base-level", - "rocksdb.estimate-pending-compaction-bytes", - "rocksdb.num-running-compactions", - "rocksdb.num-running-flushes", - "rocksdb.actual-delayed-write-rate", - "rocksdb.is-write-stopped", - "rocksdb.estimate-oldest-key-time", - "rocksdb.block-cache-capacity", - "rocksdb.block-cache-usage", - "rocksdb.block-cache-pinned-usage", -]; - -#[derive(Debug, Deserialize, Serialize)] -struct Config { - compacted: bool, - format: u64, -} - -const CURRENT_FORMAT: u64 = 0; - -impl Default for Config { - fn default() -> Self { - Config { - compacted: false, - format: CURRENT_FORMAT, - } - } -} - -fn default_opts() -> rocksdb::Options { - let mut block_opts = rocksdb::BlockBasedOptions::default(); - block_opts.set_checksum_type(rocksdb::ChecksumType::CRC32c); - - let mut opts = rocksdb::Options::default(); - opts.set_keep_log_file_num(10); - opts.set_max_open_files(16); - opts.set_compaction_style(rocksdb::DBCompactionStyle::Level); - opts.set_compression_type(rocksdb::DBCompressionType::Zstd); - opts.set_target_file_size_base(256 << 20); - opts.set_write_buffer_size(256 << 20); - opts.set_disable_auto_compactions(true); // for initial bulk load - opts.set_advise_random_on_open(false); // bulk load uses sequential I/O - opts.set_prefix_extractor(rocksdb::SliceTransform::create_fixed_prefix(8)); - opts.set_block_based_table_factory(&block_opts); - opts -} - -impl DBStore { - fn create_cf_descriptors() -> Vec { - COLUMN_FAMILIES - .iter() - .map(|&name| rocksdb::ColumnFamilyDescriptor::new(name, default_opts())) - .collect() - } - - fn open_internal(path: &Path, log_dir: Option<&Path>) -> Result { - let mut db_opts = default_opts(); - db_opts.create_if_missing(true); - db_opts.create_missing_column_families(true); - if let Some(d) = log_dir { - db_opts.set_db_log_dir(d); - } - - let db = rocksdb::DB::open_cf_descriptors(&db_opts, path, Self::create_cf_descriptors()) - .with_context(|| format!("failed to open DB: {}", path.display()))?; - let live_files = db.live_files()?; - info!( - "{:?}: {} SST files, {} GB, {} Grows", - path, - live_files.len(), - live_files.iter().map(|f| f.size).sum::() as f64 / 1e9, - live_files.iter().map(|f| f.num_entries).sum::() as f64 / 1e9 - ); - let store = DBStore { - db, - bulk_import: AtomicBool::new(true), - }; - Ok(store) - } - - fn is_legacy_format(&self) -> bool { - // In legacy DB format, all data was stored in a single (default) column family. - self.db - .iterator(rocksdb::IteratorMode::Start) - .next() - .is_some() - } - - /// Opens a new RocksDB at the specified location. - pub fn open(path: &Path, log_dir: Option<&Path>, auto_reindex: bool) -> Result { - let mut store = Self::open_internal(path, log_dir)?; - let config = store.get_config(); - debug!("DB {:?}", config); - let mut config = config.unwrap_or_default(); // use default config when DB is empty - - let reindex_cause = if store.is_legacy_format() { - Some("legacy format".to_owned()) - } else if config.format != CURRENT_FORMAT { - Some(format!( - "unsupported format {} != {}", - config.format, CURRENT_FORMAT - )) - } else { - None - }; - if let Some(cause) = reindex_cause { - if !auto_reindex { - bail!("re-index required due to {}", cause); - } - warn!( - "Database needs to be re-indexed due to {}, going to delete {}", - cause, - path.display() - ); - // close DB before deletion - drop(store); - rocksdb::DB::destroy(&default_opts(), path).with_context(|| { - format!( - "re-index required but the old database ({}) can not be deleted", - path.display() - ) - })?; - store = Self::open_internal(path, log_dir)?; - config = Config::default(); // re-init config after dropping DB - } - if config.compacted { - store.start_compactions(); - } - store.set_config(config); - Ok(store) - } - - fn config_cf(&self) -> &rocksdb::ColumnFamily { - self.db.cf_handle(CONFIG_CF).expect("missing CONFIG_CF") - } - - fn funding_cf(&self) -> &rocksdb::ColumnFamily { - self.db.cf_handle(FUNDING_CF).expect("missing FUNDING_CF") - } - - fn spending_cf(&self) -> &rocksdb::ColumnFamily { - self.db.cf_handle(SPENDING_CF).expect("missing SPENDING_CF") - } - - fn txid_cf(&self) -> &rocksdb::ColumnFamily { - self.db.cf_handle(TXID_CF).expect("missing TXID_CF") - } - - fn headers_cf(&self) -> &rocksdb::ColumnFamily { - self.db.cf_handle(HEADERS_CF).expect("missing HEADERS_CF") - } - - pub(crate) fn iter_funding( - &self, - prefix: HashPrefix, - ) -> impl Iterator + '_ { - self.iter_prefix_cf(self.funding_cf(), prefix) - } - - pub(crate) fn iter_spending( - &self, - prefix: HashPrefix, - ) -> impl Iterator + '_ { - self.iter_prefix_cf(self.spending_cf(), prefix) - } - - pub(crate) fn iter_txid( - &self, - prefix: HashPrefix, - ) -> impl Iterator + '_ { - self.iter_prefix_cf(self.txid_cf(), prefix) - } - - fn iter_cf<'a, const N: usize>( - &'a self, - cf: &rocksdb::ColumnFamily, - readopts: rocksdb::ReadOptions, - prefix: Option, - ) -> impl Iterator + '_ { - DBIterator::new(self.db.raw_iterator_cf_opt(cf, readopts), prefix) - } - - fn iter_prefix_cf( - &self, - cf: &rocksdb::ColumnFamily, - prefix: HashPrefix, - ) -> impl Iterator + '_ { - let mut opts = rocksdb::ReadOptions::default(); - opts.set_prefix_same_as_start(true); // requires .set_prefix_extractor() above. - self.iter_cf(cf, opts, Some(prefix)) - } - - pub(crate) fn iter_headers(&self) -> impl Iterator + '_ { - let mut opts = rocksdb::ReadOptions::default(); - opts.fill_cache(false); - self.iter_cf(self.headers_cf(), opts, None) - } - - pub(crate) fn get_tip(&self) -> Option> { - self.db - .get_cf(self.headers_cf(), TIP_KEY) - .expect("get_tip failed") - } - - pub(crate) fn write(&self, batch: &WriteBatch) { - let mut db_batch = rocksdb::WriteBatch::default(); - for key in &batch.funding_rows { - db_batch.put_cf(self.funding_cf(), key, b""); - } - for key in &batch.spending_rows { - db_batch.put_cf(self.spending_cf(), key, b""); - } - for key in &batch.txid_rows { - db_batch.put_cf(self.txid_cf(), key, b""); - } - for key in &batch.header_rows { - db_batch.put_cf(self.headers_cf(), key, b""); - } - db_batch.put_cf(self.headers_cf(), TIP_KEY, batch.tip_row); - - let mut opts = rocksdb::WriteOptions::new(); - let bulk_import = self.bulk_import.load(Ordering::Relaxed); - opts.set_sync(!bulk_import); - opts.disable_wal(bulk_import); - self.db.write_opt(db_batch, &opts).unwrap(); - } - - pub(crate) fn flush(&self) { - debug!("flushing DB column families"); - let mut config = self.get_config().unwrap_or_default(); - for name in COLUMN_FAMILIES { - let cf = self.db.cf_handle(name).expect("missing CF"); - self.db.flush_cf(cf).expect("CF flush failed"); - } - if !config.compacted { - for name in COLUMN_FAMILIES { - info!("starting {} compaction", name); - let cf = self.db.cf_handle(name).expect("missing CF"); - self.db.compact_range_cf(cf, None::<&[u8]>, None::<&[u8]>); - } - config.compacted = true; - self.set_config(config); - info!("finished full compaction"); - self.start_compactions(); - } - if log_enabled!(log::Level::Trace) { - let stats = self - .db - .property_value("rocksdb.dbstats") - .expect("failed to get property") - .expect("missing property"); - trace!("RocksDB stats: {}", stats); - } - } - - pub(crate) fn get_properties( - &self, - ) -> impl Iterator + '_ { - COLUMN_FAMILIES.iter().flat_map(move |cf_name| { - let cf = self.db.cf_handle(cf_name).expect("missing CF"); - DB_PROPERTIES.iter().filter_map(move |property_name| { - let value = self - .db - .property_int_value_cf(cf, *property_name) - .expect("failed to get property"); - Some((*cf_name, *property_name, value?)) - }) - }) - } - - fn start_compactions(&self) { - self.bulk_import.store(false, Ordering::Relaxed); - for name in COLUMN_FAMILIES { - let cf = self.db.cf_handle(name).expect("missing CF"); - self.db - .set_options_cf(cf, &[("disable_auto_compactions", "false")]) - .expect("failed to start auto-compactions"); - } - debug!("auto-compactions enabled"); - } - - fn set_config(&self, config: Config) { - let mut opts = rocksdb::WriteOptions::default(); - opts.set_sync(true); - opts.disable_wal(false); - let value = serde_json::to_vec(&config).expect("failed to serialize config"); - self.db - .put_cf_opt(self.config_cf(), CONFIG_KEY, value, &opts) - .expect("DB::put failed"); - } - - fn get_config(&self) -> Option { - self.db - .get_cf(self.config_cf(), CONFIG_KEY) - .expect("DB::get failed") - .map(|value| serde_json::from_slice(&value).expect("failed to deserialize Config")) - } -} - -struct DBIterator<'a, const N: usize> { - raw: rocksdb::DBRawIterator<'a>, - prefix: Option, - done: bool, -} - -impl<'a, const N: usize> DBIterator<'a, N> { - fn new(mut raw: rocksdb::DBRawIterator<'a>, prefix: Option) -> Self { - match prefix { - Some(key) => raw.seek(key), - None => raw.seek_to_first(), - }; - Self { - raw, - prefix, - done: false, - } - } -} - -impl<'a, const N: usize> Iterator for DBIterator<'a, N> { - type Item = [u8; N]; - - fn next(&mut self) -> Option { - while !self.done { - let key = match self.raw.key() { - Some(key) => key, - None => { - self.raw.status().expect("DB scan failed"); - break; // end of scan - } - }; - let prefix_match = match self.prefix { - Some(key_prefix) => key.starts_with(&key_prefix), - None => true, - }; - if !prefix_match { - break; // prefix mismatch - } - let result: Option<[u8; N]> = key.try_into().ok(); - self.raw.next(); - match result { - Some(value) => return Some(value), - None => continue, // skip keys with size != N - } - } - self.done = true; - None - } -} - -impl Drop for DBStore { - fn drop(&mut self) { - info!("closing DB at {}", self.db.path().display()); - } -} - -#[cfg(test)] -mod tests { - use super::{rocksdb, DBStore, WriteBatch, CURRENT_FORMAT}; - use std::ffi::{OsStr, OsString}; - use std::path::Path; - - #[test] - fn test_reindex_new_format() { - let dir = tempfile::tempdir().unwrap(); - { - let store = DBStore::open(dir.path(), None, false).unwrap(); - let mut config = store.get_config().unwrap(); - config.format += 1; - store.set_config(config); - }; - assert_eq!( - DBStore::open(dir.path(), None, false) - .err() - .unwrap() - .to_string(), - format!( - "re-index required due to unsupported format {} != {}", - CURRENT_FORMAT + 1, - CURRENT_FORMAT - ) - ); - { - let store = DBStore::open(dir.path(), None, true).unwrap(); - store.flush(); - let config = store.get_config().unwrap(); - assert_eq!(config.format, CURRENT_FORMAT); - assert!(!store.is_legacy_format()); - } - } - - #[test] - fn test_reindex_legacy_format() { - let dir = tempfile::tempdir().unwrap(); - { - let mut db_opts = rocksdb::Options::default(); - db_opts.create_if_missing(true); - let db = rocksdb::DB::open(&db_opts, dir.path()).unwrap(); - db.put(b"F", b"").unwrap(); // insert legacy DB compaction marker (in 'default' column family) - }; - assert_eq!( - DBStore::open(dir.path(), None, false) - .err() - .unwrap() - .to_string(), - format!("re-index required due to legacy format",) - ); - { - let store = DBStore::open(dir.path(), None, true).unwrap(); - store.flush(); - let config = store.get_config().unwrap(); - assert_eq!(config.format, CURRENT_FORMAT); - } - } - - #[test] - fn test_db_prefix_scan() { - let dir = tempfile::tempdir().unwrap(); - let store = DBStore::open(dir.path(), None, true).unwrap(); - - let items = [ - *b"ab ", - *b"abcdefgh ", - *b"abcdefghj ", - *b"abcdefghjk ", - *b"abcdefghxyz ", - *b"abcdefgi ", - *b"b ", - *b"c ", - ]; - - store.write(&WriteBatch { - txid_rows: items.to_vec(), - ..Default::default() - }); - - let rows = store.iter_txid(*b"abcdefgh"); - assert_eq!(rows.collect::>(), items[1..5]); - } - - #[test] - fn test_db_log_in_same_dir() { - let dir1 = tempfile::tempdir().unwrap(); - let _store = DBStore::open(dir1.path(), None, true).unwrap(); - - // LOG file is created in dir1 - let dir_files = list_log_files(dir1.path()); - assert_eq!(dir_files, vec![OsStr::new("LOG")]); - - let dir2 = tempfile::tempdir().unwrap(); - let dir3 = tempfile::tempdir().unwrap(); - let _store = DBStore::open(dir2.path(), Some(dir3.path()), true).unwrap(); - - // *_LOG file is not created in dir2, but in dir3 - let dir_files = list_log_files(dir2.path()); - assert_eq!(dir_files, Vec::::new()); - - let dir_files = list_log_files(dir3.path()); - assert_eq!(dir_files.len(), 1); - assert!(dir_files[0].to_str().unwrap().ends_with("_LOG")); - } - - fn list_log_files(path: &Path) -> Vec { - path.read_dir() - .unwrap() - .map(|e| e.unwrap().file_name()) - .filter(|e| e.to_str().unwrap().contains("LOG")) - .collect() - } -} diff --git a/src/electrum.rs b/src/electrum.rs index 46e0fa9..c03a4f5 100644 --- a/src/electrum.rs +++ b/src/electrum.rs @@ -1,10 +1,11 @@ -use anyhow::{bail, Context, Result}; -use bitcoin::{ +use crate::bitcoin::{ consensus::{deserialize, encode::serialize_hex}, hashes::hex::FromHex, - BlockHash, Txid, + hex::DisplayHex, + BlockHash, Transaction, Txid, }; -use crossbeam_channel::Receiver; +use anyhow::{bail, Context, Result}; +use bindex::ScriptHash; use rayon::prelude::*; use serde_derive::Deserialize; use serde_json::{self, json, Value}; @@ -12,18 +13,17 @@ use serde_json::{self, json, Value}; use std::collections::{hash_map::Entry, HashMap}; use std::fmt; use std::iter::FromIterator; +use std::net::SocketAddr; use std::str::FromStr; use crate::{ - cache::Cache, config::{Config, ELECTRS_VERSION}, - daemon::{self, extract_bitcoind_error, Daemon}, + daemon::{extract_bitcoind_error, Daemon}, merkle::Proof, metrics::{self, Histogram, Metrics}, signals::Signal, status::ScriptHashStatus, tracker::Tracker, - types::ScriptHash, }; const PROTOCOL_VERSION: &str = "1.4"; @@ -77,6 +77,29 @@ impl From<&TxGetArgs> for (Txid, bool) { } } +#[derive(Deserialize)] +#[serde(untagged)] +enum BroadcastArgs { + Package((Vec,)), + PackageVerbose((Vec, bool)), +} + +impl BroadcastArgs { + fn txs(&self) -> &[String] { + match self { + BroadcastArgs::Package((txs,)) => txs, + BroadcastArgs::PackageVerbose((txs, _)) => txs, + } + } + + fn verbose(&self) -> bool { + match self { + BroadcastArgs::Package(_) => false, + BroadcastArgs::PackageVerbose((_, verbose)) => *verbose, + } + } +} + enum StandardError { ParseError, InvalidRequest, @@ -89,7 +112,7 @@ enum RpcError { Standard(StandardError), // Electrum-specific errors BadRequest(anyhow::Error), - DaemonError(daemon::RpcError), + DaemonError(jsonrpc::error::RpcError), UnavailableIndex, } @@ -121,12 +144,11 @@ impl RpcError { /// Electrum RPC handler pub struct Rpc { tracker: Tracker, - cache: Cache, rpc_duration: Histogram, daemon: Daemon, signal: Signal, banner: String, - port: u16, + addr: SocketAddr, } impl Rpc { @@ -141,16 +163,14 @@ impl Rpc { let tracker = Tracker::new(config, metrics)?; let signal = Signal::new(); - let daemon = Daemon::connect(config, signal.exit_flag(), tracker.metrics())?; - let cache = Cache::new(tracker.metrics()); + let daemon = Daemon::connect(config, signal.exit_flag())?; Ok(Self { tracker, - cache, rpc_duration, daemon, signal, banner: config.server_banner.clone(), - port: config.electrum_rpc_addr.port(), + addr: config.electrum_rpc_addr, }) } @@ -158,24 +178,17 @@ impl Rpc { &self.signal } - pub fn new_block_notification(&self) -> Receiver<()> { - self.daemon.new_block_notification() - } - pub fn sync(&mut self) -> Result { self.tracker.sync(&self.daemon, self.signal.exit_flag()) } pub fn update_client(&self, client: &mut Client) -> Result> { - let chain = self.tracker.chain(); + let headers = self.tracker.headers(); let mut notifications = client .scripthashes .par_iter_mut() .filter_map(|(scripthash, status)| -> Option> { - match self - .tracker - .update_scripthash_status(status, &self.daemon, &self.cache) - { + match self.tracker.update_scripthash_status(status) { Ok(true) => Some(Ok(notification( "blockchain.scripthash.subscribe", &[json!(scripthash), json!(status.statushash())], @@ -188,11 +201,11 @@ impl Rpc { .context("failed to update status")?; if let Some(old_tip) = client.tip { - let new_tip = self.tracker.chain().tip(); - if old_tip != new_tip { - client.tip = Some(new_tip); - let height = chain.height(); - let header = chain.get_block_header(height).unwrap(); + let new_tip = headers.tip().unwrap(); + if old_tip != new_tip.hash() { + client.tip = Some(new_tip.hash()); + let height = headers.tip_height().unwrap(); + let header = new_tip.header(); notifications.push(notification( "blockchain.headers.subscribe", &[json!({"hex": serialize_hex(&header), "height": height})], @@ -203,34 +216,36 @@ impl Rpc { } fn headers_subscribe(&self, client: &mut Client) -> Result { - let chain = self.tracker.chain(); - client.tip = Some(chain.tip()); - let height = chain.height(); - let header = chain.get_block_header(height).unwrap(); - Ok(json!({"hex": serialize_hex(header), "height": height})) + let headers = self.tracker.headers(); + let height = headers.tip_height().unwrap(); + let header = headers.tip().unwrap(); + client.tip = Some(header.hash()); + Ok(json!({"hex": serialize_hex(header.header()), "height": height})) } fn block_header(&self, (height,): (usize,)) -> Result { - let chain = self.tracker.chain(); - let header = match chain.get_block_header(height) { + let header = match self.tracker.headers().iter_headers().nth(height) { None => bail!("no header at {}", height), Some(header) => header, }; - Ok(json!(serialize_hex(header))) + Ok(json!(serialize_hex(header.header()))) } fn block_headers(&self, (start_height, count): (usize, usize)) -> Result { - let chain = self.tracker.chain(); + let headers = self.tracker.headers(); let max_count = 2016usize; // return only the available block headers let end_height = std::cmp::min( - chain.height() + 1, + headers.tip_height().map_or(0, |h| h + 1), start_height + std::cmp::min(count, max_count), ); let heights = start_height..end_height; let count = heights.len(); - let hex_headers = - heights.filter_map(|height| chain.get_block_header(height).map(serialize_hex)); + let hex_headers = headers + .iter_headers() + .skip(start_height) + .take(count) + .map(|h| serialize_hex(h.header())); Ok(json!({"count": count, "hex": String::from_iter(hex_headers), "max": max_count})) } @@ -352,48 +367,70 @@ impl Rpc { fn new_status(&self, scripthash: ScriptHash) -> Result { let mut status = ScriptHashStatus::new(scripthash); - self.tracker - .update_scripthash_status(&mut status, &self.daemon, &self.cache)?; + self.tracker.update_scripthash_status(&mut status)?; Ok(status) } fn transaction_broadcast(&self, (tx_hex,): &(String,)) -> Result { - let tx_bytes = Vec::from_hex(tx_hex).context("non-hex transaction")?; - let tx = deserialize(&tx_bytes).context("invalid transaction")?; - let txid = self.daemon.broadcast(&tx)?; + let txid = self.daemon.broadcast(&tx_from_hex(tx_hex)?)?; Ok(json!(txid)) } + fn transaction_broadcast_package(&self, args: &BroadcastArgs) -> Result { + let txs: Vec = args + .txs() + .iter() + .map(|s| tx_from_hex(s)) + .collect::>()?; + let response = self.daemon.submitpackage(&txs)?; + if args.verbose() { + return Ok(response); + } + let build_result = || -> Option { + let success = response.get("package_msg")? == &json!("success"); + + let mut errors = vec![]; + for tx in response.get("tx-results")?.as_object()?.values() { + let tx_obj = tx.as_object()?; + if let Some(error) = tx_obj.get("error") { + let txid = tx_obj.get("txid"); + errors.push(json!({"error": error, "txid": txid})); + } + } + Some(if errors.is_empty() { + json!({"success": success}) + } else { + json!({"success": success, "errors": errors}) + }) + }; + build_result().ok_or(anyhow!("Unexpected `submitpackage` response")) + } + fn transaction_get(&self, args: &TxGetArgs) -> Result { let (txid, verbose) = args.into(); if verbose { let blockhash = self .tracker - .lookup_transaction(&self.daemon, txid)? + .lookup_transaction(txid)? .map(|(blockhash, _tx)| blockhash); - return self.daemon.get_transaction_info(&txid, blockhash); + return self.daemon.get_transaction(&txid, blockhash, verbose); } - if let Some(tx) = self.cache.get_tx(&txid, serialize_hex) { - return Ok(json!(tx)); - } - debug!("tx cache miss: txid={}", txid); - // use internal index to load confirmed transaction without an RPC - if let Some(tx) = self + // use internal index to load confirmed transaction + if let Some(tx_hex) = self .tracker - .lookup_transaction(&self.daemon, txid)? - .map(|(_blockhash, tx)| tx) + .lookup_transaction(txid)? + .map(|(_blockhash, tx)| tx.to_lower_hex_string()) { - return Ok(json!(serialize_hex(&tx))); + return Ok(json!(tx_hex)); } // load unconfirmed transaction via RPC - Ok(json!(self.daemon.get_transaction_hex(&txid, None)?)) + Ok(json!(self.daemon.get_transaction(&txid, None, verbose)?)) } fn transaction_get_merkle(&self, (txid, height): &(Txid, usize)) -> Result { - let chain = self.tracker.chain(); - let blockhash = match chain.get_block_hash(*height) { + let blockhash = match self.tracker.headers().iter_headers().nth(*height) { None => bail!("missing block at {}", height), - Some(blockhash) => blockhash, + Some(header) => header.hash(), }; let txids = self.daemon.get_block_txids(blockhash)?; match txids.iter().position(|current_txid| *current_txid == *txid) { @@ -401,9 +438,9 @@ impl Rpc { Some(position) => { let proof = Proof::create(&txids, position); Ok(json!({ - "block_height": height, - "pos": proof.position(), - "merkle": proof.to_hex(), + "block_height": height, + "pos": proof.position(), + "merkle": proof.to_hex(), })) } } @@ -413,10 +450,9 @@ impl Rpc { &self, (height, tx_pos, merkle): (usize, usize, bool), ) -> Result { - let chain = self.tracker.chain(); - let blockhash = match chain.get_block_hash(height) { + let blockhash = match self.tracker.headers().iter_headers().nth(height) { None => bail!("missing block at {}", height), - Some(blockhash) => blockhash, + Some(header) => header.hash(), }; let txids = self.daemon.get_block_txids(blockhash)?; if tx_pos >= txids.len() { @@ -425,9 +461,9 @@ impl Rpc { let txid: Txid = txids[tx_pos]; if merkle { let proof = Proof::create(&txids, tx_pos); - Ok(json!({"tx_id": txid, "merkle": proof.to_hex()})) + Ok(json!({"tx_hash": txid, "merkle": proof.to_hex()})) } else { - Ok(json!({ "tx_id": txid })) + Ok(json!({ "tx_hash": txid })) } } @@ -450,8 +486,12 @@ impl Rpc { fn features(&self) -> Result { Ok(json!({ - "genesis_hash": self.tracker.chain().get_block_hash(0), - "hosts": { "tcp_port": self.port }, + "genesis_hash": self.tracker.headers().genesis().unwrap().hash(), + "hosts": { + self.addr.ip().to_string(): { + "tcp_port": self.addr.port() + } + }, "protocol_max": PROTOCOL_VERSION, "protocol_min": PROTOCOL_VERSION, "pruning": null, @@ -557,6 +597,9 @@ impl Rpc { Params::ScriptHashSubscribe(args) => self.scripthash_subscribe(client, args), Params::ScriptHashUnsubscribe(args) => self.scripthash_unsubscribe(client, args), Params::TransactionBroadcast(args) => self.transaction_broadcast(args), + Params::TransactionBroadcastPackage(args) => { + self.transaction_broadcast_package(args) + } Params::TransactionGet(args) => self.transaction_get(args), Params::TransactionGetMerkle(args) => self.transaction_get_merkle(args), Params::TransactionFromPosition(args) => self.transaction_from_pos(*args), @@ -573,6 +616,7 @@ enum Params { BlockHeader((usize,)), BlockHeaders((usize, usize)), TransactionBroadcast((String,)), + TransactionBroadcastPackage(BroadcastArgs), Donation, EstimateFee((u16,)), Features, @@ -606,6 +650,9 @@ impl Params { "blockchain.scripthash.subscribe" => Params::ScriptHashSubscribe(convert(params)?), "blockchain.scripthash.unsubscribe" => Params::ScriptHashUnsubscribe(convert(params)?), "blockchain.transaction.broadcast" => Params::TransactionBroadcast(convert(params)?), + "blockchain.transaction.broadcast_package" => { + Params::TransactionBroadcastPackage(convert(params)?) + } "blockchain.transaction.get" => Params::TransactionGet(convert(params)?), "blockchain.transaction.get_merkle" => Params::TransactionGetMerkle(convert(params)?), "blockchain.transaction.id_from_pos" => { @@ -650,7 +697,7 @@ impl Call { Err(err) => { warn!("RPC {} failed: {:#}", self.method, err); match err - .downcast_ref::() + .downcast_ref::() .and_then(extract_bitcoind_error) { Some(e) => error_msg(&self.id, RpcError::DaemonError(e.clone())), @@ -758,9 +805,15 @@ fn check_between(version_str: &str, min_str: &str, max_str: &str) -> Result<()> Ok(()) } +fn tx_from_hex(tx_hex: &str) -> Result { + let tx_bytes = Vec::from_hex(tx_hex).context("non-hex transaction")?; + let tx = deserialize(&tx_bytes).context("invalid transaction")?; + Ok(tx) +} + #[cfg(test)] mod tests { - use super::{check_between, parse_version, Version}; + use super::*; #[test] fn test_version() { @@ -783,4 +836,19 @@ mod tests { assert!(check_between("1.4", "1.4.1", "1.5").is_err()); assert!(check_between("1.4", "1", "1").is_err()); } + + #[test] + fn test_requests() { + assert!(matches!( + parse_requests("foo"), + Err(StandardError::ParseError) + )); + assert!(matches!( + parse_requests(r"{}"), + Err(StandardError::InvalidRequest) + )); + assert!(parse_requests(r#"{"id":1,"method":"name","params":[]}"#).is_ok()); + assert!(parse_requests(r#"{"id":1,"method":"name","params":[],"unrelated":42}"#).is_ok()); + assert!(parse_requests(r#" { "id" : 1 , "method" : "name" , "params" : [ ] } "#).is_ok()); + } } diff --git a/src/index.rs b/src/index.rs deleted file mode 100644 index 5a22dc6..0000000 --- a/src/index.rs +++ /dev/null @@ -1,295 +0,0 @@ -use anyhow::{Context, Result}; -use bitcoin::consensus::{deserialize, Decodable, Encodable}; -use bitcoin::hashes::Hash; -use bitcoin::{BlockHash, OutPoint, Txid}; -use bitcoin_slices::{bsl, Visit, Visitor}; -use std::ops::ControlFlow; - -use crate::{ - chain::{Chain, NewHeader}, - daemon::Daemon, - db::{DBStore, WriteBatch}, - metrics::{self, Gauge, Histogram, Metrics}, - signals::ExitFlag, - types::{ - bsl_txid, HashPrefixRow, HeaderRow, ScriptHash, ScriptHashRow, SerBlock, SpendingPrefixRow, - TxidRow, - }, -}; - -#[derive(Clone)] -struct Stats { - update_duration: Histogram, - update_size: Histogram, - height: Gauge, - db_properties: Gauge, -} - -impl Stats { - fn new(metrics: &Metrics) -> Self { - Self { - update_duration: metrics.histogram_vec( - "index_update_duration", - "Index update duration (in seconds)", - "step", - metrics::default_duration_buckets(), - ), - update_size: metrics.histogram_vec( - "index_update_size", - "Index update size (in bytes)", - "step", - metrics::default_size_buckets(), - ), - height: metrics.gauge("index_height", "Indexed block height", "type"), - db_properties: metrics.gauge("index_db_properties", "Index DB properties", "name"), - } - } - - fn observe_duration(&self, label: &str, f: impl FnOnce() -> T) -> T { - self.update_duration.observe_duration(label, f) - } - - fn observe_size(&self, label: &str, rows: &[[u8; N]]) { - self.update_size.observe(label, (rows.len() * N) as f64); - } - - fn observe_batch(&self, batch: &WriteBatch) { - self.observe_size("write_funding_rows", &batch.funding_rows); - self.observe_size("write_spending_rows", &batch.spending_rows); - self.observe_size("write_txid_rows", &batch.txid_rows); - self.observe_size("write_header_rows", &batch.header_rows); - debug!( - "writing {} funding and {} spending rows from {} transactions, {} blocks", - batch.funding_rows.len(), - batch.spending_rows.len(), - batch.txid_rows.len(), - batch.header_rows.len() - ); - } - - fn observe_chain(&self, chain: &Chain) { - self.height.set("tip", chain.height() as f64); - } - - fn observe_db(&self, store: &DBStore) { - for (cf, name, value) in store.get_properties() { - self.db_properties - .set(&format!("{}:{}", name, cf), value as f64); - } - } -} - -/// Confirmed transactions' address index -pub struct Index { - store: DBStore, - batch_size: usize, - lookup_limit: Option, - chain: Chain, - stats: Stats, - is_ready: bool, - flush_needed: bool, -} - -impl Index { - pub(crate) fn load( - store: DBStore, - mut chain: Chain, - metrics: &Metrics, - batch_size: usize, - lookup_limit: Option, - reindex_last_blocks: usize, - ) -> Result { - if let Some(row) = store.get_tip() { - let tip = deserialize(&row).expect("invalid tip"); - let headers = store - .iter_headers() - .map(|row| HeaderRow::from_db_row(row).header); - chain.load(headers, tip); - chain.drop_last_headers(reindex_last_blocks); - }; - let stats = Stats::new(metrics); - stats.observe_chain(&chain); - stats.observe_db(&store); - Ok(Index { - store, - batch_size, - lookup_limit, - chain, - stats, - is_ready: false, - flush_needed: false, - }) - } - - pub(crate) fn chain(&self) -> &Chain { - &self.chain - } - - pub(crate) fn limit_result(&self, entries: impl Iterator) -> Result> { - let mut entries = entries.fuse(); - let result: Vec = match self.lookup_limit { - Some(lookup_limit) => entries.by_ref().take(lookup_limit).collect(), - None => entries.by_ref().collect(), - }; - if entries.next().is_some() { - bail!(">{} index entries, query may take too long", result.len()) - } - Ok(result) - } - - pub(crate) fn filter_by_txid(&self, txid: Txid) -> impl Iterator + '_ { - self.store - .iter_txid(TxidRow::scan_prefix(txid)) - .map(|row| HashPrefixRow::from_db_row(row).height()) - .filter_map(move |height| self.chain.get_block_hash(height)) - } - - pub(crate) fn filter_by_funding( - &self, - scripthash: ScriptHash, - ) -> impl Iterator + '_ { - self.store - .iter_funding(ScriptHashRow::scan_prefix(scripthash)) - .map(|row| HashPrefixRow::from_db_row(row).height()) - .filter_map(move |height| self.chain.get_block_hash(height)) - } - - pub(crate) fn filter_by_spending( - &self, - outpoint: OutPoint, - ) -> impl Iterator + '_ { - self.store - .iter_spending(SpendingPrefixRow::scan_prefix(outpoint)) - .map(|row| HashPrefixRow::from_db_row(row).height()) - .filter_map(move |height| self.chain.get_block_hash(height)) - } - - // Return `Ok(true)` when the chain is fully synced and the index is compacted. - pub(crate) fn sync(&mut self, daemon: &Daemon, exit_flag: &ExitFlag) -> Result { - let new_headers = self - .stats - .observe_duration("headers", || daemon.get_new_headers(&self.chain))?; - match (new_headers.first(), new_headers.last()) { - (Some(first), Some(last)) => { - let count = new_headers.len(); - info!( - "indexing {} blocks: [{}..{}]", - count, - first.height(), - last.height() - ); - } - _ => { - if self.flush_needed { - self.store.flush(); // full compaction is performed on the first flush call - self.flush_needed = false; - } - self.is_ready = true; - return Ok(true); // no more blocks to index (done for now) - } - } - for chunk in new_headers.chunks(self.batch_size) { - exit_flag.poll().with_context(|| { - format!( - "indexing interrupted at height: {}", - chunk.first().unwrap().height() - ) - })?; - self.sync_blocks(daemon, chunk)?; - } - self.chain.update(new_headers); - self.stats.observe_chain(&self.chain); - self.flush_needed = true; - Ok(false) // sync is not done - } - - fn sync_blocks(&mut self, daemon: &Daemon, chunk: &[NewHeader]) -> Result<()> { - let blockhashes: Vec = chunk.iter().map(|h| h.hash()).collect(); - let mut heights = chunk.iter().map(|h| h.height()); - - let mut batch = WriteBatch::default(); - - daemon.for_blocks(blockhashes, |blockhash, block| { - let height = heights.next().expect("unexpected block"); - self.stats.observe_duration("block", || { - index_single_block(blockhash, block, height, &mut batch); - }); - self.stats.height.set("tip", height as f64); - })?; - let heights: Vec<_> = heights.collect(); - assert!( - heights.is_empty(), - "some blocks were not indexed: {:?}", - heights - ); - batch.sort(); - self.stats.observe_batch(&batch); - self.stats - .observe_duration("write", || self.store.write(&batch)); - self.stats.observe_db(&self.store); - Ok(()) - } - - pub(crate) fn is_ready(&self) -> bool { - self.is_ready - } -} - -fn index_single_block( - block_hash: BlockHash, - block: SerBlock, - height: usize, - batch: &mut WriteBatch, -) { - struct IndexBlockVisitor<'a> { - batch: &'a mut WriteBatch, - height: usize, - } - - impl<'a> Visitor for IndexBlockVisitor<'a> { - fn visit_transaction(&mut self, tx: &bsl::Transaction) -> ControlFlow<()> { - let txid = bsl_txid(tx); - self.batch - .txid_rows - .push(TxidRow::row(txid, self.height).to_db_row()); - ControlFlow::Continue(()) - } - - fn visit_tx_out(&mut self, _vout: usize, tx_out: &bsl::TxOut) -> ControlFlow<()> { - let script = bitcoin::Script::from_bytes(tx_out.script_pubkey()); - // skip indexing unspendable outputs - if !script.is_op_return() { - let row = ScriptHashRow::row(ScriptHash::new(script), self.height); - self.batch.funding_rows.push(row.to_db_row()); - } - ControlFlow::Continue(()) - } - - fn visit_tx_in(&mut self, _vin: usize, tx_in: &bsl::TxIn) -> ControlFlow<()> { - let prevout: OutPoint = tx_in.prevout().into(); - // skip indexing coinbase transactions' input - if !prevout.is_null() { - let row = SpendingPrefixRow::row(prevout, self.height); - self.batch.spending_rows.push(row.to_db_row()); - } - ControlFlow::Continue(()) - } - - fn visit_block_header(&mut self, header: &bsl::BlockHeader) -> ControlFlow<()> { - let header = bitcoin::block::Header::consensus_decode(&mut header.as_ref()) - .expect("block header was already validated"); - self.batch - .header_rows - .push(HeaderRow::new(header).to_db_row()); - ControlFlow::Continue(()) - } - } - - let mut index_block = IndexBlockVisitor { batch, height }; - bsl::Block::visit(&block, &mut index_block).expect("core returned invalid block"); - - let len = block_hash - .consensus_encode(&mut (&mut batch.tip_row as &mut [u8])) - .expect("in-memory writers don't error"); - debug_assert_eq!(len, BlockHash::LEN); -} diff --git a/src/lib.rs b/src/lib.rs index d1df9c0..9b8fad8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,19 +7,12 @@ extern crate log; #[macro_use] extern crate serde_derive; -extern crate configure_me; - -mod cache; -mod chain; mod config; mod daemon; -mod db; mod electrum; -mod index; mod mempool; mod merkle; mod metrics; -mod p2p; mod server; mod signals; mod status; @@ -28,3 +21,6 @@ mod tracker; mod types; pub use server::run; + +use bindex::bitcoin; +use bindex::bitcoin_slices; diff --git a/src/mempool.rs b/src/mempool.rs index 2612f23..f797a14 100644 --- a/src/mempool.rs +++ b/src/mempool.rs @@ -5,15 +5,15 @@ use std::convert::TryFrom; use std::iter::FromIterator; use std::ops::Bound; -use bitcoin::hashes::Hash; -use bitcoin::{Amount, OutPoint, Transaction, Txid}; +use crate::bitcoin::hashes::Hash; +use crate::bitcoin::{Amount, OutPoint, Transaction, Txid}; +use bindex::ScriptHash; use serde::ser::{Serialize, SerializeSeq, Serializer}; use crate::{ daemon::Daemon, metrics::{Gauge, Metrics}, signals::ExitFlag, - types::ScriptHash, }; pub(crate) struct Entry { @@ -80,19 +80,19 @@ impl MempoolSyncUpdate { ); let chunk_entries: Vec = txids_chunk .iter() - .zip(entries.into_iter().zip(txs.into_iter())) + .zip(entries.into_iter().zip(txs)) .filter_map(|(txid, (entry, tx))| { let entry = match entry { Some(entry) => entry, None => { - warn!("missing mempool entry: {}", txid); + debug!("missing mempool entry: {}", txid); return None; } }; let tx = match tx { Some(tx) => tx, None => { - warn!("missing mempool tx: {}", txid); + debug!("missing mempool tx: {}", txid); return None; } }; @@ -211,6 +211,18 @@ impl Mempool { } pub fn sync(&mut self, daemon: &Daemon, exit_flag: &ExitFlag) { + let loaded = match daemon.get_mempool_info() { + Ok(info) => info.loaded.unwrap_or(true), + Err(e) => { + warn!("mempool sync failed: {}", e); + return; + } + }; + if !loaded { + warn!("mempool not loaded"); + return; + } + let old_txids = HashSet::::from_iter(self.entries.keys().copied()); let poll_result = MempoolSyncUpdate::poll(daemon, old_txids, exit_flag); @@ -342,7 +354,7 @@ impl Serialize for FeeHistogram { S: Serializer, { let mut seq = serializer.serialize_seq(Some(self.vsize.len()))?; - // https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-methods.html#mempool-get-fee-histogram + // https://electrum-protocol.readthedocs.io/en/latest/protocol-methods.html#mempool-get-fee-histogram let fee_rates = (0..FeeHistogram::BINS).map(|i| u64::MAX.checked_shr(i as u32).unwrap_or(0)); fee_rates @@ -356,7 +368,7 @@ impl Serialize for FeeHistogram { #[cfg(test)] mod tests { use super::FeeHistogram; - use bitcoin::Amount; + use crate::bitcoin::Amount; use serde_json::json; #[test] diff --git a/src/merkle.rs b/src/merkle.rs index 8294280..e34e8eb 100644 --- a/src/merkle.rs +++ b/src/merkle.rs @@ -1,4 +1,4 @@ -use bitcoin::{hash_types::TxMerkleNode, hashes::Hash, Txid}; +use crate::bitcoin::{hash_types::TxMerkleNode, hashes::Hash, Txid}; pub(crate) struct Proof { proof: Vec, @@ -54,7 +54,7 @@ impl Proof { #[cfg(test)] mod tests { - use bitcoin::{consensus::encode::deserialize, Block, Txid}; + use crate::bitcoin::{consensus::encode::deserialize, Block, Txid}; use std::path::Path; use super::Proof; diff --git a/src/metrics.rs b/src/metrics.rs index 99ddb26..9d6ef8b 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -5,8 +5,8 @@ mod metrics_impl { #[cfg(feature = "metrics_process")] use prometheus::process_collector::ProcessCollector; - use prometheus::{self, Encoder, HistogramOpts, HistogramVec, Registry}; - use tiny_http::{Response, Server}; + use prometheus::{self, Encoder, HistogramOpts, HistogramVec, Registry, TEXT_FORMAT}; + use tiny_http::{Header as HttpHeader, Response, Server}; use std::net::SocketAddr; @@ -33,13 +33,15 @@ mod metrics_impl { }; spawn("metrics", move || { + let content_type = HttpHeader::from_bytes(&b"Content-Type"[..], TEXT_FORMAT) + .expect("failed to create HTTP header for Prometheus text format"); for request in server.incoming_requests() { let mut buffer = vec![]; prometheus::TextEncoder::new() .encode(®.gather(), &mut buffer) .context("failed to encode metrics")?; request - .respond(Response::from_data(buffer)) + .respond(Response::from_data(buffer).with_header(content_type.clone())) .context("failed to send HTTP response")?; } Ok(()) diff --git a/src/p2p.rs b/src/p2p.rs deleted file mode 100644 index 3922f27..0000000 --- a/src/p2p.rs +++ /dev/null @@ -1,413 +0,0 @@ -use anyhow::{Context, Result}; -use bitcoin::blockdata::block::Header as BlockHeader; -use bitcoin::consensus::Encodable; -use bitcoin::{ - consensus::{ - encode::{self, ReadExt, VarInt}, - Decodable, - }, - hashes::Hash, - io, - p2p::{ - self, address, - message::{self, CommandString, NetworkMessage}, - message_blockdata::{GetHeadersMessage, Inventory}, - message_network, Magic, - }, - secp256k1::{self, rand::Rng}, - Block, BlockHash, Network, -}; -use bitcoin_slices::{bsl, Parse}; -use crossbeam_channel::{bounded, select, Receiver, Sender}; - -use std::io::Write; -use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpStream}; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; - -use crate::types::SerBlock; -use crate::{ - chain::{Chain, NewHeader}, - config::ELECTRS_VERSION, - metrics::{default_duration_buckets, default_size_buckets, Histogram, Metrics}, -}; - -enum Request { - GetNewHeaders(GetHeadersMessage), - GetBlocks(Vec), -} - -impl Request { - fn get_new_headers(chain: &Chain) -> Request { - Request::GetNewHeaders(GetHeadersMessage::new( - chain.locator(), - BlockHash::all_zeros(), - )) - } - - fn get_blocks(blockhashes: &[BlockHash]) -> Request { - Request::GetBlocks( - blockhashes - .iter() - .map(|blockhash| Inventory::WitnessBlock(*blockhash)) - .collect(), - ) - } -} - -pub(crate) struct Connection { - req_send: Sender, - blocks_recv: Receiver, - headers_recv: Receiver>, - new_block_recv: Receiver<()>, - - blocks_duration: Histogram, -} - -impl Connection { - /// Get new block headers (supporting reorgs). - /// https://en.bitcoin.it/wiki/Protocol_documentation#getheaders - /// Defined as `&mut self` to prevent concurrent invocations (https://github.com/romanz/electrs/pull/526#issuecomment-934685515). - pub(crate) fn get_new_headers(&mut self, chain: &Chain) -> Result> { - self.req_send.send(Request::get_new_headers(chain))?; - let headers = self - .headers_recv - .recv() - .context("failed to get new headers")?; - - debug!("got {} new headers", headers.len()); - let prev_blockhash = match headers.first() { - None => return Ok(vec![]), - Some(first) => first.prev_blockhash, - }; - let new_heights = match chain.get_block_height(&prev_blockhash) { - Some(last_height) => (last_height + 1).., - None => bail!("missing prev_blockhash: {}", prev_blockhash), - }; - Ok(headers - .into_iter() - .zip(new_heights) - .map(NewHeader::from) - .collect()) - } - - /// Request and process the specified blocks (in the specified order). - /// See https://en.bitcoin.it/wiki/Protocol_documentation#getblocks for details. - /// Defined as `&mut self` to prevent concurrent invocations (https://github.com/romanz/electrs/pull/526#issuecomment-934685515). - pub(crate) fn for_blocks(&mut self, blockhashes: B, mut func: F) -> Result<()> - where - B: IntoIterator, - F: FnMut(BlockHash, SerBlock), - { - self.blocks_duration.observe_duration("total", || { - let blockhashes: Vec = blockhashes.into_iter().collect(); - if blockhashes.is_empty() { - return Ok(()); - } - self.blocks_duration.observe_duration("request", || { - debug!("loading {} blocks", blockhashes.len()); - self.req_send.send(Request::get_blocks(&blockhashes)) - })?; - - for hash in blockhashes { - let block = self.blocks_duration.observe_duration("response", || { - let block = self - .blocks_recv - .recv() - .with_context(|| format!("failed to get block {}", hash))?; - let header = bsl::BlockHeader::parse(&block[..]) - .expect("core returned invalid blockheader") - .parsed_owned(); - ensure!( - &header.block_hash_sha2()[..] == hash.as_byte_array(), - "got unexpected block" - ); - Ok(block) - })?; - self.blocks_duration - .observe_duration("process", || func(hash, block)); - } - Ok(()) - }) - } - - /// Note: only a single receiver will get the notification (https://github.com/romanz/electrs/pull/526#issuecomment-934687415). - pub(crate) fn new_block_notification(&self) -> Receiver<()> { - self.new_block_recv.clone() - } - - pub(crate) fn connect( - network: Network, - address: SocketAddr, - metrics: &Metrics, - magic: Magic, - ) -> Result { - let recv_conn = TcpStream::connect(address) - .with_context(|| format!("{} p2p failed to connect: {:?}", network, address))?; - let mut send_conn = recv_conn - .try_clone() - .context("failed to clone connection")?; - - let (tx_send, tx_recv) = bounded::(1); - let (rx_send, rx_recv) = bounded::(1); - - let send_duration = metrics.histogram_vec( - "p2p_send_duration", - "Time spent sending p2p messages (in seconds)", - "step", - default_duration_buckets(), - ); - let recv_duration = metrics.histogram_vec( - "p2p_recv_duration", - "Time spent receiving p2p messages (in seconds)", - "step", - default_duration_buckets(), - ); - let parse_duration = metrics.histogram_vec( - "p2p_parse_duration", - "Time spent parsing p2p messages (in seconds)", - "step", - default_duration_buckets(), - ); - let recv_size = metrics.histogram_vec( - "p2p_recv_size", - "Size of p2p messages read (in bytes)", - "message", - default_size_buckets(), - ); - let blocks_duration = metrics.histogram_vec( - "p2p_blocks_duration", - "Time spent getting blocks via p2p protocol (in seconds)", - "step", - default_duration_buckets(), - ); - - let mut buffer = vec![]; - crate::thread::spawn("p2p_send", move || loop { - use std::net::Shutdown; - let msg = match send_duration.observe_duration("wait", || tx_recv.recv()) { - Ok(msg) => msg, - Err(_) => { - // p2p_loop is closed, so tx_send is disconnected - debug!("closing p2p_send thread: no more messages to send"); - // close the stream reader (p2p_recv thread may block on it) - if let Err(e) = send_conn.shutdown(Shutdown::Read) { - warn!("failed to shutdown p2p connection: {}", e) - } - return Ok(()); - } - }; - send_duration.observe_duration("send", || { - trace!("send: {:?}", msg); - let raw_msg = message::RawNetworkMessage::new(magic, msg); - buffer.clear(); - raw_msg - .consensus_encode(&mut buffer) - .expect("in-memory writers don't error"); - send_conn - .write_all(buffer.as_slice()) - .context("p2p failed to send") - })?; - }); - - let mut stream_reader = std::io::BufReader::new(recv_conn); - crate::thread::spawn("p2p_recv", move || loop { - let start = Instant::now(); - let raw_msg = RawNetworkMessage::consensus_decode(&mut stream_reader); - { - let duration = duration_to_seconds(start.elapsed()); - let label = format!( - "recv_{}", - raw_msg - .as_ref() - .map(|msg| msg.cmd.as_ref()) - .unwrap_or("err") - ); - recv_duration.observe(&label, duration); - } - let raw_msg = match raw_msg { - Ok(raw_msg) => { - recv_size.observe(raw_msg.cmd.as_ref(), raw_msg.raw.len() as f64); - if raw_msg.magic != magic { - bail!("unexpected magic {} (instead of {})", raw_msg.magic, magic) - } - raw_msg - } - Err(encode::Error::Io(e)) if e.kind() == io::ErrorKind::UnexpectedEof => { - debug!("closing p2p_recv thread: connection closed"); - return Ok(()); - } - Err(e) => bail!("failed to recv a message from peer: {}", e), - }; - - recv_duration.observe_duration("wait", || rx_send.send(raw_msg))?; - }); - - let (req_send, req_recv) = bounded::(1); - let (blocks_send, blocks_recv) = bounded::(10); - let (headers_send, headers_recv) = bounded::>(1); - let (new_block_send, new_block_recv) = bounded::<()>(0); - let (init_send, init_recv) = bounded::<()>(0); - - tx_send.send(build_version_message())?; - - crate::thread::spawn("p2p_loop", move || loop { - select! { - recv(rx_recv) -> result => { - let raw_msg = match result { - Ok(raw_msg) => raw_msg, - Err(_) => { // p2p_recv is closed, so rx_send is disconnected - debug!("closing p2p_loop thread: peer has disconnected"); - return Ok(()); // new_block_send is dropped, causing the server to exit - } - }; - - let label = format!("parse_{}", raw_msg.cmd.as_ref()); - let msg = match parse_duration.observe_duration(&label, || raw_msg.parse()) { - Ok(msg) => msg, - Err(err) => bail!("failed to parse {err}"), - }; - trace!("recv: {:?}", msg); - - match msg { - ParsedNetworkMessage::Version(version) => { - debug!("peer version: {:?}", version); - tx_send.send(NetworkMessage::Verack)?; - } - ParsedNetworkMessage::Inv(inventory) => { - debug!("peer inventory: {:?}", inventory); - if inventory.iter().any(|inv| matches!(inv, Inventory::Block(_))) { - let _ = new_block_send.try_send(()); // best-effort notification - } - - }, - ParsedNetworkMessage::Ping(nonce) => { - tx_send.send(NetworkMessage::Pong(nonce))?; // connection keep-alive - } - ParsedNetworkMessage::Verack => { - init_send.send(())?; // peer acknowledged our version - } - ParsedNetworkMessage::Block(block) => blocks_send.send(block)?, - ParsedNetworkMessage::Headers(headers) => headers_send.send(headers)?, - ParsedNetworkMessage::Ignored => (), - } - } - recv(req_recv) -> result => { - let req = match result { - Ok(req) => req, - Err(_) => { // self is dropped, so req_send is disconnected - debug!("closing p2p_loop thread: no more requests to handle"); - return Ok(()); - } - }; - let msg = match req { - Request::GetNewHeaders(msg) => NetworkMessage::GetHeaders(msg), - Request::GetBlocks(inv) => NetworkMessage::GetData(inv), - }; - tx_send.send(msg)?; - } - } - }); - - init_recv.recv()?; // wait until `verack` is received - - Ok(Connection { - req_send, - blocks_recv, - headers_recv, - new_block_recv, - blocks_duration, - }) - } -} - -fn build_version_message() -> NetworkMessage { - let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0); - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time error") - .as_secs() as i64; - - let services = p2p::ServiceFlags::NONE; - - NetworkMessage::Version(message_network::VersionMessage { - version: p2p::PROTOCOL_VERSION, - services, - timestamp, - receiver: address::Address::new(&addr, services), - sender: address::Address::new(&addr, services), - nonce: secp256k1::rand::thread_rng().gen(), - user_agent: format!("/electrs:{}/", ELECTRS_VERSION), - start_height: 0, - relay: false, - }) -} - -struct RawNetworkMessage { - magic: Magic, - cmd: CommandString, - raw: Vec, -} - -impl RawNetworkMessage { - fn parse(self) -> Result { - let mut raw: &[u8] = &self.raw; - let payload = match self.cmd.as_ref() { - "version" => ParsedNetworkMessage::Version(Decodable::consensus_decode(&mut raw)?), - "verack" => ParsedNetworkMessage::Verack, - "inv" => ParsedNetworkMessage::Inv(Decodable::consensus_decode(&mut raw)?), - "block" => ParsedNetworkMessage::Block(self.raw), - "headers" => { - let len = VarInt::consensus_decode(&mut raw)?.0; - let mut headers = Vec::with_capacity(len as usize); - for _ in 0..len { - headers.push(Block::consensus_decode(&mut raw)?.header); - } - ParsedNetworkMessage::Headers(headers) - } - "ping" => ParsedNetworkMessage::Ping(Decodable::consensus_decode(&mut raw)?), - "pong" => ParsedNetworkMessage::Ignored, // unused - "addr" => ParsedNetworkMessage::Ignored, // unused - "alert" => ParsedNetworkMessage::Ignored, // https://bitcoin.org/en/alert/2016-11-01-alert-retirement - _ => bail!( - "unsupported message: command={}, payload={:?}", - self.cmd, - self.raw - ), - }; - Ok(payload) - } -} - -#[derive(Debug)] -enum ParsedNetworkMessage { - Version(message_network::VersionMessage), - Verack, - Inv(Vec), - Ping(u64), - Headers(Vec), - Block(SerBlock), - Ignored, -} - -impl Decodable for RawNetworkMessage { - fn consensus_decode( - d: &mut D, - ) -> Result { - let magic = Decodable::consensus_decode(d)?; - let cmd = Decodable::consensus_decode(d)?; - - let len = u32::consensus_decode(d)?; - let _checksum = <[u8; 4]>::consensus_decode(d)?; // assume data is correct - let mut raw = vec![0u8; len as usize]; - d.read_slice(&mut raw)?; - - Ok(RawNetworkMessage { magic, cmd, raw }) - } -} - -/// `duration_to_seconds` converts Duration to seconds. -#[inline] -pub fn duration_to_seconds(d: Duration) -> f64 { - let nanos = f64::from(d.subsec_nanos()) / 1e9; - d.as_secs() as f64 + nanos -} diff --git a/src/server.rs b/src/server.rs index f50662f..221fb5a 100644 --- a/src/server.rs +++ b/src/server.rs @@ -85,7 +85,6 @@ fn serve() -> Result<()> { ); let mut rpc = Rpc::new(&config, metrics)?; - let new_block_rx = rpc.new_block_notification(); let mut peers = HashMap::::new(); loop { // initial sync and compaction may take a few hours @@ -107,14 +106,6 @@ fn serve() -> Result<()> { result.context("signal channel disconnected")?; rpc.signal().exit_flag().poll().context("RPC server interrupted")?; }, - // Handle new blocks' notifications - recv(new_block_rx) -> result => match result { - Ok(_) => (), // sync and update - Err(_) => { - info!("disconnected from bitcoind"); - return Ok(()); - } - }, // Handle Electrum RPC requests recv(server_rx) -> event => { let first = once(event.context("server disconnected")?); diff --git a/src/signals.rs b/src/signals.rs index 53c08fa..f96aca1 100644 --- a/src/signals.rs +++ b/src/signals.rs @@ -1,6 +1,9 @@ +#[cfg(not(windows))] use anyhow::Context; use crossbeam_channel::{unbounded, Receiver}; +#[cfg(not(windows))] use signal_hook::consts::signal::*; +#[cfg(not(windows))] use signal_hook::iterator::Signals; use std::sync::{ @@ -9,6 +12,7 @@ use std::sync::{ }; use std::{error, fmt}; +#[cfg(not(windows))] use crate::thread::spawn; #[derive(Debug)] @@ -53,6 +57,7 @@ pub(crate) struct Signal { } impl Signal { + #[cfg(not(windows))] pub fn new() -> Signal { let ids = vec![ SIGINT, SIGTERM, @@ -80,6 +85,27 @@ impl Signal { result } + #[cfg(windows)] + pub fn new() -> Signal { + let (tx, rx) = unbounded(); + let result = Signal { + rx, + exit: ExitFlag::new(), + }; + + let exit_flag = result.exit.clone(); + + // Handle Ctrl-C + ctrlc::set_handler(move || { + info!("notified via Ctrl-C"); + exit_flag.set(); + let _ = tx.send(()); + }) + .expect("failed to set Ctrl-C handler"); + + result + } + pub fn receiver(&self) -> &Receiver<()> { &self.rx } diff --git a/src/status.rs b/src/status.rs index 41115b3..fe16acf 100644 --- a/src/status.rs +++ b/src/status.rs @@ -1,27 +1,16 @@ -use anyhow::Result; -use bitcoin::{ +use crate::bitcoin::{ consensus::Decodable, hashes::{sha256, Hash, HashEngine}, Amount, BlockHash, OutPoint, SignedAmount, Transaction, Txid, }; -use bitcoin_slices::{bsl, Visit, Visitor}; -use rayon::prelude::*; +use anyhow::Result; +use bindex::{IndexedChain, ScriptHash}; use serde::ser::{Serialize, Serializer}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::convert::TryFrom; -use std::{ - collections::{BTreeMap, HashMap, HashSet}, - ops::ControlFlow, -}; -use crate::{ - cache::Cache, - chain::Chain, - daemon::Daemon, - index::Index, - mempool::Mempool, - types::{bsl_txid, ScriptHash, SerBlock, StatusHash}, -}; +use crate::{mempool::Mempool, types::StatusHash}; /// Given a scripthash, store relevant inputs and outputs of a specific transaction struct TxEntry { @@ -30,6 +19,7 @@ struct TxEntry { spent: Vec, // relevant spent outpoints } +// Funded outputs of a transaction struct TxOutput { index: u32, value: Amount, @@ -51,8 +41,8 @@ impl TxEntry { } // Confirmation height of a transaction or its mempool state: -// https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-history -// https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-mempool +// https://electrum-protocol.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-history +// https://electrum-protocol.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-mempool enum Height { Confirmed { height: usize }, Unconfirmed { has_unconfirmed_inputs: bool }, @@ -88,8 +78,8 @@ impl std::fmt::Display for Height { } // A single history entry: -// https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-history -// https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-mempool +// https://electrum-protocol.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-history +// https://electrum-protocol.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-mempool #[derive(Serialize)] pub(crate) struct HistoryEntry { #[serde(rename = "tx_hash")] @@ -97,12 +87,14 @@ pub(crate) struct HistoryEntry { height: Height, #[serde( skip_serializing_if = "Option::is_none", - with = "bitcoin::amount::serde::as_sat::opt" + with = "crate::bitcoin::amount::serde::as_sat::opt" )] fee: Option, } impl HistoryEntry { + // Hash to compute ScriptHash status, as defined here: + // https://electrum-protocol.readthedocs.io/en/latest/protocol-basics.html#status fn hash(&self, engine: &mut sha256::HashEngine) { let s = format!("{}:{}:", self.txid, self.height); engine.input(s.as_bytes()); @@ -127,33 +119,43 @@ impl HistoryEntry { } } +/// Make sure blocks are sorted by ascending height. +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] +struct BlockKey(usize, BlockHash); + +impl From> for BlockKey { + fn from(location: bindex::Location<'_>) -> Self { + Self(location.block_height(), location.block_hash()) + } +} + /// ScriptHash subscription status pub struct ScriptHashStatus { scripthash: ScriptHash, // specific scripthash to be queried - tip: BlockHash, // used for skipping confirmed entries' sync - confirmed: HashMap>, // confirmed entries, partitioned per block (may contain stale blocks) + confirmed: BTreeMap>, // confirmed entries (chronologically ordered), partitioned per block mempool: Vec, // unconfirmed entries history: Vec, // computed from confirmed and mempool entries statushash: Option, // computed from history } /// Specific scripthash balance +/// https://electrum-protocol.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-get-balance #[derive(Default, Eq, PartialEq, Serialize)] pub(crate) struct Balance { - #[serde(with = "bitcoin::amount::serde::as_sat", rename = "confirmed")] + #[serde(with = "crate::bitcoin::amount::serde::as_sat", rename = "confirmed")] confirmed_balance: Amount, - #[serde(with = "bitcoin::amount::serde::as_sat", rename = "unconfirmed")] + #[serde(with = "crate::bitcoin::amount::serde::as_sat", rename = "unconfirmed")] mempool_delta: SignedAmount, } -// A single unspent transaction output entry: -// https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-listunspent +/// A single unspent transaction output entry +/// https://electrum-protocol.readthedocs.io/en/latest/protocol-methods.html#blockchain-scripthash-listunspent #[derive(Serialize)] pub(crate) struct UnspentEntry { height: usize, // 0 = mempool entry tx_hash: Txid, tx_pos: u32, - #[serde(with = "bitcoin::amount::serde::as_sat")] + #[serde(with = "crate::bitcoin::amount::serde::as_sat")] value: Amount, } @@ -161,29 +163,27 @@ pub(crate) struct UnspentEntry { struct Unspent { // mapping an outpoint to its value & confirmation height outpoints: HashMap, - confirmed_balance: Amount, - mempool_delta: SignedAmount, + balance: Balance, } impl Unspent { - fn build(status: &ScriptHashStatus, chain: &Chain) -> Self { + fn build(status: &ScriptHashStatus) -> Self { let mut unspent = Unspent::default(); - + // First, add all relevant entries' funding outputs to the outpoints' map status - .confirmed_height_entries(chain) + .confirmed_height_entries() .for_each(|(height, entries)| entries.iter().for_each(|e| unspent.insert(e, height))); - status - .confirmed_entries(chain) - .for_each(|e| unspent.remove(e)); - - unspent.confirmed_balance = unspent.balance(); + // Then, remove spent outpoints from the map + status.confirmed_entries().for_each(|e| unspent.remove(e)); + unspent.balance.confirmed_balance = unspent.balance(); + // Now, do the same over the mempool (first add funding outputs, and then remove spent ones) status.mempool.iter().for_each(|e| unspent.insert(e, 0)); // mempool height = 0 status.mempool.iter().for_each(|e| unspent.remove(e)); - unspent.mempool_delta = - unspent.balance().to_signed().unwrap() - unspent.confirmed_balance.to_signed().unwrap(); - + let total_balance = unspent.balance().to_signed().unwrap(); + let confirmed_balance = unspent.balance.confirmed_balance.to_signed().unwrap(); + unspent.balance.mempool_delta = total_balance - confirmed_balance; unspent } @@ -199,6 +199,7 @@ impl Unspent { .collect() } + /// Total amount of unspent outputs fn balance(&self) -> Amount { self.outpoints .values() @@ -227,8 +228,7 @@ impl ScriptHashStatus { pub fn new(scripthash: ScriptHash) -> Self { Self { scripthash, - tip: BlockHash::all_zeros(), - confirmed: HashMap::new(), + confirmed: BTreeMap::new(), mempool: Vec::new(), history: Vec::new(), statushash: None, @@ -237,52 +237,43 @@ impl ScriptHashStatus { /// Iterate through confirmed TxEntries with their corresponding block heights. /// Skip entries from stale blocks. - fn confirmed_height_entries<'a>( - &'a self, - chain: &'a Chain, - ) -> impl Iterator + 'a { + fn confirmed_height_entries<'a>(&'a self) -> impl Iterator + 'a { self.confirmed .iter() - .filter_map(move |(blockhash, entries)| { - chain - .get_block_height(blockhash) - .map(|height| (height, &entries[..])) - }) + .map(move |(blockid, entries)| (blockid.0, &entries[..])) } /// Iterate through confirmed TxEntries. - /// Skip entries from stale blocks. - fn confirmed_entries<'a>(&'a self, chain: &'a Chain) -> impl Iterator + 'a { - self.confirmed_height_entries(chain) + fn confirmed_entries<'a>(&'a self) -> impl Iterator + 'a { + self.confirmed_height_entries() .flat_map(|(_height, entries)| entries) } /// Collect all funded and confirmed outpoints (as a set). - fn confirmed_outpoints(&self, chain: &Chain) -> HashSet { - self.confirmed_entries(chain) + fn confirmed_outpoints(&self) -> HashSet { + self.confirmed_entries() .flat_map(TxEntry::funding_outpoints) .collect() } - pub(crate) fn get_unspent(&self, chain: &Chain) -> Vec { - Unspent::build(self, chain).into_entries() + /// Collect unspent transaction entries + pub(crate) fn get_unspent(&self) -> Vec { + Unspent::build(self).into_entries() } - pub(crate) fn get_balance(&self, chain: &Chain) -> Balance { - let unspent = Unspent::build(self, chain); - Balance { - confirmed_balance: unspent.confirmed_balance, - mempool_delta: unspent.mempool_delta, - } + /// Collect unspent transaction balance + pub(crate) fn get_balance(&self) -> Balance { + Unspent::build(self).balance } + /// Collect transaction history entries pub(crate) fn get_history(&self) -> &[HistoryEntry] { &self.history } /// Collect all confirmed history entries (in block order). - fn get_confirmed_history(&self, chain: &Chain) -> Vec { - self.confirmed_height_entries(chain) + fn get_confirmed_history(&self) -> Vec { + self.confirmed_height_entries() .collect::>() .into_iter() .flat_map(|(height, entries)| { @@ -307,127 +298,89 @@ impl ScriptHashStatus { .collect() } - /// Apply func only on the new blocks (fetched from daemon). - fn for_new_blocks(&self, blockhashes: B, daemon: &Daemon, func: F) -> Result<()> - where - B: IntoIterator, - F: FnMut(BlockHash, SerBlock), - { - daemon.for_blocks( - blockhashes - .into_iter() - .filter(|blockhash| !self.confirmed.contains_key(blockhash)), - func, - ) - } - /// Get funding and spending entries from new blocks. /// Also cache relevant transactions and their merkle proofs. - fn sync_confirmed( - &self, - index: &Index, - daemon: &Daemon, - cache: &Cache, - outpoints: &mut HashSet, - ) -> Result>> { - let scripthash = self.scripthash; - let mut result = HashMap::>::new(); - - let funding_blockhashes = index.limit_result(index.filter_by_funding(scripthash))?; - self.for_new_blocks(funding_blockhashes, daemon, |blockhash, block| { - let block_entries = result.entry(blockhash).or_default(); - for filtered_outputs in filter_block_txs_outputs(block, scripthash) { - cache.add_tx(filtered_outputs.txid, move || filtered_outputs.tx); - outpoints.extend(make_outpoints( - filtered_outputs.txid, - &filtered_outputs.result, - )); - block_entries - .entry(filtered_outputs.pos) - .or_insert_with(|| TxEntry::new(filtered_outputs.txid)) - .outputs = filtered_outputs.result; + fn sync_confirmed(&mut self, index: &IndexedChain) -> Result> { + let headers = index.headers(); + let mut latest_header = None; + // Drop entries from stale blocks + while let Some(entry) = self.confirmed.last_entry() { + let BlockKey(height, hash) = entry.key(); + match headers.get_header(*hash, *height) { + Ok(header) => { + latest_header = Some(header); + break; + } + Err(err) => { + warn!("drop reorged block: {}", err); + entry.remove(); + continue; + } } - })?; - let spending_blockhashes: HashSet = outpoints - .par_iter() - .flat_map_iter(|outpoint| index.filter_by_spending(*outpoint)) - .collect(); - self.for_new_blocks(spending_blockhashes, daemon, |blockhash, block| { - let block_entries = result.entry(blockhash).or_default(); - for filtered_inputs in filter_block_txs_inputs(&block, outpoints) { - cache.add_tx(filtered_inputs.txid, move || filtered_inputs.tx); - block_entries - .entry(filtered_inputs.pos) - .or_insert_with(|| TxEntry::new(filtered_inputs.txid)) - .spent = filtered_inputs.result; - } - })?; + } + // Recompute all funded outpoints + let mut outpoints = self.confirmed_outpoints(); + // Process transactions in chronological order + for location in index.locations_by_scripthash(&self.scripthash, latest_header)? { + let tx_bytes = index.get_tx_bytes(&location)?; + let tx = Transaction::consensus_decode_from_finite_reader(&mut &tx_bytes[..])?; - Ok(result - .into_iter() - .map(|(blockhash, entries_map)| { - // sort transactions by their position in a block - let sorted_entries = entries_map - .into_iter() - .collect::>() - .into_values() - .collect(); - (blockhash, sorted_entries) - }) - .collect()) + // Check if this transaction has relevant inputs/outputs: + let spent = filter_inputs(&tx, &outpoints); + let outputs = filter_outputs(&tx, self.scripthash); + if spent.is_empty() && outputs.is_empty() { + continue; + } + + // Build new TxEntry and add new outpoints (for next transactions) + let mut tx_entry = TxEntry::new(tx.compute_txid()); + tx_entry.spent = spent; + tx_entry.outputs = outputs; + outpoints.extend(tx_entry.funding_outpoints()); + + // Add new per-block entry (if needed) + self.confirmed + .entry(BlockKey::from(location)) + .or_default() + .push(tx_entry); + } + + Ok(outpoints) } /// Get funding and spending entries from current mempool. /// Also cache relevant transactions. - fn sync_mempool( - &self, - mempool: &Mempool, - cache: &Cache, - outpoints: &mut HashSet, - ) -> Vec { + fn sync_mempool(&self, mempool: &Mempool, mut outpoints: HashSet) -> Vec { let mut result = HashMap::::new(); + // extract relevant funding transactions for entry in mempool.filter_by_funding(&self.scripthash) { let funding_outputs = filter_outputs(&entry.tx, self.scripthash); assert!(!funding_outputs.is_empty()); + // store funded outpoints (to check for spending later) outpoints.extend(make_outpoints(entry.txid, &funding_outputs)); result - .entry(entry.txid) + .entry(entry.txid) // the transaction may already exist .or_insert_with(|| TxEntry::new(entry.txid)) .outputs = funding_outputs; - cache.add_tx(entry.txid, || entry.tx.clone()); } for entry in outpoints .iter() .flat_map(|outpoint| mempool.filter_by_spending(outpoint)) { - let spent_outpoints = filter_inputs(&entry.tx, outpoints); + let spent_outpoints = filter_inputs(&entry.tx, &outpoints); assert!(!spent_outpoints.is_empty()); result - .entry(entry.txid) + .entry(entry.txid) // the transaction may already exist .or_insert_with(|| TxEntry::new(entry.txid)) .spent = spent_outpoints; - cache.add_tx(entry.txid, || entry.tx.clone()); } result.into_values().collect() } - /// Sync with currently confirmed txs and mempool, downloading non-cached transactions via p2p protocol. + /// Sync with currently confirmed txs and mempool, downloading non-cached transactions via REST API. /// After a successful sync, scripthash status is updated. - pub(crate) fn sync( - &mut self, - index: &Index, - mempool: &Mempool, - daemon: &Daemon, - cache: &Cache, - ) -> Result<()> { - let mut outpoints: HashSet = self.confirmed_outpoints(index.chain()); - - let new_tip = index.chain().tip(); - if self.tip != new_tip { - let update = self.sync_confirmed(index, daemon, cache, &mut outpoints)?; - self.confirmed.extend(update); - self.tip = new_tip; - } + pub(crate) fn sync(&mut self, index: &IndexedChain, mempool: &Mempool) -> Result<()> { + let outpoints = self.sync_confirmed(index)?; if !self.confirmed.is_empty() { debug!( "{} transactions from {} blocks", @@ -435,13 +388,13 @@ impl ScriptHashStatus { self.confirmed.len() ); } - self.mempool = self.sync_mempool(mempool, cache, &mut outpoints); + self.mempool = self.sync_mempool(mempool, outpoints); if !self.mempool.is_empty() { debug!("{} mempool transactions", self.mempool.len()); } + // update history entries and status hash self.history.clear(); - self.history - .extend(self.get_confirmed_history(index.chain())); + self.history.extend(self.get_confirmed_history()); self.history.extend(self.get_mempool_history(mempool)); self.statushash = compute_status_hash(&self.history); @@ -460,35 +413,28 @@ fn make_outpoints(txid: Txid, outputs: &[TxOutput]) -> impl Iterator Vec { let outputs = tx.output.iter().zip(0u32..); outputs - .filter_map(move |(txo, vout)| { - if ScriptHash::new(&txo.script_pubkey) == scripthash { - Some(TxOutput { - index: vout, - value: txo.value, - }) - } else { - None - } + .filter(|&(txo, _vout)| ScriptHash::new(&txo.script_pubkey) == scripthash) + .map(|(txo, vout)| TxOutput { + index: vout, + value: txo.value, }) .collect() } +/// Collect inputs that spend one of the specified outpoints. fn filter_inputs(tx: &Transaction, outpoints: &HashSet) -> Vec { - tx.input - .iter() - .filter_map(|txi| { - if outpoints.contains(&txi.previous_output) { - Some(txi.previous_output) - } else { - None - } - }) + let inputs = tx.input.iter(); + inputs + .filter(|&txi| outpoints.contains(&txi.previous_output)) + .map(|txi| txi.previous_output) .collect() } +// See https://electrum-protocol.readthedocs.io/en/latest/protocol-basics.html#status for details fn compute_status_hash(history: &[HistoryEntry]) -> Option { if history.is_empty() { return None; @@ -500,118 +446,10 @@ fn compute_status_hash(history: &[HistoryEntry]) -> Option { Some(StatusHash::from_engine(engine)) } -struct FilteredTx { - tx: Transaction, - txid: Txid, - pos: usize, - result: Vec, -} - -fn filter_block_txs_outputs(block: SerBlock, scripthash: ScriptHash) -> Vec> { - struct FindOutputs { - scripthash: ScriptHash, - result: Vec>, - buffer: Vec, - pos: usize, - } - impl Visitor for FindOutputs { - fn visit_transaction(&mut self, tx: &bsl::Transaction) -> ControlFlow<()> { - if !self.buffer.is_empty() { - let result = std::mem::take(&mut self.buffer); - let txid = bsl_txid(tx); - let tx = bitcoin::Transaction::consensus_decode(&mut tx.as_ref()) - .expect("transaction was already validated"); - self.result.push(FilteredTx:: { - tx, - txid, - pos: self.pos, - result, - }); - } - self.pos += 1; - ControlFlow::Continue(()) - } - fn visit_tx_out(&mut self, vout: usize, tx_out: &bsl::TxOut) -> ControlFlow<()> { - let current = ScriptHash::hash(tx_out.script_pubkey()); - if current == self.scripthash { - self.buffer.push(TxOutput { - index: vout as u32, - value: Amount::from_sat(tx_out.value()), - }) - } - ControlFlow::Continue(()) - } - } - let mut find_outputs = FindOutputs { - scripthash, - result: vec![], - buffer: vec![], - pos: 0, - }; - - bsl::Block::visit(&block, &mut find_outputs).expect("core returned invalid block"); - - find_outputs.result -} - -fn filter_block_txs_inputs( - block: &SerBlock, - outpoints: &HashSet, -) -> Vec> { - struct FindInputs<'a> { - outpoints: &'a HashSet, - result: Vec>, - buffer: Vec, - pos: usize, - } - - impl<'a> Visitor for FindInputs<'a> { - fn visit_transaction(&mut self, tx: &bsl::Transaction) -> ControlFlow<()> { - if !self.buffer.is_empty() { - let result = std::mem::take(&mut self.buffer); - let txid = bsl_txid(tx); - let tx = bitcoin::Transaction::consensus_decode(&mut tx.as_ref()) - .expect("transaction was already validated"); - self.result.push(FilteredTx:: { - tx, - txid, - pos: self.pos, - result, - }); - } - self.pos += 1; - ControlFlow::Continue(()) - } - fn visit_tx_in(&mut self, _vin: usize, tx_in: &bsl::TxIn) -> ControlFlow<()> { - let current: OutPoint = tx_in.prevout().into(); - if self.outpoints.contains(¤t) { - self.buffer.push(current); - } - ControlFlow::Continue(()) - } - } - - let mut find_inputs = FindInputs { - outpoints, - result: vec![], - buffer: vec![], - pos: 0, - }; - - bsl::Block::visit(block, &mut find_inputs).expect("core returned invalid block"); - - find_inputs.result -} - #[cfg(test)] mod tests { - use std::{collections::HashSet, str::FromStr}; - - use crate::types::ScriptHash; - use super::HistoryEntry; - use bitcoin::{Address, Amount}; - use bitcoin_test_data::blocks::mainnet_702861; + use crate::bitcoin::Amount; use serde_json::json; #[test] @@ -636,42 +474,4 @@ mod tests { json!({"tx_hash": "5b75086dafeede555fc8f9a810d8b10df57c46f9f176ccc3dd8d2fa20edd685b", "height": 0, "fee": 123}) ); } - - #[test] - fn test_find_outputs() { - let block = mainnet_702861().to_vec(); - - let addr = Address::from_str("1A9MXXG26vZVySrNNytQK1N8bX42ZuJ6Ax") - .unwrap() - .assume_checked(); - let scripthash = ScriptHash::new(&addr.script_pubkey()); - - let result = &super::filter_block_txs_outputs(block, scripthash)[0]; - assert_eq!( - result.txid.to_string(), - "7bcdcb44422da5a99daad47d6ba1c3d6f2e48f961a75e42c4fa75029d4b0ef49" - ); - assert_eq!(result.pos, 8); - assert_eq!(result.result[0].index, 0); - assert_eq!(result.result[0].value.to_sat(), 709503); - } - - #[test] - fn test_find_inputs() { - let block = mainnet_702861().to_vec(); - let outpoint = bitcoin::OutPoint::from_str( - "cc135e792b37a9c4ffd784f696b1e38bd1197f8e67ae1f96c9f13e4618b91866:3", - ) - .unwrap(); - let mut outpoints = HashSet::new(); - outpoints.insert(outpoint); - - let result = &super::filter_block_txs_inputs(&block, &outpoints)[0]; - assert_eq!( - result.txid.to_string(), - "7bcdcb44422da5a99daad47d6ba1c3d6f2e48f961a75e42c4fa75029d4b0ef49" - ); - assert_eq!(result.pos, 8); - assert_eq!(result.result[0], outpoint); - } } diff --git a/src/tracker.rs b/src/tracker.rs index 9ed97f3..0c7f23b 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -1,18 +1,11 @@ +use crate::bitcoin::{BlockHash, Txid}; +use crate::bitcoin_slices::{bsl, EmptyVisitor, Visit}; use anyhow::{Context, Result}; -use bitcoin::{BlockHash, Transaction, Txid}; -use bitcoin_slices::{ - bsl::{self, FindTransaction}, - Error::VisitBreak, - Visit, -}; +use bindex::IndexedChain; use crate::{ - cache::Cache, - chain::Chain, config::Config, daemon::Daemon, - db::DBStore, - index::Index, mempool::{FeeHistogram, Mempool}, metrics::Metrics, signals::ExitFlag, @@ -21,58 +14,39 @@ use crate::{ /// Electrum protocol subscriptions' tracker pub struct Tracker { - index: Index, + index: IndexedChain, mempool: Mempool, - metrics: Metrics, ignore_mempool: bool, } -pub(crate) enum Error { - NotReady, -} - impl Tracker { pub fn new(config: &Config, metrics: Metrics) -> Result { - let store = DBStore::open( - &config.db_path, - config.db_log_dir.as_deref(), - config.auto_reindex, - )?; - let chain = Chain::new(config.network); + let url = format!("http://{}", config.daemon_rpc_addr); + let index = IndexedChain::open(&config.db_dir, config.network, Some(url)) + .context("failed to open index")?; Ok(Self { - index: Index::load( - store, - chain, - &metrics, - config.index_batch_size, - config.index_lookup_limit, - config.reindex_last_blocks, - ) - .context("failed to open index")?, + index, mempool: Mempool::new(&metrics), - metrics, ignore_mempool: config.ignore_mempool, }) } - pub(crate) fn chain(&self) -> &Chain { - self.index.chain() + pub(crate) fn headers(&self) -> &bindex::Headers { + self.index.headers() } pub(crate) fn fees_histogram(&self) -> &FeeHistogram { self.mempool.fees_histogram() } - pub(crate) fn metrics(&self) -> &Metrics { - &self.metrics - } - pub(crate) fn get_unspent(&self, status: &ScriptHashStatus) -> Vec { - status.get_unspent(self.index.chain()) + status.get_unspent() } pub(crate) fn sync(&mut self, daemon: &Daemon, exit_flag: &ExitFlag) -> Result { - let done = self.index.sync(daemon, exit_flag)?; + exit_flag.poll()?; + let stats = self.index.sync(1000)?; + let done = stats.indexed_blocks == 0; if done && !self.ignore_mempool { self.mempool.sync(daemon, exit_flag); // TODO: double check tip - and retry on diff @@ -80,46 +54,36 @@ impl Tracker { Ok(done) } - pub(crate) fn status(&self) -> Result<(), Error> { - if self.index.is_ready() { - return Ok(()); - } - Err(Error::NotReady) + pub(crate) fn status(&self) -> Result<()> { + Ok(()) } - pub(crate) fn update_scripthash_status( - &self, - status: &mut ScriptHashStatus, - daemon: &Daemon, - cache: &Cache, - ) -> Result { + pub(crate) fn update_scripthash_status(&self, status: &mut ScriptHashStatus) -> Result { let prev_statushash = status.statushash(); - status.sync(&self.index, &self.mempool, daemon, cache)?; + status.sync(&self.index, &self.mempool)?; Ok(prev_statushash != status.statushash()) } pub(crate) fn get_balance(&self, status: &ScriptHashStatus) -> Balance { - status.get_balance(self.chain()) + status.get_balance() } - pub(crate) fn lookup_transaction( - &self, - daemon: &Daemon, - txid: Txid, - ) -> Result> { + pub(crate) fn lookup_transaction(&self, txid: Txid) -> Result)>> { // Note: there are two blocks with coinbase transactions having same txid (see BIP-30) - let blockhashes = self.index.filter_by_txid(txid); - let mut result = None; - daemon.for_blocks(blockhashes, |blockhash, block| { - if result.is_some() { - return; // keep first matching transaction + for loc in self.index.locations_by_txid(&txid)? { + let tx_bytes = self.index.get_tx_bytes(&loc)?; + if txid == compute_txid(&tx_bytes)? { + return Ok(Some((loc.block_hash(), tx_bytes.into_boxed_slice()))); } - let mut visitor = FindTransaction::new(txid); - result = match bsl::Block::visit(&block, &mut visitor) { - Ok(_) | Err(VisitBreak) => visitor.tx_found().map(|tx| (blockhash, tx)), - Err(e) => panic!("core returned invalid block: {:?}", e), - }; - })?; - Ok(result) + } + Ok(None) } } + +fn compute_txid(tx_bytes: &[u8]) -> Result { + let mut visit = EmptyVisitor {}; + let res = bsl::Transaction::visit(tx_bytes, &mut visit) + .map_err(|err| anyhow!("invalid transaction: {:?}", err))?; + ensure!(res.remaining().is_empty(), "non-empty remaining bytes"); + Ok(Txid::from_raw_hash(res.parsed().txid())) +} diff --git a/src/types.rs b/src/types.rs index 2d33949..54a3ae7 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,292 +1,13 @@ -use anyhow::Result; - -use std::convert::TryFrom; - -use bitcoin::blockdata::block::Header as BlockHeader; -use bitcoin::{ - consensus::encode::{deserialize, Decodable, Encodable}, - hashes::{hash_newtype, sha256, Hash}, - io, OutPoint, Script, Txid, -}; -use bitcoin_slices::bsl; - -macro_rules! impl_consensus_encoding { - ($thing:ident, $($field:ident),+) => ( - impl Encodable for $thing { - #[inline] - fn consensus_encode( - &self, - s: &mut S, - ) -> Result { - let mut len = 0; - $(len += self.$field.consensus_encode(s)?;)+ - Ok(len) - } - } - - impl Decodable for $thing { - #[inline] - fn consensus_decode( - d: &mut D, - ) -> Result<$thing, bitcoin::consensus::encode::Error> { - Ok($thing { - $($field: Decodable::consensus_decode(d)?),+ - }) - } - } - ); -} - -pub const HASH_PREFIX_LEN: usize = 8; -const HEIGHT_SIZE: usize = 4; - -pub(crate) type HashPrefix = [u8; HASH_PREFIX_LEN]; -pub(crate) type SerializedHashPrefixRow = [u8; HASH_PREFIX_ROW_SIZE]; -type Height = u32; -pub(crate) type SerBlock = Vec; - -#[derive(Debug, Serialize, Deserialize, PartialEq)] -pub(crate) struct HashPrefixRow { - prefix: HashPrefix, - height: Height, // transaction confirmed height -} - -pub const HASH_PREFIX_ROW_SIZE: usize = HASH_PREFIX_LEN + HEIGHT_SIZE; - -impl HashPrefixRow { - pub(crate) fn to_db_row(&self) -> SerializedHashPrefixRow { - let mut row = [0; HASH_PREFIX_ROW_SIZE]; - let len = self - .consensus_encode(&mut (&mut row as &mut [u8])) - .expect("in-memory writers don't error"); - debug_assert_eq!(len, HASH_PREFIX_ROW_SIZE); - row - } - - pub(crate) fn from_db_row(row: SerializedHashPrefixRow) -> Self { - deserialize(&row).expect("bad HashPrefixRow") - } - - pub fn height(&self) -> usize { - usize::try_from(self.height).expect("invalid height") - } -} - -impl_consensus_encoding!(HashPrefixRow, prefix, height); +use crate::bitcoin::hashes::{hash_newtype, sha256}; hash_newtype! { - /// https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-basics.html#script-hashes - #[hash_newtype(backward)] - pub struct ScriptHash(sha256::Hash); -} - -impl ScriptHash { - pub fn new(script: &Script) -> Self { - ScriptHash::hash(script.as_bytes()) - } - - fn prefix(&self) -> HashPrefix { - let mut prefix = HashPrefix::default(); - prefix.copy_from_slice(&self.0[..HASH_PREFIX_LEN]); - prefix - } -} - -pub(crate) struct ScriptHashRow; - -impl ScriptHashRow { - pub(crate) fn scan_prefix(scripthash: ScriptHash) -> HashPrefix { - scripthash.0[..HASH_PREFIX_LEN].try_into().unwrap() - } - - pub(crate) fn row(scripthash: ScriptHash, height: usize) -> HashPrefixRow { - HashPrefixRow { - prefix: scripthash.prefix(), - height: Height::try_from(height).expect("invalid height"), - } - } -} - -// *************************************************************************** - -hash_newtype! { - /// https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-basics.html#status + /// https://electrum-protocol.readthedocs.io/en/latest/protocol-basics.html#status pub struct StatusHash(sha256::Hash); } -// *************************************************************************** - -fn spending_prefix(prev: OutPoint) -> HashPrefix { - let txid_prefix = HashPrefix::try_from(&prev.txid[..HASH_PREFIX_LEN]).unwrap(); - let value = u64::from_be_bytes(txid_prefix); - let value = value.wrapping_add(prev.vout.into()); - value.to_be_bytes() -} - -pub(crate) struct SpendingPrefixRow; - -impl SpendingPrefixRow { - pub(crate) fn scan_prefix(outpoint: OutPoint) -> HashPrefix { - spending_prefix(outpoint) - } - - pub(crate) fn row(outpoint: OutPoint, height: usize) -> HashPrefixRow { - HashPrefixRow { - prefix: spending_prefix(outpoint), - height: Height::try_from(height).expect("invalid height"), - } - } -} - -// *************************************************************************** - -fn txid_prefix(txid: &Txid) -> HashPrefix { - let mut prefix = [0u8; HASH_PREFIX_LEN]; - prefix.copy_from_slice(&txid[..HASH_PREFIX_LEN]); - prefix -} - -pub(crate) struct TxidRow; - -impl TxidRow { - pub(crate) fn scan_prefix(txid: Txid) -> HashPrefix { - txid_prefix(&txid) - } - - pub(crate) fn row(txid: Txid, height: usize) -> HashPrefixRow { - HashPrefixRow { - prefix: txid_prefix(&txid), - height: Height::try_from(height).expect("invalid height"), - } - } -} - -// *************************************************************************** - -pub(crate) type SerializedHeaderRow = [u8; HEADER_ROW_SIZE]; - -#[derive(Debug, Serialize, Deserialize)] -pub(crate) struct HeaderRow { - pub(crate) header: BlockHeader, -} - -pub const HEADER_ROW_SIZE: usize = 80; - -impl_consensus_encoding!(HeaderRow, header); - -impl HeaderRow { - pub(crate) fn new(header: BlockHeader) -> Self { - Self { header } - } - - pub(crate) fn to_db_row(&self) -> SerializedHeaderRow { - let mut row = [0; HEADER_ROW_SIZE]; - let len = self - .consensus_encode(&mut (&mut row as &mut [u8])) - .expect("in-memory writers don't error"); - debug_assert_eq!(len, HEADER_ROW_SIZE); - row - } - - pub(crate) fn from_db_row(row: SerializedHeaderRow) -> Self { - deserialize(&row).expect("bad HeaderRow") - } -} - -pub(crate) fn bsl_txid(tx: &bsl::Transaction) -> Txid { - bitcoin::Txid::from_slice(tx.txid_sha2().as_slice()).expect("invalid txid") -} - -#[cfg(test)] -mod tests { - use crate::types::{spending_prefix, HashPrefixRow, ScriptHash, ScriptHashRow, TxidRow}; - use bitcoin::{Address, OutPoint, Txid}; - use hex_lit::hex; - use serde_json::{from_str, json}; - - use std::str::FromStr; - - #[test] - fn test_scripthash_serde() { - let hex = "\"4b3d912c1523ece4615e91bf0d27381ca72169dbf6b1c2ffcc9f92381d4984a3\""; - let scripthash: ScriptHash = from_str(hex).unwrap(); - assert_eq!(format!("\"{}\"", scripthash), hex); - assert_eq!(json!(scripthash).to_string(), hex); - } - - #[test] - fn test_scripthash_row() { - let hex = "\"4b3d912c1523ece4615e91bf0d27381ca72169dbf6b1c2ffcc9f92381d4984a3\""; - let scripthash: ScriptHash = from_str(hex).unwrap(); - let row1 = ScriptHashRow::row(scripthash, 123456); - let db_row = row1.to_db_row(); - assert_eq!(db_row, hex!("a384491d38929fcc40e20100")); - let row2 = HashPrefixRow::from_db_row(db_row); - assert_eq!(row1, row2); - } - - #[test] - fn test_scripthash() { - let addr = Address::from_str("1KVNjD3AAnQ3gTMqoTKcWFeqSFujq9gTBT") - .unwrap() - .assume_checked(); - let scripthash = ScriptHash::new(&addr.script_pubkey()); - assert_eq!( - scripthash, - "00dfb264221d07712a144bda338e89237d1abd2db4086057573895ea2659766a" - .parse() - .unwrap() - ); - } - - #[test] - fn test_txid1_prefix() { - // duplicate txids from BIP-30 - let hex = "d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599"; - let txid = Txid::from_str(hex).unwrap(); - - let row1 = TxidRow::row(txid, 91812); - let row2 = TxidRow::row(txid, 91842); - - assert_eq!(row1.to_db_row(), hex!("9985d82954e10f22a4660100")); - assert_eq!(row2.to_db_row(), hex!("9985d82954e10f22c2660100")); - } - - #[test] - fn test_txid2_prefix() { - // duplicate txids from BIP-30 - let hex = "e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468"; - let txid = Txid::from_str(hex).unwrap(); - - let row1 = TxidRow::row(txid, 91722); - let row2 = TxidRow::row(txid, 91880); - - // low-endian encoding => rows should be sorted according to block height - assert_eq!(row1.to_db_row(), hex!("68b45f58b674e94e4a660100")); - assert_eq!(row2.to_db_row(), hex!("68b45f58b674e94ee8660100")); - } - - #[test] - fn test_spending_prefix() { - let txid = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" - .parse() - .unwrap(); - - assert_eq!( - spending_prefix(OutPoint { txid, vout: 0 }), - [31, 30, 29, 28, 27, 26, 25, 24] - ); - assert_eq!( - spending_prefix(OutPoint { txid, vout: 10 }), - [31, 30, 29, 28, 27, 26, 25, 34] - ); - assert_eq!( - spending_prefix(OutPoint { txid, vout: 255 }), - [31, 30, 29, 28, 27, 26, 26, 23] - ); - assert_eq!( - spending_prefix(OutPoint { txid, vout: 256 }), - [31, 30, 29, 28, 27, 26, 26, 24] - ); - } +/// The different authentication methods for the client. +#[derive(Clone, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] +pub enum Auth { + UserPass(String, String), + CookieFile(std::path::PathBuf), } diff --git a/tests/run.sh b/tests/run.sh index e19b4d2..3a75f89 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -6,7 +6,7 @@ mkdir -p data/{bitcoin,electrum,electrs} cleanup() { trap - SIGTERM SIGINT - set +eo pipefail + set +e +o pipefail jobs for j in `jobs -rp` do @@ -16,6 +16,15 @@ cleanup() { } trap cleanup SIGINT SIGTERM EXIT +wait_for() { + CMD=$1 + shift + TEST_ARGS=$* + for _ in `seq 0 9`; do + test "$($CMD | jq -c .)" $TEST_ARGS && break || sleep 2 + done +} + BTC="bitcoin-cli -regtest -datadir=data/bitcoin" ELECTRUM="electrum --regtest" EL="$ELECTRUM --wallet=data/electrum/wallet" @@ -25,7 +34,7 @@ tail_log() { } echo "Starting $(bitcoind -version | head -n1)..." -bitcoind -regtest -datadir=data/bitcoin -printtoconsole=0 & +bitcoind -rest -regtest -datadir=data/bitcoin -printtoconsole=0 & BITCOIND_PID=$! $BTC -rpcwait getblockcount > /dev/null @@ -60,7 +69,7 @@ $EL load_wallet echo "Running integration tests:" echo " * getbalance" -test "`$EL getbalance | jq -c .`" == '{"confirmed":"550","unmatured":"4950"}' +wait_for "$EL getbalance" == '{"confirmed":"550","unmatured":"4950"}' echo " * getunusedaddress" NEW_ADDR=`$EL getunusedaddress` @@ -69,7 +78,7 @@ echo " * payto & broadcast" TXID=$($EL broadcast $($EL payto $NEW_ADDR 123 --fee 0.001 --password='')) echo " * get_tx_status" -test "`$EL get_tx_status $TXID | jq -c .`" == '{"confirmations":0}' +wait_for "$EL get_tx_status $TXID" == '{"confirmations":0}' echo " * getaddresshistory" test "`$EL getaddresshistory $NEW_ADDR | jq -c .`" == "[{\"fee\":100000,\"height\":0,\"tx_hash\":\"$TXID\"}]"