GlcoinCore/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2026-08-13 12:33:11 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge branch 'master' into nymkappa/accel-count-dashboard

Browse source
This commit is contained in:
nymkappa 2025-12-05 08:20:14 +01:00
commit ebcfa0cde6
No known key found for this signature in database
GPG key ID: A7895A957BA87DBA
619 changed files with 127209 additions and 65960 deletions
Download patch file Download diff file Expand all files Collapse all files

145
.github/workflows/backend-integration.yml vendored Normal file
View file

@ -0,0 +1,145 @@
name: Backend Integration Tests with MariaDB
on:
pull_request:
types: [opened, review_requested, synchronize]
push:
branches:
- master
jobs:
backend-integration:
if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
strategy:
matrix:
node: ["22.14.0"]
fail-fast: false
runs-on: ubuntu-latest
name: Backend Integration Tests - node ${{ matrix.node }}
steps:
- name: Checkout
uses: actions/checkout@v3
with:
path: ${{ matrix.node }}/integration
- name: Setup Node
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node }}
registry-url: "https://registry.npmjs.org"
cache: 'npm'
cache-dependency-path: '${{ matrix.node }}/integration/backend/package-lock.json'
- name: Cache node modules
uses: actions/cache@v4
with:
path: ${{ matrix.node }}/integration/backend/node_modules
key: ${{ runner.os }}-backend-integration-node-${{ matrix.node }}-${{ hashFiles('${{ matrix.node }}/integration/backend/package-lock.json') }}
restore-keys: |
${{ runner.os }}-backend-integration-node-${{ matrix.node }}-
${{ runner.os }}-backend-integration-
- name: Read rust-toolchain file from repository
id: gettoolchain
run: echo "::set-output name=toolchain::$(cat ./rust/gbt/rust-toolchain)"
working-directory: ${{ matrix.node }}/integration
- name: Cache Rust dependencies
uses: actions/cache@v4
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
${{ matrix.node }}/integration/rust/gbt/target/
key: ${{ runner.os }}-cargo-integration-${{ hashFiles('${{ matrix.node }}/integration/rust/gbt/**/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-integration-
${{ runner.os }}-cargo-
- name: Install ${{ steps.gettoolchain.outputs.toolchain }} Rust toolchain
uses: dtolnay/rust-toolchain@6d653acede28d24f02e3cd41383119e8b1b35921
with:
toolchain: ${{ steps.gettoolchain.outputs.toolchain }}
- name: Install dependencies
run: npm ci
working-directory: ${{ matrix.node }}/integration/backend
- name: Build backend
run: npm run build
working-directory: ${{ matrix.node }}/integration/backend
- name: Verify config file exists
run: |
ls -la mempool-config.test.json
echo "Current directory: ${PWD}"
echo "Config file will be: ${{ github.workspace }}/${{ matrix.node }}/integration/backend/mempool-config.test.json"
test -f mempool-config.test.json || (echo "ERROR: Config file not found!" && exit 1)
working-directory: ${{ matrix.node }}/integration/backend
- name: Run integration tests (DB auto-starts via Jest)
run: |
echo "MEMPOOL_CONFIG_FILE=$MEMPOOL_CONFIG_FILE"
npm run test:integration
working-directory: ${{ matrix.node }}/integration/backend
env:
MEMPOOL_CONFIG_FILE: ${{ github.workspace }}/${{ matrix.node }}/integration/backend/mempool-config.test.json
- name: Start MariaDB for server test
run: docker compose -f docker-compose.test.yml up -d
working-directory: ${{ matrix.node }}/integration/backend
- name: Wait for MariaDB
run: |
echo "Waiting for MariaDB to be ready..."
for i in {1..30}; do
if docker compose -f docker-compose.test.yml exec -T db-test mysqladmin ping -h localhost -u mempool_test -pmempool_test --silent 2>/dev/null; then
echo "MariaDB is ready!"
break
fi
echo "Attempt $i/30..."
sleep 2
done
sleep 3
working-directory: ${{ matrix.node }}/integration/backend
- name: Start backend server and verify connectivity
run: |
# Start server in background
node dist/index.js &
SERVER_PID=$!
# Wait for server to start
echo "Waiting for server to start..."
sleep 10
# Check if server is still running
if ps -p $SERVER_PID > /dev/null 2>&1; then
echo "Server started successfully and connected to database!"
kill $SERVER_PID 2>/dev/null || true
wait $SERVER_PID 2>/dev/null || true
exit 0
else
echo "Server failed to start or exited prematurely"
exit 1
fi
working-directory: ${{ matrix.node }}/integration/backend
env:
MEMPOOL_CONFIG_FILE: ${{ github.workspace }}/${{ matrix.node }}/integration/backend/mempool-config.test.json
- name: Cleanup containers
if: always()
run: docker compose -f docker-compose.test.yml down -v
working-directory: ${{ matrix.node }}/integration/backend
- name: Display logs on failure
if: failure()
run: |
echo "=== MariaDB logs ==="
docker compose -f docker-compose.test.yml logs db-test || true
working-directory: ${{ matrix.node }}/integration/backend

97
.github/workflows/ci.yml vendored
View file

@ -3,16 +3,19 @@ name: CI Pipeline for the Backend and Frontend
on:
pull_request:
types: [opened, review_requested, synchronize]
push:
branches:
- master
jobs:
backend:
if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
strategy:
matrix:
node: ["20", "21"]
node: ["22.14.0"]
flavor: ["dev", "prod"]
fail-fast: false
runs-on: "ubuntu-latest"
runs-on: ubuntu-latest
name: Backend (${{ matrix.flavor }}) - node ${{ matrix.node }}
steps:
@ -26,16 +29,41 @@ jobs:
with:
node-version: ${{ matrix.node }}
registry-url: "https://registry.npmjs.org"
cache: 'npm'
cache-dependency-path: '${{ matrix.node }}/${{ matrix.flavor }}/backend/package-lock.json'
- name: Cache node modules
uses: actions/cache@v4
with:
path: ${{ matrix.node }}/${{ matrix.flavor }}/backend/node_modules
key: ${{ runner.os }}-backend-${{ matrix.flavor }}-node-${{ matrix.node }}-${{ hashFiles('${{ matrix.node }}/${{ matrix.flavor }}/backend/package-lock.json') }}
restore-keys: |
${{ runner.os }}-backend-${{ matrix.flavor }}-node-${{ matrix.node }}-
${{ runner.os }}-backend-${{ matrix.flavor }}-
- name: Read rust-toolchain file from repository
id: gettoolchain
run: echo "::set-output name=toolchain::$(cat ./rust/gbt/rust-toolchain)"
working-directory: ${{ matrix.node }}/${{ matrix.flavor }}
- name: Cache Rust dependencies
uses: actions/cache@v4
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
${{ matrix.node }}/${{ matrix.flavor }}/rust/gbt/target/
key: ${{ runner.os }}-cargo-${{ matrix.flavor }}-${{ hashFiles('${{ matrix.node }}/${{ matrix.flavor }}/rust/gbt/**/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-${{ matrix.flavor }}-
${{ runner.os }}-cargo-
- name: Install ${{ steps.gettoolchain.outputs.toolchain }} Rust toolchain
# Latest version available on this commit is 1.71.1
# Commit date is Aug 3, 2023
uses: dtolnay/rust-toolchain@d8352f6b1d2e870bc5716e7a6d9b65c4cc244a1a
uses: dtolnay/rust-toolchain@6d653acede28d24f02e3cd41383119e8b1b35921
with:
toolchain: ${{ steps.gettoolchain.outputs.toolchain }}
@ -66,7 +94,10 @@ jobs:
cache:
name: "Cache assets for builds"
runs-on: "ubuntu-latest"
strategy:
matrix:
node: ["22.14.0"]
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
@ -78,6 +109,17 @@ jobs:
with:
node-version: ${{ matrix.node }}
registry-url: "https://registry.npmjs.org"
cache: 'npm'
cache-dependency-path: 'assets/frontend/package-lock.json'
- name: Cache node modules for frontend
uses: actions/cache@v4
with:
path: assets/frontend/node_modules
key: ${{ runner.os }}-cache-frontend-node-${{ matrix.node }}-${{ hashFiles('assets/frontend/package-lock.json') }}
restore-keys: |
${{ runner.os }}-cache-frontend-node-${{ matrix.node }}-
${{ runner.os }}-cache-frontend-
- name: Install (Prod dependencies only)
run: npm ci --omit=dev --omit=optional
@ -157,13 +199,13 @@ jobs:
frontend:
needs: cache
if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
strategy:
matrix:
node: ["20", "21"]
node: ["22.14.0"]
flavor: ["dev", "prod"]
fail-fast: false
runs-on: "ubuntu-latest"
runs-on: ubuntu-latest
name: Frontend (${{ matrix.flavor }}) - node ${{ matrix.node }}
steps:
@ -177,6 +219,17 @@ jobs:
with:
node-version: ${{ matrix.node }}
registry-url: "https://registry.npmjs.org"
cache: 'npm'
cache-dependency-path: '${{ matrix.node }}/${{ matrix.flavor }}/frontend/package-lock.json'
- name: Cache node modules
uses: actions/cache@v4
with:
path: ${{ matrix.node }}/${{ matrix.flavor }}/frontend/node_modules
key: ${{ runner.os }}-frontend-${{ matrix.flavor }}-node-${{ matrix.node }}-${{ hashFiles('${{ matrix.node }}/${{ matrix.flavor }}/frontend/package-lock.json') }}
restore-keys: |
${{ runner.os }}-frontend-${{ matrix.flavor }}-node-${{ matrix.node }}-
${{ runner.os }}-frontend-${{ matrix.flavor }}-
- name: Install (Prod dependencies only)
run: npm ci --omit=dev --omit=optional
@ -245,8 +298,8 @@ jobs:
VERBOSE: 1
e2e:
if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
runs-on: "ubuntu-latest"
if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
runs-on: ubuntu-latest
needs: frontend
strategy:
fail-fast: false
@ -263,10 +316,19 @@ jobs:
- name: Setup node
uses: actions/setup-node@v3
with:
node-version: 20
node-version: 22
cache: "npm"
cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json
- name: Cache node modules for e2e
uses: actions/cache@v4
with:
path: ${{ matrix.module }}/frontend/node_modules
key: ${{ runner.os }}-e2e-${{ matrix.module }}-node-22-${{ hashFiles('${{ matrix.module }}/frontend/package-lock.json') }}
restore-keys: |
${{ runner.os }}-e2e-${{ matrix.module }}-node-22-
${{ runner.os }}-e2e-${{ matrix.module }}-
- name: Restore cached mining pool assets
continue-on-error: true
id: cache-mining-pool-restore
@ -309,7 +371,7 @@ jobs:
tag: ${{ github.event_name }}
working-directory: ${{ matrix.module }}/frontend
build: npm run config:defaults:${{ matrix.module }}
start: npm run start:local-staging
start: npm run start:parameterized
wait-on: "http://localhost:4200"
wait-on-timeout: 120
record: true
@ -334,7 +396,7 @@ jobs:
tag: ${{ github.event_name }}
working-directory: ${{ matrix.module }}/frontend
build: npm run config:defaults:${{ matrix.module }}
start: npm run start:local-staging
start: npm run start:parameterized
wait-on: "http://localhost:4200"
wait-on-timeout: 120
record: true
@ -359,7 +421,7 @@ jobs:
tag: ${{ github.event_name }}
working-directory: ${{ matrix.module }}/frontend
build: npm run config:defaults:mempool
start: npm run start:local-staging
start: npm run start:parameterized
wait-on: "http://localhost:4200"
wait-on-timeout: 120
record: true
@ -375,10 +437,9 @@ jobs:
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
validate_docker_json:
if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
runs-on: "ubuntu-latest"
if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
runs-on: ubuntu-latest
name: Validate generated backend Docker JSON
steps:
@ -403,4 +464,4 @@ jobs:
- name: Validate JSON syntax
run: |
cat mempool-config.json | jq
working-directory: docker/docker/backend
working-directory: docker/docker/backend

180
.github/workflows/docker.yml vendored Normal file
View file

@ -0,0 +1,180 @@
name: Docker build on tag
env:
DOCKER_CLI_EXPERIMENTAL: enabled
TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$"
DOCKER_BUILDKIT: 1 # Enable BuildKit for better performance
COMPOSE_DOCKER_CLI_BUILD: 0
on:
push:
tags:
- v[0-9]+.[0-9]+.[0-9]+
- v[0-9]+.[0-9]+.[0-9]+-*
pull_request:
types: [opened, synchronize, reopened, labeled]
permissions:
contents: read
jobs:
build:
# Run on tag pushes OR on PRs that have the "docker" label
if: |
github.event_name == 'push' ||
(github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'docker'))
strategy:
matrix:
service:
- frontend
- backend
runs-on: ubuntu-latest
timeout-minutes: 120
name: Build and push to DockerHub
outputs:
image-digest-frontend: ${{ matrix.service == 'frontend' && steps.docker-build.outputs.digest || '' }}
image-digest-backend: ${{ matrix.service == 'backend' && steps.docker-build.outputs.digest || '' }}
steps:
- name: Replace the current swap file
shell: bash
run: |
sudo swapoff /mnt/swapfile || true
sudo rm -f /mnt/swapfile
sudo fallocate -l 16G /mnt/swapfile
sudo chmod 600 /mnt/swapfile
sudo mkswap /mnt/swapfile
sudo swapon /mnt/swapfile
- name: Show current memory and swap status
shell: bash
run: |
sudo free -h
echo
sudo swapon --show
- name: Mount a tmpfs over /var/lib/docker
shell: bash
run: |
if [ ! -d "/var/lib/docker" ]; then
echo "Directory '/var/lib/docker' not found"
exit 1
fi
sudo mount -t tmpfs -o size=12G tmpfs /var/lib/docker
sudo systemctl restart docker
sudo df -h | grep docker
# Only for tag pushes: use the Git tag as TAG
- name: Set TAG from pushed tag
if: github.event_name == 'push'
run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
- name: Add SHORT_SHA env property with commit short sha
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
SHA="${{ github.event.pull_request.head.sha }}"
else
SHA="${GITHUB_SHA}"
fi
echo "SHORT_SHA=${SHA:0:8}" >> $GITHUB_ENV
- name: Login to Docker for building
run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
- name: Checkout project
uses: actions/checkout@v4
# For PRs: use package.json version + short sha as TAG
- name: Set TAG from service package.json for pull requests
if: github.event_name == 'pull_request'
run: |
if [ "${{ matrix.service }}" = "frontend" ]; then
VERSION=$(jq -r '.version' frontend/package.json)
else
VERSION=$(jq -r '.version' backend/package.json)
fi
echo "TAG=v${VERSION}-${SHORT_SHA}" >> $GITHUB_ENV
- name: Show set environment variables
run: |
printf " TAG: %s\n" "$TAG"
printf " SHORT_SHA: %s\n" "$SHORT_SHA"
- name: Init repo for Dockerization
run: docker/init.sh "$TAG"
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: linux/amd64,linux/arm64
id: qemu
- name: Setup Docker buildx action
uses: docker/setup-buildx-action@v3
with:
platforms: linux/amd64,linux/arm64
driver-opts: |
network=host
id: buildx
- name: Available platforms
run: echo ${{ steps.buildx.outputs.platforms }}
- name: Cache Docker layers
uses: actions/cache@v3
id: cache
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ matrix.service }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-${{ matrix.service }}-
- name: Run Docker buildx for ${{ matrix.service }} against tag
id: docker-build
run: |
docker buildx build \
--cache-from "type=local,src=/tmp/.buildx-cache" \
--cache-to "type=local,dest=/tmp/.buildx-cache,mode=max" \
--platform linux/amd64,linux/arm64 \
--tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:$TAG \
--build-context rustgbt=./rust \
--build-context backend=./backend \
--output "type=registry,push=true" \
--build-arg commitHash=$SHORT_SHA \
./${{ matrix.service }}/
tag-latest:
needs: build
# Only for successful *tag pushes* and only for "plain" versions (no '-')
if: ${{ needs.build.result == 'success' && github.event_name == 'push' && !contains(github.ref_name, '-') }}
runs-on: ubuntu-latest
timeout-minutes: 30
name: Tag release build as latest
strategy:
matrix:
service:
- frontend
- backend
steps:
- name: Set env variables
run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: linux/amd64,linux/arm64
- name: Setup Docker buildx action
uses: docker/setup-buildx-action@v3
with:
platforms: linux/amd64,linux/arm64
driver-opts: |
network=host
- name: Login to Docker Hub
run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
- name: Tag as latest for ${{ matrix.service }}
run: |
docker buildx imagetools create \
--tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:latest \
${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:$TAG

273
.github/workflows/e2e_parameterized.yml vendored Normal file
View file

@ -0,0 +1,273 @@
name: 'Parameterized e2e tests'
on:
workflow_dispatch:
inputs:
ref:
description: 'Branch name or Pull Request number (e.g., master or 6102)'
required: true
default: 'master'
type: string
mempool_hostname:
description: 'Mempool Hostname'
required: true
default: 'mempool.space'
type: string
liquid_hostname:
description: 'Liquid Hostname'
required: true
default: 'liquid.network'
type: string
jobs:
cache:
name: "Cache assets for builds"
runs-on: ubuntu-latest
steps:
- name: Determine checkout ref
id: determine-ref
run: |
REF_INPUT="${{ github.event.inputs.ref }}"
if [[ "$REF_INPUT" =~ ^[0-9]+$ ]]; then
echo "ref=refs/pull/$REF_INPUT/head" >> $GITHUB_OUTPUT
else
echo "ref=$REF_INPUT" >> $GITHUB_OUTPUT
fi
- name: Checkout
uses: actions/checkout@v3
with:
ref: ${{ steps.determine-ref.outputs.ref }}
path: assets
- name: Setup Node
uses: actions/setup-node@v3
with:
node-version: 22.14.0
registry-url: "https://registry.npmjs.org"
- name: Install (Prod dependencies only)
run: npm ci --omit=dev --omit=optional
working-directory: assets/frontend
- name: Restore cached mining pool assets
continue-on-error: true
id: cache-mining-pool-restore
uses: actions/cache/restore@v4
with:
path: |
mining-pool-assets.zip
key: mining-pool-assets-cache
- name: Restore promo video assets
continue-on-error: true
id: cache-promo-video-restore
uses: actions/cache/restore@v4
with:
path: |
promo-video-assets.zip
key: promo-video-assets-cache
- name: Unzip assets before building (src/resources)
continue-on-error: true
run: unzip -o mining-pool-assets.zip -d assets/frontend/src/resources/mining-pools
- name: Unzip assets before building (src/resources)
continue-on-error: true
run: unzip -o promo-video-assets.zip -d assets/frontend/src/resources/promo-video
# - name: Unzip assets before building (dist)
# continue-on-error: true
# run: unzip assets.zip -d assets/frontend/dist/mempool/browser/resources
- name: Sync-assets
run: npm run sync-assets-dev
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
MEMPOOL_CDN: 1
VERBOSE: 1
working-directory: assets/frontend
- name: Zip mining-pool assets
run: zip -jrq mining-pool-assets.zip assets/frontend/src/resources/mining-pools/*
- name: Zip promo-video assets
run: zip -jrq promo-video-assets.zip assets/frontend/src/resources/promo-video/*
- name: Upload mining pool assets as artifact
uses: actions/upload-artifact@v4
with:
name: mining-pool-assets
path: mining-pool-assets.zip
- name: Upload promo video assets as artifact
uses: actions/upload-artifact@v4
with:
name: promo-video-assets
path: promo-video-assets.zip
- name: Save mining pool assets cache
id: cache-mining-pool-save
uses: actions/cache/save@v4
with:
path: |
mining-pool-assets.zip
key: mining-pool-assets-cache
- name: Save promo video assets cache
id: cache-promo-video-save
uses: actions/cache/save@v4
with:
path: |
promo-video-assets.zip
key: promo-video-assets-cache
e2e:
runs-on: ubuntu-latest
needs: cache
strategy:
fail-fast: false
matrix:
module: ["mempool", "liquid", "testnet4"]
name: E2E tests for ${{ matrix.module }}
steps:
- name: Determine checkout ref
id: determine-ref
run: |
REF_INPUT="${{ github.event.inputs.ref }}"
if [[ "$REF_INPUT" =~ ^[0-9]+$ ]]; then
echo "ref=refs/pull/$REF_INPUT/head" >> $GITHUB_OUTPUT
else
echo "ref=$REF_INPUT" >> $GITHUB_OUTPUT
fi
- name: Checkout
uses: actions/checkout@v3
with:
ref: ${{ steps.determine-ref.outputs.ref }}
path: ${{ matrix.module }}
- name: Setup node
uses: actions/setup-node@v3
with:
node-version: 22.14.0
cache: "npm"
cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json
- name: Restore cached mining pool assets
continue-on-error: true
id: cache-mining-pool-restore
uses: actions/cache/restore@v4
with:
path: |
mining-pool-assets.zip
key: mining-pool-assets-cache
- name: Restore cached promo video assets
continue-on-error: true
id: cache-promo-video-restore
uses: actions/cache/restore@v4
with:
path: |
promo-video-assets.zip
key: promo-video-assets-cache
- name: Download artifact
uses: actions/download-artifact@v4
with:
name: mining-pool-assets
- name: Unzip assets before building (src/resources)
run: unzip -o mining-pool-assets.zip -d ${{ matrix.module }}/frontend/src/resources/mining-pools
- name: Download artifact
uses: actions/download-artifact@v4
with:
name: promo-video-assets
- name: Unzip assets before building (src/resources)
run: unzip -o promo-video-assets.zip -d ${{ matrix.module }}/frontend/src/resources/promo-video
# mempool
- name: Chrome browser tests (${{ matrix.module }})
if: ${{ matrix.module == 'mempool' }}
uses: cypress-io/github-action@v5
with:
tag: ${{ github.event_name }}
working-directory: ${{ matrix.module }}/frontend
build: npm run config:defaults:${{ matrix.module }}
start: npm run start:parameterized
wait-on: "http://localhost:4200"
wait-on-timeout: 120
record: true
parallel: true
spec: |
cypress/e2e/mainnet/*.spec.ts
cypress/e2e/signet/*.spec.ts
group: Tests on Chrome (${{ matrix.module }})
browser: "chrome"
ci-build-id: "${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}"
env:
COMMIT_INFO_MESSAGE: ${{ github.event_name }} (${{ github.sha }}) - ref ${{ github.event.inputs.ref }} - ${{ github.event.inputs.mempool_hostname }} - ${{ github.event.inputs.liquid_hostname }}
MEMPOOL_HOSTNAME: ${{ github.event.inputs.mempool_hostname }}
LIQUID_HOSTNAME: ${{ github.event.inputs.liquid_hostname }}
MEMPOOL_CI_API_KEY: ${{ secrets.MEMPOOL_CI_API_KEY }}
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
# liquid
- name: Chrome browser tests (${{ matrix.module }})
if: ${{ matrix.module == 'liquid' }}
uses: cypress-io/github-action@v5
with:
tag: ${{ github.event_name }}
working-directory: ${{ matrix.module }}/frontend
build: npm run config:defaults:${{ matrix.module }}
start: npm run start:parameterized
wait-on: "http://localhost:4200"
wait-on-timeout: 120
record: true
parallel: true
spec: |
cypress/e2e/liquid/liquid.spec.ts
cypress/e2e/liquidtestnet/liquidtestnet.spec.ts
group: Tests on Chrome (${{ matrix.module }})
browser: "chrome"
ci-build-id: "${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}"
env:
COMMIT_INFO_MESSAGE: ${{ github.event_name }} (${{ github.sha }}) - ref ${{ github.event.inputs.ref }} - ${{ github.event.inputs.mempool_hostname }} - ${{ github.event.inputs.liquid_hostname }}
MEMPOOL_HOSTNAME: ${{ github.event.inputs.mempool_hostname }}
LIQUID_HOSTNAME: ${{ github.event.inputs.liquid_hostname }}
MEMPOOL_CI_API_KEY: ${{ secrets.MEMPOOL_CI_API_KEY }}
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
# testnet
- name: Chrome browser tests (${{ matrix.module }})
if: ${{ matrix.module == 'testnet4' }}
uses: cypress-io/github-action@v5
with:
tag: ${{ github.event_name }}
working-directory: ${{ matrix.module }}/frontend
build: npm run config:defaults:mempool
start: npm run start:parameterized
wait-on: "http://localhost:4200"
wait-on-timeout: 120
record: true
parallel: true
spec: |
cypress/e2e/testnet4/*.spec.ts
group: Tests on Chrome (${{ matrix.module }})
browser: "chrome"
ci-build-id: "${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}"
env:
COMMIT_INFO_MESSAGE: ${{ github.event_name }} (${{ github.sha }}) - ref ${{ github.event.inputs.ref }} - ${{ github.event.inputs.mempool_hostname }} - ${{ github.event.inputs.liquid_hostname }}
MEMPOOL_HOSTNAME: ${{ github.event.inputs.mempool_hostname }}
LIQUID_HOSTNAME: ${{ github.event.inputs.liquid_hostname }}
MEMPOOL_CI_API_KEY: ${{ secrets.MEMPOOL_CI_API_KEY }}
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}

2
.github/workflows/get_backend_block_height.yml vendored
View file

@ -4,7 +4,7 @@ on: [workflow_dispatch]
jobs:
print-backend-sha:
runs-on: 'ubuntu-latest'
runs-on: ubuntu-latest
name: Get block height
steps:
- name: Checkout

2
.github/workflows/get_backend_hash.yml vendored
View file

@ -4,7 +4,7 @@ on: [workflow_dispatch]
jobs:
print-backend-sha:
runs-on: 'ubuntu-latest'
runs-on: ubuntu-latest
name: Print backend hashes
steps:
- name: Checkout

2
.github/workflows/get_image_digest.yml vendored
View file

@ -10,7 +10,7 @@ on:
type: string
jobs:
print-images-sha:
runs-on: 'ubuntu-latest'
runs-on: ubuntu-latest
name: Print digest for images
steps:
- name: Checkout

107
.github/workflows/on-tag.yml vendored
View file

@ -1,107 +0,0 @@
name: Docker build on tag
env:
DOCKER_CLI_EXPERIMENTAL: enabled
TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$"
DOCKER_BUILDKIT: 0
COMPOSE_DOCKER_CLI_BUILD: 0
on:
push:
tags:
- v[0-9]+.[0-9]+.[0-9]+
- v[0-9]+.[0-9]+.[0-9]+-*
permissions:
contents: read
jobs:
build:
strategy:
matrix:
service:
- frontend
- backend
runs-on: ubuntu-latest
timeout-minutes: 120
name: Build and push to DockerHub
steps:
# Workaround based on JonasAlfredsson/docker-on-tmpfs@v1.0.1
- name: Replace the current swap file
shell: bash
run: |
sudo swapoff /mnt/swapfile
sudo rm -v /mnt/swapfile
sudo fallocate -l 13G /mnt/swapfile
sudo chmod 600 /mnt/swapfile
sudo mkswap /mnt/swapfile
sudo swapon /mnt/swapfile
- name: Show current memory and swap status
shell: bash
run: |
sudo free -h
echo
sudo swapon --show
- name: Mount a tmpfs over /var/lib/docker
shell: bash
run: |
if [ ! -d "/var/lib/docker" ]; then
echo "Directory '/var/lib/docker' not found"
exit 1
fi
sudo mount -t tmpfs -o size=10G tmpfs /var/lib/docker
sudo systemctl restart docker
sudo df -h | grep docker
- name: Set env variables
run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
- name: Show set environment variables
run: |
printf " TAG: %s\n" "$TAG"
- name: Add SHORT_SHA env property with commit short sha
run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV
- name: Login to Docker for building
run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
- name: Checkout project
uses: actions/checkout@v4
- name: Init repo for Dockerization
run: docker/init.sh "$TAG"
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
id: qemu
- name: Setup Docker buildx action
uses: docker/setup-buildx-action@v3
id: buildx
- name: Available platforms
run: echo ${{ steps.buildx.outputs.platforms }}
- name: Cache Docker layers
uses: actions/cache@v3
id: cache
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
- name: Run Docker buildx for ${{ matrix.service }} against tag
run: |
docker buildx build \
--cache-from "type=local,src=/tmp/.buildx-cache" \
--cache-to "type=local,dest=/tmp/.buildx-cache" \
--platform linux/amd64,linux/arm64 \
--tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:$TAG \
--tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:latest \
--build-context rustgbt=./rust \
--build-context backend=./backend \
--output "type=registry" ./${{ matrix.service }}/ \
--build-arg commitHash=$SHORT_SHA

2
.nvmrc
View file

@ -1 +1 @@
v20.8.0
v22

6
LICENSE
View file

@ -1,5 +1,5 @@
The Mempool Open Source Project®
Copyright (c) 2019-2024 Mempool Space K.K. and other shadowy super-coders
Copyright (c) 2019-2025 Mempool Space K.K. and other shadowy super-coders
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU Affero General Public License as published by the Free
@ -10,8 +10,8 @@ However, this copyright license does not include an implied right or license
to use any trademarks, service marks, logos, or trade names of Mempool Space K.K.
or any other contributor to The Mempool Open Source Project.
The Mempool Open Source Project®, Mempool Accelerator™, Mempool Enterprise®,
Mempool Liquidity™, mempool.space®, Be your own explorer™, Explore the full
The Mempool Open Source Project®, Mempool Accelerator®, Mempool Enterprise®,
Mempool Wallet™, mempool.space®, Be your own explorer™, Explore the full
Bitcoin ecosystem™, Mempool Goggles™, the mempool Logo, the mempool Square Logo,
the mempool block visualization Logo, the mempool Blocks Logo, the mempool
transaction Logo, the mempool Blocks 3 | 2 Logo, the mempool research Logo,

20
backend/docker-compose.test.yml Normal file
View file

@ -0,0 +1,20 @@
version: "3.7"
services:
db-test:
image: mariadb:10.5.21
environment:
MYSQL_DATABASE: "mempool_test"
MYSQL_USER: "mempool_test"
MYSQL_PASSWORD: "mempool_test"
MYSQL_ROOT_PASSWORD: "admin"
ports:
- "33306:3306"
healthcheck:
test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "mempool_test", "-pmempool_test"]
interval: 5s
timeout: 5s
retries: 10
tmpfs:
- /var/lib/mysql

6
backend/jest.config.ts
View file

@ -7,7 +7,7 @@ const config: Config.InitialOptions = {
automock: false,
collectCoverage: true,
collectCoverageFrom: ["./src/**/**.ts"],
coverageProvider: "babel",
coverageProvider: "v8",
coverageThreshold: {
global: {
lines: 1
@ -16,5 +16,9 @@ const config: Config.InitialOptions = {
setupFiles: [
"./testSetup.ts",
],
testPathIgnorePatterns: [
"/node_modules/",
"/__integration_tests__/",
],
}
export default config;

21
backend/jest.integration.config.ts Normal file
View file

@ -0,0 +1,21 @@
import type { Config } from "@jest/types"
const config: Config.InitialOptions = {
preset: "ts-jest",
testEnvironment: "node",
verbose: true,
automock: false,
collectCoverage: false,
coverageProvider: "v8",
testMatch: [
"**/__integration_tests__/**/*.test.ts"
],
globalSetup: "./jest.integration.setup.ts", // Start database before all tests
setupFiles: [
"./testSetup.integration.ts",
],
globalTeardown: "./jest.integration.teardown.ts", // Stop database after all tests
maxWorkers: 1, // Force sequential execution
}
export default config;

72
backend/jest.integration.setup.ts Normal file
View file

@ -0,0 +1,72 @@
// Setup that runs BEFORE setupFiles
// This ensures MEMPOOL_CONFIG_FILE is set before any modules are loaded
import * as path from 'path';
import { execSync } from 'child_process';
// Set the config file path if not already set
if (!process.env.MEMPOOL_CONFIG_FILE) {
process.env.MEMPOOL_CONFIG_FILE = path.join(__dirname, 'mempool-config.test.json');
}
// Helper to get docker compose command (v1 or v2)
function getDockerComposeCmd(): string {
try {
execSync('docker compose version', { stdio: 'pipe' });
return 'docker compose';
} catch {
try {
execSync('docker-compose version', { stdio: 'pipe' });
return 'docker-compose';
} catch {
throw new Error('Neither "docker compose" nor "docker-compose" is available');
}
}
}
// Start the Docker test database container
module.exports = async () => {
// Skip if SKIP_DB_SETUP is set (e.g., when test-with-db.sh manages the database)
if (process.env.SKIP_DB_SETUP) {
console.log('Skipping database setup (managed externally)');
return;
}
console.log('Starting test database container...');
try {
const composeFile = path.join(__dirname, 'docker-compose.test.yml');
const dockerComposeCmd = getDockerComposeCmd();
// Start the container
execSync(`${dockerComposeCmd} -f "${composeFile}" up -d`, {
stdio: 'inherit',
cwd: __dirname
});
// Wait for database to be ready
console.log('Waiting for database to be ready...');
let attempts = 0;
const maxAttempts = 30;
while (attempts < maxAttempts) {
try {
execSync(`${dockerComposeCmd} -f "${composeFile}" exec -T db-test mysqladmin ping -h localhost -u mempool_test -pmempool_test --silent`, {
cwd: __dirname,
stdio: 'pipe'
});
console.log('Database is ready!');
break;
} catch (e) {
attempts++;
if (attempts >= maxAttempts) {
throw new Error('Database did not start in time');
}
// Wait 1 second before retrying
await new Promise(resolve => setTimeout(resolve, 1000));
}
}
} catch (error) {
console.error('Failed to start test database:', error instanceof Error ? error.message : error);
throw error;
}
};

102
backend/jest.integration.teardown.ts Normal file
View file

@ -0,0 +1,102 @@
import DB from './src/database';
import logger from './src/logger';
import mempool from './src/api/mempool';
import { execSync } from 'child_process';
import * as path from 'path';
// Helper to get docker compose command (v1 or v2)
function getDockerComposeCmd(): string {
try {
execSync('docker compose version', { stdio: 'pipe' });
return 'docker compose';
} catch {
try {
execSync('docker-compose version', { stdio: 'pipe' });
return 'docker-compose';
} catch {
throw new Error('Neither "docker compose" nor "docker-compose" is available');
}
}
}
module.exports = async () => {
try {
// Final cleanup after all tests
const tables = [
'blocks_audits',
'blocks_summaries',
'blocks_prices',
'blocks_templates',
'cpfp_clusters',
'blocks',
'difficulty_adjustments',
'hashrates',
'prices',
'node_records',
'nodes_sockets',
'nodes',
'lightning_stats',
'transactions',
'elements_pegs',
'federation_txos',
'pools'
];
await DB.query('SET FOREIGN_KEY_CHECKS = 0');
for (const table of tables) {
try {
// Use 'silent' error logging to avoid noise for optional tables that don't exist
await DB.query(`TRUNCATE TABLE ${table}`, [], 'silent');
} catch (e) {
// Table might not exist - silently ignore
}
}
await DB.query('SET FOREIGN_KEY_CHECKS = 1');
logger.info('Integration tests cleanup completed');
// Close the database connection pool to prevent Jest from hanging
await DB.close();
logger.info('Database connection pool closed');
// Clean up singleton resources that have timers or sockets
mempool.destroy();
logger.info('Mempool resources cleaned up');
// Close logger's UDP socket last (after all logging is done)
logger.close();
// Stop and remove the Docker test database container
// Skip if SKIP_DB_TEARDOWN is set (e.g., when test-with-db.sh manages the database)
if (!process.env.SKIP_DB_TEARDOWN) {
try {
const composeFile = path.join(__dirname, 'docker-compose.test.yml');
const dockerComposeCmd = getDockerComposeCmd();
execSync(`${dockerComposeCmd} -f "${composeFile}" down -v`, {
stdio: 'inherit',
cwd: __dirname
});
console.log('Test database container stopped and removed');
} catch (error) {
console.error('Failed to stop Docker container:', error instanceof Error ? error.message : error);
}
} else {
console.log('Skipping Docker cleanup (managed externally)');
}
} catch (error) {
// Use console.error since logger might be closed
console.error('Failed to cleanup after integration tests:', error instanceof Error ? error.message : error);
} finally {
// Ensure we always try to close connections even if cleanup fails
try {
await DB.close();
mempool.destroy();
logger.close();
} catch (e) {
// Ignore errors on close
}
}
};

163
backend/mempool-config.test.json Normal file
View file

@ -0,0 +1,163 @@
{
"MEMPOOL": {
"OFFICIAL": false,
"NETWORK": "mainnet",
"BACKEND": "none",
"ENABLED": true,
"HTTP_PORT": 8998,
"SPAWN_CLUSTER_PROCS": 0,
"API_URL_PREFIX": "/api/v1/",
"POLL_RATE_MS": 2000,
"CACHE_DIR": "./cache",
"CACHE_ENABLED": false,
"CLEAR_PROTECTION_MINUTES": 20,
"RECOMMENDED_FEE_PERCENTILE": 50,
"BLOCK_WEIGHT_UNITS": 4000000,
"INITIAL_BLOCKS_AMOUNT": 8,
"MEMPOOL_BLOCKS_AMOUNT": 8,
"INDEXING_BLOCKS_AMOUNT": 11000,
"BLOCKS_SUMMARIES_INDEXING": false,
"GOGGLES_INDEXING": false,
"USE_SECOND_NODE_FOR_MINFEE": false,
"EXTERNAL_ASSETS": [],
"EXTERNAL_MAX_RETRY": 1,
"EXTERNAL_RETRY_INTERVAL": 0,
"USER_AGENT": "mempool",
"STDOUT_LOG_MIN_PRIORITY": "debug",
"AUTOMATIC_POOLS_UPDATE": false,
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
"POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
"POOLS_UPDATE_DELAY": 604800,
"AUDIT": false,
"RUST_GBT": true,
"LIMIT_GBT": false,
"CPFP_INDEXING": false,
"DISK_CACHE_BLOCK_INTERVAL": 6,
"MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
"ALLOW_UNREACHABLE": true,
"PRICE_UPDATES_PER_HOUR": 1,
"MAX_TRACKED_ADDRESSES": 100,
"UNIX_SOCKET_PATH": ""
},
"CORE_RPC": {
"HOST": "127.0.0.1",
"PORT": 8332,
"USERNAME": "mempool",
"PASSWORD": "mempool",
"TIMEOUT": 60000,
"COOKIE": false,
"COOKIE_PATH": "/path/to/bitcoin/.cookie",
"DEBUG_LOG_PATH": "/path/to/bitcoin/debug.log"
},
"ELECTRUM": {
"HOST": "127.0.0.1",
"PORT": 50002,
"TLS_ENABLED": true
},
"ESPLORA": {
"REST_API_URL": "http://127.0.0.1:3000",
"UNIX_SOCKET_PATH": "/tmp/esplora-bitcoin-mainnet",
"BATCH_QUERY_BASE_SIZE": 1000,
"RETRY_UNIX_SOCKET_AFTER": 30000,
"REQUEST_TIMEOUT": 10000,
"FALLBACK_TIMEOUT": 5000,
"FALLBACK": [],
"MAX_BEHIND_TIP": 2
},
"SECOND_CORE_RPC": {
"HOST": "127.0.0.1",
"PORT": 8332,
"USERNAME": "mempool",
"PASSWORD": "mempool",
"TIMEOUT": 60000,
"COOKIE": false,
"COOKIE_PATH": "/path/to/bitcoin/.cookie"
},
"DATABASE": {
"ENABLED": true,
"HOST": "127.0.0.1",
"PORT": 33306,
"SOCKET": "",
"DATABASE": "mempool_test",
"USERNAME": "mempool_test",
"PASSWORD": "mempool_test",
"TIMEOUT": 180000,
"PID_DIR": ""
},
"SYSLOG": {
"ENABLED": false,
"HOST": "127.0.0.1",
"PORT": 514,
"MIN_PRIORITY": "info",
"FACILITY": "local7"
},
"STATISTICS": {
"ENABLED": true,
"TX_PER_SECOND_SAMPLE_PERIOD": 150
},
"MAXMIND": {
"ENABLED": false,
"GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
"GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
"GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
},
"LIGHTNING": {
"ENABLED": false,
"BACKEND": "lnd",
"STATS_REFRESH_INTERVAL": 600,
"GRAPH_REFRESH_INTERVAL": 600,
"LOGGER_UPDATE_INTERVAL": 30,
"FORENSICS_INTERVAL": 43200,
"FORENSICS_RATE_LIMIT": 20
},
"LND": {
"TLS_CERT_PATH": "tls.cert",
"MACAROON_PATH": "readonly.macaroon",
"REST_API_URL": "https://localhost:8080",
"TIMEOUT": 10000
},
"CLIGHTNING": {
"SOCKET": "lightning-rpc"
},
"SOCKS5PROXY": {
"ENABLED": false,
"USE_ONION": true,
"HOST": "127.0.0.1",
"PORT": 9050,
"USERNAME": "",
"PASSWORD": ""
},
"EXTERNAL_DATA_SERVER": {
"MEMPOOL_API": "https://mempool.space/api/v1",
"MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1",
"LIQUID_API": "https://liquid.network/api/v1",
"LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1"
},
"REDIS": {
"ENABLED": false,
"UNIX_SOCKET_PATH": "/tmp/redis.sock",
"BATCH_QUERY_BASE_SIZE": 5000
},
"REPLICATION": {
"ENABLED": false,
"AUDIT": false,
"AUDIT_START_HEIGHT": 774000,
"STATISTICS": false,
"STATISTICS_START_TIME": 1481932800,
"SERVERS": []
},
"MEMPOOL_SERVICES": {
"API": "https://mempool.space/api/v1/services",
"ACCELERATIONS": false
},
"STRATUM": {
"ENABLED": false,
"API": "http://localhost:1234"
},
"FIAT_PRICE": {
"ENABLED": false,
"PAID": false,
"API_KEY": ""
}
}

8929
backend/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

21
backend/package.json
View file

@ -1,6 +1,6 @@
{
"name": "mempool-backend",
"version": "3.1.0-dev",
"version": "3.3-dev",
"description": "Bitcoin mempool visualizer and blockchain explorer backend",
"license": "GNU Affero General Public License v3.0",
"homepage": "https://mempool.space",
@ -34,20 +34,21 @@
"reindex-all-blocks": "npm run start-production --update-pools --reindex-blocks",
"test": "./node_modules/.bin/jest --coverage",
"test:ci": "CI=true ./node_modules/.bin/jest --coverage",
"test:integration": "./node_modules/.bin/jest --config=jest.integration.config.ts --runInBand --forceExit",
"test:with-db": "bash ./scripts/test-with-db.sh",
"lint": "./node_modules/.bin/eslint . --ext .ts",
"lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
"prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
},
"dependencies": {
"@babel/core": "^7.25.2",
"@mempool/electrum-client": "1.1.9",
"@types/node": "^18.15.3",
"axios": "1.7.2",
"axios": "1.12.2",
"bitcoinjs-lib": "~6.1.3",
"crypto-js": "~4.2.0",
"express": "~4.21.1",
"maxmind": "~4.3.11",
"mysql2": "~3.11.0",
"mysql2": "~3.14.1",
"rust-gbt": "file:./rust-gbt",
"redis": "^4.7.0",
"socks-proxy-agent": "~7.0.0",
@ -55,20 +56,22 @@
"ws": "~8.18.0"
},
"devDependencies": {
"@babel/code-frame": "^7.18.6",
"@babel/core": "^7.25.2",
"@types/compression": "^1.7.2",
"@types/crypto-js": "^4.1.1",
"@types/express": "^4.17.17",
"@types/jest": "^29.5.0",
"@types/jest": "^30.0.0",
"@types/ws": "~8.5.10",
"@typescript-eslint/eslint-plugin": "^5.55.0",
"@typescript-eslint/parser": "^5.55.0",
"eslint": "^8.36.0",
"eslint-config-prettier": "^8.8.0",
"jest": "^29.5.0",
"jest": "^30.0.0",
"prettier": "^3.0.0",
"ts-jest": "^29.1.1",
"ts-jest": "^29.4.5",
"ts-node": "^10.9.1"
},
"overrides": {
"js-yaml": "^4.1.1",
"glob": "^11.1.0"
}
}

40
backend/scripts/debug-integration-tests.sh Executable file
View file

@ -0,0 +1,40 @@
#!/bin/bash
# Script to debug integration tests
# Usage: ./scripts/debug-integration-tests.sh [test-file-pattern]
set -e
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'
# Get the directory of this script
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
BACKEND_DIR="$( cd "$SCRIPT_DIR/.." && pwd )"
cd "$BACKEND_DIR"
# Check if MariaDB is running
if ! docker-compose -f docker-compose.test.yml ps | grep -q "Up"; then
echo -e "${YELLOW}Starting MariaDB container...${NC}"
docker-compose -f docker-compose.test.yml up -d
# Wait for MariaDB
echo -e "${YELLOW}Waiting for MariaDB...${NC}"
sleep 5
fi
# Run tests with pattern if provided
if [ -n "$1" ]; then
echo -e "${GREEN}Running tests matching: $1${NC}"
MEMPOOL_CONFIG_FILE="$BACKEND_DIR/mempool-config.test.json" npx jest --config=jest.integration.config.ts --runInBand --verbose "$1"
else
echo -e "${GREEN}Running all integration tests${NC}"
MEMPOOL_CONFIG_FILE="$BACKEND_DIR/mempool-config.test.json" npm run test:integration
fi
echo -e "${GREEN}Tests completed${NC}"

111
backend/scripts/test-with-db.sh Executable file
View file

@ -0,0 +1,111 @@
#!/bin/bash
set -e
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
echo -e "${GREEN}Starting integration tests with MariaDB...${NC}"
# Get the directory of this script
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
BACKEND_DIR="$( cd "$SCRIPT_DIR/.." && pwd )"
cd "$BACKEND_DIR"
# Detect docker compose command (v1 or v2)
if docker compose version &> /dev/null; then
DOCKER_COMPOSE="docker compose"
elif docker-compose version &> /dev/null; then
DOCKER_COMPOSE="docker-compose"
else
echo -e "${RED}Error: Neither 'docker compose' nor 'docker-compose' is available${NC}"
exit 1
fi
echo -e "${YELLOW}Using: ${DOCKER_COMPOSE}${NC}"
# Function to cleanup on exit
cleanup() {
echo -e "\n${YELLOW}Cleaning up...${NC}"
# Kill the backend server if it's running
if [ ! -z "$SERVER_PID" ]; then
kill $SERVER_PID 2>/dev/null || true
fi
# Stop containers and remove volumes, but don't fail on network errors
$DOCKER_COMPOSE -f docker-compose.test.yml down -v 2>&1 | grep -v "Resource is still in use" || true
}
# Set trap to cleanup on exit
trap cleanup EXIT INT TERM
# Stop any existing test containers
echo -e "${YELLOW}Stopping any existing test containers...${NC}"
$DOCKER_COMPOSE -f docker-compose.test.yml down -v 2>/dev/null || true
# Start MariaDB container
echo -e "${GREEN}Starting MariaDB container...${NC}"
$DOCKER_COMPOSE -f docker-compose.test.yml up -d
# Wait for MariaDB to be ready
echo -e "${YELLOW}Waiting for MariaDB to be ready...${NC}"
max_attempts=30
attempt=0
while [ $attempt -lt $max_attempts ]; do
if $DOCKER_COMPOSE -f docker-compose.test.yml exec -T db-test mysqladmin ping -h localhost -u mempool_test -pmempool_test --silent 2>/dev/null; then
echo -e "${GREEN}MariaDB is ready!${NC}"
break
fi
attempt=$((attempt + 1))
echo -e "${YELLOW}Attempt $attempt/$max_attempts - waiting for MariaDB...${NC}"
sleep 2
done
if [ $attempt -eq $max_attempts ]; then
echo -e "${RED}MariaDB did not start in time${NC}"
exit 1
fi
# Additional wait to ensure MariaDB is fully initialized
sleep 3
# Build the backend
echo -e "${GREEN}Building backend...${NC}"
npm run build
# Run integration tests with absolute path to config
# SKIP_DB_SETUP=1 and SKIP_DB_TEARDOWN=1 tell Jest that we're managing the database lifecycle
echo -e "${GREEN}Running integration tests...${NC}"
export MEMPOOL_CONFIG_FILE="$BACKEND_DIR/mempool-config.test.json"
export SKIP_DB_SETUP=1
export SKIP_DB_TEARDOWN=1
npm run test:integration
# Start the backend server in the background
# MEMPOOL_CONFIG_FILE is already exported above
echo -e "${GREEN}Starting backend server...${NC}"
node dist/index.js &
SERVER_PID=$!
# Wait for server to start and verify connection
echo -e "${YELLOW}Waiting for server to start and connect to database...${NC}"
sleep 10
# Check if server is still running
if ps -p $SERVER_PID > /dev/null 2>&1; then
echo -e "${GREEN}Server started successfully and connected to database!${NC}"
# Kill the server (it will be in the cleanup function too, but do it here as well)
kill $SERVER_PID 2>/dev/null || true
wait $SERVER_PID 2>/dev/null || true
SERVER_PID=""
else
echo -e "${RED}Server failed to start or exited prematurely${NC}"
exit 1
fi
echo -e "${GREEN}All tests passed successfully!${NC}"

134
backend/src/__integration_tests__/blocks-repository.test.ts Normal file
View file

@ -0,0 +1,134 @@
import BlocksRepository from '../repositories/BlocksRepository';
import { setupTestDatabase, waitForDatabase, cleanupTestData, insertTestPool, insertTestBlock } from './test-helpers';
describe('BlocksRepository Integration Tests', () => {
let defaultPoolId: number;
beforeAll(async () => {
await waitForDatabase();
await setupTestDatabase();
}, 120000);
beforeEach(async () => {
await cleanupTestData();
// Create a default pool for all blocks
defaultPoolId = await insertTestPool({
name: 'Unknown',
slug: 'unknown',
addresses: '[]',
regexes: '[]'
});
});
afterAll(async () => {
await cleanupTestData();
});
test('should insert and retrieve a block', async () => {
const blockHash = '00000000000000000001a0e3e9b2e6d6e8a4b0e9b2e6d6e8a4b0e9b2e6d6e8a4';
const height = 800000;
await insertTestBlock({
height: height,
hash: blockHash,
blockTimestamp: new Date('2023-07-16T00:00:00Z'),
size: 1500000,
weight: 3999000,
tx_count: 3000,
difficulty: 53911173001054.59,
poolId: defaultPoolId
});
const block = await BlocksRepository.$getBlockByHeight(height);
expect(block).toBeDefined();
expect(block!.height).toBe(height);
expect(block!.id).toBe(blockHash);
});
test('should get block by hash', async () => {
const blockHash = '00000000000000000002b0e3e9b2e6d6e8a4b0e9b2e6d6e8a4b0e9b2e6d6e8a4';
const height = 800001;
await insertTestBlock({
height: height,
hash: blockHash,
tx_count: 2500,
poolId: defaultPoolId
});
const block = await BlocksRepository.$getBlockByHash(blockHash);
expect(block).toBeDefined();
expect(block!.id).toBe(blockHash);
expect(block!.height).toBe(height);
});
test('should handle non-existent block', async () => {
const block = await BlocksRepository.$getBlockByHeight(999999);
expect(block).toBeNull();
});
test('should check for missing blocks in range', async () => {
// Insert blocks with a gap
await insertTestBlock({
height: 800100,
hash: '0000000000000000000100000000000000000000000000000000000000000001',
poolId: defaultPoolId
});
await insertTestBlock({
height: 800102,
hash: '0000000000000000000100000000000000000000000000000000000000000003',
poolId: defaultPoolId
});
const missingBlocks = await BlocksRepository.$getMissingBlocksBetweenHeights(800100, 800102);
expect(missingBlocks).toContain(800101);
});
test('should get latest block height', async () => {
await insertTestBlock({
height: 800200,
hash: '0000000000000000000200000000000000000000000000000000000000000001',
poolId: defaultPoolId
});
await insertTestBlock({
height: 800201,
hash: '0000000000000000000200000000000000000000000000000000000000000002',
poolId: defaultPoolId
});
const height = await BlocksRepository.$mostRecentBlockHeight();
expect(height).toBe(800201);
});
test('should handle block with pool association', async () => {
// Insert a pool and get its auto-generated ID
const testPoolId = await insertTestPool({
name: 'Test Pool',
slug: 'test-pool',
addresses: '[]',
regexes: '[]'
});
const blockHash = '0000000000000000000300000000000000000000000000000000000000000001';
await insertTestBlock({
height: 800300,
hash: blockHash,
poolId: testPoolId
});
const block = await BlocksRepository.$getBlockByHash(blockHash);
expect(block).toBeDefined();
expect(block).not.toBeNull();
// The pool should be populated with the test pool's data
if (block && block.extras?.pool) {
expect(block.extras.pool.name).toBe('Test Pool');
expect(block.extras.pool.slug).toBe('test-pool');
}
});
});

45
backend/src/__integration_tests__/database-connection.test.ts Normal file
View file

@ -0,0 +1,45 @@
import DB from '../database';
import config from '../config';
import { setupTestDatabase, waitForDatabase, getTestDatabaseConfig } from './test-helpers';
describe('Database Connection Integration Tests', () => {
beforeAll(async () => {
// Wait for database to be ready
await waitForDatabase();
}, 60000);
test('should connect to the test database', async () => {
const dbConfig = getTestDatabaseConfig();
expect(dbConfig.enabled).toBe(true);
expect(dbConfig.database).toBe('mempool_test');
expect(dbConfig.port).toBe(33306);
});
test('should execute a simple query', async () => {
const [result] = await DB.query<any>('SELECT 1 as value');
expect(result).toHaveLength(1);
expect(result[0].value).toBe(1);
});
test('should execute a query with parameters', async () => {
const [result] = await DB.query<any>('SELECT ? as sum', [42]);
expect(result).toHaveLength(1);
expect(result[0].sum).toBe(42);
});
test('should check database connection', async () => {
await expect(DB.checkDbConnection()).resolves.not.toThrow();
});
test('should handle query timeout configuration', async () => {
expect(config.DATABASE.TIMEOUT).toBeGreaterThan(0);
});
test('should have correct database configuration', () => {
expect(config.DATABASE.HOST).toBe('127.0.0.1');
expect(config.DATABASE.USERNAME).toBe('mempool_test');
expect(config.DATABASE.PASSWORD).toBe('mempool_test');
expect(config.DATABASE.DATABASE).toBe('mempool_test');
});
});

97
backend/src/__integration_tests__/database-migration.test.ts Normal file
View file

@ -0,0 +1,97 @@
import DB from '../database';
import { setupTestDatabase, waitForDatabase } from './test-helpers';
describe('Database Migration Integration Tests', () => {
beforeAll(async () => {
await waitForDatabase();
await setupTestDatabase();
}, 120000);
test('should create state table', async () => {
const [result] = await DB.query<any>(
`SELECT COUNT(*) as count
FROM information_schema.tables
WHERE table_schema = 'mempool_test'
AND table_name = 'state'`
);
expect(result[0].count).toBe(1);
});
test('should have schema version in state table', async () => {
const [result] = await DB.query<any>("SELECT number FROM state WHERE name = 'schema_version'");
expect(result).toHaveLength(1);
expect(result[0].number).toBeGreaterThan(0);
});
test('should create blocks table', async () => {
const [result] = await DB.query<any>(
`SELECT COUNT(*) as count
FROM information_schema.tables
WHERE table_schema = 'mempool_test'
AND table_name = 'blocks'`
);
expect(result[0].count).toBe(1);
});
test('should create pools table', async () => {
const [result] = await DB.query<any>(
`SELECT COUNT(*) as count
FROM information_schema.tables
WHERE table_schema = 'mempool_test'
AND table_name = 'pools'`
);
expect(result[0].count).toBe(1);
});
test('should create hashrates table', async () => {
const [result] = await DB.query<any>(
`SELECT COUNT(*) as count
FROM information_schema.tables
WHERE table_schema = 'mempool_test'
AND table_name = 'hashrates'`
);
expect(result[0].count).toBe(1);
});
test('should create prices table', async () => {
const [result] = await DB.query<any>(
`SELECT COUNT(*) as count
FROM information_schema.tables
WHERE table_schema = 'mempool_test'
AND table_name = 'prices'`
);
expect(result[0].count).toBe(1);
});
test('blocks table should have required columns', async () => {
const [columns] = await DB.query<any>(
`SELECT COLUMN_NAME
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = 'mempool_test'
AND TABLE_NAME = 'blocks'`
);
const columnNames = columns.map((col: any) => col.COLUMN_NAME);
expect(columnNames).toContain('height');
expect(columnNames).toContain('hash');
expect(columnNames).toContain('blockTimestamp');
expect(columnNames).toContain('size');
expect(columnNames).toContain('weight');
expect(columnNames).toContain('tx_count');
});
test('pools table should have required columns', async () => {
const [columns] = await DB.query<any>(
`SELECT COLUMN_NAME
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = 'mempool_test'
AND TABLE_NAME = 'pools'`
);
const columnNames = columns.map((col: any) => col.COLUMN_NAME);
expect(columnNames).toContain('id');
expect(columnNames).toContain('name');
expect(columnNames).toContain('slug');
});
});

122
backend/src/__integration_tests__/pools-repository.test.ts Normal file
View file

@ -0,0 +1,122 @@
import PoolsRepository from '../repositories/PoolsRepository';
import { setupTestDatabase, waitForDatabase, cleanupTestData, insertTestPool } from './test-helpers';
describe('PoolsRepository Integration Tests', () => {
beforeAll(async () => {
await waitForDatabase();
await setupTestDatabase();
}, 120000);
beforeEach(async () => {
await cleanupTestData();
});
afterAll(async () => {
await cleanupTestData();
});
test('should insert and retrieve a pool', async () => {
const poolData = {
name: 'Foundry USA',
slug: 'foundryusa',
link: 'https://foundrydigital.com',
addresses: JSON.stringify(['bc1qxhmdufsvnuaaaer4ynz88fspdsxq2h9e9cetdj']),
regexes: JSON.stringify(['/Foundry USA Pool/'])
};
const poolId = await insertTestPool(poolData);
expect(poolId).toBeGreaterThan(0);
const pools = await PoolsRepository.$getPools();
const insertedPool = pools.find(p => p.id === poolId);
expect(insertedPool).toBeDefined();
expect(insertedPool?.name).toBe(poolData.name);
expect(insertedPool?.slug).toBe(poolData.slug);
});
test('should get pool by slug', async () => {
await insertTestPool({
name: 'AntPool',
slug: 'antpool',
link: 'https://antpool.com'
});
const pool = await PoolsRepository.$getPool('antpool');
expect(pool).toBeDefined();
expect(pool!.name).toBe('AntPool');
expect(pool!.slug).toBe('antpool');
});
test('should get all pools', async () => {
await insertTestPool({
name: 'Pool 1',
slug: 'pool-1'
});
await insertTestPool({
name: 'Pool 2',
slug: 'pool-2'
});
await insertTestPool({
name: 'Pool 3',
slug: 'pool-3'
});
const pools = await PoolsRepository.$getPools();
expect(pools.length).toBeGreaterThanOrEqual(3);
});
test('should handle pool with addresses', async () => {
const addresses = ['bc1qtest1', 'bc1qtest2', '3TestAddress'];
await insertTestPool({
name: 'Multi Address Pool',
slug: 'multi-address-pool',
addresses: JSON.stringify(addresses)
});
const pool = await PoolsRepository.$getPool('multi-address-pool', false);
expect(pool).toBeDefined();
const poolAddresses = JSON.parse(pool!.addresses);
expect(poolAddresses).toHaveLength(3);
expect(poolAddresses).toContain('bc1qtest1');
});
test('should handle pool with regexes', async () => {
const regexes = ['/Pool Name/', '/Alternative Name/'];
await insertTestPool({
name: 'Regex Pool',
slug: 'regex-pool',
regexes: JSON.stringify(regexes)
});
const pool = await PoolsRepository.$getPool('regex-pool', false);
expect(pool).toBeDefined();
const poolRegexes = JSON.parse(pool!.regexes);
expect(poolRegexes).toHaveLength(2);
expect(poolRegexes[0]).toBe('/Pool Name/');
});
test('should handle non-existent pool', async () => {
const pool = await PoolsRepository.$getPool('non-existent-pool-slug');
expect(pool).toBeNull();
});
test('should update pool information', async () => {
const poolDbId = await insertTestPool({
name: 'Original Pool Name',
slug: 'original-pool'
});
// Update the pool name
await PoolsRepository.$renameMiningPool(poolDbId, 'updated-pool', 'Updated Pool Name');
const pool = await PoolsRepository.$getPool('updated-pool');
expect(pool).toBeDefined();
expect(pool!.name).toBe('Updated Pool Name');
});
});

190
backend/src/__integration_tests__/test-helpers.ts Normal file
View file

@ -0,0 +1,190 @@
import DB from '../database';
import config from '../config';
import logger from '../logger';
import databaseMigration from '../api/database-migration';
/**
* Initialize the test database with schema migrations
*/
export async function setupTestDatabase(): Promise<void> {
try {
await DB.checkDbConnection();
await databaseMigration.$initializeOrMigrateDatabase();
} catch (error) {
logger.err('Failed to setup test database: ' + (error instanceof Error ? error.message : error));
throw error;
}
}
/**
* Clean up all data from test tables (but preserve schema)
* This runs between each test to ensure isolation
*/
export async function cleanupTestData(): Promise<void> {
// Order matters: delete child tables before parent tables
const tables = [
'blocks_audits',
'blocks_summaries',
'blocks_prices',
'blocks_templates',
'cpfp_clusters',
'blocks', // blocks references pools
'difficulty_adjustments',
'hashrates',
'prices',
'node_records',
'nodes_sockets',
'nodes',
'lightning_stats',
'transactions',
'elements_pegs',
'federation_txos',
'pools'
];
try {
// Disable foreign key checks temporarily for faster cleanup
await DB.query('SET FOREIGN_KEY_CHECKS = 0');
for (const table of tables) {
try {
// Use 'silent' error logging to avoid noise for optional tables that don't exist
await DB.query(`TRUNCATE TABLE ${table}`, [], 'silent');
} catch (e) {
// Table might not exist, that's okay for optional tables
// Silently ignore - no need to log since these are expected for optional features
}
}
// Re-enable foreign key checks
await DB.query('SET FOREIGN_KEY_CHECKS = 1');
} catch (error) {
// Try to re-enable foreign keys even if cleanup failed
try {
await DB.query('SET FOREIGN_KEY_CHECKS = 1');
} catch (e) {
// Ignore
}
logger.err('Failed to cleanup test data: ' + (error instanceof Error ? error.message : error));
throw error;
}
}
/**
* Wait for database to be ready
*/
export async function waitForDatabase(maxRetries = 30, retryInterval = 1000): Promise<void> {
for (let i = 0; i < maxRetries; i++) {
try {
await DB.query('SELECT 1');
logger.info('Database is ready');
return;
} catch (error) {
logger.debug(`Waiting for database... attempt ${i + 1}/${maxRetries}`);
await new Promise(resolve => setTimeout(resolve, retryInterval));
}
}
throw new Error('Database did not become ready in time');
}
/**
* Get database configuration for tests
*/
export function getTestDatabaseConfig() {
return {
host: config.DATABASE.HOST,
port: config.DATABASE.PORT,
database: config.DATABASE.DATABASE,
username: config.DATABASE.USERNAME,
enabled: config.DATABASE.ENABLED
};
}
/**
* Insert a test pool into the database
*/
export async function insertTestPool(poolData: {
id?: number;
name: string;
link?: string;
slug: string;
addresses?: string;
regexes?: string;
}) {
const [result] = await DB.query<any>(
`INSERT INTO pools (unique_id, name, link, slug, addresses, regexes)
VALUES (?, ?, ?, ?, ?, ?)`,
[
poolData.id || -1,
poolData.name,
poolData.link || '',
poolData.slug,
poolData.addresses || '[]',
poolData.regexes || '[]'
]
);
return result.insertId;
}
/**
* Insert a test block into the database
*/
export async function insertTestBlock(blockData: {
height: number;
hash: string;
blockTimestamp?: Date;
size?: number;
weight?: number;
tx_count?: number;
difficulty?: number;
poolId?: number | null;
}) {
const timestamp = blockData.blockTimestamp || new Date();
const size = blockData.size || 1000000;
const weight = blockData.weight || 4000000;
const txCount = blockData.tx_count || 2000;
await DB.query(
`INSERT INTO blocks (
height, hash, blockTimestamp, size, weight, tx_count,
difficulty, pool_id, version, bits, nonce, merkle_root,
previous_block_hash, median_timestamp, stale,
fees, fee_span, median_fee,
avg_tx_size, total_inputs, total_outputs, total_output_amt,
segwit_total_txs, segwit_total_size, segwit_total_weight,
header, utxoset_change
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
blockData.height,
blockData.hash,
timestamp,
size,
weight,
txCount,
blockData.difficulty || 1.0,
blockData.poolId !== undefined ? blockData.poolId : null,
0x20000000,
0x1d00ffff,
0,
'0000000000000000000000000000000000000000000000000000000000000000',
'0000000000000000000000000000000000000000000000000000000000000000',
timestamp,
0, // stale = false
// Required fields with defaults
50000000, // fees (in sats)
JSON.stringify([0, 0, 0, 0, 0, 0, 0]), // fee_span (JSON array)
10000, // median_fee (in sats)
size / txCount, // avg_tx_size
txCount * 2, // total_inputs (estimate)
txCount * 2, // total_outputs (estimate)
2100000000000000, // total_output_amt (21M BTC in sats, estimate)
txCount, // segwit_total_txs (assume all segwit for test)
size, // segwit_total_size
weight, // segwit_total_weight
'00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', // header (160 chars)
0 // utxoset_change
]
);
}

55
backend/src/__tests__/api/fee-api.ts Normal file
View file

@ -0,0 +1,55 @@
import feeApi from '../../api/fee-api';
import { IBitcoinApi } from '../../api/bitcoin/bitcoin-api.interface';
const feeMempoolBlocks = require('./test-data/fee-mempool-blocks.json');
const subSatMempoolInfo: IBitcoinApi.MempoolInfo = {
mempoolminfee: 0.000001, // 0.1 sat/vbyte
loaded: true,
size: 100,
bytes: 10000,
usage: 10000,
total_fee: 10000,
maxmempool: 10000,
minrelaytxfee: 0.000001, // 0.1 sat/vbyte
};
const mempoolInfo: IBitcoinApi.MempoolInfo = {
mempoolminfee: 0.00001,
loaded: true,
size: 100,
bytes: 10000,
usage: 10000,
total_fee: 10000,
maxmempool: 10000,
minrelaytxfee: 0.00001,
};
describe('Fee API', () => {
test('should calculate recommended fees properly for sub-sat mempool', () => {
const fee = feeApi.calculateRecommendedFee(feeMempoolBlocks.subsat, subSatMempoolInfo);
expect(fee.fastestFee).toBe(2);
expect(fee.halfHourFee).toBe(1);
expect(fee.hourFee).toBe(1);
expect(fee.economyFee).toBe(1);
expect(fee.minimumFee).toBe(1);
});
test('should calculate recommended fees properly for full but low fee mempool', () => {
const fee = feeApi.calculateRecommendedFee(feeMempoolBlocks.lowfee, mempoolInfo);
expect(fee.fastestFee).toBe(2);
expect(fee.halfHourFee).toBe(2);
expect(fee.hourFee).toBe(2);
expect(fee.economyFee).toBe(2);
expect(fee.minimumFee).toBe(1);
});
test('should calculate recommended fees properly for empty mempool', () => {
const fee = feeApi.calculateRecommendedFee(feeMempoolBlocks.empty, mempoolInfo);
expect(fee.fastestFee).toBe(1);
expect(fee.halfHourFee).toBe(1);
expect(fee.hourFee).toBe(1);
expect(fee.economyFee).toBe(1);
expect(fee.minimumFee).toBe(1);
});
});

289
backend/src/__tests__/api/test-data/fee-mempool-blocks.json Normal file
View file

@ -0,0 +1,289 @@
{
"subsat": [
{
"blockSize": 1746106,
"blockVSize": 997953.25,
"nTx": 3750,
"totalFees": 1764766,
"medianFee": 1.0023586640951265,
"feeRange": [
0.6082474226804123,
0.6082474226804123,
0.6082474226804123,
0.6082474226804123,
1.2076788830715532,
3.166077738515901,
73.61111111111111
]
},
{
"blockSize": 1809871,
"blockVSize": 997963,
"nTx": 4861,
"totalFees": 588436,
"medianFee": 0.6082474226804123,
"feeRange": [
0.5156626506024097,
0.5649475055559813,
0.5670103092783505,
0.6082474226804123,
0.6082474226804123,
0.6082474226804123,
0.6082474226804123
]
},
{
"blockSize": 2917870,
"blockVSize": 997821.25,
"nTx": 1221,
"totalFees": 535389,
"medianFee": 0.5200034509958588,
"feeRange": [
0.35051546391752575,
0.5649475055559813,
0.5649475055559813,
0.5649475055559813,
0.5649475055559813,
0.5649475055559813,
0.5649475055559813
]
},
{
"blockSize": 3937966,
"blockVSize": 997875.25,
"nTx": 18,
"totalFees": 506205,
"medianFee": 0.5100003123470801,
"feeRange": [
0.35051546391752575,
0.35051546391752575,
0.35051546391752575,
0.5099980684971892,
0.5100003123470801,
0.5100020185186144,
0.5100025244589157
]
},
{
"blockSize": 3917639,
"blockVSize": 997810.25,
"nTx": 62,
"totalFees": 503529,
"medianFee": 0.5099975959779666,
"feeRange": [
0.35051546391752575,
0.35051546391752575,
0.35051546391752575,
0.35051546391752575,
0.35051546391752575,
0.5099971889756266,
0.5100002265056967
]
},
{
"blockSize": 3694897,
"blockVSize": 997893.25,
"nTx": 43,
"totalFees": 503307,
"medianFee": 0.5099952023028946,
"feeRange": [
0.35051546391752575,
0.35051546391752575,
0.35051546391752575,
0.35051546391752575,
0.3704911425797351,
0.5099952023028946,
0.5099967157103613
]
},
{
"blockSize": 2486759,
"blockVSize": 997857.5,
"nTx": 498,
"totalFees": 386911,
"medianFee": 0.36001120750903104,
"feeRange": [
0.35051546391752575,
0.35051546391752575,
0.35051546391752575,
0.35051546391752575,
0.35051546391752575,
0.35051546391752575,
0.5000054632270189
]
},
{
"blockSize": 6681226,
"blockVSize": 3385616.5,
"nTx": 10222,
"totalFees": 1197838,
"medianFee": 0.35051546391752575,
"feeRange": [
0.21576460085542437,
0.35051546391752575,
0.35051546391752575,
0.35051546391752575,
0.35051546391752575,
0.35051546391752575,
0.3599967979827291,
0.3600004002561639,
0.3600040026016911,
0.3600040026016911,
0.3600040026016911
]
}
],
"empty": [{
"blockSize": 10000,
"blockVSize": 10000,
"nTx": 2000,
"totalFees": 1197838,
"medianFee": 1,
"feeRange": [
1,
1,
1,
1,
1,
1,
1
]
}
],
"lowfee": [
{
"blockSize": 1746106,
"blockVSize": 997953.25,
"nTx": 3750,
"totalFees": 1764766,
"medianFee": 1.0023586640951265,
"feeRange": [
1.6082474226804123,
1.6082474226804123,
1.6082474226804123,
1.6082474226804123,
1.2076788830715532,
3.166077738515901,
73.61111111111111
]
},
{
"blockSize": 1809871,
"blockVSize": 997963,
"nTx": 4861,
"totalFees": 588436,
"medianFee": 1.6082474226804123,
"feeRange": [
1.5156626506024097,
1.5649475055559813,
1.5670103092783505,
1.6082474226804123,
1.6082474226804123,
1.6082474226804123,
1.6082474226804123
]
},
{
"blockSize": 2917870,
"blockVSize": 997821.25,
"nTx": 1221,
"totalFees": 535389,
"medianFee": 1.5200034509958588,
"feeRange": [
1.35051546391752575,
1.5649475055559813,
1.5649475055559813,
1.5649475055559813,
1.5649475055559813,
1.5649475055559813,
1.5649475055559813
]
},
{
"blockSize": 3937966,
"blockVSize": 997875.25,
"nTx": 18,
"totalFees": 506205,
"medianFee": 1.5100003123470801,
"feeRange": [
1.35051546391752575,
1.35051546391752575,
1.35051546391752575,
1.5099980684971892,
1.5100003123470801,
1.5100020185186144,
1.5100025244589157
]
},
{
"blockSize": 3917639,
"blockVSize": 997811.25,
"nTx": 62,
"totalFees": 503529,
"medianFee": 1.5099975959779666,
"feeRange": [
1.35051546391752575,
1.35051546391752575,
1.35051546391752575,
1.35051546391752575,
1.35051546391752575,
1.5099971889756266,
1.5100002265056967
]
},
{
"blockSize": 3694897,
"blockVSize": 997893.25,
"nTx": 43,
"totalFees": 503307,
"medianFee": 1.5099952023028946,
"feeRange": [
1.35051546391752575,
1.35051546391752575,
1.35051546391752575,
1.35051546391752575,
1.3704911425797351,
1.5099952023028946,
1.5099967157103613
]
},
{
"blockSize": 2486759,
"blockVSize": 997857.5,
"nTx": 498,
"totalFees": 386911,
"medianFee": 1.36001120750903104,
"feeRange": [
1.35051546391752575,
1.35051546391752575,
1.35051546391752575,
1.35051546391752575,
1.35051546391752575,
1.35051546391752575,
1.5000054632270189
]
},
{
"blockSize": 6681226,
"blockVSize": 3385616.5,
"nTx": 10222,
"totalFees": 1197838,
"medianFee": 1.35051546391752575,
"feeRange": [
1.21576460085542437,
1.35051546391752575,
1.35051546391752575,
1.35051546391752575,
1.35051546391752575,
1.35051546391752575,
1.3599967979827291,
1.3600004002561639,
1.3600040026016911,
1.3600040026016911,
1.3600040026016911
]
}
]
}

99
backend/src/__tests__/api/test-data/non-standard-txs.json
View file

@ -1,52 +1,55 @@
[
{
"txid": "50136231cb7eeeffb17fc41d1cca213426abe5bf3760e3d6421cad0c0edad367",
"version": 1,
"locktime": 0,
"vin": [
{
"txid": "c7f86fb7b830124057475b282809f3474ef3565daa3de0b599980fb9e84ab019",
"vout": 4217,
"prevout": {
"scriptpubkey": "001466197b5eadd8067ec194a457e1044b6d1fbdd3b3",
"scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 66197b5eadd8067ec194a457e1044b6d1fbdd3b3",
"scriptpubkey_type": "v0_p2wpkh",
"scriptpubkey_address": "bc1qvcvhkh4dmqr8asv553t7zpztd50mm5ang4na33",
"value": 106
},
"scriptsig": "",
"scriptsig_asm": "",
"witness": [
"3043021f2af6060a142c6cfd7428adad6a50745d2424813d7ced5c0bbcca85e70de1be022021440ca1c8c3ed49ecd1b64dca6911adcd430c5d3dd60d77ffe0072953999f5b01",
"02ead5c34e3d2c506574b562f857576e11380b6ba15d9f0ad7b7303fdaa9c1513d"
],
"is_coinbase": false,
"sequence": 4294967295
}
],
"vout": [
{
"scriptpubkey": "6a023a29",
"scriptpubkey_asm": "OP_RETURN OP_PUSHBYTES_2 3a29",
"scriptpubkey_type": "op_return",
"value": 0
{
"txid": "f859a4e6692c3f15a279449eac191ecb9d17d2d6c003bbae9a55e5da07147eab",
"version": 1,
"locktime": 0,
"size": 170,
"weight": 371,
"fee": 11586,
"vin": [
{
"is_coinbase": false,
"prevout": {
"value": 11586,
"scriptpubkey": "512088939b42f2ed113dd4756055b179cd784c2cfa52cad29046411c28335aa2055c",
"scriptpubkey_address": "bc1p3zfekshja5gnm4r4vp2mz7wd0pxze7jjetffq3jprs5rxk4zq4wqesykl2",
"scriptpubkey_asm": "OP_PUSHNUM_1 OP_PUSHBYTES_32 88939b42f2ed113dd4756055b179cd784c2cfa52cad29046411c28335aa2055c",
"scriptpubkey_type": "v1_p2tr"
},
"scriptsig": "",
"scriptsig_asm": "",
"sequence": 4294967295,
"txid": "9b93dab94a3334b6119f75d107da6bcef9435fa52ec6c0de14aa8a094794551a",
"vout": 0,
"witness": [
"6840b6fa27a00ba001cc92797ce4f3ab7b7a32c21d1fce49e893b42e506bd92e8db187966a84ef799915cf671334cc59779915b192bfb66b2afcf384bb61d0f4",
"500049276d20616e20616e6e6578212041726520796f7520616e20616e6e65783f00"
],
"inner_redeemscript_asm": "",
"inner_witnessscript_asm": ""
}
],
"vout": [
{
"value": 0,
"scriptpubkey": "6a05616e6e6578",
"scriptpubkey_address": "",
"scriptpubkey_asm": "OP_RETURN OP_PUSHBYTES_5 616e6e6578",
"scriptpubkey_type": "op_return"
}
],
"status": {
"confirmed": true,
"block_height": 896088,
"block_hash": "0000000000000000000148370c6ad0eceb23d0de54ea86a362679cee7fcd3f4a",
"block_time": 1746868490
},
{
"scriptpubkey": "6a036d7648",
"scriptpubkey_asm": "OP_RETURN OP_PUSHBYTES_3 6d7648",
"scriptpubkey_type": "op_return",
"value": 0
}
],
"size": 186,
"weight": 420,
"sigops": 1,
"fee": 106,
"status": {
"confirmed": true,
"block_height": 836361,
"block_hash": "0000000000000000000341cc26cda4af82cd25f7063c448772228cbf2836915b",
"block_time": 1711448028
"order": 2877166599,
"vsize": 93,
"adjustedVsize": 92.75,
"sigops": 0,
"feePerVsize": 124.91644204851752,
"adjustedFeePerVsize": 124.91644204851752,
"effectiveFeePerVsize": 124.91644204851752
}
}
]

2
backend/src/__tests__/config.test.ts
View file

@ -51,7 +51,7 @@ describe('Mempool Backend Config', () => {
MAX_PUSH_TX_SIZE_WEIGHT: 400000,
ALLOW_UNREACHABLE: true,
PRICE_UPDATES_PER_HOUR: 1,
MAX_TRACKED_ADDRESSES: 1,
MAX_TRACKED_ADDRESSES: 1
});
expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

14
backend/src/api/acceleration/acceleration.routes.ts
View file

@ -11,6 +11,7 @@ class AccelerationRoutes {
public initRoutes(app: Application): void {
app
.get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations', this.$getAcceleratorAccelerations.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/:txid', this.$getAcceleratorAcceleration.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/history', this.$getAcceleratorAccelerationsHistory.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/history/aggregated', this.$getAcceleratorAccelerationsHistoryAggregated.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/stats', this.$getAcceleratorAccelerationsStats.bind(this))
@ -23,6 +24,19 @@ class AccelerationRoutes {
res.status(200).send(Object.values(accelerations));
}
private async $getAcceleratorAcceleration(req: Request, res: Response): Promise<void> {
if (req.params.txid) {
const acceleration = await AccelerationRepository.$getAccelerationInfoForTxid(req.params.txid);
if (acceleration) {
res.status(200).send(acceleration);
} else {
res.status(404).send('Acceleration not found');
}
} else {
res.status(400).send('txid is required');
}
}
private async $getAcceleratorAccelerationsHistory(req: Request, res: Response): Promise<void> {
const history = await AccelerationRepository.$getAccelerationInfo(null, req.query.blockHeight ? parseInt(req.query.blockHeight as string, 10) : null);
res.status(200).send(history.map(accel => ({

2
backend/src/api/audit.ts
View file

@ -55,7 +55,7 @@ class Audit {
} else if (mempool[txid]?.lastBoosted != null && (now - (mempool[txid]?.lastBoosted || 0)) <= PROPAGATION_MARGIN) {
// tx was recently cpfp'd, miner may not have the latest effective rate
fresh.push(txid);
} else {
} else if (mempool[txid].effectiveFeePerVsize >= 1) { // transactions paying < 1 sat/vbyte are never considered censored
isCensored[txid] = true;
}
displacedWeight += mempool[txid]?.weight || 0;

7
backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts
View file

@ -8,19 +8,22 @@ export interface AbstractBitcoinApi {
$getMempoolTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]>;
$getAllMempoolTransactions(lastTxid?: string, max_txs?: number);
$getTransactionHex(txId: string): Promise<string>;
$getTransactionMerkleProof(txId: string): Promise<IEsploraApi.MerkleProof>;
$getBlockHeightTip(): Promise<number>;
$getBlockHashTip(): Promise<string>;
$getTxIdsForBlock(hash: string): Promise<string[]>;
$getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]>;
$getTxIdsForBlock(hash: string, fallbackToCore?: boolean): Promise<string[]>;
$getTxsForBlock(hash: string, fallbackToCore?: boolean): Promise<IEsploraApi.Transaction[]>;
$getBlockHash(height: number): Promise<string>;
$getBlockHeader(hash: string): Promise<string>;
$getBlock(hash: string): Promise<IEsploraApi.Block>;
$getRawBlock(hash: string): Promise<Buffer>;
$getAddress(address: string): Promise<IEsploraApi.Address>;
$getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
$getAddressUtxos(address: string): Promise<IEsploraApi.UTXO[]>;
$getAddressPrefix(prefix: string): string[];
$getScriptHash(scripthash: string): Promise<IEsploraApi.ScriptHash>;
$getScriptHashTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
$getScriptHashUtxos(scripthash: string): Promise<IEsploraApi.UTXO[]>;
$sendRawTransaction(rawTransaction: string): Promise<string>;
$testMempoolAccept(rawTransactions: string[], maxfeerate?: number): Promise<TestMempoolAcceptResult[]>;
$submitPackage(rawTransactions: string[], maxfeerate?: number, maxburnamount?: number): Promise<SubmitPackageResult>;

44
backend/src/api/bitcoin/bitcoin-api.ts
View file

@ -6,6 +6,7 @@ import blocks from '../blocks';
import mempool from '../mempool';
import { TransactionExtended } from '../../mempool.interfaces';
import transactionUtils from '../transaction-utils';
import { Common } from '../common';
class BitcoinApi implements AbstractBitcoinApi {
private rawMempoolCache: IBitcoinApi.RawMempool | null = null;
@ -94,6 +95,10 @@ class BitcoinApi implements AbstractBitcoinApi {
});
}
$getTransactionMerkleProof(txId: string): Promise<IEsploraApi.MerkleProof> {
throw new Error('Method getTransactionMerkleProof not supported by the Bitcoin RPC API.');
}
$getBlockHeightTip(): Promise<number> {
return this.bitcoindClient.getBlockCount();
}
@ -102,16 +107,22 @@ class BitcoinApi implements AbstractBitcoinApi {
return this.bitcoindClient.getBestBlockHash();
}
$getTxIdsForBlock(hash: string): Promise<string[]> {
$getTxIdsForBlock(hash: string, fallbackToCore = false): Promise<string[]> {
return this.bitcoindClient.getBlock(hash, 1)
.then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx);
}
async $getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]> {
async $getTxsForBlock(hash: string, fallbackToCore = false): Promise<IEsploraApi.Transaction[]> {
const verboseBlock: IBitcoinApi.VerboseBlock = await this.bitcoindClient.getBlock(hash, 2);
const transactions: IEsploraApi.Transaction[] = [];
for (const tx of verboseBlock.tx) {
const converted = await this.$convertTransaction(tx, true);
const converted = await this.$convertTransaction(tx, true, false, verboseBlock.confirmations === -1);
converted.status = {
confirmed: true,
block_height: verboseBlock.height,
block_hash: hash,
block_time: verboseBlock.time,
};
transactions.push(converted);
}
return transactions;
@ -148,6 +159,10 @@ class BitcoinApi implements AbstractBitcoinApi {
throw new Error('Method getAddressTransactions not supported by the Bitcoin RPC API.');
}
$getAddressUtxos(address: string): Promise<IEsploraApi.UTXO[]> {
throw new Error('Method getAddressUtxos not supported by the Bitcoin RPC API.');
}
$getScriptHash(scripthash: string): Promise<IEsploraApi.ScriptHash> {
throw new Error('Method getScriptHash not supported by the Bitcoin RPC API.');
}
@ -156,6 +171,10 @@ class BitcoinApi implements AbstractBitcoinApi {
throw new Error('Method getScriptHashTransactions not supported by the Bitcoin RPC API.');
}
$getScriptHashUtxos(scripthash: string): Promise<IEsploraApi.UTXO[]> {
throw new Error('Method getScriptHashUtxos not supported by the Bitcoin RPC API.');
}
$getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
return this.bitcoindClient.getRawMemPool();
}
@ -264,7 +283,7 @@ class BitcoinApi implements AbstractBitcoinApi {
return this.bitcoindClient.getNetworkHashPs(120, blockHeight);
}
protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean, lazyPrevouts = false): Promise<IEsploraApi.Transaction> {
protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean, lazyPrevouts = false, allowMissingPrevouts = false): Promise<IEsploraApi.Transaction> {
let esploraTransaction: IEsploraApi.Transaction = {
txid: transaction.txid,
version: transaction.version,
@ -293,7 +312,7 @@ class BitcoinApi implements AbstractBitcoinApi {
is_coinbase: !!vin.coinbase,
prevout: null,
scriptsig: vin.scriptSig && vin.scriptSig.hex || vin.coinbase || '',
scriptsig_asm: vin.scriptSig && transactionUtils.convertScriptSigAsm(vin.scriptSig.hex) || '',
scriptsig_asm: vin.scriptSig ? transactionUtils.convertScriptSigAsm(vin.scriptSig.hex) : (vin.coinbase ? transactionUtils.convertScriptSigAsm(vin.coinbase) : ''),
sequence: vin.sequence,
txid: vin.txid || '',
vout: vin.vout || 0,
@ -313,7 +332,13 @@ class BitcoinApi implements AbstractBitcoinApi {
}
if (addPrevout) {
esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, false, lazyPrevouts);
try {
esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, false, lazyPrevouts);
} catch (e) {
if (!allowMissingPrevouts) {
throw e;
}
}
} else if (!transaction.confirmations) {
esploraTransaction = await this.$appendMempoolFeeData(esploraTransaction);
}
@ -360,6 +385,7 @@ class BitcoinApi implements AbstractBitcoinApi {
}
protected async $addPrevouts(transaction: TransactionExtended): Promise<TransactionExtended> {
let addedPrevouts = false;
for (const vin of transaction.vin) {
if (vin.prevout) {
continue;
@ -367,6 +393,12 @@ class BitcoinApi implements AbstractBitcoinApi {
const innerTx = await this.$getRawTransaction(vin.txid, false, false);
vin.prevout = innerTx.vout[vin.vout];
transactionUtils.addInnerScriptsToVin(vin);
addedPrevouts = true;
}
if (addedPrevouts) {
// re-calculate transaction flags now that we have full prevout data
transaction.flags = undefined; // clear existing flags to force full classification
transaction.flags = Common.getTransactionFlags(transaction, transaction.status?.block_height ?? blocks.getCurrentBlockHeight());
}
return transaction;
}

263
backend/src/api/bitcoin/bitcoin.routes.ts
View file

@ -21,6 +21,8 @@ import transactionRepository from '../../repositories/TransactionRepository';
import rbfCache from '../rbf-cache';
import { calculateMempoolTxCpfp } from '../cpfp';
import { handleError } from '../../utils/api';
import poolsUpdater from '../../tasks/pools-updater';
import chainTips from '../chain-tips';
const TXID_REGEX = /^[a-f0-9]{64}$/i;
const BLOCK_HASH_REGEX = /^[a-f0-9]{64}$/i;
@ -34,6 +36,7 @@ class BitcoinRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.$getCpfpInfo)
.get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', this.getDifficultyChange)
.get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', this.getRecommendedFees)
.get(config.MEMPOOL.API_URL_PREFIX + 'fees/precise', this.getPreciseRecommendedFees)
.get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', this.getMempoolBlocks)
.get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', this.getBackendInfo)
.get(config.MEMPOOL.API_URL_PREFIX + 'init-data', this.getInitData)
@ -54,8 +57,16 @@ class BitcoinRoutes {
.post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'chain-tips', this.getChainTips.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'stale-tips', this.getStaleTips.bind(this))
.post(config.MEMPOOL.API_URL_PREFIX + 'prevouts', this.$getPrevouts)
.post(config.MEMPOOL.API_URL_PREFIX + 'cpfp', this.getCpfpLocalTxs)
// Temporarily add txs/package endpoint for all backends until esplora supports it
.post(config.MEMPOOL.API_URL_PREFIX + 'txs/package', this.$submitPackage)
// Internal routes
.get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/definition/list', this.getBlockDefinitionHashes)
.get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/definition/current', this.getCurrentBlockDefinitionHash)
.get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/:definitionHash', this.getBlocksByDefinitionHash)
;
if (config.MEMPOOL.BACKEND !== 'esplora') {
@ -69,6 +80,7 @@ class BitcoinRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/hex', this.getRawTransaction)
.get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', this.getTransactionStatus)
.get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', this.getTransactionOutspends)
.get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/merkle-proof', this.getTransactionMerkleProof)
.get(config.MEMPOOL.API_URL_PREFIX + 'txs/outspends', this.$getBatchedOutspends)
.get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', this.getBlockHeader)
.get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/hash', this.getBlockTipHash)
@ -80,9 +92,11 @@ class BitcoinRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', this.getAddress)
.get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', this.getAddressTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/summary', this.getAddressTransactionSummary)
.get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/utxo', this.getAddressUtxo)
.get(config.MEMPOOL.API_URL_PREFIX + 'scripthash/:scripthash', this.getScriptHash)
.get(config.MEMPOOL.API_URL_PREFIX + 'scripthash/:scripthash/txs', this.getScriptHashTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'scripthash/:scripthash/txs/summary', this.getScriptHashTransactionSummary)
.get(config.MEMPOOL.API_URL_PREFIX + 'scripthash/:scripthash/utxo', this.getScriptHashUtxo)
.get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', this.getAddressPrefix)
;
}
@ -109,6 +123,16 @@ class BitcoinRoutes {
res.json(result);
}
private getPreciseRecommendedFees(req: Request, res: Response) {
if (!mempool.isInSync()) {
res.statusCode = 503;
res.send('Service Unavailable');
return;
}
const result = feeApi.getPreciseRecommendedFee();
res.json(result);
}
private getMempoolBlocks(req: Request, res: Response) {
try {
const result = mempoolBlocks.getMempoolBlocks();
@ -399,8 +423,8 @@ class BitcoinRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * cacheDuration).toUTCString());
res.json(block);
} catch (e) {
handleError(req, res, 500, 'Failed to get block');
} catch (e: any) {
handleError(req, res, e?.response?.status === 404 ? 404 : 500, 'Failed to get block');
}
}
@ -462,7 +486,7 @@ class BitcoinRoutes {
private async getBlocks(req: Request, res: Response) {
try {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(await blocks.$getBlocks(height, 15));
@ -476,7 +500,7 @@ class BitcoinRoutes {
private async getBlocksByBulk(req: Request, res: Response) {
try {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid - Not implemented
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid - Not implemented
handleError(req, res, 404, `This API is only available for Bitcoin networks`);
return;
}
@ -516,6 +540,46 @@ class BitcoinRoutes {
}
}
private async getChainTips(req: Request, res: Response) {
try {
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
const tips = await chainTips.getChainTips();
if (tips.length > 0) {
res.json(tips);
} else {
handleError(req, res, 503, `Temporarily unavailable`);
return;
}
} else { // Liquid
handleError(req, res, 404, `This API is only available for Bitcoin networks`);
return;
}
} catch (e) {
handleError(req, res, 500, 'Failed to get chain tips');
}
}
private async getStaleTips(req: Request, res: Response) {
try {
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
const tips = await chainTips.getStaleTips();
if (tips.length > 0) {
res.json(tips);
} else {
handleError(req, res, 503, `Temporarily unavailable`);
return;
}
} else { // Liquid
handleError(req, res, 404, `This API is only available for Bitcoin networks`);
return;
}
} catch (e) {
handleError(req, res, 500, 'Failed to get stale tips');
}
}
private async getLegacyBlocks(req: Request, res: Response) {
try {
const returnBlocks: IEsploraApi.Block[] = [];
@ -637,6 +701,28 @@ class BitcoinRoutes {
}
}
private async getAddressUtxo(req: Request, res: Response): Promise<void> {
if (config.MEMPOOL.BACKEND === 'none') {
handleError(req, res, 405, 'Address lookups cannot be used with bitcoind as backend.');
return;
}
if (!ADDRESS_REGEX.test(req.params.address)) {
handleError(req, res, 501, `Invalid address`);
return;
}
try {
const addressData = await bitcoinApi.$getAddressUtxos(req.params.address);
res.json(addressData);
} catch (e) {
if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
handleError(req, res, 413, e.message);
return;
}
handleError(req, res, 500, 'Failed to get address');
}
}
private async getAddressTransactionSummary(req: Request, res: Response): Promise<void> {
if (config.MEMPOOL.BACKEND !== 'esplora') {
handleError(req, res, 405, 'Address summary lookups require mempool/electrs backend.');
@ -696,6 +782,30 @@ class BitcoinRoutes {
}
}
private async getScriptHashUtxo(req: Request, res: Response): Promise<void> {
if (config.MEMPOOL.BACKEND === 'none') {
handleError(req, res, 405, 'Address lookups cannot be used with bitcoind as backend.');
return;
}
if (!SCRIPT_HASH_REGEX.test(req.params.scripthash)) {
handleError(req, res, 501, `Invalid scripthash`);
return;
}
try {
// electrum expects scripthashes in little-endian
const electrumScripthash = req.params.scripthash.match(/../g)?.reverse().join('') ?? '';
const addressData = await bitcoinApi.$getScriptHashUtxos(electrumScripthash);
res.json(addressData);
} catch (e) {
if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
handleError(req, res, 413, e.message);
return;
}
handleError(req, res, 500, 'Failed to get script hash');
}
}
private async getScriptHashTransactionSummary(req: Request, res: Response): Promise<void> {
if (config.MEMPOOL.BACKEND !== 'esplora') {
handleError(req, res, 405, 'Scripthash summary lookups require mempool/electrs backend.');
@ -739,6 +849,52 @@ class BitcoinRoutes {
}
}
private async getBlockDefinitionHashes(req: Request, res: Response): Promise<void> {
try {
const result = await blocks.$getBlockDefinitionHashes();
if (!result) {
handleError(req, res, 503, `Service Temporarily Unavailable`);
return;
}
res.setHeader('content-type', 'application/json');
res.send(result);
} catch (e) {
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
private async getCurrentBlockDefinitionHash(req: Request, res: Response): Promise<void> {
try {
const currentSha = await poolsUpdater.getShaFromDb();
if (!currentSha) {
handleError(req, res, 503, `Service Temporarily Unavailable`);
return;
}
res.setHeader('content-type', 'text/plain');
res.send(currentSha);
} catch (e) {
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
private async getBlocksByDefinitionHash(req: Request, res: Response): Promise<void> {
try {
if (typeof(req.params.definitionHash) !== 'string') {
res.status(400).send('Parameter "hash" must be a valid string');
return;
}
const blocksHash = await blocks.$getBlocksByDefinitionHash(req.params.definitionHash as string);
if (!blocksHash) {
handleError(req, res, 503, `Service Temporarily Unavailable`);
return;
}
res.setHeader('content-type', 'application/json');
res.send(blocksHash);
} catch (e) {
handleError(req, res, 500, e instanceof Error ? e.message : e);
}
}
private getBlockTipHeight(req: Request, res: Response) {
try {
const result = blocks.getCurrentBlockHeight();
@ -868,6 +1024,19 @@ class BitcoinRoutes {
}
}
private async getTransactionMerkleProof(req: Request, res: Response): Promise<void> {
if (!TXID_REGEX.test(req.params.txId)) {
handleError(req, res, 501, `Invalid transaction ID`);
return;
}
try {
const result = await bitcoinApi.$getTransactionMerkleProof(req.params.txId);
res.json(result);
} catch (e) {
handleError(req, res, 500, e instanceof Error ? e.message : 'Failed to get transaction merkle proof');
}
}
private getDifficultyChange(req: Request, res: Response) {
try {
const da = difficultyAdjustment.getDifficultyAdjustment();
@ -930,6 +1099,92 @@ class BitcoinRoutes {
}
}
private async $getPrevouts(req: Request, res: Response) {
try {
const outpoints = req.body;
if (!Array.isArray(outpoints) || outpoints.some((item) => !/^[a-fA-F0-9]{64}$/.test(item.txid) || typeof item.vout !== 'number')) {
handleError(req, res, 400, 'Invalid outpoints format');
return;
}
if (outpoints.length > 100) {
handleError(req, res, 400, 'Too many outpoints requested');
return;
}
const result = Array(outpoints.length).fill(null);
const memPool = mempool.getMempool();
for (let i = 0; i < outpoints.length; i++) {
const outpoint = outpoints[i];
let prevout: IEsploraApi.Vout | null = null;
let unconfirmed: boolean | null = null;
const mempoolTx = memPool[outpoint.txid];
if (mempoolTx) {
if (outpoint.vout < mempoolTx.vout.length) {
prevout = mempoolTx.vout[outpoint.vout];
unconfirmed = true;
}
} else {
try {
const rawPrevout = await bitcoinClient.getTxOut(outpoint.txid, outpoint.vout, false);
if (rawPrevout) {
prevout = {
value: Math.round(rawPrevout.value * 100000000),
scriptpubkey: rawPrevout.scriptPubKey.hex,
scriptpubkey_asm: rawPrevout.scriptPubKey.asm ? transactionUtils.convertScriptSigAsm(rawPrevout.scriptPubKey.hex) : '',
scriptpubkey_type: transactionUtils.translateScriptPubKeyType(rawPrevout.scriptPubKey.type),
scriptpubkey_address: rawPrevout.scriptPubKey && rawPrevout.scriptPubKey.address ? rawPrevout.scriptPubKey.address : '',
};
unconfirmed = false;
}
} catch (e) {
// Ignore bitcoin client errors, just leave prevout as null
}
}
if (prevout) {
result[i] = { prevout, unconfirmed };
}
}
res.json(result);
} catch (e) {
handleError(req, res, 500, 'Failed to get prevouts');
}
}
private getCpfpLocalTxs(req: Request, res: Response) {
try {
const transactions = req.body;
if (!Array.isArray(transactions) || transactions.some(tx =>
!tx || typeof tx !== 'object' ||
!/^[a-fA-F0-9]{64}$/.test(tx.txid) ||
typeof tx.weight !== 'number' ||
typeof tx.sigops !== 'number' ||
typeof tx.fee !== 'number' ||
!Array.isArray(tx.vin) ||
!Array.isArray(tx.vout)
)) {
handleError(req, res, 400, 'Invalid transactions format');
return;
}
if (transactions.length > 1) {
handleError(req, res, 400, 'More than one transaction is not supported yet');
return;
}
const cpfpInfo = calculateMempoolTxCpfp(transactions[0], mempool.getMempool(), true);
res.json([cpfpInfo]);
} catch (e) {
handleError(req, res, 500, 'Failed to calculate CPFP info');
}
}
}
export default new BitcoinRoutes();

7
backend/src/api/bitcoin/electrum-api.interface.ts
View file

@ -9,4 +9,11 @@ export namespace IElectrumApi {
tx_hash: string;
fee?: number;
}
export interface ScriptHashUtxos {
tx_pos: number;
value: number;
tx_hash: string;
height: number;
}
}

52
backend/src/api/bitcoin/electrum-api.ts
View file

@ -160,6 +160,15 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
}
}
async $getAddressUtxos(address: string): Promise<IEsploraApi.UTXO[]> {
const addressInfo = await this.bitcoindClient.validateAddress(address);
if (!addressInfo || !addressInfo.isvalid) {
return [];
}
const scripthash = this.encodeScriptHash(addressInfo.scriptPubKey);
return this.$getScriptHashUtxos(scripthash);
}
async $getScriptHashTransactions(scripthash: string, lastSeenTxId?: string): Promise<IEsploraApi.Transaction[]> {
try {
loadingIndicators.setProgress('address-' + scripthash, 0);
@ -197,6 +206,49 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
}
}
async $getScriptHashUtxos(scripthash: string): Promise<IEsploraApi.UTXO[]> {
const utxos = await this.$getScriptHashUnspent(scripthash);
const result: IEsploraApi.UTXO[] = [];
for(let utxo of utxos) {
if(utxo.height===0) {
//Unconfirmed
result.push({
txid: utxo.tx_hash,
vout: utxo.tx_pos,
status: {
confirmed: false
},
value: utxo.value
});
} else {
//Confirmed
const blockHash = await this.$getBlockHash(utxo.height);
const block = await this.$getBlock(blockHash);
result.push({
txid: utxo.tx_hash,
vout: utxo.tx_pos,
status: {
confirmed: true,
block_height: utxo.height,
block_hash: blockHash,
block_time: block.timestamp
},
value: utxo.value
});
}
}
return result;
}
private $getScriptHashUnspent(scriptHash: string): Promise<IElectrumApi.ScriptHashUtxos[]> {
return this.electrumClient.blockchainScripthash_listunspent(scriptHash);
}
async $getTransactionMerkleProof(txId: string): Promise<IEsploraApi.MerkleProof> {
const tx = await this.$getRawTransaction(txId);
return this.electrumClient.blockchainTransaction_getMerkle(txId, tx.status.block_height);
}
private $getScriptHashBalance(scriptHash: string): Promise<IElectrumApi.ScriptHashBalance> {
return this.electrumClient.blockchainScripthash_getBalance(this.encodeScriptHash(scriptHash));
}

18
backend/src/api/bitcoin/esplora-api.interface.ts
View file

@ -186,4 +186,22 @@ export namespace IEsploraApi {
time: number;
tx_position?: number;
}
export interface MerkleProof {
merkle: string[];
block_height: number;
pos: number;
}
export interface UTXO {
txid: string;
vout: number;
status: {
confirmed: boolean;
block_height?: number;
block_hash?: string;
block_time?: number;
},
value: number;
}
}

97
backend/src/api/bitcoin/esplora-api.ts
View file

@ -1,12 +1,14 @@
import config from '../../config';
import axios, { isAxiosError } from 'axios';
import http from 'http';
import https from 'https';
import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-factory';
import { IEsploraApi } from './esplora-api.interface';
import logger from '../../logger';
import { Common } from '../common';
import { SubmitPackageResult, TestMempoolAcceptResult } from './bitcoin-api.interface';
import os from 'os';
import { bitcoinCoreApi } from './bitcoin-api-factory';
interface FailoverHost {
host: string,
rtts: number[],
@ -23,6 +25,7 @@ interface FailoverHost {
publicDomain: string,
hashes: {
frontend?: string,
hybrid?: string,
backend?: string,
electrs?: string,
lastUpdated: number,
@ -36,7 +39,7 @@ class FailoverRouter {
maxHeight: number = 0;
hosts: FailoverHost[];
multihost: boolean;
gitHashInterval: number = 600000; // 10 minutes
gitHashInterval: number = 60000; // 1 minute
pollInterval: number = 60000; // 1 minute
pollTimer: NodeJS.Timeout | null = null;
pollConnection = axios.create();
@ -111,7 +114,7 @@ class FailoverRouter {
for (const host of this.hosts) {
try {
const result = await (host.socket
? this.pollConnection.get<number>('/blocks/tip/height', { socketPath: host.host, timeout: config.ESPLORA.FALLBACK_TIMEOUT })
? this.pollConnection.get<number>('http://api/blocks/tip/height', { socketPath: host.host, timeout: config.ESPLORA.FALLBACK_TIMEOUT })
: this.pollConnection.get<number>(host.host + '/blocks/tip/height', { timeout: config.ESPLORA.FALLBACK_TIMEOUT })
);
if (result) {
@ -142,7 +145,8 @@ class FailoverRouter {
if (Date.now() - host.hashes.lastUpdated > this.gitHashInterval) {
await Promise.all([
this.$updateFrontendGitHash(host),
this.$updateBackendGitHash(host)
this.$updateBackendGitHash(host),
config.MEMPOOL.OFFICIAL ? this.$updateHybridGitHash(host) : Promise.resolve(),
]);
host.hashes.lastUpdated = Date.now();
}
@ -251,6 +255,47 @@ class FailoverRouter {
if (match && match[1]?.length) {
host.hashes.frontend = match[1];
}
const hybridMatch = response.data.match(/GIT_COMMIT_HASH_MEMPOOL_SPACE\s*=\s*['"](.*?)['"]/);
if (hybridMatch && hybridMatch[1]?.length) {
host.hashes.hybrid = hybridMatch[1];
}
} catch (e) {
// failed to get frontend build hash - do nothing
}
}
private async $updateHybridGitHash(host: FailoverHost): Promise<void> {
try {
const response: string = await new Promise((resolve, reject) => {
const req = https.request({
hostname: host.publicDomain.replace('https://', '').replace('http://', ''),
port: 443,
path: '/en-US/resources/config.js',
method: 'GET',
headers: {
'Host': 'mempool.space'
},
timeout: config.ESPLORA.FALLBACK_TIMEOUT,
}, (res) => {
let data = '';
res.on('data', chunk => data += chunk);
res.on('end', () => {
if (res.statusCode === 200) {
resolve(data);
} else {
reject(new Error(`Failed to get hybrid git hash: ${res.statusCode}`));
}
});
});
req.on('error', (e) => {
reject(e);
});
req.end();
});
const match = response.match(/GIT_COMMIT_HASH_MEMPOOL_SPACE\s*=\s*['"](.*?)['"]/);
if (match && match[1]?.length) {
host.hashes.hybrid = match[1];
}
} catch (e) {
// failed to get frontend build hash - do nothing
}
@ -288,7 +333,7 @@ class FailoverRouter {
let url;
if (host.socket) {
axiosConfig = { socketPath: host.host, timeout: config.ESPLORA.REQUEST_TIMEOUT, responseType };
url = path;
url = 'http://api' + path;
} else {
axiosConfig = { timeout: config.ESPLORA.REQUEST_TIMEOUT, responseType };
url = host.host + path;
@ -352,6 +397,10 @@ class ElectrsApi implements AbstractBitcoinApi {
return this.failoverRouter.$get<string>('/tx/' + txId + '/hex');
}
$getTransactionMerkleProof(txId: string): Promise<IEsploraApi.MerkleProof> {
return this.failoverRouter.$get<IEsploraApi.MerkleProof>('/tx/' + txId + '/merkle-proof');
}
$getBlockHeightTip(): Promise<number> {
return this.failoverRouter.$get<number>('/blocks/tip/height');
}
@ -360,12 +409,36 @@ class ElectrsApi implements AbstractBitcoinApi {
return this.failoverRouter.$get<string>('/blocks/tip/hash');
}
$getTxIdsForBlock(hash: string): Promise<string[]> {
return this.failoverRouter.$get<string[]>('/block/' + hash + '/txids');
async $getTxIdsForBlock(hash: string, fallbackToCore = false): Promise<string[]> {
try {
const txids = await this.failoverRouter.$get<string[]>('/block/' + hash + '/txids');
return txids;
} catch (e) {
if (fallbackToCore && isAxiosError(e) && e.response?.status === 404) {
// might be a stale block, see if Core has it?
const coreBlock = await bitcoinCoreApi.$getBlock(hash);
if (coreBlock?.stale) {
return bitcoinCoreApi.$getTxIdsForBlock(hash);
}
}
throw e;
}
}
$getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]> {
return this.failoverRouter.$get<IEsploraApi.Transaction[]>('/internal/block/' + hash + '/txs');
async $getTxsForBlock(hash: string, fallbackToCore = false): Promise<IEsploraApi.Transaction[]> {
try {
const txs = await this.failoverRouter.$get<IEsploraApi.Transaction[]>('/internal/block/' + hash + '/txs');
return txs;
} catch (e) {
if (fallbackToCore && isAxiosError(e) && e.response?.status === 404) {
// might be a stale block, see if Core has it?
const coreBlock = await bitcoinCoreApi.$getBlock(hash);
if (coreBlock?.stale) {
return bitcoinCoreApi.$getTxsForBlock(hash);
}
}
throw e;
}
}
$getBlockHash(height: number): Promise<string> {
@ -393,6 +466,10 @@ class ElectrsApi implements AbstractBitcoinApi {
throw new Error('Method getAddressTransactions not implemented.');
}
$getAddressUtxos(address: string): Promise<IEsploraApi.UTXO[]> {
return this.failoverRouter.$get<IEsploraApi.UTXO[]>('/address/' + address + '/utxo');
}
$getScriptHash(scripthash: string): Promise<IEsploraApi.ScriptHash> {
throw new Error('Method getScriptHash not implemented.');
}
@ -401,6 +478,10 @@ class ElectrsApi implements AbstractBitcoinApi {
throw new Error('Method getScriptHashTransactions not implemented.');
}
$getScriptHashUtxos(scripthash: string): Promise<IEsploraApi.UTXO[]> {
throw new Error('Method getScriptHashUtxos not implemented.');
}
$getAddressPrefix(prefix: string): string[] {
throw new Error('Method not implemented.');
}

406
backend/src/api/blocks.ts
View file

@ -33,8 +33,8 @@ import AccelerationRepository from '../repositories/AccelerationRepository';
import { calculateFastBlockCpfp, calculateGoodBlockCpfp } from './cpfp';
import mempool from './mempool';
import CpfpRepository from '../repositories/CpfpRepository';
import accelerationApi from './services/acceleration';
import { parseDATUMTemplateCreator } from '../utils/bitcoin-script';
import database from '../database';
class Blocks {
private blocks: BlockExtended[] = [];
@ -94,17 +94,19 @@ class Blocks {
txIds: string[] | null = null,
quiet: boolean = false,
addMempoolData: boolean = false,
stale: boolean = false,
): Promise<TransactionExtended[]> {
const isEsplora = config.MEMPOOL.BACKEND === 'esplora';
const transactionMap: { [txid: string]: TransactionExtended } = {};
if (!txIds) {
txIds = await bitcoinApi.$getTxIdsForBlock(blockHash);
txIds = await bitcoinApi.$getTxIdsForBlock(blockHash, stale);
}
const mempool = memPool.getMempool();
let foundInMempool = 0;
let totalFound = 0;
let missing = 0;
// Copy existing transactions from the mempool
if (!onlyCoinbase) {
@ -136,14 +138,17 @@ class Blocks {
} catch (e) {
const msg = `Cannot fetch coinbase tx ${txIds[0]}. Reason: ` + (e instanceof Error ? e.message : e);
logger.err(msg);
throw new Error(msg);
// tolerate this error for stale blocks (the cb transaction won't be accessible via normal RPCs)
if (!stale) {
throw new Error(msg);
}
}
}
// Fetch remaining txs in bulk
if (isEsplora && (txIds.length - totalFound > 500)) {
if ((isEsplora && (txIds.length - totalFound > 500)) || stale) {
try {
const rawTransactions = await bitcoinApi.$getTxsForBlock(blockHash);
const rawTransactions = await bitcoinApi.$getTxsForBlock(blockHash, stale);
for (const tx of rawTransactions) {
if (!transactionMap[tx.txid]) {
transactionMap[tx.txid] = addMempoolData ? transactionUtils.extendMempoolTransaction(tx) : transactionUtils.extendTransaction(tx);
@ -184,7 +189,6 @@ class Blocks {
}
// Require all transactions to be present
// (we should have thrown an error already if a tx request failed)
if (txIds.some(txid => !transactionMap[txid])) {
const msg = `Failed to fetch ${txIds.length - totalFound} transactions from block`;
logger.err(msg);
@ -244,7 +248,7 @@ class Blocks {
*/
private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
const blk: Partial<BlockExtended> = Object.assign({}, block);
const extras: Partial<BlockExtension> = {};
@ -267,7 +271,7 @@ class Blocks {
extras.segwitTotalSize = 0;
extras.segwitTotalWeight = 0;
} else {
const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id);
const stats: IBitcoinApi.BlockStats = await this.$getBlockStats(block, transactions);
let feeStats = {
medianFee: stats.feerate_percentiles[2], // 50th percentiles
feeRange: [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(),
@ -296,7 +300,7 @@ class Blocks {
extras.medianFeeAmt = extras.feePercentiles[3];
}
}
extras.virtualSize = block.weight / 4.0;
if (coinbaseTx?.vout.length > 0) {
extras.coinbaseAddress = coinbaseTx.vout[0].scriptpubkey_address ?? null;
@ -323,7 +327,7 @@ class Blocks {
extras.totalInputAmt = null;
}
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
let pool: PoolTag;
if (coinbaseTx !== undefined) {
pool = await this.$findBlockMiner(coinbaseTx);
@ -368,6 +372,79 @@ class Blocks {
return <BlockExtended>blk;
}
private async $getBlockStats(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<IBitcoinApi.BlockStats> {
if (!block.stale) {
return bitcoinClient.getBlockStats(block.id);
}
// TODO: make these match the definitions used by the RPC response
const totalFee = transactions.reduce((acc, tx) => acc + tx.fee, 0);
const totalVsize = transactions.reduce((acc, tx) => acc + tx.vsize, 0);
const totalReward = transactions[0].vout.reduce((acc, vout) => acc + vout.value, 0);
const sortedByFee = transactions.sort((a, b) => a.fee - b.fee);
const sortedByVsize = transactions.sort((a, b) => a.vsize - b.vsize);
const sortedByFeerate = transactions.sort((a, b) => (a.fee / a.weight) - (b.fee / b.weight));
const sortedFeerates = sortedByFeerate.map(tx => (tx.fee / (tx.weight / 4)));
const avgfee = totalFee / transactions.length;
const avgfeerate = totalFee / (block.weight / 4);
const avgtxsize = totalVsize / transactions.length;
const medianfee = sortedByFee[Math.floor(transactions.length / 2)].fee;
const mediantime = block.timestamp;
const mediantxsize = sortedByVsize[Math.floor(transactions.length / 2)].vsize;
const minfee = sortedByFee[0].fee;
const maxfee = sortedByFee[sortedByFee.length - 1].fee;
const minfeerate = sortedFeerates[0];
const maxfeerate = sortedFeerates[sortedFeerates.length - 1];
const mintxsize = sortedByVsize[0].vsize;
const maxtxsize = sortedByVsize[sortedByVsize.length - 1].vsize;
const ins = transactions.reduce((acc, tx) => acc + tx.vin.length, 0);
const outs = transactions.reduce((acc, tx) => acc + tx.vout.length, 0);
const subsidy = totalReward - totalFee;
const swtotal_size = 0;
const swtotal_weight = 0;
const swtxs = 0;
const time = block.timestamp;
const total_out = transactions.reduce((acc, tx) => acc + tx.vout.reduce((acc, vout) => acc + vout.value, 0), 0);
const total_size = block.size;
const total_weight = block.weight;
const totalfee = totalFee;
const txs = transactions.length;
const utxo_increase = 0;
const utxo_size_inc = 0;
return {
avgfee,
avgfeerate,
avgtxsize,
blockhash: block.id,
feerate_percentiles: [minfeerate, sortedFeerates[Math.floor(transactions.length / 4)], medianfee, sortedFeerates[Math.floor(transactions.length * 3 / 4)], maxfeerate],
height: block.height,
ins,
maxfee,
maxfeerate,
maxtxsize,
medianfee,
mediantime,
mediantxsize,
minfee,
minfeerate,
mintxsize,
outs,
subsidy,
swtotal_size,
swtotal_weight,
swtxs,
time,
total_out,
total_size,
total_weight,
totalfee,
txs,
utxo_increase,
utxo_size_inc,
};
}
/**
* Try to find which miner found the block
* @param txMinerInfo
@ -452,16 +529,7 @@ class Blocks {
indexedThisRun = 0;
}
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendMempoolTransaction(tx));
const cpfpSummary = await this.$indexCPFP(block.hash, block.height, txs);
if (cpfpSummary) {
await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary
}
} else {
await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary
}
await this.$indexBlockSummary(block.hash, block.height, block.stale);
// Logging
indexedThisRun++;
@ -479,6 +547,18 @@ class Blocks {
}
}
public async $indexBlockSummary(hash: string, height: number, stale?: boolean): Promise<void> {
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(hash, stale)).map(tx => transactionUtils.extendMempoolTransaction(tx));
const cpfpSummary = await this.$indexCPFP(hash, height, txs, stale);
if (cpfpSummary) {
await this.$getStrippedBlockTransactions(hash, true, true, cpfpSummary, height); // This will index the block summary
}
} else {
await this.$getStrippedBlockTransactions(hash, true, true); // This will index the block summary
}
}
/**
* [INDEXING] Index transaction CPFP data for all blocks
*/
@ -582,8 +662,7 @@ class Blocks {
return;
}
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
const currentBlockHeight = blockchainInfo.blocks;
const currentBlockHeight = this.getCurrentBlockHeight();
const targetSummaryVersion: number = 1;
const targetTemplateVersion: number = 1;
@ -626,7 +705,7 @@ class Blocks {
if (unclassifiedBlocks[height]) {
const blockHash = unclassifiedBlocks[height];
// fetch transactions
txs = (await bitcoinApi.$getTxsForBlock(blockHash)).map(tx => transactionUtils.extendMempoolTransaction(tx)) || [];
txs = (await bitcoinApi.$getTxsForBlock(blockHash, true)).map(tx => transactionUtils.extendMempoolTransaction(tx)) || [];
// add CPFP
const cpfpSummary = calculateGoodBlockCpfp(height, txs, []);
// classify
@ -804,7 +883,7 @@ class Blocks {
}
const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, block.timestamp, true, null, true);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, block.timestamp, !block.stale, null, true, block.stale);
const blockExtended = await this.$getBlockExtended(block, transactions);
newlyIndexed++;
@ -836,6 +915,7 @@ class Blocks {
let fastForwarded = false;
let handledBlocks = 0;
const lastBlockHeight = this.currentBlockHeight;
const blockHeightTip = await bitcoinCoreApi.$getBlockHeightTip();
this.updateTimerProgress(timer, 'got block height tip');
@ -844,16 +924,6 @@ class Blocks {
} else {
this.currentBlockHeight = this.blocks[this.blocks.length - 1].height;
}
if (this.currentBlockHeight >= 503) {
try {
const quarterEpochBlockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight - 503);
const quarterEpochBlock = await bitcoinApi.$getBlock(quarterEpochBlockHash);
this.quarterEpochBlockTime = quarterEpochBlock?.timestamp;
} catch (e) {
this.quarterEpochBlockTime = null;
logger.warn('failed to update last epoch block time: ' + (e instanceof Error ? e.message : e));
}
}
if (blockHeightTip - this.currentBlockHeight > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 2) {
logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${config.MEMPOOL.INITIAL_BLOCKS_AMOUNT} recent blocks`);
@ -892,17 +962,20 @@ class Blocks {
}
}
const heightChanged = lastBlockHeight !== this.currentBlockHeight;
// make sure to update the quarter epoch block time now if we won't do it inside the loop
if (this.currentBlockHeight >= blockHeightTip && (heightChanged || this.quarterEpochBlockTime == null)) {
await this.updateQuarterEpochBlockTime();
}
while (this.currentBlockHeight < blockHeightTip) {
if (this.currentBlockHeight === 0) {
this.currentBlockHeight = blockHeightTip;
await this.updateQuarterEpochBlockTime();
} else {
this.currentBlockHeight++;
await this.updateQuarterEpochBlockTime();
logger.debug(`New block found (#${this.currentBlockHeight})!`);
// skip updating the orphan block cache if we've fallen behind the chain tip
if (this.currentBlockHeight >= blockHeightTip - 2) {
this.updateTimerProgress(timer, `getting orphaned blocks for ${this.currentBlockHeight}`);
await chainTips.updateOrphanedBlocks();
}
}
this.updateTimerProgress(timer, `getting block data for ${this.currentBlockHeight}`);
@ -931,38 +1004,7 @@ class Blocks {
if (Common.indexingEnabled()) {
if (!fastForwarded) {
const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
this.updateTimerProgress(timer, `got block by height for ${this.currentBlockHeight}`);
if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) {
logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`, logger.tags.mining);
// We assume there won't be a reorg with more than 10 block depth
this.updateTimerProgress(timer, `rolling back diverged chain from ${this.currentBlockHeight}`);
await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
await HashratesRepository.$deleteLastEntries();
await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
await AccelerationRepository.$deleteAccelerationsFrom(lastBlock.height - 10);
this.blocks = this.blocks.slice(0, -10);
this.updateTimerProgress(timer, `rolled back chain divergence from ${this.currentBlockHeight}`);
for (let i = 10; i >= 0; --i) {
const newBlock = await this.$indexBlock(lastBlock.height - i);
this.blocks.push(newBlock);
this.updateTimerProgress(timer, `reindexed block`);
let newCpfpSummary;
if (config.MEMPOOL.CPFP_INDEXING) {
newCpfpSummary = await this.$indexCPFP(newBlock.id, lastBlock.height - i);
this.updateTimerProgress(timer, `reindexed block cpfp`);
}
await this.$getStrippedBlockTransactions(newBlock.id, true, true, newCpfpSummary, newBlock.height);
this.updateTimerProgress(timer, `reindexed block summary`);
}
await mining.$indexDifficultyAdjustments();
await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
this.updateTimerProgress(timer, `reindexed difficulty adjustments`);
logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`, logger.tags.mining);
indexer.reindex();
websocketHandler.handleReorg();
}
await this.$handleReorgs(blockExtended, timer);
}
await blocksRepository.$saveBlockInDatabase(blockExtended);
@ -1034,6 +1076,12 @@ class Blocks {
this.currentBits = block.bits;
}
// skip updating the orphan block cache if we've fallen behind the chain tip
if (this.currentBlockHeight >= blockHeightTip - 2) {
this.updateTimerProgress(timer, `getting orphaned blocks for ${this.currentBlockHeight}`);
await chainTips.updateOrphanedBlocks();
}
// wait for pending async callbacks to finish
this.updateTimerProgress(timer, `waiting for async callbacks to complete for ${this.currentBlockHeight}`);
await Promise.all(callbackPromises);
@ -1098,21 +1146,121 @@ class Blocks {
}
}
/**
* Index a block if it's missing from the database. Returns the block after indexing
*/
public async $indexBlock(height: number): Promise<BlockExtended> {
if (Common.indexingEnabled()) {
private async updateQuarterEpochBlockTime(): Promise<void> {
if (this.currentBlockHeight >= 503) {
try {
const quarterEpochBlockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight - 503);
const quarterEpochBlock = await bitcoinApi.$getBlock(quarterEpochBlockHash);
this.quarterEpochBlockTime = quarterEpochBlock?.timestamp;
} catch (e) {
this.quarterEpochBlockTime = null;
logger.warn('failed to update last epoch block time: ' + (e instanceof Error ? e.message : e));
}
}
}
public async $indexBlockByHeight(height: number, skipDb = false): Promise<BlockExtended> {
if (Common.indexingEnabled() && !skipDb) {
const dbBlock = await blocksRepository.$getBlockByHeight(height);
if (dbBlock !== null) {
return dbBlock;
}
}
// not already indexed
const hash = await bitcoinApi.$getBlockHash(height);
return this.$indexBlock(hash);
}
const blockHash = await bitcoinApi.$getBlockHash(height);
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, block.timestamp, true);
private async $handleReorgs(blockExtended: BlockExtended, timer: any): Promise<void> {
let forkTail = blockExtended;
let currentlyIndexed = await blocksRepository.$getBlockByHeight(forkTail.height - 1);
this.updateTimerProgress(timer, `got block by height at previous tip ${forkTail.height - 1}`);
// previous blockhash is not what we expected: there has been a reorg
if (currentlyIndexed !== null && forkTail.previousblockhash !== currentlyIndexed.id) {
logger.warn(`Chain divergence detected at block ${blockExtended.height}, re-indexing most recent data`, logger.tags.mining);
this.updateTimerProgress(timer, `reconnecting diverged chain from ${this.currentBlockHeight}`);
const newBlocks: BlockExtended[] = [];
// walk back along the chain until we reach the fork point
while (currentlyIndexed !== null && forkTail.previousblockhash !== currentlyIndexed.id) {
const newBlock = await this.$indexBlock(forkTail.previousblockhash);
await blocksRepository.$setCanonicalBlockAtHeight(newBlock.id, newBlock.height);
newBlocks.push(newBlock);
this.updateTimerProgress(timer, `reindexed block at ${newBlock.height} (${newBlock.id})`);
let newCpfpSummary;
if (config.MEMPOOL.CPFP_INDEXING) {
newCpfpSummary = await this.$indexCPFP(newBlock.id, newBlock.height);
this.updateTimerProgress(timer, `reindexed block cpfp`);
}
await this.$getStrippedBlockTransactions(newBlock.id, true, true, newCpfpSummary, newBlock.height);
this.updateTimerProgress(timer, `reindexed block summary`);
forkTail = newBlock;
currentlyIndexed = await blocksRepository.$getBlockByHeight(forkTail.height - 1);
this.updateTimerProgress(timer, `got block by height for ${forkTail.height - 1}`);
}
// rebuild the block cache
let currentBlock = forkTail;
const cachedBlocksByHash = {};
for (const cached of this.blocks) {
cachedBlocksByHash[cached.id] = cached;
}
while (currentBlock.height > 0 && newBlocks.length < (config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4)) {
const newBlock = cachedBlocksByHash[currentBlock.previousblockhash] || await blocksRepository.$getBlockByHash(currentBlock.previousblockhash);
if (newBlock) {
newBlocks.push(newBlock);
currentBlock = newBlock;
} else {
break;
}
}
this.updateTimerProgress(timer, `rebuilt block cache`);
// force re-indexing of block-related data
await HashratesRepository.$deleteHashratesFromTimestamp(forkTail.timestamp - 604800);
await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(forkTail.height);
await cpfpRepository.$deleteClustersFrom(forkTail.height);
await AccelerationRepository.$deleteAccelerationsFrom(forkTail.height);
chainTips.clearOrphanCacheAboveHeight(forkTail.height);
this.updateTimerProgress(timer, `deleted stale block data`);
this.blocks = newBlocks.reverse();
if (this.blocks.length > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4) {
this.blocks = this.blocks.slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4);
}
this.updateTimerProgress(timer, `connected new best chain from ${forkTail.height} to ${this.currentBlockHeight}`);
await mining.$indexDifficultyAdjustments();
this.updateTimerProgress(timer, `reindexed difficulty adjustments`);
logger.info(`Re-indexed ${this.currentBlockHeight - forkTail.height} blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`, logger.tags.mining);
indexer.reindex();
websocketHandler.handleReorg();
}
}
/**
* Index a block if it's missing from the database. Returns the block after indexing
*/
public async $indexBlock(hash: string, block?: IEsploraApi.Block, skipDb = false): Promise<BlockExtended> {
if (Common.indexingEnabled() && !skipDb) {
const dbBlock = await blocksRepository.$getBlockByHash(hash);
if (dbBlock !== null) {
return dbBlock;
}
}
if (!block) {
// dont' bother trying to fetch orphan blocks from esplora
block = await (chainTips.isOrphaned(hash) ? bitcoinCoreApi.$getBlock(hash) : bitcoinApi.$getBlock(hash));
}
const transactions = await this.$getTransactionsExtended(hash, block.height, block.timestamp, !block.stale, null, false, false, block.stale);
const blockExtended = await this.$getBlockExtended(block, transactions);
if (block.stale) {
blockExtended.canonical = await bitcoinApi.$getBlockHash(block.height);
}
if (Common.indexingEnabled()) {
await blocksRepository.$saveBlockInDatabase(blockExtended);
@ -1121,38 +1269,25 @@ class Blocks {
return blockExtended;
}
public async $indexStaleBlock(hash: string): Promise<BlockExtended> {
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash);
const transactions = await this.$getTransactionsExtended(hash, block.height, block.timestamp, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
blockExtended.canonical = await bitcoinApi.$getBlockHash(block.height);
return blockExtended;
}
/**
* Get one block by its hash
*/
public async $getBlock(hash: string): Promise<BlockExtended | IEsploraApi.Block> {
public async $getBlock(hash: string, skipMemoryCache: boolean = false): Promise<BlockExtended | IEsploraApi.Block> {
// Check the memory cache
const blockByHash = this.getBlocks().find((b) => b.id === hash);
if (blockByHash) {
return blockByHash;
if (!skipMemoryCache) {
const blockByHash = this.getBlocks().find((b) => b.id === hash);
if (blockByHash) {
return blockByHash;
}
}
// Not Bitcoin network, return the block as it from the bitcoin backend
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) === false) {
return await bitcoinCoreApi.$getBlock(hash);
}
// Bitcoin network, add our custom data on top
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash);
if (block.stale) {
return await this.$indexStaleBlock(hash);
} else {
return await this.$indexBlock(block.height);
}
return await this.$indexBlock(hash);
}
public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,
@ -1200,20 +1335,19 @@ class Blocks {
};
summaryVersion = cpfpSummary.version;
} else {
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
summary = this.summarizeBlockTransactions(hash, height || 0, txs);
summaryVersion = 1;
} else {
// Call Core RPC
const block = await bitcoinClient.getBlock(hash, 2);
summary = this.summarizeBlock(block);
height = block.height;
}
const txs = (await bitcoinApi.$getTxsForBlock(hash, true)).map(tx => transactionUtils.extendTransaction(tx));
summary = this.summarizeBlockTransactions(hash, height || 0, txs);
summaryVersion = 1;
}
if (height == null) {
const block = await bitcoinApi.$getBlock(hash);
height = block.height;
// If the block is orphaned, use the height from the chaintips cache
const orphanedBlock = chainTips.getOrphanedBlock(hash);
if (orphanedBlock) {
height = orphanedBlock.height;
} else {
const block = await bitcoinApi.$getBlock(hash);
height = block.height;
}
}
// Index the response if needed
@ -1260,7 +1394,7 @@ class Blocks {
returnBlocks.push(block);
} else {
// Using indexing (find by height, index on the fly, save in database)
block = await this.$indexBlock(currentHeight);
block = await this.$indexBlockByHeight(currentHeight);
returnBlocks.push(block);
}
currentHeight--;
@ -1285,7 +1419,7 @@ class Blocks {
while (fromHeight <= toHeight) {
let block: BlockExtended | null = await blocksRepository.$getBlockByHeight(fromHeight);
if (!block) {
await this.$indexBlock(fromHeight);
await this.$indexBlockByHeight(fromHeight);
block = await blocksRepository.$getBlockByHeight(fromHeight);
if (!block) {
continue;
@ -1342,7 +1476,7 @@ class Blocks {
let summary;
let summaryVersion = 0;
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(cleanBlock.hash)).map(tx => transactionUtils.extendTransaction(tx));
const txs = (await bitcoinApi.$getTxsForBlock(cleanBlock.hash, cleanBlock.stale)).map(tx => transactionUtils.extendTransaction(tx));
summary = this.summarizeBlockTransactions(cleanBlock.hash, cleanBlock.height, txs);
summaryVersion = 1;
} else {
@ -1391,7 +1525,7 @@ class Blocks {
}
public async $getBlockAuditSummary(hash: string): Promise<BlockAudit | null> {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) && Common.auditIndexingEnabled()) {
return BlocksAuditsRepository.$getBlockAudit(hash);
} else {
return null;
@ -1399,7 +1533,7 @@ class Blocks {
}
public async $getBlockTxAuditSummary(hash: string, txid: string): Promise<TransactionAudit | null> {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) && Common.auditIndexingEnabled()) {
return BlocksAuditsRepository.$getBlockTxAudit(hash, txid);
} else {
return null;
@ -1422,11 +1556,11 @@ class Blocks {
return this.currentBlockHeight;
}
public async $indexCPFP(hash: string, height: number, txs?: MempoolTransactionExtended[]): Promise<CpfpSummary | null> {
public async $indexCPFP(hash: string, height: number, txs?: MempoolTransactionExtended[], stale?: boolean): Promise<CpfpSummary | null> {
let transactions = txs;
if (!transactions) {
if (config.MEMPOOL.BACKEND === 'esplora') {
transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendMempoolTransaction(tx));
transactions = (await bitcoinApi.$getTxsForBlock(hash, true)).map(tx => transactionUtils.extendMempoolTransaction(tx));
}
if (!transactions) {
const block = await bitcoinClient.getBlock(hash, 2);
@ -1440,7 +1574,9 @@ class Blocks {
if (transactions?.length != null) {
const summary = calculateFastBlockCpfp(height, transactions);
await this.$saveCpfp(hash, height, summary);
if (!stale) {
await this.$saveCpfp(hash, height, summary);
}
const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions);
await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats);
@ -1462,6 +1598,36 @@ class Blocks {
// not a fatal error, we'll try again next time the indexer runs
}
}
public async $getBlockDefinitionHashes(): Promise<string[] | null> {
try {
const [rows]: any = await database.query(`SELECT DISTINCT(definition_hash) FROM blocks WHERE stale = 0`);
if (rows && Array.isArray(rows)) {
return rows.map(r => r.definition_hash);
} else {
logger.debug(`Unable to retrieve list of blocks.definition_hash from db (no result)`);
return null;
}
} catch (e) {
logger.debug(`Unable to retrieve list of blocks.definition_hash from db (exception: ${e})`);
return null;
}
}
public async $getBlocksByDefinitionHash(definitionHash: string): Promise<string[] | null> {
try {
const [rows]: any = await database.query(`SELECT hash FROM blocks WHERE definition_hash = ? AND stale = 0`, [definitionHash]);
if (rows && Array.isArray(rows)) {
return rows.map(r => r.hash);
} else {
logger.debug(`Unable to retrieve list of blocks for definition hash ${definitionHash} from db (no result)`);
return null;
}
} catch (e) {
logger.debug(`Unable to retrieve list of blocks for definition hash ${definitionHash} from db (exception: ${e})`);
return null;
}
}
}
export default new Blocks();

154
backend/src/api/chain-tips.ts
View file

@ -1,5 +1,12 @@
import config from '../config';
import logger from '../logger';
import { BlockExtended } from '../mempool.interfaces';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
import bitcoinClient from './bitcoin/bitcoin-client';
import { IEsploraApi } from './bitcoin/esplora-api.interface';
import blocks from './blocks';
import { Common } from './common';
export interface ChainTip {
height: number;
@ -8,6 +15,11 @@ export interface ChainTip {
status: 'invalid' | 'active' | 'valid-fork' | 'valid-headers' | 'headers-only';
};
export interface StaleTip extends ChainTip {
stale: BlockExtended;
canonical: BlockExtended;
}
export interface OrphanedBlock {
height: number;
hash: string;
@ -17,18 +29,31 @@ export interface OrphanedBlock {
class ChainTips {
private chainTips: ChainTip[] = [];
private staleTips: Record<number, StaleTip> = {};
private orphanedBlocks: { [hash: string]: OrphanedBlock } = {};
private blockCache: { [hash: string]: OrphanedBlock } = {};
private orphansByHeight: { [height: number]: OrphanedBlock[] } = {};
private indexingOrphanedBlocks = false;
private indexingQueue: { blockhash?: string, block?: IEsploraApi.Block, tip: OrphanedBlock }[] = [];
private staleTipsCacheSize = 50;
private maxIndexingQueueSize = 100;
public async updateOrphanedBlocks(): Promise<void> {
try {
this.chainTips = await bitcoinClient.getChainTips();
const activeTipHeight = this.chainTips.find(tip => tip.status === 'active')?.height || (await bitcoinApi.$getBlockHeightTip());
let minIndexHeight = 0;
const indexedBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, activeTipHeight);
if (indexedBlockAmount > 0) {
minIndexHeight = Math.max(0, activeTipHeight - indexedBlockAmount + 1);
}
const start = Date.now();
const breakAt = start + 10000;
let newOrphans = 0;
this.orphanedBlocks = {};
const newOrphanedBlocks = {};
for (const chain of this.chainTips) {
if (chain.status === 'valid-fork' || chain.status === 'valid-headers') {
@ -37,16 +62,35 @@ class ChainTips {
do {
let orphan = this.blockCache[hash];
if (!orphan) {
const block = await bitcoinClient.getBlock(hash);
if (block && block.confirmations === -1) {
const block = await bitcoinCoreApi.$getBlock(hash);
if (block && block.stale) {
newOrphans++;
orphan = {
height: block.height,
hash: block.hash,
hash: block.id,
status: chain.status,
prevhash: block.previousblockhash,
};
this.blockCache[hash] = orphan;
// don't index stale blocks below the INDEXING_BLOCKS_AMOUNT cutoff
if (block.height >= minIndexHeight) {
if (this.indexingQueue.length < this.maxIndexingQueueSize) {
this.indexingQueue.push({ block, tip: orphan });
} else {
// re-fetch blocks lazily if the queue is big to keep memory usage sane
this.indexingQueue.push({ blockhash: hash, tip: orphan });
}
}
// make sure the cached canonical block at this height is correct & up to date
if (block.height >= (activeTipHeight - (config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4))) {
const cachedBlocks = blocks.getBlocks();
for (const cachedBlock of cachedBlocks) {
if (cachedBlock.height === block.height) {
// ensure this stale block is included in the orphans list
cachedBlock.extras.orphans = Array.from(new Set([...(cachedBlock.extras.orphans || []), orphan]));
}
}
}
}
}
if (orphan) {
@ -55,7 +99,7 @@ class ChainTips {
hash = orphan?.prevhash;
} while (hash && (Date.now() < breakAt));
for (const orphan of orphans) {
this.orphanedBlocks[orphan.hash] = orphan;
newOrphanedBlocks[orphan.hash] = orphan;
}
}
if (Date.now() >= breakAt) {
@ -65,6 +109,7 @@ class ChainTips {
}
this.orphansByHeight = {};
this.orphanedBlocks = newOrphanedBlocks;
const allOrphans = Object.values(this.orphanedBlocks);
for (const orphan of allOrphans) {
if (!this.orphansByHeight[orphan.height]) {
@ -73,12 +118,82 @@ class ChainTips {
this.orphansByHeight[orphan.height].push(orphan);
}
const heightsToKeep = new Set(this.chainTips.filter(tip => tip.status !== 'active').map(tip => tip.height));
const heightsToRemove: number[] = Object.keys(this.staleTips).map(Number).filter(height => !heightsToKeep.has(height));
for (const height of heightsToRemove) {
delete this.staleTips[height];
}
this.trimStaleTipsCache();
// index new orphaned blocks in the background
void this.$indexOrphanedBlocks();
logger.debug(`Updated orphaned blocks cache. Fetched ${newOrphans} new orphaned blocks. Total ${allOrphans.length}`);
} catch (e) {
logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`);
}
}
private async $indexOrphanedBlocks(): Promise<void> {
if (this.indexingOrphanedBlocks) {
return;
}
this.indexingOrphanedBlocks = true;
while (this.indexingQueue.length > 0) {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion, prefer-const
let { blockhash, block, tip } = this.indexingQueue.shift()!;
if (!block && !blockhash) {
continue;
}
if (blockhash && !block) {
block = await bitcoinCoreApi.$getBlock(blockhash);
}
if (!block) {
continue;
}
try {
let staleBlock: BlockExtended | undefined;
const alreadyIndexed = await BlocksSummariesRepository.$isSummaryIndexed(block.id);
const needToCache = Object.keys(this.staleTips).length < this.staleTipsCacheSize || block.height > Object.keys(this.staleTips).map(Number).sort((a, b) => b - a)[this.staleTipsCacheSize - 1];
if (!alreadyIndexed) {
staleBlock = await blocks.$indexBlock(block.id, block, true);
await blocks.$indexBlockSummary(block.id, block.height, true);
// don't DDOS core by indexing too fast
await Common.sleep$(5000);
} else if (needToCache) {
staleBlock = await blocks.$getBlock(block.id, true) as BlockExtended;
}
if (staleBlock && needToCache) {
const canonicalBlock = await blocks.$indexBlockByHeight(staleBlock.height);
this.staleTips[staleBlock.height] = {
height: staleBlock.height,
hash: staleBlock.id,
branchlen: tip.height - staleBlock.height,
status: tip.status,
stale: staleBlock,
canonical: canonicalBlock,
};
this.trimStaleTipsCache();
}
} catch (e) {
logger.err(`Failed to index orphaned block ${block.id} at height ${block.height}. Reason: ${e instanceof Error ? e.message : e}`);
}
}
this.indexingOrphanedBlocks = false;
}
private trimStaleTipsCache(): void {
const staleTipHeights = Object.keys(this.staleTips).map(Number).sort((a, b) => b - a);
if (staleTipHeights.length > this.staleTipsCacheSize) {
const heightsToDiscard = staleTipHeights.slice(this.staleTipsCacheSize);
for (const height of heightsToDiscard) {
delete this.staleTips[height];
}
}
}
public getOrphanedBlocksAtHeight(height: number | undefined): OrphanedBlock[] {
if (height === undefined) {
return [];
@ -86,6 +201,35 @@ class ChainTips {
return this.orphansByHeight[height] || [];
}
public getChainTips(): ChainTip[] {
return this.chainTips;
}
public getStaleTips(): StaleTip[] {
return Object.values(this.staleTips).sort((a, b) => b.height - a.height);
}
clearOrphanCacheAboveHeight(height: number): void {
for (const h in this.orphansByHeight) {
if (Number(h) > height) {
const orphans = this.orphansByHeight[h];
delete this.orphansByHeight[h];
for (const o of orphans) {
delete this.orphanedBlocks[o.hash];
delete this.blockCache[o.hash];
}
}
}
}
public isOrphaned(hash: string): boolean {
return !!this.orphanedBlocks[hash] || this.blockCache[hash]?.status === 'valid-fork' || this.blockCache[hash]?.status === 'valid-headers';
}
public getOrphanedBlock(hash: string): OrphanedBlock | undefined {
return this.orphanedBlocks[hash] || this.blockCache[hash];
}
}
export default new ChainTips();

190
backend/src/api/common.ts
View file

@ -8,6 +8,7 @@ import transactionUtils from './transaction-utils';
import { isPoint } from '../utils/secp256k1';
import logger from '../logger';
import { getVarIntLength, opcodes, parseMultisigScript } from '../utils/bitcoin-script';
import { IEsploraApi } from './bitcoin/esplora-api.interface';
// Bitcoin Core default policy settings
const MAX_STANDARD_TX_WEIGHT = 400_000;
@ -23,15 +24,15 @@ const MAX_STANDARD_SCRIPTSIG_SIZE = 1650;
const DUST_RELAY_TX_FEE = 3;
const MAX_OP_RETURN_RELAY = 83;
const DEFAULT_PERMIT_BAREMULTISIG = true;
const MAX_TX_LEGACY_SIGOPS = 2_500 * 4; // witness-adjusted sigops
export class Common {
static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ?
'144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49'
: '6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d';
static _isLiquid = config.MEMPOOL.NETWORK === 'liquid' || config.MEMPOOL.NETWORK === 'liquidtestnet';
static isLiquid(): boolean {
return this._isLiquid;
return config?.MEMPOOL?.NETWORK === 'liquid' || config?.MEMPOOL?.NETWORK === 'liquidtestnet';
}
static median(numbers: number[]) {
@ -224,6 +225,11 @@ export class Common {
return true;
}
// legacy sigops
if (this.isNonStandardLegacySigops(tx, height)) {
return true;
}
// input validation
for (const vin of tx.vin) {
if (vin.is_coinbase) {
@ -252,14 +258,40 @@ export class Common {
}
} else if (['unknown', 'provably_unspendable', 'empty'].includes(vin.prevout?.scriptpubkey_type || '')) {
return true;
} else if (this.isNonStandardAnchor(tx, height)) {
} else if (vin.prevout?.scriptpubkey_type === 'anchor' && this.isNonStandardAnchor(vin, height)) {
return true;
}
// TODO: bad-witness-nonstandard
// bad-witness-nonstandard
if (vin.prevout?.scriptpubkey_type === 'v1_p2tr' && vin.witness?.length) {
const hasAnnex = vin.witness.length > 1 && vin.witness[vin.witness.length - 1].startsWith('50');
// annex is non-standard
if (hasAnnex) {
return true;
}
if (vin.witness.length > (hasAnnex ? 2 : 1)) {
// script path spend
const controlBlock = vin.witness[vin.witness.length - (hasAnnex ? 2 : 1)];
// control block is required
if (!controlBlock.length) {
return false;
} else {
// Leaf version must be 0xc0 (aka Tapscript, see BIP 342)
if ((parseInt(controlBlock.slice(0, 2), 16) & 0xfe) !== 0xc0) {
return false;
}
}
// remaining witness items (except for the script) must be within MAX_STANDARD_TAPSCRIPT_STACK_ITEM_SIZE limit
if (vin.witness.slice(0, vin.witness.length - (hasAnnex ? 3 : 2)).some(v => v.length > 160)) {
return false;
}
}
}
// TODO: other bad-witness-nonstandard cases
}
// output validation
let opreturnCount = 0;
let opreturnBytes = 0;
for (const vout of tx.vout) {
// scriptpubkey
if (['nonstandard', 'provably_unspendable', 'empty'].includes(vout.scriptpubkey_type)) {
@ -283,10 +315,7 @@ export class Common {
}
} else if (vout.scriptpubkey_type === 'op_return') {
opreturnCount++;
if ((vout.scriptpubkey.length / 2) > MAX_OP_RETURN_RELAY) {
// over default datacarrier limit
return true;
}
opreturnBytes += vout.scriptpubkey.length / 2;
}
// dust
// (we could probably hardcode this for the different output types...)
@ -303,14 +332,16 @@ export class Common {
}
if (vout.value < (dustSize * DUST_RELAY_TX_FEE)) {
// under minimum output size
return true;
return !Common.isStandardEphemeralDust(tx, height);
}
}
}
// multi-op-return
if (opreturnCount > 1) {
return true;
// op_return
if (opreturnCount > 0) {
if (!this.isStandardOpReturn(opreturnBytes, opreturnCount, height)) {
return true;
}
}
// TODO: non-mandatory-script-verify-flag
@ -370,11 +401,12 @@ export class Common {
'signet': 211_000,
'': 863_500,
};
static isNonStandardAnchor(tx: TransactionExtended, height?: number): boolean {
static isNonStandardAnchor(vin: IEsploraApi.Vin, height?: number): boolean {
if (
height != null
&& this.ANCHOR_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
&& height <= this.ANCHOR_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
&& vin.prevout?.scriptpubkey === '51024e73'
) {
// anchor outputs were non-standard to spend before v28.x (scheduled for 2024/09/30 https://github.com/bitcoin/bitcoin/issues/29891)
return true;
@ -382,6 +414,69 @@ export class Common {
return false;
}
// Ephemeral dust is a new concept that allows a single dust output in a transaction, provided the transaction is zero fee
static EPHEMERAL_DUST_STANDARDNESS_ACTIVATION_HEIGHT = {
'testnet4': 90_500,
'testnet': 4_550_000,
'signet': 260_000,
'': 905_000,
};
static isStandardEphemeralDust(tx: TransactionExtended, height?: number): boolean {
if (
tx.fee === 0
&& (height == null || (
this.EPHEMERAL_DUST_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
&& height >= this.EPHEMERAL_DUST_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
))
) {
return true;
}
return false;
}
// OP_RETURN size & count limits were lifted in v28.3/v29.2/v30.0
static OP_RETURN_STANDARDNESS_ACTIVATION_HEIGHT = {
'testnet4': 108_000,
'testnet': 4_750_000,
'signet': 276_500,
'': 921_000,
};
static MAX_DATACARRIER_BYTES = 83;
static isStandardOpReturn(bytes: number, outputs: number,height?: number): boolean {
if (
(height == null || (
this.OP_RETURN_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
&& height >= this.OP_RETURN_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
)) // limits lifted
|| // OR
(bytes <= this.MAX_DATACARRIER_BYTES && outputs <= 1) // below old limits
) {
return true;
}
return false;
}
// New legacy sigops limit started to be enforced in v30.0
static LEGACY_SIGOPS_STANDARDNESS_ACTIVATION_HEIGHT = {
'testnet4': 108_000,
'testnet': 4_750_000,
'signet': 276_500,
'': 921_000,
};
static isNonStandardLegacySigops(tx: TransactionExtended, height?: number): boolean {
if (
height == null || (
this.LEGACY_SIGOPS_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
&& height >= this.LEGACY_SIGOPS_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
)
) {
if (!transactionUtils.checkSigopsBIP54(tx, MAX_TX_LEGACY_SIGOPS)) {
return true;
}
}
return false;
}
static getNonWitnessSize(tx: TransactionExtended): number {
let weight = tx.weight;
let hasWitness = false;
@ -462,6 +557,51 @@ export class Common {
return flags;
}
static inputIsMaybeInscription(vin: IEsploraApi.Vin): boolean {
// check if this is actually a taproot input
let isTaproot = false;
let isNotTaproot = false;
// taproot inputs have no scriptsig (otherwise probably wrapped segwit)
if (vin.scriptsig?.length) {
isNotTaproot = true;
}
// p2wpkh consist of a DER-encoded signature followed by a compressed pubkey
if (!isNotTaproot
&& this.isDERSig(vin.witness[0])
&& (vin.witness[1].startsWith('02') || vin.witness[1].startsWith('03'))
&& vin.witness[1].length === 66) {
isNotTaproot = true;
}
// p2wsh could be almost anything, but ends with a script which
// probably doesn't match a valid taproot control block length (32 + 33m)
if (!isNotTaproot
&& vin.witness.length >= 2
) {
const hasAnnex = vin.witness[vin.witness.length - 1].startsWith('50');
const controlBlock = vin.witness[vin.witness.length - (hasAnnex ? 2 : 1)];
if ((controlBlock.length - 66) % 64 === 0) {
isNotTaproot = true;
}
}
// signed taproot inscriptions should have 3 non-annex witness items:
// 1) schnorr signature
// 2) inscription script
// 3) control block (length 33 + 32m)
if (!isTaproot
&& vin.witness.length >= 3
&& vin.witness[0].length === 128 || vin.witness[0].length === 130
&& (vin.witness[2].length - 66) % 64 === 0
) {
isTaproot = true;
}
return isTaproot || !isNotTaproot;
}
static getTransactionFlags(tx: TransactionExtended, height?: number): number {
let flags = tx.flags ? BigInt(tx.flags) : 0n;
@ -513,12 +653,17 @@ export class Common {
flags |= TransactionFlags.p2tr;
if (vin.witness?.length) {
flags = Common.isInscription(vin, flags);
const hasAnnex = vin.witness.length > 1 && vin.witness[vin.witness.length - 1].startsWith('50');
if (hasAnnex) {
flags |= TransactionFlags.annex;
}
}
} break;
}
} else {
// no prevouts, optimistically check witness-bearing inputs
if (vin.witness?.length >= 2) {
if (vin.witness?.length >= 2 && Common.inputIsMaybeInscription(vin)) {
// try to parse the witness as a taproot inscription
try {
flags = Common.isInscription(vin, flags);
} catch {
@ -709,7 +854,7 @@ export class Common {
static indexingEnabled(): boolean {
return (
['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) &&
['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) &&
config.DATABASE.ENABLED === true &&
config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== 0
);
@ -722,6 +867,13 @@ export class Common {
);
}
static auditIndexingEnabled(): boolean {
return (
Common.indexingEnabled() &&
config.MEMPOOL.AUDIT === true
);
}
static gogglesIndexingEnabled(): boolean {
return (
Common.blocksSummariesIndexingEnabled() &&
@ -856,14 +1008,16 @@ export class Common {
}
}
static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string, acceleration?: boolean }[]): EffectiveFeeStats {
static calcEffectiveFeeStatistics(transactions: { weight: number, fee?: number, effectiveFeePerVsize?: number, txid: string, acceleration?: boolean }[]): EffectiveFeeStats {
const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate);
const totalWeight = transactions.reduce((acc, tx) => acc + tx.weight, 0);
let weightCount = 0;
// include any unused space
let weightCount = config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight;
let medianFee = 0;
let medianWeight = 0;
// calculate the "medianFee" as the average fee rate of the middle 0.25% weight units of transactions
// calculate the "medianFee" as the average fee rate of the middle 0.25% weight units of the conceptual block
const halfWidth = config.MEMPOOL.BLOCK_WEIGHT_UNITS / 800;
const leftBound = Math.floor((config.MEMPOOL.BLOCK_WEIGHT_UNITS / 2) - halfWidth);
const rightBound = Math.ceil((config.MEMPOOL.BLOCK_WEIGHT_UNITS / 2) + halfWidth);

33
backend/src/api/cpfp.ts
View file

@ -167,8 +167,10 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran
/**
* Takes a mempool transaction and a copy of the current mempool, and calculates the CPFP data for
* that transaction (and all others in the same cluster)
* If the passed transaction is not guaranteed to be in the mempool, set localTx to true: this will
* prevent updating the CPFP data of other transactions in the cluster
*/
export function calculateMempoolTxCpfp(tx: MempoolTransactionExtended, mempool: { [txid: string]: MempoolTransactionExtended }): CpfpInfo {
export function calculateMempoolTxCpfp(tx: MempoolTransactionExtended, mempool: { [txid: string]: MempoolTransactionExtended }, localTx: boolean = false): CpfpInfo {
if (tx.cpfpUpdated && Date.now() < (tx.cpfpUpdated + CPFP_UPDATE_INTERVAL)) {
tx.cpfpDirty = false;
return {
@ -198,17 +200,26 @@ export function calculateMempoolTxCpfp(tx: MempoolTransactionExtended, mempool:
totalFee += tx.fees.base;
}
const effectiveFeePerVsize = totalFee / totalVsize;
for (const tx of cluster.values()) {
mempool[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize;
mempool[tx.txid].ancestors = Array.from(tx.ancestors.values()).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fees.base }));
mempool[tx.txid].descendants = Array.from(cluster.values()).filter(entry => entry.txid !== tx.txid && !tx.ancestors.has(entry.txid)).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fees.base }));
mempool[tx.txid].bestDescendant = null;
mempool[tx.txid].cpfpChecked = true;
mempool[tx.txid].cpfpDirty = true;
mempool[tx.txid].cpfpUpdated = Date.now();
}
tx = mempool[tx.txid];
if (localTx) {
tx.effectiveFeePerVsize = effectiveFeePerVsize;
tx.ancestors = Array.from(cluster.get(tx.txid)?.ancestors.values() || []).map(ancestor => ({ txid: ancestor.txid, weight: ancestor.weight, fee: ancestor.fees.base }));
tx.descendants = Array.from(cluster.values()).filter(entry => entry.txid !== tx.txid && !cluster.get(tx.txid)?.ancestors.has(entry.txid)).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fees.base }));
tx.bestDescendant = null;
} else {
for (const tx of cluster.values()) {
mempool[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize;
mempool[tx.txid].ancestors = Array.from(tx.ancestors.values()).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fees.base }));
mempool[tx.txid].descendants = Array.from(cluster.values()).filter(entry => entry.txid !== tx.txid && !tx.ancestors.has(entry.txid)).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fees.base }));
mempool[tx.txid].bestDescendant = null;
mempool[tx.txid].cpfpChecked = true;
mempool[tx.txid].cpfpDirty = true;
mempool[tx.txid].cpfpUpdated = Date.now();
}
tx = mempool[tx.txid];
}
return {
ancestors: tx.ancestors || [],

76
backend/src/api/database-migration.ts
View file

@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 94;
private static currentVersion = 104;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@ -104,7 +104,7 @@ class DatabaseMigration {
private async $createMissingTablesAndIndexes(databaseSchemaVersion: number) {
await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion);
const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
const isBitcoin = ['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK);
await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
@ -1118,6 +1118,66 @@ class DatabaseMigration {
}
await this.updateToSchemaVersion(94);
}
// blocks pools-v2.json hash
if (databaseSchemaVersion < 95) {
let poolJsonSha = 'f737d86571d190cf1a1a3cf5fd86b33ba9624254'; // https://github.com/mempool/mining-pools/commit/f737d86571d190cf1a1a3cf5fd86b33ba9624254
const [poolJsonShaDb]: any[] = await DB.query(`SELECT string FROM state WHERE name = 'pools_json_sha'`);
if (poolJsonShaDb?.length > 0) {
poolJsonSha = poolJsonShaDb[0].string;
}
await this.$executeQuery(`ALTER TABLE blocks ADD definition_hash varchar(255) NOT NULL DEFAULT "${poolJsonSha}"`);
await this.$executeQuery('ALTER TABLE blocks ADD INDEX `definition_hash` (`definition_hash`)');
await this.updateToSchemaVersion(95);
}
if (databaseSchemaVersion < 96) {
await this.$executeQuery(`ALTER TABLE blocks_audits MODIFY time timestamp NOT NULL DEFAULT 0`);
await this.updateToSchemaVersion(96);
}
// Make definition_hash nullable
if (databaseSchemaVersion < 97) {
let poolJsonSha = '895cf0903e771beb647d0c1356bb4b8f4f123af7'; // https://github.com/mempool/mining-pools/commit/895cf0903e771beb647d0c1356bb4b8f4f123af7
const [poolJsonShaDb]: any[] = await DB.query(`SELECT string FROM state WHERE name = 'pools_json_sha'`);
if (poolJsonShaDb?.length > 0) {
poolJsonSha = poolJsonShaDb[0].string;
}
await this.$executeQuery(`ALTER TABLE blocks MODIFY COLUMN definition_hash varchar(255) NULL DEFAULT "${poolJsonSha}"`);
await this.updateToSchemaVersion(97);
}
// reindex mainnet Goggles flags for mined block templates above height 896070
// (since the first annex transaction at height 896071)
// (safe to make this conditional on the network since it doesn't change the database schema)
if (databaseSchemaVersion < 98 && config.MEMPOOL.NETWORK === 'mainnet') {
await this.$executeQuery('UPDATE blocks_summaries SET version = 0 WHERE height >= 896070;');
await this.updateToSchemaVersion(98);
}
// Add vsize_0 to statistics table
if (databaseSchemaVersion < 99) {
await this.$executeQuery('ALTER TABLE statistics ADD COLUMN vsize_0 int(11) NOT NULL DEFAULT 0');
await this.updateToSchemaVersion(99);
}
// Add "block indexed at version" index_version column to the blocks table
// to be used for lazy migrations & reindexing tasks
if (databaseSchemaVersion < 100) {
await this.$executeQuery('ALTER TABLE `blocks` ADD index_version INT NOT NULL DEFAULT 0');
await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `index_version` (`index_version`)');
await this.updateToSchemaVersion(100);
}
if (databaseSchemaVersion < 102) {
await this.$executeQuery('ALTER TABLE `blocks` ADD stale BOOL NOT NULL DEFAULT 0');
await this.updateToSchemaVersion(102);
}
if (databaseSchemaVersion < 103) {
await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `stale` (`stale`)');
await this.updateToSchemaVersion(103);
}
}
/**
@ -1226,7 +1286,7 @@ class DatabaseMigration {
*/
private getMigrationQueriesFromVersion(version: number): string[] {
const queries: string[] = [];
const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
const isBitcoin = ['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK);
if (version < 1) {
if (config.MEMPOOL.NETWORK !== 'liquid' && config.MEMPOOL.NETWORK !== 'liquidtestnet') {
@ -1250,6 +1310,16 @@ class DatabaseMigration {
queries.push(`DELETE FROM state WHERE name = 'last_weekly_hashrates_indexing'`);
}
if (version < 101) {
queries.push(`DELETE FROM prices WHERE USD = -1`);
}
if (version < 104) {
queries.push(`ALTER TABLE blocks DROP PRIMARY KEY`);
queries.push(`ALTER TABLE blocks ADD PRIMARY KEY (hash)`);
queries.push(`ALTER TABLE blocks ADD INDEX (height)`);
}
return queries;
}

2
backend/src/api/disk-cache.ts
View file

@ -252,7 +252,7 @@ class DiskCache {
}
if (rbfData?.rbf) {
rbfCache.load({
await rbfCache.load({
txs: rbfData.rbf.txs.map(([txid, entry]) => ({ value: entry })),
trees: rbfData.rbf.trees,
expiring: rbfData.rbf.expiring.map(([txid, value]) => ({ key: txid, value })),

84
backend/src/api/fee-api.ts
View file

@ -1,4 +1,5 @@
import { MempoolBlock } from '../mempool.interfaces';
import { IBitcoinApi } from './bitcoin/bitcoin-api.interface';
import config from '../config';
import mempool from './mempool';
import projectedBlocks from './mempool-blocks';
@ -16,29 +17,58 @@ interface RecommendedFees {
class FeeApi {
constructor() { }
defaultFee = isLiquid ? 0.1 : 1;
minimumIncrement = isLiquid ? 0.1 : 1;
minFastestFee = isLiquid ? 0.1 : 1;
minHalfHourFee = isLiquid ? 0.1 : 0.5;
priorityFactor = isLiquid ? 0 : 0.5;
public getRecommendedFee(): RecommendedFees {
const pBlocks = projectedBlocks.getMempoolBlocks();
const mPool = mempool.getMempoolInfo();
const minimumFee = this.roundUpToNearest(mPool.mempoolminfee * 100000, this.minimumIncrement);
const defaultMinFee = Math.max(minimumFee, this.defaultFee);
return this.calculateRecommendedFee(pBlocks, mPool);
}
public getPreciseRecommendedFee(): RecommendedFees {
const pBlocks = projectedBlocks.getMempoolBlocks();
const mPool = mempool.getMempoolInfo();
// minimum non-zero minrelaytxfee / incrementalrelayfee is 1 sat/kvB = 0.001 sat/vB
const recommendations = this.calculateRecommendedFee(pBlocks, mPool, 0.001);
// enforce floor & offset for highest priority recommendations while <100% hashrate accepts sub-sat fees
recommendations.fastestFee = Math.max(recommendations.fastestFee + this.priorityFactor, this.minFastestFee);
recommendations.halfHourFee = Math.max(recommendations.halfHourFee + (this.priorityFactor / 2), this.minHalfHourFee);
return {
'fastestFee': Math.round(recommendations.fastestFee * 1000) / 1000,
'halfHourFee': Math.round(recommendations.halfHourFee * 1000) / 1000,
'hourFee': Math.round(recommendations.hourFee * 1000) / 1000,
'economyFee': Math.round(recommendations.economyFee * 1000) / 1000,
'minimumFee': Math.round(recommendations.minimumFee * 1000) / 1000,
};
}
public calculateRecommendedFee(pBlocks: MempoolBlock[], mPool: IBitcoinApi.MempoolInfo, minIncrement: number = this.minimumIncrement): RecommendedFees {
const purgeRate = this.roundUpToNearest(mPool.mempoolminfee * 100000, minIncrement);
const minimumFee = Math.max(purgeRate, minIncrement);
if (!pBlocks.length) {
return {
'fastestFee': defaultMinFee,
'halfHourFee': defaultMinFee,
'hourFee': defaultMinFee,
'fastestFee': minimumFee,
'halfHourFee': minimumFee,
'hourFee': minimumFee,
'economyFee': minimumFee,
'minimumFee': minimumFee,
};
}
const firstMedianFee = this.optimizeMedianFee(pBlocks[0], pBlocks[1]);
const secondMedianFee = pBlocks[1] ? this.optimizeMedianFee(pBlocks[1], pBlocks[2], firstMedianFee) : this.defaultFee;
const thirdMedianFee = pBlocks[2] ? this.optimizeMedianFee(pBlocks[2], pBlocks[3], secondMedianFee) : this.defaultFee;
const firstMedianFee = this.optimizeMedianFee(pBlocks[0], pBlocks[1], undefined, minimumFee, minIncrement);
const secondMedianFee = pBlocks[1] ? this.optimizeMedianFee(pBlocks[1], pBlocks[2], firstMedianFee, minimumFee, minIncrement) : minimumFee;
const thirdMedianFee = pBlocks[2] ? this.optimizeMedianFee(pBlocks[2], pBlocks[3], secondMedianFee, minimumFee, minIncrement) : minimumFee;
// explicitly enforce a minimum of ceil(mempoolminfee) on all recommendations.
// simply rounding up recommended rates is insufficient, as the purging rate
// can exceed the median rate of projected blocks in some extreme scenarios
// (see https://bitcoin.stackexchange.com/a/120024)
let fastestFee = Math.max(minimumFee, firstMedianFee);
let halfHourFee = Math.max(minimumFee, secondMedianFee);
let hourFee = Math.max(minimumFee, thirdMedianFee);
@ -49,33 +79,39 @@ class FeeApi {
halfHourFee = Math.max(halfHourFee, hourFee, economyFee);
hourFee = Math.max(hourFee, economyFee);
// explicitly enforce a minimum of ceil(mempoolminfee) on all recommendations.
// simply rounding up recommended rates is insufficient, as the purging rate
// can exceed the median rate of projected blocks in some extreme scenarios
// (see https://bitcoin.stackexchange.com/a/120024)
return {
'fastestFee': fastestFee,
'halfHourFee': halfHourFee,
'hourFee': hourFee,
'economyFee': economyFee,
'minimumFee': minimumFee,
'fastestFee': this.roundToNearest(fastestFee, minIncrement),
'halfHourFee': this.roundToNearest(halfHourFee, minIncrement),
'hourFee': this.roundToNearest(hourFee, minIncrement),
'economyFee': this.roundToNearest(economyFee, minIncrement),
'minimumFee': this.roundToNearest(minimumFee, minIncrement),
};
}
private optimizeMedianFee(pBlock: MempoolBlock, nextBlock: MempoolBlock | undefined, previousFee?: number): number {
private optimizeMedianFee(pBlock: MempoolBlock, nextBlock: MempoolBlock | undefined, previousFee: number | undefined, minFee: number, minIncrement: number = this.minimumIncrement): number {
const useFee = previousFee ? (pBlock.medianFee + previousFee) / 2 : pBlock.medianFee;
if (pBlock.blockVSize <= 500000) {
return this.defaultFee;
if (pBlock.blockVSize <= 500000 || pBlock.medianFee < minFee) {
return minFee;
}
if (pBlock.blockVSize <= 950000 && !nextBlock) {
const multiplier = (pBlock.blockVSize - 500000) / 500000;
return Math.max(Math.round(useFee * multiplier), this.defaultFee);
return Math.max(this.roundToNearest(useFee * multiplier, minIncrement), minFee);
}
return this.roundUpToNearest(useFee, this.minimumIncrement);
return Math.max(this.roundUpToNearest(useFee, minIncrement), minFee);
}
private roundUpToNearest(value: number, nearest: number): number {
return Math.ceil(value / nearest) * nearest;
if (nearest !== 0) {
return Math.ceil(value / nearest) * nearest;
}
return value;
}
private roundToNearest(value: number, nearest: number): number {
if (nearest !== 0) {
return Math.round(value / nearest) * nearest;
}
return value;
}
}

30
backend/src/api/liquid/liquid.routes.ts
View file

@ -4,6 +4,7 @@ import config from '../../config';
import elementsParser from './elements-parser';
import icons from './icons';
import { handleError } from '../../utils/api';
import PricesRepository from '../../repositories/PricesRepository';
class LiquidRoutes {
public initRoutes(app: Application) {
@ -31,6 +32,7 @@ class LiquidRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/utxos/emergency-spent', this.$getEmergencySpentUtxos)
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/utxos/emergency-spent/stats', this.$getEmergencySpentUtxosStats)
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/status', this.$getFederationAuditStatus)
.get(config.MEMPOOL.API_URL_PREFIX + 'historical-price', this.$getHistoricalPrice)
;
}
}
@ -255,6 +257,34 @@ class LiquidRoutes {
}
}
private async $getHistoricalPrice(req: Request, res: Response): Promise<void> {
try {
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
if (['testnet', 'signet', 'liquidtestnet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
handleError(req, res, 400, 'Prices are not available on testnets.');
return;
}
const timestamp = parseInt(req.query.timestamp as string, 10) || 0;
const currency = req.query.currency as string;
let response;
if (timestamp && currency) {
response = await PricesRepository.$getNearestHistoricalPrice(timestamp, currency);
} else if (timestamp) {
response = await PricesRepository.$getNearestHistoricalPrice(timestamp);
} else if (currency) {
response = await PricesRepository.$getHistoricalPrices(currency);
} else {
response = await PricesRepository.$getHistoricalPrices();
}
res.status(200).send(response);
} catch (e) {
handleError(req, res, 500, 'Failed to get historical prices');
}
}
}
export default new LiquidRoutes();

2
backend/src/api/mempool-blocks.ts
View file

@ -46,7 +46,7 @@ class MempoolBlocks {
}
public async updatePools$(): Promise<void> {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) === false) {
this.pools = {};
return;
}

32
backend/src/api/mempool.ts
View file

@ -21,8 +21,7 @@ class Mempool {
private mempoolCandidates: { [txid: string ]: boolean } = {};
private spendMap = new Map<string, MempoolTransactionExtended>();
private recentlyDeleted: MempoolTransactionExtended[][] = []; // buffer of transactions deleted in recent mempool updates
private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
maxmempool: 300000000, mempoolminfee: Common.isLiquid() ? 0.00000100 : 0.00001000, minrelaytxfee: Common.isLiquid() ? 0.00000100 : 0.00001000 };
private mempoolInfo: IBitcoinApi.MempoolInfo;
private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[],
deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[]) => void) | undefined;
private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[],
@ -44,11 +43,36 @@ class Mempool {
private timer = new Date().getTime();
private missingTxCount = 0;
private mainLoopTimeout: number = 120000;
private txPerSecondInterval: NodeJS.Timeout | null = null;
public limitGBT = config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE && config.MEMPOOL.LIMIT_GBT;
constructor() {
setInterval(this.updateTxPerSecond.bind(this), 1000);
// Initialize mempoolInfo here to avoid circular dependency issues
// Use config directly instead of Common.isLiquid() to break circular dependency
const isLiquid = config.MEMPOOL.NETWORK === 'liquid' || config.MEMPOOL.NETWORK === 'liquidtestnet';
this.mempoolInfo = {
loaded: false,
size: 0,
bytes: 0,
usage: 0,
total_fee: 0,
maxmempool: 300000000,
mempoolminfee: isLiquid ? 0.00000100 : 0.00001000,
minrelaytxfee: isLiquid ? 0.00000100 : 0.00001000
};
this.txPerSecondInterval = setInterval(this.updateTxPerSecond.bind(this), 1000);
}
/**
* Cleanup resources (timers, etc.)
* This should only be called when shutting down or in test teardown
*/
public destroy(): void {
if (this.txPerSecondInterval) {
clearInterval(this.txPerSecondInterval);
this.txPerSecondInterval = null;
}
}
/**
@ -411,7 +435,7 @@ class Mempool {
}
}
public async getNextCandidates(minFeeTransactions: string[], blockHeight: number, deletedTransactions: MempoolTransactionExtended[]): Promise<GbtCandidates | undefined> {
public getNextCandidates(minFeeTransactions: string[], blockHeight: number, deletedTransactions: MempoolTransactionExtended[]): GbtCandidates | undefined {
if (this.limitGBT) {
const deletedTxsMap = {};
for (const tx of deletedTransactions) {

12
backend/src/api/mining/mining-routes.ts
View file

@ -53,7 +53,7 @@ class MiningRoutes {
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
if (['testnet', 'signet', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) {
if (['testnet', 'signet', 'liquidtestnet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
handleError(req, res, 400, 'Prices are not available on testnets.');
return;
}
@ -394,7 +394,7 @@ class MiningRoutes {
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
handleError(req, res, 400, 'Acceleration data is not available.');
return;
}
@ -409,7 +409,7 @@ class MiningRoutes {
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
handleError(req, res, 400, 'Acceleration data is not available.');
return;
}
@ -425,7 +425,7 @@ class MiningRoutes {
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
handleError(req, res, 400, 'Acceleration data is not available.');
return;
}
@ -440,7 +440,7 @@ class MiningRoutes {
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
handleError(req, res, 400, 'Acceleration data is not available.');
return;
}
@ -455,7 +455,7 @@ class MiningRoutes {
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
handleError(req, res, 400, 'Acceleration data is not available.');
return;
}

81
backend/src/api/mining/mining.ts
View file

@ -30,6 +30,12 @@ class Mining {
public reindexHashrateRequested = false;
public reindexDifficultyAdjustmentRequested = false;
private genesisData: {
timestamp: number,
bits: number,
difficulty: number,
} | null = null;
/**
* Get historical blocks health
*/
@ -224,8 +230,8 @@ class Mining {
try {
const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0));
const genesisTimestamp = genesisBlock.timestamp * 1000;
const genesisData = await this.getGenesisData();
const genesisTimestamp = genesisData.timestamp * 1000;
const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
const hashrates: any[] = [];
@ -256,31 +262,36 @@ class Mining {
const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
null, fromTimestamp / 1000, toTimestamp / 1000);
const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
blockStats.lastBlockHeight);
let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp / 1000, toTimestamp / 1000);
const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0);
if (totalBlocks > 0) {
pools = pools.map((pool: any) => {
pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
pool.share = (pool.blockCount / totalBlocks);
return pool;
});
if (blockStats.blockCount <= 0) {
logger.debug(`No block found between ${fromTimestamp / 1000} and ${toTimestamp / 1000}, skipping hashrate indexing for this period`, logger.tags.mining);
} else {
const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
blockStats.lastBlockHeight);
for (const pool of pools) {
hashrates.push({
hashrateTimestamp: toTimestamp / 1000,
avgHashrate: pool['hashrate'] ,
poolId: pool.poolId,
share: pool['share'],
type: 'weekly',
let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp / 1000, toTimestamp / 1000);
const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0);
if (totalBlocks > 0) {
pools = pools.map((pool: any) => {
pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
pool.share = (pool.blockCount / totalBlocks);
return pool;
});
}
newlyIndexed += hashrates.length / Math.max(1, pools.length);
await HashratesRepository.$saveHashrates(hashrates);
hashrates.length = 0;
for (const pool of pools) {
hashrates.push({
hashrateTimestamp: toTimestamp / 1000,
avgHashrate: pool['hashrate'] ,
poolId: pool.poolId,
share: pool['share'],
type: 'weekly',
});
}
newlyIndexed += hashrates.length / Math.max(1, pools.length);
await HashratesRepository.$saveHashrates(hashrates);
hashrates.length = 0;
}
}
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
@ -335,8 +346,8 @@ class Mining {
const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
try {
const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0));
const genesisTimestamp = genesisBlock.timestamp * 1000;
const genesisData = await this.getGenesisData();
const genesisTimestamp = genesisData.timestamp * 1000;
const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
const lastMidnight = this.getDateMidnight(new Date());
let toTimestamp = Math.round(lastMidnight.getTime());
@ -445,14 +456,14 @@ class Mining {
// gets {time, height, difficulty, bits} of blocks in ascending order of height
const blocks: any = await BlocksRepository.$getBlocksDifficulty();
const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0));
let currentDifficulty = genesisBlock.difficulty;
let currentBits = genesisBlock.bits;
const genesisData = await this.getGenesisData();
let currentDifficulty = genesisData.difficulty;
let currentBits = genesisData.bits;
let totalIndexed = 0;
if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) {
await DifficultyAdjustmentsRepository.$saveAdjustments({
time: genesisBlock.timestamp,
time: genesisData.timestamp,
height: 0,
difficulty: currentDifficulty,
adjustment: 0.0,
@ -689,6 +700,18 @@ class Mining {
}
return blocks[0];
}
private async getGenesisData(): Promise<{timestamp: number, bits: number, difficulty: number}> {
if (this.genesisData == null) {
const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0));
this.genesisData = {
timestamp: genesisBlock.timestamp,
bits: genesisBlock.bits,
difficulty: genesisBlock.difficulty,
};
}
return this.genesisData;
}
}
export default new Mining();

29
backend/src/api/pools-parser.ts
View file

@ -8,6 +8,7 @@ import mining from './mining/mining';
import transactionUtils from './transaction-utils';
import BlocksRepository from '../repositories/BlocksRepository';
import redisCache from './redis-cache';
import blocks from './blocks';
class PoolsParser {
miningPools: any[] = [];
@ -19,15 +20,6 @@ class PoolsParser {
'addresses': '[]',
'slug': 'unknown'
};
private uniqueLogs: string[] = [];
private uniqueLog(loggerFunction: any, msg: string): void {
if (this.uniqueLogs.includes(msg)) {
return;
}
this.uniqueLogs.push(msg);
loggerFunction(msg);
}
public setMiningPools(pools): void {
for (const pool of pools) {
@ -51,6 +43,8 @@ class PoolsParser {
await this.$insertUnknownPool();
let reindexUnknown = false;
let clearCache = false;
for (const pool of this.miningPools) {
if (!pool.id) {
@ -87,17 +81,20 @@ class PoolsParser {
logger.debug(`Inserting new mining pool ${pool.name}`);
await PoolsRepository.$insertNewMiningPool(pool, slug);
reindexUnknown = true;
clearCache = true;
} else {
if (poolDB.name !== pool.name) {
// Pool has been renamed
const newSlug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
logger.warn(`Renaming ${poolDB.name} mining pool to ${pool.name}. Slug has been updated. Maybe you want to make a redirection from 'https://mempool.space/mining/pool/${poolDB.slug}' to 'https://mempool.space/mining/pool/${newSlug}`);
await PoolsRepository.$renameMiningPool(poolDB.id, newSlug, pool.name);
clearCache = true;
}
if (poolDB.link !== pool.link) {
// Pool link has changed
logger.debug(`Updating link for ${pool.name} mining pool`);
await PoolsRepository.$updateMiningPoolLink(poolDB.id, pool.link);
clearCache = true;
}
if (JSON.stringify(pool.addresses) !== poolDB.addresses ||
JSON.stringify(pool.regexes) !== poolDB.regexes) {
@ -105,6 +102,7 @@ class PoolsParser {
logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool.`);
await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes);
reindexUnknown = true;
clearCache = true;
await this.$reindexBlocksForPool(poolDB.id);
}
}
@ -120,6 +118,17 @@ class PoolsParser {
}
await this.$reindexBlocksForPool(unknownPool.id);
}
// refresh the in-memory block cache with the reindexed data
if (clearCache) {
for (const block of blocks.getBlocks()) {
const reindexedBlock = await blocks.$indexBlock(block.id);
block.extras.pool = reindexedBlock.extras.pool;
}
// update persistent cache with the reindexed data
diskCache.$saveCacheToDisk();
redisCache.$updateBlocks(blocks.getBlocks());
}
}
public matchBlockMiner(scriptsig: string, addresses: string[], pools: PoolTag[]): PoolTag | undefined {
@ -186,7 +195,7 @@ class PoolsParser {
let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52
if (config.MEMPOOL.NETWORK === 'testnet') {
firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581
} else if (config.MEMPOOL.NETWORK === 'signet') {
} else if (['signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
firstKnownBlockPool = 0;
}

4
backend/src/api/rbf-cache.ts
View file

@ -484,7 +484,7 @@ class RbfCache {
return deflated;
}
async importTree(mempool, root, txid, deflated, txs: Map<string, MempoolTransactionExtended>, mined: boolean = false): Promise<RbfTree | void> {
importTree(mempool, root, txid, deflated, txs: Map<string, MempoolTransactionExtended>, mined: boolean = false): RbfTree | void {
const treeInfo = deflated[txid];
const replaces: RbfTree[] = [];
@ -503,7 +503,7 @@ class RbfCache {
// recursively reconstruct child trees
for (const childId of treeInfo.replaces) {
const replaced = await this.importTree(mempool, root, childId, deflated, txs, mined);
const replaced = this.importTree(mempool, root, childId, deflated, txs, mined);
if (replaced) {
this.replacedBy.set(replaced.tx.txid, txid);
if (mempool[replaced.tx.txid]) {

16
backend/src/api/services/services-routes.ts
View file

@ -7,6 +7,7 @@ class ServicesRoutes {
public initRoutes(app: Application): void {
app
.get(config.MEMPOOL.API_URL_PREFIX + 'wallet/:walletId', this.$getWallet)
.get(config.MEMPOOL.API_URL_PREFIX + 'treasuries', this.$getTreasuries)
;
}
@ -17,11 +18,24 @@ class ServicesRoutes {
res.setHeader('Expires', new Date(Date.now() + 1000 * 5).toUTCString());
const walletId = req.params.walletId;
const wallet = await WalletApi.getWallet(walletId);
res.status(200).send(wallet);
if (wallet === null) {
res.status(404).send('No such wallet');
} else {
res.status(200).send(wallet);
}
} catch (e) {
handleError(req, res, 500, 'Failed to get wallet');
}
}
private async $getTreasuries(req: Request, res: Response): Promise<void> {
try {
const treasuries = await WalletApi.getTreasuries();
res.status(200).send(treasuries);
} catch (e) {
handleError(req, res, 500, 'Failed to get treasuries');
}
}
}
export default new ServicesRoutes();

201
backend/src/api/services/wallets.ts
View file

@ -4,6 +4,7 @@ import { IEsploraApi } from '../bitcoin/esplora-api.interface';
import bitcoinApi from '../bitcoin/bitcoin-api-factory';
import axios from 'axios';
import { TransactionExtended } from '../../mempool.interfaces';
import { promises as fsPromises } from 'fs';
interface WalletAddress {
address: string;
@ -25,21 +26,125 @@ interface Wallet {
lastPoll: number;
}
interface Treasury {
id: number,
name: string,
wallet: string,
enterprise: string,
verifiedAddresses: string[],
balances: { balance: number, time: number }[], // off-chain balances
}
const POLL_FREQUENCY = 5 * 60 * 1000; // 5 minutes
class WalletApi {
private treasuries: Treasury[] = [];
private wallets: Record<string, Wallet> = {};
private syncing = false;
private lastSync = 0;
private isSaving = false;
private cacheSchemaVersion = 1;
private static TMP_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/tmp-wallets-cache.json';
private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/wallets-cache.json';
constructor() {
this.wallets = config.WALLETS.ENABLED ? (config.WALLETS.WALLETS as string[]).reduce((acc, wallet) => {
acc[wallet] = { name: wallet, addresses: {}, lastPoll: 0 };
return acc;
}, {} as Record<string, Wallet>) : {};
// Load cache on startup
if (config.WALLETS.ENABLED) {
this.$loadCache();
}
}
public getWallet(wallet: string): Record<string, WalletAddress> {
return this.wallets?.[wallet]?.addresses || {};
private async $loadCache(): Promise<void> {
try {
const cacheData = await fsPromises.readFile(WalletApi.FILE_NAME, 'utf8');
if (!cacheData) {
return;
}
const data = JSON.parse(cacheData);
if (data.cacheSchemaVersion !== this.cacheSchemaVersion) {
logger.notice('Wallets cache contains an outdated schema version. Clearing it.');
return this.$wipeCache();
}
this.wallets = data.wallets;
this.treasuries = data.treasuries || [];
// Reset lastSync time to force transaction history refresh
for (const wallet of Object.values(this.wallets)) {
wallet.lastPoll = 0;
for (const address of Object.values(wallet.addresses)) {
address.lastSync = 0;
}
}
logger.info('Restored wallets data from disk cache');
} catch (e) {
logger.warn('Failed to parse wallets cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
}
}
private async $saveCache(): Promise<void> {
if (this.isSaving || !config.WALLETS.ENABLED) {
return;
}
try {
this.isSaving = true;
logger.debug('Writing wallets data to disk cache...');
const cacheData = {
cacheSchemaVersion: this.cacheSchemaVersion,
wallets: this.wallets,
treasuries: this.treasuries,
};
await fsPromises.writeFile(
WalletApi.TMP_FILE_NAME,
JSON.stringify(cacheData),
{ flag: 'w' }
);
await fsPromises.rename(WalletApi.TMP_FILE_NAME, WalletApi.FILE_NAME);
logger.debug('Wallets data saved to disk cache');
} catch (e) {
logger.warn('Error writing to wallets cache file: ' + (e instanceof Error ? e.message : e));
} finally {
this.isSaving = false;
}
}
private async $wipeCache(): Promise<void> {
try {
await fsPromises.unlink(WalletApi.FILE_NAME);
} catch (e: any) {
if (e?.code !== 'ENOENT') {
logger.err(`Cannot wipe wallets cache file ${WalletApi.FILE_NAME}. Exception ${JSON.stringify(e)}`);
}
}
}
public getWallet(wallet: string): Record<string, WalletAddress> | null {
if (wallet in this.wallets) {
return this.wallets?.[wallet]?.addresses || {};
} else {
return null;
}
}
public getWallets(): string[] {
return Object.keys(this.wallets);
}
public getTreasuries(): Treasury[] {
return this.treasuries?.filter(treasury => !!this.wallets[treasury.wallet]) || [];
}
// resync wallet addresses from the services backend
@ -47,7 +152,63 @@ class WalletApi {
if (!config.WALLETS.ENABLED || this.syncing) {
return;
}
this.syncing = true;
if (config.WALLETS.AUTO && (Date.now() - this.lastSync) > POLL_FREQUENCY) {
try {
// update list of active wallets
this.lastSync = Date.now();
const response = await axios.get(config.MEMPOOL_SERVICES.API + `/wallets`);
const walletList: string[] = response.data;
if (walletList) {
// create a quick lookup dictionary of active wallets
const newWallets: Record<string, boolean> = Object.fromEntries(
walletList.map(wallet => [wallet, true])
);
for (const wallet of walletList) {
// don't overwrite existing wallets
if (!this.wallets[wallet]) {
this.wallets[wallet] = { name: wallet, addresses: {}, lastPoll: 0 };
}
}
// remove wallets that are no longer active
for (const wallet of Object.keys(this.wallets)) {
if (!newWallets[wallet]) {
delete this.wallets[wallet];
}
}
}
// update list of treasuries
const treasuriesResponse = await axios.get(config.MEMPOOL_SERVICES.API + `/treasuries`);
this.treasuries = treasuriesResponse.data || [];
} catch (e) {
logger.err(`Error updating active wallets: ${(e instanceof Error ? e.message : e)}`);
}
try {
// update list of active treasuries
this.lastSync = Date.now();
const response = await axios.get(config.MEMPOOL_SERVICES.API + `/treasuries`);
const treasuries: Treasury[] = response.data;
if (treasuries) {
this.treasuries = treasuries;
}
} catch (e) {
logger.err(`Error updating active treasuries: ${(e instanceof Error ? e.message : e)}`);
}
// insert dummy address data to represent off-chain balance history
for (const treasury of this.treasuries) {
if (treasury.balances?.length) {
if (this.wallets[treasury.wallet]) {
this.wallets[treasury.wallet].addresses['private'] = convertBalancesToWalletAddress(treasury.wallet, treasury.balances);
}
}
}
}
for (const walletKey of Object.keys(this.wallets)) {
const wallet = this.wallets[walletKey];
if (wallet.lastPoll < (Date.now() - POLL_FREQUENCY)) {
@ -61,17 +222,21 @@ class WalletApi {
}
// remove old addresses
for (const address of Object.keys(wallet.addresses)) {
if (!addresses[address]) {
if (address !== 'private' && !addresses[address]) {
delete wallet.addresses[address];
}
}
wallet.lastPoll = Date.now();
logger.debug(`Synced ${Object.keys(wallet.addresses).length} addresses for wallet ${wallet.name}`);
// Update cache
await this.$saveCache();
} catch (e) {
logger.err(`Error syncing wallet ${wallet.name}: ${(e instanceof Error ? e.message : e)}`);
}
}
}
this.syncing = false;
}
@ -150,4 +315,34 @@ class WalletApi {
}
}
function convertBalancesToWalletAddress(wallet: string, balances: { balance: number, time: number }[]): WalletAddress {
// represent the off-chain balance as a series of transactions modifying a single notional UTXO
const sortedBalances = balances.sort((a, b) => a.time - b.time);
const walletAddress: WalletAddress = {
address: 'private',
active: false,
stats: {
funded_txo_count: 0,
funded_txo_sum: sortedBalances[sortedBalances.length - 1].balance,
spent_txo_count: 0,
spent_txo_sum: 0,
tx_count: 0,
},
transactions: [],
lastSync: sortedBalances[sortedBalances.length - 1].time,
};
let lastBalance = 0;
for (const [index, entry] of sortedBalances.entries()) {
const diff = entry.balance - lastBalance;
walletAddress.transactions.push({
txid: `${wallet}-private-${index}`,
value: diff,
height: index,
time: entry.time,
});
lastBalance = entry.balance;
}
return walletAddress;
}
export default new WalletApi();

88
backend/src/api/statistics/statistics-api.ts
View file

@ -16,6 +16,7 @@ class StatisticsApi {
fee_data,
total_fee,
min_fee,
vsize_0,
vsize_1,
vsize_2,
vsize_3,
@ -55,7 +56,7 @@ class StatisticsApi {
vsize_1800,
vsize_2000
)
VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)`;
const [result]: any = await DB.query(query);
return result.insertId;
@ -75,6 +76,7 @@ class StatisticsApi {
fee_data,
total_fee,
min_fee,
vsize_0,
vsize_1,
vsize_2,
vsize_3,
@ -115,7 +117,7 @@ class StatisticsApi {
vsize_2000
)
VALUES (${convertToDatetime ? `FROM_UNIXTIME(${statistics.added})` : statistics.added}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
const params: (string | number)[] = [
statistics.unconfirmed_transactions,
@ -125,6 +127,7 @@ class StatisticsApi {
statistics.fee_data,
statistics.total_fee,
statistics.min_fee,
statistics.vsize_0,
statistics.vsize_1,
statistics.vsize_2,
statistics.vsize_3,
@ -177,6 +180,7 @@ class StatisticsApi {
CAST(avg(unconfirmed_transactions) as DOUBLE) as unconfirmed_transactions,
CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second,
CAST(avg(min_fee) as DOUBLE) as min_fee,
CAST(avg(vsize_0) as DOUBLE) as vsize_0,
CAST(avg(vsize_1) as DOUBLE) as vsize_1,
CAST(avg(vsize_2) as DOUBLE) as vsize_2,
CAST(avg(vsize_3) as DOUBLE) as vsize_3,
@ -227,6 +231,7 @@ class StatisticsApi {
CAST(avg(unconfirmed_transactions) as DOUBLE) as unconfirmed_transactions,
CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second,
CAST(avg(min_fee) as DOUBLE) as min_fee,
vsize_0,
vsize_1,
vsize_2,
vsize_3,
@ -414,6 +419,7 @@ class StatisticsApi {
total_fee: s.total_fee,
min_fee: s.min_fee,
vsizes: [
s.vsize_0,
s.vsize_1,
s.vsize_2,
s.vsize_3,
@ -459,6 +465,7 @@ class StatisticsApi {
public mapOptimizedStatisticToStatistic(statistic: OptimizedStatistic[]): Statistic[] {
return statistic.map((s) => {
const completeVsizes = s.vsizes.length === 37 ? [0, ...s.vsizes] : s.vsizes;
return {
added: s.added,
unconfirmed_transactions: s.count,
@ -468,44 +475,45 @@ class StatisticsApi {
total_fee: s.total_fee || 0,
min_fee: s.min_fee,
fee_data: '',
vsize_1: s.vsizes[0],
vsize_2: s.vsizes[1],
vsize_3: s.vsizes[2],
vsize_4: s.vsizes[3],
vsize_5: s.vsizes[4],
vsize_6: s.vsizes[5],
vsize_8: s.vsizes[6],
vsize_10: s.vsizes[7],
vsize_12: s.vsizes[8],
vsize_15: s.vsizes[9],
vsize_20: s.vsizes[10],
vsize_30: s.vsizes[11],
vsize_40: s.vsizes[12],
vsize_50: s.vsizes[13],
vsize_60: s.vsizes[14],
vsize_70: s.vsizes[15],
vsize_80: s.vsizes[16],
vsize_90: s.vsizes[17],
vsize_100: s.vsizes[18],
vsize_125: s.vsizes[19],
vsize_150: s.vsizes[20],
vsize_175: s.vsizes[21],
vsize_200: s.vsizes[22],
vsize_250: s.vsizes[23],
vsize_300: s.vsizes[24],
vsize_350: s.vsizes[25],
vsize_400: s.vsizes[26],
vsize_500: s.vsizes[27],
vsize_600: s.vsizes[28],
vsize_700: s.vsizes[29],
vsize_800: s.vsizes[30],
vsize_900: s.vsizes[31],
vsize_1000: s.vsizes[32],
vsize_1200: s.vsizes[33],
vsize_1400: s.vsizes[34],
vsize_1600: s.vsizes[35],
vsize_1800: s.vsizes[36],
vsize_2000: s.vsizes[37],
vsize_0: completeVsizes[0],
vsize_1: completeVsizes[1],
vsize_2: completeVsizes[2],
vsize_3: completeVsizes[3],
vsize_4: completeVsizes[4],
vsize_5: completeVsizes[5],
vsize_6: completeVsizes[6],
vsize_8: completeVsizes[7],
vsize_10: completeVsizes[8],
vsize_12: completeVsizes[9],
vsize_15: completeVsizes[10],
vsize_20: completeVsizes[11],
vsize_30: completeVsizes[12],
vsize_40: completeVsizes[13],
vsize_50: completeVsizes[14],
vsize_60: completeVsizes[15],
vsize_70: completeVsizes[16],
vsize_80: completeVsizes[17],
vsize_90: completeVsizes[18],
vsize_100: completeVsizes[19],
vsize_125: completeVsizes[20],
vsize_150: completeVsizes[21],
vsize_175: completeVsizes[22],
vsize_200: completeVsizes[23],
vsize_250: completeVsizes[24],
vsize_300: completeVsizes[25],
vsize_350: completeVsizes[26],
vsize_400: completeVsizes[27],
vsize_500: completeVsizes[28],
vsize_600: completeVsizes[29],
vsize_700: completeVsizes[30],
vsize_800: completeVsizes[31],
vsize_900: completeVsizes[32],
vsize_1000: completeVsizes[33],
vsize_1200: completeVsizes[34],
vsize_1400: completeVsizes[35],
vsize_1600: completeVsizes[36],
vsize_1800: completeVsizes[37],
vsize_2000: completeVsizes[38],
}
});
}

3
backend/src/api/statistics/statistics.ts
View file

@ -73,7 +73,7 @@ class Statistics {
const totalWeight = memPoolArray.map((tx) => tx.vsize).reduce((acc, curr) => acc + curr) * 4;
const totalFee = memPoolArray.map((tx) => tx.fee).reduce((acc, curr) => acc + curr);
const logFees = [1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200,
const logFees = [0, 1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200,
250, 300, 350, 400, 500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000];
const weightVsizeFees: { [feePerWU: number]: number } = {};
@ -109,6 +109,7 @@ class Statistics {
total_fee: totalFee,
fee_data: '',
min_fee: minFee,
vsize_0: weightVsizeFees['0'] || 0,
vsize_1: weightVsizeFees['1'] || 0,
vsize_2: weightVsizeFees['2'] || 0,
vsize_3: weightVsizeFees['3'] || 0,

61
backend/src/api/transaction-utils.ts
View file

@ -145,6 +145,9 @@ class TransactionUtils {
return str;
}
/**
* Calculate the witness-adjusted sigops cost of an asm script
*/
public countScriptSigops(script: string, isRawScript: boolean = false, witness: boolean = false): number {
if (!script?.length) {
return 0;
@ -213,6 +216,41 @@ class TransactionUtils {
return sigops;
}
/**
* see https://github.com/bitcoin/bitcoin/blob/25c45bb0d0bd6618ec9296a1a43605657124e5de/src/policy/policy.cpp#L166-L193
* returns true if the transactions is permitted under bip54 sigops rules
*
* "Unlike the existing block wide sigop limit which counts sigops present in the block
* itself (including the scriptPubKey which is not executed until spending later), BIP54
* counts sigops in the block where they are potentially executed (only).
* This means sigops in the spent scriptPubKey count toward the limit.
* `fAccurate` means correctly accounting sigops for CHECKMULTISIGs(VERIFY) with 16 pubkeys
* or fewer. This method of accounting was introduced by BIP16, and BIP54 reuses it.
* The GetSigOpCount call on the previous scriptPubKey counts both bare and P2SH sigops."
*/
public checkSigopsBIP54(tx: TransactionExtended, limit): boolean {
let sigops = 0;
for (const input of tx.vin) {
if (input.scriptsig_asm) {
sigops += this.countScriptSigops(input.scriptsig_asm);
}
if (input.prevout) {
// P2SH redeem script
if (input.prevout.scriptpubkey_type === 'p2sh' && input.inner_redeemscript_asm) {
sigops += this.countScriptSigops(input.inner_redeemscript_asm);
} else {
// prevout scriptpubkey
sigops += this.countScriptSigops(input.prevout.scriptpubkey_asm);
}
}
if (sigops > limit) {
return false;
}
}
return true;
}
// returns the most significant 4 bytes of the txid as an integer
public txidToOrdering(txid: string): number {
return parseInt(
@ -420,6 +458,29 @@ class TransactionUtils {
return { prioritized, deprioritized };
}
// Copied from https://github.com/mempool/mempool/blob/14e49126c3ca8416a8d7ad134a95c5e090324d69/backend/src/api/bitcoin/bitcoin-api.ts#L324
public translateScriptPubKeyType(outputType: string): string {
const map = {
'pubkey': 'p2pk',
'pubkeyhash': 'p2pkh',
'scripthash': 'p2sh',
'witness_v0_keyhash': 'v0_p2wpkh',
'witness_v0_scripthash': 'v0_p2wsh',
'witness_v1_taproot': 'v1_p2tr',
'nonstandard': 'nonstandard',
'multisig': 'multisig',
'anchor': 'anchor',
'nulldata': 'op_return'
};
if (map[outputType]) {
return map[outputType];
} else {
return 'unknown';
}
}
}
export default new TransactionUtils();

28
backend/src/api/websocket-handler.ts
View file

@ -102,7 +102,7 @@ class WebsocketHandler {
'backendInfo': backendInfo.getBackendInfo(),
'loadingIndicators': loadingIndicators.getLoadingIndicators(),
'da': da?.previousTime ? da : undefined,
'fees': feeApi.getRecommendedFee(),
'fees': feeApi.getPreciseRecommendedFee(),
});
}
@ -639,7 +639,7 @@ class WebsocketHandler {
}
memPool.removeFromSpendMap(deletedTransactions);
memPool.addToSpendMap(newTransactions);
const recommendedFees = feeApi.getRecommendedFee();
const recommendedFees = feeApi.getPreciseRecommendedFee();
const latestTransactions = memPool.getLatestTransactions();
@ -1011,15 +1011,19 @@ class WebsocketHandler {
const blockTransactions = structuredClone(transactions);
this.printLogs();
await statistics.runStatistics();
if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
await statistics.runStatistics();
}
const _memPool = memPool.getMempool();
const candidateTxs = await memPool.getMempoolCandidates();
const candidateTxs = memPool.getMempoolCandidates();
let candidates: GbtCandidates | undefined = (memPool.limitGBT && candidateTxs) ? { txs: candidateTxs, added: [], removed: [] } : undefined;
let transactionIds: string[] = (memPool.limitGBT) ? Object.keys(candidates?.txs || {}) : Object.keys(_memPool);
const accelerations = Object.values(mempool.getAccelerations());
await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, structuredClone(transactions));
if (config.DATABASE.ENABLED) {
const accelerations = Object.values(mempool.getAccelerations());
await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, structuredClone(transactions));
}
const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
memPool.handleRbfTransactions(rbfTransactions);
@ -1095,7 +1099,9 @@ class WebsocketHandler {
if (config.CORE_RPC.DEBUG_LOG_PATH && block.extras) {
const firstSeen = getRecentFirstSeen(block.id);
if (firstSeen) {
BlocksRepository.$saveFirstSeenTime(block.id, firstSeen);
if (config.DATABASE.ENABLED) {
BlocksRepository.$saveFirstSeenTime(block.id, firstSeen);
}
block.extras.firstSeen = firstSeen;
}
}
@ -1112,7 +1118,7 @@ class WebsocketHandler {
if (memPool.limitGBT) {
const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null;
const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1;
candidates = await memPool.getNextCandidates(minFeeMempool, minFeeTip, transactions);
candidates = memPool.getNextCandidates(minFeeMempool, minFeeTip, transactions);
transactionIds = Object.keys(candidates?.txs || {});
} else {
candidates = undefined;
@ -1131,7 +1137,7 @@ class WebsocketHandler {
const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
const da = difficultyAdjustment.getDifficultyAdjustment();
const fees = feeApi.getRecommendedFee();
const fees = feeApi.getPreciseRecommendedFee();
const mempoolInfo = memPool.getMempoolInfo();
// pre-compute address transactions
@ -1392,7 +1398,9 @@ class WebsocketHandler {
});
}
await statistics.runStatistics();
if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
await statistics.runStatistics();
}
}
public handleNewStratumJob(job: StratumJob): void {

2
backend/src/config.ts
View file

@ -164,6 +164,7 @@ interface IConfig {
},
WALLETS: {
ENABLED: boolean;
AUTO: boolean;
WALLETS: string[];
},
STRATUM: {
@ -334,6 +335,7 @@ const defaults: IConfig = {
},
'WALLETS': {
'ENABLED': false,
'AUTO': false,
'WALLETS': [],
},
'STRATUM': {

24
backend/src/database.ts
View file

@ -89,8 +89,9 @@ import { execSync } from 'child_process';
OkPacket[] | ResultSetHeader>(queries: { query, params }[], errorLogLevel: LogLevel | 'silent' = 'debug'): Promise<[T, FieldPacket[]][]>
{
const pool = await this.getPool();
const connection = await pool.getConnection();
let connection;
try {
connection = await pool.getConnection();
await connection.beginTransaction();
const results: [T, FieldPacket[]][] = [];
@ -104,10 +105,14 @@ import { execSync } from 'child_process';
return results;
} catch (e) {
logger.warn('Could not complete db transaction, rolling back: ' + (e instanceof Error ? e.message : e));
this.$rollbackAtomic(connection);
if (connection) {
await this.$rollbackAtomic(connection);
}
throw e;
} finally {
connection.release();
if (connection) {
connection.release();
}
}
}
@ -175,6 +180,19 @@ import { execSync } from 'child_process';
}
return this.pool;
}
/**
* Close the database connection pool
* This should only be called when the application is shutting down
* or at the end of test suites
*/
public async close(): Promise<void> {
if (this.pool !== null) {
await this.pool.end();
this.pool = null;
logger.debug('Database connection pool closed');
}
}
}
export default new DB();

50
backend/src/index.ts
View file

@ -100,14 +100,20 @@ class Server {
logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
// Register cleanup listeners for exit events
['exit', 'SIGHUP', 'SIGINT', 'SIGTERM', 'SIGUSR1', 'SIGUSR2'].forEach(event => {
process.on(event, () => { this.onExit(event); });
['SIGHUP', 'SIGINT', 'SIGTERM', 'SIGUSR1', 'SIGUSR2'].forEach(event => {
process.on(event, () => { this.forceExit(event); });
});
process.on('exit', () => {
logger.debug(`'exit' event triggered`);
this.exitCleanup();
});
process.on('uncaughtException', (error) => {
this.onUnhandledException('uncaughtException', error);
console.error(`uncaughtException:`, error);
this.forceExit('uncaughtException', 1);
});
process.on('unhandledRejection', (reason, promise) => {
this.onUnhandledException('unhandledRejection', reason);
console.error(`unhandledRejection:`, reason, promise);
this.forceExit('unhandledRejection', 1);
});
if (config.MEMPOOL.BACKEND === 'esplora') {
@ -131,11 +137,14 @@ class Server {
this.app
.use((req: Request, res: Response, next: NextFunction) => {
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS');
res.setHeader('Access-Control-Allow-Headers', 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With');
res.setHeader('Access-Control-Expose-Headers', 'X-Total-Count,X-Mempool-Auth');
next();
})
.use(express.urlencoded({ extended: true }))
.use(express.text({ type: ['text/plain', 'application/base64'] }))
.use(express.json())
.use(express.urlencoded({ extended: true, limit: '10mb' }))
.use(express.text({ type: ['text/plain', 'application/base64'], limit: '10mb' }))
.use(express.json({ limit: '10mb' }))
;
if (config.DATABASE.ENABLED && config.FIAT_PRICE.ENABLED) {
@ -152,8 +161,15 @@ class Server {
this.setUpWebsocketHandling();
await poolsUpdater.updatePoolsJson(); // Needs to be done before loading the disk cache because we sometimes wipe it
if (config.DATABASE.ENABLED === true && config.MEMPOOL.ENABLED && ['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) && !poolsUpdater.currentSha) {
logger.err(`Failed to retreive pools-v2.json sha, cannot run block indexing. Please make sure you've set valid urls in your mempool-config.json::MEMPOOL::POOLS_JSON_URL and mempool-config.json::MEMPOOL::POOLS_JSON_TREE_UR, aborting now`);
return process.exit(1);
}
await syncAssets.syncAssets$();
await mempoolBlocks.updatePools$();
if (config.DATABASE.ENABLED) {
await mempoolBlocks.updatePools$();
}
if (config.MEMPOOL.ENABLED) {
if (config.MEMPOOL.CACHE_ENABLED) {
await diskCache.$loadMempoolCache();
@ -377,8 +393,16 @@ class Server {
}
}
onExit(exitEvent, code = 0): void {
logger.debug(`onExit for signal: ${exitEvent}`);
forceExit(exitEvent, code?: number): void {
logger.debug(`triggering exit for signal: ${exitEvent}`);
if (code != null) {
// override the default exit code
process.exitCode = code;
}
process.exit();
}
exitCleanup(): void {
if (config.DATABASE.ENABLED) {
DB.releasePidLock();
}
@ -388,12 +412,6 @@ class Server {
if (this.wssUnixSocket) {
this.wssUnixSocket.close();
}
process.exit(code);
}
onUnhandledException(type, error): void {
console.error(`${type}:`, error);
this.onExit(type, 1);
}
}

36
backend/src/indexer.ts
View file

@ -11,6 +11,7 @@ import auditReplicator from './replication/AuditReplication';
import statisticsReplicator from './replication/StatisticsReplication';
import AccelerationRepository from './repositories/AccelerationRepository';
import BlocksAuditsRepository from './repositories/BlocksAuditsRepository';
import BlocksRepository from './repositories/BlocksRepository';
export interface CoreIndex {
name: string;
@ -25,6 +26,7 @@ class Indexer {
private indexerRunning = false;
private tasksRunning: { [key in TaskName]?: boolean; } = {};
private tasksScheduled: { [key in TaskName]?: NodeJS.Timeout; } = {};
private reindexTimeout: NodeJS.Timeout | undefined;
private coreIndexes: CoreIndex[] = [];
public indexerIsRunning(): boolean {
@ -75,10 +77,23 @@ class Indexer {
public reindex(): void {
if (Common.indexingEnabled()) {
if (this.reindexTimeout) {
clearTimeout(this.reindexTimeout);
this.reindexTimeout = undefined;
}
this.runIndexer = true;
}
}
private scheduleNextRun(timeout: number): void {
if (!this.reindexTimeout) { // Only one future run should be planned, ignore if already scheduled
this.reindexTimeout = setTimeout(() => {
this.reindexTimeout = undefined;
this.reindex();
}, timeout);
}
}
/**
* schedules a single task to run in `timeout` ms
* only one task of each type may be scheduled
@ -119,7 +134,7 @@ class Indexer {
switch (task) {
case 'blocksPrices': {
if (!['testnet', 'signet'].includes(config.MEMPOOL.NETWORK) && config.FIAT_PRICE.ENABLED) {
if (!['testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) && config.FIAT_PRICE.ENABLED) {
let lastestPriceId;
try {
lastestPriceId = await PricesRepository.$getLatestPriceId();
@ -137,7 +152,11 @@ class Indexer {
case 'coinStatsIndex': {
logger.debug(`Indexing coinStatsIndex now`);
await mining.$indexCoinStatsIndex();
try {
await mining.$indexCoinStatsIndex();
} catch (e) {
logger.debug(`failed to index coinstatsindex: ` + (e instanceof Error ? e.message : e));
}
} break;
}
@ -151,6 +170,9 @@ class Indexer {
return;
}
this.runIndexer = false;
this.indexerRunning = true;
if (config.FIAT_PRICE.ENABLED) {
try {
await priceUpdater.$run();
@ -165,9 +187,6 @@ class Indexer {
return;
}
this.runIndexer = false;
this.indexerRunning = true;
logger.debug(`Running mining indexer`);
await this.checkAvailableCoreIndexes();
@ -177,7 +196,7 @@ class Indexer {
if (chainValid === false) {
// Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration
logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`, logger.tags.mining);
setTimeout(() => this.reindex(), 10000);
this.scheduleNextRun(10000);
this.indexerRunning = false;
return;
}
@ -194,12 +213,13 @@ class Indexer {
await statisticsReplicator.$sync();
await AccelerationRepository.$indexPastAccelerations();
await BlocksAuditsRepository.$migrateAuditsV0toV1();
await BlocksRepository.$migrateBlocks();
// do not wait for classify blocks to finish
blocks.$classifyBlocks();
} catch (e) {
this.indexerRunning = false;
logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));
setTimeout(() => this.reindex(), 10000);
this.scheduleNextRun(10000);
this.indexerRunning = false;
return;
}
@ -208,7 +228,7 @@ class Indexer {
const runEvery = 1000 * 3600; // 1 hour
logger.debug(`Indexing completed. Next run planned at ${new Date(new Date().getTime() + runEvery).toUTCString()}`);
setTimeout(() => this.reindex(), runEvery);
this.scheduleNextRun(runEvery);
}
}

16
backend/src/logger.ts
View file

@ -67,6 +67,8 @@ class Logger {
}
}
this.client = dgram.createSocket('udp4');
// Unref the socket so it doesn't prevent Node.js from exiting
this.client.unref();
this.network = this.getNetwork();
}
@ -153,6 +155,20 @@ class Logger {
months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
return months[month] + ' ' + day + ' ' + hours + ':' + minutes + ':' + seconds;
}
/**
* Close the UDP socket used for syslog
* This should only be called when shutting down or in test teardown
*/
public close(): void {
if (this.client) {
// Unref allows Node.js to exit even if the socket is open
this.client.unref();
this.client.close(() => {
// Socket closed callback
});
}
}
}
export type LogLevel = 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';

5
backend/src/mempool.interfaces.ts
View file

@ -274,6 +274,7 @@ export const TransactionFlags = {
fake_pubkey: 0b00000010_00000000_00000000_00000000n,
inscription: 0b00000100_00000000_00000000_00000000n,
fake_scripthash: 0b00001000_00000000_00000000_00000000n,
annex: 0b00010000_00000000_00000000_00000000n,
// heuristics
coinjoin: 0b00000001_00000000_00000000_00000000_00000000n,
consolidation: 0b00000010_00000000_00000000_00000000_00000000n,
@ -325,6 +326,8 @@ export interface BlockExtension {
// Requires coinstatsindex, will be set to NULL otherwise
utxoSetSize: number | null;
totalInputAmt: number | null;
// pools-v2.json git hash
definitionHash: string | undefined;
}
/**
@ -334,6 +337,7 @@ export interface BlockExtension {
export interface BlockExtended extends IEsploraApi.Block {
extras: BlockExtension;
canonical?: string;
indexVersion?: number;
}
export interface BlockSummary {
@ -403,6 +407,7 @@ export interface Statistic {
fee_data: string;
min_fee: number;
vsize_0: number;
vsize_1: number;
vsize_2: number;
vsize_3: number;

28
backend/src/repositories/AccelerationRepository.ts
View file

@ -60,6 +60,32 @@ class AccelerationRepository {
}
}
public async $getAccelerationInfoForTxid(txid: string): Promise<PublicAcceleration | null> {
const [rows] = await DB.query(`
SELECT *, UNIX_TIMESTAMP(requested) as requested_timestamp, UNIX_TIMESTAMP(added) as block_timestamp FROM accelerations
JOIN pools on pools.unique_id = accelerations.pool
WHERE txid = ?
`, [txid]) as RowDataPacket[][];
if (rows?.length) {
const row = rows[0];
return {
txid: row.txid,
height: row.height,
added: row.requested_timestamp || row.block_timestamp,
pool: {
id: row.id,
slug: row.slug,
name: row.name,
},
effective_vsize: row.effective_vsize,
effective_fee: row.effective_fee,
boost_rate: row.boost_rate,
boost_cost: row.boost_cost,
};
}
return null;
}
public async $getAccelerationInfo(poolSlug: string | null = null, height: number | null = null, interval: string | null = null): Promise<PublicAcceleration[]> {
if (!interval || !['24h', '3d', '1w', '1m'].includes(interval)) {
interval = '1m';
@ -256,7 +282,7 @@ class AccelerationRepository {
try {
while (!done) {
// don't DDoS the services backend
Common.sleep$(500 + (Math.random() * 1000));
await Common.sleep$(500 + (Math.random() * 1000));
const accelerations = await accelerationApi.$fetchAccelerationHistory(page);
page++;
if (!accelerations?.length) {

374
backend/src/repositories/BlocksRepository.ts
View file

@ -1,4 +1,4 @@
import bitcoinApi from '../api/bitcoin/bitcoin-api-factory';
import bitcoinApi, { bitcoinCoreApi } from '../api/bitcoin/bitcoin-api-factory';
import { BlockExtended, BlockExtension, BlockPrice, EffectiveFeeStats } from '../mempool.interfaces';
import DB from '../database';
import logger from '../logger';
@ -15,8 +15,10 @@ import blocks from '../api/blocks';
import BlocksAuditsRepository from './BlocksAuditsRepository';
import transactionUtils from '../api/transaction-utils';
import { parseDATUMTemplateCreator } from '../utils/bitcoin-script';
import poolsUpdater from '../tasks/pools-updater';
interface DatabaseBlock {
index_version: number;
id: string;
height: number;
version: number;
@ -58,9 +60,11 @@ interface DatabaseBlock {
utxoSetSize: number;
totalInputAmt: number;
firstSeen: number;
stale: boolean;
}
const BLOCK_DB_FIELDS = `
blocks.index_version AS indexVersion,
blocks.hash AS id,
blocks.height,
blocks.version,
@ -101,10 +105,13 @@ const BLOCK_DB_FIELDS = `
blocks.utxoset_change AS utxoSetChange,
blocks.utxoset_size AS utxoSetSize,
blocks.total_input_amt AS totalInputAmt,
UNIX_TIMESTAMP(blocks.first_seen) AS firstSeen
UNIX_TIMESTAMP(blocks.first_seen) AS firstSeen,
blocks.stale
`;
class BlocksRepository {
static version = 1;
/**
* Save indexed block data in the database
*/
@ -114,16 +121,17 @@ class BlocksRepository {
try {
const query = `INSERT INTO blocks(
height, hash, blockTimestamp, size,
weight, tx_count, coinbase_raw, difficulty,
pool_id, fees, fee_span, median_fee,
reward, version, bits, nonce,
merkle_root, previous_block_hash, avg_fee, avg_fee_rate,
median_timestamp, header, coinbase_address, coinbase_addresses,
coinbase_signature, utxoset_size, utxoset_change, avg_tx_size,
total_inputs, total_outputs, total_input_amt, total_output_amt,
fee_percentiles, segwit_total_txs, segwit_total_size, segwit_total_weight,
median_fee_amt, coinbase_signature_ascii
height, hash, blockTimestamp, size,
weight, tx_count, coinbase_raw, difficulty,
pool_id, fees, fee_span, median_fee,
reward, version, bits, nonce,
merkle_root, previous_block_hash, avg_fee, avg_fee_rate,
median_timestamp, header, coinbase_address, coinbase_addresses,
coinbase_signature, utxoset_size, utxoset_change, avg_tx_size,
total_inputs, total_outputs, total_input_amt, total_output_amt,
fee_percentiles, segwit_total_txs, segwit_total_size, segwit_total_weight,
median_fee_amt, coinbase_signature_ascii, definition_hash, index_version,
stale
) VALUE (
?, ?, FROM_UNIXTIME(?), ?,
?, ?, ?, ?,
@ -134,7 +142,8 @@ class BlocksRepository {
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?
?, ?, ?, ?,
?
)`;
const poolDbId = await PoolsRepository.$getPoolByUniqueId(block.extras.pool.id);
@ -181,12 +190,24 @@ class BlocksRepository {
block.extras.segwitTotalWeight,
block.extras.medianFeeAmt,
truncatedCoinbaseSignatureAscii,
poolsUpdater.currentSha,
BlocksRepository.version,
(block.stale ? 1 : 0),
];
await DB.query(query, params);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`, logger.tags.mining);
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart or if a stale block is reconnected
if (!block.stale) {
logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, setting as canonical`, logger.tags.mining);
try {
await this.$setCanonicalBlockAtHeight(block.id, block.height);
} catch (e: any) {
logger.err(`Cannot set canonical block at height ${block.height}. Reason: ` + (e instanceof Error ? e.message : e));
}
} else {
logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`, logger.tags.mining);
}
} else {
logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
throw e;
@ -251,7 +272,11 @@ class BlocksRepository {
* Get all block height that have not been indexed between [startHeight, endHeight]
*/
public async $getMissingBlocksBetweenHeights(startHeight: number, endHeight: number): Promise<number[]> {
if (startHeight < endHeight) {
// Ensure startHeight is the lower value and endHeight is the higher value
const minHeight = Math.min(startHeight, endHeight);
const maxHeight = Math.max(startHeight, endHeight);
if (minHeight === maxHeight) {
return [];
}
@ -259,13 +284,13 @@ class BlocksRepository {
const [rows]: any[] = await DB.query(`
SELECT height
FROM blocks
WHERE height <= ? AND height >= ?
ORDER BY height DESC;
`, [startHeight, endHeight]);
WHERE height >= ? AND height <= ? AND stale = 0
ORDER BY height ASC;
`, [minHeight, maxHeight]);
const indexedBlockHeights: number[] = [];
rows.forEach((row: any) => { indexedBlockHeights.push(row.height); });
const seekedBlocks: number[] = Array.from(Array(startHeight - endHeight + 1).keys(), n => n + endHeight).reverse();
const seekedBlocks: number[] = Array.from(Array(maxHeight - minHeight + 1).keys(), n => n + minHeight);
const missingBlocksHeights = seekedBlocks.filter(x => indexedBlockHeights.indexOf(x) === -1);
return missingBlocksHeights;
@ -285,7 +310,7 @@ class BlocksRepository {
let query = `SELECT count(height) as count, pools.id as poolId
FROM blocks
JOIN pools on pools.id = blocks.pool_id
WHERE tx_count = 1`;
WHERE tx_count = 1 AND stale = 0`;
if (poolId) {
query += ` AND pool_id = ?`;
@ -328,20 +353,16 @@ class BlocksRepository {
const params: any[] = [];
let query = `SELECT count(height) as blockCount
FROM blocks`;
FROM blocks
WHERE stale = 0`;
if (poolId) {
query += ` WHERE pool_id = ?`;
query += ` AND pool_id = ?`;
params.push(poolId);
}
if (interval) {
if (poolId) {
query += ` AND`;
} else {
query += ` WHERE`;
}
query += ` blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
query += ` AND blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
}
try {
@ -365,19 +386,15 @@ class BlocksRepository {
let query = `SELECT
count(height) as blockCount,
max(height) as lastBlockHeight
FROM blocks`;
FROM blocks
WHERE stale = 0`;
if (poolId) {
query += ` WHERE pool_id = ?`;
query += ` AND pool_id = ?`;
params.push(poolId);
}
if (poolId) {
query += ` AND`;
} else {
query += ` WHERE`;
}
query += ` blockTimestamp BETWEEN FROM_UNIXTIME('${from}') AND FROM_UNIXTIME('${to}')`;
query += ` AND blockTimestamp BETWEEN FROM_UNIXTIME('${from}') AND FROM_UNIXTIME('${to}')`;
try {
const [rows] = await DB.query(query, params);
@ -395,7 +412,7 @@ class BlocksRepository {
const params: any[] = [];
let query = `SELECT count(height) as blockCount
FROM blocks
WHERE height <= ${startHeight} AND height >= ${endHeight}`;
WHERE height <= ${startHeight} AND height >= ${endHeight} AND stale = 0`;
try {
const [rows] = await DB.query(query, params);
@ -415,7 +432,7 @@ class BlocksRepository {
SELECT AVG(blocks_audits.match_rate) AS avg_match_rate
FROM blocks
JOIN blocks_audits ON blocks.height = blocks_audits.height
WHERE blocks.pool_id = ?
WHERE blocks.pool_id = ? AND stale = 0
`;
params.push(poolId);
@ -439,7 +456,7 @@ class BlocksRepository {
const query = `
SELECT sum(reward) as total_reward
FROM blocks
WHERE blocks.pool_id = ?
WHERE blocks.pool_id = ? AND stale = 0
`;
params.push(poolId);
@ -461,6 +478,7 @@ class BlocksRepository {
public async $oldestBlockTimestamp(): Promise<number> {
const query = `SELECT UNIX_TIMESTAMP(blockTimestamp) as blockTimestamp
FROM blocks
WHERE stale = 0
ORDER BY height
LIMIT 1;`;
@ -492,7 +510,7 @@ class BlocksRepository {
SELECT ${BLOCK_DB_FIELDS}
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE pool_id = ?`;
WHERE pool_id = ? AND stale = 0`;
params.push(pool.id);
if (startHeight !== undefined) {
@ -527,7 +545,7 @@ class BlocksRepository {
SELECT ${BLOCK_DB_FIELDS}
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE blocks.height = ?`,
WHERE blocks.height = ? AND stale = 0`,
[height]
);
@ -542,12 +560,36 @@ class BlocksRepository {
}
}
/**
* Get one block by hash
*/
public async $getBlockByHash(hash: string): Promise<BlockExtended | null> {
try {
const [rows]: any[] = await DB.query(`
SELECT ${BLOCK_DB_FIELDS}
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE blocks.hash = ?`,
[hash]
);
if (rows.length <= 0) {
return null;
}
return await this.formatDbBlockIntoExtendedBlock(rows[0] as DatabaseBlock);
} catch (e) {
logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Return blocks difficulty
*/
public async $getBlocksDifficulty(): Promise<object[]> {
try {
const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty, bits FROM blocks ORDER BY height ASC`);
const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty, bits FROM blocks WHERE stale = 0 ORDER BY height ASC`);
return rows;
} catch (e) {
logger.err('Cannot get blocks difficulty list from the db. Reason: ' + (e instanceof Error ? e.message : e));
@ -566,7 +608,7 @@ class BlocksRepository {
try {
// Get first block at or after the given timestamp
const query = `SELECT height, hash, blockTimestamp as timestamp FROM blocks
WHERE blockTimestamp <= FROM_UNIXTIME(?)
WHERE blockTimestamp <= FROM_UNIXTIME(?) AND stale = 0
ORDER BY blockTimestamp DESC
LIMIT 1`;
const params = [timestamp];
@ -595,6 +637,7 @@ class BlocksRepository {
SELECT MIN(height) as startBlock, MAX(height) as endBlock, SUM(reward) as totalReward, SUM(fees) as totalFee, SUM(tx_count) as totalTx
FROM
(SELECT height, reward, fees, tx_count FROM blocks
WHERE stale = 0
ORDER by height DESC
LIMIT ?) as sub`;
@ -608,44 +651,83 @@ class BlocksRepository {
}
/**
* Check if the chain of block hash is valid and delete data from the stale branch if needed
* Check if the canonical chain of blocks is valid and fix it if needed
*/
public async $validateChain(): Promise<boolean> {
try {
const start = new Date().getTime();
const tip = await bitcoinApi.$getBlockHashTip();
let firstBadBlockHeight: number | null = null;
const [blocks]: any[] = await DB.query(`
SELECT
height,
hash,
previous_block_hash,
UNIX_TIMESTAMP(blockTimestamp) AS timestamp
UNIX_TIMESTAMP(blockTimestamp) AS timestamp,
stale
FROM blocks
ORDER BY height
ORDER BY height DESC
`);
let partialMsg = false;
let idx = 1;
while (idx < blocks.length) {
if (blocks[idx].height - 1 !== blocks[idx - 1].height) {
if (partialMsg === false) {
logger.info('Some blocks are not indexed, skipping missing blocks during chain validation');
partialMsg = true;
}
++idx;
continue;
const blocksByHash = {};
const blocksByHeight = {};
let minHeight = Infinity;
for (const block of blocks) {
blocksByHash[block.hash] = block;
if (!blocksByHeight[block.height]) {
blocksByHeight[block.height] = [block];
} else {
blocksByHeight[block.height].push(block);
}
if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) {
logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}`);
await this.$deleteBlocksFrom(blocks[idx - 1].height);
await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);
await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(blocks[idx - 1].height);
return false;
}
++idx;
minHeight = block.height;
}
logger.debug(`${idx} blocks hash validated in ${new Date().getTime() - start} ms`);
// ensure that indexed blocks are correctly classified as stale or canonical
// iterate back to genesis, resetting canonical status where necessary
let hash = tip;
const tipHeight = blocksByHash[hash].height || (await bitcoinApi.$getBlock(hash))?.height;
// stop at the last canonical block we're supposed to have indexed already
let lastIndexedBlockHeight = minHeight;
const indexedBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, tipHeight);
if (indexedBlockAmount > 0) {
lastIndexedBlockHeight = Math.max(0, tipHeight - indexedBlockAmount + 1);
}
for (let height = tipHeight; height > lastIndexedBlockHeight; height--) {
const block = blocksByHash[hash];
if (!block) {
// block hasn't been indexed
// mark any other blocks at this height as stale
if (blocksByHeight[height]?.length > 1) {
await this.$setCanonicalBlockAtHeight(null, height);
}
} else if (block.stale) {
// block is marked stale, but shouldn't be
await this.$setCanonicalBlockAtHeight(block.hash, height);
firstBadBlockHeight = height;
}
hash = block?.previous_block_hash;
if (!hash) {
if (height < minHeight) {
// we haven't indexed anything below this height anyway
height = -1;
break;
} else {
logger.info('Some blocks are not indexed, looking up prevhashes directly for chain validation');
hash = await bitcoinApi.$getBlockHash(height - 1);
}
}
}
if (firstBadBlockHeight != null) {
logger.warn(`Chain divergence detected at block ${firstBadBlockHeight}`);
await HashratesRepository.$deleteHashratesFromTimestamp(blocksByHash[firstBadBlockHeight].timestamp - 604800);
await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(firstBadBlockHeight);
return false;
}
logger.debug(`validated best chain of ${tipHeight} blocks in ${new Date().getTime() - start} ms`);
return true;
} catch (e) {
logger.err('Cannot validate chain of block hash. Reason: ' + (e instanceof Error ? e.message : e));
@ -653,19 +735,6 @@ class BlocksRepository {
}
}
/**
* Delete blocks from the database from blockHeight
*/
public async $deleteBlocksFrom(blockHeight: number) {
logger.info(`Delete newer blocks from height ${blockHeight} from the database`, logger.tags.mining);
try {
await DB.query(`DELETE FROM blocks where height >= ${blockHeight}`);
} catch (e) {
logger.err('Cannot delete indexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
}
}
/**
* Get the historical averaged block fees
*/
@ -679,12 +748,13 @@ class BlocksRepository {
FROM blocks
JOIN blocks_prices on blocks_prices.height = blocks.height
JOIN prices on prices.id = blocks_prices.price_id
WHERE stale = 0
`;
if (interval !== null) {
query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
query += ` AND blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
} else if (timespan) {
query += ` WHERE blockTimestamp BETWEEN FROM_UNIXTIME(${timespan.from}) AND FROM_UNIXTIME(${timespan.to})`;
query += ` AND blockTimestamp BETWEEN FROM_UNIXTIME(${timespan.from}) AND FROM_UNIXTIME(${timespan.to})`;
}
query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
@ -710,10 +780,11 @@ class BlocksRepository {
FROM blocks
JOIN blocks_prices on blocks_prices.height = blocks.height
JOIN prices on prices.id = blocks_prices.price_id
WHERE stale = 0
`;
if (interval !== null) {
query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
query += ` AND blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
}
query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
@ -741,10 +812,11 @@ class BlocksRepository {
CAST(AVG(JSON_EXTRACT(fee_span, '$[4]')) as INT) as avgFee_75,
CAST(AVG(JSON_EXTRACT(fee_span, '$[5]')) as INT) as avgFee_90,
CAST(AVG(JSON_EXTRACT(fee_span, '$[6]')) as INT) as avgFee_100
FROM blocks`;
FROM blocks
WHERE stale = 0`;
if (interval !== null) {
query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
query += ` AND blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
}
query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
@ -766,10 +838,11 @@ class BlocksRepository {
CAST(AVG(height) as INT) as avgHeight,
CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(size) as INT) as avgSize
FROM blocks`;
FROM blocks
WHERE stale = 0`;
if (interval !== null) {
query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
query += ` AND blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
}
query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
@ -791,10 +864,11 @@ class BlocksRepository {
CAST(AVG(height) as INT) as avgHeight,
CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(weight) as INT) as avgWeight
FROM blocks`;
FROM blocks
WHERE stale = 0`;
if (interval !== null) {
query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
query += ` AND blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
}
query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
@ -809,11 +883,12 @@ class BlocksRepository {
/**
* Get a list of blocks that have been indexed
* (includes stale blocks)
*/
public async $getIndexedBlocks(): Promise<{ height: number, hash: string }[]> {
public async $getIndexedBlocks(): Promise<{ height: number, hash: string, stale: boolean }[]> {
try {
const [rows] = await DB.query(`SELECT height, hash FROM blocks ORDER BY height DESC`) as RowDataPacket[][];
return rows as { height: number, hash: string }[];
const [rows] = await DB.query(`SELECT height, hash, stale FROM blocks ORDER BY height DESC`) as RowDataPacket[][];
return rows as { height: number, hash: string, stale: boolean }[];
} catch (e) {
logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
@ -858,7 +933,7 @@ class BlocksRepository {
*/
public async $getOldestConsecutiveBlock(): Promise<any> {
try {
const [rows]: any = await DB.query(`SELECT height, UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty, bits FROM blocks ORDER BY height DESC`);
const [rows]: any = await DB.query(`SELECT height, UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty, bits FROM blocks WHERE stale = 0 ORDER BY height DESC`);
for (let i = 0; i < rows.length - 1; ++i) {
if (rows[i].height - rows[i + 1].height > 1) {
return rows[i];
@ -880,7 +955,9 @@ class BlocksRepository {
SELECT UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.height
FROM blocks
LEFT JOIN blocks_prices ON blocks.height = blocks_prices.height
LEFT JOIN prices ON blocks_prices.price_id = prices.id
WHERE blocks_prices.height IS NULL
OR prices.id IS NULL
ORDER BY blocks.height
`);
return rows;
@ -900,6 +977,7 @@ class BlocksRepository {
query += ` (${price.height}, ${price.priceId}),`;
}
query = query.slice(0, -1);
query += ` ON DUPLICATE KEY UPDATE price_id = VALUES(price_id)`;
await DB.query(query);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
@ -919,7 +997,7 @@ class BlocksRepository {
SELECT height, hash
FROM blocks
WHERE height >= ${minHeight} AND height <= ${maxHeight} AND
(utxoset_size IS NULL OR total_input_amt IS NULL)
(utxoset_size IS NULL OR total_input_amt IS NULL) AND stale = 0
`);
return blocks;
} catch (e) {
@ -930,6 +1008,7 @@ class BlocksRepository {
/**
* Get all indexed blocks with missing coinbase addresses
* (includes stale blocks)
*/
public async $getBlocksWithoutCoinbaseAddresses(): Promise<any> {
try {
@ -1013,9 +1092,9 @@ class BlocksRepository {
public async $savePool(id: string, poolId: number): Promise<void> {
try {
await DB.query(`
UPDATE blocks SET pool_id = ?
UPDATE blocks SET pool_id = ?, definition_hash = ?
WHERE hash = ?`,
[poolId, id]
[poolId, poolsUpdater.currentSha, id]
);
} catch (e) {
logger.err(`Cannot update block pool. Reason: ` + (e instanceof Error ? e.message : e));
@ -1041,6 +1120,34 @@ class BlocksRepository {
}
}
/**
* Change which block at a height belongs to the canonical chain
*
* @param hash
* @param height
*/
public async $setCanonicalBlockAtHeight(hash: string | null, height: number): Promise<void> {
try {
// do this first, so that we fail if the block hasn't actually been indexed yet
if (hash) {
await DB.query(`
UPDATE blocks SET stale = 0
WHERE hash = ?`,
[hash]
);
}
// all other blocks at this height must be stale
await DB.query(`
UPDATE blocks SET stale = 1
WHERE height = ? AND hash != ?`,
[height, hash ?? '']
);
} catch (e) {
logger.err(`Cannot set canonical block at height. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Convert a mysql row block into a BlockExtended. Note that you
* must provide the correct field into dbBlk object param
@ -1065,7 +1172,7 @@ class BlocksRepository {
blk.weight = dbBlk.weight;
blk.previousblockhash = dbBlk.previousblockhash;
blk.mediantime = dbBlk.mediantime;
blk.indexVersion = dbBlk.index_version;
// BlockExtension
extras.totalFees = dbBlk.totalFees;
extras.medianFee = dbBlk.medianFee;
@ -1124,11 +1231,10 @@ class BlocksRepository {
{
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
if (extras.feePercentiles === null) {
let summary;
let summaryVersion = 0;
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(dbBlk.id)).map(tx => transactionUtils.extendTransaction(tx));
const txs = (await bitcoinApi.$getTxsForBlock(dbBlk.id, dbBlk.stale)).map(tx => transactionUtils.extendTransaction(tx));
summary = blocks.summarizeBlockTransactions(dbBlk.id, dbBlk.height, txs);
summaryVersion = 1;
} else {
@ -1153,6 +1259,74 @@ class BlocksRepository {
blk.extras = <BlockExtension>extras;
return <BlockExtended>blk;
}
// Execute reindexing tasks & lazy schema migrations
public async $migrateBlocks(): Promise<number> {
let blocksMigrated = 0;
blocksMigrated = await this.$migrateBlocksToV1();
if (blocksMigrated > 0) {
// return early, run the next migration on the next indexing loop
return blocksMigrated;
}
return 0;
}
// migration to fix median fee bug
private async $migrateBlocksToV1(): Promise<number> {
let blocksMigrated = 0;
try {
// median fee bug only affects mmre-than-half but less-than-completely full blocks
const minWeight = config.MEMPOOL.BLOCK_WEIGHT_UNITS / 2 - (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 800);
const maxWeight = config.MEMPOOL.BLOCK_WEIGHT_UNITS - (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 400);
const [rows]: any[] = await DB.query(`
SELECT height, hash, index_version
FROM blocks
WHERE index_version < 1
AND weight >= ?
AND weight <= ?
ORDER BY height DESC
`, [minWeight, maxWeight]);
const blocksToMigrate = rows.length;
let timer = Date.now() / 1000;
const startedAt = Date.now() / 1000;
for (const row of rows) {
// fetch block summary
const transactions = await blocks.$getStrippedBlockTransactions(row.hash);
// recalculate effective fee statistics using latest methodology
const feeStats = Common.calcEffectiveFeeStatistics(transactions.map(tx => ({
weight: tx.vsize * 4,
effectiveFeePerVsize: tx.rate,
txid: tx.txid,
acceleration: tx.acc,
})));
// update block db
await DB.query(`
UPDATE blocks SET index_version = 1, median_fee = ?, fee_span = ?
WHERE hash = ?`,
[feeStats.medianFee, JSON.stringify(feeStats.feeRange), row.hash]
);
const elapsedSeconds = (Date.now() / 1000) - timer;
if (elapsedSeconds > 5) {
const runningFor = (Date.now() / 1000) - startedAt;
const blockPerSeconds = blocksMigrated / elapsedSeconds;
const progress = Math.round(blocksMigrated / blocksToMigrate * 10000) / 100;
logger.debug(`Migrating blocks to version 1 | ~${blockPerSeconds.toFixed(2)} blocks/sec | height: ${row.height} | total: ${blocksMigrated}/${blocksToMigrate} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`);
timer = Date.now() / 1000;
}
blocksMigrated++;
}
logger.notice(`Migrating blocks to version 1 completed: migrated ${blocksMigrated} blocks`);
} catch (e) {
logger.err(`Migrating blocks to version 1 failed. Trying again later. Reason: ${(e instanceof Error ? e.message : e)}`);
throw e;
}
return blocksMigrated;
}
}
export default new BlocksRepository();

14
backend/src/repositories/BlocksSummariesRepository.ts
View file

@ -1,4 +1,5 @@
import { RowDataPacket } from 'mysql2';
import { Common } from '../api/common';
import DB from '../database';
import logger from '../logger';
import { BlockSummary, TransactionClassified } from '../mempool.interfaces';
@ -192,6 +193,19 @@ class BlocksSummariesRepository {
return null;
}
}
public async $isSummaryIndexed(id: string): Promise<boolean> {
if (!Common.blocksSummariesIndexingEnabled()) {
return false;
}
try {
const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries WHERE id = ?`, [id]);
return rows.length > 0;
} catch (e) {
logger.err(`Cannot check if block summary is indexed. Reason: ` + (e instanceof Error ? e.message : e));
}
return false;
}
}
export default new BlocksSummariesRepository();

2
backend/src/repositories/HashratesRepository.ts
View file

@ -93,7 +93,7 @@ class HashratesRepository {
const [rows]: any[] = await DB.query(query);
return rows.map(row => row.timestamp);
} catch (e) {
logger.err('Cannot retreive indexed weekly hashrate timestamps. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
logger.err('Cannot retrieve indexed weekly hashrate timestamps. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
throw e;
}
}

8
backend/src/repositories/PoolsRepository.ts
View file

@ -45,10 +45,11 @@ class PoolsRepository {
FROM blocks
JOIN pools on pools.id = pool_id
LEFT JOIN blocks_audits ON blocks_audits.height = blocks.height
WHERE blocks.stale = 0
`;
if (interval) {
query += ` WHERE blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
query += ` AND blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
}
query += ` GROUP BY pool_id
@ -70,6 +71,7 @@ class PoolsRepository {
const query = `SELECT COUNT(height) as blockCount, pools.id as poolId, pools.name as poolName
FROM pools
LEFT JOIN blocks on pools.id = blocks.pool_id AND blocks.blockTimestamp BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?)
WHERE blocks.stale = 0
GROUP BY pools.id`;
try {
@ -100,7 +102,7 @@ class PoolsRepository {
if (parse) {
rows[0].regexes = JSON.parse(rows[0].regexes);
}
if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
if (['testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
rows[0].addresses = []; // pools-v2.json only contains mainnet addresses
} else if (parse) {
rows[0].addresses = JSON.parse(rows[0].addresses);
@ -132,7 +134,7 @@ class PoolsRepository {
if (parse) {
rows[0].regexes = JSON.parse(rows[0].regexes);
}
if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
if (['testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
rows[0].addresses = []; // pools.json only contains mainnet addresses
} else if (parse) {
rows[0].addresses = JSON.parse(rows[0].addresses);

5
backend/src/repositories/PricesRepository.ts
View file

@ -296,6 +296,7 @@ class PricesRepository {
id,
USD
FROM prices
WHERE USD >= 0
ORDER BY time
`);
return times as {time: number, id: number, USD: number}[];
@ -305,6 +306,7 @@ class PricesRepository {
const [rates] = await DB.query(`
SELECT ${ApiPriceFields}
FROM prices
WHERE USD >= 0
ORDER BY time DESC
LIMIT 1`
);
@ -321,6 +323,7 @@ class PricesRepository {
SELECT ${ApiPriceFields}
FROM prices
WHERE UNIX_TIMESTAMP(time) < ?
AND USD >= 0
ORDER BY time DESC
LIMIT 1`,
[timestamp]
@ -332,6 +335,7 @@ class PricesRepository {
const [latestPrices] = await DB.query(`
SELECT ${ApiPriceFields}
FROM prices
WHERE USD >= 0
ORDER BY time DESC
LIMIT 1
`);
@ -429,6 +433,7 @@ class PricesRepository {
const [rates] = await DB.query(`
SELECT ${ApiPriceFields}
FROM prices
WHERE USD >= 0
ORDER BY time DESC
`);
if (!Array.isArray(rates)) {

19
backend/src/tasks/lightning/network-sync.service.ts
View file

@ -269,17 +269,16 @@ class NetworkSyncService {
private async $scanForClosedChannels(): Promise<void> {
let currentBlockHeight = blocks.getCurrentBlockHeight();
if (config.MEMPOOL.ENABLED === false) { // https://github.com/mempool/mempool/issues/3582
currentBlockHeight = await bitcoinApi.$getBlockHeightTip();
}
if (this.closedChannelsScanBlock === currentBlockHeight) {
logger.debug(`We've already scan closed channels for this block, skipping.`);
return;
}
let progress = 0;
try {
if (config.MEMPOOL.ENABLED === false) { // https://github.com/mempool/mempool/issues/3582
currentBlockHeight = await bitcoinApi.$getBlockHeightTip();
}
if (this.closedChannelsScanBlock === currentBlockHeight) {
logger.debug(`We've already scan closed channels for this block, skipping.`);
return;
}
let progress = 0;
let log = `Starting closed channels scan`;
if (this.closedChannelsScanBlock > 0) {
log += `. Last scan was at block ${this.closedChannelsScanBlock}`;

8
backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts
View file

@ -79,6 +79,10 @@ class FundingTxFetcher {
}
const parts = channelId.split('x');
if (parts.length < 3) {
logger.debug(`Channel ID ${channelId} does not seem valid, should contains at least 3 parts separated by 'x'`, logger.tags.ln);
return null;
}
const blockHeight = parts[0];
const txIdx = parts[1];
const outputIdx = parts[2];
@ -99,6 +103,10 @@ class FundingTxFetcher {
}
const txid = block.tx[txIdx];
if (!txid) {
logger.debug(`Cannot cache ${channelId} funding tx. TX index ${txIdx} does not exist in block ${block.hash ?? block.id}`, logger.tags.ln);
return null;
}
const rawTx = await bitcoinClient.getRawTransaction(txid);
const tx = await bitcoinClient.decodeRawTransaction(rawTx);

17
backend/src/tasks/pools-updater.ts
View file

@ -31,7 +31,7 @@ class PoolsUpdater {
}
public async updatePoolsJson(): Promise<void> {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false ||
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) === false ||
config.MEMPOOL.ENABLED === false
) {
return;
@ -45,15 +45,15 @@ class PoolsUpdater {
this.lastRun = now;
try {
if (config.DATABASE.ENABLED === true) {
this.currentSha = await this.getShaFromDb();
}
const githubSha = await this.fetchPoolsSha(); // Fetch pools-v2.json sha from github
if (githubSha === null) {
return;
}
if (config.DATABASE.ENABLED === true) {
this.currentSha = await this.getShaFromDb();
}
logger.debug(`pools-v2.json sha | Current: ${this.currentSha} | Github: ${githubSha}`, this.tag);
if (this.currentSha !== null && this.currentSha === githubSha) {
return;
@ -88,8 +88,8 @@ class PoolsUpdater {
try {
await DB.query('START TRANSACTION;');
await poolsParser.migratePoolsJson();
await this.updateDBSha(githubSha);
await poolsParser.migratePoolsJson();
await DB.query('COMMIT;');
} catch (e) {
logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, this.tag);
@ -98,7 +98,8 @@ class PoolsUpdater {
logger.info(`Mining pools-v2.json (${githubSha}) import completed`, this.tag);
} catch (e) {
this.lastRun = now - 600; // Try again in 10 minutes
// fast-forward lastRun to 10 minutes before the next scheduled update
this.lastRun = now - Math.max(config.MEMPOOL.POOLS_UPDATE_DELAY - 600, 600);
logger.err(`PoolsUpdater failed. Will try again in 10 minutes. Exception: ${JSON.stringify(e)}`, this.tag);
}
}
@ -121,7 +122,7 @@ class PoolsUpdater {
/**
* Fetch our latest pools-v2.json sha from the db
*/
private async getShaFromDb(): Promise<string | null> {
public async getShaFromDb(): Promise<string | null> {
try {
const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
return (rows.length > 0 ? rows[0].string : null);

34
backend/src/tasks/price-updater.ts
View file

@ -56,6 +56,7 @@ class PriceUpdater {
private feeds: PriceFeed[] = [];
private currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
private latestPrices: ApiPrice;
private latestGoodPrices: ApiPrice;
private currencyConversionFeed: ConversionFeed | undefined;
private newCurrencies: string[] = ['BGN', 'BRL', 'CNY', 'CZK', 'DKK', 'HKD', 'HRK', 'HUF', 'IDR', 'ILS', 'INR', 'ISK', 'KRW', 'MXN', 'MYR', 'NOK', 'NZD', 'PHP', 'PLN', 'RON', 'RUB', 'SEK', 'SGD', 'THB', 'TRY', 'ZAR'];
private lastTimeConversionsRatesFetched: number = 0;
@ -64,6 +65,7 @@ class PriceUpdater {
constructor() {
this.latestPrices = this.getEmptyPricesObj();
this.latestGoodPrices = this.getEmptyPricesObj();
this.feeds.push(new BitflyerApi()); // Does not have historical endpoint
this.feeds.push(new KrakenApi());
@ -76,7 +78,7 @@ class PriceUpdater {
}
public getLatestPrices(): ApiPrice {
return this.latestPrices;
return this.latestGoodPrices;
}
public getEmptyPricesObj(): ApiPrice {
@ -128,10 +130,11 @@ class PriceUpdater {
*/
public async $initializeLatestPriceWithDb(): Promise<void> {
this.latestPrices = await PricesRepository.$getLatestConversionRates();
this.latestGoodPrices = JSON.parse(JSON.stringify(this.latestPrices));
}
public async $run(): Promise<void> {
if (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet') {
if (['testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
// Coins have no value on testnet/signet, so we want to always show 0
return;
}
@ -179,6 +182,14 @@ class PriceUpdater {
this.running = false;
}
private setLatestPrice(currency, price): void {
this.latestPrices[currency] = price;
if (price > 0) {
this.latestGoodPrices[currency] = price;
this.latestGoodPrices.time = Math.round(new Date().getTime() / 1000);
}
}
private getMillisecondsSinceBeginningOfHour(): number {
const now = new Date();
const beginningOfHour = new Date(now);
@ -245,16 +256,16 @@ class PriceUpdater {
// Compute average price, non weighted
prices = prices.filter(price => price > 0);
if (prices.length === 0) {
this.latestPrices[currency] = -1;
this.setLatestPrice(currency, -1);
} else {
this.latestPrices[currency] = Math.round(getMedian(prices));
this.setLatestPrice(currency, Math.round(getMedian(prices)));
}
}
if (config.FIAT_PRICE.API_KEY && this.latestPrices.USD > 0 && Object.keys(this.latestConversionsRatesFromFeed).length > 0) {
for (const conversionCurrency of this.newCurrencies) {
if (this.latestConversionsRatesFromFeed[conversionCurrency] > 0 && this.latestPrices.USD * this.latestConversionsRatesFromFeed[conversionCurrency] < MAX_PRICES[conversionCurrency]) {
this.latestPrices[conversionCurrency] = Math.round(this.latestPrices.USD * this.latestConversionsRatesFromFeed[conversionCurrency]);
this.setLatestPrice(conversionCurrency, Math.round(this.latestPrices.USD * this.latestConversionsRatesFromFeed[conversionCurrency]));
}
}
}
@ -271,18 +282,19 @@ class PriceUpdater {
}
this.latestPrices.time = Math.round(new Date().getTime() / 1000);
logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
if (this.ratesChangedCallback) {
this.ratesChangedCallback(this.latestPrices);
}
if (!forceUpdate) {
this.cyclePosition++;
}
if (this.latestPrices.USD === -1) {
this.latestPrices = await PricesRepository.$getLatestConversionRates();
logger.warn(`No BTC price available, falling back to latest known price: ${JSON.stringify(this.latestGoodPrices)}`);
} else {
logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestGoodPrices)}`);
}
if (this.ratesChangedCallback && this.latestGoodPrices.USD > 0) {
this.ratesChangedCallback(this.latestGoodPrices);
}
}

2
backend/src/utils/axios-query.ts
View file

@ -18,7 +18,7 @@ export async function query(path, throwOnFail: boolean = false): Promise<object
headers: {
'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
},
timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 20000
};
let retry = 0;
let lastError: any = null;

14
backend/src/utils/file-read.ts
View file

@ -24,7 +24,7 @@ function extractDateFromLogLine(line: string): number | undefined {
const dateStr = dateMatch[0];
const date = new Date(dateStr);
let timestamp = Math.floor(date.getTime() / 1000); // Remove decimal (microseconds are added later)
const timestamp = Math.floor(date.getTime() / 1000); // Remove decimal (microseconds are added later)
const timePart = dateStr.split('T')[1];
const microseconds = timePart.split('.')[1] || '';
@ -41,14 +41,18 @@ export function getRecentFirstSeen(hash: string): number | undefined {
if (debugLogPath) {
try {
// Read the last few lines of debug.log
const lines = readFile(debugLogPath, 2048);
const lines = readFile(debugLogPath, 4096).reverse();
for (let i = lines.length - 1; i >= 0; i--) {
const line = lines[i];
if (line && line.includes(`Saw new header hash=${hash}`)) {
let bestMatch: number | undefined;
for (const line of lines) {
if (line.includes(`Saw new header hash=${hash}`) || line.includes(`Saw new cmpctblock header hash=${hash}`)) {
return extractDateFromLogLine(line);
} else if (line.includes(`UpdateTip: new best=${hash}`)) {
bestMatch = extractDateFromLogLine(line);
}
}
return bestMatch;
} catch (e) {
logger.err(`Cannot parse block first seen time from Core logs. Reason: ` + (e instanceof Error ? e.message : e));
}

14
backend/testSetup.integration.ts Normal file
View file

@ -0,0 +1,14 @@
// Integration test setup - uses real implementations, not mocks
//
// Note: We don't mock ./mempool-config.json here because:
// 1. The MEMPOOL_CONFIG_FILE env var points to mempool-config.test.json
// 2. config.ts will load that file via require() if env var is set
// 3. If we mock it, we interfere with the actual config loading
//
// Don't mock these for integration tests - we want real implementations:
// - logger.ts (need real logging)
// - config.ts (need real config from mempool-config.test.json)
// - rbf-cache.ts (might be used by repositories)
// - mempool.ts (might be used by repositories)
// - memory-cache.ts (might be used by repositories)

3
contributors/AaronDewes.txt Normal file
View file

@ -0,0 +1,3 @@
I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of October 31, 2025.
Signed: AaronDewes

3
contributors/achow101.txt Normal file
View file

@ -0,0 +1,3 @@
I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of October 12, 2025.
Signed: achow101

3
contributors/adambor.txt Normal file
View file

@ -0,0 +1,3 @@
I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of July 12, 2025.
Signed: adambor

29
docker/backend/Dockerfile
View file

@ -1,20 +1,20 @@
FROM node:20.15.0-buster-slim AS builder
FROM rust:1.84-bookworm AS builder
ARG commitHash
ENV MEMPOOL_COMMIT_HASH=${commitHash}
WORKDIR /build
RUN apt-get update && \
apt-get install -y curl ca-certificates && \
curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && \
apt-get install -y nodejs=22.14.0-1nodesource1 build-essential python3 pkg-config && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
COPY . .
RUN apt-get update
RUN apt-get install -y build-essential python3 pkg-config curl ca-certificates
# Install Rust via rustup
RUN CPU_ARCH=$(uname -m); if [ "$CPU_ARCH" = "armv7l" ]; then c_rehash; fi
#RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable
#Workaround to run on github actions from https://github.com/rust-lang/rustup/issues/2700#issuecomment-1367488985
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sed 's#/proc/self/exe#\/bin\/sh#g' | sh -s -- -y --default-toolchain stable
ENV PATH="/root/.cargo/bin:$PATH"
ENV PATH="/usr/local/cargo/bin:$PATH"
COPY --from=backend . .
COPY --from=rustgbt . ../rust/
@ -24,7 +24,14 @@ RUN npm install --omit=dev --omit=optional
WORKDIR /build
RUN npm run package
FROM node:20.15.0-buster-slim
FROM rust:1.84-bookworm AS runtime
RUN apt-get update && \
apt-get install -y curl ca-certificates && \
curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && \
apt-get install -y nodejs=22.14.0-1nodesource1 && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
WORKDIR /backend

2
docker/backend/start.sh
View file

@ -26,7 +26,7 @@ __MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=1}
__MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=0}
__MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
__MEMPOOL_AUTOMATIC_POOLS_UPDATE__=${MEMPOOL_AUTOMATIC_POOLS_UPDATE:=false}
__MEMPOOL_AUTOMATIC_POOLS_UPDATE__=${MEMPOOL_AUTOMATIC_POOLS_UPDATE:=true}
__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json}
__MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
__MEMPOOL_POOLS_UPDATE_DELAY__=${MEMPOOL_POOLS_UPDATE_DELAY:=604800}

2
docker/frontend/Dockerfile
View file

@ -1,4 +1,4 @@
FROM node:20.15.0-buster-slim AS builder
FROM node:22.14.0-bookworm-slim AS builder
ARG commitHash
ENV DOCKER_COMMIT_HASH=${commitHash}

2
docker/frontend/entrypoint.sh
View file

@ -45,6 +45,7 @@ __SERVICES_API__=${SERVICES_API:=https://mempool.space/api/v1/services}
__PUBLIC_ACCELERATIONS__=${PUBLIC_ACCELERATIONS:=false}
__HISTORICAL_PRICE__=${HISTORICAL_PRICE:=true}
__ADDITIONAL_CURRENCIES__=${ADDITIONAL_CURRENCIES:=false}
__STRATUM_ENABLED__=${STRATUM_ENABLED:=false}
# Export as environment variables to be used by envsubst
export __MAINNET_ENABLED__
@ -76,6 +77,7 @@ export __SERVICES_API__
export __PUBLIC_ACCELERATIONS__
export __HISTORICAL_PRICE__
export __ADDITIONAL_CURRENCIES__
export __STRATUM_ENABLED__
folder=$(find /var/www/mempool -name "config.js" | xargs dirname)
echo ${folder}

16
frontend/angular.json
View file

@ -181,8 +181,8 @@
"inject": false
},
{
"input": "src/theme-wiz.scss",
"bundleName": "wiz",
"input": "src/theme-softsimon.scss",
"bundleName": "softsimon",
"inject": false
},
{
@ -261,20 +261,14 @@
"proxyConfig": "proxy.conf.mixed.js",
"verbose": true
},
"staging": {
"proxyConfig": "proxy.conf.js",
"disableHostCheck": true,
"host": "0.0.0.0",
"verbose": true
},
"local-prod": {
"proxyConfig": "proxy.conf.js",
"disableHostCheck": true,
"host": "0.0.0.0",
"verbose": false
},
"local-staging": {
"proxyConfig": "proxy.conf.staging.js",
"parameterized": {
"proxyConfig": "proxy.conf.parameterized.js",
"disableHostCheck": true,
"host": "0.0.0.0",
"verbose": false
@ -371,4 +365,4 @@
}
}
}
}
}

2
frontend/custom-meta-config.json
View file

@ -25,7 +25,7 @@
"component": "twitter",
"mobileOrder": 5,
"props": {
"handle": "Metaplanet_JP"
"handle": "Metaplanet"
}
},
{

51
frontend/custom-river-config.json Normal file
View file

@ -0,0 +1,51 @@
{
"theme": "wiz",
"enterprise": "river",
"branding": {
"name": "river",
"title": "river",
"site_id": 22,
"header_img": "/resources/riverlogo.svg",
"footer_img": "/resources/riverlogo.svg"
},
"dashboard": {
"widgets": [
{
"component": "fees",
"mobileOrder": 4
},
{
"component": "walletBalance",
"mobileOrder": 1,
"props": {
"wallet": "RIVER"
}
},
{
"component": "twitter",
"mobileOrder": 5,
"props": {
"handle": "River"
}
},
{
"component": "wallet",
"mobileOrder": 2,
"props": {
"wallet": "RIVER",
"period": "all"
}
},
{
"component": "blocks"
},
{
"component": "walletTransactions",
"mobileOrder": 3,
"props": {
"wallet": "RIVER"
}
}
]
}
}

42
frontend/custom-strategy-config.json Normal file
View file

@ -0,0 +1,42 @@
{
"theme": "wiz",
"enterprise": "strategy",
"branding": {
"name": "strategy",
"title": "strategy",
"site_id": 22,
"header_img": "/resources/strategylogo.svg",
"footer_img": "/resources/strategylogo.svg"
},
"dashboard": {
"widgets": [
{
"component": "fees",
"mobileOrder": 1
},
{
"component": "difficulty",
"mobileOrder": 4
},
{
"component": "twitter",
"mobileOrder": 2,
"props": {
"handle": "MicroStrategy"
}
},
{
"component": "goggles",
"mobileOrder": 3
},
{
"component": "blocks",
"mobileOrder": 5
},
{
"component": "transactions",
"mobileOrder": 6
}
]
}
}

23
frontend/custom-sv-config.json
View file

@ -16,10 +16,10 @@
"mobileOrder": 4
},
{
"component": "balance",
"component": "walletBalance",
"mobileOrder": 1,
"props": {
"address": "32ixEdVJWo3kmvJGMTZq5jAQVZZeuwnqzo"
"wallet": "ONBTC"
}
},
{
@ -30,21 +30,28 @@
}
},
{
"component": "address",
"component": "wallet",
"mobileOrder": 2,
"props": {
"address": "32ixEdVJWo3kmvJGMTZq5jAQVZZeuwnqzo",
"period": "1m"
"wallet": "ONBTC",
"period": "1m",
"label": "bitcoin.gob.sv",
"image": "/resources/elsalvador-flag.svg"
}
},
{
"component": "blocks"
"component": "simpleproof_cubo",
"mobileOrder": 6,
"props": {
"label": "CUBO+ Certificates",
"key": "cubo_certificates"
}
},
{
"component": "addressTransactions",
"component": "walletTransactions",
"mobileOrder": 3,
"props": {
"address": "32ixEdVJWo3kmvJGMTZq5jAQVZZeuwnqzo"
"wallet": "ONBTC"
}
}
]

48
frontend/custom-xxi-config.json Normal file
View file

@ -0,0 +1,48 @@
{
"theme": "wiz",
"enterprise": "xxi",
"branding": {
"name": "xxi",
"title": "Twenty One",
"site_id": 23,
"header_img": "/resources/xxi/xxilogo.png",
"footer_img": "/resources/xxi/xxilogo.png"
},
"dashboard": {
"widgets": [
{
"component": "fees",
"mobileOrder": 4
},
{
"component": "walletBalance",
"mobileOrder": 1,
"props": {
"wallet": "xxi"
}
},
{
"component": "goggles",
"mobileOrder": 5
},
{
"component": "wallet",
"mobileOrder": 2,
"props": {
"wallet": "xxi",
"period": "all"
}
},
{
"component": "blocks"
},
{
"component": "walletTransactions",
"mobileOrder": 3,
"props": {
"wallet": "xxi"
}
}
]
}
}

15
frontend/cypress/e2e/liquid/liquid.spec.ts
View file

@ -57,11 +57,6 @@ describe('Liquid', () => {
});
});
it('loads the tv page - desktop', () => {
cy.visit(`${basePath}/tv`);
cy.waitForSkeletonGone();
});
it('loads the graphs page - mobile', () => {
cy.visit(`${basePath}`)
cy.waitForSkeletonGone();
@ -144,10 +139,10 @@ describe('Liquid', () => {
it('show unblinded TX', () => {
cy.visit(`${basePath}/tx/f2f41c0850e8e7e3f1af233161fd596662e67c11ef10ed15943884186fbb7f46#blinded=100000,6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d,0ab9f70650f16b1db8dfada05237f7d0d65191c3a13183da8a2ddddfbde9a2ad,fd98b2edc5530d76acd553f206a431f4c1fab27e10e290ad719582af878e98fc,2364760,6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d,90c7a43b15b905bca045ca42a01271cfe71d2efe3133f4197792c24505cb32ed,12eb5959d9293b8842e7dd8bc9aa9639fd3fd031c5de3ba911adeca94eb57a3a`);
cy.waitForSkeletonGone();
cy.get('.table-tx-vin tr:nth-child(1) .amount').should('contain.text', '0.02465000 L-BTC');
cy.get('.table-tx-vin tr:nth-child(1) .amount').should('contain.text', '0.02465000 LBTC');
cy.get('.table-tx-vin tr').should('have.class', 'assetBox');
cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', '0.00100000 L-BTC');
cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', '0.02364760 L-BTC');
cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', '0.00100000 LBTC');
cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', '0.02364760 LBTC');
cy.get('.table-tx-vout tr').should('have.class', 'assetBox');
});
@ -174,13 +169,13 @@ describe('Liquid', () => {
cy.visit(`${basePath}/tx/f2f41c0850e8e7e3f1af233161fd596662e67c11ef10ed15943884186fbb7f46#blinded=100000,6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d,0ab9f70650f16b1db8dfada05237f7d0d65191c3a13183da8a2ddddfbde9a2ad,fd98b2edc5530d76acd553f206a431f4c1fab27e10e290ad719582af878e98fc`);
cy.waitForSkeletonGone();
cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', 'assetBox');
cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', '0.00100000 L-BTC');
cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', '0.00100000 LBTC');
});
it('show second unblinded vout', () => {
cy.visit(`${basePath}/tx/f2f41c0850e8e7e3f1af233161fd596662e67c11ef10ed15943884186fbb7f46#blinded=2364760,6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d,90c7a43b15b905bca045ca42a01271cfe71d2efe3133f4197792c24505cb32ed,12eb5959d9293b8842e7dd8bc9aa9639fd3fd031c5de3ba911adeca94eb57a3a`);
cy.get('.table-tx-vout tr:nth-child(2').should('have.class', 'assetBox');
cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', '0.02364760 L-BTC');
cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', '0.02364760 LBTC');
});
it('show invalid error unblinded TX', () => {

13
frontend/cypress/e2e/liquidtestnet/liquidtestnet.spec.ts
View file

@ -57,11 +57,6 @@ describe('Liquid Testnet', () => {
cy.waitForSkeletonGone();
});
it('loads the tv page - desktop', () => {
cy.visit(`${basePath}/tv`);
cy.waitForSkeletonGone();
});
it('loads the graphs page - mobile', () => {
cy.visit(`${basePath}`)
cy.waitForSkeletonGone();
@ -108,10 +103,10 @@ describe('Liquid Testnet', () => {
it('show unblinded TX', () => {
cy.visit(`${basePath}/tx/c3d908ab77891e4c569b0df71aae90f4720b157019ebb20db176f4f9c4d626b8#blinded=100000,144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49,df290ead654d7d110ebc5aaf0bcf11d5b5d360431a467f1cde0a856fde986893,33cb3a2fd2e76643843691cf44a78c5cd28ec652a414da752160ad63fbd37bc9,49741,144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49,edb0713bcbfcb3daabf601cb50978439667d208e15fed8a5ebbfea5696cda1d5,4de70115501e8c7d6bd763e229bf42781edeacf6e75e1d7bdfa4c63104bc508a`);
cy.waitForSkeletonGone();
cy.get('.table-tx-vin tr:nth-child(1) .amount').should('contain.text', '0.00100000 tL-BTC');
cy.get('.table-tx-vin tr:nth-child(1) .amount').should('contain.text', '0.00100000 tLBTC');
cy.get('.table-tx-vin tr').should('have.class', 'assetBox');
cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', '0.00050000 tL-BTC');
cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', '0.00049741 tL-BTC');
cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', '0.00050000 tLBTC');
cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', '0.00049741 tLBTC');
cy.get('.table-tx-vout tr').should('have.class', 'assetBox');
});
@ -138,7 +133,7 @@ describe('Liquid Testnet', () => {
cy.visit(`${basePath}/tx/0877bc0c7aa5c2b8d0e4b15450425879b8783c40e341806037a605ef836fb886#blinded=5000,38fca2d939696061a8f76d4e6b5eecd54e3b4221c846f24a6b279e79952850a5,328de54e90e867a9154b4f1eb7fcab86267e880fa2ee9e53b41a91e61dab86e6,8885831e6b089eaf06889d53a24843f0da533d300a7b1527b136883a6819f3ae,5000,38fca2d939696061a8f76d4e6b5eecd54e3b4221c846f24a6b279e79952850a5,aca78b953615d69ae0ae68c4c5c3c0ee077c10bc20ad3f0c5960706004e6cb56,d2ec175afe5f761e2dbd443faf46abbb7091f341deb3387e5787d812bdb2df9f,100000,144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49,4b54a4ca809b3844f34dd88b68617c4c866d92a02211f02ba355755bac20a1c6,eddd02e92b0cfbad8cab89828570a50f2c643bb2a54d886c86e25ce47e818685,99729,144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49,8b86d565c9549eb0352bb81ee576d01d064435b64fddcc045decebeb1d9913ce,b082ce3448d40d47b5b39f15d72b285f4a1046b636b56c25f32f498ece29d062,10000,38fca2d939696061a8f76d4e6b5eecd54e3b4221c846f24a6b279e79952850a5,62b04ee86198d6b41681cdd0acb450ab366af727a010aaee8ba0b9e69ff43896,3f98429bca9b538dc943c22111f25d9c4448d45a63ff0f4e58b22fd434c0365e`);
cy.waitForSkeletonGone();
cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', 'assetBox');
cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', '0.00099729 tL-BTC');
cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', '0.00099729 tLBTC');
});
it('show second unblinded vout (asset)', () => {

241
frontend/cypress/e2e/mainnet/mainnet.spec.ts
View file

@ -1,4 +1,4 @@
import { emitMempoolInfo, dropWebSocket } from '../../support/websocket';
import { emitMempoolInfo, dropWebSocket, receiveWebSocketMessageFromServer } from '../../support/websocket';
const baseModule = Cypress.env('BASE_MODULE');
@ -216,6 +216,69 @@ describe('Mainnet', () => {
cy.get('[data-cy="tx-1"] .table-tx-vout .highlight').its('length').should('equal', 2);
cy.get('[data-cy="tx-1"] .table-tx-vout .highlight').invoke('text').should('contain', `${address}`);
});
describe('address poisoning', () => {
it('highlights potential address poisoning attacks on outputs, prefix and infix', () => {
const txid = '152a5dea805f95d6f83e50a9fd082630f542a52a076ebabdb295723eaf53fa30';
const prefix = '1DonatePLease';
const infix1 = 'SenderAddressXVXCmAY';
const infix2 = '5btcToSenderXXWBoKhB';
cy.visit(`/tx/${txid}`);
cy.waitForSkeletonGone();
cy.get('.alert-mempool').should('exist');
cy.get('.poison-alert').its('length').should('equal', 2);
cy.get('.prefix')
.should('have.length', 2)
.each(($el) => {
cy.wrap($el).should('have.text', prefix);
});
cy.get('.infix')
.should('have.length', 2)
.then(($infixes) => {
cy.wrap($infixes[0]).should('have.text', infix1);
cy.wrap($infixes[1]).should('have.text', infix2);
});
});
it('highlights potential address poisoning attacks on inputs and outputs, prefix, infix and postfix', () => {
const txid = '44544516084ea916ff1eb69c675c693e252addbbaf77102ffff86e3979ac6132';
const prefix = 'bc1qge8';
const infix1 = '6gqjnk8aqs3nvv7ejrvcd4zq6qur3';
const infix2 = 'xyxjex6zzzx5g8hh65vsel4e548p2';
const postfix1 = '6p6e3r';
const postfix2 = '6p6e3r';
cy.visit(`/tx/${txid}`);
cy.waitForSkeletonGone();
cy.get('.alert-mempool').should('exist');
cy.get('.poison-alert').its('length').should('equal', 2);
cy.get('.prefix')
.should('have.length', 2)
.each(($el) => {
cy.wrap($el).should('have.text', prefix);
});
cy.get('.infix')
.should('have.length', 2)
.then(($infixes) => {
cy.wrap($infixes[0]).should('have.text', infix1);
cy.wrap($infixes[1]).should('have.text', infix2);
});
cy.get('.postfix')
.should('have.length', 2)
.then(($postfixes) => {
cy.wrap($postfixes[0]).should('include.text', postfix1);
cy.wrap($postfixes[1]).should('include.text', postfix2);
});
});
});
});
describe('blocks navigation', () => {
@ -397,6 +460,7 @@ describe('Mainnet', () => {
cy.get('#dropdownFees').should('be.visible');
cy.get('.btn-group').should('be.visible');
});
it('check buttons - tablet', () => {
cy.viewport('ipad-2');
cy.visit('/graphs');
@ -405,6 +469,7 @@ describe('Mainnet', () => {
cy.get('#dropdownFees').should('be.visible');
cy.get('.btn-group').should('be.visible');
});
it('check buttons - desktop', () => {
cy.viewport('macbook-16');
cy.visit('/graphs');
@ -415,26 +480,6 @@ describe('Mainnet', () => {
});
});
it('loads the tv screen - desktop', () => {
cy.viewport('macbook-16');
cy.visit('/graphs/mempool');
cy.waitForSkeletonGone();
cy.get('#btn-tv').click().then(() => {
cy.viewport('macbook-16');
cy.get('.chart-holder');
cy.get('.blockchain-wrapper').should('be.visible');
cy.get('#mempool-block-0').should('be.visible');
});
});
it('loads the tv screen - mobile', () => {
cy.viewport('iphone-6');
cy.visit('/tv');
cy.waitForSkeletonGone();
cy.get('.chart-holder');
cy.get('.blockchain-wrapper').should('not.visible');
});
it('loads the api screen', () => {
cy.visit('/');
cy.waitForSkeletonGone();
@ -516,7 +561,44 @@ describe('Mainnet', () => {
});
describe('RBF transactions', () => {
it('shows RBF transactions properly (mobile)', () => {
it('RBF page gets updated over websockets', () => {
cy.intercept('/api/v1/replacements', {
statusCode: 200,
body: []
});
cy.intercept('/api/v1/fullrbf/replacements', {
statusCode: 200,
body: []
});
cy.mockMempoolSocketV2();
cy.visit('/rbf');
cy.get('.no-replacements');
cy.get('.tree').should('have.length', 0);
receiveWebSocketMessageFromServer({
params: {
file: {
path: 'rbf_page/rbf_01.json'
}
}
});
cy.get('.tree').should('have.length', 1);
receiveWebSocketMessageFromServer({
params: {
file: {
path: 'rbf_page/rbf_02.json'
}
}
});
cy.get('.tree').should('have.length', 2);
});
it('shows RBF transactions properly (mobile - details)', () => {
cy.intercept('/api/v1/tx/21518a98d1aa9df524865d2f88c578499f524eb1d0c4d3e70312ab863508692f/cached', {
fixture: 'mainnet_tx_cached.json'
}).as('cached_tx');
@ -527,7 +609,7 @@ describe('Mainnet', () => {
cy.viewport('iphone-xr');
cy.mockMempoolSocket();
cy.visit('/tx/21518a98d1aa9df524865d2f88c578499f524eb1d0c4d3e70312ab863508692f');
cy.visit('/tx/21518a98d1aa9df524865d2f88c578499f524eb1d0c4d3e70312ab863508692f?mode=details');
cy.waitForSkeletonGone();
@ -545,7 +627,120 @@ describe('Mainnet', () => {
}
});
cy.get('.alert-mempool').should('be.visible');
});
it('shows RBF transactions properly (mobile - tracker)', () => {
cy.mockMempoolSocketV2();
cy.viewport('iphone-xr');
// API Mocks
cy.intercept('/api/v1/mining/pools/1w', {
fixture: 'details_rbf/api_mining_pools_1w.json'
}).as('api_mining_1w');
cy.intercept('/api/tx/242f3fff9ca7d5aea7a7a57d886f3fa7329e24fac948598a991b3a3dd631cd29', {
statusCode: 404,
body: 'Transaction not found'
}).as('api_tx01_404');
cy.intercept('/api/v1/tx/242f3fff9ca7d5aea7a7a57d886f3fa7329e24fac948598a991b3a3dd631cd29/cached', {
fixture: 'details_rbf/tx01_api_cached.json'
}).as('api_tx01_cached');
cy.intercept('/api/v1/tx/242f3fff9ca7d5aea7a7a57d886f3fa7329e24fac948598a991b3a3dd631cd29/rbf', {
fixture: 'details_rbf/tx01_api_rbf.json'
}).as('api_tx01_rbf');
cy.visit('/tx/242f3fff9ca7d5aea7a7a57d886f3fa7329e24fac948598a991b3a3dd631cd29?mode=tracker');
cy.wait('@api_tx01_rbf');
// Start sending mocked WS messages
receiveWebSocketMessageFromServer({
params: {
file: {
path: 'details_rbf/tx01_ws_stratum_jobs.json'
}
}
});
receiveWebSocketMessageFromServer({
params: {
file: {
path: 'details_rbf/tx01_ws_blocks_01.json'
}
}
});
receiveWebSocketMessageFromServer({
params: {
file: {
path: 'details_rbf/tx01_ws_tx_replaced.json'
}
}
});
receiveWebSocketMessageFromServer({
params: {
file: {
path: 'details_rbf/tx01_ws_mempool_blocks_01.json'
}
}
});
cy.get('.alert-replaced').should('be.visible');
cy.get('.explainer').should('be.visible');
cy.get('svg[data-icon=timeline]').should('be.visible');
// Second TX setup
cy.intercept('/api/tx/b6a53d8a8025c0c5b194345925a99741b645cae0b1f180f0613273029f2bf698', {
fixture: 'details_rbf/tx02_api_tx.json'
}).as('tx02_api');
cy.intercept('/api/v1/transaction-times?txId%5B%5D=b6a53d8a8025c0c5b194345925a99741b645cae0b1f180f0613273029f2bf698', {
fixture: 'details_rbf/tx02_api_tx_times.json'
}).as('tx02_api_tx_times');
cy.intercept('/api/v1/tx/b6a53d8a8025c0c5b194345925a99741b645cae0b1f180f0613273029f2bf698/rbf', {
fixture: 'details_rbf/tx02_api_rbf.json'
}).as('tx02_api_rbf');
cy.intercept('/api/v1/cpfp/b6a53d8a8025c0c5b194345925a99741b645cae0b1f180f0613273029f2bf698', {
fixture: 'details_rbf/tx02_api_cpfp.json'
}).as('tx02_api_cpfp');
// Go to the replacement tx
cy.get('.alert-replaced a').click();
cy.wait('@tx02_api_cpfp');
receiveWebSocketMessageFromServer({
params: {
file: {
path: 'details_rbf/tx02_ws_tx_position.json'
}
}
});
receiveWebSocketMessageFromServer({
params: {
file: {
path: 'details_rbf/tx02_ws_mempool_blocks_01.json'
}
}
});
cy.get('svg[data-icon=hourglass-half]').should('be.visible');
receiveWebSocketMessageFromServer({
params: {
file: {
path: 'details_rbf/tx02_ws_block.json'
}
}
});
cy.get('app-confirmations');
cy.get('svg[data-icon=circle-check]').should('be.visible');
});
it('shows RBF transactions properly (desktop)', () => {

24
frontend/cypress/e2e/signet/signet.spec.ts
View file

@ -60,30 +60,6 @@ describe('Signet', () => {
});
});
describe.skip('tv mode', () => {
it('loads the tv screen - desktop', () => {
cy.viewport('macbook-16');
cy.visit('/signet/graphs');
cy.waitForSkeletonGone();
cy.get('#btn-tv').click().then(() => {
cy.get('.chart-holder').should('be.visible');
cy.get('#mempool-block-0').should('be.visible');
cy.get('.tv-only').should('not.exist');
});
});
it('loads the tv screen - mobile', () => {
cy.visit('/signet/graphs');
cy.waitForSkeletonGone();
cy.get('#btn-tv').click().then(() => {
cy.viewport('iphone-8');
cy.get('.chart-holder').should('be.visible');
cy.get('.tv-only').should('not.exist');
cy.get('#mempool-block-0').should('be.visible');
});
});
});
it('loads the api screen', () => {
cy.visit('/signet');
cy.waitForSkeletonGone();

24
frontend/cypress/e2e/testnet4/testnet4.spec.ts
View file

@ -60,30 +60,6 @@ describe('Testnet4', () => {
});
});
describe('tv mode', () => {
it('loads the tv screen - desktop', () => {
cy.viewport('macbook-16');
cy.visit('/testnet4/graphs');
cy.waitForSkeletonGone();
cy.get('#btn-tv').click().then(() => {
cy.wait(1000);
cy.get('.tv-only').should('not.exist');
cy.get('#mempool-block-0').should('be.visible');
});
});
it('loads the tv screen - mobile', () => {
cy.visit('/testnet4/graphs');
cy.waitForSkeletonGone();
cy.get('#btn-tv').click().then(() => {
cy.viewport('iphone-6');
cy.wait(1000);
cy.get('.tv-only').should('not.exist');
});
});
});
it('loads the api screen', () => {
cy.visit('/testnet4');
cy.waitForSkeletonGone();

2
frontend/cypress/fixtures/assets.minimal.json
View file

@ -13,7 +13,7 @@
],
"6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d": [
null,
"L-BTC",
"LBTC",
"Liquid Bitcoin",
8
],

60
frontend/cypress/fixtures/details_rbf/api_accelerator_estimate.json Normal file
View file

@ -0,0 +1,60 @@
{
"txSummary": {
"txid": "b6a53d8a8025c0c5b194345925a99741b645cae0b1f180f0613273029f2bf698",
"effectiveVsize": 224,
"effectiveFee": 960,
"ancestorCount": 1
},
"cost": 1000,
"targetFeeRate": 3,
"nextBlockFee": 672,
"userBalance": 0,
"mempoolBaseFee": 50000,
"vsizeFee": 0,
"pools": [
36,
102,
112,
44,
4,
2,
6,
94,
143,
43,
105,
115,
142,
111
],
"options": [
{
"fee": 1000
},
{
"fee": 2000
},
{
"fee": 10000
}
],
"hasAccess": false,
"availablePaymentMethods": {
"bitcoin": {
"enabled": true,
"min": 1000,
"max": 10000000
},
"applePay": {
"enabled": true,
"min": 10,
"max": 1000
},
"googlePay": {
"enabled": true,
"min": 10,
"max": 1000
}
},
"unavailable": false
}

3
frontend/cypress/fixtures/details_rbf/api_accelerator_version.json Normal file
View file

@ -0,0 +1,3 @@
{
"gitCommit": "62f80296"
}

1
frontend/cypress/fixtures/details_rbf/api_block_history.json Normal file
View file

@ -0,0 +1 @@
[]

260
frontend/cypress/fixtures/details_rbf/api_mining_pools_1w.json Normal file
View file

@ -0,0 +1,260 @@
{
"pools": [
{
"poolId": 112,
"name": "Foundry USA",
"link": "https://foundrydigital.com",
"blockCount": 323,
"rank": 1,
"emptyBlocks": 0,
"slug": "foundryusa",
"avgMatchRate": 99.96,
"avgFeeDelta": "-0.01971455",
"poolUniqueId": 111
},
{
"poolId": 45,
"name": "AntPool",
"link": "https://www.antpool.com",
"blockCount": 171,
"rank": 2,
"emptyBlocks": 0,
"slug": "antpool",
"avgMatchRate": 99.99,
"avgFeeDelta": "-0.04227368",
"poolUniqueId": 44
},
{
"poolId": 74,
"name": "ViaBTC",
"link": "https://viabtc.com",
"blockCount": 166,
"rank": 3,
"emptyBlocks": 0,
"slug": "viabtc",
"avgMatchRate": 99.99,
"avgFeeDelta": "-0.02530964",
"poolUniqueId": 73
},
{
"poolId": 37,
"name": "F2Pool",
"link": "https://www.f2pool.com",
"blockCount": 104,
"rank": 4,
"emptyBlocks": 0,
"slug": "f2pool",
"avgMatchRate": 99.99,
"avgFeeDelta": "-0.03299327",
"poolUniqueId": 36
},
{
"poolId": 116,
"name": "MARA Pool",
"link": "https://marapool.com",
"blockCount": 66,
"rank": 5,
"emptyBlocks": 0,
"slug": "marapool",
"avgMatchRate": 99.97,
"avgFeeDelta": "0.02366061",
"poolUniqueId": 115
},
{
"poolId": 103,
"name": "SpiderPool",
"link": "https://www.spiderpool.com",
"blockCount": 46,
"rank": 6,
"emptyBlocks": 1,
"slug": "spiderpool",
"avgMatchRate": 97.82,
"avgFeeDelta": "-0.07258913",
"poolUniqueId": 102
},
{
"poolId": 142,
"name": "SECPOOL",
"link": "https://www.secpool.com",
"blockCount": 30,
"rank": 7,
"emptyBlocks": 1,
"slug": "secpool",
"avgMatchRate": 96.67,
"avgFeeDelta": "-0.06596000",
"poolUniqueId": 141
},
{
"poolId": 106,
"name": "Binance Pool",
"link": "https://pool.binance.com",
"blockCount": 28,
"rank": 8,
"emptyBlocks": 0,
"slug": "binancepool",
"avgMatchRate": 99.99,
"avgFeeDelta": "-0.05834286",
"poolUniqueId": 105
},
{
"poolId": 5,
"name": "Luxor",
"link": "https://mining.luxor.tech",
"blockCount": 28,
"rank": 9,
"emptyBlocks": 0,
"slug": "luxor",
"avgMatchRate": 100,
"avgFeeDelta": "-0.05496071",
"poolUniqueId": 4
},
{
"poolId": 143,
"name": "OCEAN",
"link": "https://ocean.xyz/",
"blockCount": 12,
"rank": 10,
"emptyBlocks": 0,
"slug": "ocean",
"avgMatchRate": 91.9,
"avgFeeDelta": "-0.14650833",
"poolUniqueId": 142
},
{
"poolId": 44,
"name": "Braiins Pool",
"link": "https://braiins.com/pool",
"blockCount": 12,
"rank": 11,
"emptyBlocks": 0,
"slug": "braiinspool",
"avgMatchRate": 100,
"avgFeeDelta": "-0.03553333",
"poolUniqueId": 43
},
{
"poolId": 113,
"name": "SBI Crypto",
"link": "https://sbicrypto.com",
"blockCount": 8,
"rank": 12,
"emptyBlocks": 0,
"slug": "sbicrypto",
"avgMatchRate": 98.65,
"avgFeeDelta": "-0.04246250",
"poolUniqueId": 112
},
{
"poolId": 152,
"name": "Carbon Negative",
"link": "https://github.com/bitcoin-data/mining-pools/issues/48",
"blockCount": 7,
"rank": 13,
"emptyBlocks": 0,
"slug": "carbonnegative",
"avgMatchRate": 99.75,
"avgFeeDelta": "-0.04407143",
"poolUniqueId": 151
},
{
"poolId": 7,
"name": "BTC.com",
"link": "https://pool.btc.com",
"blockCount": 5,
"rank": 14,
"emptyBlocks": 0,
"slug": "btccom",
"avgMatchRate": 99.98,
"avgFeeDelta": "-0.02496000",
"poolUniqueId": 6
},
{
"poolId": 162,
"name": "Mining Squared",
"link": "https://pool.bsquared.network/",
"blockCount": 4,
"rank": 15,
"emptyBlocks": 0,
"slug": "miningsquared",
"avgMatchRate": 100,
"avgFeeDelta": "-0.00915000",
"poolUniqueId": 161
},
{
"poolId": 95,
"name": "Poolin",
"link": "https://www.poolin.com",
"blockCount": 4,
"rank": 16,
"emptyBlocks": 0,
"slug": "poolin",
"avgMatchRate": 100,
"avgFeeDelta": "-0.26485000",
"poolUniqueId": 94
},
{
"poolId": 1,
"name": "Unknown",
"link": "https://learnmeabitcoin.com/technical/coinbase-transaction",
"blockCount": 4,
"rank": 17,
"emptyBlocks": 0,
"slug": "unknown",
"avgMatchRate": 100,
"avgFeeDelta": "-0.06490000",
"poolUniqueId": 0
},
{
"poolId": 144,
"name": "WhitePool",
"link": "https://whitebit.com/mining-pool",
"blockCount": 3,
"rank": 18,
"emptyBlocks": 0,
"slug": "whitepool",
"avgMatchRate": 100,
"avgFeeDelta": "-0.01293333",
"poolUniqueId": 143
},
{
"poolId": 3,
"name": "ULTIMUSPOOL",
"link": "https://www.ultimuspool.com",
"blockCount": 1,
"rank": 19,
"emptyBlocks": 0,
"slug": "ultimuspool",
"avgMatchRate": 100,
"avgFeeDelta": "-0.16130000",
"poolUniqueId": 2
},
{
"poolId": 50,
"name": "Solo CK",
"link": "https://solo.ckpool.org",
"blockCount": 1,
"rank": 20,
"emptyBlocks": 0,
"slug": "solock",
"avgMatchRate": 100,
"avgFeeDelta": "-0.01510000",
"poolUniqueId": 49
},
{
"poolId": 158,
"name": "BitFuFuPool",
"link": "https://www.bitfufu.com/pool",
"blockCount": 1,
"rank": 21,
"emptyBlocks": 0,
"slug": "bitfufupool",
"avgMatchRate": 100,
"avgFeeDelta": "-0.01630000",
"poolUniqueId": 157
}
],
"blockCount": 1024,
"lastEstimatedHashrate": 786391245138648900000,
"lastEstimatedHashrate3d": 797683179385121300000,
"lastEstimatedHashrate1w": 827836055441520300000
}

Some files were not shown because too many files have changed in this diff Show more