Merge pull request #6693 from mempool/jramos/x-viewing

fix: viewing link in x app

.github/dependabot.yml

@@ -6,6 +6,11 @@ updates:
     schedule:
       interval: daily
     open-pull-requests-limit: 10
+    cooldown:
+      default-days: 7
+      semver-major-days: 30
+      semver-minor-days: 14
+      semver-patch-days: 7
     ignore:
       - dependency-name: "*"
         update-types:
@@ -33,6 +38,11 @@ updates:
     schedule:
       interval: daily
     open-pull-requests-limit: 10
+    cooldown:
+      default-days: 7
+      semver-major-days: 30
+      semver-minor-days: 14
+      semver-patch-days: 7
     ignore:
       - dependency-name: "*"
         update-types:
@@ -44,6 +54,8 @@ updates:
     directory: "/docker/backend"
     schedule:
       interval: weekly
+    cooldown:
+      default-days: 7
     ignore:
       - dependency-name: "*"
         update-types:
@@ -53,6 +65,8 @@ updates:
     directory: "/docker/frontend"
     schedule:
       interval: weekly
+    cooldown:
+      default-days: 7
     ignore:
       - dependency-name: "*"
         update-types:
@@ -62,6 +76,8 @@ updates:
     directory: "/"
     schedule:
       interval: weekly
+    cooldown:
+      default-days: 7
     ignore:
       - dependency-name: "*"
         update-types:

.github/nginx-check/Dockerfile.docker

@@ -0,0 +1,17 @@
+FROM nginx:1.30.1-alpine
+
+# Validate the config set loaded by the mempool/frontend docker image.
+# Build context: repo root
+COPY nginx.conf http-basic.conf /etc/nginx/
+COPY nginx-mempool.conf /etc/nginx/conf.d/
+
+# Replay the transforms applied by docker/init.sh (with the default values
+# docker/frontend/entrypoint.sh substitutes at container start)
+RUN sed -i \
+      -e "s!127.0.0.1:80!0.0.0.0:8080!g" \
+      -e "s!127.0.0.1!0.0.0.0!g" \
+      -e "s!user nobody;!!g" \
+      -e "s!/etc/nginx/nginx-mempool.conf!/etc/nginx/conf.d/nginx-mempool.conf!g" \
+      /etc/nginx/nginx.conf
+
+CMD ["nginx", "-t"]

.github/nginx-check/Dockerfile.production

@@ -0,0 +1,13 @@
+FROM nginx:1.30.1-alpine
+
+# Stage the config the way production/install does: the repo is available at
+# ${NGINX_ETC_FOLDER}/mempool and production/nginx/nginx.conf becomes the main
+# config, with __NGINX_USER__ and __NGINX_ETC_FOLDER__ substituted.
+# Build context: production/
+COPY nginx /etc/nginx/mempool/production/nginx
+
+RUN cp /etc/nginx/mempool/production/nginx/nginx.conf /etc/nginx/nginx.conf && \
+    sed -i "s!__NGINX_USER__!nginx!" /etc/nginx/nginx.conf && \
+    sed -i "s!__NGINX_ETC_FOLDER__!/etc/nginx!" /etc/nginx/nginx.conf
+
+CMD ["nginx", "-t"]

.github/workflows/backend-integration.yml

@@ -2,30 +2,33 @@ name: Backend Integration Tests with MariaDB
 
 on:
   pull_request:
-    types: [opened, review_requested, synchronize]
+    types: [opened, synchronize]
   push:
     branches:
       - master
 
+permissions:
+  contents: read
+
 jobs:
   backend-integration:
     if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
     strategy:
       matrix:
-        node: ["22.14.0"]
+        node: ["24.13.0"]
       fail-fast: false
-    runs-on: ubuntu-latest
+    runs-on: mempool-ci
 
     name: Backend Integration Tests - node ${{ matrix.node }}
     
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: ${{ matrix.node }}/integration
 
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@3235b876344d2a9aa001b8d1453c930bba69e610 # v3
         with:
           node-version: ${{ matrix.node }}
           registry-url: "https://registry.npmjs.org"
@@ -33,7 +36,7 @@ jobs:
           cache-dependency-path: '${{ matrix.node }}/integration/backend/package-lock.json'
 
       - name: Cache node modules
-        uses: actions/cache@v4
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: ${{ matrix.node }}/integration/backend/node_modules
           key: ${{ runner.os }}-backend-integration-node-${{ matrix.node }}-${{ hashFiles('${{ matrix.node }}/integration/backend/package-lock.json') }}
@@ -47,7 +50,7 @@ jobs:
         working-directory: ${{ matrix.node }}/integration
 
       - name: Cache Rust dependencies
-        uses: actions/cache@v4
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             ~/.cargo/bin/
@@ -61,12 +64,12 @@ jobs:
             ${{ runner.os }}-cargo-
 
       - name: Install ${{ steps.gettoolchain.outputs.toolchain }} Rust toolchain
-        uses: dtolnay/rust-toolchain@6d653acede28d24f02e3cd41383119e8b1b35921
+        uses: dtolnay/rust-toolchain@f7ccc83f9ed1e5b9c81d8a67d7ad1a747e22a561
         with:
           toolchain: ${{ steps.gettoolchain.outputs.toolchain }}
 
       - name: Install dependencies
-        run: npm ci
+        run: bash meta/scripts/check-install-scripts.sh && npm ci
         working-directory: ${{ matrix.node }}/integration/backend
 
       - name: Build backend

.github/workflows/ci.yml

@@ -2,38 +2,44 @@ name: CI Pipeline for the Backend and Frontend
 
 on:
   pull_request:
-    types: [opened, review_requested, synchronize]
+    types: [opened, synchronize]
   push:
     branches:
       - master
 
+permissions:
+  contents: read
+
 jobs:
   backend:
     if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
     strategy:
       matrix:
-        node: ["22.14.0"]
+        node: ["24.13.0"]
         flavor: ["dev", "prod"]
       fail-fast: false
-    runs-on: ubuntu-latest
+    runs-on: mempool-ci
 
     name: Backend (${{ matrix.flavor }}) - node ${{ matrix.node }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: ${{ matrix.node }}/${{ matrix.flavor }}
 
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
         with:
           node-version: ${{ matrix.node }}
           registry-url: "https://registry.npmjs.org"
           cache: 'npm'
           cache-dependency-path: '${{ matrix.node }}/${{ matrix.flavor }}/backend/package-lock.json'
 
+      - name: Install npm 11.12.0
+        run: npm install -g npm@11.12.0
+
       - name: Cache node modules
-        uses: actions/cache@v4
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: ${{ matrix.node }}/${{ matrix.flavor }}/backend/node_modules
           key: ${{ runner.os }}-backend-${{ matrix.flavor }}-node-${{ matrix.node }}-${{ hashFiles('${{ matrix.node }}/${{ matrix.flavor }}/backend/package-lock.json') }}
@@ -47,7 +53,7 @@ jobs:
         working-directory: ${{ matrix.node }}/${{ matrix.flavor }}
 
       - name: Cache Rust dependencies
-        uses: actions/cache@v4
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             ~/.cargo/bin/
@@ -63,18 +69,18 @@ jobs:
       - name: Install ${{ steps.gettoolchain.outputs.toolchain }} Rust toolchain
         # Latest version available on this commit is 1.71.1
         # Commit date is Aug 3, 2023
-        uses: dtolnay/rust-toolchain@6d653acede28d24f02e3cd41383119e8b1b35921
+        uses: dtolnay/rust-toolchain@f7ccc83f9ed1e5b9c81d8a67d7ad1a747e22a561
         with:
           toolchain: ${{ steps.gettoolchain.outputs.toolchain }}
 
       - name: Install
         if: ${{ matrix.flavor == 'dev'}}
-        run: npm ci
+        run: bash meta/scripts/check-install-scripts.sh && npm ci
         working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
 
       - name: Install (Prod dependencies only)
         if: ${{ matrix.flavor == 'prod'}}
-        run: npm ci --omit=dev --omit=optional
+        run: bash meta/scripts/check-install-scripts.sh && npm ci --omit=dev --omit=optional
         working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
 
       - name: Lint
@@ -96,24 +102,27 @@ jobs:
     name: "Cache assets for builds"
     strategy:
       matrix:
-        node: ["22.14.0"]
-    runs-on: ubuntu-latest
+        node: ["24.13.0"]
+    runs-on: mempool-ci
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: assets
 
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
         with:
           node-version: ${{ matrix.node }}
           registry-url: "https://registry.npmjs.org"
           cache: 'npm'
           cache-dependency-path: 'assets/frontend/package-lock.json'
 
+      - name: Install npm 11.12.0
+        run: npm install -g npm@11.12.0
+
       - name: Cache node modules for frontend
-        uses: actions/cache@v4
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: assets/frontend/node_modules
           key: ${{ runner.os }}-cache-frontend-node-${{ matrix.node }}-${{ hashFiles('assets/frontend/package-lock.json') }}
@@ -122,13 +131,13 @@ jobs:
             ${{ runner.os }}-cache-frontend-
 
       - name: Install (Prod dependencies only)
-        run: npm ci --omit=dev --omit=optional
+        run: bash meta/scripts/check-install-scripts.sh && npm ci --omit=dev --omit=optional
         working-directory: assets/frontend
 
       - name: Restore cached mining pool assets
         continue-on-error: true
         id: cache-mining-pool-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             mining-pool-assets.zip
@@ -137,7 +146,7 @@ jobs:
       - name: Restore promo video assets
         continue-on-error: true
         id: cache-promo-video-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             promo-video-assets.zip
@@ -170,20 +179,20 @@ jobs:
         run: zip -jrq promo-video-assets.zip assets/frontend/src/resources/promo-video/*
 
       - name: Upload mining pool assets as artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: mining-pool-assets
           path: mining-pool-assets.zip
 
       - name: Upload promo video assets as artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: promo-video-assets
           path: promo-video-assets.zip
 
       - name: Save mining pool assets cache
         id: cache-mining-pool-save
-        uses: actions/cache/save@v4
+        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             mining-pool-assets.zip
@@ -191,7 +200,7 @@ jobs:
 
       - name: Save promo video assets cache
         id: cache-promo-video-save
-        uses: actions/cache/save@v4
+        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             promo-video-assets.zip
@@ -202,28 +211,31 @@ jobs:
     if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
     strategy:
       matrix:
-        node: ["22.14.0"]
+        node: ["24.13.0"]
         flavor: ["dev", "prod"]
       fail-fast: false
-    runs-on: ubuntu-latest
+    runs-on: mempool-ci
 
     name: Frontend (${{ matrix.flavor }}) - node ${{ matrix.node }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: ${{ matrix.node }}/${{ matrix.flavor }}
 
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
         with:
           node-version: ${{ matrix.node }}
           registry-url: "https://registry.npmjs.org"
           cache: 'npm'
           cache-dependency-path: '${{ matrix.node }}/${{ matrix.flavor }}/frontend/package-lock.json'
 
+      - name: Install npm 11.12.0
+        run: npm install -g npm@11.12.0
+
       - name: Cache node modules
-        uses: actions/cache@v4
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: ${{ matrix.node }}/${{ matrix.flavor }}/frontend/node_modules
           key: ${{ runner.os }}-frontend-${{ matrix.flavor }}-node-${{ matrix.node }}-${{ hashFiles('${{ matrix.node }}/${{ matrix.flavor }}/frontend/package-lock.json') }}
@@ -232,13 +244,13 @@ jobs:
             ${{ runner.os }}-frontend-${{ matrix.flavor }}-
 
       - name: Install (Prod dependencies only)
-        run: npm ci --omit=dev --omit=optional
+        run: bash meta/scripts/check-install-scripts.sh && npm ci --omit=dev --omit=optional
         if: ${{ matrix.flavor == 'prod'}}
         working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
 
       - name: Install
         if: ${{ matrix.flavor == 'dev'}}
-        run: npm ci
+        run: bash meta/scripts/check-install-scripts.sh && npm ci
         working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
 
       - name: Lint
@@ -252,7 +264,7 @@ jobs:
       - name: Restore cached mining pool assets
         continue-on-error: true
         id: cache-mining-pool-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             mining-pool-assets.zip
@@ -261,14 +273,14 @@ jobs:
       - name: Restore promo video assets
         continue-on-error: true
         id: cache-promo-video-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             promo-video-assets.zip
           key: promo-video-assets-cache
 
       - name: Download artifact
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
           name: mining-pool-assets
 
@@ -276,7 +288,7 @@ jobs:
         run: unzip -o mining-pool-assets.zip -d ${{ matrix.node }}/${{ matrix.flavor }}/frontend/src/resources/mining-pools
 
       - name: Download artifact
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
           name: promo-video-assets
 
@@ -299,40 +311,44 @@ jobs:
   
   e2e:
     if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
-    runs-on: ubuntu-latest
+    runs-on: mempool-ci
     needs: frontend
     strategy:
       fail-fast: false
       matrix:
+        node: ["24.13.0"]
         module: ["mempool", "liquid", "testnet4"]
 
     name: E2E tests for ${{ matrix.module }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: ${{ matrix.module }}
 
       - name: Setup node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
         with:
-          node-version: 22
+          node-version: ${{ matrix.node }}
           cache: "npm"
           cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json
 
+      - name: Install npm 11.12.0
+        run: npm install -g npm@11.12.0
+
       - name: Cache node modules for e2e
-        uses: actions/cache@v4
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: ${{ matrix.module }}/frontend/node_modules
-          key: ${{ runner.os }}-e2e-${{ matrix.module }}-node-22-${{ hashFiles('${{ matrix.module }}/frontend/package-lock.json') }}
+          key: ${{ runner.os }}-e2e-${{ matrix.module }}-node-${{ matrix.node }}-${{ hashFiles('${{ matrix.module }}/frontend/package-lock.json') }}
           restore-keys: |
-            ${{ runner.os }}-e2e-${{ matrix.module }}-node-22-
+            ${{ runner.os }}-e2e-${{ matrix.module }}-node-${{ matrix.node }}-
             ${{ runner.os }}-e2e-${{ matrix.module }}-
 
       - name: Restore cached mining pool assets
         continue-on-error: true
         id: cache-mining-pool-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             mining-pool-assets.zip
@@ -341,14 +357,14 @@ jobs:
       - name: Restore cached promo video assets
         continue-on-error: true
         id: cache-promo-video-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             promo-video-assets.zip
           key: promo-video-assets-cache
 
       - name: Download artifact
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
           name: mining-pool-assets
 
@@ -356,7 +372,7 @@ jobs:
         run: unzip -o mining-pool-assets.zip -d ${{ matrix.module }}/frontend/src/resources/mining-pools
 
       - name: Download artifact
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
           name: promo-video-assets
 
@@ -366,7 +382,7 @@ jobs:
       # mempool
       - name: Chrome browser tests (${{ matrix.module }})
         if: ${{ matrix.module == 'mempool' }}
-        uses: cypress-io/github-action@v5
+        uses: cypress-io/github-action@248bde77443c376edc45906ede03a1aba9da0462 # v5
         with:
           tag: ${{ github.event_name }}
           working-directory: ${{ matrix.module }}/frontend
@@ -391,7 +407,7 @@ jobs:
       # liquid
       - name: Chrome browser tests (${{ matrix.module }})
         if: ${{ matrix.module == 'liquid' }}
-        uses: cypress-io/github-action@v5
+        uses: cypress-io/github-action@248bde77443c376edc45906ede03a1aba9da0462 # v5
         with:
           tag: ${{ github.event_name }}
           working-directory: ${{ matrix.module }}/frontend
@@ -416,7 +432,7 @@ jobs:
       # testnet
       - name: Chrome browser tests (${{ matrix.module }})
         if: ${{ matrix.module == 'testnet4' }}
-        uses: cypress-io/github-action@v5
+        uses: cypress-io/github-action@248bde77443c376edc45906ede03a1aba9da0462 # v5
         with:
           tag: ${{ github.event_name }}
           working-directory: ${{ matrix.module }}/frontend
@@ -439,12 +455,12 @@ jobs:
           CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
   validate_docker_json:
     if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
-    runs-on: ubuntu-latest
+    runs-on: mempool-ci
     name: Validate generated backend Docker JSON
 
     steps: 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: docker
       

.github/workflows/dependabot-provenance-check.yml

@@ -0,0 +1,102 @@
+name: Dependabot Provenance Check
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions:
+  contents: read
+  pull-requests: write
+  issues: write
+
+jobs:
+  check-provenance:
+    if: >-
+      github.event.pull_request.user.login == 'dependabot[bot]' &&
+      !contains(github.event.pull_request.labels.*.name, 'provenance-exception')
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - name: Fetch Dependabot metadata
+        id: metadata
+        uses: dependabot/fetch-metadata@21025c705c08248db411dc16f3619e6b5f9ea21a # v2
+        with:
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+
+      - name: Setup Node
+        if: steps.metadata.outputs.package-ecosystem == 'npm_and_yarn'
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: "24.13.0"
+
+      - name: Check npm provenance attestation
+        if: steps.metadata.outputs.package-ecosystem == 'npm_and_yarn'
+        id: npm-provenance
+        run: |
+          IFS=',' read -ra DEPS <<< "${{ steps.metadata.outputs.dependency-names }}"
+          UPDATED_DEPS_JSON='${{ steps.metadata.outputs.updated-dependencies-json }}'
+
+          FAILED_DEPS=""
+          for DEP in "${DEPS[@]}"; do
+            DEP=$(echo "$DEP" | xargs)
+
+            NEW_VERSION=$(echo "$UPDATED_DEPS_JSON" | jq -r --arg name "$DEP" '
+              .[] | select(.dependencyName == $name) | .newVersion
+            ' | head -n 1)
+
+            if [ -z "$NEW_VERSION" ] || [ "$NEW_VERSION" = "null" ]; then
+              echo "::error::Could not determine updated version for dependency $DEP from Dependabot metadata"
+              FAILED_DEPS="${FAILED_DEPS:+$FAILED_DEPS, }$DEP"
+              continue
+            fi
+            echo "::group::Checking provenance for $DEP@$NEW_VERSION"
+
+            ATTESTATIONS=$(npm view "$DEP@$NEW_VERSION" --json 2>/dev/null | jq -r '.dist.attestations // empty')
+
+            if [ -z "$ATTESTATIONS" ]; then
+              echo "::error::No provenance attestation found for $DEP@$NEW_VERSION"
+              FAILED_DEPS="${FAILED_DEPS:+$FAILED_DEPS, }$DEP"
+            else
+              echo "Provenance attestation found for $DEP@$NEW_VERSION"
+            fi
+            echo "::endgroup::"
+          done
+
+          if [ -n "$FAILED_DEPS" ]; then
+            echo "failed=true" >> "$GITHUB_OUTPUT"
+            echo "failed-deps=$FAILED_DEPS" >> "$GITHUB_OUTPUT"
+          else
+            echo "failed=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Close PR if provenance check failed
+        if: steps.npm-provenance.outputs.failed == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          FAILED_DEPS: ${{ steps.npm-provenance.outputs.failed-deps }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          COMMENT_BODY_FILE="$(mktemp)"
+          cat > "$COMMENT_BODY_FILE" <<EOF
+          ## Provenance Attestation Check Failed
+          
+          This pull request has been automatically closed because the following dependencies do not have [provenance attestation](https://docs.npmjs.com/generating-provenance-statements):
+          
+          **Failed:** $FAILED_DEPS
+          
+          ### What is provenance attestation?
+          
+          Provenance attestation cryptographically proves that a package was built from a specific source repository using a specific build process. This helps protect against supply chain attacks by verifying the link between published packages and their source code.
+          
+          ### What to do
+          
+          - Check if a newer version of the dependency publishes provenance
+          - Contact the package maintainer to request [npm provenance](https://docs.npmjs.com/generating-provenance-statements) support
+          - If this dependency is trusted and an exception is warranted, a maintainer can reopen this PR and add the \`provenance-exception\` label
+          EOF
+          
+          gh pr comment "$PR_NUMBER" --body-file "$COMMENT_BODY_FILE"
+          gh pr close "$PR_NUMBER"

.github/workflows/docker.yml

@@ -17,22 +17,228 @@ permissions:
   contents: read
 
 jobs:
+  test-images:
+    # Always run on tag pushes and all pull requests
+    runs-on: mempool-ci
+    timeout-minutes: 30
+    name: Test built Docker images
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - name: Add SHORT_SHA env property with commit short sha
+        run: |
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            SHA="${{ github.event.pull_request.head.sha }}"
+          else
+            SHA="${GITHUB_SHA}"
+          fi
+          echo "SHORT_SHA=${SHA:0:8}" >> $GITHUB_ENV
+
+      - name: Set TAG from pushed tag or package.json
+        run: |
+          if [ "${{ github.event_name }}" = "push" ]; then
+            TAG="${GITHUB_REF/refs\/tags\//}"
+          else
+            FRONTEND_VERSION=$(jq -r '.version' frontend/package.json)
+            BACKEND_VERSION=$(jq -r '.version' backend/package.json)
+            if [ "$FRONTEND_VERSION" != "$BACKEND_VERSION" ]; then
+              echo "Error: Frontend version ($FRONTEND_VERSION) and backend version ($BACKEND_VERSION) do not match"
+              exit 1
+            fi
+            TAG="v${FRONTEND_VERSION}-${SHORT_SHA}"
+          fi
+          echo "TAG=${TAG}" >> $GITHUB_ENV
+
+      - name: Show set environment variables
+        run: |
+          printf "      TAG: %s\n"  "$TAG"
+          printf " SHORT_SHA: %s\n" "$SHORT_SHA"
+
+      - name: Init repo for Dockerization
+        run: docker/init.sh "$TAG"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+
+      - name: Build frontend image locally
+        run: |
+          docker buildx build \
+            --tag test-frontend:$TAG \
+            --build-arg commitHash=$SHORT_SHA \
+            --load \
+            --platform linux/amd64 \
+            ./frontend/
+
+      - name: Build backend image locally
+        run: |
+          docker buildx build \
+            --tag test-backend:$TAG \
+            --build-context rustgbt=./rust \
+            --build-context backend=./backend \
+            --build-arg commitHash=$SHORT_SHA \
+            --load \
+            --platform linux/amd64 \
+            ./backend/
+
+      - name: Prepare docker-compose test file
+        run: |
+          cat > /tmp/modify_compose.py << 'SCRIPT_END'
+          import re
+          import os
+          import sys
+          
+          # Read the base docker-compose file
+          with open('docker/docker-compose.yml', 'r') as f:
+              content = f.read()
+          
+          # Get TAG from environment
+          tag = os.environ.get('TAG', '')
+          
+          # Replace image names with locally built test images
+          content = content.replace('image: mempool/frontend:latest', f'image: test-frontend:{tag}')
+          content = content.replace('image: mempool/backend:latest', f'image: test-backend:{tag}')
+          
+          # Change web port mapping from 80:8080 to 8080:8080
+          content = content.replace('- 80:8080', '- 8080:8080')
+          
+          # Remove volumes from api service
+          content = re.sub(r'    volumes:\n      - \.\/data:\/backend\/cache\n', '', content)
+          
+          # For db service: remove user and volumes, add tmpfs and healthcheck
+          # Remove user line from db service (only the one in db service)
+          lines = content.split('\n')
+          in_db_service = False
+          new_lines = []
+          for i, line in enumerate(lines):
+              if line.strip().startswith('db:'):
+                  in_db_service = True
+              elif line.strip() and not line.startswith(' ') and not line.startswith('\t'):
+                  in_db_service = False
+              if in_db_service and line.strip() == 'user: "1000:1000"':
+                  continue
+              new_lines.append(line)
+          content = '\n'.join(new_lines)
+          
+          # Remove volumes section from db service
+          content = re.sub(r'    volumes:\n      - \.\/mysql\/data:\/var\/lib\/mysql\n', '', content)
+          
+          # Add tmpfs after stop_grace_period in db service (healthcheck already exists in base file)
+          db_stop_grace = '    stop_grace_period: 1m'
+          db_additions = '    stop_grace_period: 1m\n    tmpfs:\n      - /var/lib/mysql'
+          content = content.replace(db_stop_grace, db_additions, 1)
+          
+          # Add depends_on to web service after ports
+          web_ports = '    ports:\n      - 8080:8080'
+          web_with_depends = '    ports:\n      - 8080:8080\n    depends_on:\n      - api\n      - db'
+          content = content.replace(web_ports, web_with_depends, 1)
+          
+          # Add depends_on to api service after command
+          api_command = '    command: "./wait-for-it.sh db:3306 --timeout=720 --strict -- ./start.sh"'
+          api_with_depends = '    command: "./wait-for-it.sh db:3306 --timeout=720 --strict -- ./start.sh"\n    depends_on:\n      - db'
+          content = content.replace(api_command, api_with_depends, 1)
+          
+          # Write the modified content
+          with open('docker-compose.test.yml', 'w') as f:
+              f.write(content)
+          
+          print("Generated docker-compose.test.yml")
+          SCRIPT_END
+          python3 /tmp/modify_compose.py
+          cat docker-compose.test.yml
+
+      - name: Start containers
+        run: |
+          docker compose -f docker-compose.test.yml up -d
+
+      - name: Wait for services to be ready
+        run: |
+          echo "Waiting for all services (web, api, db) to be healthy..."
+          timeout=120
+          elapsed=0
+          while [ $elapsed -lt $timeout ]; do
+            # Check health status for all services
+            PS_OUTPUT=$(docker compose -f docker-compose.test.yml ps)
+            HEALTHY_COUNT=$(echo "$PS_OUTPUT" | grep -c "(healthy)" || true)
+            if [ "$HEALTHY_COUNT" -ge 3 ]; then
+              echo "All services are healthy!"
+              echo "$PS_OUTPUT"
+              break
+            fi
+            echo "Waiting for services to be healthy... (${elapsed}s/${timeout}s)"
+            echo "$PS_OUTPUT"
+            sleep 2
+            elapsed=$((elapsed + 2))
+          done
+          if [ $elapsed -ge $timeout ]; then
+            echo "Services did not become healthy in time"
+            docker compose -f docker-compose.test.yml ps
+            docker compose -f docker-compose.test.yml logs
+            exit 1
+          fi
+
+      - name: Verify containers are healthy
+        run: |
+          echo "Checking container health status..."
+          PS_OUTPUT=$(docker compose -f docker-compose.test.yml ps)
+          echo "$PS_OUTPUT"
+          
+          # Check that all three services (web, api, db) are healthy
+          HEALTHY_COUNT=$(echo "$PS_OUTPUT" | grep -c "(healthy)" || true)
+          if [ "$HEALTHY_COUNT" -lt 3 ]; then
+            echo "Not all containers are healthy. Expected 3 healthy services, found $HEALTHY_COUNT"
+            docker compose -f docker-compose.test.yml logs
+            exit 1
+          fi
+          
+          # Verify each service individually for better error messages
+          if ! echo "$PS_OUTPUT" | grep -q "web.*(healthy)"; then
+            echo "Web service is not healthy"
+            docker compose -f docker-compose.test.yml logs web
+            exit 1
+          fi
+          if ! echo "$PS_OUTPUT" | grep -q "api.*(healthy)"; then
+            echo "API service is not healthy"
+            docker compose -f docker-compose.test.yml logs api
+            exit 1
+          fi
+          if ! echo "$PS_OUTPUT" | grep -q "db.*(healthy)"; then
+            echo "Database service is not healthy"
+            docker compose -f docker-compose.test.yml logs db
+            exit 1
+          fi
+          
+          echo "All containers are healthy!"
+
+      - name: Show container logs
+        if: failure()
+        run: |
+          docker compose -f docker-compose.test.yml logs
+
+      - name: Clean up containers
+        if: always()
+        run: |
+          docker compose -f docker-compose.test.yml down -v
+
   build:
-    # Run on tag pushes OR on PRs that have the "docker" label
+    needs: test-images
+    # Run on tag pushes OR on PRs with "docker-push" label (after test-images passes)
     if: |
-      github.event_name == 'push' ||
-      (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'docker'))
+      needs.test-images.result == 'success' &&
+      (github.event_name == 'push' ||
+       (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'docker-push')))
     strategy:
       matrix:
         service:
           - frontend
           - backend
-    runs-on: ubuntu-latest
+    runs-on: mempool-ci
     timeout-minutes: 120
     name: Build and push to DockerHub
     outputs:
       image-digest-frontend: ${{ matrix.service == 'frontend' && steps.docker-build.outputs.digest || '' }}
       image-digest-backend: ${{ matrix.service == 'backend' && steps.docker-build.outputs.digest || '' }}
+      tag: ${{ matrix.service == 'frontend' && (steps.set-tag-push.outputs.tag || steps.set-tag-pr.outputs.tag) || '' }}
     steps:
       - name: Replace the current swap file
         shell: bash
@@ -65,7 +271,11 @@ jobs:
       # Only for tag pushes: use the Git tag as TAG
       - name: Set TAG from pushed tag
         if: github.event_name == 'push'
-        run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
+        id: set-tag-push
+        run: |
+          TAG="${GITHUB_REF/refs\/tags\//}"
+          echo "TAG=${TAG}" >> $GITHUB_ENV
+          echo "tag=${TAG}" >> $GITHUB_OUTPUT
 
       - name: Add SHORT_SHA env property with commit short sha
         run: |
@@ -78,21 +288,27 @@ jobs:
 
 
       - name: Login to Docker for building
-        run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Checkout project
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
 
       # For PRs: use package.json version + short sha as TAG
       - name: Set TAG from service package.json for pull requests
         if: github.event_name == 'pull_request'
+        id: set-tag-pr
         run: |
           if [ "${{ matrix.service }}" = "frontend" ]; then
             VERSION=$(jq -r '.version' frontend/package.json)
           else
             VERSION=$(jq -r '.version' backend/package.json)
           fi
-          echo "TAG=v${VERSION}-${SHORT_SHA}" >> $GITHUB_ENV
+          TAG="v${VERSION}-${SHORT_SHA}"
+          echo "TAG=${TAG}" >> $GITHUB_ENV
+          echo "tag=${TAG}" >> $GITHUB_OUTPUT
 
       - name: Show set environment variables
         run: |
@@ -103,13 +319,13 @@ jobs:
         run: docker/init.sh "$TAG"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
         with:
           platforms: linux/amd64,linux/arm64
         id: qemu
 
       - name: Setup Docker buildx action
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
         with:
           platforms: linux/amd64,linux/arm64
           driver-opts: |
@@ -120,7 +336,7 @@ jobs:
         run: echo ${{ steps.buildx.outputs.platforms }}
 
       - name: Cache Docker layers
-        uses: actions/cache@v3
+        uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3
         id: cache
         with:
           path: /tmp/.buildx-cache
@@ -144,9 +360,9 @@ jobs:
 
   tag-latest:
     needs: build
-    # Only for successful *tag pushes* and only for "plain" versions (no '-')
+    # Only for successful tag pushes (not PRs with docker-push label) and only for "plain" versions (no '-')
     if: ${{ needs.build.result == 'success' && github.event_name == 'push' && !contains(github.ref_name, '-') }}
-    runs-on: ubuntu-latest
+    runs-on: mempool-ci
     timeout-minutes: 30
     name: Tag release build as latest
     strategy:
@@ -159,19 +375,22 @@ jobs:
         run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
         with:
           platforms: linux/amd64,linux/arm64
 
       - name: Setup Docker buildx action
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
         with:
           platforms: linux/amd64,linux/arm64
           driver-opts: |
             network=host
 
       - name: Login to Docker Hub
-        run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Tag as latest for ${{ matrix.service }}
         run: |

.github/workflows/e2e_parameterized.yml

@@ -17,12 +17,15 @@ on:
         description: 'Liquid Hostname'
         required: true
         default: 'liquid.network'
-        type: string 
+        type: string
+
+permissions:
+  contents: read
 
 jobs:
   cache:
     name: "Cache assets for builds"
-    runs-on: ubuntu-latest
+    runs-on: mempool-ci
     steps:
       - name: Determine checkout ref
         id: determine-ref
@@ -35,25 +38,25 @@ jobs:
           fi
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           ref: ${{ steps.determine-ref.outputs.ref }}
           path: assets
 
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@3235b876344d2a9aa001b8d1453c930bba69e610 # v3
         with:
-          node-version: 22.14.0
+          node-version: 24.13.0
           registry-url: "https://registry.npmjs.org"
 
       - name: Install (Prod dependencies only)
-        run: npm ci --omit=dev --omit=optional
+        run: bash meta/scripts/check-install-scripts.sh && npm ci --omit=dev --omit=optional
         working-directory: assets/frontend
 
       - name: Restore cached mining pool assets
         continue-on-error: true
         id: cache-mining-pool-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             mining-pool-assets.zip
@@ -62,7 +65,7 @@ jobs:
       - name: Restore promo video assets
         continue-on-error: true
         id: cache-promo-video-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             promo-video-assets.zip
@@ -95,20 +98,20 @@ jobs:
         run: zip -jrq promo-video-assets.zip assets/frontend/src/resources/promo-video/*
 
       - name: Upload mining pool assets as artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: mining-pool-assets
           path: mining-pool-assets.zip
 
       - name: Upload promo video assets as artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: promo-video-assets
           path: promo-video-assets.zip
 
       - name: Save mining pool assets cache
         id: cache-mining-pool-save
-        uses: actions/cache/save@v4
+        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             mining-pool-assets.zip
@@ -116,14 +119,14 @@ jobs:
 
       - name: Save promo video assets cache
         id: cache-promo-video-save
-        uses: actions/cache/save@v4
+        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             promo-video-assets.zip
           key: promo-video-assets-cache
 
   e2e:
-    runs-on: ubuntu-latest
+    runs-on: mempool-ci
     needs: cache
     strategy:
       fail-fast: false
@@ -143,22 +146,22 @@ jobs:
           fi
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           ref: ${{ steps.determine-ref.outputs.ref }}
           path: ${{ matrix.module }}
 
       - name: Setup node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@3235b876344d2a9aa001b8d1453c930bba69e610 # v3
         with:
-          node-version: 22.14.0
+          node-version: 24.13.0
           cache: "npm"
           cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json
 
       - name: Restore cached mining pool assets
         continue-on-error: true
         id: cache-mining-pool-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             mining-pool-assets.zip
@@ -167,14 +170,14 @@ jobs:
       - name: Restore cached promo video assets
         continue-on-error: true
         id: cache-promo-video-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             promo-video-assets.zip
           key: promo-video-assets-cache
 
       - name: Download artifact
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
           name: mining-pool-assets
 
@@ -182,7 +185,7 @@ jobs:
         run: unzip -o mining-pool-assets.zip -d ${{ matrix.module }}/frontend/src/resources/mining-pools
 
       - name: Download artifact
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
           name: promo-video-assets
 
@@ -192,7 +195,7 @@ jobs:
     # mempool
       - name: Chrome browser tests (${{ matrix.module }})
         if: ${{ matrix.module == 'mempool' }}
-        uses: cypress-io/github-action@v5
+        uses: cypress-io/github-action@248bde77443c376edc45906ede03a1aba9da0462 # v5
         with:
           tag: ${{ github.event_name }}
           working-directory: ${{ matrix.module }}/frontend
@@ -220,7 +223,7 @@ jobs:
       # liquid
       - name: Chrome browser tests (${{ matrix.module }})
         if: ${{ matrix.module == 'liquid' }}
-        uses: cypress-io/github-action@v5
+        uses: cypress-io/github-action@248bde77443c376edc45906ede03a1aba9da0462 # v5
         with:
           tag: ${{ github.event_name }}
           working-directory: ${{ matrix.module }}/frontend
@@ -248,7 +251,7 @@ jobs:
       # testnet
       - name: Chrome browser tests (${{ matrix.module }})
         if: ${{ matrix.module == 'testnet4' }}
-        uses: cypress-io/github-action@v5
+        uses: cypress-io/github-action@248bde77443c376edc45906ede03a1aba9da0462 # v5
         with:
           tag: ${{ github.event_name }}
           working-directory: ${{ matrix.module }}/frontend

.github/workflows/get_backend_block_height.yml

@@ -2,13 +2,16 @@ name: 'Check if servers are in sync'
 
 on: [workflow_dispatch]
 
+permissions:
+  contents: read
+
 jobs:
   print-backend-sha:
-    runs-on: ubuntu-latest
+    runs-on: mempool-ci
     name: Get block height
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: repo
 

.github/workflows/get_backend_hash.yml

@@ -2,13 +2,16 @@ name: 'Print backend hashes'
 
 on: [workflow_dispatch]
 
+permissions:
+  contents: read
+
 jobs:
   print-backend-sha:
-    runs-on: ubuntu-latest
+    runs-on: mempool-ci
     name: Print backend hashes
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: repo
 

.github/workflows/get_image_digest.yml

@@ -7,14 +7,18 @@ on:
         description: 'Image Version'
         required: false
         default: 'latest'
-        type: string 
+        type: string
+
+permissions:
+  contents: read
+
 jobs:
   print-images-sha:
-    runs-on: ubuntu-latest
+    runs-on: mempool-ci
     name: Print digest for images
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: digest
 

.github/workflows/project-review-status.yml

@@ -1,84 +1,21 @@
-# Workflow: Automatically set project status to "Review Needed" when a reviewer is requested
-name: Set Project Status on Review Request
+name: Project Board Automation
 
-# Trigger: Runs whenever a reviewer is requested on a pull request
 on:
   pull_request:
     types: [review_requested]
+  issues:
+    types: [opened]
+
+permissions:
+  contents: read
 
 jobs:
-  update-project-status:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Update Project Status to Review Needed
-        uses: actions/github-script@v7
-        with:
-          # Use the PAT stored in repository secrets (has project write access)
-          github-token: ${{ secrets.PROJECT_TOKEN }}
-          script: |
-            // GraphQL query to find the PR's project items
-            // This fetches all projects the PR is linked to
-            const query = `
-              query($owner: String!, $repo: String!, $pr: Int!) {
-                repository(owner: $owner, name: $repo) {
-                  pullRequest(number: $pr) {
-                    projectItems(first: 10) {
-                      nodes {
-                        id
-                        project {
-                          number
-                        }
-                      }
-                    }
-                  }
-                }
-              }
-            `;
-
-            // Execute the query with current repo/PR context
-            const result = await github.graphql(query, {
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              pr: context.payload.pull_request.number
-            });
-
-            // Find the project item that belongs to project #8
-            const projectItems = result.repository.pullRequest.projectItems.nodes;
-            const projectItem = projectItems.find(item => item.project.number === 8);
-
-            // Exit early if PR isn't in project #8
-            if (!projectItem) {
-              console.log('PR is not in project #8, skipping...');
-              return;
-            }
-
-            // GraphQL mutation to update the Status field
-            const mutation = `
-              mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!) {
-                updateProjectV2ItemFieldValue(
-                  input: {
-                    projectId: $projectId
-                    itemId: $itemId
-                    fieldId: $fieldId
-                    value: { singleSelectOptionId: $optionId }
-                  }
-                ) {
-                  projectV2Item {
-                    id
-                  }
-                }
-              }
-            `;
-
-            // Execute the mutation using IDs stored in repository variables
-            // PROJECT_ID: The project's unique identifier
-            // STATUS_FIELD_ID: The "Status" field's unique identifier
-            // REVIEW_NEEDED_OPTION_ID: The "Review Needed" option's unique identifier
-            await github.graphql(mutation, {
-              projectId: "${{ secrets.PROJECT_ID }}",
-              itemId: projectItem.id,
-              fieldId: "${{ secrets.STATUS_FIELD_ID }}",
-              optionId: "${{ secrets.REVIEW_NEEDED_OPTION_ID }}"
-            });
-
-            console.log('Successfully updated project status to Review Needed');
+  project-automation:
+    uses: mempool/.github/.github/workflows/project-board-automation.yml@267e0f38846bfb8cf1b118798dfb12def19ccb35 # master
+    with:
+      project-number: 8
+    secrets:
+      PROJECT_TOKEN: ${{ secrets.PROJECT_TOKEN }}
+      PROJECT_ID: ${{ secrets.PROJECT_ID }}
+      STATUS_FIELD_ID: ${{ secrets.STATUS_FIELD_ID }}
+      REVIEW_NEEDED_OPTION_ID: ${{ secrets.REVIEW_NEEDED_OPTION_ID }}

.github/workflows/server-config.yml

@@ -0,0 +1,32 @@
+name: Validate server config
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+      - "nginx.conf"
+      - "http-basic.conf"
+      - "nginx-mempool.conf"
+      - "production/nginx/**"
+      - ".github/nginx-check/**"
+      - ".github/workflows/server-config.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  check_config:
+    runs-on: "ubuntu-latest"
+
+    name: Validate nginx config
+    steps:
+      - name: Checkout
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
+
+      - name: Validate the production nginx config
+        run: |
+          docker run --rm $(docker build -q -f .github/nginx-check/Dockerfile.production production)
+
+      - name: Validate the docker frontend nginx config
+        run: |
+          docker run --rm $(docker build -q -f .github/nginx-check/Dockerfile.docker .)

.github/workflows/supply-chain-audit.yml

@@ -0,0 +1,58 @@
+name: Supply Chain Audit
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+  push:
+    branches:
+      - master
+
+permissions:
+  contents: read
+
+jobs:
+  backend:
+    if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
+    runs-on: mempool-ci
+    name: Backend install-script audit
+    steps:
+      - name: Checkout
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
+
+      - name: Setup Node
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: "24.13.0"
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Install npm 11.12.0
+        run: npm install -g npm@11.12.0
+
+      - name: Audit backend install scripts
+        run: bash backend/meta/scripts/check-install-scripts.sh
+
+      - name: Run backend safe install
+        run: bash backend/meta/scripts/safe-install.sh
+
+  frontend:
+    if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
+    runs-on: mempool-ci
+    name: Frontend install-script audit
+    steps:
+      - name: Checkout
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
+
+      - name: Setup Node
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: "24.13.0"
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Install npm 11.12.0
+        run: npm install -g npm@11.12.0
+
+      - name: Audit frontend install scripts
+        run: bash frontend/meta/scripts/check-install-scripts.sh
+
+      - name: Run frontend safe install
+        run: bash frontend/meta/scripts/safe-install.sh

.nvmrc

@@ -1 +1 @@
-v22
+v24.13.0

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 # Contributing to The Mempool Open Source Project
 
-Thank you for contributing to The Mempool Open Source Project managed by Mempool Space K.K. (“Mempool”).
+Thank you for contributing to The Mempool Open Source Project managed by Mempool Holdings S.A. de C.V. in El Salvador (“Mempool”).
 
 In order to clarify the intellectual property license granted with Contributions from any person or entity, Mempool must have a statement on file from each Contributor indicating their agreement to the Contributor License Agreement (“Agreement”). This license is for your protection as a Contributor as well as the protection of Mempool and its other contributors and users; it does not change your rights to use your own Contributions for any other purpose.
 

LICENSE

@@ -1,5 +1,5 @@
 The Mempool Open Source Project®
-Copyright (c) 2019-2025 Mempool Space K.K. and other shadowy super-coders
+Copyright (c) 2019-2026 Mempool Holdings S.A. de C.V. and other shadowy super-coders
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU Affero General Public License as published by the Free
@@ -7,17 +7,17 @@ Software Foundation, either version 3 of the License or any later version
 approved by a proxy statement published on <https://mempool.space/about>.
 
 However, this copyright license does not include an implied right or license
-to use any trademarks, service marks, logos, or trade names of Mempool Space K.K.
+to use any trademarks, service marks, logos, or trade names of Mempool Holdings
 or any other contributor to The Mempool Open Source Project.
 
 The Mempool Open Source Project®, Mempool Accelerator®, Mempool Enterprise®, 
 Mempool Wallet™, mempool.space®, Be your own explorer™, Explore the full 
-Bitcoin ecosystem™, Mempool Goggles™, the mempool Logo, the mempool Square Logo, 
+Bitcoin ecosystem™, Mempool Goggles®, the mempool Logo, the mempool Square Logo, 
 the mempool block visualization Logo, the mempool Blocks Logo, the mempool 
 transaction Logo, the mempool Blocks 3 | 2 Logo, the mempool research Logo, 
 the mempool.space Vertical Logo, and the mempool.space Horizontal Logo are 
-registered trademarks or trademarks of Mempool Space K.K in Japan, 
-the United States, and/or other countries.
+registered trademarks or trademarks of Mempool Holdings S.A. de C.V. in El
+Salvador, Japan, the United States, and/or other countries.
 
 See our full Trademark Policy and Guidelines for more details, published on 
 <https://mempool.space/trademark-policy>.

backend/.eslintignore

@@ -1,2 +1,4 @@
 node_modules
-dist
+dist
+eslint-local-rules
+.eslintrc.js

backend/.eslintrc.js

@@ -1,8 +1,13 @@
-{
+module.exports = {
   "root": true,
   "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "project": "./tsconfig.json",
+    "tsconfigRootDir": __dirname
+  },
   "plugins": [
-    "@typescript-eslint"
+    "@typescript-eslint",
+    "local-rules"
   ],
   "extends": [
     "eslint:recommended",
@@ -10,6 +15,16 @@
     "plugin:@typescript-eslint/recommended",
     "prettier"
   ],
+  "ignorePatterns": ["dist", "eslint-local-rules", ".eslintrc.js", "testSetup*.ts", "jest.integration.*.ts", "__tests__", "__e2e__", "*.config.ts"],
+  "overrides": [
+    {
+      "files": ["src/__integration_tests__/**/*"],
+      "rules": {
+        "@typescript-eslint/no-floating-promises": "off",
+        "local-rules/no-unhandled-await": "off"
+      }
+    }
+  ],
   "rules": {
     "@typescript-eslint/ban-ts-comment": 1,
     "@typescript-eslint/ban-types": 1,
@@ -21,6 +36,8 @@
     "@typescript-eslint/no-var-requires": 1,
     "@typescript-eslint/explicit-function-return-type": 1,
     "@typescript-eslint/no-unused-vars": 1,
+    "@typescript-eslint/no-floating-promises": "error",
+    "local-rules/no-unhandled-await": "error",
     "no-console": 1,
     "no-constant-condition": 1,
     "no-dupe-else-if": 1,

backend/.npmrc

@@ -0,0 +1 @@
+min-release-age=7

backend/eslint-local-rules/index.js

@@ -0,0 +1,398 @@
+'use strict';
+
+module.exports = {
+  'no-unhandled-await': {
+    meta: {
+      type: 'problem',
+      docs: {
+        description: 'forbid unhandled await unless callee is @asyncSafe, context is @asyncUnsafe, or rejection is explicitly handled',
+      },
+      schema: [{
+        type: 'object',
+        properties: {
+          safeTag:   { type: 'string' }, // jsdoc tag that marks a callee safe (default '@asyncSafe')
+          unsafeTag: { type: 'string' }, // comment/jsdoc that marks a context unsafe (default '@asyncUnsafe')
+          allowAllSettled:     { type: 'boolean' },
+          allowCatchMethod:    { type: 'boolean' },
+          allowThenWithTwoArgs:{ type: 'boolean' },
+        },
+        additionalProperties: false,
+      }],
+      messages: {
+        unhandled:
+          'await of non-@asyncSafe callee in @asyncSafe context; use try/catch or annotate callee (@asyncSafe) or context (@asyncUnsafe)',
+        unhandledVoid:
+          'void of non-@asyncSafe callee; annotate callee with @asyncSafe or handle the promise properly',
+      },
+    },
+
+    create(context) {
+      const src = context.getSourceCode();
+      const opt = Object.assign(
+        {
+          safeTag: '@asyncSafe',
+          unsafeTag: '@asyncUnsafe',
+          allowAllSettled: true,
+          allowCatchMethod: true,
+          allowThenWithTwoArgs: true,
+        },
+        (context.options && context.options[0]) || {}
+      );
+
+      // optional typescript API (for cross-file/class resolution)
+      let ts = null, services = null, checker = null, es2ts = null;
+      try {
+        // eslint will only populate parserServices if @typescript-eslint/parser + parserOptions.project are set
+        services = context.parserServices || null;
+        // @ts-ignore
+        if (services && (services.program || services.esTreeNodeToTSNodeMap)) {
+          // eslint-disable-next-line @typescript-eslint/no-var-requires
+          ts = require('typescript');
+          // @ts-ignore
+          const program = services.program;
+          // @ts-ignore
+          es2ts = services.esTreeNodeToTSNodeMap;
+          checker = program?.getTypeChecker?.();
+        }
+      } catch { /* noop */ }
+
+      const hasRange = (n) => n && Array.isArray(n.range);
+      const inside = (n, o) => hasRange(n) && hasRange(o) && n.range[0] >= o.range[0] && n.range[1] <= o.range[1];
+      const before = (a, b) => hasRange(a) && hasRange(b) && a.range[0] < b.range[0];
+
+      const isFnNode = (n) =>
+        n &&
+        (n.type === 'FunctionDeclaration' ||
+         n.type === 'FunctionExpression' ||
+         n.type === 'ArrowFunctionExpression' ||
+         n.type === 'MethodDefinition');
+
+      const isCommentWith = (c, tag) => typeof c?.value === 'string' && c.value.includes(tag);
+
+      // --- jsdoc tag utils ----------------------------------------------------
+      const stripAt = (s) => (s || '').replace(/^@/, '');
+
+      function tsNodeHasJsDocTag(tsNode, tag) {
+        if (!ts || !tsNode) return false;
+        try {
+          const want = stripAt(tag);
+          const tags = ts.getJSDocTags(tsNode) || [];
+          return tags.some((t) => {
+            const n = t.tagName && (t.tagName.escapedText || t.tagName.getText?.());
+            return String(n) === want;
+          });
+        } catch { return false; }
+      }
+
+      function leadingCommentsHaveTag(node, tag) {
+        if (!node) return false;
+        const lead = src.getCommentsBefore(node) || [];
+        return lead.some((c) => isCommentWith(c, tag));
+      }
+
+      function isExportWrapper(node) {
+        return node?.type === 'ExportNamedDeclaration' || node?.type === 'ExportDefaultDeclaration';
+      }
+
+      // does a given function *definition* carry tag in its leading comments?
+      function fnHasTag(fnNode, tag) {
+        if (!fnNode) return false;
+
+        // 1) method definitions: jsdoc sits on the MethodDefinition
+        if (fnNode.type === 'MethodDefinition') {
+          return leadingCommentsHaveTag(fnNode, tag);
+        }
+
+        // 2) function declarations (also handle `export` wrappers)
+        if (fnNode.type === 'FunctionDeclaration') {
+          if (leadingCommentsHaveTag(fnNode, tag)) return true;
+          const p = fnNode.parent;
+          if (isExportWrapper(p) && leadingCommentsHaveTag(p, tag)) return true;
+          const gp = p && p.parent;
+          if (isExportWrapper(gp) && leadingCommentsHaveTag(gp, tag)) return true; // belt & suspenders
+          return false;
+        }
+
+        // 3) function/arrow expressions
+        if (fnNode.type === 'FunctionExpression' || fnNode.type === 'ArrowFunctionExpression') {
+          // tag directly on the expression
+          if (leadingCommentsHaveTag(fnNode, tag)) return true;
+
+          const p = fnNode.parent;
+
+          // tag on class members that wrap the fn expr (class fields or methods-as-values)
+          if (
+            p?.type === 'MethodDefinition' ||
+            p?.type === 'PropertyDefinition' ||     // ts/estree: class field
+            p?.type === 'ClassProperty'             // older @typescript-eslint
+          ) {
+            if (leadingCommentsHaveTag(p, tag)) return true;
+          }
+
+          // tag on a variable declarator (const fn = async () => {})
+          if (p?.type === 'VariableDeclarator') {
+            if (leadingCommentsHaveTag(p, tag)) return true;
+            if (p.parent && leadingCommentsHaveTag(p.parent, tag)) return true; // VariableDeclaration
+            // handle: export const fn = async () => {}
+            const exp = p.parent && p.parent.parent;
+            if (isExportWrapper(exp) && leadingCommentsHaveTag(exp, tag)) return true;
+          }
+
+          // handle: export default (async () => {...})  or export default (async function(){})
+          if (isExportWrapper(p) && leadingCommentsHaveTag(p, tag)) return true;
+        }
+
+        return false;
+      }
+
+
+      // nearest class body ancestor (if any)
+      function nearestClassBody() {
+        const anc = context.getAncestors();
+        for (let i = anc.length - 1; i >= 0; i--) {
+          if (anc[i]?.type === 'ClassBody') return anc[i];
+        }
+        return null;
+      }
+
+      // within the same class, find a method by name
+      function findMethodInCurrentClass(propertyName) {
+        const body = nearestClassBody();
+        if (!body) return null;
+        for (const el of body.body || []) {
+          if (el?.type === 'MethodDefinition') {
+            // only handle simple identifiers (not computed) rn
+            if (el.key?.type === 'Identifier' && el.key.name === propertyName) return el;
+          }
+        }
+        return null;
+      }
+
+      // nearest function ancestor
+      function nearestFn() {
+        const anc = context.getAncestors();
+        for (let i = anc.length - 1; i >= 0; i--) if (isFnNode(anc[i])) return anc[i];
+        return null;
+      }
+
+      // nearest block or program ancestor
+      function nearestBlockOrProgram() {
+        const anc = context.getAncestors();
+        for (let i = anc.length - 1; i >= 0; i--) {
+          const a = anc[i];
+          if (a?.type === 'BlockStatement' || a?.type === 'Program') return a;
+        }
+        return null;
+      }
+
+      // context is @asyncUnsafe if the function has the tag OR there is a tagged comment earlier in the same block/program
+      function contextIsAnnotatedUnsafe(node) {
+        const fn = nearestFn();
+        if (fnHasTag(fn, opt.unsafeTag)) return true;
+
+        const blk = nearestBlockOrProgram();
+        if (!blk) return false;
+        const lead = src.getCommentsBefore(node) || [];
+        return lead.some((c) => isCommentWith(c, opt.unsafeTag) && inside(c, blk) && before(c, node));
+      }
+
+      // in try { ... } ?
+      function inTryBlock(node) {
+        const anc = context.getAncestors();
+        for (let i = anc.length - 1; i >= 0; i--) {
+          const a = anc[i];
+          if (a?.type === 'TryStatement' && a.block && inside(node, a.block)) return true;
+        }
+        return false;
+      }
+
+      const unwrapChain = (e) => (e && e.type === 'ChainExpression' ? e.expression : e);
+
+      function isHandledAwaitArg(node) {
+        const arg = unwrapChain(node.argument);
+        if (!arg) return false;
+
+        // await Promise.allSettled(...)
+        if (
+          opt.allowAllSettled &&
+          arg.type === 'CallExpression' &&
+          arg.callee?.type === 'MemberExpression' &&
+          arg.callee.object?.type === 'Identifier' &&
+          arg.callee.object.name === 'Promise' &&
+          arg.callee.property?.type === 'Identifier' &&
+          arg.callee.property.name === 'allSettled'
+        ) return true;
+
+        // await p.catch(...)
+        if (
+          opt.allowCatchMethod &&
+          arg.type === 'CallExpression' &&
+          arg.callee?.type === 'MemberExpression' &&
+          arg.callee.property?.type === 'Identifier' &&
+          arg.callee.property.name === 'catch'
+        ) return true;
+
+        // await p.then(onFulfilled, onRejected)
+        if (
+          opt.allowThenWithTwoArgs &&
+          arg.type === 'CallExpression' &&
+          arg.callee?.type === 'MemberExpression' &&
+          arg.callee.property?.type === 'Identifier' &&
+          arg.callee.property.name === 'then' &&
+          Array.isArray(arg.arguments) &&
+          arg.arguments.length >= 2
+        ) return true;
+
+        return false;
+      }
+
+      // resolve identifier → local function def in scope (best-effort)
+      function resolveFnFromIdentifier(id) {
+        const name = id?.name;
+        if (!name) return null;
+        let scope = context.getScope();
+        while (scope) {
+          const v = (scope.set && scope.set.get(name)) || scope.variables?.find((vv) => vv.name === name);
+          if (v && v.defs && v.defs.length) {
+            for (const d of v.defs) {
+              const dn = d.node;
+              if (!dn) continue;
+              if (dn.type === 'FunctionDeclaration') return dn;
+              if (dn.type === 'VariableDeclarator') {
+                const init = dn.init;
+                if (init && (init.type === 'ArrowFunctionExpression' || init.type === 'FunctionExpression')) {
+                  return init;
+                }
+              }
+            }
+          }
+          scope = scope.upper;
+        }
+        return null;
+      }
+
+      // typescript-powered resolution: member call callee → ts declarations → jsdoc tags
+      function tsMemberIsAnnotatedSafe(memberExpr) {
+        if (!ts || !checker || !es2ts) return false;
+        const prop = memberExpr.property;
+        if (!prop || prop.type !== 'Identifier') return false; // skip computed/strings rn
+
+        try {
+          const tsObj = es2ts.get(unwrapChain(memberExpr.object));
+          if (!tsObj) return false;
+          let type = checker.getTypeAtLocation(tsObj);
+          if (!type) return false;
+          // normalize to apparent type (unions, etc.)
+          const apparent = checker.getApparentType ? checker.getApparentType(type) : type;
+          const name = prop.name;
+
+          // ts <5 vs >=5 api differences
+          const sym = (apparent.getProperty && apparent.getProperty(name)) ||
+                      (checker.getPropertyOfType && checker.getPropertyOfType(apparent, name));
+          if (!sym || !Array.isArray(sym.declarations)) return false;
+
+          for (const decl of sym.declarations) {
+            // method, function, property with function type — accept any with @asyncSafe
+            if (tsNodeHasJsDocTag(decl, opt.safeTag)) return true;
+            // for class methods, also check the parent (sometimes the tag is on the signature)
+            if (decl.parent && tsNodeHasJsDocTag(decl.parent, opt.safeTag)) return true;
+          }
+        } catch { /* ignore */ }
+        return false;
+      }
+
+      // estree-only: this.method() inside same class
+      function thisMethodIsAnnotatedSafe(memberExpr) {
+        if (memberExpr.object?.type !== 'ThisExpression') return false;
+        const prop = memberExpr.property;
+        if (!prop || prop.type !== 'Identifier') return false;
+        const m = findMethodInCurrentClass(prop.name);
+        return fnHasTag(m, opt.safeTag);
+      }
+
+      function calleeIsAnnotatedSafe(callExpr) {
+        const arg = unwrapChain(callExpr);
+        if (!arg) return false;
+
+        if (arg.type === 'CallExpression') {
+          const c = unwrapChain(arg.callee);
+          if (!c) return false;
+
+          // direct identifier call
+          if (c.type === 'Identifier') {
+            const def = resolveFnFromIdentifier(c);
+            if (fnHasTag(def, opt.safeTag)) return true;
+
+            // ts fallback for imported funcs
+            if (ts && checker && es2ts) {
+              try {
+                const tsCallee = es2ts.get(c);
+                const sym = checker.getSymbolAtLocation?.(tsCallee);
+                const decls = sym?.declarations || [];
+                for (const d of decls) {
+                  if (tsNodeHasJsDocTag(d, opt.safeTag) || (d.parent && tsNodeHasJsDocTag(d.parent, opt.safeTag))) {
+                    return true;
+                  }
+                }
+              } catch { /* noop */ }
+            }
+            return false;
+          }
+
+          // member call: this.m(), obj.m()
+          if (c.type === 'MemberExpression') {
+            // 1) easy path: this.method inside same class
+            if (thisMethodIsAnnotatedSafe(c)) return true;
+
+            // 2) ts-powered cross-file/class/instance resolution
+            if (tsMemberIsAnnotatedSafe(c)) return true;
+
+            return false;
+          }
+
+          // function expressions / arrow directly inline
+          if (c.type === 'FunctionExpression' || c.type === 'ArrowFunctionExpression') {
+            return fnHasTag(c, opt.safeTag);
+          }
+
+          // dynamic/new/etc → treat as unsafe
+          return false;
+        }
+
+        // awaiting a non-call promise value → treat as unsafe
+        return false;
+      }
+
+      // --- main ---------------------------------------------------------------
+      return {
+        AwaitExpression(node) {
+          // handled patterns → ok
+          if (inTryBlock(node) || isHandledAwaitArg(node)) return;
+
+          // callee carries @asyncSafe? → ok anywhere
+          if (calleeIsAnnotatedSafe(node.argument)) return;
+
+          // context explicitly @asyncUnsafe? → ok to bubble
+          if (contextIsAnnotatedUnsafe(node)) return;
+
+          // default: context is safe, callee is unsafe → error
+          context.report({ node, messageId: 'unhandled' });
+        },
+
+        // void someAsyncCall() — only allowed if callee is @asyncSafe
+        UnaryExpression(node) {
+          if (node.operator !== 'void') return;
+
+          const arg = unwrapChain(node.argument);
+          if (!arg || arg.type !== 'CallExpression') return;
+
+          // callee carries @asyncSafe? → ok
+          if (calleeIsAnnotatedSafe(node.argument)) return;
+
+          // void of non-safe callee → error
+          context.report({ node, messageId: 'unhandledVoid' });
+        },
+      };
+    },
+  },
+};

backend/jest.config.ts

@@ -1,24 +1,26 @@
-import type { Config } from "@jest/types"
+import type { Config } from '@jest/types';
 
 const config: Config.InitialOptions = {
-  preset: "ts-jest",
-  testEnvironment: "node",
+  preset: 'ts-jest',
+  testEnvironment: 'node',
   verbose: true,
   automock: false,
   collectCoverage: true,
-  collectCoverageFrom: ["./src/**/**.ts"],
-  coverageProvider: "v8",
+  collectCoverageFrom: ['./src/**/**.ts'],
+  coverageProvider: 'v8',
   coverageThreshold: {
     global: {
       lines: 1
     }
   },
   setupFiles: [
-    "./testSetup.ts",
+    './testSetup.ts',
   ],
   testPathIgnorePatterns: [
-    "/node_modules/",
-    "/__integration_tests__/",
+    '/dist/',
+    '/node_modules/',
+    '/__integration_tests__/',
+    'test-utils\\.ts$',
   ],
-}
+};
 export default config;

backend/jest.integration.config.ts

@@ -1,21 +1,21 @@
-import type { Config } from "@jest/types"
+import type { Config } from '@jest/types';
 
 const config: Config.InitialOptions = {
-  preset: "ts-jest",
-  testEnvironment: "node",
+  preset: 'ts-jest',
+  testEnvironment: 'node',
   verbose: true,
   automock: false,
   collectCoverage: false,
-  coverageProvider: "v8",
+  coverageProvider: 'v8',
   testMatch: [
-    "**/__integration_tests__/**/*.test.ts"
+    '**/__integration_tests__/**/*.test.ts'
   ],
-  globalSetup: "./jest.integration.setup.ts",  // Start database before all tests
+  globalSetup: './jest.integration.setup.ts',  // Start database before all tests
   setupFiles: [
-    "./testSetup.integration.ts",
+    './testSetup.integration.ts',
   ],
-  globalTeardown: "./jest.integration.teardown.ts",  // Stop database after all tests
+  globalTeardown: './jest.integration.teardown.ts',  // Stop database after all tests
   maxWorkers: 1,  // Force sequential execution
-}
+};
 export default config;
 

backend/jest.integration.setup.ts

@@ -35,18 +35,18 @@ module.exports = async () => {
   try {
     const composeFile = path.join(__dirname, 'docker-compose.test.yml');
     const dockerComposeCmd = getDockerComposeCmd();
-    
+
     // Start the container
-    execSync(`${dockerComposeCmd} -f "${composeFile}" up -d`, { 
+    execSync(`${dockerComposeCmd} -f "${composeFile}" up -d`, {
       stdio: 'inherit',
       cwd: __dirname
     });
-    
+
     // Wait for database to be ready
     console.log('Waiting for database to be ready...');
     let attempts = 0;
     const maxAttempts = 30;
-    
+
     while (attempts < maxAttempts) {
       try {
         execSync(`${dockerComposeCmd} -f "${composeFile}" exec -T db-test mysqladmin ping -h localhost -u mempool_test -pmempool_test --silent`, {

backend/jest.integration.teardown.ts

@@ -43,7 +43,7 @@ module.exports = async () => {
     ];
 
     await DB.query('SET FOREIGN_KEY_CHECKS = 0');
-    
+
     for (const table of tables) {
       try {
         // Use 'silent' error logging to avoid noise for optional tables that don't exist
@@ -52,29 +52,29 @@ module.exports = async () => {
         // Table might not exist - silently ignore
       }
     }
-    
+
     await DB.query('SET FOREIGN_KEY_CHECKS = 1');
-    
+
     logger.info('Integration tests cleanup completed');
-    
+
     // Close the database connection pool to prevent Jest from hanging
     await DB.close();
     logger.info('Database connection pool closed');
-    
+
     // Clean up singleton resources that have timers or sockets
     mempool.destroy();
     logger.info('Mempool resources cleaned up');
-    
+
     // Close logger's UDP socket last (after all logging is done)
     logger.close();
-    
+
     // Stop and remove the Docker test database container
     // Skip if SKIP_DB_TEARDOWN is set (e.g., when test-with-db.sh manages the database)
     if (!process.env.SKIP_DB_TEARDOWN) {
       try {
         const composeFile = path.join(__dirname, 'docker-compose.test.yml');
         const dockerComposeCmd = getDockerComposeCmd();
-        execSync(`${dockerComposeCmd} -f "${composeFile}" down -v`, { 
+        execSync(`${dockerComposeCmd} -f "${composeFile}" down -v`, {
           stdio: 'inherit',
           cwd: __dirname
         });

backend/mempool-config.sample.json

@@ -31,6 +31,8 @@
     "AUDIT": false,
     "RUST_GBT": true,
     "LIMIT_GBT": false,
+    "CLUSTER_MEMPOOL": false,
+    "CLUSTER_MEMPOOL_INDEXING": false,
     "CPFP_INDEXING": false,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,

backend/mempool-config.test.json

@@ -31,6 +31,8 @@
     "AUDIT": false,
     "RUST_GBT": true,
     "LIMIT_GBT": false,
+    "CLUSTER_MEMPOOL": false,
+    "CLUSTER_MEMPOOL_INDEXING": false,
     "CPFP_INDEXING": false,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,

backend/meta/scripts/check-install-scripts.sh

@@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+#
+# Audit backend install-script sources before running a full npm install.
+# Exits non-zero if any package not on the whitelist has hasInstallScript: true.
+#
+# Usage (from repo root):
+#   backend/meta/scripts/check-install-scripts.sh
+#
+
+set -euo pipefail
+
+BACKEND_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+LOCKFILE="${BACKEND_DIR}/package-lock.json"
+RUST_GBT_DIR="${BACKEND_DIR}/../rust/gbt"
+TMP_AUDIT_DIR=""
+
+BACKEND_ALLOWED=(
+  "mempool-backend"
+  "fsevents"
+  "unrs-resolver"
+)
+
+RUST_GBT_ALLOWED=()
+
+is_allowed() {
+  local pkg="$1"
+  shift
+  local allowed
+  for allowed in "$@"; do
+    if [[ "$pkg" == "$allowed" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+cleanup() {
+  if [[ -n "${TMP_AUDIT_DIR}" && -d "${TMP_AUDIT_DIR}" ]]; then
+    rm -rf "${TMP_AUDIT_DIR}"
+  fi
+}
+
+trap cleanup EXIT
+
+collect_has_install_script_packages() {
+  local lockfile="$1"
+  if [[ ! -f "$lockfile" ]]; then
+    echo "No package-lock.json found at ${lockfile}"
+    return 1
+  fi
+
+  LOCKFILE_PATH="$lockfile" node - <<'NODE'
+const fs = require('fs');
+
+const lock = JSON.parse(fs.readFileSync(process.env.LOCKFILE_PATH, 'utf8'));
+const pkgs = lock.packages || {};
+
+for (const [path, meta] of Object.entries(pkgs)) {
+  if (meta && meta.hasInstallScript) {
+    const name = path === '' ? (lock.name || '(root)') : path.replace(/^.*node_modules\//, '');
+    console.log(name);
+  }
+}
+NODE
+}
+
+audit_lockfile() {
+  local lockfile="$1"
+  local label="$2"
+  shift 2
+  local allowed=("$@")
+  local found_violations=0
+  local pkg
+
+  while IFS= read -r pkg; do
+    [[ -z "$pkg" ]] && continue
+    if ! is_allowed "$pkg" "${allowed[@]}"; then
+      echo "VIOLATION: unauthorized install script in '${pkg}' (${label}: ${lockfile})"
+      found_violations=1
+    fi
+  done < <(collect_has_install_script_packages "$lockfile")
+
+  if [[ "$found_violations" -eq 1 ]]; then
+    echo ""
+    echo "FAILED: Found packages with install scripts not on the whitelist."
+    return 1
+  fi
+
+  echo "OK: All ${label} install scripts are from whitelisted packages."
+}
+
+get_rust_gbt_napi_cli_version() {
+  local rust_gbt_package_json="${RUST_GBT_DIR}/package.json"
+  if [[ ! -f "$rust_gbt_package_json" ]]; then
+    echo "No rust/gbt package.json found at ${rust_gbt_package_json}"
+    return 1
+  fi
+
+  RUST_GBT_PACKAGE_JSON="$rust_gbt_package_json" node - <<'NODE'
+const fs = require('fs');
+
+const pkg = JSON.parse(fs.readFileSync(process.env.RUST_GBT_PACKAGE_JSON, 'utf8'));
+const buildScript = (pkg.scripts && pkg.scripts.build) || '';
+const match = buildScript.match(/npm install --no-save @napi-rs\/cli@([^\s]+)/);
+
+if (!match) {
+  console.error('Could not determine the @napi-rs/cli version from rust/gbt/package.json.');
+  process.exit(1);
+}
+
+process.stdout.write(match[1]);
+NODE
+}
+
+generate_rust_gbt_audit_lockfile() {
+  local napi_cli_version="$1"
+
+  TMP_AUDIT_DIR="$(mktemp -d "${TMPDIR:-/tmp}/backend-rust-gbt-audit.XXXXXX")"
+  cat > "${TMP_AUDIT_DIR}/package.json" <<EOF
+{
+  "name": "rust-gbt-install-audit",
+  "private": true,
+  "dependencies": {
+    "@napi-rs/cli": "${napi_cli_version}"
+  }
+}
+EOF
+
+  (
+    cd "${TMP_AUDIT_DIR}"
+    npm install --package-lock-only --ignore-scripts --no-audit --no-fund >/dev/null
+  )
+
+  printf '%s\n' "${TMP_AUDIT_DIR}/package-lock.json"
+}
+
+audit_rust_gbt_preinstall_chain() {
+  local napi_cli_version
+  local rust_gbt_lockfile
+
+  napi_cli_version="$(get_rust_gbt_napi_cli_version)"
+  rust_gbt_lockfile="$(generate_rust_gbt_audit_lockfile "${napi_cli_version}")"
+  if [[ "${#RUST_GBT_ALLOWED[@]}" -gt 0 ]]; then
+    audit_lockfile "${rust_gbt_lockfile}" "backend preinstall toolchain" "${RUST_GBT_ALLOWED[@]}"
+  else
+    audit_lockfile "${rust_gbt_lockfile}" "backend preinstall toolchain"
+  fi
+}
+
+audit_lockfile "${LOCKFILE}" "backend lockfile" "${BACKEND_ALLOWED[@]}"
+audit_rust_gbt_preinstall_chain
+
+echo "OK: Backend install-script audit passed."

backend/meta/scripts/safe-install.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+#
+# Safely install backend npm dependencies by refreshing the lockfile without
+# running install scripts, auditing it, then doing the real install.
+#
+# Usage (from repo root):
+#   backend/meta/scripts/safe-install.sh
+#
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BACKEND_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+LOCKFILE="${BACKEND_DIR}/package-lock.json"
+RESTORE_LOCKFILE_DONE=0
+
+# Back up the lockfile so we can restore on failure
+if [[ -f "$LOCKFILE" ]]; then
+  cp "$LOCKFILE" "${LOCKFILE}.bak"
+fi
+
+restore_lockfile() {
+  trap - ERR INT TERM
+  if [[ "${RESTORE_LOCKFILE_DONE}" -eq 1 ]]; then
+    return
+  fi
+  RESTORE_LOCKFILE_DONE=1
+
+  if [[ -f "${LOCKFILE}.bak" ]]; then
+    mv "${LOCKFILE}.bak" "$LOCKFILE"
+    echo "Restored original package-lock.json."
+  elif [[ -f "$LOCKFILE" ]]; then
+    rm "$LOCKFILE"
+    echo "Removed generated package-lock.json."
+  fi
+}
+
+trap restore_lockfile ERR INT TERM
+
+echo "==> Refreshing backend lockfile (--ignore-scripts --package-lock-only)..."
+(cd "$BACKEND_DIR" && npm install --ignore-scripts --package-lock-only --no-audit --no-fund)
+
+echo ""
+echo "==> Auditing lockfile for install scripts..."
+bash "${SCRIPT_DIR}/check-install-scripts.sh"
+
+trap - ERR INT TERM
+rm -f "${LOCKFILE}.bak"
+
+echo ""
+echo "==> Installing backend (npm ci)..."
+(cd "$BACKEND_DIR" && npm ci)
+
+echo ""
+echo "Done."

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-backend",
-  "version": "3.3-dev",
+  "version": "3.4-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",
@@ -43,28 +43,29 @@
   "dependencies": {
     "@mempool/electrum-client": "1.1.9",
     "@types/node": "^18.15.3",
-    "axios": "1.12.2",
+    "axios": "1.16.1",
     "bitcoinjs-lib": "~6.1.3",
     "crypto-js": "~4.2.0",
     "express": "~4.22.1",
     "maxmind": "~4.3.11",
-    "mysql2": "~3.16.0",
+    "mysql2": "~3.22.4",
     "rust-gbt": "file:./rust-gbt",
     "redis": "^4.7.0",
     "socks-proxy-agent": "~7.0.0",
     "typescript": "~4.9.3",
-    "ws": "~8.18.0"
+    "ws": "~8.21.0"
   },
   "devDependencies": {
     "@types/compression": "^1.7.2",
     "@types/crypto-js": "^4.1.1",
     "@types/express": "^4.17.17",
     "@types/jest": "^30.0.0",
-    "@types/ws": "~8.5.10",
+    "@types/ws": "~8.18.1",
     "@typescript-eslint/eslint-plugin": "^5.55.0",
     "@typescript-eslint/parser": "^5.55.0",
     "eslint": "^8.36.0",
     "eslint-config-prettier": "^8.8.0",
+    "eslint-plugin-local-rules": "^3.0.2",
     "jest": "^30.0.0",
     "prettier": "^3.0.0",
     "ts-jest": "^29.4.5",

backend/src/__e2e__/cluster-mempool/harness/rpc-client.ts

@@ -0,0 +1,137 @@
+import http from 'http';
+
+export interface RpcConfig {
+  host: string;
+  port: number;
+  user: string;
+  pass: string;
+}
+
+interface RpcResponse {
+  result: any;
+  error: { code: number; message: string } | null;
+  id: string;
+}
+
+export class RpcClient {
+  private config: RpcConfig;
+  private idCounter = 0;
+
+  constructor(config: RpcConfig) {
+    this.config = config;
+  }
+
+  private async call(method: string, params: any[] = []): Promise<any> {
+    const id = String(++this.idCounter);
+    const body = JSON.stringify({ jsonrpc: '2.0', id, method, params });
+    const auth = Buffer.from(`${this.config.user}:${this.config.pass}`).toString('base64');
+
+    return new Promise((resolve, reject) => {
+      const req = http.request(
+        {
+          hostname: this.config.host,
+          port: this.config.port,
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Basic ${auth}`,
+            'Content-Length': Buffer.byteLength(body),
+          },
+        },
+        (res) => {
+          let data = '';
+          res.on('data', (chunk) => (data += chunk));
+          res.on('end', () => {
+            try {
+              const parsed: RpcResponse = JSON.parse(data);
+              if (parsed.error) {
+                reject(new Error(`RPC error ${parsed.error.code}: ${parsed.error.message}`));
+              } else {
+                resolve(parsed.result);
+              }
+            } catch (e) {
+              reject(new Error(`Failed to parse RPC response: ${data.slice(0, 500)}`));
+            }
+          });
+        },
+      );
+      req.on('error', reject);
+      req.write(body);
+      req.end();
+    });
+  }
+
+  async getRawMempool(): Promise<string[]> {
+    return this.call('getrawmempool');
+  }
+
+  async getRawTransaction(txid: string, verbose = true): Promise<any> {
+    return this.call('getrawtransaction', [txid, verbose]);
+  }
+
+  async getBlockTemplate(rules: string[] = ['segwit']): Promise<any> {
+    return this.call('getblocktemplate', [{ rules }]);
+  }
+
+  async getMempoolEntry(txid: string): Promise<any> {
+    return this.call('getmempoolentry', [txid]);
+  }
+
+  async getMempoolCluster(txid: string): Promise<any> {
+    return this.call('getmempoolcluster', [txid]);
+  }
+
+  async getBlockCount(): Promise<number> {
+    return this.call('getblockcount');
+  }
+
+  async batch(calls: { method: string; params: any[] }[]): Promise<any[]> {
+    if (calls.length === 0) {
+      return [];
+    }
+    const bodies = calls.map((c) => ({
+      jsonrpc: '2.0',
+      id: String(++this.idCounter),
+      method: c.method,
+      params: c.params,
+    }));
+    const body = JSON.stringify(bodies);
+    const auth = Buffer.from(`${this.config.user}:${this.config.pass}`).toString('base64');
+
+    return new Promise((resolve, reject) => {
+      const req = http.request(
+        {
+          hostname: this.config.host,
+          port: this.config.port,
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Basic ${auth}`,
+            'Content-Length': Buffer.byteLength(body),
+          },
+        },
+        (res) => {
+          let data = '';
+          res.on('data', (chunk) => (data += chunk));
+          res.on('end', () => {
+            try {
+              const parsed: RpcResponse[] = JSON.parse(data);
+              const results = parsed.map((r) => {
+                if (r.error) {
+                  return { error: r.error };
+                }
+                return r.result;
+              });
+              resolve(results);
+            } catch (e) {
+              reject(new Error(`Failed to parse batch RPC response: ${data.slice(0, 500)}`));
+            }
+          });
+        },
+      );
+      req.on('error', reject);
+      req.write(body);
+      req.end();
+    });
+  }
+}

backend/src/__e2e__/cluster-mempool/harness/run-harness.ts

@@ -0,0 +1,463 @@
+/**
+ * Cluster Mempool Test Harness
+ *
+ * Standalone script that compares our ClusterMempool block template ordering
+ * against Bitcoin Core's getblocktemplate output.
+ *
+ * Uses the backend's own transactionUtils to fetch and convert transactions,
+ * ensuring fields (sigops, adjustedVsize, etc.) match exactly.
+ *
+ * Usage:
+ *   npx ts-node src/__tests__/cluster-mempool/harness/run-harness.ts [options]
+ *
+ * Options:
+ *   --host <host>       Override CORE_RPC host
+ *   --port <port>       Override CORE_RPC port
+ *   --user <user>       Override CORE_RPC username
+ *   --pass <pass>       Override CORE_RPC password
+ *   --interval <ms>     Comparison interval in ms (default: 30000)
+ *   --poll <ms>         Mempool poll interval in ms (default: 1000)
+ *   --max <n>           Max comparisons before exit (0 = unlimited, default: 0)
+ */
+
+import { RpcClient, RpcConfig } from './rpc-client';
+
+// ─── CLI Parsing (must happen before backend imports) ───────────────────────
+
+interface CliOptions {
+  rpcOverrides: Partial<RpcConfig>;
+  comparisonInterval: number;
+  pollInterval: number;
+  maxComparisons: number;
+}
+
+function parseArgs(): CliOptions {
+  const args = process.argv.slice(2);
+  const overrides: Partial<RpcConfig> = {};
+  let comparisonInterval = 30_000;
+  let pollInterval = 1000;
+  let maxComparisons = 0;
+
+  for (let i = 0; i < args.length; i++) {
+    switch (args[i]) {
+      case '--host':
+        overrides.host = args[++i];
+        break;
+      case '--port':
+        overrides.port = parseInt(args[++i], 10);
+        break;
+      case '--user':
+        overrides.user = args[++i];
+        break;
+      case '--pass':
+        overrides.pass = args[++i];
+        break;
+      case '--interval':
+        comparisonInterval = parseInt(args[++i], 10);
+        break;
+      case '--poll':
+        pollInterval = parseInt(args[++i], 10);
+        break;
+      case '--max':
+        maxComparisons = parseInt(args[++i], 10);
+        break;
+    }
+  }
+
+  return { rpcOverrides: overrides, comparisonInterval, pollInterval, maxComparisons };
+}
+
+const cliOptions = parseArgs();
+
+// ─── Backend Module Loading ─────────────────────────────────────────────────
+// Import config first, apply CLI overrides, then import modules that depend on it.
+// This works because Node's require() caches modules — when transactionUtils
+// (via bitcoinClient) reads config, it sees our modified values.
+
+const config = require('../../../config').default;
+
+if (cliOptions.rpcOverrides.host) { config.CORE_RPC.HOST = cliOptions.rpcOverrides.host; }
+if (cliOptions.rpcOverrides.port) { config.CORE_RPC.PORT = cliOptions.rpcOverrides.port; }
+if (cliOptions.rpcOverrides.user) { config.CORE_RPC.USERNAME = cliOptions.rpcOverrides.user; }
+if (cliOptions.rpcOverrides.pass) { config.CORE_RPC.PASSWORD = cliOptions.rpcOverrides.pass; }
+
+// Now load modules that depend on config
+const bitcoinApi = require('../../../api/bitcoin/bitcoin-api-factory').default;
+const transactionUtils = require('../../../api/transaction-utils').default;
+const { ClusterMempool } = require('../../../cluster-mempool/cluster-mempool');
+
+import { MempoolTransactionExtended } from '../../../mempool.interfaces';
+import { MempoolDiff } from '../../../cluster-mempool/cluster-mempool';
+
+// ─── Main Harness ───────────────────────────────────────────────────────────
+
+const TX_FETCH_BATCH_SIZE = 1000;
+
+class Harness {
+  private rpc: RpcClient;
+  private clusterMempool: InstanceType<typeof ClusterMempool> | null = null;
+  private mempool: { [txid: string]: MempoolTransactionExtended } = {};
+  private knownTxids = new Set<string>();
+  private lastBlockHeight = -1;
+  private comparisonInterval: number;
+  private pollInterval: number;
+  private maxComparisons: number;
+  private pollTimer: ReturnType<typeof setInterval> | null = null;
+  private polling = false;
+  private nextComparisonTime = 0;
+  private stats = {
+    templateMatches: 0,
+    templateMismatches: 0,
+    comparisons: 0,
+  };
+
+  constructor(rpcConfig: RpcConfig, opts: CliOptions) {
+    this.rpc = new RpcClient(rpcConfig);
+    this.comparisonInterval = opts.comparisonInterval;
+    this.pollInterval = opts.pollInterval;
+    this.maxComparisons = opts.maxComparisons;
+  }
+
+  async run(): Promise<void> {
+    console.log('=== Cluster Mempool Test Harness ===');
+    console.log(`RPC: ${config.CORE_RPC.HOST}:${config.CORE_RPC.PORT}`);
+    console.log(`Backend: ${config.MEMPOOL.BACKEND}`);
+    console.log(`Poll interval: ${this.pollInterval}ms`);
+    console.log(`Comparison interval: ${this.comparisonInterval}ms`);
+    console.log();
+
+    // Step 1: Fetch full mempool
+    console.log('Fetching full mempool...');
+    const txids: string[] = await bitcoinApi.$getRawMempool();
+    console.log(`Got ${txids.length} txids from getrawmempool`);
+
+    const fetchStart = Date.now();
+    await this.fetchTransactions(txids);
+    const fetchTime = Date.now() - fetchStart;
+    console.log(`Fetched ${Object.keys(this.mempool).length} transactions in ${fetchTime}ms`);
+
+    // Step 2: Initialize ClusterMempool
+    console.log('Building cluster mempool...');
+    const buildStart = Date.now();
+    this.clusterMempool = new ClusterMempool(this.mempool);
+    const buildTime = Date.now() - buildStart;
+    console.log(`Cluster mempool built in ${buildTime}ms`);
+    console.log(`  Clusters: ${this.clusterMempool.getClusterCount()}`);
+    console.log(`  Transactions: ${this.clusterMempool.getTxCount()}`);
+    console.log();
+
+    // Record initial block height
+    this.lastBlockHeight = await this.rpc.getBlockCount();
+
+    // Run initial comparison, then start polling
+    await this.runComparison();
+    this.nextComparisonTime = Date.now() + this.comparisonInterval;
+    this.pollTimer = setInterval(() => this.pollMempool(), this.pollInterval);
+
+    console.log('\nHarness running. Press Ctrl+C to stop.\n');
+
+    process.on('SIGINT', () => this.shutdown());
+    process.on('SIGTERM', () => this.shutdown());
+  }
+
+  private async fetchTransactions(txids: string[]): Promise<MempoolTransactionExtended[]> {
+    const added: MempoolTransactionExtended[] = [];
+
+    for (let offset = 0; offset < txids.length; offset += TX_FETCH_BATCH_SIZE) {
+      let batch = txids.slice(offset, offset + TX_FETCH_BATCH_SIZE);
+      let txs: MempoolTransactionExtended[] = [];
+      let tries = 0;
+      while (batch.length && tries < 20) {
+        try {
+          tries++;
+          txs = txs.concat(await transactionUtils.$getMempoolTransactionsExtended(batch, false, false, false));
+          let missing: string[] = [];
+          console.log(`txs: ${txs.length} of ${batch.length} fetched`);
+          for (const txid of batch) {
+            if (!txs.some(tx => tx.txid === txid)) {
+              console.log(`missing ${txid} at offset ${offset}, retrying`);
+              missing.push(txid);
+            }
+          }
+          batch = missing;
+          if (batch.length) {
+            await new Promise(resolve => setTimeout(resolve, 500));
+          }
+        } catch (err: any) {
+          console.log(`  Fetch batch failed at offset ${offset}: ${err.message}, retrying`);
+        }
+      }
+
+      for (const tx of txs) {
+        this.mempool[tx.txid] = tx;
+        this.knownTxids.add(tx.txid);
+        added.push(tx);
+      }
+
+      if (txids.length > TX_FETCH_BATCH_SIZE && (offset + batch.length) % 5000 < TX_FETCH_BATCH_SIZE) {
+        console.log(`  ${offset + batch.length} / ${txids.length} fetched...`);
+      }
+    }
+
+    return added;
+  }
+
+  private async pollMempool(): Promise<void> {
+    if (this.polling || !this.clusterMempool) {
+      return;
+    }
+    this.polling = true;
+
+    try {
+      // Check for new block
+      const height = await this.rpc.getBlockCount();
+      const newBlock = height > this.lastBlockHeight;
+      if (newBlock) {
+        console.log(`\n--- New block detected (height ${height}) ---`);
+        this.lastBlockHeight = height;
+      }
+
+      // Get current mempool
+      const currentTxids = await bitcoinApi.$getRawMempool();
+      const currentSet = new Set(currentTxids);
+
+      // Find added and removed
+      const addedTxids: string[] = [];
+      for (const txid of currentTxids) {
+        if (!this.knownTxids.has(txid)) {
+          addedTxids.push(txid);
+        }
+      }
+      const removedTxids: string[] = [];
+      for (const txid of this.knownTxids) {
+        if (!currentSet.has(txid)) {
+          removedTxids.push(txid);
+        }
+      }
+
+      if (addedTxids.length === 0 && removedTxids.length === 0) {
+        if (newBlock) {
+          await this.runComparison();
+        }
+        return;
+      }
+
+      const added = await this.fetchTransactions(addedTxids);
+
+      const removed = removedTxids
+        .map(txid => this.mempool[txid])
+        .filter((tx): tx is MempoolTransactionExtended => !!tx);
+      const diff: MempoolDiff = { added, removed, accelerations: {} };
+      const t0 = Date.now();
+      this.clusterMempool.applyMempoolChange(diff);
+      const dt = Date.now() - t0;
+      for (const txid of removedTxids) {
+        delete this.mempool[txid];
+        this.knownTxids.delete(txid);
+      }
+      console.log(
+        `Applied${newBlock ? ' block' : ''} diff: +${added.length} -${removedTxids.length} txs in ${dt}ms` +
+        ` (${this.clusterMempool.getTxCount()} txs, ${this.clusterMempool.getClusterCount()} clusters)`
+      );
+
+      if (newBlock) {
+        await this.runComparison();
+      } else if (Date.now() >= this.nextComparisonTime) {
+        await this.runComparison();
+        this.nextComparisonTime = Date.now() + this.comparisonInterval;
+      }
+    } catch (err: any) {
+      console.error(`Poll error: ${err.message}`);
+    } finally {
+      this.polling = false;
+    }
+  }
+
+  private async runComparison(): Promise<void> {
+    if (!this.clusterMempool) {
+      return;
+    }
+    this.stats.comparisons++;
+    console.log(`\n=== Comparison #${this.stats.comparisons} ===`);
+
+    // Step 1: Get Core's block template
+    let template: any;
+    try {
+      template = await this.rpc.getBlockTemplate(['segwit']);
+    } catch (err: any) {
+      console.log(`getblocktemplate failed: ${err.message}`);
+      return;
+    }
+    const coreTxids: string[] = template.transactions.map((t: any) => t.txid);
+    const coreSet = new Set(coreTxids);
+
+    const missingTxids: string[] = [];
+    for (const txid of coreSet) {
+      if (!this.mempool[txid]) {
+        missingTxids.push(txid);
+      }
+    }
+    if (missingTxids.length) {
+      const added = await this.fetchTransactions(missingTxids);
+      if (added.length) {
+        console.log(`Reconciled: +${added.length} missing txs`);
+        this.clusterMempool.applyMempoolChange({ added, removed: [], accelerations: {} });
+      }
+    }
+
+    const ourBlocks = this.clusterMempool.getBlocks(1, true);
+    const ourTxids: string[] = ourBlocks[0]?.txids || [];
+
+    await this.compareTemplateOrdering(coreTxids, ourTxids);
+
+    this.printStats();
+
+    if (this.maxComparisons > 0 && this.stats.comparisons >= this.maxComparisons) {
+      console.log(`\nReached ${this.maxComparisons} comparisons, stopping.`);
+      this.shutdown();
+    }
+  }
+
+  private async compareTemplateOrdering(coreTxids: string[], ourTxids: string[]): Promise<void> {
+    const coreSet = new Set(coreTxids);
+    const ourSet = new Set(ourTxids);
+
+    let inBoth = 0;
+    let onlyCore = 0;
+    let onlyOurs = 0;
+    for (const txid of coreSet) {
+      if (ourSet.has(txid)) {
+        inBoth++;
+      } else {
+        onlyCore++;
+      }
+    }
+    for (const txid of ourSet) {
+      if (!coreSet.has(txid)) {
+        onlyOurs++;
+      }
+    }
+
+    console.log(`Template sets: Core ${coreTxids.length} txs, Ours ${ourTxids.length} txs`);
+    console.log(`  In both: ${inBoth}, Only Core: ${onlyCore}, Only ours: ${onlyOurs}`);
+
+    // Build position maps for the intersection
+    const corePos = new Map<string, number>();
+    for (let i = 0; i < coreTxids.length; i++) {
+      if (ourSet.has(coreTxids[i])) {
+        corePos.set(coreTxids[i], i);
+      }
+    }
+
+    // Filter to shared txids in each order
+    const sharedInCoreOrder = coreTxids.filter(t => ourSet.has(t));
+    const sharedInOurOrder = ourTxids.filter(t => coreSet.has(t));
+
+    let exactMatches = 0;
+    let firstDivergence = -1;
+    for (let i = 0; i < sharedInCoreOrder.length; i++) {
+      if (sharedInCoreOrder[i] === sharedInOurOrder[i]) {
+        exactMatches++;
+      } else if (firstDivergence === -1) {
+        firstDivergence = i;
+      }
+    }
+
+    const orderMatchRate = sharedInCoreOrder.length > 0
+      ? (exactMatches / sharedInCoreOrder.length * 100).toFixed(1)
+      : 'N/A';
+    console.log(`  Ordering: ${orderMatchRate}% exact position match (${exactMatches}/${sharedInCoreOrder.length} shared txs)`);
+
+    if (firstDivergence >= 0) {
+      const coreTx = sharedInCoreOrder[firstDivergence];
+      const ourTx = sharedInOurOrder[firstDivergence];
+      console.log(`\n  First divergence at position ${firstDivergence}:`);
+      console.log(`    Core wants: ${coreTx}`);
+      console.log(`    We placed:  ${ourTx} (Core has this at position ${corePos.get(ourTx)})`);
+
+      for (const [label, txid] of [['Core tx', coreTx], ['Our tx', ourTx]] as const) {
+        const inOurMempool = !!this.mempool[txid];
+        const info = this.clusterMempool?.getClusterInfo(txid);
+        console.log(`\n    --- ${label}: ${txid} ---`);
+        console.log(`    In our mempool: ${inOurMempool}`);
+        if (inOurMempool) {
+          const tx = this.mempool[txid];
+          console.log(`    fee=${tx.fee} weight=${tx.weight} vsize=${tx.vsize} sigops=${tx.sigops} adjustedVsize=${tx.adjustedVsize}`);
+        }
+        if (info) {
+          console.log(`    clusterId=${info.clusterId} chunkIndex=${info.chunkIndex} chunkFeerate=${info.chunkFeerate.toFixed(6)}`);
+          const cluster = this.clusterMempool?.getCluster(info.clusterId);
+          if (cluster) {
+            console.log(`    Cluster has ${cluster.chunks.length} chunk(s):`);
+            for (let ci = 0; ci < cluster.chunks.length; ci++) {
+              const c = cluster.chunks[ci];
+              console.log(`      chunk[${ci}]: ${c.txs.length} txs, feerate=${c.feerate.toFixed(6)}`);
+              if (ci === info.chunkIndex) {
+                const chunkTxids = c.txs.map((idx: number) => cluster.txs[idx]?.txid).filter(Boolean);
+                for (const tid of chunkTxids) {
+                  const t = this.mempool[tid];
+                  const parents = t?.vin
+                    ?.filter(v => !v.is_coinbase && this.mempool[v.txid] && chunkTxids.includes(v.txid))
+                    .map(v => v.txid.substring(0, 12)) || [];
+                  console.log(`        ${tid.substring(0, 12)}  fee=${t?.fee} size=${(t?.weight || 0) / 4} sigops=${t?.sigops} parents=[${parents.join(', ')}]`);
+                }
+              }
+            }
+          }
+        } else {
+          console.log(`    NOT in our cluster mempool`);
+        }
+      }
+
+      for (const [label, txid] of [['Core', coreTx], ['Our', ourTx]] as const) {
+        try {
+          const coreCluster = await this.rpc.getMempoolCluster(txid);
+          console.log(`\n    --- Core cluster for ${label} tx ${txid.substring(0, 12)} ---`);
+          console.log(`    Raw response: ${JSON.stringify(coreCluster).substring(0, 2000)}`);
+        } catch (e: any) {
+          console.log(`    Failed to get Core cluster for ${label} tx: ${e.message}`);
+        }
+      }
+
+      this.stats.templateMismatches++;
+    } else if (sharedInCoreOrder.length === coreTxids.length && sharedInCoreOrder.length === ourTxids.length) {
+      console.log(`  PERFECT MATCH`);
+      this.stats.templateMatches++;
+    } else {
+      console.log(`  Ordering matches for shared txs, but sets differ`);
+      this.stats.templateMatches++;
+    }
+  }
+
+  private printStats(): void {
+    console.log('\n--- Cumulative Stats ---');
+    const total = this.stats.templateMatches + this.stats.templateMismatches;
+    console.log(`Template comparisons: ${total} (${this.stats.templateMatches} perfect, ${this.stats.templateMismatches} divergent)`);
+  }
+
+  private shutdown(): void {
+    console.log('\nShutting down...');
+    if (this.pollTimer) {
+      clearInterval(this.pollTimer);
+    }
+    this.printStats();
+    process.exit(0);
+  }
+}
+
+// ─── Entry Point ────────────────────────────────────────────────────────────
+
+const rpcConfig: RpcConfig = {
+  host: config.CORE_RPC.HOST,
+  port: config.CORE_RPC.PORT,
+  user: config.CORE_RPC.USERNAME,
+  pass: config.CORE_RPC.PASSWORD,
+};
+
+console.log(`Connecting to Bitcoin Core at ${rpcConfig.host}:${rpcConfig.port}`);
+
+const harness = new Harness(rpcConfig, cliOptions);
+harness.run().catch((err) => {
+  console.error('Harness failed:', err);
+  process.exit(1);
+});

backend/src/__fixtures__/mempool-config.template.json

@@ -32,6 +32,8 @@
     "AUDIT": true,
     "RUST_GBT": false,
     "LIMIT_GBT": false,
+    "CLUSTER_MEMPOOL": false,
+    "CLUSTER_MEMPOOL_INDEXING": false,
     "CPFP_INDEXING": true,
     "MAX_BLOCKS_BULK_QUERY": 999,
     "DISK_CACHE_BLOCK_INTERVAL": 999,

backend/src/__integration_tests__/blocks-repository.test.ts

@@ -40,7 +40,7 @@ describe('BlocksRepository Integration Tests', () => {
     });
 
     const block = await BlocksRepository.$getBlockByHeight(height);
-    
+
     expect(block).toBeDefined();
     expect(block!.height).toBe(height);
     expect(block!.id).toBe(blockHash);
@@ -58,7 +58,7 @@ describe('BlocksRepository Integration Tests', () => {
     });
 
     const block = await BlocksRepository.$getBlockByHash(blockHash);
-    
+
     expect(block).toBeDefined();
     expect(block!.id).toBe(blockHash);
     expect(block!.height).toBe(height);
@@ -71,36 +71,36 @@ describe('BlocksRepository Integration Tests', () => {
 
   test('should check for missing blocks in range', async () => {
     // Insert blocks with a gap
-    await insertTestBlock({ 
-      height: 800100, 
+    await insertTestBlock({
+      height: 800100,
       hash: '0000000000000000000100000000000000000000000000000000000000000001',
       poolId: defaultPoolId
     });
-    await insertTestBlock({ 
-      height: 800102, 
+    await insertTestBlock({
+      height: 800102,
       hash: '0000000000000000000100000000000000000000000000000000000000000003',
       poolId: defaultPoolId
     });
 
     const missingBlocks = await BlocksRepository.$getMissingBlocksBetweenHeights(800100, 800102);
-    
+
     expect(missingBlocks).toContain(800101);
   });
 
   test('should get latest block height', async () => {
-    await insertTestBlock({ 
-      height: 800200, 
+    await insertTestBlock({
+      height: 800200,
       hash: '0000000000000000000200000000000000000000000000000000000000000001',
       poolId: defaultPoolId
     });
-    await insertTestBlock({ 
-      height: 800201, 
+    await insertTestBlock({
+      height: 800201,
       hash: '0000000000000000000200000000000000000000000000000000000000000002',
       poolId: defaultPoolId
     });
 
     const height = await BlocksRepository.$mostRecentBlockHeight();
-    
+
     expect(height).toBe(800201);
   });
 
@@ -121,7 +121,7 @@ describe('BlocksRepository Integration Tests', () => {
     });
 
     const block = await BlocksRepository.$getBlockByHash(blockHash);
-    
+
     expect(block).toBeDefined();
     expect(block).not.toBeNull();
     // The pool should be populated with the test pool's data

backend/src/__integration_tests__/database-migration.test.ts

@@ -18,7 +18,7 @@ describe('Database Migration Integration Tests', () => {
   });
 
   test('should have schema version in state table', async () => {
-    const [result] = await DB.query<any>("SELECT number FROM state WHERE name = 'schema_version'");
+    const [result] = await DB.query<any>('SELECT number FROM state WHERE name = \'schema_version\'');
     expect(result).toHaveLength(1);
     expect(result[0].number).toBeGreaterThan(0);
   });
@@ -70,7 +70,7 @@ describe('Database Migration Integration Tests', () => {
        WHERE TABLE_SCHEMA = 'mempool_test' 
        AND TABLE_NAME = 'blocks'`
     );
-    
+
     const columnNames = columns.map((col: any) => col.COLUMN_NAME);
     expect(columnNames).toContain('height');
     expect(columnNames).toContain('hash');
@@ -87,7 +87,7 @@ describe('Database Migration Integration Tests', () => {
        WHERE TABLE_SCHEMA = 'mempool_test' 
        AND TABLE_NAME = 'pools'`
     );
-    
+
     const columnNames = columns.map((col: any) => col.COLUMN_NAME);
     expect(columnNames).toContain('id');
     expect(columnNames).toContain('name');

backend/src/__integration_tests__/pools-repository.test.ts

@@ -43,7 +43,7 @@ describe('PoolsRepository Integration Tests', () => {
     });
 
     const pool = await PoolsRepository.$getPool('antpool');
-    
+
     expect(pool).toBeDefined();
     expect(pool!.name).toBe('AntPool');
     expect(pool!.slug).toBe('antpool');
@@ -64,7 +64,7 @@ describe('PoolsRepository Integration Tests', () => {
     });
 
     const pools = await PoolsRepository.$getPools();
-    
+
     expect(pools.length).toBeGreaterThanOrEqual(3);
   });
 
@@ -77,7 +77,7 @@ describe('PoolsRepository Integration Tests', () => {
     });
 
     const pool = await PoolsRepository.$getPool('multi-address-pool', false);
-    
+
     expect(pool).toBeDefined();
     const poolAddresses = JSON.parse(pool!.addresses);
     expect(poolAddresses).toHaveLength(3);
@@ -93,7 +93,7 @@ describe('PoolsRepository Integration Tests', () => {
     });
 
     const pool = await PoolsRepository.$getPool('regex-pool', false);
-    
+
     expect(pool).toBeDefined();
     const poolRegexes = JSON.parse(pool!.regexes);
     expect(poolRegexes).toHaveLength(2);

backend/src/__integration_tests__/test-helpers.ts

@@ -45,7 +45,7 @@ export async function cleanupTestData(): Promise<void> {
   try {
     // Disable foreign key checks temporarily for faster cleanup
     await DB.query('SET FOREIGN_KEY_CHECKS = 0');
-    
+
     for (const table of tables) {
       try {
         // Use 'silent' error logging to avoid noise for optional tables that don't exist
@@ -55,7 +55,7 @@ export async function cleanupTestData(): Promise<void> {
         // Silently ignore - no need to log since these are expected for optional features
       }
     }
-    
+
     // Re-enable foreign key checks
     await DB.query('SET FOREIGN_KEY_CHECKS = 1');
   } catch (error) {
@@ -143,7 +143,7 @@ export async function insertTestBlock(blockData: {
   const size = blockData.size || 1000000;
   const weight = blockData.weight || 4000000;
   const txCount = blockData.tx_count || 2000;
-  
+
   await DB.query(
     `INSERT INTO blocks (
       height, hash, blockTimestamp, size, weight, tx_count, 

backend/src/__tests__/api/common.ts

@@ -30,11 +30,34 @@ describe('Common', () => {
         expect(Common.isNonStandard(tx)).toEqual(true);
       });
     });
-  
+
     test('should not misclassify as nonstandard transactions', () => {
       randomTransactions.forEach((tx) => {
         expect(Common.isNonStandard(tx)).toEqual(false);
       });
     });
   });
+
+  describe('Effective Fee Statistics', () => {
+    test('returns safe defaults for blocks with only coinbase', () => {
+      const coinbaseTx = { weight: 1000, fee: 0, txid: 'coinbase0' };
+      const result = Common.calcEffectiveFeeStatistics([coinbaseTx]);
+
+      expect(result.medianFee).toBe(0);
+      expect(result.feeRange).toEqual([0, 0, 0, 0, 0, 0, 0]);
+    });
+
+    test('excludes coinbase from fee stats when multiple txs', () => {
+      const coinbaseTx = { weight: 1000, fee: 0, txid: 'coinbase0' };
+      const tx1 = { weight: 400, fee: 100, txid: 'tx1' }; // vsize 100, rate 1 sat/vB
+      const tx2 = { weight: 400, fee: 250, txid: 'tx2' }; // vsize 100, rate 2.5 sat/vB
+
+      const result = Common.calcEffectiveFeeStatistics([coinbaseTx, tx1, tx2]);
+
+      // Verify that coinbase (fee 0) was excluded from stats
+      // Fee range min/max should be > 0 (not affected by coinbase's 0 fee)
+      expect(result.feeRange[0]).toBeGreaterThan(0); // min fee
+      expect(result.feeRange[6]).toBeGreaterThan(0); // max fee
+    });
+  });
 });

backend/src/__tests__/cluster-mempool/cluster-mempool.test.ts

@@ -0,0 +1,544 @@
+import { ClusterMempool } from '../../cluster-mempool/cluster-mempool';
+import { makeTx, txid } from './test-utils';
+import { MempoolTransactionExtended } from '../../mempool.interfaces';
+
+function buildMempool(txs: MempoolTransactionExtended[]): { [txid: string]: MempoolTransactionExtended } {
+  const mempool: { [txid: string]: MempoolTransactionExtended } = {};
+  for (const tx of txs) {
+    mempool[tx.txid] = tx;
+  }
+  return mempool;
+}
+
+describe('ClusterMempool', () => {
+  describe('constructor', () => {
+    it('should build clusters from mempool', () => {
+      const parentId = txid('a1');
+      const childId = txid('a2');
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        makeTx(childId, 5000, 100, [parentId]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(2);
+    });
+
+    it('should create separate clusters for unrelated txs', () => {
+      const mempool = buildMempool([
+        makeTx(txid('a1'), 100, 100),
+        makeTx(txid('b1'), 200, 100),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(2);
+    });
+  });
+
+  describe('getClusterInfo', () => {
+    it('should return cluster info for a known tx', () => {
+      const parentId = txid('a1');
+      const childId = txid('a2');
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        makeTx(childId, 5000, 100, [parentId]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const info = cm.getClusterInfo(parentId);
+      expect(info).not.toBeNull();
+      expect(info?.chunkFeerate).toBeGreaterThan(0);
+    });
+
+    it('should return null for unknown tx', () => {
+      const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterInfo(txid('zz'))).toBeNull();
+    });
+  });
+
+  describe('getCluster', () => {
+    it('should return cluster data with correct topology', () => {
+      const parentId = txid('a1');
+      const childId = txid('a2');
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        makeTx(childId, 5000, 100, [parentId]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const info = cm.getClusterInfo(parentId);
+      expect(info).not.toBeNull();
+      const data = cm.getCluster(info!.clusterId);
+      expect(data).not.toBeNull();
+      expect(data!.txs.length).toBe(2);
+      expect(data!.chunks.length).toBeGreaterThan(0);
+    });
+
+    it('should return null for unknown cluster id', () => {
+      const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getCluster(9999)).toBeNull();
+    });
+  });
+
+  describe('applyMempoolChange', () => {
+    it('should handle adding a new singleton tx', () => {
+      const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+      const initialCount = cm.getClusterCount();
+
+      cm.applyMempoolChange({
+        added: [makeTx(txid('b1'), 200, 100)],
+        removed: [],
+        accelerations: {},
+      });
+
+      expect(cm.getClusterCount()).toBe(initialCount + 1);
+      expect(cm.getTxCount()).toBe(2);
+    });
+
+    it('should skip duplicate tx additions', () => {
+      const mempool = buildMempool([]);
+      const cm = new ClusterMempool(mempool);
+      const tx = makeTx(txid('a1'), 100, 100);
+      mempool[tx.txid] = tx;
+
+      cm.applyMempoolChange({
+        added: [tx, tx],
+        removed: [],
+        accelerations: {},
+      });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(1);
+      expect(cm.getBlocks(1)[0].txids).toEqual([tx.txid]);
+    });
+
+    it('should handle removing a tx', () => {
+      const parentId = txid('a1');
+      const childId = txid('a2');
+      const childTx = makeTx(childId, 5000, 100, [parentId]);
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        childTx,
+      ]);
+      const cm = new ClusterMempool(mempool);
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [childTx],
+        accelerations: {},
+      });
+
+      expect(cm.getTxCount()).toBe(1);
+      expect(cm.getClusterInfo(childId)).toBeNull();
+      expect(cm.getClusterInfo(parentId)).not.toBeNull();
+    });
+
+    it('should clean spentBy when removed tx is already missing from mempool', () => {
+      const parentId = txid('a1');
+      const childId = txid('a2');
+      const childTx = makeTx(childId, 5000, 100, [parentId]);
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        childTx,
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const spentBy = (cm as unknown as { spentBy: Map<string, string> }).spentBy;
+
+      expect(spentBy.get(`${parentId}:0`)).toBe(childId);
+
+      delete mempool[childId];
+      cm.applyMempoolChange({
+        added: [],
+        removed: [childTx],
+        accelerations: {},
+      });
+
+      expect(spentBy.has(`${parentId}:0`)).toBe(false);
+    });
+
+    it('should not delete spentBy for a replacement transaction', () => {
+      const parentId = txid('a1');
+      const replacedId = txid('a2');
+      const replacementId = txid('a3');
+      const replacedTx = makeTx(replacedId, 5000, 100, [parentId]);
+      const replacementTx = makeTx(replacementId, 6000, 100, [parentId]);
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        replacementTx,
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const spentBy = (cm as unknown as { spentBy: Map<string, string> }).spentBy;
+
+      expect(spentBy.get(`${parentId}:0`)).toBe(replacementId);
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [replacedTx],
+        accelerations: {},
+      });
+
+      expect(spentBy.get(`${parentId}:0`)).toBe(replacementId);
+    });
+
+    it('should split cluster when middle tx is removed', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const bTx = makeTx(b, 200, 100, [a]);
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        bTx,
+        makeTx(c, 300, 100, [b]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [bTx],
+        accelerations: {},
+      });
+
+      expect(cm.getTxCount()).toBe(2);
+      const infoA = cm.getClusterInfo(a);
+      const infoC = cm.getClusterInfo(c);
+      expect(infoA).not.toBeNull();
+      expect(infoC).not.toBeNull();
+      expect(infoA!.clusterId).not.toBe(infoC!.clusterId);
+    });
+
+    it('should handle fee changes via acceleration', () => {
+      const parentId = txid('a1');
+      const childId = txid('a2');
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        makeTx(childId, 100, 100, [parentId]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const infoBefore = cm.getClusterInfo(childId);
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [],
+        accelerations: { [childId]: { feeDelta: 49900 } },
+      });
+
+      const infoAfter = cm.getClusterInfo(childId);
+      expect(infoAfter).not.toBeNull();
+      expect(infoAfter!.clusterId).not.toBe(infoBefore!.clusterId);
+    });
+
+    it('should merge clusters when new tx connects them', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        makeTx(b, 200, 100),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(2);
+
+      const bridgeTx = makeTx(txid('c1'), 300, 100, [a, b]);
+      cm.applyMempoolChange({
+        added: [bridgeTx],
+        removed: [],
+        accelerations: {},
+      });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(3);
+    });
+  });
+
+  describe('cluster merging', () => {
+    it('should merge 3 separate clusters when new tx bridges them', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        makeTx(b, 200, 100),
+        makeTx(c, 300, 100),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(3);
+
+      const bridge = makeTx(txid('d1'), 400, 100, [a, b, c]);
+      mempool[bridge.txid] = bridge;
+      cm.applyMempoolChange({ added: [bridge], removed: [], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(4);
+    });
+
+    it('should grow cluster by 1 when new tx has parents in same cluster', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        makeTx(b, 200, 100, [a]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+
+      const c = makeTx(txid('c1'), 300, 100, [a]);
+      mempool[c.txid] = c;
+      cm.applyMempoolChange({ added: [c], removed: [], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(3);
+    });
+
+    it('should grow chain incrementally', () => {
+      const a = txid('a1');
+      const mempool = buildMempool([makeTx(a, 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+
+      const b = makeTx(txid('b1'), 200, 100, [a]);
+      mempool[b.txid] = b;
+      cm.applyMempoolChange({ added: [b], removed: [], accelerations: {} });
+
+      const c = makeTx(txid('c1'), 300, 100, [txid('b1')]);
+      mempool[c.txid] = c;
+      cm.applyMempoolChange({ added: [c], removed: [], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(3);
+    });
+  });
+
+  describe('cluster splitting', () => {
+    it('should split star into singletons when center is removed', () => {
+      const center = txid('center');
+      const leaves = Array.from({ length: 5 }, (_, i) => txid(`leaf${i}`));
+      const centerTx = makeTx(center, 100, 100);
+      for (let i = 1; i < 5; i++) {
+        centerTx.vout.push({
+          scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000,
+        });
+      }
+      const leafTxs = leaves.map((l, i) => {
+        const tx = makeTx(l, 200, 100, [center]);
+        tx.vin[0].vout = i;
+        return tx;
+      });
+      const mempool = buildMempool([centerTx, ...leafTxs]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+
+      cm.applyMempoolChange({ added: [], removed: [centerTx], accelerations: {} });
+
+      expect(cm.getTxCount()).toBe(5);
+      expect(cm.getClusterCount()).toBe(5);
+    });
+
+    it('should shrink cluster without splitting when leaf is removed', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const aTx = makeTx(a, 100, 100);
+      aTx.vout.push({ scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000 });
+      const bTx = makeTx(b, 200, 100, [a]);
+      bTx.vin[0].vout = 0;
+      const cTx = makeTx(c, 300, 100, [a]);
+      cTx.vin[0].vout = 1;
+      const mempool = buildMempool([aTx, bTx, cTx]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+
+      cm.applyMempoolChange({ added: [], removed: [cTx], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(2);
+    });
+
+    it('should produce singleton when tx is removed from 2-tx cluster', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const bTx = makeTx(b, 200, 100, [a]);
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        bTx,
+      ]);
+      const cm = new ClusterMempool(mempool);
+
+      cm.applyMempoolChange({ added: [], removed: [bTx], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(1);
+      expect(cm.getClusterInfo(a)).not.toBeNull();
+    });
+
+    it('should create 3+ components when removing a tx that bridges multiple subgraphs', () => {
+      const hub = txid('hub');
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const hubTx = makeTx(hub, 100, 100);
+      hubTx.vout.push(
+        { scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000 },
+        { scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000 },
+      );
+      const txA = makeTx(a, 200, 100, [hub]);
+      txA.vin[0].vout = 0;
+      const txB = makeTx(b, 300, 100, [hub]);
+      txB.vin[0].vout = 1;
+      const txC = makeTx(c, 400, 100, [hub]);
+      txC.vin[0].vout = 2;
+      const mempool = buildMempool([hubTx, txA, txB, txC]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+
+      cm.applyMempoolChange({ added: [], removed: [hubTx], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(3);
+      expect(cm.getTxCount()).toBe(3);
+    });
+  });
+
+  describe('fee changes via accelerations', () => {
+    it('should increase chunk feerate when acceleration added', () => {
+      const a = txid('a1');
+      const mempool = buildMempool([makeTx(a, 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+      const infoBefore = cm.getClusterInfo(a);
+      expect(infoBefore).not.toBeNull();
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [],
+        accelerations: { [a]: { feeDelta: 9900 } },
+      });
+
+      const infoAfter = cm.getClusterInfo(a);
+      expect(infoAfter).not.toBeNull();
+      expect(infoAfter!.chunkFeerate).toBeGreaterThan(infoBefore!.chunkFeerate);
+    });
+
+    it('should decrease chunk feerate when acceleration removed', () => {
+      const a = txid('a1');
+      const mempool = buildMempool([makeTx(a, 100, 100)]);
+      const cm = new ClusterMempool(mempool, { [a]: { feeDelta: 9900 } });
+      const infoBefore = cm.getClusterInfo(a);
+      expect(infoBefore).not.toBeNull();
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [],
+        accelerations: {},
+      });
+
+      const infoAfter = cm.getClusterInfo(a);
+      expect(infoAfter).not.toBeNull();
+      expect(infoAfter!.chunkFeerate).toBeLessThan(infoBefore!.chunkFeerate);
+    });
+
+    it('should reorder chunks when acceleration shifts priorities', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        makeTx(b, 200, 100, [a]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const infoBBefore = cm.getClusterInfo(b);
+      expect(infoBBefore).not.toBeNull();
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [],
+        accelerations: { [b]: { feeDelta: 49800 } },
+      });
+
+      const infoBAfter = cm.getClusterInfo(b);
+      expect(infoBAfter).not.toBeNull();
+      expect(infoBAfter!.chunkFeerate).toBeGreaterThan(infoBBefore!.chunkFeerate);
+    });
+  });
+
+  describe('getBlocks', () => {
+    it('should return projected blocks', () => {
+      const txs: MempoolTransactionExtended[] = [];
+      for (let i = 0; i < 10; i++) {
+        txs.push(makeTx(txid(`t${i}`), 1000 * (i + 1), 100));
+      }
+      const mempool = buildMempool(txs);
+      const cm = new ClusterMempool(mempool);
+      const blocks = cm.getBlocks(3);
+      expect(blocks.length).toBeGreaterThan(0);
+      expect(blocks[0].txids.length).toBeGreaterThan(0);
+    });
+
+    it('should return empty array for empty mempool', () => {
+      const cm = new ClusterMempool({});
+      const blocks = cm.getBlocks(3);
+      expect(blocks.length).toBe(0);
+    });
+
+    it('should respect chunk ordering for single-cluster mempool', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const mempool = buildMempool([
+        makeTx(a, 3000, 100),
+        makeTx(b, 200, 100, [a]),
+        makeTx(c, 100, 100, [b]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const blocks = cm.getBlocks(1);
+      expect(blocks.length).toBe(1);
+      const txids = blocks[0].txids;
+      expect(txids.indexOf(a)).toBeLessThan(txids.indexOf(b));
+      expect(txids.indexOf(b)).toBeLessThan(txids.indexOf(c));
+    });
+
+    it('should maintain topological validity within blocks', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const d = txid('d1');
+      const aTx = makeTx(a, 1000, 100);
+      aTx.vout.push({ scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000 });
+      const bTx = makeTx(b, 500, 100, [a]);
+      bTx.vin[0].vout = 0;
+      const cTx = makeTx(c, 500, 100, [a]);
+      cTx.vin[0].vout = 1;
+      const mempool = buildMempool([aTx, bTx, cTx, makeTx(d, 200, 100, [b, c])]);
+      const cm = new ClusterMempool(mempool);
+      const blocks = cm.getBlocks(1);
+      const txids = blocks[0].txids;
+
+      expect(txids.indexOf(a)).toBeLessThan(txids.indexOf(b));
+      expect(txids.indexOf(a)).toBeLessThan(txids.indexOf(c));
+      expect(txids.indexOf(b)).toBeLessThan(txids.indexOf(d));
+      expect(txids.indexOf(c)).toBeLessThan(txids.indexOf(d));
+    });
+  });
+
+  describe('empty and degenerate cases', () => {
+    it('should handle empty diff with no changes', () => {
+      const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+      const countBefore = cm.getClusterCount();
+      const txCountBefore = cm.getTxCount();
+
+      cm.applyMempoolChange({ added: [], removed: [], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(countBefore);
+      expect(cm.getTxCount()).toBe(txCountBefore);
+    });
+
+    it('should not crash when removing nonexistent tx', () => {
+      const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [makeTx(txid('nonexistent'), 100, 100)],
+        accelerations: {},
+      });
+
+      expect(cm.getTxCount()).toBe(1);
+    });
+  });
+});

backend/src/__tests__/cluster-mempool/depgraph.test.ts

@@ -0,0 +1,494 @@
+import { DepGraph, sortTopological, subgraph } from '../../cluster-mempool/depgraph';
+import { buildChain, buildFanOut, buildDiamond, buildStar } from './test-utils';
+
+describe('DepGraph', () => {
+  describe('addTransaction', () => {
+    it('should add a transaction and return a ClusterTx', () => {
+      const dg = new DepGraph();
+      const tx = dg.addTransaction('tx0', 1000, 100);
+      expect(dg.size).toBe(1);
+      expect(tx.effectiveFee).toBe(1000);
+      expect(tx.weight).toBe(100);
+      expect(tx.txid).toBe('tx0');
+    });
+
+    it('should assign distinct ClusterTx objects', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      expect(a).not.toBe(b);
+      expect(b).not.toBe(c);
+      expect(dg.size).toBe(3);
+    });
+
+    it('should include self in ancestors and descendants', () => {
+      const dg = new DepGraph();
+      const tx = dg.addTransaction('tx0', 100, 10);
+      expect(tx.ancestors.has(tx)).toBe(true);
+      expect(tx.descendants.has(tx)).toBe(true);
+    });
+
+    it('should handle large clusters', () => {
+      const dg = new DepGraph();
+      for (let i = 0; i < 100; i++) {
+        dg.addTransaction(`tx${i}`, 100, 10);
+      }
+      expect(dg.size).toBe(100);
+    });
+  });
+
+  describe('addDependency', () => {
+    it('should establish parent-child relationship', () => {
+      const dg = new DepGraph();
+      const parent = dg.addTransaction('parent', 100, 10);
+      const child = dg.addTransaction('child', 200, 20);
+      dg.addDependency(parent, child);
+
+      expect(child.ancestors.has(parent)).toBe(true);
+      expect(parent.descendants.has(child)).toBe(true);
+      expect(child.parents.has(parent)).toBe(true);
+      expect(parent.children.has(child)).toBe(true);
+    });
+
+    it('should propagate ancestors transitively', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+
+      expect(c.ancestors.has(a)).toBe(true);
+      expect(c.ancestors.has(b)).toBe(true);
+      expect(c.ancestors.has(c)).toBe(true);
+
+      expect(a.descendants.has(a)).toBe(true);
+      expect(a.descendants.has(b)).toBe(true);
+      expect(a.descendants.has(c)).toBe(true);
+    });
+
+    it('should handle diamond dependencies', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      const d = dg.addTransaction('d', 400, 40);
+      dg.addDependency(a, b);
+      dg.addDependency(a, c);
+      dg.addDependency(b, d);
+      dg.addDependency(c, d);
+
+      expect(d.ancestors.size).toBe(4);
+      expect(a.descendants.size).toBe(4);
+    });
+  });
+
+  describe('removeTransactions', () => {
+    it('should remove transactions and update sets', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+
+      dg.removeTransactions(new Set([b]));
+      expect(dg.size).toBe(2);
+      expect(dg.hasTx(b)).toBe(false);
+      expect(c.ancestors.has(b)).toBe(false);
+      expect(a.descendants.has(b)).toBe(false);
+    });
+
+    it('should handle slot reuse after removal', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      dg.addTransaction('b', 200, 20);
+      dg.removeTransactions(new Set([a]));
+      const c = dg.addTransaction('c', 300, 30);
+      expect(dg.size).toBe(2);
+      expect(c.txid).toBe('c');
+    });
+  });
+
+  describe('dependsOn (via ancestors)', () => {
+    it('should correctly identify dependencies', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+
+      expect(b.ancestors.has(a)).toBe(true);
+      expect(a.ancestors.has(b)).toBe(false);
+      expect(c.ancestors.has(a)).toBe(false);
+    });
+  });
+
+  describe('findConnectedComponents', () => {
+    it('should find a single component', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addDependency(a, b);
+
+      const components = dg.findConnectedComponents();
+      expect(components.length).toBe(1);
+      expect(components[0].size).toBe(2);
+    });
+
+    it('should find multiple disconnected components', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      const d = dg.addTransaction('d', 400, 40);
+      dg.addDependency(a, b);
+      dg.addDependency(c, d);
+
+      const components = dg.findConnectedComponents();
+      expect(components.length).toBe(2);
+    });
+
+    it('should handle isolated transactions', () => {
+      const dg = new DepGraph();
+      dg.addTransaction('a', 100, 10);
+      dg.addTransaction('b', 200, 20);
+      dg.addTransaction('c', 300, 30);
+
+      const components = dg.findConnectedComponents();
+      expect(components.length).toBe(3);
+    });
+  });
+
+  describe('parents / children (direct)', () => {
+    it('should return only direct parents, not transitive', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+
+      expect(c.parents.has(b)).toBe(true);
+      expect(c.parents.has(a)).toBe(false);
+    });
+
+    it('should return only direct children', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+
+      expect(a.children.has(b)).toBe(true);
+      expect(a.children.has(c)).toBe(false);
+    });
+  });
+
+  describe('appendTopo', () => {
+    it('should output in topological order', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+
+      const output = sortTopological(new Set([c, a, b]));
+      expect(output.indexOf(a)).toBeLessThan(output.indexOf(b));
+      expect(output.indexOf(b)).toBeLessThan(output.indexOf(c));
+    });
+  });
+
+  describe('restrict', () => {
+    it('should create a subgraph with correct deps', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+
+      const { depgraph: sub, txMap } = subgraph(new Set([a, b]));
+      expect(sub.size).toBe(2);
+      const newA = txMap.get(a)!;
+      const newB = txMap.get(b)!;
+      expect(newB.ancestors.has(newA)).toBe(true);
+    });
+  });
+
+  describe('graceful error handling', () => {
+    it('should no-op addDependency with non-member txs', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const dg2 = new DepGraph();
+      const foreign = dg2.addTransaction('foreign', 200, 20);
+      dg.addDependency(a, foreign);
+      dg.addDependency(foreign, a);
+      expect(a.ancestors.size).toBe(1);
+    });
+  });
+
+  describe('deep chain topology', () => {
+    it('should track ancestors and descendants at each depth', () => {
+      const { depgraph, txs } = buildChain(20, 100, 10);
+      expect(depgraph.size).toBe(20);
+
+      expect(txs[19].ancestors.size).toBe(20);
+      expect(txs[0].descendants.size).toBe(20);
+
+      expect(txs[10].ancestors.size).toBe(11);
+      expect(txs[10].descendants.size).toBe(10);
+
+      expect(txs[10].parents.size).toBe(1);
+      expect(txs[10].parents.has(txs[9])).toBe(true);
+      expect(txs[10].children.size).toBe(1);
+      expect(txs[10].children.has(txs[11])).toBe(true);
+    });
+  });
+
+  describe('wide fan-out topology', () => {
+    it('should track parent-children relationships for 1 parent with 10 children', () => {
+      const { depgraph, parent, children } = buildFanOut(10, 100, 10, 50, 10);
+      expect(depgraph.size).toBe(11);
+      expect(parent.children.size).toBe(10);
+      expect(parent.descendants.size).toBe(11);
+
+      for (const child of children) {
+        expect(child.ancestors.size).toBe(2);
+        expect(child.ancestors.has(parent)).toBe(true);
+        expect(child.parents.size).toBe(1);
+        expect(child.parents.has(parent)).toBe(true);
+      }
+    });
+  });
+
+  describe('wide fan-in topology', () => {
+    it('should track many parents converging to one child', () => {
+      const dg = new DepGraph();
+      const parents: any[] = [];
+      for (let i = 0; i < 10; i++) {
+        parents.push(dg.addTransaction(`p${i}`, 100, 10));
+      }
+      const child = dg.addTransaction('child', 500, 50);
+      for (const p of parents) {
+        dg.addDependency(p, child);
+      }
+
+      expect(child.parents.size).toBe(10);
+      expect(child.ancestors.size).toBe(11);
+      for (const p of parents) {
+        expect(p.descendants.has(child)).toBe(true);
+      }
+    });
+  });
+
+  describe('multiple diamonds in sequence', () => {
+    it('should handle A→B,C→D→E,F→G topology', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 100, 10);
+      const c = dg.addTransaction('c', 100, 10);
+      const d = dg.addTransaction('d', 100, 10);
+      const e = dg.addTransaction('e', 100, 10);
+      const f = dg.addTransaction('f', 100, 10);
+      const g = dg.addTransaction('g', 100, 10);
+      dg.addDependency(a, b);
+      dg.addDependency(a, c);
+      dg.addDependency(b, d);
+      dg.addDependency(c, d);
+      dg.addDependency(d, e);
+      dg.addDependency(d, f);
+      dg.addDependency(e, g);
+      dg.addDependency(f, g);
+
+      expect(g.ancestors.size).toBe(7);
+      expect(a.descendants.size).toBe(7);
+      expect(d.parents.size).toBe(2);
+      expect(d.children.size).toBe(2);
+    });
+  });
+
+  describe('disconnected subgraphs', () => {
+    it('should coexist in one DepGraph', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addDependency(a, b);
+
+      const c = dg.addTransaction('c', 300, 30);
+      const d = dg.addTransaction('d', 400, 40);
+      dg.addDependency(c, d);
+
+      const e = dg.addTransaction('e', 500, 50);
+
+      expect(dg.size).toBe(5);
+      expect(b.ancestors.has(c)).toBe(false);
+      expect(a.descendants.has(d)).toBe(false);
+      expect(e.ancestors.size).toBe(1);
+      expect(dg.findConnectedComponents().length).toBe(3);
+    });
+  });
+
+  describe('removeTransactions edge cases', () => {
+    it('should remove a leaf without affecting siblings', () => {
+      const { depgraph, parent, children } = buildFanOut(3, 100, 10, 50, 10);
+      depgraph.removeTransactions(new Set([children[2]]));
+      expect(depgraph.size).toBe(3);
+      expect(parent.children.size).toBe(2);
+      expect(depgraph.hasTx(children[0])).toBe(true);
+      expect(depgraph.hasTx(children[1])).toBe(true);
+    });
+
+    it('should remove a root without affecting unrelated txs', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addDependency(a, b);
+      dg.removeTransactions(new Set([a]));
+      expect(dg.size).toBe(1);
+      expect(dg.hasTx(a)).toBe(false);
+      expect(b.ancestors.size).toBe(1);
+      expect(b.ancestors.has(b)).toBe(true);
+    });
+
+    it('should break transitive edges when middle of chain is removed', () => {
+      const { depgraph, txs } = buildChain(5, 100, 10);
+      depgraph.removeTransactions(new Set([txs[2]]));
+      expect(depgraph.size).toBe(4);
+      expect(txs[0].descendants.has(txs[3])).toBe(false);
+      expect(txs[0].descendants.has(txs[1])).toBe(true);
+      expect(txs[3].ancestors.has(txs[0])).toBe(false);
+      expect(txs[3].descendants.has(txs[4])).toBe(true);
+    });
+
+    it('should handle batch removal of multiple txs', () => {
+      const { depgraph, txs } = buildChain(5, 100, 10);
+      depgraph.removeTransactions(new Set([txs[1], txs[3]]));
+      expect(depgraph.size).toBe(3);
+      expect(depgraph.hasTx(txs[0])).toBe(true);
+      expect(depgraph.hasTx(txs[2])).toBe(true);
+      expect(depgraph.hasTx(txs[4])).toBe(true);
+    });
+
+    it('should result in empty graph when all txs removed', () => {
+      const { depgraph, txs } = buildChain(3, 100, 10);
+      depgraph.removeTransactions(new Set(txs));
+      expect(depgraph.size).toBe(0);
+      expect(depgraph.getTxs().size).toBe(0);
+    });
+
+    it('should produce clean state when new tx is added after removal', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addDependency(a, b);
+      dg.removeTransactions(new Set([a]));
+
+      const c = dg.addTransaction('c', 300, 30);
+      expect(c.ancestors.size).toBe(1);
+      expect(c.descendants.size).toBe(1);
+      expect(c.ancestors.has(c)).toBe(true);
+      expect(b.ancestors.has(c)).toBe(false);
+    });
+  });
+
+  describe('findConnectedComponents after removal', () => {
+    it('should split into 2 components when bridge tx is removed', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      const d = dg.addTransaction('d', 400, 40);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+      dg.addDependency(d, c);
+
+      dg.removeTransactions(new Set([b]));
+      const components = dg.findConnectedComponents();
+      expect(components.length).toBe(2);
+      const sizes = components.map(comp => comp.size).sort((x, y) => x - y);
+      expect(sizes).toEqual([1, 2]);
+    });
+
+    it('should produce N singletons when center of star is removed', () => {
+      const { depgraph, center, leaves } = buildStar(5, 100, 10, 50, 10);
+      depgraph.removeTransactions(new Set([center]));
+      const components = depgraph.findConnectedComponents();
+      expect(components.length).toBe(5);
+      for (const comp of components) {
+        expect(comp.size).toBe(1);
+      }
+    });
+
+    it('should remain 1 component when non-bridge tx is removed', () => {
+      const { depgraph, txs } = buildDiamond([100, 200, 300, 400], [10, 20, 30, 40]);
+      depgraph.removeTransactions(new Set([txs[1]]));
+      const components = depgraph.findConnectedComponents();
+      expect(components.length).toBe(1);
+    });
+  });
+
+  describe('restrict edge cases', () => {
+    it('should restrict to a single tx', () => {
+      const { depgraph, txs } = buildChain(3, 100, 10);
+      const { depgraph: sub } = subgraph(new Set([txs[1]]));
+      expect(sub.size).toBe(1);
+    });
+
+    it('should preserve edges within partial chain subset', () => {
+      const { depgraph, txs } = buildChain(5, 100, 10);
+      const subset = new Set([txs[0], txs[1], txs[2]]);
+      const { depgraph: sub, txMap } = subgraph(subset);
+      expect(sub.size).toBe(3);
+
+      const newA = txMap.get(txs[0]);
+      const newB = txMap.get(txs[1]);
+      const newC = txMap.get(txs[2]);
+      if (!newA || !newB || !newC) {
+        throw new Error('txMap missing entries');
+      }
+      expect(newB.ancestors.has(newA)).toBe(true);
+      expect(newC.ancestors.has(newB)).toBe(true);
+    });
+  });
+
+  describe('sortTopological edge cases', () => {
+    it('should handle subset with no internal dependencies', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+
+      const output = sortTopological(new Set([a, b, c]));
+      expect(output.length).toBe(3);
+      expect(new Set(output).size).toBe(3);
+    });
+
+    it('should handle single-tx subset', () => {
+      const { txs } = buildChain(3, 100, 10);
+      const output = sortTopological(new Set([txs[1]]));
+      expect(output).toEqual([txs[1]]);
+    });
+  });
+
+  describe('addDependency edge cases', () => {
+    it('should be idempotent when adding the same dependency twice', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addDependency(a, b);
+      dg.addDependency(a, b);
+      expect(b.ancestors.size).toBe(2);
+      expect(a.descendants.size).toBe(2);
+    });
+
+    it('should handle redundant edge when parent is already transitive ancestor', () => {
+      const { depgraph, txs } = buildChain(3, 100, 10);
+      depgraph.addDependency(txs[0], txs[2]);
+      expect(txs[2].ancestors.size).toBe(3);
+      expect(txs[2].parents.size).toBe(2);
+    });
+  });
+});

backend/src/__tests__/cluster-mempool/linearize.test.ts

@@ -0,0 +1,485 @@
+import { DepGraph } from '../../cluster-mempool/depgraph';
+import { chunkify, postLinearize, spanningForestLinearize, linearizeCluster } from '../../cluster-mempool/linearize';
+import { buildChain, buildFanOut, buildStar, verifyLinearization, verifyTopologicalOrder } from './test-utils';
+
+describe('chunkify', () => {
+  it('should create one chunk per tx when feerates are decreasing', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 300, 10);
+    const b = dg.addTransaction('b', 200, 10);
+    const c = dg.addTransaction('c', 100, 10);
+
+    const chunks = chunkify([a, b, c]);
+    expect(chunks.length).toBe(3);
+    expect(chunks[0].txs).toEqual([a]);
+    expect(chunks[1].txs).toEqual([b]);
+    expect(chunks[2].txs).toEqual([c]);
+  });
+
+  it('should merge all into one chunk when feerates are increasing', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 10);
+    const b = dg.addTransaction('b', 200, 10);
+    const c = dg.addTransaction('c', 300, 10);
+
+    const chunks = chunkify([a, b, c]);
+    expect(chunks.length).toBe(1);
+    expect(chunks[0].txs).toEqual([a, b, c]);
+    expect(chunks[0].fee).toBe(600);
+    expect(chunks[0].weight).toBe(30);
+  });
+
+  it('should NOT merge equal feerates (matching Core behavior)', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 10);
+    const b = dg.addTransaction('b', 100, 10);
+
+    const chunks = chunkify([a, b]);
+    expect(chunks.length).toBe(2);
+  });
+
+  it('should handle single transaction', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 500, 50);
+
+    const chunks = chunkify([a]);
+    expect(chunks.length).toBe(1);
+    expect(chunks[0].fee).toBe(500);
+    expect(chunks[0].weight).toBe(50);
+  });
+
+  it('should handle empty linearization', () => {
+    const chunks = chunkify([]);
+    expect(chunks.length).toBe(0);
+  });
+
+  it('should produce decreasing chunk feerates', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 10);
+    const b = dg.addTransaction('b', 50, 10);
+    const c = dg.addTransaction('c', 200, 10);
+    const d = dg.addTransaction('d', 30, 10);
+
+    const chunks = chunkify([a, c, b, d]);
+    for (let i = 1; i < chunks.length; i++) {
+      const prevRate = chunks[i - 1].fee / chunks[i - 1].weight;
+      const curRate = chunks[i].fee / chunks[i].weight;
+      expect(prevRate).toBeGreaterThanOrEqual(curRate);
+    }
+  });
+});
+
+describe('postLinearize', () => {
+  it('should improve a bad linearization', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 500, 100);
+
+    const result = postLinearize([a, b]);
+    expect(result[0]).toBe(b);
+    expect(result[1]).toBe(a);
+  });
+
+  it('should not violate dependencies', () => {
+    const dg = new DepGraph();
+    const parent = dg.addTransaction('parent', 100, 100);
+    const child = dg.addTransaction('child', 500, 100);
+    dg.addDependency(parent, child);
+
+    const result = postLinearize([parent, child]);
+    expect(result.indexOf(parent)).toBeLessThan(result.indexOf(child));
+  });
+
+  it('should handle already-optimal ordering', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 500, 100);
+    const b = dg.addTransaction('b', 100, 100);
+
+    const result = postLinearize([a, b]);
+    expect(result).toEqual([a, b]);
+  });
+});
+
+describe('spanningForestLinearize', () => {
+  it('should sort independent transactions by feerate', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 500, 100);
+    const c = dg.addTransaction('c', 300, 100);
+
+    const result = spanningForestLinearize(dg.getTxs(), 75000);
+    expect(result[0]).toBe(b);
+    expect(result[1]).toBe(c);
+    expect(result[2]).toBe(a);
+  });
+
+  it('should respect dependencies', () => {
+    const dg = new DepGraph();
+    const parent = dg.addTransaction('parent', 100, 100);
+    const child = dg.addTransaction('child', 500, 100);
+    dg.addDependency(parent, child);
+
+    const result = spanningForestLinearize(dg.getTxs(), 75000);
+    expect(result.indexOf(parent)).toBeLessThan(result.indexOf(child));
+  });
+
+  it('should handle CPFP pattern', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 10000, 100);
+    dg.addDependency(a, b);
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(1);
+    expect(chunks[0].txs.length).toBe(2);
+  });
+
+  it('should separate high and low feerate independent txs', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 100);
+    const b = dg.addTransaction('b', 100, 100);
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(2);
+    expect(chunks[0].txs).toContain(a);
+    expect(chunks[1].txs).toContain(b);
+  });
+
+  it('should handle single transaction', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 100);
+
+    const result = spanningForestLinearize(dg.getTxs(), 75000);
+    expect(result).toEqual([a]);
+  });
+
+  it('should handle empty graph', () => {
+    const dg = new DepGraph();
+    const result = spanningForestLinearize(dg.getTxs(), 75000);
+    expect(result).toEqual([]);
+  });
+});
+
+describe('minimize', () => {
+  it('should keep equal-feerate chain as individual chunks', () => {
+    const dg = new DepGraph();
+    const txs: any[] = [];
+    for (let i = 0; i < 5; i++) {
+      txs.push(dg.addTransaction(`tx${i}`, 19, 140));
+    }
+    for (let i = 1; i < 5; i++) {
+      dg.addDependency(txs[i - 1], txs[i]);
+    }
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(5);
+    for (const chunk of chunks) {
+      expect(chunk.txs.length).toBe(1);
+    }
+  });
+
+  it('should merge chain where child has strictly higher feerate', () => {
+    const dg = new DepGraph();
+    const parent = dg.addTransaction('parent', 100, 200);
+    const child = dg.addTransaction('child', 900, 100);
+    dg.addDependency(parent, child);
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(1);
+    expect(chunks[0].txs.length).toBe(2);
+  });
+
+  it('should split parent-child with equal feerate', () => {
+    const dg = new DepGraph();
+    const parent = dg.addTransaction('parent', 1720, 344);
+    const child = dg.addTransaction('child', 1240, 248);
+    dg.addDependency(parent, child);
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(2);
+  });
+
+  it('should split disconnected equal-feerate components', () => {
+    const dg = new DepGraph();
+    dg.addTransaction('a', 100, 100);
+    dg.addTransaction('b', 100, 100);
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(2);
+    expect(chunks[0].txs.length).toBe(1);
+    expect(chunks[1].txs.length).toBe(1);
+  });
+});
+
+describe('chunkify edge cases', () => {
+  it('should produce N separate chunks when all feerates are equal', () => {
+    const dg = new DepGraph();
+    const txs: any[] = [];
+    for (let i = 0; i < 5; i++) {
+      txs.push(dg.addTransaction(`tx${i}`, 100, 10));
+    }
+    const chunks = chunkify(txs);
+    expect(chunks.length).toBe(5);
+  });
+
+  it('should handle alternating high/low feerates', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 10);
+    const b = dg.addTransaction('b', 100, 10);
+    const c = dg.addTransaction('c', 1000, 10);
+    const d = dg.addTransaction('d', 100, 10);
+
+    const chunks = chunkify([a, b, c, d]);
+    for (let i = 1; i < chunks.length; i++) {
+      const prevRate = chunks[i - 1].fee * chunks[i].weight;
+      const curRate = chunks[i].fee * chunks[i - 1].weight;
+      expect(prevRate).toBeGreaterThanOrEqual(curRate);
+    }
+  });
+
+  it('should merge all when single very high feerate tx is at the end', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 10, 10);
+    const b = dg.addTransaction('b', 10, 10);
+    const c = dg.addTransaction('c', 10, 10);
+    const d = dg.addTransaction('d', 10000, 10);
+
+    const chunks = chunkify([a, b, c, d]);
+    expect(chunks.length).toBe(1);
+    expect(chunks[0].txs.length).toBe(4);
+  });
+
+  it('should maintain non-increasing feerates for 50+ tx linearization', () => {
+    const dg = new DepGraph();
+    const txs: any[] = [];
+    for (let i = 0; i < 50; i++) {
+      txs.push(dg.addTransaction(`tx${i}`, 5000 - i * 100, 100));
+    }
+    const chunks = chunkify(txs);
+    for (let i = 1; i < chunks.length; i++) {
+      const prevRate = chunks[i - 1].fee * chunks[i].weight;
+      const curRate = chunks[i].fee * chunks[i - 1].weight;
+      expect(prevRate).toBeGreaterThanOrEqual(curRate);
+    }
+  });
+
+  it('should handle zero-fee transaction', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 100);
+    const b = dg.addTransaction('b', 0, 100);
+
+    const chunks = chunkify([a, b]);
+    expect(chunks.length).toBe(2);
+    expect(chunks[1].fee).toBe(0);
+  });
+});
+
+describe('postLinearize edge cases', () => {
+  it('should sort three independent txs by feerate', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 500, 100);
+    const c = dg.addTransaction('c', 300, 100);
+
+    const result = postLinearize([a, c, b]);
+    expect(result[0]).toBe(b);
+    expect(result[2]).toBe(a);
+  });
+
+  it('should respect parent-child dependency even when child has higher feerate', () => {
+    const dg = new DepGraph();
+    const parent = dg.addTransaction('parent', 100, 100);
+    const child = dg.addTransaction('child', 1000, 100);
+    dg.addDependency(parent, child);
+
+    const result = postLinearize([parent, child]);
+    expect(result[0]).toBe(parent);
+    expect(result[1]).toBe(child);
+  });
+
+  it('should handle chain A→B→C with CPFP-like feerates', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 200, 100);
+    const c = dg.addTransaction('c', 10000, 100);
+    dg.addDependency(a, b);
+    dg.addDependency(b, c);
+
+    const result = postLinearize([a, b, c]);
+    verifyTopologicalOrder(result);
+  });
+});
+
+describe('SFL adversarial topologies', () => {
+  it('should handle comb pattern: one root with many children at different feerates', () => {
+    const dg = new DepGraph();
+    const root = dg.addTransaction('root', 100, 100);
+    for (let i = 0; i < 8; i++) {
+      const child = dg.addTransaction(`child${i}`, (i + 1) * 500, 100);
+      dg.addDependency(root, child);
+    }
+
+    const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000);
+    verifyLinearization(dg.getTxs(), linearization, chunks);
+  });
+
+  it('should handle inverted tree: many leaves → intermediates → root', () => {
+    const dg = new DepGraph();
+    const root = dg.addTransaction('root', 100, 100);
+    const mid1 = dg.addTransaction('mid1', 200, 100);
+    const mid2 = dg.addTransaction('mid2', 300, 100);
+    dg.addDependency(root, mid1);
+    dg.addDependency(root, mid2);
+
+    for (let i = 0; i < 4; i++) {
+      const leaf = dg.addTransaction(`leaf${i}`, 5000, 100);
+      dg.addDependency(i < 2 ? mid1 : mid2, leaf);
+    }
+
+    const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000);
+    verifyLinearization(dg.getTxs(), linearization, chunks);
+  });
+
+  it('should handle two parallel chains with shared root', () => {
+    const dg = new DepGraph();
+    const root = dg.addTransaction('root', 100, 100);
+
+    let prev1: any = root;
+    for (let i = 0; i < 5; i++) {
+      const tx = dg.addTransaction(`chain1_${i}`, 200, 100);
+      dg.addDependency(prev1, tx);
+      prev1 = tx;
+    }
+
+    let prev2: any = root;
+    for (let i = 0; i < 3; i++) {
+      const tx = dg.addTransaction(`chain2_${i}`, 300, 100);
+      dg.addDependency(prev2, tx);
+      prev2 = tx;
+    }
+
+    const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000);
+    verifyLinearization(dg.getTxs(), linearization, chunks);
+  });
+
+  it('should handle deep CPFP: low-fee chain with high-fee tip', () => {
+    const { depgraph, txs } = buildChain(6, 10, 100);
+    txs[5].effectiveFee = 50000;
+
+    const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), 75000);
+    verifyLinearization(depgraph.getTxs(), linearization, chunks);
+    expect(chunks[0].txs.length).toBeGreaterThan(1);
+  });
+
+  it('should find better result than ancestor-feerate for overlapping high-feerate subsets', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 100, 100);
+    const c = dg.addTransaction('c', 10000, 100);
+    dg.addDependency(a, c);
+    dg.addDependency(b, c);
+
+    const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000 );
+    verifyLinearization(dg.getTxs(), linearization, chunks);
+    const firstChunkFee = chunks[0].fee;
+    const firstChunkSize = chunks[0].weight;
+    expect(firstChunkFee / firstChunkSize).toBeGreaterThan(100 / 100);
+  });
+});
+
+describe('linearizeCluster', () => {
+  it('should produce valid topological linearization', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 10);
+    const b = dg.addTransaction('b', 200, 20);
+    const c = dg.addTransaction('c', 300, 30);
+    dg.addDependency(a, b);
+    dg.addDependency(b, c);
+
+    const { linearization } = linearizeCluster(dg.getTxs(), 75000);
+    expect(linearization.indexOf(a)).toBeLessThan(linearization.indexOf(b));
+    expect(linearization.indexOf(b)).toBeLessThan(linearization.indexOf(c));
+  });
+
+  it('should produce monotonically decreasing chunk feerates', () => {
+    const dg = new DepGraph();
+    for (let i = 0; i < 10; i++) {
+      dg.addTransaction(`tx${i}`, Math.floor(Math.random() * 10000) + 100, Math.floor(Math.random() * 500) + 50);
+    }
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    for (let i = 1; i < chunks.length; i++) {
+      const prevRate = chunks[i - 1].fee * chunks[i].weight;
+      const curRate = chunks[i].fee * chunks[i - 1].weight;
+      expect(prevRate).toBeGreaterThanOrEqual(curRate);
+    }
+  });
+
+  it('should handle complex diamond dependency graph', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 100);
+    const b = dg.addTransaction('b', 500, 100);
+    const c = dg.addTransaction('c', 100, 100);
+    const d = dg.addTransaction('d', 300, 100);
+
+    dg.addDependency(a, b);
+    dg.addDependency(a, c);
+    dg.addDependency(b, d);
+    dg.addDependency(c, d);
+
+    const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000);
+    verifyLinearization(dg.getTxs(), linearization, chunks);
+  });
+
+  it('should produce at-least-as-good result when given suboptimal hint', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 500, 100);
+    const b = dg.addTransaction('b', 100, 100);
+    const c = dg.addTransaction('c', 1000, 100);
+
+    const suboptimal = [b, a, c];
+    const { chunks: hintChunks } = linearizeCluster(dg.getTxs(), 75000, suboptimal);
+    const { chunks: freshChunks } = linearizeCluster(dg.getTxs(), 75000);
+
+    const hintFirstFeerate = hintChunks[0].fee * freshChunks[0].weight;
+    const freshFirstFeerate = freshChunks[0].fee * hintChunks[0].weight;
+    expect(hintFirstFeerate).toBeGreaterThanOrEqual(freshFirstFeerate - 1);
+  });
+
+  it('should preserve an already-optimal linearization', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 100);
+    const b = dg.addTransaction('b', 500, 100);
+    const c = dg.addTransaction('c', 100, 100);
+
+    const optimal = [a, b, c];
+    const { linearization } = linearizeCluster(dg.getTxs(), 75000, optimal);
+    expect(linearization).toEqual(optimal);
+  });
+
+  it('should produce valid linearizations on repeated calls', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 100, 100);
+    const c = dg.addTransaction('c', 100, 100);
+    dg.addDependency(a, c);
+    dg.addDependency(b, c);
+
+    const result1 = linearizeCluster(dg.getTxs(), 75000);
+    const result2 = linearizeCluster(dg.getTxs(), 75000);
+    verifyLinearization(dg.getTxs(), result1.linearization, result1.chunks);
+    verifyLinearization(dg.getTxs(), result2.linearization, result2.chunks);
+  });
+
+  it('should pass invariant checks for fan-out topology', () => {
+    const { depgraph } = buildFanOut(6, 100, 100, 500, 100);
+    const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), 75000);
+    verifyLinearization(depgraph.getTxs(), linearization, chunks);
+  });
+
+  it('should pass invariant checks for star topology', () => {
+    const { depgraph } = buildStar(5, 100, 100, 300, 100);
+    const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), 75000);
+    verifyLinearization(depgraph.getTxs(), linearization, chunks);
+  });
+});

backend/src/__tests__/cluster-mempool/test-utils.ts

@@ -0,0 +1,175 @@
+import { MempoolTransactionExtended } from '../../mempool.interfaces';
+import { ClusterTx, DepGraph } from '../../cluster-mempool/depgraph';
+import { LinearizationChunk } from '../../cluster-mempool/linearize';
+
+export function makeTx(
+  txid: string,
+  fee: number,
+  vsize: number,
+  parentTxids: string[] = [],
+): MempoolTransactionExtended {
+  const vin = parentTxids.length > 0
+    ? parentTxids.map(ptxid => ({
+        txid: ptxid,
+        vout: 0,
+        is_coinbase: false,
+        scriptsig: '',
+        scriptsig_asm: '',
+        inner_redeemscript_asm: '',
+        inner_witnessscript_asm: '',
+        sequence: 0,
+        witness: [] as string[],
+        prevout: null,
+      }))
+    : [{
+        txid: '0000000000000000000000000000000000000000000000000000000000000000',
+        vout: 0,
+        is_coinbase: false,
+        scriptsig: '',
+        scriptsig_asm: '',
+        inner_redeemscript_asm: '',
+        inner_witnessscript_asm: '',
+        sequence: 0,
+        witness: [] as string[],
+        prevout: null,
+      }];
+
+  return {
+    txid,
+    version: 2,
+    locktime: 0,
+    size: vsize,
+    weight: vsize * 4,
+    fee,
+    vin,
+    vout: [{
+      scriptpubkey: '',
+      scriptpubkey_asm: '',
+      scriptpubkey_type: 'v0_p2wpkh',
+      value: 50000,
+    }],
+    status: { confirmed: false },
+    vsize,
+    feePerVsize: fee / vsize,
+    effectiveFeePerVsize: fee / vsize,
+    order: 0,
+    sigops: 0,
+    adjustedVsize: vsize,
+    adjustedFeePerVsize: fee / vsize,
+  } as MempoolTransactionExtended;
+}
+
+export function txid(short: string): string {
+  return short.padStart(64, '0');
+}
+
+export function buildChain(
+  n: number,
+  baseFee: number,
+  baseSize: number,
+): { depgraph: DepGraph; txs: ClusterTx[] } {
+  const depgraph = new DepGraph();
+  const txs: ClusterTx[] = [];
+  for (let i = 0; i < n; i++) {
+    txs.push(depgraph.addTransaction(`chain_${i}`, baseFee, baseSize));
+  }
+  for (let i = 1; i < n; i++) {
+    depgraph.addDependency(txs[i - 1], txs[i]);
+  }
+  return { depgraph, txs };
+}
+
+export function buildFanOut(
+  nChildren: number,
+  parentFee: number,
+  parentSize: number,
+  childFee: number,
+  childSize: number,
+): { depgraph: DepGraph; parent: ClusterTx; children: ClusterTx[] } {
+  const depgraph = new DepGraph();
+  const parent = depgraph.addTransaction('fanout_parent', parentFee, parentSize);
+  const children: ClusterTx[] = [];
+  for (let i = 0; i < nChildren; i++) {
+    const child = depgraph.addTransaction(`fanout_child_${i}`, childFee, childSize);
+    depgraph.addDependency(parent, child);
+    children.push(child);
+  }
+  return { depgraph, parent, children };
+}
+
+export function buildDiamond(
+  fees: [number, number, number, number],
+  sizes: [number, number, number, number],
+): { depgraph: DepGraph; txs: [ClusterTx, ClusterTx, ClusterTx, ClusterTx] } {
+  const depgraph = new DepGraph();
+  const a = depgraph.addTransaction('diamond_a', fees[0], sizes[0]);
+  const b = depgraph.addTransaction('diamond_b', fees[1], sizes[1]);
+  const c = depgraph.addTransaction('diamond_c', fees[2], sizes[2]);
+  const d = depgraph.addTransaction('diamond_d', fees[3], sizes[3]);
+  depgraph.addDependency(a, b);
+  depgraph.addDependency(a, c);
+  depgraph.addDependency(b, d);
+  depgraph.addDependency(c, d);
+  return { depgraph, txs: [a, b, c, d] };
+}
+
+export function buildStar(
+  nLeaves: number,
+  centerFee: number,
+  centerSize: number,
+  leafFee: number,
+  leafSize: number,
+): { depgraph: DepGraph; center: ClusterTx; leaves: ClusterTx[] } {
+  const depgraph = new DepGraph();
+  const center = depgraph.addTransaction('star_center', centerFee, centerSize);
+  const leaves: ClusterTx[] = [];
+  for (let i = 0; i < nLeaves; i++) {
+    const leaf = depgraph.addTransaction(`star_leaf_${i}`, leafFee, leafSize);
+    depgraph.addDependency(center, leaf);
+    leaves.push(leaf);
+  }
+  return { depgraph, center, leaves };
+}
+
+export function verifyTopologicalOrder(ordering: ClusterTx[]): void {
+  const positionMap = new Map<ClusterTx, number>();
+  for (let i = 0; i < ordering.length; i++) {
+    positionMap.set(ordering[i], i);
+  }
+  for (const tx of ordering) {
+    for (const parent of tx.parents) {
+      const parentPos = positionMap.get(parent);
+      const childPos = positionMap.get(tx);
+      if (parentPos !== undefined && childPos !== undefined) {
+        expect(parentPos).toBeLessThan(childPos);
+      }
+    }
+  }
+}
+
+export function verifyLinearization(
+  txs: Set<ClusterTx>,
+  linearization: ClusterTx[],
+  chunks: LinearizationChunk[],
+): void {
+  expect(linearization.length).toBe(txs.size);
+  const linSet = new Set(linearization);
+  expect(linSet.size).toBe(linearization.length);
+  for (const tx of txs) {
+    expect(linSet.has(tx)).toBe(true);
+  }
+
+  verifyTopologicalOrder(linearization);
+
+  for (let i = 1; i < chunks.length; i++) {
+    const prevRate = chunks[i - 1].fee * chunks[i].weight;
+    const curRate = chunks[i].fee * chunks[i - 1].weight;
+    expect(prevRate).toBeGreaterThanOrEqual(curRate);
+  }
+
+  const chunkTxs = chunks.flatMap(c => c.txs);
+  expect(chunkTxs.length).toBe(linearization.length);
+  for (let i = 0; i < chunkTxs.length; i++) {
+    expect(chunkTxs[i]).toBe(linearization[i]);
+  }
+}

backend/src/__tests__/config.test.ts

@@ -45,6 +45,8 @@ describe('Mempool Backend Config', () => {
         AUDIT: false,
         RUST_GBT: true,
         LIMIT_GBT: false,
+        CLUSTER_MEMPOOL: false,
+        CLUSTER_MEMPOOL_INDEXING: false,
         CPFP_INDEXING: false,
         MAX_BLOCKS_BULK_QUERY: 0,
         DISK_CACHE_BLOCK_INTERVAL: 6,
@@ -144,7 +146,7 @@ describe('Mempool Backend Config', () => {
       });
 
       expect(config.MEMPOOL_SERVICES).toStrictEqual({
-        API: "",
+        API: '',
         ACCELERATIONS: false,
       });
 

backend/src/api/about.routes.ts

@@ -1,7 +1,9 @@
-import { Application } from "express";
-import config from "../config";
-import axios from "axios";
-import logger from "../logger";
+import { Application } from 'express';
+import config from '../config';
+import axios from 'axios';
+import logger from '../logger';
+
+const PROXY_PATH_SEGMENT_REGEX = /^(?!\.{1,2}$)[^\p{Cc}/?#\\]{1,256}$/u;
 
 class AboutRoutes {
   public initRoutes(app: Application) {
@@ -15,8 +17,13 @@ class AboutRoutes {
         }
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => {
+        if (!PROXY_PATH_SEGMENT_REGEX.test(req.params.id)) {
+          res.status(400).end();
+          return;
+        }
+
         try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${req.params.id}`, {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${encodeURIComponent(req.params.id)}`, {
             responseType: 'stream', timeout: 10000
           });
           response.data.pipe(res);
@@ -33,8 +40,13 @@ class AboutRoutes {
         }
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => {
+        if (!PROXY_PATH_SEGMENT_REGEX.test(req.params.id)) {
+          res.status(400).end();
+          return;
+        }
+
         try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${req.params.id}`, {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${encodeURIComponent(req.params.id)}`, {
             responseType: 'stream', timeout: 10000
           });
           response.data.pipe(res);
@@ -51,8 +63,13 @@ class AboutRoutes {
         }
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => {
+        if (!PROXY_PATH_SEGMENT_REGEX.test(req.params.id)) {
+          res.status(400).end();
+          return;
+        }
+
         try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${req.params.id}`, {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${encodeURIComponent(req.params.id)}`, {
             responseType: 'stream', timeout: 10000
           });
           response.data.pipe(res);
@@ -61,7 +78,7 @@ class AboutRoutes {
         }
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'services/sponsors', async (req, res) => {
-        const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+        const url = `${config.MEMPOOL_SERVICES.API}/sponsors`;
         try {
           const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
           response.data.pipe(res);
@@ -71,7 +88,12 @@ class AboutRoutes {
         }
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'services/account/images/:username/:md5', async (req, res) => {
-        const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+        if (!PROXY_PATH_SEGMENT_REGEX.test(req.params.username) || !PROXY_PATH_SEGMENT_REGEX.test(req.params.md5)) {
+          res.status(400).end();
+          return;
+        }
+
+        const url = `${config.MEMPOOL_SERVICES.API}/account/images/${encodeURIComponent(req.params.username)}/${encodeURIComponent(req.params.md5)}`;
         try {
           const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
           response.data.pipe(res);
@@ -84,4 +106,4 @@ class AboutRoutes {
   }
 }
 
-export default new AboutRoutes();
+export default new AboutRoutes();

backend/src/api/acceleration/acceleration.routes.ts

@@ -5,16 +5,18 @@ import logger from '../../logger';
 import mempool from '../mempool';
 import AccelerationRepository from '../../repositories/AccelerationRepository';
 
+const TXID_REGEX = /^[a-f0-9]{64}$/i;
+
 class AccelerationRoutes {
   private tag = 'Accelerator';
 
   public initRoutes(app: Application): void {
     app
       .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations', this.$getAcceleratorAccelerations.bind(this))
-      .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/:txid', this.$getAcceleratorAcceleration.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/history', this.$getAcceleratorAccelerationsHistory.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/history/aggregated', this.$getAcceleratorAccelerationsHistoryAggregated.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/stats', this.$getAcceleratorAccelerationsStats.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/:txid', this.$getAcceleratorAcceleration.bind(this))
       .post(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/estimate', this.$getAcceleratorEstimate.bind(this))
     ;
   }
@@ -24,8 +26,9 @@ class AccelerationRoutes {
     res.status(200).send(Object.values(accelerations));
   }
 
+  /** @asyncUnsafe */
   private async $getAcceleratorAcceleration(req: Request, res: Response): Promise<void> {
-    if (req.params.txid) {
+    if (req.params.txid && TXID_REGEX.test(req.params.txid)) {
       const acceleration = await AccelerationRepository.$getAccelerationInfoForTxid(req.params.txid);
       if (acceleration) {
         res.status(200).send(acceleration);
@@ -33,10 +36,11 @@ class AccelerationRoutes {
         res.status(404).send('Acceleration not found');
       }
     } else {
-      res.status(400).send('txid is required');
+      res.status(400).send('invalid txid');
     }
   }
 
+  /** @asyncUnsafe */
   private async $getAcceleratorAccelerationsHistory(req: Request, res: Response): Promise<void> {
     const history = await AccelerationRepository.$getAccelerationInfo(null, req.query.blockHeight ? parseInt(req.query.blockHeight as string, 10) : null);
     res.status(200).send(history.map(accel => ({
@@ -53,9 +57,9 @@ class AccelerationRoutes {
   }
 
   private async $getAcceleratorAccelerationsHistoryAggregated(req: Request, res: Response): Promise<void> {
-    const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+    const url = `${config.MEMPOOL_SERVICES.API}/accelerator/accelerations/history/aggregated`;
     try {
-      const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
+      const response = await axios.get(url, { params: req.query, responseType: 'stream', timeout: 10000 });
       for (const key in response.headers) {
         res.setHeader(key, response.headers[key]);
       }
@@ -67,9 +71,9 @@ class AccelerationRoutes {
   }
 
   private async $getAcceleratorAccelerationsStats(req: Request, res: Response): Promise<void> {
-    const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+    const url = `${config.MEMPOOL_SERVICES.API}/accelerator/accelerations/stats`;
     try {
-      const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
+      const response = await axios.get(url, { params: req.query, responseType: 'stream', timeout: 10000 });
       for (const key in response.headers) {
         res.setHeader(key, response.headers[key]);
       }
@@ -81,7 +85,7 @@ class AccelerationRoutes {
   }
 
   private async $getAcceleratorEstimate(req: Request, res: Response): Promise<void> {
-    const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+    const url = `${config.MEMPOOL_SERVICES.API}/accelerator/estimate`;
     try {
       const response = await axios.post(url, req.body, { responseType: 'stream', timeout: 10000 });
       for (const key in response.headers) {
@@ -95,4 +99,4 @@ class AccelerationRoutes {
   }
 }
 
-export default new AccelerationRoutes();
+export default new AccelerationRoutes();

backend/src/api/audit.ts

@@ -6,11 +6,28 @@ import transactionUtils from './transaction-utils';
 
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
 
+export interface AuditResult {
+  unseen: string[];
+  censored: string[];
+  added: string[];
+  prioritized: string[];
+  fresh: string[];
+  sigop: string[];
+  fullrbf: string[];
+  accelerated: string[];
+  matchRate: number;
+  similarity: number;
+}
+
 class Audit {
-  auditBlock(height: number, transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended })
-   : { unseen: string[], censored: string[], added: string[], prioritized: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } {
+  auditBlock(
+    height: number,
+    transactions: MempoolTransactionExtended[],
+    projectedBlocks: MempoolBlockWithTransactions[],
+    mempool: { [txId: string]: MempoolTransactionExtended }
+  ): AuditResult {
     if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
-      return { unseen: [], censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 1, similarity: 1 };
+      return { unseen: [], censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], matchRate: 100, similarity: 1 };
     }
 
     const matches: string[] = []; // present in both mined block and template
@@ -176,6 +193,8 @@ class Audit {
     }
     const similarity = projectedWeight ? matchedWeight / projectedWeight : 1;
 
+    const matchRate = Math.round(score * 100 * 100) / 100;
+
     return {
       unseen,
       censored: Object.keys(isCensored),
@@ -185,7 +204,7 @@ class Audit {
       sigop: [],
       fullrbf: rbf,
       accelerated,
-      score,
+      matchRate,
       similarity,
     };
   }

backend/src/api/backend-info.ts

@@ -3,9 +3,12 @@ import path from 'path';
 import os from 'os';
 import { IBackendInfo } from '../mempool.interfaces';
 import config from '../config';
+import bitcoinClient from './bitcoin/bitcoin-client';
+import logger from '../logger';
 
 class BackendInfo {
   private backendInfo: IBackendInfo;
+  private timer;
 
   constructor() {
     // This file is created by ./fetch-version.ts during building
@@ -26,7 +29,28 @@ class BackendInfo {
       gitCommit: versionInfo.gitCommit,
       lightning: config.LIGHTNING.ENABLED,
       backend: config.MEMPOOL.BACKEND,
+      coreVersion: '?',
+      osVersion: `${os.type()} ${os.release()}`,
     };
+
+    this.timer = setInterval(async () => {
+      try {
+        await this.$updateCoreVersion();
+      } catch (e) {
+        logger.err(`Exception in $updateCoreVersion. Reason: ${(e instanceof Error ? e.message : e)}`);
+      }
+    }, 10 * 60 * 1000); // every 10 minutes
+    void this.$updateCoreVersion(); // starting immediately
+  }
+
+  /** @asyncSafe */
+  private async $updateCoreVersion(): Promise<void> {
+    try {
+      const networkInfo = await bitcoinClient.getNetworkInfo();
+      this.backendInfo.coreVersion = networkInfo.subversion;
+    } catch (e) {
+      logger.err(`Exception in $updateCoreVersion. Reason: ${(e instanceof Error ? e.message : e)}`);
+    }
   }
 
   public getBackendInfo(): IBackendInfo {

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -57,4 +57,19 @@ export interface HealthCheckHost {
   unreachable: boolean;
   checked: boolean;
   lastChecked: number;
+  hashes?: {
+    frontend?: string;
+    hybrid?: string;
+    backend?: string;
+    electrs?: string;
+    ssr?: string;
+    core?: string;
+    os?: string;
+    lastUpdated?: number;
+  };
+  liquidAudit?: {
+    pegRatio: number;
+    bitcoinLastBlockUpdate: number;
+    liquidLastBlockUpdate: number;
+  };
 }

backend/src/api/bitcoin/bitcoin-api.interface.ts

@@ -53,6 +53,7 @@ export namespace IBitcoinApi {
     nTx: number;                     //  (numeric) The number of transactions in the block
     previousblockhash: string;       //  (string) The hash of the previous block
     nextblockhash: string;           //  (string) The hash of the next block
+    dynamic_parameters?: any;        //  (object) Elements only: dynamic parameters at this block
   }
 
   export interface Transaction {
@@ -90,6 +91,7 @@ export namespace IBitcoinApi {
     };
     sequence: number;                //  (numeric) The script sequence number
     txinwitness?: string[];          //  (string) hex-encoded witness data
+    pegin_witness?: string[];        //  (string) Elements peg-in witness
     coinbase?: string;
     is_pegin?: boolean;              //  (boolean) Elements peg-in
   }
@@ -165,44 +167,44 @@ export namespace IBitcoinApi {
     timeout: number;                 // (numeric) the median time past of a block at which the deployment is considered failed if not yet locked in
     since: number;                   // (numeric) height of the first block to which the status applies
     statistics: {                    // (object) numeric statistics about BIP9 signalling for a softfork (only for started status)
-      period: number;                // (numeric) the length in blocks of the BIP9 signalling period 
-      threshold: number;             // (numeric) the number of blocks with the version bit set required to activate the feature 
-      elapsed: number;               // (numeric) the number of blocks elapsed since the beginning of the current period 
-      count: number;                 // (numeric) the number of blocks with the version bit set in the current period 
-      possible: boolean;             // (boolean) returns false if there are not enough blocks left in this period to pass activation threshold 
+      period: number;                // (numeric) the length in blocks of the BIP9 signalling period
+      threshold: number;             // (numeric) the number of blocks with the version bit set required to activate the feature
+      elapsed: number;               // (numeric) the number of blocks elapsed since the beginning of the current period
+      count: number;                 // (numeric) the number of blocks with the version bit set in the current period
+      possible: boolean;             // (boolean) returns false if there are not enough blocks left in this period to pass activation threshold
     }
   }
 
   export interface BlockStats {
-    "avgfee": number;
-    "avgfeerate": number;
-    "avgtxsize": number;
-    "blockhash": string;
-    "feerate_percentiles": [number, number, number, number, number];
-    "height": number;
-    "ins": number;
-    "maxfee": number;
-    "maxfeerate": number;
-    "maxtxsize": number;
-    "medianfee": number;
-    "mediantime": number;
-    "mediantxsize": number;
-    "minfee": number;
-    "minfeerate": number;
-    "mintxsize": number;
-    "outs": number;
-    "subsidy": number;
-    "swtotal_size": number;
-    "swtotal_weight": number;
-    "swtxs": number;
-    "time": number;
-    "total_out": number;
-    "total_size": number;
-    "total_weight": number;
-    "totalfee": number;
-    "txs": number;
-    "utxo_increase": number;
-    "utxo_size_inc": number;
+    'avgfee': number;
+    'avgfeerate': number;
+    'avgtxsize': number;
+    'blockhash': string;
+    'feerate_percentiles': [number, number, number, number, number];
+    'height': number;
+    'ins': number;
+    'maxfee': number;
+    'maxfeerate': number;
+    'maxtxsize': number;
+    'medianfee': number;
+    'mediantime': number;
+    'mediantxsize': number;
+    'minfee': number;
+    'minfeerate': number;
+    'mintxsize': number;
+    'outs': number;
+    'subsidy': number;
+    'swtotal_size': number;
+    'swtotal_weight': number;
+    'swtxs': number;
+    'time': number;
+    'total_out': number;
+    'total_size': number;
+    'total_weight': number;
+    'totalfee': number;
+    'txs': number;
+    'utxo_increase': number;
+    'utxo_size_inc': number;
   }
 }
 
@@ -213,26 +215,26 @@ export interface TestMempoolAcceptResult {
   vsize?: number,
   fees?: {
     base: number,
-    "effective-feerate": number,
-    "effective-includes": string[],
+    'effective-feerate': number,
+    'effective-includes': string[],
   },
   ['reject-reason']?: string,
 }
 
 export interface SubmitPackageResult {
   package_msg: string;
-  "tx-results": { [wtxid: string]: TxResult };
-  "replaced-transactions"?: string[];
+  'tx-results': { [wtxid: string]: TxResult };
+  'replaced-transactions'?: string[];
 }
 
 export interface TxResult {
   txid: string;
-  "other-wtxid"?: string;
+  'other-wtxid'?: string;
   vsize?: number;
   fees?: {
     base: number;
-    "effective-feerate"?: number;
-    "effective-includes"?: string[];
+    'effective-feerate'?: number;
+    'effective-includes'?: string[];
   };
   error?: string;
 }

backend/src/api/bitcoin/bitcoin-api.ts

@@ -112,6 +112,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       .then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx);
   }
 
+  /** @asyncUnsafe */
   async $getTxsForBlock(hash: string, fallbackToCore = false): Promise<IEsploraApi.Transaction[]> {
     const verboseBlock: IBitcoinApi.VerboseBlock = await this.bitcoindClient.getBlock(hash, 2);
     const transactions: IEsploraApi.Transaction[] = [];
@@ -130,7 +131,7 @@ class BitcoinApi implements AbstractBitcoinApi {
 
   $getRawBlock(hash: string): Promise<Buffer> {
     return this.bitcoindClient.getBlock(hash, 0)
-      .then((raw: string) => Buffer.from(raw, "hex"));
+      .then((raw: string) => Buffer.from(raw, 'hex'));
   }
 
   $getBlockHash(height: number): Promise<string> {
@@ -219,6 +220,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     return this.bitcoindClient.submitPackage(rawTransactions, maxfeerate ?? undefined, maxburnamount ?? undefined);
   }
 
+  /** @asyncUnsafe */
   async $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
     const txOut = await this.bitcoindClient.getTxOut(txId, vout, false);
     return {
@@ -229,6 +231,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     };
   }
 
+  /** @asyncUnsafe */
   async $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
     const outSpends: IEsploraApi.Outspend[] = [];
     const tx = await this.$getRawTransaction(txId, true, false);
@@ -247,6 +250,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     return outSpends;
   }
 
+  /** @asyncUnsafe */
   async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
     const outspends: IEsploraApi.Outspend[][] = [];
     for (const tx of txId) {
@@ -260,6 +264,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     return this.$getBatchedOutspends(txId);
   }
 
+  /** @asyncUnsafe */
   async $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]> {
     const outspends: IEsploraApi.Outspend[] = [];
     for (const outpoint of outpoints) {
@@ -269,6 +274,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     return outspends;
   }
 
+  /** @asyncUnsafe */
   async $getCoinbaseTx(blockhash: string): Promise<IEsploraApi.Transaction> {
     const txids = await this.$getTxIdsForBlock(blockhash);
     return this.$getRawTransaction(txids[0]);
@@ -283,6 +289,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     return this.bitcoindClient.getNetworkHashPs(120, blockHeight);
   }
 
+  /** @asyncUnsafe */
   protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean, lazyPrevouts = false, allowMissingPrevouts = false): Promise<IEsploraApi.Transaction> {
     let esploraTransaction: IEsploraApi.Transaction = {
       txid: transaction.txid,
@@ -367,6 +374,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     }
   }
 
+  /** @asyncUnsafe */
   private async $appendMempoolFeeData(transaction: IEsploraApi.Transaction): Promise<IEsploraApi.Transaction> {
     if (transaction.fee) {
       return transaction;
@@ -384,6 +392,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     return transaction;
   }
 
+  /** @asyncUnsafe */
   protected async $addPrevouts(transaction: TransactionExtended): Promise<TransactionExtended> {
     let addedPrevouts = false;
     for (const vin of transaction.vin) {
@@ -423,6 +432,7 @@ class BitcoinApi implements AbstractBitcoinApi {
   }
 
 
+  /** @asyncUnsafe */
   private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean, lazyPrevouts: boolean): Promise<IEsploraApi.Transaction> {
     if (transaction.vin[0].is_coinbase) {
       transaction.fee = 0;

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -12,7 +12,7 @@ import backendInfo from '../backend-info';
 import transactionUtils from '../transaction-utils';
 import { IEsploraApi } from './esplora-api.interface';
 import loadingIndicators from '../loading-indicators';
-import { TransactionExtended } from '../../mempool.interfaces';
+import { CpfpInfo, TransactionExtended } from '../../mempool.interfaces';
 import logger from '../../logger';
 import blocks from '../blocks';
 import bitcoinClient from './bitcoin-client';
@@ -23,11 +23,14 @@ import { calculateMempoolTxCpfp } from '../cpfp';
 import { handleError } from '../../utils/api';
 import poolsUpdater from '../../tasks/pools-updater';
 import chainTips from '../chain-tips';
+import FlagValueRepository, { INTERVAL_PRESETS } from '../../repositories/FlagValueRepository';
 
 const TXID_REGEX = /^[a-f0-9]{64}$/i;
 const BLOCK_HASH_REGEX = /^[a-f0-9]{64}$/i;
 const ADDRESS_REGEX = /^[a-z0-9]{2,120}$/i;
 const SCRIPT_HASH_REGEX = /^([a-f0-9]{2})+$/i;
+const MAX_TRANSACTION_TIMES = 100;
+const JUST_NUMBERS_REGEX = /^[1-9]\d*$/;
 
 class BitcoinRoutes {
   public initRoutes(app: Application) {
@@ -37,6 +40,7 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', this.getDifficultyChange)
       .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', this.getRecommendedFees)
       .get(config.MEMPOOL.API_URL_PREFIX + 'fees/precise', this.getPreciseRecommendedFees)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'fee-estimates', this.getPreciseRecommendedFeesEsploraTransformed)
       .get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', this.getMempoolBlocks)
       .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', this.getBackendInfo)
       .get(config.MEMPOOL.API_URL_PREFIX + 'init-data', this.getInitData)
@@ -59,6 +63,7 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'chain-tips', this.getChainTips.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'stale-tips', this.getStaleTips.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'stale-tips/:height', this.getStaleTips.bind(this))
       .post(config.MEMPOOL.API_URL_PREFIX + 'prevouts', this.$getPrevouts)
       .post(config.MEMPOOL.API_URL_PREFIX + 'cpfp', this.getCpfpLocalTxs)
       // Temporarily add txs/package endpoint for all backends until esplora supports it
@@ -67,6 +72,10 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/definition/list', this.getBlockDefinitionHashes)
       .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/definition/current', this.getCurrentBlockDefinitionHash)
       .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/:definitionHash', this.getBlocksByDefinitionHash)
+
+      .get(config.MEMPOOL.API_URL_PREFIX + 'goggles/:interval/', this.getTxCountPerFlagValue)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'goggles/:interval/:bucketSize', this.getTxCountPerFlagValue)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'goggles/:interval/:bucketSize/:op/:mask', this.getTxCountPerFlagValue)
       ;
 
       if (config.MEMPOOL.BACKEND !== 'esplora') {
@@ -133,6 +142,58 @@ class BitcoinRoutes {
     res.json(result);
   }
 
+  private getPreciseRecommendedFeesEsploraTransformed(req: Request, res: Response) {
+    if (!mempool.isInSync()) {
+      res.statusCode = 503;
+      res.send('Service Unavailable');
+      return;
+    }
+    const result = feeApi.getPreciseRecommendedFee();
+
+    // Note: Manually tested against rust-esplora-client 0.12.3
+    // to make sure the status code and extra header didn't blow everything up.
+    // 3xx redirects with a Location header were silently followed and caused the client to
+    // blow up, 4xx and 5xx caused errors in the client as well, so only 2xx was acceptable for this endpoint.
+
+    // HTTP 203 Non-Authoritative Information is used to indicate
+    // that the response is not the original from the server, but a transformed version of it.
+    res.statusCode = 203;
+    res.setHeader(
+      'x-warning-from-mempool',
+      `This endpoint is deprecated and will be removed in a future release. Please use /api/v1/fees/precise instead.`,
+    );
+    res.json({
+      '1': result.fastestFee,
+      '2': result.fastestFee,
+      '3': result.halfHourFee,
+      '4': result.halfHourFee,
+      '5': result.halfHourFee,
+      '6': result.hourFee,
+      '7': result.hourFee,
+      '8': result.hourFee,
+      '9': result.hourFee,
+      '10': result.hourFee,
+      '11': result.hourFee,
+      '12': result.hourFee,
+      '13': result.hourFee,
+      '14': result.hourFee,
+      '15': result.hourFee,
+      '16': result.hourFee,
+      '17': result.hourFee,
+      '18': result.hourFee,
+      '19': result.hourFee,
+      '20': result.hourFee,
+      '21': result.hourFee,
+      '22': result.hourFee,
+      '23': result.hourFee,
+      '24': result.hourFee,
+      '25': result.hourFee,
+      '144': result.economyFee,
+      '504': result.economyFee,
+      '1008': result.minimumFee,
+    });
+  }
+
   private getMempoolBlocks(req: Request, res: Response) {
     try {
       const result = mempoolBlocks.getMempoolBlocks();
@@ -143,17 +204,21 @@ class BitcoinRoutes {
   }
 
   private getTransactionTimes(req: Request, res: Response) {
-    if (!Array.isArray(req.query.txId)) {
-      handleError(req, res, 500, 'Not an array');
+    if (!req.query.txId || typeof req.query.txId !== 'object') {
+      handleError(req, res, 400, 'invalid txId format');
       return;
     }
+
+    const requestedTxIds = Object.values(req.query.txId);
+    if (requestedTxIds.length > MAX_TRANSACTION_TIMES) {
+      handleError(req, res, 400, 'Too many txids requested');
+      return;
+    }
+
     const txIds: string[] = [];
-    for (const _txId in req.query.txId) {
-      if (typeof req.query.txId[_txId] === 'string') {
-        const txid = req.query.txId[_txId].toString();
-        if (TXID_REGEX.test(txid)) {
-          txIds.push(txid);
-        }
+    for (const txid of requestedTxIds) {
+      if (typeof txid === 'string' && TXID_REGEX.test(txid)) {
+        txIds.push(txid);
       }
     }
 
@@ -164,7 +229,7 @@ class BitcoinRoutes {
   private async $getBatchedOutspends(req: Request, res: Response): Promise<IEsploraApi.Outspend[][] | void> {
     const txids_csv = req.query.txids;
     if (!txids_csv || typeof txids_csv !== 'string') {
-      handleError(req, res, 500, 'Invalid txids format');
+      handleError(req, res, 400, 'Invalid txids format');
       return;
     }
     const txids = txids_csv.split(',');
@@ -187,18 +252,18 @@ class BitcoinRoutes {
 
   private async $getCpfpInfo(req: Request, res: Response) {
     if (!TXID_REGEX.test(req.params.txId)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
 
     const tx = mempool.getMempool()[req.params.txId];
     if (tx) {
       if (tx?.cpfpChecked) {
-        res.json({
-          ancestors: tx.ancestors,
+        const response: CpfpInfo & { acceleratedBy?: number[], acceleratedAt?: number, feeDelta?: number } = {
+          ancestors: tx.ancestors || [],
           bestDescendant: tx.bestDescendant || null,
-          descendants: tx.descendants || null,
-          effectiveFeePerVsize: tx.effectiveFeePerVsize || null,
+          descendants: tx.descendants,
+          effectiveFeePerVsize: tx.effectiveFeePerVsize,
           sigops: tx.sigops,
           fee: tx.fee,
           adjustedVsize: tx.adjustedVsize,
@@ -206,7 +271,14 @@ class BitcoinRoutes {
           acceleratedBy: tx.acceleratedBy || undefined,
           acceleratedAt: tx.acceleratedAt || undefined,
           feeDelta: tx.feeDelta || undefined,
-        });
+        };
+        if (config.MEMPOOL.CLUSTER_MEMPOOL && tx.clusterId != null) {
+          const cluster = mempool.clusterMempool?.getClusterForApi(req.params.txId);
+          if (cluster) {
+            response.cluster = cluster;
+          }
+        }
+        res.json(response);
         return;
       }
 
@@ -242,7 +314,7 @@ class BitcoinRoutes {
 
   private async getTransaction(req: Request, res: Response) {
     if (!TXID_REGEX.test(req.params.txId)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -261,7 +333,7 @@ class BitcoinRoutes {
 
   private async getRawTransaction(req: Request, res: Response) {
     if (!TXID_REGEX.test(req.params.txId)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -349,7 +421,7 @@ class BitcoinRoutes {
 
   private async getTransactionStatus(req: Request, res: Response) {
     if (!TXID_REGEX.test(req.params.txId)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -368,7 +440,7 @@ class BitcoinRoutes {
 
   private async getStrippedBlockTransactions(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     try {
@@ -382,11 +454,11 @@ class BitcoinRoutes {
 
   private async getStrippedBlockTransaction(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     if (!TXID_REGEX.test(req.params.txid)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -404,7 +476,7 @@ class BitcoinRoutes {
 
   private async getBlock(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     try {
@@ -430,7 +502,7 @@ class BitcoinRoutes {
 
   private async getBlockHeader(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     try {
@@ -444,7 +516,7 @@ class BitcoinRoutes {
 
   private async getBlockAuditSummary(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     try {
@@ -463,11 +535,11 @@ class BitcoinRoutes {
 
   private async $getBlockTxAuditSummary(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     if (!TXID_REGEX.test(req.params.txid)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -486,7 +558,7 @@ class BitcoinRoutes {
 
   private async getBlocks(req: Request, res: Response) {
     try {
-      if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
+      if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
         const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
         res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
         res.json(await blocks.$getBlocks(height, 15));
@@ -500,7 +572,7 @@ class BitcoinRoutes {
 
   private async getBlocksByBulk(req: Request, res: Response) {
     try {
-      if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid - Not implemented
+      if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid - Not implemented
         handleError(req, res, 404, `This API is only available for Bitcoin networks`);
         return;
       }
@@ -542,7 +614,7 @@ class BitcoinRoutes {
 
   private async getChainTips(req: Request, res: Response) {
     try {
-      if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
+      if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
         res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
         const tips = await chainTips.getChainTips();
         if (tips.length > 0) {
@@ -562,15 +634,19 @@ class BitcoinRoutes {
 
   private async getStaleTips(req: Request, res: Response) {
     try {
-      if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
-        res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-        const tips = await chainTips.getStaleTips();
-        if (tips.length > 0) {
-          res.json(tips);
-        } else {
-          handleError(req, res, 503, `Temporarily unavailable`);
-          return;
+      if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
+        let fromHeight: number | undefined;
+        if (req.params.height !== undefined) {
+          fromHeight = parseInt(req.params.height, 10);
+          if (isNaN(fromHeight) || fromHeight < 0) {
+            handleError(req, res, 400, `Parameter 'height' must be a block height (integer)`);
+            return;
+          }
         }
+
+        res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+        const tips = await chainTips.$getStaleTipsPage(fromHeight, 25);
+        res.json(tips);
       } else { // Liquid
         handleError(req, res, 404, `This API is only available for Bitcoin networks`);
         return;
@@ -617,7 +693,7 @@ class BitcoinRoutes {
 
   private async getBlockTransactions(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     try {
@@ -659,7 +735,7 @@ class BitcoinRoutes {
       return;
     }
     if (!ADDRESS_REGEX.test(req.params.address)) {
-      handleError(req, res, 501, `Invalid address`);
+      handleError(req, res, 400, `Invalid address`);
       return;
     }
 
@@ -667,6 +743,10 @@ class BitcoinRoutes {
       const addressData = await bitcoinApi.$getAddress(req.params.address);
       res.json(addressData);
     } catch (e) {
+      if (e instanceof Error && e.message === 'Invalid Bitcoin address') {
+        res.status(400).send(e.message);
+        return;
+      }
       if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
         handleError(req, res, 413, e.message);
         return;
@@ -681,7 +761,7 @@ class BitcoinRoutes {
       return;
     }
     if (!ADDRESS_REGEX.test(req.params.address)) {
-      handleError(req, res, 501, `Invalid address`);
+      handleError(req, res, 400, `Invalid address`);
       return;
     }
 
@@ -693,6 +773,10 @@ class BitcoinRoutes {
       const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, lastTxId);
       res.json(transactions);
     } catch (e) {
+      if (e instanceof Error && e.message === 'Invalid Bitcoin address') {
+        res.status(400).send(e.message);
+        return;
+      }
       if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
         handleError(req, res, 413, e.message);
         return;
@@ -707,7 +791,7 @@ class BitcoinRoutes {
       return;
     }
     if (!ADDRESS_REGEX.test(req.params.address)) {
-      handleError(req, res, 501, `Invalid address`);
+      handleError(req, res, 400, `Invalid address`);
       return;
     }
 
@@ -715,6 +799,10 @@ class BitcoinRoutes {
       const addressData = await bitcoinApi.$getAddressUtxos(req.params.address);
       res.json(addressData);
     } catch (e) {
+      if (e instanceof Error && e.message === 'Invalid Bitcoin address') {
+        res.status(400).send(e.message);
+        return;
+      }
       if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
         handleError(req, res, 413, e.message);
         return;
@@ -736,7 +824,7 @@ class BitcoinRoutes {
       return;
     }
     if (!SCRIPT_HASH_REGEX.test(req.params.scripthash)) {
-      handleError(req, res, 501, `Invalid scripthash`);
+      handleError(req, res, 400, `Invalid scripthash`);
       return;
     }
 
@@ -760,7 +848,7 @@ class BitcoinRoutes {
       return;
     }
     if (!SCRIPT_HASH_REGEX.test(req.params.scripthash)) {
-      handleError(req, res, 501, `Invalid scripthash`);
+      handleError(req, res, 400, `Invalid scripthash`);
       return;
     }
 
@@ -788,7 +876,7 @@ class BitcoinRoutes {
       return;
     }
     if (!SCRIPT_HASH_REGEX.test(req.params.scripthash)) {
-      handleError(req, res, 501, `Invalid scripthash`);
+      handleError(req, res, 400, `Invalid scripthash`);
       return;
     }
 
@@ -921,7 +1009,7 @@ class BitcoinRoutes {
 
   private async getRawBlock(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     try {
@@ -935,7 +1023,7 @@ class BitcoinRoutes {
 
   private async getTxIdsForBlock(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     try {
@@ -948,7 +1036,7 @@ class BitcoinRoutes {
 
   private async validateAddress(req: Request, res: Response) {
     if (!ADDRESS_REGEX.test(req.params.address)) {
-      handleError(req, res, 501, `Invalid address`);
+      handleError(req, res, 400, `Invalid address`);
       return;
     }
     try {
@@ -961,7 +1049,7 @@ class BitcoinRoutes {
 
   private async getRbfHistory(req: Request, res: Response) {
     if (!TXID_REGEX.test(req.params.txId)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -996,7 +1084,7 @@ class BitcoinRoutes {
 
   private async getCachedTx(req: Request, res: Response) {
     if (!TXID_REGEX.test(req.params.txId)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -1013,7 +1101,7 @@ class BitcoinRoutes {
 
   private async getTransactionOutspends(req: Request, res: Response) {
     if (!TXID_REGEX.test(req.params.txId)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -1026,7 +1114,7 @@ class BitcoinRoutes {
 
   private async getTransactionMerkleProof(req: Request, res: Response): Promise<void> {
     if (!TXID_REGEX.test(req.params.txId)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -1050,6 +1138,63 @@ class BitcoinRoutes {
     }
   }
 
+  private async getTxCountPerFlagValue(req: Request, res: Response) {
+    try {
+      if (!Common.blocksSummariesIndexingEnabled()) {
+        handleError(req, res, 404, `Block summaries indexing is required for this API`);
+        return;
+      }
+
+      const presets = INTERVAL_PRESETS;
+      const operations = ['and', 'or', 'nor', undefined];
+      const intervals = Object.keys(presets);
+      const interval = req.params.interval;
+
+      if (!intervals.includes(interval)) {
+        handleError(req, res, 400, `Invalid interval, must be one of ${intervals.toString()}`);
+        return;
+      }
+
+      const validBucketSizes = presets[interval].bucketSizes;
+      const rawBucketSize = req.params.bucketSize;
+      const bucketSize: number = rawBucketSize === undefined ? validBucketSizes[0] : Number(rawBucketSize);
+      if (!Number.isInteger(bucketSize) || !validBucketSizes.includes(bucketSize)) {
+        handleError(req, res, 400, `Invalid bucket size, must be ${validBucketSizes.toString()}`);
+        return;
+      }
+
+      if (!operations.includes(req.params.op)) {
+        handleError(req, res, 400, `Invalid operation, must be 'and', 'or', 'nor' or undefined.`);
+        return;
+      }
+
+      if (req.params.mask && !JUST_NUMBERS_REGEX.test(req.params.mask)) {
+        handleError(req, res, 400, `Invalid mask value, must be a positive integer`);
+        return;
+      }
+
+      const op = (req.params.op) as 'and' | 'or' | 'nor' | undefined;
+      const mask = BigInt(req.params.mask ?? 0n);
+
+      const { tip, tail }  = await FlagValueRepository.$getTipAndTailIndexedByBucketSize(bucketSize) || { tip: undefined, tail: undefined };
+
+      if (tip === undefined || tail === undefined) {
+        handleError(req, res, 400, `Failed to get latest indexed flag values for ${interval}`);
+        return;
+      }
+
+      const totalCount = await FlagValueRepository.$getTotalBlocksIndexedByBucketSize(bucketSize === 1 ? 1008 : bucketSize) ?? tip - tail;
+
+      const startHeight = presets[interval].retentionSpan !== -1 ? (tip - presets[interval].retentionSpan) : -1;
+      const txsCount = await FlagValueRepository.$queryTxCountBasedOnMask(mask, bucketSize, op, startHeight);
+      res.header('X-total-count', totalCount.toString());
+      res.header('Expires', new Date(Date.now() + 1000 * 3600 * 24 * (presets[interval].bucketSizes[0] / 144)).toUTCString());
+      res.send(txsCount);
+    } catch (e: any) {
+      handleError(req, res, 400, e instanceof Error ? e.message : 'Failed to get flag values');
+    }
+  }
+
   private async $postTransaction(req: Request, res: Response) {
     res.setHeader('content-type', 'text/plain');
     try {

backend/src/api/bitcoin/electrum-api.ts

@@ -5,7 +5,7 @@ import { IEsploraApi } from './esplora-api.interface';
 import { IElectrumApi } from './electrum-api.interface';
 import BitcoinApi from './bitcoin-api';
 import logger from '../../logger';
-import crypto from "crypto-js";
+import crypto from 'crypto-js';
 import loadingIndicators from '../loading-indicators';
 import memoryCache from '../memory-cache';
 
@@ -40,26 +40,11 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
       });
   }
 
+  /** @asyncUnsafe */
   async $getAddress(address: string): Promise<IEsploraApi.Address> {
     const addressInfo = await this.bitcoindClient.validateAddress(address);
     if (!addressInfo || !addressInfo.isvalid) {
-      return ({
-        'address': address,
-        'chain_stats': {
-          'funded_txo_count': 0,
-          'funded_txo_sum': 0,
-          'spent_txo_count': 0,
-          'spent_txo_sum': 0,
-          'tx_count': 0
-        },
-        'mempool_stats': {
-          'funded_txo_count': 0,
-          'funded_txo_sum': 0,
-          'spent_txo_count': 0,
-          'spent_txo_sum': 0,
-          'tx_count': 0
-        }
-      });
+      throw new Error('Invalid Bitcoin address');
     }
 
     try {
@@ -91,10 +76,11 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
     }
   }
 
+  /** @asyncUnsafe */
   async $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]> {
     const addressInfo = await this.bitcoindClient.validateAddress(address);
     if (!addressInfo || !addressInfo.isvalid) {
-      return [];
+      throw new Error('Invalid Bitcoin address');
     }
 
     try {
@@ -160,10 +146,11 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
     }
   }
 
+  /** @asyncUnsafe */
   async $getAddressUtxos(address: string): Promise<IEsploraApi.UTXO[]> {
     const addressInfo = await this.bitcoindClient.validateAddress(address);
     if (!addressInfo || !addressInfo.isvalid) {
-      return [];
+      throw new Error('Invalid Bitcoin address');
     }
     const scripthash = this.encodeScriptHash(addressInfo.scriptPubKey);
     return this.$getScriptHashUtxos(scripthash);
@@ -206,10 +193,11 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
     }
   }
 
+  /** @asyncUnsafe */
   async $getScriptHashUtxos(scripthash: string): Promise<IEsploraApi.UTXO[]> {
     const utxos = await this.$getScriptHashUnspent(scripthash);
     const result: IEsploraApi.UTXO[] = [];
-    for(let utxo of utxos) {
+    for(const utxo of utxos) {
       if(utxo.height===0) {
         //Unconfirmed
         result.push({
@@ -244,6 +232,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
     return this.electrumClient.blockchainScripthash_listunspent(scriptHash);
   }
 
+  /** @asyncUnsafe */
   async $getTransactionMerkleProof(txId: string): Promise<IEsploraApi.MerkleProof> {
     const tx = await this.$getRawTransaction(txId);
     return this.electrumClient.blockchainTransaction_getMerkle(txId, tx.status.block_height);

backend/src/api/bitcoin/esplora-api.ts

@@ -28,14 +28,22 @@ interface FailoverHost {
     hybrid?: string,
     backend?: string,
     electrs?: string,
+    ssr?: string,
+    core?: string,
+    os?: string,
     lastUpdated: number,
+  },
+  liquidAudit?: {
+    pegRatio: number,
+    bitcoinLastBlockUpdate: number,
+    liquidLastBlockUpdate: number,
   }
 }
 
 class FailoverRouter {
   activeHost: FailoverHost;
   fallbackHost: FailoverHost;
-  maxSlippage: number = config.ESPLORA.MAX_BEHIND_TIP ?? 2;
+  maxSlippage: number = config.ESPLORA.MAX_BEHIND_TIP ?? (Common.isLiquid() ? 8 : 2);
   maxHeight: number = 0;
   hosts: FailoverHost[];
   multihost: boolean;
@@ -98,11 +106,12 @@ class FailoverRouter {
     });
 
     if (this.multihost) {
-      this.pollHosts();
+      void this.pollHosts();
     }
   }
 
   // start polling hosts to measure availability & rtt
+  /** @asyncSafe */
   private async pollHosts(): Promise<void> {
     if (this.pollTimer) {
       clearTimeout(this.pollTimer);
@@ -141,11 +150,14 @@ class FailoverRouter {
             }
           }
 
+          await this.$updateLiquidAudit(host);
+
           // Check front and backend git hashes less often
           if (Date.now() - host.hashes.lastUpdated > this.gitHashInterval) {
             await Promise.all([
               this.$updateFrontendGitHash(host),
-              this.$updateBackendGitHash(host),
+              this.$updateBackendVersions(host),
+              this.$updateSSRGitHash(host),
               config.MEMPOOL.OFFICIAL ? this.$updateHybridGitHash(host) : Promise.resolve(),
             ]);
             host.hashes.lastUpdated = Date.now();
@@ -191,7 +203,7 @@ class FailoverRouter {
 
     const elapsed = Date.now() - start;
 
-    this.pollTimer = setTimeout(() => { this.pollHosts(); }, Math.max(1, this.pollInterval - elapsed));
+    this.pollTimer = setTimeout(() => { void this.pollHosts(); }, Math.max(1, this.pollInterval - elapsed));
   }
 
   private formatRanking(index: number, host: FailoverHost, active: FailoverHost, maxHeight: number): string {
@@ -250,7 +262,12 @@ class FailoverRouter {
   private async $updateFrontendGitHash(host: FailoverHost): Promise<void> {
     try {
       const url = `${host.publicDomain}/resources/config.js`;
-      const response = await this.pollConnection.get<string>(url, { timeout: config.ESPLORA.FALLBACK_TIMEOUT });
+      const response = await this.pollConnection.get<string>(
+        url, {
+          timeout: config.ESPLORA.FALLBACK_TIMEOUT,
+          headers: Common.isLiquid() ? { 'Host': 'liquid.network' } : undefined
+        }
+      );
       const match = response.data.match(/GIT_COMMIT_HASH\s*=\s*['"](.*?)['"]/);
       if (match && match[1]?.length) {
         host.hashes.frontend = match[1];
@@ -273,7 +290,7 @@ class FailoverRouter {
           path: '/en-US/resources/config.js',
           method: 'GET',
           headers: {
-            'Host': 'mempool.space'
+            'Host': Common.isLiquid() ? 'liquid.network' : 'mempool.space'
           },
           timeout: config.ESPLORA.FALLBACK_TIMEOUT,
         }, (res) => {
@@ -301,18 +318,83 @@ class FailoverRouter {
     }
   }
 
-  private async $updateBackendGitHash(host: FailoverHost): Promise<void> {
+  private async $updateBackendVersions(host: FailoverHost): Promise<void> {
     try {
       const url = `${host.publicDomain}/api/v1/backend-info`;
-      const response = await this.pollConnection.get<any>(url, { timeout: config.ESPLORA.FALLBACK_TIMEOUT });
+      const response = await this.pollConnection.get<any>(
+        url, {
+          timeout: config.ESPLORA.FALLBACK_TIMEOUT,
+          headers: Common.isLiquid() ? { 'Host': 'liquid.network' } : undefined
+        }
+      );
       if (response.data?.gitCommit) {
         host.hashes.backend = response.data.gitCommit;
       }
+      if (response.data?.coreVersion) {
+        host.hashes.core = response.data.coreVersion;
+      }
+      if (response.data?.osVersion) {
+        host.hashes.os = response.data.osVersion;
+      }
     } catch (e) {
       // failed to get backend build hash - do nothing
     }
   }
 
+  private async $updateSSRGitHash(host: FailoverHost): Promise<void> {
+    try {
+      const url = `${host.publicDomain}/ssr/api/status`;
+      const response = await this.pollConnection.get<any>(
+        url, {
+          timeout: config.ESPLORA.FALLBACK_TIMEOUT,
+          headers: Common.isLiquid() ? { 'Host': 'liquid.network' } : undefined
+        }
+      );
+      if (response.data?.gitHash) {
+        host.hashes.ssr = response.data.gitHash;
+      }
+    } catch (e) {
+      // failed to get ssr build hash - do nothing
+    }
+  }
+
+  private async $updateLiquidAudit(host: FailoverHost): Promise<void> {
+    if (config.MEMPOOL.NETWORK !== 'liquid') {
+      return;
+    }
+    try {
+      const [reservesResponse, pegsResponse] = await Promise.all([
+        this.pollConnection.get<any>(
+          `${host.publicDomain}/api/v1/liquid/reserves`, {
+            timeout: config.ESPLORA.FALLBACK_TIMEOUT,
+            headers: { 'Host': 'liquid.network' }
+          }
+        ),
+        this.pollConnection.get<any>(
+          `${host.publicDomain}/api/v1/liquid/pegs`, {
+            timeout: config.ESPLORA.FALLBACK_TIMEOUT,
+            headers: { 'Host': 'liquid.network' }
+          }
+        ),
+      ]);
+
+      const reservesAmount = Number(reservesResponse.data?.amount);
+      const pegsAmount = Number(pegsResponse.data?.amount);
+      const bitcoinLastBlockUpdate = Number(reservesResponse.data?.lastBlockUpdate);
+      const liquidLastBlockUpdate = Number(pegsResponse.data?.lastBlockUpdate);
+
+      if (Number.isFinite(reservesAmount) && Number.isFinite(pegsAmount) && Number.isFinite(bitcoinLastBlockUpdate) && Number.isFinite(liquidLastBlockUpdate) && pegsAmount > 0) {
+        host.liquidAudit = {
+          pegRatio: (reservesAmount / pegsAmount) * 100,
+          bitcoinLastBlockUpdate,
+          liquidLastBlockUpdate,
+        };
+      }
+    } catch (e) {
+      // failed to get liquid audit values - do nothing
+    }
+  }
+
   // returns the public mempool domain corresponding to an esplora server url
   // (a bit of a hack to avoid manually specifying frontend & backend URLs for each esplora server)
   private extractPublicDomain(url: string): string {
@@ -409,6 +491,7 @@ class ElectrsApi implements AbstractBitcoinApi {
     return this.failoverRouter.$get<string>('/blocks/tip/hash');
   }
 
+  /** @asyncUnsafe */
   async $getTxIdsForBlock(hash: string, fallbackToCore = false): Promise<string[]> {
     try {
       const txids = await this.failoverRouter.$get<string[]>('/block/' + hash + '/txids');
@@ -425,6 +508,7 @@ class ElectrsApi implements AbstractBitcoinApi {
     }
   }
 
+  /** @asyncUnsafe */
   async $getTxsForBlock(hash: string, fallbackToCore = false): Promise<IEsploraApi.Transaction[]> {
     try {
       const txs = await this.failoverRouter.$get<IEsploraApi.Transaction[]>('/internal/block/' + hash + '/txs');
@@ -518,6 +602,7 @@ class ElectrsApi implements AbstractBitcoinApi {
     return this.failoverRouter.$post<IEsploraApi.Outspend[]>('/internal/txs/outspends/by-outpoint', outpoints.map(out => `${out.txid}:${out.vout}`), 'json');
   }
 
+  /** @asyncUnsafe */
   async $getCoinbaseTx(blockhash: string): Promise<IEsploraApi.Transaction> {
     const txid = await this.failoverRouter.$get<string>(`/block/${blockhash}/txid/0`);
     return this.failoverRouter.$get<IEsploraApi.Transaction>('/tx/' + txid);
@@ -544,6 +629,7 @@ class ElectrsApi implements AbstractBitcoinApi {
         checked: !!host.checked,
         lastChecked: host.lastChecked || 0,
         hashes: host.hashes,
+        ...(config.MEMPOOL.NETWORK === 'liquid' ? { liquidAudit: host.liquidAudit } : {}),
       }));
     } else {
       return [];

backend/src/api/block-processor.ts

@@ -0,0 +1,262 @@
+import config from '../config';
+import logger from '../logger';
+import {
+  BlockExtended,
+  BlockSummary,
+  PoolTag,
+  MempoolTransactionExtended,
+  CpfpSummary,
+  TemplateAlgorithm,
+  MempoolBlockWithTransactions,
+} from '../mempool.interfaces';
+import { IEsploraApi } from './bitcoin/esplora-api.interface';
+import { Acceleration } from './services/acceleration';
+import { calculateGoodBlockCpfp, calculateClusterMempoolBlockCpfp, calculateFastBlockCpfp, BlockCpfpData } from './cpfp';
+import mempoolBlocks from './mempool-blocks';
+import memPool from './mempool';
+import Audit, { AuditResult } from './audit';
+import blocks from './blocks';
+import transactionUtils from './transaction-utils';
+import { ClusterMempool } from '../cluster-mempool/cluster-mempool';
+import { Common } from './common';
+import accelerationApi from './services/acceleration';
+
+interface ProcessedAudit extends AuditResult {
+  expectedFees: number;
+  expectedWeight: number;
+  projectedBlocks: MempoolBlockWithTransactions[];
+}
+
+export interface BlockProcessingResult {
+  templateAlgorithm: TemplateAlgorithm;
+  cpfpSummary: CpfpSummary;
+  blockExtended: BlockExtended;
+  blockSummary: BlockSummary;
+  auditResult?: ProcessedAudit;
+}
+
+const CM_ACTIVATION_HEIGHT: { [network: string]: number } = {
+  'mainnet': 940000,
+  'testnet': 4860000,
+  'testnet4': 125000,
+  'signet': 294000,
+  'regtest': 0,
+};
+
+class BlockProcessor {
+
+  /** @asyncUnsafe */
+  public async $processNewBlock(
+    block: IEsploraApi.Block,
+    transactions: MempoolTransactionExtended[],
+    pool: PoolTag,
+    accelerations: Record<string, Acceleration>
+  ): Promise<BlockProcessingResult> {
+    const poolAccelerations = Object.values(accelerations)
+      .filter(a => a.pools.includes(pool.uniqueId))
+      .map(a => ({ txid: a.txid, max_bid: a.feeDelta }));
+
+    const { templateAlgorithm, cpfpSummary } = detectTemplateAlgorithm(
+      block.height,
+      transactions,
+      poolAccelerations
+    );
+
+
+    const blockExtended = await blocks.$getBlockExtended(block, cpfpSummary.transactions, pool);
+    const blockSummary = blocks.summarizeBlockTransactions(block.id, block.height, cpfpSummary.transactions);
+
+    let auditResult: ProcessedAudit | undefined;
+    if (config.MEMPOOL.AUDIT && memPool.isInSync()) {
+      auditResult = await this.$runAudit(
+        blockExtended,
+        transactions,
+        templateAlgorithm,
+        pool,
+        accelerations
+      );
+
+      if (blockExtended.extras) {
+        blockExtended.extras.matchRate = auditResult.matchRate;
+        blockExtended.extras.expectedFees = auditResult.expectedFees;
+        blockExtended.extras.expectedWeight = auditResult.expectedWeight;
+        blockExtended.extras.similarity = auditResult.similarity;
+      }
+    } else if (blockExtended.extras) {
+      const mBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
+      if (mBlocks?.length && mBlocks[0].transactions) {
+        blockExtended.extras.similarity = Common.getSimilarity(mBlocks[0], transactions);
+      }
+    }
+
+    return {
+      templateAlgorithm,
+      cpfpSummary,
+      blockExtended,
+      blockSummary,
+      auditResult,
+    };
+  }
+
+  private async $runAudit(
+    block: BlockExtended,
+    transactions: MempoolTransactionExtended[],
+    templateAlgorithm: TemplateAlgorithm,
+    pool: PoolTag,
+    accelerations: Record<string, Acceleration>
+  ): Promise<ProcessedAudit> {
+    const auditMempool = memPool.getMempool();
+    const isAccelerated = accelerationApi.isAcceleratedBlock(block, Object.values(accelerations));
+
+    const candidateTxs = memPool.getMempoolCandidates();
+    const candidates = (memPool.limitGBT && candidateTxs)
+      ? { txs: candidateTxs, added: [], removed: [] }
+      : undefined;
+    const transactionIds = (memPool.limitGBT)
+      ? Object.keys(candidates?.txs || {})
+      : Object.keys(auditMempool);
+
+    let projectedBlocks: MempoolBlockWithTransactions[];
+
+    if (templateAlgorithm === TemplateAlgorithm.clusterMempool) {
+      const clusterMempool = memPool.clusterMempool ?? new ClusterMempool(auditMempool, accelerations, true, 75000);
+      const cmBlocks = clusterMempool.getBlocks(config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT) ?? [];
+      projectedBlocks = mempoolBlocks.processClusterMempoolBlocks(
+        cmBlocks,
+        auditMempool,
+        accelerations,
+        false,
+        pool.uniqueId
+      );
+    } else if (config.MEMPOOL.RUST_GBT) {
+      const added = memPool.limitGBT ? (candidates?.added || []) : [];
+      const removed = memPool.limitGBT ? (candidates?.removed || []) : [];
+      projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(
+        transactionIds,
+        auditMempool,
+        added,
+        removed,
+        candidates,
+        isAccelerated,
+        pool.uniqueId,
+        true
+      );
+    } else {
+      projectedBlocks = await mempoolBlocks.$makeBlockTemplates(
+        transactionIds,
+        auditMempool,
+        candidates,
+        false,
+        isAccelerated,
+        pool.uniqueId
+      );
+    }
+
+    const auditResult = Audit.auditBlock(block.height, transactions, projectedBlocks, auditMempool);
+
+    const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
+
+    let totalFees = 0;
+    let totalWeight = 0;
+    for (const tx of stripped) {
+      totalFees += tx.fee;
+      totalWeight += (tx.vsize * 4);
+    }
+
+    return {
+      ...auditResult,
+      expectedFees: totalFees,
+      expectedWeight: totalWeight,
+      projectedBlocks,
+    };
+  }
+}
+
+function saveCpfpDataToTransactions(transactions: MempoolTransactionExtended[], cpfpData: BlockCpfpData): void {
+  for (const tx of transactions) {
+    if (cpfpData.txs[tx.txid]) {
+      Object.assign(tx, cpfpData.txs[tx.txid]);
+    }
+  }
+}
+
+export function saveCpfpDataToCpfpSummary(transactions: MempoolTransactionExtended[], cpfpData: BlockCpfpData): CpfpSummary {
+  saveCpfpDataToTransactions(transactions, cpfpData);
+  return {
+    transactions,
+    clusters: cpfpData.clusters,
+    version: cpfpData.version,
+  };
+}
+
+/**
+ *
+ * @param height
+ * @param blockTransactions
+ * @param poolAccelerations
+ * @param fast
+ *
+ * saves effective fee rates from detected algorithm to blockTransactions
+ */
+export function detectTemplateAlgorithm(
+  height: number,
+  blockTransactions: MempoolTransactionExtended[],
+  poolAccelerations: { txid: string; max_bid: number }[],
+  fast: boolean = false
+): { templateAlgorithm: TemplateAlgorithm; cpfpSummary: CpfpSummary } {
+
+  const legacyCpfpData = fast ? calculateFastBlockCpfp(
+    height,
+    blockTransactions,
+  ) : calculateGoodBlockCpfp(
+    height,
+    blockTransactions,
+    poolAccelerations
+  );
+
+  if (!config.MEMPOOL.CLUSTER_MEMPOOL_INDEXING) {
+    return {
+      templateAlgorithm: TemplateAlgorithm.legacy,
+      cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, legacyCpfpData),
+    };
+  }
+
+  const network = config.MEMPOOL.NETWORK || 'mainnet';
+  const activationHeight = CM_ACTIVATION_HEIGHT[network] ?? Infinity;
+
+  if (height < activationHeight) {
+    return {
+      templateAlgorithm: TemplateAlgorithm.legacy,
+      cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, legacyCpfpData),
+    };
+  }
+
+  const clusterCpfpData = calculateClusterMempoolBlockCpfp(
+    height,
+    blockTransactions,
+    poolAccelerations
+  );
+
+  const clusterTxs = blockTransactions.map(tx => ({ txid: tx.txid, rate: clusterCpfpData.txs[tx.txid].effectiveFeePerVsize ?? tx.effectiveFeePerVsize }));
+  const legacyTxs = blockTransactions.map(tx => ({ txid: tx.txid, rate: legacyCpfpData.txs[tx.txid].effectiveFeePerVsize ?? tx.effectiveFeePerVsize }));
+  const clusterPrioritization = transactionUtils.identifyPrioritizedTransactions(clusterTxs, 'rate');
+  const legacyPrioritization = transactionUtils.identifyPrioritizedTransactions(legacyTxs, 'rate');
+
+  const clusterCount = clusterPrioritization.prioritized.length + clusterPrioritization.deprioritized.length;
+  const legacyCount = legacyPrioritization.prioritized.length + legacyPrioritization.deprioritized.length;
+
+  if (clusterCount < legacyCount) {
+    saveCpfpDataToTransactions(blockTransactions, clusterCpfpData);
+    return {
+      templateAlgorithm: TemplateAlgorithm.clusterMempool,
+      cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, clusterCpfpData),
+    };
+  } else {
+    return {
+      templateAlgorithm: TemplateAlgorithm.legacy,
+      cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, legacyCpfpData),
+    };
+  }
+}
+
+export default new BlockProcessor();

backend/src/api/blocks.ts

@@ -2,7 +2,7 @@ import config from '../config';
 import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 import logger from '../logger';
 import memPool from './mempool';
-import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified, BlockAudit, TransactionAudit } from '../mempool.interfaces';
+import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified, BlockAudit, TransactionAudit, TemplateAlgorithm } from '../mempool.interfaces';
 import { Common } from './common';
 import diskCache from './disk-cache';
 import transactionUtils from './transaction-utils';
@@ -28,13 +28,19 @@ import chainTips from './chain-tips';
 import websocketHandler from './websocket-handler';
 import redisCache from './redis-cache';
 import rbfCache from './rbf-cache';
+import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
+import mempoolBlocks from './mempool-blocks';
+import statistics from './statistics/statistics';
 import { calcBitsDifference } from './difficulty-adjustment';
 import AccelerationRepository from '../repositories/AccelerationRepository';
-import { calculateFastBlockCpfp, calculateGoodBlockCpfp } from './cpfp';
+import { calculateGoodBlockCpfp } from './cpfp';
+import blockProcessor, { BlockProcessingResult, detectTemplateAlgorithm, saveCpfpDataToCpfpSummary } from './block-processor';
 import mempool from './mempool';
 import CpfpRepository from '../repositories/CpfpRepository';
-import { parseDATUMTemplateCreator } from '../utils/bitcoin-script';
+import { parseDATUMTemplateCreator, parseDMNDTemplateCreator } from '../utils/bitcoin-script';
 import database from '../database';
+import { getBlockFirstSeenFromLogs, getOldestLogTimestampFromLogs, scanLogsForBlocksFirstSeen } from '../utils/file-read';
+import FlagValueRepository, { INDEXING_PRESETS } from '../repositories/FlagValueRepository';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -45,10 +51,12 @@ class Blocks {
   private previousDifficultyRetarget = 0;
   private quarterEpochBlockTime: number | null = null;
   private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
-  private newAsyncBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: MempoolTransactionExtended[]) => Promise<void>)[] = [];
   private classifyingBlocks: boolean = false;
+  private oldestCoreLogTimestamp: number | undefined | null = undefined;
 
   private mainLoopTimeout: number = 120000;
+  private indexingFlagValues: boolean = false;
+  private flagValuesDeleteQueue: number[]= [];
 
   constructor() { }
 
@@ -72,10 +80,6 @@ class Blocks {
     this.newBlockCallbacks.push(fn);
   }
 
-  public setNewAsyncBlockCallback(fn: (block: BlockExtended, txIds: string[], transactions: MempoolTransactionExtended[]) => Promise<void>) {
-    this.newAsyncBlockCallbacks.push(fn);
-  }
-
   /**
    * Return the list of transaction for a block
    * @param blockHash
@@ -85,6 +89,8 @@ class Blocks {
    * @param quiet - don't print non-essential logs
    * @param addMempoolData - calculate sigops etc
    * @returns Promise<TransactionExtended[]>
+   *
+   * @asyncUnsafe
    */
   private async $getTransactionsExtended(
     blockHash: string,
@@ -106,7 +112,7 @@ class Blocks {
     const mempool = memPool.getMempool();
     let foundInMempool = 0;
     let totalFound = 0;
-    let missing = 0;
+    const missing = 0;
 
     // Copy existing transactions from the mempool
     if (!onlyCoinbase) {
@@ -245,8 +251,10 @@ class Blocks {
    * @param block
    * @param transactions
    * @returns BlockExtended
+   *
+   * @asyncUnsafe
    */
-  private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
+  public async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[], providedPool?: PoolTag): Promise<BlockExtended> {
     const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
 
     const blk: Partial<BlockExtended> = Object.assign({}, block);
@@ -327,9 +335,11 @@ class Blocks {
       extras.totalInputAmt = null;
     }
 
-    if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
+    if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
       let pool: PoolTag;
-      if (coinbaseTx !== undefined) {
+      if (providedPool) {
+        pool = providedPool;
+      } else if (coinbaseTx !== undefined) {
         pool = await this.$findBlockMiner(coinbaseTx);
       } else {
         if (config.DATABASE.ENABLED === true) {
@@ -352,6 +362,8 @@ class Blocks {
 
         if (extras.pool.name === 'OCEAN') {
           extras.pool.minerNames = parseDATUMTemplateCreator(extras.coinbaseRaw);
+        } else if (extras.pool.name === 'DMND') {
+          extras.pool.minerNames = parseDMNDTemplateCreator(extras.coinbaseRaw);
         }
       }
 
@@ -366,13 +378,21 @@ class Blocks {
           extras.expectedWeight = auditScore.expectedWeight;
         }
       }
+
+      extras.firstSeen = null;
+      if (config.CORE_RPC.DEBUG_LOG_PATH) {
+        const oldestLog = this.getOldestCoreLogTimestamp();
+        if (oldestLog) {
+          extras.firstSeen = getBlockFirstSeenFromLogs(block.id, block.timestamp, oldestLog);
+        }
+      }
     }
 
     blk.extras = <BlockExtension>extras;
     return <BlockExtended>blk;
   }
 
-  private async $getBlockStats(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<IBitcoinApi.BlockStats> {
+  public async $getBlockStats(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<IBitcoinApi.BlockStats> {
     if (!block.stale) {
       return bitcoinClient.getBlockStats(block.id);
     }
@@ -449,6 +469,8 @@ class Blocks {
    * Try to find which miner found the block
    * @param txMinerInfo
    * @returns
+   *
+   * @asyncUnsafe
    */
   private async $findBlockMiner(txMinerInfo: TransactionMinerInfo | undefined): Promise<PoolTag> {
     if (txMinerInfo === undefined || txMinerInfo.vout.length < 1) {
@@ -480,6 +502,129 @@ class Blocks {
     }
   }
 
+  /** @asyncUnsafe */
+  private async $applyBlockTransactionsToMempool(
+    txIds: string[],
+    transactions: MempoolTransactionExtended[]
+  ): Promise<{ rbfTransactions: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }}}> {
+    const _memPool = memPool.getMempool();
+
+    const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
+    memPool.handleRbfTransactions(rbfTransactions);
+    memPool.removeFromSpendMap(transactions);
+
+    if (config.MEMPOOL.CLUSTER_MEMPOOL) {
+      memPool.clusterMempool?.applyMempoolChange({
+        added: [],
+        removed: transactions,
+        accelerations: mempool.getAccelerations(),
+      });
+    }
+
+    for (const txId of txIds) {
+      delete _memPool[txId];
+      rbfCache.mined(txId);
+    }
+    redisCache.queueTransactionsForRemoval(txIds);
+
+    let candidates;
+    let transactionIds: string[];
+
+    if (memPool.limitGBT) {
+      const minFeeMempool = await bitcoinSecondClient.getRawMemPool();
+      const minFeeTip = await bitcoinSecondClient.getBlockCount();
+      candidates = memPool.getNextCandidates(minFeeMempool, minFeeTip, transactions);
+      transactionIds = Object.keys(candidates?.txs || {});
+    } else {
+      candidates = undefined;
+      transactionIds = Object.keys(memPool.getMempool());
+    }
+
+    if (config.MEMPOOL.CLUSTER_MEMPOOL) {
+      const cmBlocks = mempool.clusterMempool?.getBlocks(config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT) ?? [];
+      mempoolBlocks.processClusterMempoolBlocks(cmBlocks, _memPool, mempool.getAccelerations());
+    } else if (config.MEMPOOL.RUST_GBT) {
+      const added = memPool.limitGBT ? (candidates?.added || []) : [];
+      const removed = memPool.limitGBT ? (candidates?.removed || []) : transactions;
+      await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, _memPool, added, removed, candidates, true);
+    } else {
+      await mempoolBlocks.$makeBlockTemplates(transactionIds, _memPool, candidates, true, true);
+    }
+
+    return { rbfTransactions };
+  }
+
+  /** @asyncUnsafe */
+  private async $saveBlockData(
+    processingResult: BlockProcessingResult,
+    timer: number
+  ): Promise<void> {
+    const blockExtended = processingResult.blockExtended;
+    const cpfpSummary = processingResult.cpfpSummary;
+
+    let latestPriceId;
+    try {
+      latestPriceId = await PricesRepository.$getLatestPriceId();
+      this.updateTimerProgress(timer, `got latest price id ${this.currentBlockHeight}`);
+    } catch (e) {
+      logger.debug('failed to fetch latest price id from db: ' + (e instanceof Error ? e.message : e));
+    }
+    if (priceUpdater.historyInserted === true && latestPriceId !== null) {
+      await blocksRepository.$saveBlockPrices([{
+        height: blockExtended.height,
+        priceId: latestPriceId,
+      }]);
+      this.updateTimerProgress(timer, `saved prices for ${this.currentBlockHeight}`);
+    } else {
+      logger.debug(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining);
+      indexer.scheduleSingleTask('blocksPrices', 10000);
+    }
+
+    if (Common.blocksSummariesIndexingEnabled() === true) {
+      // indexes the summary as a side effect
+      await this.$getStrippedBlockTransactions(blockExtended.id, true, false, cpfpSummary, blockExtended.height);
+      this.updateTimerProgress(timer, `saved block summary for ${this.currentBlockHeight}`);
+    }
+
+    if (config.MEMPOOL.CPFP_INDEXING) {
+      // can be slow, and isn't critical, so don't await
+      void this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary);
+      this.updateTimerProgress(timer, `saved cpfp for ${this.currentBlockHeight}`);
+    }
+
+    if (processingResult.auditResult) {
+      void BlocksSummariesRepository.$saveTemplate({
+        height: blockExtended.height,
+        template: {
+          id: blockExtended.id,
+          transactions: processingResult.auditResult.projectedBlocks[0].transactions,
+        },
+        version: 1,
+      });
+      this.updateTimerProgress(timer, `saved audit template for ${this.currentBlockHeight}`);
+
+      void BlocksAuditsRepository.$saveAudit({
+        version: 1,
+        templateAlgorithm: processingResult.templateAlgorithm,
+        time: blockExtended.timestamp,
+        height: blockExtended.height,
+        hash: blockExtended.id,
+        unseenTxs: processingResult.auditResult.unseen,
+        addedTxs: processingResult.auditResult.added,
+        prioritizedTxs: processingResult.auditResult.prioritized,
+        missingTxs: processingResult.auditResult.censored,
+        freshTxs: processingResult.auditResult.fresh,
+        sigopTxs: processingResult.auditResult.sigop,
+        fullrbfTxs: processingResult.auditResult.fullrbf,
+        acceleratedTxs: processingResult.auditResult.accelerated,
+        matchRate: processingResult.auditResult.matchRate,
+        expectedFees: processingResult.auditResult.expectedFees,
+        expectedWeight: processingResult.auditResult.expectedWeight,
+      });
+      this.updateTimerProgress(timer, `saved audit results for ${this.currentBlockHeight}`);
+    }
+  }
+
   /**
    * [INDEXING] Index all blocks summaries for the block txs visualization
    */
@@ -547,6 +692,156 @@ class Blocks {
     }
   }
 
+  /**
+   * [INDEXING] Index all blocks flag values for the goggles graph rendering
+   *
+   *  @asyncSafe
+   */
+  public async $generateFlagValuesDatabase(): Promise<void> {
+    const MAX_BLOCKS_PERQUERY = 144;
+    if (this.indexingFlagValues) {
+      return;
+    }
+
+    if (Common.blocksSummariesIndexingEnabled() === false || Common.isLiquid()) {
+      return;
+    }
+
+    this.indexingFlagValues = true;
+
+    const tipOfSummaries = await BlocksSummariesRepository.$getTipIndexed();
+    if (!tipOfSummaries) {
+      this.indexingFlagValues = false;
+      return;
+    }
+
+    let newlyIndexedBuckets = 0;
+
+    while (this.flagValuesDeleteQueue.length > 0) { // Deletion of in-queue heights due to reorg
+      const deletionHeight = this.flagValuesDeleteQueue.shift();
+      if (deletionHeight === undefined) {
+        continue;
+      }
+      await FlagValueRepository.$deleteFlagValuesFromHeight(deletionHeight);
+    }
+
+    for (const preset of INDEXING_PRESETS) {
+      let seedHeight = preset.retentionSpan > -1 ? tipOfSummaries - preset.retentionSpan : 0;
+      if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT > 0) {
+        seedHeight = Math.max(seedHeight, tipOfSummaries - config.MEMPOOL.INDEXING_BLOCKS_AMOUNT + 1);
+      }
+      const firstBucket = Math.floor((tipOfSummaries + 1) / preset.bucketSize) * preset.bucketSize - preset.bucketSize;
+      const lastBucket = Math.max(0, Math.floor(seedHeight / preset.bucketSize) * preset.bucketSize);
+
+      // Deletion of flag values out of retention span
+      const tipAndTailOfFlagValues = await FlagValueRepository.$getTipAndTailIndexedByBucketSize(preset.bucketSize);
+      if (tipAndTailOfFlagValues && lastBucket > tipAndTailOfFlagValues.tail) { // Drop buckets that fell out of block span
+        logger.debug(`Deleting all the flag values ${preset.name} below height #${lastBucket}`, logger.tags.goggles);
+        await FlagValueRepository.$deleteFlagValuesBelowHeight(lastBucket, preset.bucketSize);
+      }
+
+      if (firstBucket < lastBucket) {
+        continue; // no complete bucket in range
+      }
+
+      const indexedBuckets = await FlagValueRepository.$getIndexedStartHeights(preset.bucketSize, firstBucket, lastBucket);
+      const isBucketIndexed = {};
+      // We map the buckets that are already indexed to skip them
+      for (const startHeight of indexedBuckets) {
+        isBucketIndexed[startHeight] = true;
+      }
+
+      logger.debug(`Processing and indexing flag values from #${firstBucket} to #${lastBucket} ${preset.name}`, logger.tags.goggles);
+
+      let timer = Date.now() / 1000;
+      const startedAt = Date.now() / 1000;
+      let blocksComputedInTotal = 0;
+      let blocksComputedThisRun = 0;
+      const blocksToCompute = firstBucket + preset.bucketSize - lastBucket - (indexedBuckets.length * preset.bucketSize);
+      for (let bucketStart = firstBucket; bucketStart >= lastBucket; bucketStart -= preset.bucketSize) {
+        if (isBucketIndexed[bucketStart]) {
+          continue; // already indexed
+        }
+        try {
+          const bucketFirstHeight = bucketStart + preset.bucketSize - 1;
+          const bucketLastHeight = bucketStart - 1;
+
+          let step = bucketFirstHeight;
+
+          const dataPerFlag: Record<string, Record<string, number>> = {};
+          let sumTimestamps = 0;
+          let nBlocks = 0;
+          let incomplete = false;
+
+          // Incrementalized logic capped by max blocks per query, not bucket size
+          while (step > bucketLastHeight) {
+            const blocksPerQuery = Math.min(step - bucketLastHeight, MAX_BLOCKS_PERQUERY);
+            const cappedLastHeight = step - blocksPerQuery;
+
+            const blocks = await BlocksSummariesRepository.$getSummariesBetweenHeights(step, cappedLastHeight);
+            await Common.sleep$(250); // Don't query/index flag values too fast
+
+            if (!blocks || blocks.length < blocksPerQuery) {
+              incomplete = true;
+              break; // Incomplete bucket
+            }
+
+            // Flag values processing
+            for (const block of blocks) {
+              const txData = JSON.parse(block.transactions).map((tx) => ({flags: tx.flags, vsize: tx.vsize}));
+              for (const data of txData) {
+                if (dataPerFlag[data.flags] === undefined || Object.keys(dataPerFlag[data.flags]).length === 0) {
+                  dataPerFlag[data.flags] = {
+                    txCount: 0,
+                    vSizeTotal: 0
+                  };
+                }
+                dataPerFlag[data.flags].txCount = dataPerFlag[data.flags].txCount + 1;
+                dataPerFlag[data.flags].vSizeTotal = dataPerFlag[data.flags].vSizeTotal + data.vsize;
+              }
+              sumTimestamps += block.timestamp;
+              blocksComputedInTotal++;
+              blocksComputedThisRun++;
+              nBlocks++;
+            }
+
+            // Logging
+            const elapsedSeconds = (Date.now() / 1000) - timer;
+            if (elapsedSeconds > 5) {
+              const runningFor = (Date.now() / 1000) - startedAt;
+              const blocksPerSecond = blocksComputedThisRun / elapsedSeconds;
+              const completion = (blocksComputedInTotal / blocksToCompute) * 100;
+              logger.debug(`Indexing flag values ${preset.name} | ${blocksComputedInTotal}/${blocksToCompute} (${completion.toFixed(2)}%) | ~${blocksPerSecond.toFixed(2)} blocks/sec | elapsed: ${runningFor.toFixed(2)} seconds`,logger.tags.goggles);
+              timer = Date.now() / 1000;
+              blocksComputedThisRun = 0;
+            }
+
+            step -= blocksPerQuery;
+          }
+
+          if (incomplete) {
+            continue;
+          }
+
+          const avgTimestamp = sumTimestamps / nBlocks;
+          await FlagValueRepository.$saveBatchFlagValues(preset.bucketSize, bucketStart, dataPerFlag, avgTimestamp);
+          nBlocks = 0;
+          newlyIndexedBuckets++;
+        } catch (e) {
+          logger.err(`Failed to index flag values between #${bucketStart} and #${bucketStart + preset.bucketSize - 1}. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.goggles);
+        }
+      }
+      logger.debug(`Successfully indexed #${blocksComputedInTotal} blocks ${preset.name} in ${((Date.now() / 1000) - startedAt).toFixed(2)} seconds`, logger.tags.goggles);
+    }
+    if (newlyIndexedBuckets > 0) {
+      logger.notice(`Flag values indexing completed: indexed ${newlyIndexedBuckets} buckets`, logger.tags.goggles);
+    } else {
+      logger.debug(`Flag values indexing completed: indexed ${newlyIndexedBuckets} buckets`, logger.tags.goggles);
+    }
+    this.indexingFlagValues = false;
+  }
+
+  /** @asyncUnsafe */
   public async $indexBlockSummary(hash: string, height: number, stale?: boolean): Promise<void> {
     if (config.MEMPOOL.BACKEND === 'esplora') {
       const txs = (await bitcoinApi.$getTxsForBlock(hash, stale)).map(tx => transactionUtils.extendMempoolTransaction(tx));
@@ -610,6 +905,8 @@ class Blocks {
 
   /**
    * [INDEXING] Index expected fees & weight for all audited blocks
+   *
+   * @asyncUnsafe
    */
   public async $generateAuditStats(): Promise<void> {
     const blockIds = await BlocksAuditsRepository.$getBlocksWithoutSummaries();
@@ -650,6 +947,8 @@ class Blocks {
 
   /**
    * [INDEXING] Index transaction classification flags for Goggles
+   *
+   * @asyncSafe
    */
   public async $classifyBlocks(): Promise<void> {
     if (this.classifyingBlocks) {
@@ -707,7 +1006,8 @@ class Blocks {
           // fetch transactions
           txs = (await bitcoinApi.$getTxsForBlock(blockHash, true)).map(tx => transactionUtils.extendMempoolTransaction(tx)) || [];
           // add CPFP
-          const cpfpSummary = calculateGoodBlockCpfp(height, txs, []);
+          const blockCpfpData = calculateGoodBlockCpfp(height, txs, []);
+          const cpfpSummary = saveCpfpDataToCpfpSummary(txs, blockCpfpData);
           // classify
           const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, height, cpfpSummary.transactions);
           await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 2);
@@ -744,7 +1044,8 @@ class Blocks {
               }
               templateTxs.push(tx || templateTx);
             }
-            const cpfpSummary = calculateGoodBlockCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as MempoolTransactionExtended[], []);
+            const blockCpfpData = calculateGoodBlockCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as MempoolTransactionExtended[], []);
+            const cpfpSummary = saveCpfpDataToCpfpSummary(templateTxs as MempoolTransactionExtended[], blockCpfpData);
             // classify
             const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, height, cpfpSummary.transactions);
             const classifiedTxMap: { [txid: string]: TransactionClassified } = {};
@@ -830,6 +1131,7 @@ class Blocks {
 
   /**
    * [INDEXING] Index all blocks metadata for the mining dashboard
+   * @asyncSafe
    */
   public async $generateBlockDatabase(): Promise<boolean> {
     try {
@@ -907,6 +1209,46 @@ class Blocks {
     return await BlocksRepository.$validateChain();
   }
 
+  /**
+   * [INDEXING] Index all blocks first seen time from Bitcoin Core debug logs
+   * 
+   * @asyncUnsafe
+   */
+  public async $indexBlocksFirstSeen(): Promise<void> {
+    const previous = this.oldestCoreLogTimestamp;
+    const oldestLogTimestamp = this.getOldestCoreLogTimestamp(true);
+    const hasLogFileChanged = previous !== undefined && oldestLogTimestamp !== previous;
+
+    if (!oldestLogTimestamp) {
+      return;
+    }
+
+    // If the log file changed since last run, re-try to index blocks marked with sentinel value
+    const blocks = await BlocksRepository.$getBlocksWithoutFirstSeen(hasLogFileChanged);
+
+    if (!blocks?.length) {
+      return;
+    }
+    logger.debug(`Indexing ${blocks.length} block first seen times${hasLogFileChanged ? ' (log file changed since last run)' : ''}`);
+    const startedAt = Date.now();
+    const results = scanLogsForBlocksFirstSeen(blocks, oldestLogTimestamp);
+    const foundCount = results.filter(result => result.firstSeen !== null).length;
+    logger.debug(`Found first seen times of ${foundCount} / ${results.length} blocks in Core logs, saving to database...`);
+    await BlocksRepository.$saveFirstSeenTimes(results);
+
+    const blocksByHash = new Map<string, BlockExtended>(this.blocks.map<[string, BlockExtended]>(block => [block.id, block]));
+
+    for (const { hash, firstSeen } of results) {
+      const cachedBlock = blocksByHash.get(hash);
+      if (cachedBlock?.extras) {
+        cachedBlock.extras.firstSeen = firstSeen;
+      }
+    }
+
+    logger.debug(`Indexed ${foundCount} / ${blocks.length} block first seen times in ${((Date.now() - startedAt) / 1000).toFixed(2)} seconds`);
+  }
+
+  /** @asyncUnsafe */
   public async $updateBlocks(): Promise<number> {
     // warn if this run stalls the main loop for more than 2 minutes
     const timer = this.startTimer();
@@ -992,59 +1334,55 @@ class Blocks {
         }
       }
 
-      let accelerations = Object.values(mempool.getAccelerations());
-      if (accelerations?.length > 0) {
-        const pool = await this.$findBlockMiner(transactionUtils.stripCoinbaseTransaction(transactions[0]));
-        accelerations = accelerations.filter(a => a.pools.includes(pool.uniqueId));
-      }
-      const cpfpSummary: CpfpSummary = calculateGoodBlockCpfp(block.height, transactions, accelerations.map(a => ({ txid: a.txid, max_bid: a.feeDelta })));
-      const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions);
-      const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, block.height, cpfpSummary.transactions);
+      const pool = await this.$findBlockMiner(transactionUtils.stripCoinbaseTransaction(transactions[0]));
+      const accelerations = mempool.getAccelerations();
+
+      const processingResult = await blockProcessor.$processNewBlock(
+        block,
+        transactions,
+        pool,
+        accelerations
+      );
+
+      const blockExtended = processingResult.blockExtended;
+      const blockSummary = processingResult.blockSummary;
+      const cpfpSummary = processingResult.cpfpSummary;
       this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`);
 
-      if (Common.indexingEnabled()) {
-        if (!fastForwarded) {
-          await this.$handleReorgs(blockExtended, timer);
-        }
+      if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
+        await statistics.runStatistics();
+      }
 
+      const { rbfTransactions } = await this.$applyBlockTransactionsToMempool(txIds, cpfpSummary.transactions);
+      this.updateTimerProgress(timer, `applied mempool changes for ${this.currentBlockHeight}`);
+
+      if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
+        await statistics.runStatistics();
+      }
+
+      if (Common.indexingEnabled() && !fastForwarded) {
+        await this.$handleReorgs(blockExtended, timer);
+      }
+
+      await websocketHandler.handleNewBlock(blockExtended, txIds, cpfpSummary.transactions, rbfTransactions);
+      this.updateTimerProgress(timer, `sent websocket updates for ${this.currentBlockHeight}`);
+
+      if (Common.indexingEnabled()) {
         await blocksRepository.$saveBlockInDatabase(blockExtended);
         this.updateTimerProgress(timer, `saved ${this.currentBlockHeight} to database`);
 
-        if (!fastForwarded) {
-          let lastestPriceId;
-          try {
-            lastestPriceId = await PricesRepository.$getLatestPriceId();
-            this.updateTimerProgress(timer, `got latest price id ${this.currentBlockHeight}`);
-          } catch (e) {
-            logger.debug('failed to fetch latest price id from db: ' + (e instanceof Error ? e.message : e));
-          }
-          if (priceUpdater.historyInserted === true && lastestPriceId !== null) {
-            await blocksRepository.$saveBlockPrices([{
-              height: blockExtended.height,
-              priceId: lastestPriceId,
-            }]);
-            this.updateTimerProgress(timer, `saved prices for ${this.currentBlockHeight}`);
-          } else {
-            logger.debug(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining);
-            indexer.scheduleSingleTask('blocksPrices', 10000);
-          }
+        await AccelerationRepository.$indexAccelerationsForBlock(
+          blockExtended,
+          Object.values(accelerations),
+          cpfpSummary.transactions
+        );
+        this.updateTimerProgress(timer, `indexed accelerations for ${this.currentBlockHeight}`);
 
-          // Save blocks summary for visualization if it's enabled
-          if (Common.blocksSummariesIndexingEnabled() === true) {
-            await this.$getStrippedBlockTransactions(blockExtended.id, true, false, cpfpSummary, blockExtended.height);
-            this.updateTimerProgress(timer, `saved block summary for ${this.currentBlockHeight}`);
-          }
-          if (config.MEMPOOL.CPFP_INDEXING) {
-            this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary);
-            this.updateTimerProgress(timer, `saved cpfp for ${this.currentBlockHeight}`);
-          }
+        if (!fastForwarded) {
+          await this.$saveBlockData(processingResult, timer);
         }
       }
 
-      // start async callbacks
-      this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
-      const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, cpfpSummary.transactions));
-
       if (block.height % 2016 === 0) {
         if (Common.indexingEnabled()) {
           let adjustment;
@@ -1082,11 +1420,6 @@ class Blocks {
         await chainTips.updateOrphanedBlocks();
       }
 
-      // wait for pending async callbacks to finish
-      this.updateTimerProgress(timer, `waiting for async callbacks to complete for ${this.currentBlockHeight}`);
-      await Promise.all(callbackPromises);
-      this.updateTimerProgress(timer, `async callbacks completed for ${this.currentBlockHeight}`);
-
       this.blocks.push(blockExtended);
       if (this.blocks.length > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4) {
         this.blocks = this.blocks.slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4);
@@ -1103,14 +1436,14 @@ class Blocks {
         this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
       }
       if (config.MEMPOOL.CACHE_ENABLED && !memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
-        diskCache.$saveCacheToDisk();
+        void diskCache.$saveCacheToDisk();
       }
 
       // Update Redis cache
       if (config.REDIS.ENABLED) {
         await redisCache.$updateBlocks(this.blocks);
         await redisCache.$updateBlockSummaries(this.blockSummaries);
-        await redisCache.$removeTransactions(txIds);
+        await redisCache.$removeTransactions();
         await rbfCache.updateCache();
       }
 
@@ -1146,6 +1479,7 @@ class Blocks {
     }
   }
 
+  /** @asyncUnsafe */
   private async updateQuarterEpochBlockTime(): Promise<void> {
     if (this.currentBlockHeight >= 503) {
       try {
@@ -1159,6 +1493,10 @@ class Blocks {
     }
   }
 
+  /**
+   * Index a block if it's missing from the database. Returns the block after indexing
+   * @asyncUnsafe
+   */
   public async $indexBlockByHeight(height: number, skipDb = false): Promise<BlockExtended> {
     if (Common.indexingEnabled() && !skipDb) {
       const dbBlock = await blocksRepository.$getBlockByHeight(height);
@@ -1171,6 +1509,7 @@ class Blocks {
     return this.$indexBlock(hash);
   }
 
+  /** @asyncUnsafe */
   private async $handleReorgs(blockExtended: BlockExtended, timer: any): Promise<void> {
     let forkTail = blockExtended;
     let currentlyIndexed = await blocksRepository.$getBlockByHeight(forkTail.height - 1);
@@ -1222,6 +1561,7 @@ class Blocks {
       await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(forkTail.height);
       await cpfpRepository.$deleteClustersFrom(forkTail.height);
       await AccelerationRepository.$deleteAccelerationsFrom(forkTail.height);
+      this.flagValuesDeleteQueue.push(forkTail.height);
       chainTips.clearOrphanCacheAboveHeight(forkTail.height);
       this.updateTimerProgress(timer, `deleted stale block data`);
 
@@ -1242,6 +1582,8 @@ class Blocks {
 
   /**
    * Index a block if it's missing from the database. Returns the block after indexing
+   *
+   * @asyncUnsafe
    */
   public async $indexBlock(hash: string, block?: IEsploraApi.Block, skipDb = false): Promise<BlockExtended> {
     if (Common.indexingEnabled() && !skipDb) {
@@ -1271,6 +1613,7 @@ class Blocks {
 
   /**
    * Get one block by its hash
+   * @asyncUnsafe
    */
   public async $getBlock(hash: string, skipMemoryCache: boolean = false): Promise<BlockExtended | IEsploraApi.Block> {
     // Check the memory cache
@@ -1282,7 +1625,7 @@ class Blocks {
     }
 
     // Not Bitcoin network, return the block as it from the bitcoin backend
-    if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) === false) {
+    if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) === false) {
       return await bitcoinCoreApi.$getBlock(hash);
     }
 
@@ -1290,6 +1633,7 @@ class Blocks {
     return await this.$indexBlock(hash);
   }
 
+  /** @asyncUnsafe */
   public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,
     skipDBLookup = false, cpfpSummary?: CpfpSummary, blockHeight?: number): Promise<TransactionClassified[]>
   {
@@ -1358,6 +1702,7 @@ class Blocks {
     return summary.transactions;
   }
 
+  /** @asyncUnsafe */
   public async $getSingleTxFromSummary(hash: string, txid: string): Promise<TransactionClassified | null> {
     const txs = await this.$getStrippedBlockTransactions(hash);
     return txs.find(tx => tx.txid === txid) || null;
@@ -1365,15 +1710,16 @@ class Blocks {
 
   /**
    * Get 15 blocks
-   * 
+   *
    * Internally this function uses two methods to get the blocks, and
    * the method is automatically selected:
    *  - Using previous block hash links
    *  - Using block height
-   * 
-   * @param fromHeight 
-   * @param limit 
-   * @returns 
+   *
+   * @param fromHeight
+   * @param limit
+   * @returns
+   * @asyncUnsafe
    */
   public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
     let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight;
@@ -1405,9 +1751,10 @@ class Blocks {
 
   /**
    * Used for bulk block data query
-   * 
-   * @param fromHeight 
-   * @param toHeight 
+   *
+   * @param fromHeight
+   * @param toHeight
+   * @asyncUnsafe
    */
   public async $getBlocksBetweenHeight(fromHeight: number, toHeight: number): Promise<any> {
     if (!Common.indexingEnabled()) {
@@ -1525,7 +1872,7 @@ class Blocks {
   }
 
   public async $getBlockAuditSummary(hash: string): Promise<BlockAudit | null> {
-    if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) && Common.auditIndexingEnabled()) {
+    if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) && Common.auditIndexingEnabled()) {
       return BlocksAuditsRepository.$getBlockAudit(hash);
     } else {
       return null;
@@ -1533,7 +1880,7 @@ class Blocks {
   }
 
   public async $getBlockTxAuditSummary(hash: string, txid: string): Promise<TransactionAudit | null> {
-    if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) && Common.auditIndexingEnabled()) {
+    if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) && Common.auditIndexingEnabled()) {
       return BlocksAuditsRepository.$getBlockTxAudit(hash, txid);
     } else {
       return null;
@@ -1556,6 +1903,7 @@ class Blocks {
     return this.currentBlockHeight;
   }
 
+/** @asyncUnsafe */
   public async $indexCPFP(hash: string, height: number, txs?: MempoolTransactionExtended[], stale?: boolean): Promise<CpfpSummary | null> {
     let transactions = txs;
     if (!transactions) {
@@ -1572,22 +1920,23 @@ class Blocks {
     }
 
     if (transactions?.length != null) {
-      const summary = calculateFastBlockCpfp(height, transactions);
+      const { cpfpSummary } = await detectTemplateAlgorithm(height, transactions, [], true);
 
-      if (!stale) {
-        await this.$saveCpfp(hash, height, summary);
+      if (!stale && Common.cpfpIndexingEnabled() === true) {
+        await this.$saveCpfp(hash, height, cpfpSummary);
       }
 
-      const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions);
+      const effectiveFeeStats = Common.calcEffectiveFeeStatistics(cpfpSummary.transactions);
       await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats);
 
-      return summary;
+      return cpfpSummary;
     } else {
       logger.err(`Cannot index CPFP for block ${height} - missing transaction data`);
       return null;
     }
   }
 
+  /** @asyncSafe */
   public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise<void> {
     try {
       const result = await cpfpRepository.$batchSaveClusters(cpfpSummary.clusters);
@@ -1628,6 +1977,30 @@ class Blocks {
       return null;
     }
   }
+
+  public getOldestCoreLogTimestamp(forceRefresh = false): number | null {
+    if (!forceRefresh && this.oldestCoreLogTimestamp !== undefined) {
+      return this.oldestCoreLogTimestamp;
+    }
+    const debugLogPath = config.CORE_RPC.DEBUG_LOG_PATH;
+    if (!debugLogPath) {
+      this.oldestCoreLogTimestamp = null;
+      return null;
+    }
+    try {
+      this.oldestCoreLogTimestamp = getOldestLogTimestampFromLogs(debugLogPath);
+      if (this.oldestCoreLogTimestamp !== null) {
+        logger.info(`Core debug log entries date back to ${new Date(this.oldestCoreLogTimestamp * 1000).toISOString()}`);
+      } else {
+        logger.err(`Could not find oldest timestamp in Core debug log file at ${debugLogPath}`);
+      }
+      return this.oldestCoreLogTimestamp;
+    } catch (e) {
+      this.oldestCoreLogTimestamp = null;
+      logger.err(`Could not read Core debug log file at ${debugLogPath}. Reason: ${e instanceof Error ? e.message : e}`);
+      return null;
+    }
+  }
 }
 
 export default new Blocks();

backend/src/api/chain-tips.ts

@@ -2,6 +2,7 @@ import config from '../config';
 import logger from '../logger';
 import { BlockExtended } from '../mempool.interfaces';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
+import BlocksRepository from '../repositories/BlocksRepository';
 import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 import bitcoinClient from './bitcoin/bitcoin-client';
 import { IEsploraApi } from './bitcoin/esplora-api.interface';
@@ -23,25 +24,29 @@ export interface StaleTip extends ChainTip {
 export interface OrphanedBlock {
   height: number;
   hash: string;
+  branchlen: number;
   status: 'valid-fork' | 'valid-headers' | 'headers-only';
   prevhash: string;
 }
 
 class ChainTips {
   private chainTips: ChainTip[] = [];
-  private staleTips: Record<number, StaleTip> = {};
+  private validChainTips: ChainTip[] = []; // 'valid-fork' and 'valid-headers' only, in descending height order
+  private staleBlocks: Record<string, BlockExtended> = {};
   private orphanedBlocks: { [hash: string]: OrphanedBlock } = {};
   private blockCache: { [hash: string]: OrphanedBlock } = {};
   private orphansByHeight: { [height: number]: OrphanedBlock[] } = {};
   private indexingOrphanedBlocks = false;
   private indexingQueue: { blockhash?: string, block?: IEsploraApi.Block, tip: OrphanedBlock }[] = [];
 
-  private staleTipsCacheSize = 50;
+  private staleBlocksCacheSize = 50;
   private maxIndexingQueueSize = 100;
 
+  /** @asyncSafe */
   public async updateOrphanedBlocks(): Promise<void> {
     try {
       this.chainTips = await bitcoinClient.getChainTips();
+      this.validChainTips = this.chainTips.filter(tip => tip.status === 'valid-fork' || tip.status === 'valid-headers').sort((a, b) => b.height - a.height);
 
       const activeTipHeight = this.chainTips.find(tip => tip.status === 'active')?.height || (await bitcoinApi.$getBlockHeightTip());
       let minIndexHeight = 0;
@@ -68,6 +73,7 @@ class ChainTips {
                 orphan = {
                   height: block.height,
                   hash: block.id,
+                  branchlen: chain.branchlen,
                   status: chain.status,
                   prevhash: block.previousblockhash,
                 };
@@ -118,13 +124,7 @@ class ChainTips {
         this.orphansByHeight[orphan.height].push(orphan);
       }
 
-      const heightsToKeep = new Set(this.chainTips.filter(tip => tip.status !== 'active').map(tip => tip.height));
-      const heightsToRemove: number[] = Object.keys(this.staleTips).map(Number).filter(height => !heightsToKeep.has(height));
-      for (const height of heightsToRemove) {
-        delete this.staleTips[height];
-      }
-
-      this.trimStaleTipsCache();
+      this.trimStaleBlocksCache();
 
       // index new orphaned blocks in the background
       void this.$indexOrphanedBlocks();
@@ -135,6 +135,7 @@ class ChainTips {
     }
   }
 
+  /** @asyncSafe */
   private async $indexOrphanedBlocks(): Promise<void> {
     if (this.indexingOrphanedBlocks) {
       return;
@@ -146,16 +147,16 @@ class ChainTips {
       if (!block && !blockhash) {
         continue;
       }
-      if (blockhash && !block) {
-        block = await bitcoinCoreApi.$getBlock(blockhash);
-      }
-      if (!block) {
-        continue;
-      }
       try {
+        if (blockhash && !block) {
+          block = await bitcoinCoreApi.$getBlock(blockhash);
+        }
+        if (!block) {
+          continue;
+        }
         let staleBlock: BlockExtended | undefined;
         const alreadyIndexed = await BlocksSummariesRepository.$isSummaryIndexed(block.id);
-        const needToCache = Object.keys(this.staleTips).length < this.staleTipsCacheSize || block.height > Object.keys(this.staleTips).map(Number).sort((a, b) => b - a)[this.staleTipsCacheSize - 1];
+        const needToCache = this.shouldCacheStaleBlock(block.id, block.height);
         if (!alreadyIndexed) {
           staleBlock = await blocks.$indexBlock(block.id, block, true);
           await blocks.$indexBlockSummary(block.id, block.height, true);
@@ -166,30 +167,53 @@ class ChainTips {
         }
 
         if (staleBlock && needToCache) {
-          const canonicalBlock = await blocks.$indexBlockByHeight(staleBlock.height);
-          this.staleTips[staleBlock.height] = {
-            height: staleBlock.height,
-            hash: staleBlock.id,
-            branchlen: tip.height - staleBlock.height,
-            status: tip.status,
-            stale: staleBlock,
-            canonical: canonicalBlock,
-          };
-          this.trimStaleTipsCache();
+          // ensure the canonical block is correctly indexed
+          await blocks.$indexBlockByHeight(staleBlock.height);
+          this.cacheStaleBlock(staleBlock);
         }
       } catch (e) {
-        logger.err(`Failed to index orphaned block ${block.id} at height ${block.height}. Reason: ${e instanceof Error ? e.message : e}`);
+        logger.err(`Failed to index orphaned block ${block?.id} at height ${block?.height}. Reason: ${e instanceof Error ? e.message : e}`);
       }
     }
     this.indexingOrphanedBlocks = false;
   }
 
-  private trimStaleTipsCache(): void {
-    const staleTipHeights = Object.keys(this.staleTips).map(Number).sort((a, b) => b - a);
-    if (staleTipHeights.length > this.staleTipsCacheSize) {
-      const heightsToDiscard = staleTipHeights.slice(this.staleTipsCacheSize);
-      for (const height of heightsToDiscard) {
-        delete this.staleTips[height];
+  private shouldCacheStaleBlock(hash: string, height: number): boolean {
+    // already cached
+    if (this.staleBlocks[hash]) {
+      return false;
+    }
+
+    // cache is not full
+    const cachedBlocks = Object.values(this.staleBlocks);
+    if (cachedBlocks.length < this.staleBlocksCacheSize) {
+      return true;
+    }
+
+    // otherwise cache if this block is newer than the oldest in the cache
+    const oldestCachedHeight = cachedBlocks.reduce((min, block) => Math.min(min, block.height), Infinity);
+    return height >= oldestCachedHeight;
+  }
+
+  private cacheStaleBlock(block: BlockExtended): void {
+    this.staleBlocks[block.id] = block;
+    this.trimStaleBlocksCache();
+  }
+
+  // evict the oldest stale blocks until the cache is within the size limit
+  private trimStaleBlocksCache(): void {
+    // sort by height
+    const cachedBlocks = Object.values(this.staleBlocks).sort((a, b) => {
+      if (b.height !== a.height) {
+        return b.height - a.height;
+      }
+      // tie-break by hash
+      return a.id.localeCompare(b.id);
+    });
+    // delete everything beyond the size limit
+    if (cachedBlocks.length > this.staleBlocksCacheSize) {
+      for (const block of cachedBlocks.slice(this.staleBlocksCacheSize)) {
+        delete this.staleBlocks[block.id];
       }
     }
   }
@@ -206,8 +230,51 @@ class ChainTips {
     return this.chainTips;
   }
 
-  public getStaleTips(): StaleTip[] {
-    return Object.values(this.staleTips).sort((a, b) => b.height - a.height);
+  /**
+   * get paginated stale chain tips
+   * @param fromHeight - start height (exclusive)
+   * @param count - requested page size (target, but not strictly enforced)
+   *
+   * @asyncSafe
+   */
+  public async $getStaleTipsPage(fromHeight: number | undefined, count: number): Promise<StaleTip[]> {
+    const start = fromHeight === undefined ? 0 : this.validChainTips.findIndex(tip => tip.height < fromHeight);
+    // no tips beyond the requested height, we can return early
+    if (start === -1) {
+      return [];
+    }
+
+    // fill the response array with hydrated tip data
+    const tips: StaleTip[] = [];
+    let lastHeight;
+    for (let index = start; index < this.validChainTips.length; index++) {
+      const staleTip = this.validChainTips[index];
+      // stretch the page to include any remaining blocks at the last included height to avoid pagination gaps with a height-based cursor
+      if (tips.length >= count) {
+        if (staleTip.height !== lastHeight) {
+          break;
+        }
+      }
+      // fetch blocks from caches if available, or DB otherwise
+      const canonical = blocks.getBlocks().find(block => block.height === staleTip.height) || await BlocksRepository.$getBlockByHeight(staleTip.height);
+      let stale: BlockExtended | null | undefined = this.staleBlocks[staleTip.hash];
+      if (!stale) {
+        stale = await BlocksRepository.$getBlockByHash(staleTip.hash);
+      }
+      // skip tips with missing block data
+      if (!canonical || !stale) {
+        continue;
+      }
+
+      tips.push({
+        ...staleTip,
+        stale,
+        canonical,
+      });
+      lastHeight = staleTip.height;
+    }
+
+    return tips;
   }
 
   clearOrphanCacheAboveHeight(height: number): void {
@@ -232,4 +299,4 @@ class ChainTips {
   }
 }
 
-export default new ChainTips();
+export default new ChainTips();

backend/src/api/common.ts

@@ -376,6 +376,7 @@ export class Common {
     'testnet4': 42_000,
     'testnet': 2_900_000,
     'signet': 211_000,
+    'regtest': 0,
     '': 863_500,
   };
   static isNonStandardVersion(tx: TransactionExtended, height?: number): boolean {
@@ -399,6 +400,7 @@ export class Common {
     'testnet4': 42_000,
     'testnet': 2_900_000,
     'signet': 211_000,
+    'regtest': 0,
     '': 863_500,
   };
   static isNonStandardAnchor(vin: IEsploraApi.Vin, height?: number): boolean {
@@ -419,6 +421,7 @@ export class Common {
     'testnet4': 90_500,
     'testnet': 4_550_000,
     'signet': 260_000,
+    'regtest': 0,
     '': 905_000,
   };
   static isStandardEphemeralDust(tx: TransactionExtended, height?: number): boolean {
@@ -439,6 +442,7 @@ export class Common {
     'testnet4': 108_000,
     'testnet': 4_750_000,
     'signet': 276_500,
+    'regtest': 0,
     '': 921_000,
   };
   static MAX_DATACARRIER_BYTES = 83;
@@ -461,6 +465,7 @@ export class Common {
     'testnet4': 108_000,
     'testnet': 4_750_000,
     'signet': 276_500,
+    'regtest': 0,
     '': 921_000,
   };
   static isNonStandardLegacySigops(tx: TransactionExtended, height?: number): boolean {
@@ -744,7 +749,15 @@ export class Common {
     // fast but bad heuristic to detect possible coinjoins
     // (at least 5 inputs and 5 outputs, less than half of which are unique amounts, with no address reuse)
     const addressReuse = Object.keys(reusedOutputAddresses).reduce((acc, key) => Math.max(acc, (reusedInputAddresses[key] || 0) + (reusedOutputAddresses[key] || 0)), 0) > 1;
-    if (!addressReuse && tx.vin.length >= 5 && tx.vout.length >= 5 && (Object.keys(inValues).length + Object.keys(outValues).length) <= (tx.vin.length + tx.vout.length) / 2 ) {
+    const tokenRelated = (flags & (TransactionFlags.inscription | TransactionFlags.op_return)) !== 0n;
+    if (!addressReuse &&
+        tx.vin.length >= 5 &&
+        tx.vout.length >= 5 &&
+        (Object.keys(inValues).length + Object.keys(outValues).length) <= (tx.vin.length + tx.vout.length) / 2 &&
+        !tokenRelated &&
+        tx.vin.length / tx.vout.length < 5 &&
+        tx.vin.length / tx.vout.length > 0.2
+      ) {
       flags |= TransactionFlags.coinjoin;
     }
     // more than 5:1 input:output ratio
@@ -797,6 +810,7 @@ export class Common {
     return txs.map(Common.stripTransaction);
   }
 
+  /** @asyncSafe */
   static sleep$(ms: number): Promise<void> {
     return new Promise((resolve) => {
        setTimeout(() => {
@@ -854,7 +868,7 @@ export class Common {
 
   static indexingEnabled(): boolean {
     return (
-      ['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) &&
+      ['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) &&
       config.DATABASE.ENABLED === true &&
       config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== 0
     );
@@ -911,7 +925,7 @@ export class Common {
     if (id.indexOf('/') !== -1) {
       id = id.slice(0, -2);
     }
-    
+
     if (id.indexOf('x') !== -1) { // Already a short id
       return id;
     }
@@ -1024,8 +1038,18 @@ export class Common {
   }
 
   static calcEffectiveFeeStatistics(transactions: { weight: number, fee?: number, effectiveFeePerVsize?: number, txid: string, acceleration?: boolean }[]): EffectiveFeeStats {
-    const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate);
-    const totalWeight = transactions.reduce((acc, tx) => acc + tx.weight, 0);
+    // return early with safe default values
+    if (transactions.length <= 1) {
+      return {
+        medianFee: 0,
+        feeRange: [0, 0, 0, 0, 0, 0, 0],
+      };
+    }
+    // assume the first transaction is a coinbase if the fee is falsy (0 or undefined)
+    const nonCoinbaseTransactions = transactions[0].fee ? transactions : transactions.slice(1);
+
+    const sortedTxs = nonCoinbaseTransactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate);
+    const totalWeight = nonCoinbaseTransactions.reduce((acc, tx) => acc + tx.weight, 0);
 
     // include any unused space
     let weightCount = config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight;
@@ -1054,7 +1078,7 @@ export class Common {
     // b) the minimum effective fee rate in the last 2% of transactions (in block order)
     const minFee = Math.min(
       Common.getNthPercentile(1, sortedTxs).rate,
-      transactions.slice(-transactions.length / 50).reduce((min, tx) => { return Math.min(min, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, Infinity)
+      nonCoinbaseTransactions.slice(Math.ceil(nonCoinbaseTransactions.length * 49 / 50)).reduce((min, tx) => { return Math.min(min, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, Infinity)
     );
 
     // maximum effective fee heuristic:
@@ -1063,7 +1087,7 @@ export class Common {
     // b) the maximum effective fee rate in the first 2% of transactions (in block order)
     const maxFee = Math.max(
       Common.getNthPercentile(99, sortedTxs).rate,
-      transactions.slice(0, transactions.length / 50).reduce((max, tx) => { return Math.max(max, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, 0)
+      nonCoinbaseTransactions.slice(0, nonCoinbaseTransactions.length / 50).reduce((max, tx) => { return Math.max(max, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, 0)
     );
 
     return {
@@ -1077,11 +1101,14 @@ export class Common {
   }
 
   static getNthPercentile(n: number, sortedDistribution: any[]): any {
+    if (sortedDistribution.length === 0) {
+      return { rate: 0 };
+    }
     return sortedDistribution[Math.floor((sortedDistribution.length - 1) * (n / 100))];
   }
 
   static getTransactionFromRequest(req: Request, form: boolean): string {
-    let rawTx: any = typeof req.body === 'object' && form
+    const rawTx: any = typeof req.body === 'object' && form
       ? Object.values(req.body)[0] as any
       : req.body;
     if (typeof rawTx !== 'string') {
@@ -1182,7 +1209,7 @@ export class Common {
             }
           }
         }
-      })
+      });
     }
 
     // Pass through the input string untouched
@@ -1220,14 +1247,14 @@ export class Common {
 /**
  * Class to calculate average fee rates of a list of transactions
  * at certain weight percentiles, in a single pass
- * 
+ *
  * init with:
  *   maxWeight - the total weight to measure percentiles relative to (e.g. 4MW for a single block)
  *   percentileBandWidth - how many weight units to average over for each percentile (as a % of maxWeight)
  *   percentiles - an array of weight percentiles to compute, in %
- * 
+ *
  * then call .processNext(tx) for each transaction, in descending order
- * 
+ *
  * retrieve the final results with .getFeeStats()
  */
 export class OnlineFeeStatsCalculator {

backend/src/api/cpfp.ts

@@ -1,20 +1,30 @@
-import { Ancestor, CpfpCluster, CpfpInfo, CpfpSummary, MempoolTransactionExtended, TransactionExtended } from '../mempool.interfaces';
+import { Ancestor, CpfpCluster, CpfpInfo, MempoolTransactionExtended, TemplateAlgorithm, TransactionExtended } from '../mempool.interfaces';
 import { GraphTx, convertToGraphTx, expandRelativesGraph, initializeRelatives, makeBlockTemplate, mempoolComparator, removeAncestors, setAncestorScores } from './mini-miner';
 import memPool from './mempool';
 import { Acceleration } from './acceleration/acceleration';
+import { ClusterMempool } from '../cluster-mempool/cluster-mempool';
 
 const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction
 const MAX_CLUSTER_ITERATIONS = 100;
 
-export function calculateFastBlockCpfp(height: number, transactions: MempoolTransactionExtended[], saveRelatives: boolean = false): CpfpSummary {
+type TransactionCpfpData = Partial<CpfpInfo & { cpfpDirty?: boolean, clusterId?: number, chunkIndex?: number }>;
+export interface BlockCpfpData {
+  txs: Record<string, TransactionCpfpData>,
+  clusters: CpfpCluster[];
+  version: number;
+}
+
+export function calculateFastBlockCpfp(height: number, transactions: MempoolTransactionExtended[], saveRelatives: boolean = false): BlockCpfpData {
   const clusters: CpfpCluster[] = []; // list of all cpfp clusters in this block
   const clusterMap: { [txid: string]: CpfpCluster } = {}; // map transactions to their cpfp cluster
   let clusterTxs: TransactionExtended[] = []; // working list of elements of the current cluster
   let ancestors: { [txid: string]: boolean } = {}; // working set of ancestors of the current cluster root
   const txMap: { [txid: string]: TransactionExtended } = {};
+  const cpfpData: Record<string, TransactionCpfpData> = {};
   // initialize the txMap
   for (const tx of transactions) {
     txMap[tx.txid] = tx;
+    cpfpData[tx.txid] = {};
   }
   // reverse pass to identify CPFP clusters
   for (let i = transactions.length - 1; i >= 0; i--) {
@@ -38,7 +48,9 @@ export function calculateFastBlockCpfp(height: number, transactions: MempoolTran
         clusters.push(cluster);
       }
       clusterTxs.forEach(tx => {
-        txMap[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize;
+        cpfpData[tx.txid] = {
+          effectiveFeePerVsize
+        };
         if (cluster) {
           clusterMap[tx.txid] = cluster;
         }
@@ -54,17 +66,19 @@ export function calculateFastBlockCpfp(height: number, transactions: MempoolTran
   }
   // forward pass to enforce ancestor rate caps
   for (const tx of transactions) {
-    let minAncestorRate = tx.effectiveFeePerVsize;
+    const txRate = cpfpData[tx.txid]?.effectiveFeePerVsize ?? tx.effectiveFeePerVsize;
+    let minAncestorRate = txRate;
     for (const vin of tx.vin) {
-      if (txMap[vin.txid]?.effectiveFeePerVsize) {
-        minAncestorRate = Math.min(minAncestorRate, txMap[vin.txid].effectiveFeePerVsize);
+      const vinRate = cpfpData[vin.txid]?.effectiveFeePerVsize ?? txMap[vin.txid]?.effectiveFeePerVsize;
+      if (vinRate) {
+        minAncestorRate = Math.min(minAncestorRate, vinRate);
       }
     }
     // check rounded values to skip cases with almost identical fees
     const roundedMinAncestorRate = Math.ceil(minAncestorRate);
-    const roundedEffectiveFeeRate = Math.floor(tx.effectiveFeePerVsize);
+    const roundedEffectiveFeeRate = Math.floor(txRate);
     if (roundedMinAncestorRate < roundedEffectiveFeeRate) {
-      tx.effectiveFeePerVsize = minAncestorRate;
+      cpfpData[tx.txid].effectiveFeePerVsize = minAncestorRate;
       if (!clusterMap[tx.txid]) {
         // add a single-tx cluster to record the dependent rate
         const cluster = {
@@ -84,23 +98,25 @@ export function calculateFastBlockCpfp(height: number, transactions: MempoolTran
   if (saveRelatives) {
     for (const cluster of clusters) {
       cluster.txs.forEach((member, index) => {
-        txMap[member.txid].descendants = cluster.txs.slice(0, index).reverse();
-        txMap[member.txid].ancestors = cluster.txs.slice(index + 1).reverse();
-        txMap[member.txid].effectiveFeePerVsize = cluster.effectiveFeePerVsize;
+        cpfpData[member.txid].descendants = cluster.txs.slice(0, index).reverse();
+        cpfpData[member.txid].ancestors = cluster.txs.slice(index + 1).reverse();
+        cpfpData[member.txid].effectiveFeePerVsize = cluster.effectiveFeePerVsize;
       });
     }
   }
   return {
-    transactions,
+    txs: cpfpData,
     clusters,
     version: 1,
   };
 }
 
-export function calculateGoodBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): CpfpSummary {
+export function calculateGoodBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): BlockCpfpData {
   const txMap: { [txid: string]: MempoolTransactionExtended } = {};
+  const cpfpData: Record<string, TransactionCpfpData> = {};
   for (const tx of transactions) {
     txMap[tx.txid] = tx;
+    cpfpData[tx.txid] = {};
   }
   const template = makeBlockTemplate(transactions, accelerations, 1, Infinity, Infinity);
   const clusters = new Map<string, string[]>();
@@ -110,15 +126,15 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran
     if (cluster.length > 1 && root && !clusters.has(root)) {
       clusters.set(root, cluster);
     }
-    txMap[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize;
+    cpfpData[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize;
   }
 
   const clusterArray: CpfpCluster[] = [];
 
   for (const cluster of clusters.values()) {
     for (const txid of cluster) {
-      const mempoolTx = txMap[txid];
-      if (mempoolTx) {
+      const mempoolTxCpfpData = cpfpData[txid];
+      if (mempoolTxCpfpData) {
         const ancestors: Ancestor[] = [];
         const descendants: Ancestor[] = [];
         let matched = false;
@@ -138,10 +154,10 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran
             }
           }
         });
-        if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) {
-          mempoolTx.cpfpDirty = true;
+        if (mempoolTxCpfpData.ancestors?.length !== ancestors.length || mempoolTxCpfpData.descendants?.length !== descendants.length) {
+          mempoolTxCpfpData.cpfpDirty = true;
         }
-        Object.assign(mempoolTx, { ancestors, descendants, bestDescendant: null, cpfpChecked: true });
+        Object.assign(mempoolTxCpfpData, { ancestors, descendants, bestDescendant: null, cpfpChecked: true });
       }
     }
     const root = cluster[cluster.length - 1];
@@ -153,17 +169,75 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran
         fee: txMap[txid].fee,
         weight: (txMap[txid].adjustedVsize * 4) || txMap[txid].weight,
       })),
-      effectiveFeePerVsize: txMap[root].effectiveFeePerVsize,
+      effectiveFeePerVsize: cpfpData[root].effectiveFeePerVsize ?? txMap[root].effectiveFeePerVsize,
     });
   }
 
   return {
-    transactions: transactions.map(tx => txMap[tx.txid]),
+    txs: cpfpData,
     clusters: clusterArray,
     version: 2,
   };
 }
 
+export function calculateClusterMempoolBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): BlockCpfpData {
+  const txMap: { [txid: string]: MempoolTransactionExtended } = {};
+  const cpfpData: Record<string, TransactionCpfpData> = {};
+  for (const tx of transactions) {
+    txMap[tx.txid] = tx;
+    cpfpData[tx.txid] = {};
+  }
+
+  const accelMap: { [txid: string]: { feeDelta: number } } = {};
+  for (const acc of accelerations) {
+    accelMap[acc.txid] = { feeDelta: acc.max_bid };
+  }
+
+  const cm = new ClusterMempool(txMap, accelMap, false, 25000);
+
+  const seenClusters = new Set<number>();
+  const clusters: CpfpCluster[] = [];
+
+  for (const txid in cpfpData) {
+    const txCpfpData = cm.getCpfpDataForTx(txid);
+    if (!txCpfpData) {
+      continue;
+    }
+    cpfpData[txid].effectiveFeePerVsize = txCpfpData.effectiveFeePerVsize;
+    cpfpData[txid].clusterId = txCpfpData.clusterId;
+    cpfpData[txid].chunkIndex = txCpfpData.chunkIndex;
+    cpfpData[txid].ancestors = txCpfpData.ancestors;
+    cpfpData[txid].descendants = txCpfpData.descendants;
+    if (txCpfpData.clusterId !== undefined && !seenClusters.has(txCpfpData.clusterId)) {
+      seenClusters.add(txCpfpData.clusterId);
+
+      const clusterData = cm.getCluster(txCpfpData.clusterId);
+      if (clusterData && clusterData.txs.length > 1) {
+        let totalFee = 0;
+        let totalWeight = 0;
+        for (const t of clusterData.txs) {
+          totalFee += t.fee;
+          totalWeight += t.weight;
+        }
+        clusters.push({
+          root: clusterData.txs[0].txid,
+          height,
+          txs: clusterData.txs.map(t => ({ txid: t.txid, weight: t.weight, fee: t.fee })),
+          effectiveFeePerVsize: totalFee / (totalWeight / 4),
+          templateAlgorithm: TemplateAlgorithm.clusterMempool,
+          clusterData,
+        });
+      }
+    }
+  }
+
+  return {
+    txs: cpfpData,
+    clusters,
+    version: 3,
+  };
+}
+
 /**
  * Takes a mempool transaction and a copy of the current mempool, and calculates the CPFP data for
  * that transaction (and all others in the same cluster)
@@ -236,7 +310,7 @@ export function calculateMempoolTxCpfp(tx: MempoolTransactionExtended, mempool:
 /**
    * Given a root transaction and a list of in-mempool ancestors,
    * Calculate the CPFP cluster
-   * 
+   *
    * @param tx
    * @param ancestors
    */

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 104;
+  private static currentVersion = 112;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -28,6 +28,7 @@ class DatabaseMigration {
 
   /**
    * Entry point
+   * @asyncUnsafe
    */
   public async $initializeOrMigrateDatabase(): Promise<void> {
     logger.debug('MIGRATIONS: Running migrations');
@@ -100,11 +101,12 @@ class DatabaseMigration {
 
   /**
    * Create all missing tables
+   * @asyncUnsafe
    */
   private async $createMissingTablesAndIndexes(databaseSchemaVersion: number) {
     await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion);
 
-    const isBitcoin = ['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK);
+    const isBitcoin = ['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK);
 
     await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
     await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
@@ -566,8 +568,8 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `blocks_templates` ADD INDEX `version` (`version`)');
       await this.updateToSchemaVersion(67);
     }
-    
-    if (databaseSchemaVersion < 68 && config.MEMPOOL.NETWORK === "liquid") {
+
+    if (databaseSchemaVersion < 68 && config.MEMPOOL.NETWORK === 'liquid') {
       await this.$executeQuery('TRUNCATE TABLE elements_pegs');
       await this.$executeQuery('ALTER TABLE elements_pegs ADD PRIMARY KEY (txid, txindex);');
       await this.$executeQuery(`UPDATE state SET number = 0 WHERE name = 'last_elements_block';`);
@@ -931,24 +933,24 @@ class DatabaseMigration {
 
         // Version 34
         await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_tor_nodes int(11) NOT NULL DEFAULT "0"');
-    
+
         // Version 35
         await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"');
         await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);');
 
         // Version 36
         await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"');
-    
+
         // Version 37
         await this.$executeQuery(this.getCreateLNNodesSocketsTableQuery(), await this.$checkIfTableExists('nodes_sockets'));
-        
+
         // Version 38
         await this.$executeQuery(`TRUNCATE lightning_stats`);
         await this.$executeQuery(`TRUNCATE node_stats`);
         await this.$executeQuery('ALTER TABLE `lightning_stats` CHANGE `added` `added` timestamp NULL');
         await this.$executeQuery('ALTER TABLE `node_stats` CHANGE `added` `added` timestamp NULL');
         await this.updateToSchemaVersion(38);
-      
+
         // Version 39
         await this.$executeQuery('ALTER TABLE `nodes` ADD alias_search TEXT NULL DEFAULT NULL AFTER `alias`');
         await this.$executeQuery('ALTER TABLE nodes ADD FULLTEXT(alias_search)');
@@ -963,7 +965,7 @@ class DatabaseMigration {
 
         // Version 42
         await this.$executeQuery('ALTER TABLE `channels` ADD closing_resolved tinyint(1) DEFAULT 0');
-      
+
         // Version 43
         await this.$executeQuery(this.getCreateLNNodeRecordsTableQuery(), await this.$checkIfTableExists('nodes_records'));
 
@@ -972,7 +974,7 @@ class DatabaseMigration {
 
         // Version 45
         await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fresh_txs JSON DEFAULT "[]"');
-    
+
         // Version 48
         await this.$executeQuery('ALTER TABLE `channels` ADD source_checked tinyint(1) DEFAULT 0');
         await this.$executeQuery('ALTER TABLE `channels` ADD closing_fee bigint(20) unsigned DEFAULT 0');
@@ -1002,13 +1004,13 @@ class DatabaseMigration {
         // Version 62
         await this.$executeQuery('ALTER TABLE `blocks_audits` ADD expected_fees BIGINT UNSIGNED DEFAULT NULL');
         await this.$executeQuery('ALTER TABLE `blocks_audits` ADD expected_weight BIGINT UNSIGNED DEFAULT NULL');
-      
+
         // Version 63
         await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fullrbf_txs JSON DEFAULT "[]"');
-    
+
         // Version 64
         await this.$executeQuery('ALTER TABLE `nodes` ADD features text NULL');
-    
+
         // Version 65
         await this.$executeQuery('ALTER TABLE `blocks_audits` ADD accelerated_txs JSON DEFAULT "[]"');
 
@@ -1044,8 +1046,8 @@ class DatabaseMigration {
             ADD INDEX \`closing_reason\` (\`closing_reason\`),
             ADD INDEX \`closing_resolved\` (\`closing_resolved\`)
         `);
-        
-        // Version 86        
+
+        // Version 86
         await this.$executeQuery(`
           ALTER TABLE \`nodes\`
             ADD INDEX \`status\` (\`status\`),
@@ -1058,20 +1060,20 @@ class DatabaseMigration {
         // Version 87
         await this.$executeQuery('ALTER TABLE `nodes_sockets` ADD INDEX `type` (`type`)');
         await this.updateToSchemaVersion(87);
-        
+
         // Version 88
         await this.$executeQuery('ALTER TABLE `lightning_stats` ADD INDEX `added` (`added`)');
-    
+
         // Version 89
         await this.$executeQuery('ALTER TABLE `geo_names` ADD INDEX `names` (`names`)');
-    
+
         // Version 90
         await this.$executeQuery('ALTER TABLE `hashrates` ADD INDEX `type` (`type`)');
 
         // Version 91
         await this.$executeQuery('ALTER TABLE `blocks_audits` ADD INDEX `time` (`time`)');
       }
-      
+
       if (config.MEMPOOL.NETWORK !== 'liquid') {
         // Apply all the liquid specific migrations to all other networks
         // Version 68
@@ -1093,7 +1095,7 @@ class DatabaseMigration {
             ADD INDEX \`bitcoinaddress\` (\`bitcoinaddress\`),
             ADD INDEX \`bitcointxid\` (\`bitcointxid\`)
         `);
-    
+
         // Version 93
         await this.$executeQuery(`
           ALTER TABLE \`federation_txos\`
@@ -1178,6 +1180,86 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `stale` (`stale`)');
       await this.updateToSchemaVersion(103);
     }
+
+    // reindex liquid federation addresses and txos when needed, and add hardcoded federation addresses
+    // (safe to make this conditional on the network since it doesn't change the database schema)
+    if (databaseSchemaVersion < 105 && config.MEMPOOL.NETWORK === 'liquid') {
+      // Hardcoded federation addresses
+      await this.$executeQuery(`INSERT IGNORE INTO federation_addresses (bitcoinaddress) VALUES ('3G6neksSBMp51kHJ2if8SeDUrzT8iVETWT')`);
+      await this.$executeQuery(`INSERT IGNORE INTO federation_addresses (bitcoinaddress) VALUES ('bc1qwnevjp8nsq7adu3hxlvdvslrf242q4vuavfg0y929jp2zntp3vgq7cq6z2')`);
+
+      // Rollback only on up to date instances
+      const [stateRows]: any[] = await DB.query(`SELECT name, number FROM state WHERE name IN ('last_elements_block', 'last_bitcoin_block_audit')`);
+      const lastElementsBlock = Number(stateRows?.find((row: any) => row.name === 'last_elements_block')?.number ?? 0);
+      const lastBlockAudit = Number(stateRows?.find((row: any) => row.name === 'last_bitcoin_block_audit')?.number ?? 0);
+      if (lastElementsBlock > 3686608 && lastBlockAudit > 929700) {
+        await this.$executeQuery('DELETE FROM elements_pegs WHERE block > 3686608');
+        await this.$executeQuery('DELETE FROM federation_txos WHERE blocknumber > 929701');
+        await this.$executeQuery(`UPDATE federation_txos SET lastblockupdate = 929700 WHERE unspent = 1;`);
+        await this.$executeQuery(`UPDATE state SET number = 3686608 WHERE name = 'last_elements_block';`);
+        await this.$executeQuery(`UPDATE state SET number = 929700 WHERE name = 'last_bitcoin_block_audit';`);
+      }
+      await this.updateToSchemaVersion(105);
+    }
+
+    // another liquid failure, fix bad timelocks on federation txos
+    // (safe to make this conditional on the network since it doesn't change the database schema)
+    if (databaseSchemaVersion < 106 && config.MEMPOOL.NETWORK === 'liquid') {
+      // In a specific setup it's possible that 3G6neksSBMp51kHJ2if8SeDUrzT8iVETWT and bc1qwnevjp8nsq7adu3hxlvdvslrf242q4vuavfg0y929jp2zntp3vgq7cq6z2
+      // were set with a timelock of 2016 instead of 4032
+      // This rollbacks the tables to before bc1qwnevjp8nsq7adu3hxlvdvslrf242q4vuavfg0y929jp2zntp3vgq7cq6z2 is used, and 
+      // manually fixes the timelock for 3G6neksSBMp51kHJ2if8SeDUrzT8iVETWT
+      const [stateRows]: any[] = await DB.query(`SELECT name, number FROM state WHERE name IN ('last_elements_block', 'last_bitcoin_block_audit')`);
+      const lastElementsBlock = Number(stateRows?.find((row: any) => row.name === 'last_elements_block')?.number ?? 0);
+      const lastBlockAudit = Number(stateRows?.find((row: any) => row.name === 'last_bitcoin_block_audit')?.number ?? 0);
+      if (lastElementsBlock > 3686608 && lastBlockAudit > 929700) {
+        await this.$executeQuery('DELETE FROM elements_pegs WHERE block > 3686608');
+        await this.$executeQuery('DELETE FROM federation_txos WHERE blocknumber > 929701');
+        await this.$executeQuery(`UPDATE federation_txos SET lastblockupdate = 929700 WHERE unspent = 1;`);
+        await this.$executeQuery(`UPDATE federation_txos SET timelock = 4032 WHERE bitcoinaddress = '3G6neksSBMp51kHJ2if8SeDUrzT8iVETWT';`);
+        await this.$executeQuery(`UPDATE state SET number = 3686608 WHERE name = 'last_elements_block';`);
+        await this.$executeQuery(`UPDATE state SET number = 929700 WHERE name = 'last_bitcoin_block_audit';`);
+      }
+      await this.updateToSchemaVersion(106);
+    }
+
+    if (databaseSchemaVersion < 107) {
+      await this.$executeQuery('ALTER TABLE `federation_txos` DROP FOREIGN KEY IF EXISTS `federation_txos_ibfk_1`');
+      await this.$executeQuery('DROP TABLE IF EXISTS `federation_addresses`');
+      await this.updateToSchemaVersion(107);
+    }
+
+    if (databaseSchemaVersion < 108) {
+      await this.$executeQuery(this.getCreateFederationPegScriptsTableQuery(), await this.$checkIfTableExists('federation_peg_scripts'),);
+      await this.updateToSchemaVersion(108);
+    }
+
+    // safe to make this conditional on the network since it doesn't change the database schema
+    if (databaseSchemaVersion < 109 && config.MEMPOOL.NETWORK === 'liquid') {
+      // hardcode current and past federation peg scripts to avoid reindexing existing instances
+      await this.$executeQuery(`
+          INSERT IGNORE INTO federation_peg_scripts (address, fedpegscript, timelock, blocknumber) VALUES
+            ('3G6neksSBMp51kHJ2if8SeDUrzT8iVETWT', '745c87635b21020e0338c96a8870479f2396c373cc7696ba124e8635d41b0ea581112b678172612102675333a4e4b8fb51d9d4e22fa5a8eaced3fdac8a8cbf9be8c030f75712e6af992102896807d54bc55c24981f24a453c60ad3e8993d693732288068a23df3d9f50d4821029e51a5ef5db3137051de8323b001749932f2ff0d34c82e96a2c2461de96ae56c2102a4e1a9638d46923272c266631d94d36bdb03a64ee0e14c7518e49d2f29bc40102102f8a00b269f8c5e59c67d36db3cdc11b11b21f64b4bffb2815e9100d9aa8daf072103079e252e85abffd3c401a69b087e590a9b86f33f574f08129ccbd3521ecf516b2103111cf405b627e22135b3b3733a4a34aa5723fb0f58379a16d32861bf576b0ec2210318f331b3e5d38156da6633b31929c5b220349859cc9ca3d33fb4e68aa08401742103230dae6b4ac93480aeab26d000841298e3b8f6157028e47b0897c1e025165de121035abff4281ff00660f99ab27bb53e6b33689c2cd8dcd364bc3c90ca5aea0d71a62103bd45cddfacf2083b14310ae4a84e25de61e451637346325222747b157446614c2103cc297026b06c71cbfa52089149157b5ff23de027ac5ab781800a578192d175462103d3bde5d63bdb3a6379b461be64dad45eabff42f758543a9645afd42f6d4248282103ed1e8d5109c9ed66f7941bc53cc71137baa76d50d274bda8d5e8ffbd6e61fe9a5f6702c00fb275522103aab896d53a8e7d6433137bbba940f9c521e085dd07e60994579b64a6d992cf79210291b7d0b1b692f8f524516ed950872e5da10fb1b808b5a526dedc6fed1cf29807210386aa9372fbab374593466bc5451dc59954e90787f08060964d95c87ef34ca5bb5368ae', 4032, 0),
+            ('3EiAcrzq1cELXScc98KeCswGWZaPGceT1d', '745c87635b21020e0338c96a8870479f2396c373cc7696ba124e8635d41b0ea581112b678172612102675333a4e4b8fb51d9d4e22fa5a8eaced3fdac8a8cbf9be8c030f75712e6af992102896807d54bc55c24981f24a453c60ad3e8993d693732288068a23df3d9f50d4821029e51a5ef5db3137051de8323b001749932f2ff0d34c82e96a2c2461de96ae56c2102a4e1a9638d46923272c266631d94d36bdb03a64ee0e14c7518e49d2f29bc40102102f8a00b269f8c5e59c67d36db3cdc11b11b21f64b4bffb2815e9100d9aa8daf072103079e252e85abffd3c401a69b087e590a9b86f33f574f08129ccbd3521ecf516b2103111cf405b627e22135b3b3733a4a34aa5723fb0f58379a16d32861bf576b0ec2210318f331b3e5d38156da6633b31929c5b220349859cc9ca3d33fb4e68aa08401742103230dae6b4ac93480aeab26d000841298e3b8f6157028e47b0897c1e025165de121035abff4281ff00660f99ab27bb53e6b33689c2cd8dcd364bc3c90ca5aea0d71a62103bd45cddfacf2083b14310ae4a84e25de61e451637346325222747b157446614c2103cc297026b06c71cbfa52089149157b5ff23de027ac5ab781800a578192d175462103d3bde5d63bdb3a6379b461be64dad45eabff42f758543a9645afd42f6d4248282103ed1e8d5109c9ed66f7941bc53cc71137baa76d50d274bda8d5e8ffbd6e61fe9a5f6702e007b275522103aab896d53a8e7d6433137bbba940f9c521e085dd07e60994579b64a6d992cf79210291b7d0b1b692f8f524516ed950872e5da10fb1b808b5a526dedc6fed1cf29807210386aa9372fbab374593466bc5451dc59954e90787f08060964d95c87ef34ca5bb5368ae', 2016, 0),
+            ('bc1qxvay4an52gcghxq5lavact7r6qe9l4laedsazz8fj2ee2cy47tlqff4aj4', '5b21020e0338c96a8870479f2396c373cc7696ba124e8635d41b0ea581112b678172612102675333a4e4b8fb51d9d4e22fa5a8eaced3fdac8a8cbf9be8c030f75712e6af992102896807d54bc55c24981f24a453c60ad3e8993d693732288068a23df3d9f50d4821029e51a5ef5db3137051de8323b001749932f2ff0d34c82e96a2c2461de96ae56c2102a4e1a9638d46923272c266631d94d36bdb03a64ee0e14c7518e49d2f29bc401021031c41fdbcebe17bec8d49816e00ca1b5ac34766b91c9f2ac37d39c63e5e008afb2103079e252e85abffd3c401a69b087e590a9b86f33f574f08129ccbd3521ecf516b2103111cf405b627e22135b3b3733a4a34aa5723fb0f58379a16d32861bf576b0ec2210318f331b3e5d38156da6633b31929c5b220349859cc9ca3d33fb4e68aa08401742103230dae6b4ac93480aeab26d000841298e3b8f6157028e47b0897c1e025165de121035abff4281ff00660f99ab27bb53e6b33689c2cd8dcd364bc3c90ca5aea0d71a62103bd45cddfacf2083b14310ae4a84e25de61e451637346325222747b157446614c2103cc297026b06c71cbfa52089149157b5ff23de027ac5ab781800a578192d175462103d3bde5d63bdb3a6379b461be64dad45eabff42f758543a9645afd42f6d4248282103ed1e8d5109c9ed66f7941bc53cc71137baa76d50d274bda8d5e8ffbd6e61fe9a5fae736402c00fb269522103aab896d53a8e7d6433137bbba940f9c521e085dd07e60994579b64a6d992cf79210291b7d0b1b692f8f524516ed950872e5da10fb1b808b5a526dedc6fed1cf29807210386aa9372fbab374593466bc5451dc59954e90787f08060964d95c87ef34ca5bb53ae68', 4032, 2197440),
+            ('bc1qwnevjp8nsq7adu3hxlvdvslrf242q4vuavfg0y929jp2zntp3vgq7cq6z2', '5b21020e0338c96a8870479f2396c373cc7696ba124e8635d41b0ea581112b678172612102675333a4e4b8fb51d9d4e22fa5a8eaced3fdac8a8cbf9be8c030f75712e6af9921021fad939f956beae2e6765b450552f1a75236c8b7314ec30474d8f4978b252ab221029e51a5ef5db3137051de8323b001749932f2ff0d34c82e96a2c2461de96ae56c2102a4e1a9638d46923272c266631d94d36bdb03a64ee0e14c7518e49d2f29bc401021031c41fdbcebe17bec8d49816e00ca1b5ac34766b91c9f2ac37d39c63e5e008afb2102e323a079079582cf6bb68583df94bd68c9590e5904062b503ef5476109ad739e2103111cf405b627e22135b3b3733a4a34aa5723fb0f58379a16d32861bf576b0ec2210318f331b3e5d38156da6633b31929c5b220349859cc9ca3d33fb4e68aa08401742103230dae6b4ac93480aeab26d000841298e3b8f6157028e47b0897c1e025165de121035abff4281ff00660f99ab27bb53e6b33689c2cd8dcd364bc3c90ca5aea0d71a62103bd45cddfacf2083b14310ae4a84e25de61e451637346325222747b157446614c2103cc297026b06c71cbfa52089149157b5ff23de027ac5ab781800a578192d175462103d3bde5d63bdb3a6379b461be64dad45eabff42f758543a9645afd42f6d4248282103ed1e8d5109c9ed66f7941bc53cc71137baa76d50d274bda8d5e8ffbd6e61fe9a5fae736402c00fb269522103aab896d53a8e7d6433137bbba940f9c521e085dd07e60994579b64a6d992cf79210291b7d0b1b692f8f524516ed950872e5da10fb1b808b5a526dedc6fed1cf29807210386aa9372fbab374593466bc5451dc59954e90787f08060964d95c87ef34ca5bb53ae68', 4032, 3729600);
+        `);
+      await this.updateToSchemaVersion(109);
+    }
+    if (databaseSchemaVersion < 110 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD template_algo TINYINT UNSIGNED NOT NULL DEFAULT 0');
+      await this.updateToSchemaVersion(110);
+    }
+
+    if (databaseSchemaVersion < 111 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `compact_cpfp_clusters` ADD template_algo TINYINT UNSIGNED NOT NULL DEFAULT 0');
+      await this.updateToSchemaVersion(111);
+    }
+
+    if (databaseSchemaVersion < 112) {
+      await this.$executeQuery(this.getCreateFlagsValuesTableQuery(), await this.$checkIfTableExists('flag_values'));
+      await this.updateToSchemaVersion(112);
+    }
   }
 
   /**
@@ -1224,6 +1306,7 @@ class DatabaseMigration {
 
   /**
    * Check if 'table' exists in the database
+   * @asyncUnsafe
    */
   private async $checkIfTableExists(table: string): Promise<boolean> {
     const query = `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '${config.DATABASE.DATABASE}' AND TABLE_NAME = '${table}'`;
@@ -1233,6 +1316,7 @@ class DatabaseMigration {
 
   /**
    * Get current database version
+   * @asyncUnsafe
    */
   private async $getSchemaVersionFromDatabase(): Promise<number> {
     const query = `SELECT number FROM state WHERE name = 'schema_version';`;
@@ -1242,6 +1326,7 @@ class DatabaseMigration {
 
   /**
    * Create the `state` table
+   * @asyncUnsafe
    */
   private async $createMigrationStateTable(): Promise<void> {
     const query = `CREATE TABLE IF NOT EXISTS state (
@@ -1259,6 +1344,7 @@ class DatabaseMigration {
 
   /**
    * We actually execute the migrations queries here
+   * @asyncUnsafe
    */
   private async $migrateTableSchemaFromVersion(version: number): Promise<void> {
     const transactionQueries: string[] = [];
@@ -1286,7 +1372,7 @@ class DatabaseMigration {
    */
   private getMigrationQueriesFromVersion(version: number): string[] {
     const queries: string[] = [];
-    const isBitcoin = ['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK);
+    const isBitcoin = ['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK);
 
     if (version < 1) {
       if (config.MEMPOOL.NETWORK !== 'liquid' && config.MEMPOOL.NETWORK !== 'liquidtestnet') {
@@ -1325,11 +1411,13 @@ class DatabaseMigration {
 
   /**
    * Save the schema version in the database
+   * @asyncUnsafe
    */
   private getUpdateToLatestSchemaVersionQuery(): string {
     return `UPDATE state SET number = ${DatabaseMigration.currentVersion} WHERE name = 'schema_version';`;
   }
 
+  /** @asyncUnsafe */
   private async updateToSchemaVersion(version): Promise<void> {
     await this.$executeQuery(`UPDATE state SET number = ${version} WHERE name = 'schema_version';`);
   }
@@ -1456,11 +1544,21 @@ class DatabaseMigration {
       pegtxid varchar(65) NOT NULL,
       pegindex int(11) NOT NULL,
       pegblocktime int(11) unsigned NOT NULL,
-      PRIMARY KEY (txid, txindex), 
+      PRIMARY KEY (txid, txindex),
       FOREIGN KEY (bitcoinaddress) REFERENCES federation_addresses (bitcoinaddress)
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateFederationPegScriptsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS federation_peg_scripts (
+      address varchar(100) NOT NULL,
+      fedpegscript text NOT NULL,
+      timelock int(11) unsigned NOT NULL,
+      blocknumber int(11) unsigned NOT NULL,
+      PRIMARY KEY (address)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
   private getCreatePoolsTableQuery(): string {
     return `CREATE TABLE IF NOT EXISTS pools (
       id int(11) NOT NULL AUTO_INCREMENT,
@@ -1748,6 +1846,19 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateFlagsValuesTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS flag_values (
+      bucket_size enum('1', '1008', '4032') NOT NULL,
+      start_height int unsigned NOT NULL,
+      avg_timestamp timestamp NOT NULL,
+      flag_value bigint unsigned NOT NULL,
+      tx_count int unsigned NOT NULL,
+      vsize_total int unsigned NOT NULL,
+      PRIMARY KEY (bucket_size, start_height, flag_value)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8`;
+  }
+
+  /** @asyncUnsafe */
   public async $blocksReindexingTruncate(): Promise<void> {
     logger.warn(`Truncating pools, blocks, hashrates and difficulty_adjustments tables for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
     await Common.sleep$(5000);

backend/src/api/disk-cache.ts

@@ -33,11 +33,12 @@ class DiskCache {
       return;
     }
     process.on('SIGINT', (e) => {
-      this.$saveCacheToDisk(true);
+      void this.$saveCacheToDisk(true);
       process.exit(0);
     });
   }
 
+  /** @asyncSafe */
   async $saveCacheToDisk(sync: boolean = false): Promise<void> {
     if (!cluster.isPrimary || !config.MEMPOOL.CACHE_ENABLED) {
       return;
@@ -174,6 +175,7 @@ class DiskCache {
     }
   }
 
+  /** @asyncSafe */
   async $loadMempoolCache(): Promise<void> {
     if (!config.MEMPOOL.CACHE_ENABLED || !fs.existsSync(DiskCache.FILE_NAME)) {
       return;

backend/src/api/explorer/channels.api.ts

@@ -53,7 +53,7 @@ class ChannelsApi {
           GROUP BY nodes_1.public_key, nodes_2.public_key
           ORDER BY channels.capacity DESC
           LIMIT 10000
-        `;        
+        `;
       }
 
       const [rows]: any = await DB.query(query, params);
@@ -241,10 +241,10 @@ class ChannelsApi {
       let [feeRates2]: any = await DB.query(query);
       feeRates2 = feeRates2.map(rate => rate.node2_fee_rate);
 
-      let feeRates = (feeRates1.concat(feeRates2)).sort((a, b) => a - b);
+      const feeRates = (feeRates1.concat(feeRates2)).sort((a, b) => a - b);
       let avgFeeRate = 0;
       for (const rate of feeRates) {
-        avgFeeRate += rate; 
+        avgFeeRate += rate;
       }
       avgFeeRate /= feeRates.length;
       const medianFeeRate = feeRates[Math.floor(feeRates.length / 2)];
@@ -257,14 +257,14 @@ class ChannelsApi {
       let [baseFees2]: any = await DB.query(query);
       baseFees2 = baseFees2.map(rate => rate.node2_base_fee_mtokens);
 
-      let baseFees = (baseFees1.concat(baseFees2)).sort((a, b) => a - b);
+      const baseFees = (baseFees1.concat(baseFees2)).sort((a, b) => a - b);
       let avgBaseFee = 0;
       for (const fee of baseFees) {
-        avgBaseFee += fee; 
+        avgBaseFee += fee;
       }
       avgBaseFee /= baseFees.length;
       const medianBaseFee = feeRates[Math.floor(baseFees.length / 2)];
-      
+
       return {
         avgCapacity: parseInt(avgCapacity[0].avgCapacity, 10),
         avgFeeRate: avgFeeRate,
@@ -272,7 +272,7 @@ class ChannelsApi {
         medianCapacity: medianCapacity,
         medianFeeRate: medianFeeRate,
         medianBaseFee: medianBaseFee,
-      }
+      };
 
     } catch (e) {
       logger.err(`Cannot calculate channels statistics. Reason: ${e instanceof Error ? e.message : e}`);
@@ -298,6 +298,7 @@ class ChannelsApi {
     }
   }
 
+  /** @asyncSafe */
   public async $getChannelByClosingId(transactionId: string): Promise<any> {
     try {
       const query = `
@@ -338,6 +339,7 @@ class ChannelsApi {
     }
   }
 
+  /** @asyncSafe */
   public async $updateClosingInfo(channelInfo: { id: string, node1_closing_balance: number, node2_closing_balance: number, closed_by: string | null, closing_fee: number, outputs: ILightningApi.ForensicOutput[]}): Promise<void> {
     try {
       const query = `
@@ -363,6 +365,7 @@ class ChannelsApi {
     }
   }
 
+  /** @asyncSafe */
   public async $updateOpeningInfo(channelInfo: { id: string, node1_funding_balance: number, node2_funding_balance: number, funding_ratio: number, single_funded: boolean | void }): Promise<void> {
     try {
       const query = `
@@ -456,7 +459,7 @@ class ChannelsApi {
         allChannels = allChannels.slice(0, 1000);
       }
 
-      const channels: any[] = []
+      const channels: any[] = [];
       for (const row of allChannels) {
         let channel;
         if (index >= 0) {
@@ -578,6 +581,7 @@ class ChannelsApi {
 
   /**
    * Save or update a channel present in the graph
+   * @asyncUnsafe
    */
   public async $saveChannel(channel: ILightningApi.Channel, status = 1): Promise<void> {
     if (!channel.chan_point?.length) {
@@ -717,6 +721,7 @@ class ChannelsApi {
     }
   }
 
+  /** @asyncSafe */
   public async $getLatestChannelUpdateForNode(publicKey: string): Promise<number> {
     try {
       const query = `

backend/src/api/explorer/channels.routes.ts

@@ -78,17 +78,14 @@ class ChannelsRoutes {
 
   private async $getChannelsByTransactionIds(req: Request, res: Response): Promise<void> {
     try {
-      if (!Array.isArray(req.query.txId)) {
-        handleError(req, res, 400, 'Not an array');
+      if (!req.query.txId || typeof req.query.txId !== 'object') {
+        handleError(req, res, 400, 'invalid txId format');
         return;
       }
       const txIds: string[] = [];
-      for (const _txId in req.query.txId) {
-        if (typeof req.query.txId[_txId] === 'string') {
-          const txid = req.query.txId[_txId].toString();
-          if (TXID_REGEX.test(txid)) {
-            txIds.push(txid);
-          }
+      for (const txid of Object.values(req.query.txId)) {
+        if (typeof txid === 'string' && TXID_REGEX.test(txid)) {
+          txIds.push(txid);
         }
       }
       const channels = await channelsApi.$getChannelsByTransactionId(txIds);

backend/src/api/explorer/nodes.api.ts

@@ -34,7 +34,7 @@ class NodesApi {
       `;
 
       const [maximums]: any[] = await DB.query(query);
-      
+
       return {
         maxLiquidity: maximums[0].maxLiquidity,
         maxChannels: maximums[0].maxChannels,
@@ -78,7 +78,7 @@ class NodesApi {
       node.city = JSON.parse(node.city);
       node.country = JSON.parse(node.country);
 
-      // Features      
+      // Features
       node.features = JSON.parse(node.features);
       node.featuresBits = null;
       if (node.features) {
@@ -87,7 +87,7 @@ class NodesApi {
           maxBit = Math.max(maxBit, feature.bit);
         }
         maxBit = Math.ceil(maxBit / 4) * 4 - 1;
-        
+
         node.featuresBits = new Array(maxBit + 1).fill(0);
         for (const feature of node.features) {
           node.featuresBits[feature.bit] = 1;
@@ -143,6 +143,7 @@ class NodesApi {
     }
   }
 
+  /** @asyncUnsafe */
   public async $getActiveChannelsStats(node_public_key: string): Promise<unknown> {
     const query = `
       SELECT count(short_id) as active_channel_count, sum(capacity) as capacity
@@ -394,7 +395,7 @@ class NodesApi {
     try {
       const publicKeySearch = search.replace(/[^a-zA-Z0-9]/g, '') + '%';
       const aliasSearch = search
-        .replace(/[-_.]/g, ' ') // Replace all -_. characters with empty space. Eg: "ln.nicehash" becomes "ln nicehash".  
+        .replace(/[-_.]/g, ' ') // Replace all -_. characters with empty space. Eg: "ln.nicehash" becomes "ln nicehash".
         .replace(/[^a-zA-Z0-9 ]/g, '') // Remove all special characters and keep just A to Z, 0 to 9.
         .split(' ')
         .filter(key => key.length)
@@ -455,7 +456,7 @@ class NodesApi {
         } else if (ispList[isp2].ids.includes(channel.isp2ID) === false) {
           ispList[isp2].ids.push(channel.isp2ID);
         }
-        
+
         ispList[isp1].capacity += channel.capacity;
         ispList[isp1].channels += 1;
         ispList[isp1].nodes[channel.node1PublicKey] = true;
@@ -463,7 +464,7 @@ class NodesApi {
         ispList[isp2].channels += 1;
         ispList[isp2].nodes[channel.node2PublicKey] = true;
       }
-      
+
       const ispRanking: any[] = [];
       for (const isp of Object.keys(ispList)) {
         ispRanking.push([
@@ -494,7 +495,7 @@ class NodesApi {
       `;
       const [clearnetCapacity]: any = await DB.query(query);
 
-      // Get the total capacity of all channels which have both nodes on Tor 
+      // Get the total capacity of all channels which have both nodes on Tor
       query = `
         SELECT SUM(capacity) as capacity
         FROM (
@@ -642,11 +643,11 @@ class NodesApi {
       for (const country of nodesCountPerCountry) {
         nodesPerCountry.push({
           name: JSON.parse(country.names),
-          iso: country.iso_code, 
+          iso: country.iso_code,
           count: country.nodesCount,
           share: Math.floor(country.nodesCount / nodesWithAS[0].total * 10000) / 100,
           capacity: country.capacity,
-        })
+        });
       }
 
       return nodesPerCountry;
@@ -658,6 +659,7 @@ class NodesApi {
 
   /**
    * Save or update a node present in the graph
+   * @asyncSafe
    */
   public async $saveNode(node: ILightningApi.Node): Promise<void> {
     try {
@@ -665,7 +667,7 @@ class NodesApi {
       if ((node.last_update ?? 0) < 1514736061) { // January 1st 2018
         node.last_update = null;
       }
-  
+
       const uniqueAddr = [...new Set(node.addresses?.map(a => a.addr))];
       const formattedSockets = (uniqueAddr.join(',')) ?? '';
 
@@ -727,6 +729,7 @@ class NodesApi {
 
   /**
    * Set all nodes not in `nodesPubkeys` as inactive (status = 0)
+   * @asyncSafe
    */
    public async $setNodesInactive(graphNodesPubkeys: string[]): Promise<void> {
     if (graphNodesPubkeys.length === 0) {

backend/src/api/explorer/nodes.routes.ts

@@ -39,7 +39,7 @@ class NodesRoutes {
   private async $getNodeGroup(req: Request, res: Response) {
     try {
       let nodesList;
-      let nodes: any[] = [];
+      const nodes: any[] = [];
       switch (config.MEMPOOL.NETWORK) {
         case 'testnet':
           nodesList = [
@@ -174,7 +174,7 @@ class NodesRoutes {
           ];
       }
 
-      for (let pubKey of nodesList) {
+      for (const pubKey of nodesList) {
         try {
           const node = await nodesApi.$getNode(pubKey);
           if (node) {

backend/src/api/fetch-version.ts

@@ -1,5 +1,5 @@
 import fs from 'fs';
-import path from "path";
+import path from 'path';
 const { spawnSync } = require('child_process');
 
 function getVersion(): string {
@@ -29,9 +29,9 @@ function getGitCommit(): string {
 const versionInfo = {
   version: getVersion(),
   gitCommit: getGitCommit()
-}
+};
 
 fs.writeFileSync(
   path.join(__dirname, 'version.json'),
-  JSON.stringify(versionInfo, null, 2) + "\n"
+  JSON.stringify(versionInfo, null, 2) + '\n'
 );

backend/src/api/lightning/clightning/clightning-client.ts

@@ -116,7 +116,7 @@ class LightningError extends Error {
 
 const defaultRpcPath = path.join(homedir(), '.lightning')
   , fStat = (...p) => statSync(path.join(...p))
-  , fExists = (...p) => existsSync(path.join(...p))
+  , fExists = (...p) => existsSync(path.join(...p));
 
 export default class CLightningClient extends EventEmitter implements AbstractLightningApi {
   private rpcPath: string;
@@ -141,9 +141,9 @@ export default class CLightningClient extends EventEmitter implements AbstractLi
       // main data directory provided, default to using the bitcoin mainnet subdirectory
       // to be removed in v0.2.0
       else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) {
-        logger.warn(`${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`, logger.tags.ln)
-        logger.warn(`specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`, logger.tags.ln)
-        rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc')
+        logger.warn(`${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`, logger.tags.ln);
+        logger.warn(`specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`, logger.tags.ln);
+        rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc');
       }
     }
 
@@ -249,6 +249,7 @@ export default class CLightningClient extends EventEmitter implements AbstractLi
       }));
   }
 
+  /** @asyncUnsafe */
   async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
     const listnodes: any[] = await this.call('listnodes');
     const listchannels: any[] = await this.call('listchannels');

backend/src/api/lightning/clightning/clightning-convert.ts

@@ -156,6 +156,7 @@ export function convertNode(clNode: any): ILightningApi.Node {
 
 /**
  * Convert clightning "listchannels" response to lnd "describegraph.edges" format
+ * @asyncUnsafe
  */
 export async function convertAndmergeBidirectionalChannels(clChannels: any[]): Promise<ILightningApi.Channel[]> {
   logger.debug(`Converting clightning nodes and channels to lnd graph format`, logger.tags.ln);
@@ -212,6 +213,7 @@ export async function convertAndmergeBidirectionalChannels(clChannels: any[]): P
 /**
  * Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format
  * In this case, clightning knows the channel policy for both nodes
+ * @asyncUnsafe
  */
 async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILightningApi.Channel | null> {
   const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0);
@@ -238,6 +240,7 @@ async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILigh
 /**
  * Convert one clightning "getchannels" entry into a full a lnd "describegraph.edges" format
  * In this case, clightning knows the channel policy of only one node
+ * @asyncUnsafe
  */
 async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Channel | null> {
   const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannel.short_channel_id);

backend/src/api/lightning/lnd/lnd-api.ts

@@ -40,16 +40,17 @@ class LndApi implements AbstractLightningApi {
       .then((response) => response.data);
   }
 
+  /** @asyncUnsafe */
   async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
     const graph = await axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
       .then((response) => response.data);
 
     for (const node of graph.nodes) {
       const nodeFeatures: ILightningApi.Feature[] = [];
-      for (const bit in node.features) {        
+      for (const bit in node.features) {
         nodeFeatures.push({
           bit: parseInt(bit, 10),
-          name: node.features[bit].name,  
+          name: node.features[bit].name,
           is_required: node.features[bit].is_required,
           is_known: node.features[bit].is_known,
         });

backend/src/api/liquid/elements-parser.ts

@@ -4,13 +4,23 @@ import bitcoinSecondClient from '../bitcoin/bitcoin-second-client';
 import { Common } from '../common';
 import DB from '../../database';
 import logger from '../../logger';
+import * as bitcoinjs from 'bitcoinjs-lib';
 
-const federationChangeAddresses = ['bc1qxvay4an52gcghxq5lavact7r6qe9l4laedsazz8fj2ee2cy47tlqff4aj4', '3EiAcrzq1cELXScc98KeCswGWZaPGceT1d'];
 const auditBlockOffsetWithTip = 1; // Wait for 1 block confirmation before processing the block in the audit process to reduce the risk of reorgs
+const auditSyncToleranceBlocks = 105; // Audit lag from bitcoin tip to avoid potential unsynced state on peg-ins, which require 102 confirmations
+const DYNAFED_ACTIVATION_HEIGHT = 1517040;
+const DYNAFED_CHECK_INTERVAL = 20160;
+const FEDERATION_TIMELOCK_FALLBACK = 0x0000ffff; // Preventive timelock in case an error occurs while extracting it from the fedpegscript
+interface FederationAddress {
+  fedpegscript: string;
+  address: string;
+  timelock: number;
+  blocknumber: number;
+}
 
 class ElementsParser {
   private isRunning = false;
-  private isUtxosUpdatingRunning = false;
+  private federationPegScripts: FederationAddress[] = [];
 
   constructor() { }
 
@@ -20,15 +30,23 @@ class ElementsParser {
     }
     try {
       this.isRunning = true;
+      const startedAt = Date.now() / 1000;
+      const stopAt = startedAt + 3600; // Limit one parse run to 1 hour to keep the tip sufficiently up to date
       const result = await bitcoinClient.getChainTips();
       const tip = result[0].height;
       const latestBlockHeight = await this.$getLatestBlockHeightFromDatabase();
       for (let height = latestBlockHeight + 1; height <= tip; height++) {
-        const blockHash: IBitcoinApi.ChainTips = await bitcoinClient.getBlockHash(height);
+        if ((Date.now() / 1000) >= stopAt) {
+          logger.debug(`Reached max Elements parsing runtime of one hour, pausing parse to resume later`);
+          break;
+        }
+        const blockHash: string = await bitcoinClient.getBlockHash(height);
         const block: IBitcoinApi.Block = await bitcoinClient.getBlock(blockHash, 2);
+        await this.$updateFederationPegScripts(block);
         await this.$parseBlock(block);
         await this.$saveLatestBlockToDatabase(block.height);
       }
+      await this.$updateFederationUtxos(stopAt);
       this.isRunning = false;
     } catch (e) {
       this.isRunning = false;
@@ -36,6 +54,7 @@ class ElementsParser {
     }
   }
 
+  /** @asyncUnsafe */
   protected async $parseBlock(block: IBitcoinApi.Block) {
     for (const tx of block.tx) {
       await this.$parseInputs(tx, block);
@@ -43,6 +62,7 @@ class ElementsParser {
     }
   }
 
+  /** @asyncUnsafe */
   protected async $parseInputs(tx: IBitcoinApi.Transaction, block: IBitcoinApi.Block) {
     for (const [index, input] of tx.vin.entries()) {
       if (input.is_pegin) {
@@ -51,15 +71,18 @@ class ElementsParser {
     }
   }
 
+  /** @asyncUnsafe */
   protected async $parsePegIn(input: IBitcoinApi.Vin, vindex: number, txid: string, block: IBitcoinApi.Block) {
     const bitcoinTx: IBitcoinApi.Transaction = await bitcoinSecondClient.getRawTransaction(input.txid, true);
     const bitcoinBlock: IBitcoinApi.Block = await bitcoinSecondClient.getBlock(bitcoinTx.blockhash);
     const prevout = bitcoinTx.vout[input.vout || 0];
     const outputAddress = prevout.scriptPubKey.address || (prevout.scriptPubKey.addresses && prevout.scriptPubKey.addresses[0]) || '';
+    const timelock = await this.$resolvePegInTimelock(input, outputAddress);
     await this.$savePegToDatabase(block.height, block.time, prevout.value * 100000000, txid, vindex,
-      outputAddress, bitcoinTx.txid, prevout.n, bitcoinBlock.height, bitcoinBlock.time, 1);
+      outputAddress, bitcoinTx.txid, prevout.n, bitcoinBlock.height, bitcoinBlock.time, 1, timelock);
   }
 
+  /** @asyncUnsafe */
   protected async $parseOutputs(tx: IBitcoinApi.Transaction, block: IBitcoinApi.Block) {
     for (const output of tx.vout) {
       if (output.scriptPubKey.pegout_chain) {
@@ -74,8 +97,9 @@ class ElementsParser {
     }
   }
 
+  /** @asyncUnsafe */
   protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string,
-    txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, bitcoinblock: number, bitcoinBlockTime: number, final_tx: number): Promise<void> {
+    txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, bitcoinblock: number, bitcoinBlockTime: number, final_tx: number, pegInTimelock: number = FEDERATION_TIMELOCK_FALLBACK): Promise<void> {
     const query = `INSERT IGNORE INTO elements_pegs(
         block, datetime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
       ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`;
@@ -87,27 +111,146 @@ class ElementsParser {
     logger.debug(`Saved L-BTC peg from Liquid block height #${height} with TXID ${txid}.`);
 
     if (amount > 0) { // Peg-in
-  
-      // Add the address to the federation addresses table
-      await DB.query(`INSERT IGNORE INTO federation_addresses (bitcoinaddress) VALUES (?)`, [bitcoinaddress]);
-
       // Add the UTXO to the federation txos table
       const query_utxos = `INSERT IGNORE INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate, timelock, expiredAt, emergencyKey, pegtxid, pegindex, pegblocktime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
-      const params_utxos: (string | number)[] = [bitcointxid, bitcoinindex, bitcoinaddress, amount, bitcoinblock, bitcoinBlockTime, 1, bitcoinblock - 1, 0, 4032, 0, 0, txid, txindex, blockTime];
+      const params_utxos: (string | number)[] = [bitcointxid, bitcoinindex, bitcoinaddress, amount, bitcoinblock, bitcoinBlockTime, 1, bitcoinblock - 1, 0, pegInTimelock, 0, 0, txid, txindex, blockTime];
       await DB.query(query_utxos, params_utxos);
-      const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`)
+      const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`);
       await this.$saveLastBlockAuditToDatabase(minBlockUpdate[0]['lastblockupdate']);
       logger.debug(`Saved new Federation UTXO ${bitcointxid}:${bitcoinindex} belonging to ${bitcoinaddress} to federation txos`);
 
     }
   }
 
+  /** @asyncUnsafe */
+  private async $getFederationPegScripts(): Promise<FederationAddress[]> {
+    if (this.federationPegScripts.length === 0) {
+      const [rows] = await DB.query(`SELECT fedpegscript, address, timelock, blocknumber FROM federation_peg_scripts ORDER BY blocknumber DESC`);
+      this.federationPegScripts = rows as FederationAddress[];
+    }
+    return this.federationPegScripts;
+  }
+
+  /** @asyncUnsafe */
+  private async $updateFederationPegScripts(block: IBitcoinApi.Block): Promise<void> {
+    const height = block.height;
+    if (height < DYNAFED_ACTIVATION_HEIGHT || height % DYNAFED_CHECK_INTERVAL !== 0) {
+      return;
+    }
+
+    const fedpegscript = block.dynamic_parameters?.current?.fedpegscript as string | undefined;
+    const fedpegProgram = block.dynamic_parameters?.current?.fedpeg_program as string | undefined;
+    if (!fedpegscript || !fedpegProgram) {
+      logger.err(`Missing fedpeg fields at height ${height}, skipping dynamic federation address update.`);
+      return;
+    }
+
+    if ((await this.$getFederationPegScripts()).some(entry => entry.fedpegscript === fedpegscript)) {
+      logger.debug(`Fedpegscript at height ${height} is already known, skipping.`);
+      return;
+    }
+
+    logger.debug(`New fedpegscript found at height ${height}, deriving federation address.`);
+    let address: string;
+    try {
+      address = bitcoinjs.address.fromOutputScript(Buffer.from(fedpegProgram, 'hex'), bitcoinjs.networks.bitcoin);
+    } catch (e) {
+      logger.err(`Unable to derive federation address from fedpeg program at height ${height}: ${e instanceof Error ? e.message : 'Error'}`);
+      return;
+    }
+
+    let timelock = this.extractTimelockFromFedpegscript(fedpegscript);
+    if (timelock === null) {
+      timelock = FEDERATION_TIMELOCK_FALLBACK;
+      logger.err(`Unable to extract federation address timelock from fedpegscript at height ${height}, using fallback timelock ${FEDERATION_TIMELOCK_FALLBACK}.`);
+    }
+
+    this.federationPegScripts.unshift({ fedpegscript, address, timelock, blocknumber: height });
+    logger.debug(`Derived new federation address ${address} with timelock ${timelock} from fedpegscript at height ${height}. Adding to database.`);
+    try {
+      await DB.query(
+        `INSERT INTO federation_peg_scripts (fedpegscript, address, timelock, blocknumber) VALUES (?, ?, ?, ?)`,
+        [fedpegscript, address, timelock, height]
+      );
+      logger.debug(`Added new federation address ${address} to the database.`);
+    } catch (e) {
+      logger.err(`Error while inserting new federation address ${address} into the database: ${e instanceof Error ? e.message : 'Error'}`);
+    }
+  }
+
+  private extractTimelockFromFedpegscript(fedpegscript: string): number | null {
+    const chunks = bitcoinjs.script.decompile(Buffer.from(fedpegscript, 'hex'));
+    if (!chunks) {
+      return null;
+    }
+
+    for (let i = 0; i < chunks.length; i++) {
+      if (chunks[i] !== bitcoinjs.opcodes.OP_CHECKSEQUENCEVERIFY) {
+        continue;
+      }
+
+      const previous = chunks[i - 1];
+      if (previous === undefined) {
+        return null;
+      }
+
+      if (Buffer.isBuffer(previous)) {
+        try {
+          return bitcoinjs.script.number.decode(previous, 5, true);
+        } catch (e) {
+          return null;
+        }
+      }
+
+      if (typeof previous === 'number') {
+        if (previous === bitcoinjs.opcodes.OP_0) {
+          return 0;
+        }
+        if (previous >= bitcoinjs.opcodes.OP_1 && previous <= bitcoinjs.opcodes.OP_16) {
+          return previous - bitcoinjs.opcodes.OP_1 + 1;
+        }
+      }
+
+      return null;
+    }
+
+    return null;
+  }
+
+  /** @asyncUnsafe */
+  private async $resolvePegInTimelock(input: IBitcoinApi.Vin, bitcoinaddress: string): Promise<number> {
+    const claimScript = input.pegin_witness?.[3];
+    if (!claimScript) {
+      logger.err(`Missing claim_script for peg-in input to address ${bitcoinaddress}, using fallback timelock ${FEDERATION_TIMELOCK_FALLBACK}.`);
+      return FEDERATION_TIMELOCK_FALLBACK;
+    }
+
+    // Check claim script against each fedpegscript most recent first
+    for (const pegScript of (await this.$getFederationPegScripts())) {
+      try {
+        const tweakResult = await bitcoinClient.tweakFedPegScript(claimScript, pegScript.fedpegscript);
+        const matches = tweakResult?.p2wsh === bitcoinaddress || tweakResult?.p2shwsh === bitcoinaddress;
+        if (matches) {
+          return pegScript.timelock;
+        }
+      } catch (e) {
+        const message = e instanceof Error ? e.message : String(e);
+        logger.err(`tweakfedpegscript failed for address ${bitcoinaddress}: ${message}`);
+      }
+    }
+
+    logger.err(`No matching fedpegscript found for address ${bitcoinaddress}, using fallback timelock ${FEDERATION_TIMELOCK_FALLBACK}.`);
+    return FEDERATION_TIMELOCK_FALLBACK;
+  }
+
+  /** @asyncUnsafe */
   protected async $getLatestBlockHeightFromDatabase(): Promise<number> {
     const query = `SELECT number FROM state WHERE name = 'last_elements_block'`;
     const [rows] = await DB.query(query);
     return rows[0]['number'];
   }
 
+  /** @asyncUnsafe */
   protected async $saveLatestBlockToDatabase(blockHeight: number) {
     const query = `UPDATE state SET number = ? WHERE name = 'last_elements_block'`;
     await DB.query(query, [blockHeight]);
@@ -115,19 +258,12 @@ class ElementsParser {
 
   ///////////// FEDERATION AUDIT //////////////
 
-  public async $updateFederationUtxos() {
-    if (this.isUtxosUpdatingRunning) {
-      return;
-    }
-
-    this.isUtxosUpdatingRunning = true;
-
+  public async $updateFederationUtxos(stopAt: number) {
     try {
       let auditProgress = await this.$getAuditProgress();
       // If no peg in transaction was found in the database, return
       if (!auditProgress.lastBlockAudit) {
         logger.debug(`No Federation UTXOs found in the database. Waiting for some to be confirmed before starting the Federation UTXOs audit`);
-        this.isUtxosUpdatingRunning = false;
         return;
       }
 
@@ -135,7 +271,6 @@ class ElementsParser {
       // If the bitcoin blockchain is not synced yet, return
       if (bitcoinBlocksToSync.bitcoinHeaders > bitcoinBlocksToSync.bitcoinBlocks + 1) {
         logger.debug(`Bitcoin client is not synced yet. ${bitcoinBlocksToSync.bitcoinHeaders - bitcoinBlocksToSync.bitcoinBlocks} blocks remaining to sync before the Federation audit process can start`);
-        this.isUtxosUpdatingRunning = false;
         return;
       }
 
@@ -154,19 +289,28 @@ class ElementsParser {
 
         // Get the peg-out addresses that need to be scanned
         const redeemAddresses = await this.$getRedeemAddressesToScan();
+        const redeemAddressesByAddress = new Map<string, any[]>();
+        for (const redeemAddress of redeemAddresses) {
+          const entries = redeemAddressesByAddress.get(redeemAddress.bitcoinaddress);
+          if (entries) {
+            entries.push(redeemAddress);
+          } else {
+            redeemAddressesByAddress.set(redeemAddress.bitcoinaddress, [redeemAddress]);
+          }
+        }
 
         // The fast way: check if these UTXOs are still unspent as of the current block with gettxout
-        let spentAsTip: any[];
-        let unspentAsTip: any[];
+        let spentAsTip: Map<string, any>;
+        let unspentAsTip: Map<string, any>;
         if (auditProgress.confirmedTip - auditProgress.lastBlockAudit <= 150) { // If the audit status is not too far in the past, we can use gettxout (fast way)
           const utxosToParse = await this.$getFederationUtxosToParse(utxos);
           spentAsTip = utxosToParse.spentAsTip;
           unspentAsTip = utxosToParse.unspentAsTip;
           logger.debug(`Found ${utxos.length} Federation UTXOs and ${redeemAddresses.length} Peg-Out Addresses to scan in Bitcoin block height #${auditProgress.lastBlockAudit} / #${auditProgress.confirmedTip}`);
-          logger.debug(`${unspentAsTip.length} / ${utxos.length} Federation UTXOs are unspent as of tip`);
+          logger.debug(`${unspentAsTip.size} / ${utxos.length} Federation UTXOs are unspent as of tip`);
         } else { // If the audit status is too far in the past, it is useless and wasteful to look for still unspent txos since they will all be spent as of the tip
-          spentAsTip = utxos;
-          unspentAsTip = [];
+          spentAsTip = new Map<string, any>(utxos.map(utxo => [`${utxo.txid}:${utxo.txindex}`, utxo]));
+          unspentAsTip = new Map<string, any>();
 
           // Logging
           const elapsedSeconds = (Date.now() / 1000) - timer;
@@ -174,7 +318,7 @@ class ElementsParser {
             const runningFor = (Date.now() / 1000) - startedAt;
             const blockPerSeconds = indexedThisRun / elapsedSeconds;
             indexingSpeeds.push(blockPerSeconds);
-            if (indexingSpeeds.length > 100) indexingSpeeds.shift(); // Keep the length of the up to 100 last indexing speeds
+            if (indexingSpeeds.length > 100) {indexingSpeeds.shift();} // Keep the length of the up to 100 last indexing speeds
             const meanIndexingSpeed = indexingSpeeds.reduce((a, b) => a + b, 0) / indexingSpeeds.length;
             const eta = (auditProgress.confirmedTip - auditProgress.lastBlockAudit) / meanIndexingSpeed;
             logger.debug(`Scanning ${utxos.length} Federation UTXOs and ${redeemAddresses.length} Peg-Out Addresses at Bitcoin block height #${auditProgress.lastBlockAudit} / #${auditProgress.confirmedTip} | ~${meanIndexingSpeed.toFixed(2)} blocks/sec | elapsed: ${(runningFor / 60).toFixed(0)} minutes | ETA: ${(eta / 60).toFixed(0)} minutes`);
@@ -186,53 +330,60 @@ class ElementsParser {
         // The slow way: parse the block to look for the spending tx
         const blockHash: IBitcoinApi.ChainTips = await bitcoinSecondClient.getBlockHash(auditProgress.lastBlockAudit);
         const block: IBitcoinApi.Block = await bitcoinSecondClient.getBlock(blockHash, 2);
-        await this.$parseBitcoinBlock(block, spentAsTip, unspentAsTip, auditProgress.confirmedTip, redeemAddresses);
+        await this.$parseBitcoinBlock(block, spentAsTip, unspentAsTip, auditProgress.confirmedTip, redeemAddressesByAddress);
 
         // Finally, update the lastblockupdate of the remaining UTXOs and save to the database
-        const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`)
+        const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`);
         await this.$saveLastBlockAuditToDatabase(minBlockUpdate[0]['lastblockupdate']);
 
         auditProgress = await this.$getAuditProgress();
         auditProgress.lastBlockAudit++;
         indexedThisRun++;
+
+        if ((Date.now() / 1000) >= stopAt) {
+          logger.debug(`Reached max federation audit runtime, pausing audit to let Liquid parsing run and resuming later`);
+          break;
+        }
       }
 
-      this.isUtxosUpdatingRunning = false;
     } catch (e) {
-      this.isUtxosUpdatingRunning = false;
       throw new Error(e instanceof Error ? e.message : 'Error');
-    } 
+    }
   }
 
   // Get the UTXOs that need to be scanned in block height (UTXOs that were last updated in the block height - 1)
-  protected async $getFederationUtxosToScan(height: number) { 
+  /** @asyncUnsafe */
+  protected async $getFederationUtxosToScan(height: number) {
     const query = `SELECT txid, txindex, bitcoinaddress, amount, blocknumber, timelock, expiredAt FROM federation_txos WHERE lastblockupdate = ? AND unspent = 1`;
     const [rows] = await DB.query(query, [height - 1]);
     return rows as any[];
   }
 
   // Returns the UTXOs that are spent as of tip and need to be scanned
-  protected async $getFederationUtxosToParse(utxos: any[]): Promise<any> {
-    const spentAsTip: any[] = [];
-    const unspentAsTip: any[] = [];
+  /** @asyncUnsafe */
+  protected async $getFederationUtxosToParse(utxos: any[]): Promise<{ spentAsTip: Map<string, any>; unspentAsTip: Map<string, any> }> {
+    const spentAsTip = new Map<string, any>();
+    const unspentAsTip = new Map<string, any>();
 
     for (const utxo of utxos) {
       const result = await bitcoinSecondClient.getTxOut(utxo.txid, utxo.txindex, false);
-      result ? unspentAsTip.push(utxo) : spentAsTip.push(utxo);
+      const key = `${utxo.txid}:${utxo.txindex}`;
+      result ? unspentAsTip.set(key, utxo) : spentAsTip.set(key, utxo);
     }
-    
+
     return {spentAsTip, unspentAsTip};
   }
 
-  protected async $parseBitcoinBlock(block: IBitcoinApi.Block, spentAsTip: any[], unspentAsTip: any[], confirmedTip: number, redeemAddressesData: any[] = []) {
-    const redeemAddresses: string[] = redeemAddressesData.map(redeemAddress => redeemAddress.bitcoinaddress);
+  /** @asyncUnsafe */
+  protected async $parseBitcoinBlock(block: IBitcoinApi.Block, spentAsTip: Map<string, any>, unspentAsTip: Map<string, any>, confirmedTip: number, redeemAddressesByAddress: Map<string, any[]>) {
+    const federationTimelockByAddress = new Map<string, number>((await this.$getFederationPegScripts()).map(fedAddress => [fedAddress.address, fedAddress.timelock]));
     for (const tx of block.tx) {
       let mightRedeemInThisTx = false;
       // Check if the Federation UTXOs that was spent as of tip are spent in this block
       for (const input of tx.vin) {
-        const txo = spentAsTip.find(txo => txo.txid === input.txid && txo.txindex === input.vout);
+        const txo = spentAsTip.get(`${input.txid!}:${input.vout!}`);
         if (txo) {
-          mightRedeemInThisTx = true; // A Federation UTXO is spent in this block: we might find a peg-out address in the outputs
+          mightRedeemInThisTx = true; // A Federation UTXO is spent in this tx: a peg-out redeem output may be present
           if (txo.expiredAt > 0 ) {
             if (input.txinwitness?.length !== 13) { // Check if the witness data of the input contains the 11 signatures: if it doesn't, emergency keys are being used
               await DB.query(`UPDATE federation_txos SET unspent = 0, lastblockupdate = ?, lasttimeupdate = ?, emergencyKey = 1 WHERE txid = ? AND txindex = ?`, [block.height, block.time, txo.txid, txo.txindex]);
@@ -245,36 +396,44 @@ class ElementsParser {
             await DB.query(`UPDATE federation_txos SET unspent = 0, lastblockupdate = ?, lasttimeupdate = ? WHERE txid = ? AND txindex = ?`, [block.height, block.time, txo.txid, txo.txindex]);
             logger.debug(`Federation UTXO ${txo.txid}:${txo.txindex} (${txo.amount} sats) was spent in block ${block.height}`);
           }
-          // Remove the TXO from the utxo array
-          spentAsTip.splice(spentAsTip.indexOf(txo), 1);
+          // Remove the TXO from the map
+          spentAsTip.delete(`${txo.txid}:${txo.txindex}`);
         }
       }
       // Check if an output is sent to a change address of the federation
       for (const output of tx.vout) {
-        if (output.scriptPubKey.address && federationChangeAddresses.includes(output.scriptPubKey.address)) {
-          // Check that the UTXO was not already added in the DB by previous scans
-          const [rows_check] = await DB.query(`SELECT txid FROM federation_txos WHERE txid = ? AND txindex = ?`, [tx.txid, output.n]) as any[];
-          if (rows_check.length === 0) {
-            const timelock = output.scriptPubKey.address === federationChangeAddresses[0] ? 4032 : 2016; // P2WSH change address has a 4032 timelock, P2SH change address has a 2016 timelock
-            const query_utxos = `INSERT INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate, timelock, expiredAt, emergencyKey, pegtxid, pegindex, pegblocktime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
-            const params_utxos: (string | number)[] = [tx.txid, output.n, output.scriptPubKey.address, output.value * 100000000, block.height, block.time, 1, block.height, 0, timelock, 0, 0, '', 0, 0];
-            await DB.query(query_utxos, params_utxos);
-            // Add the UTXO to the utxo array
-            spentAsTip.push({
-              txid: tx.txid,
-              txindex: output.n,
-              bitcoinaddress: output.scriptPubKey.address,
-              amount: output.value * 100000000,
-              blocknumber: block.height,
-              timelock: timelock,
-              expiredAt: 0,
-            });
-            logger.debug(`Added new Federation UTXO ${tx.txid}:${output.n} (${Math.round(output.value * 100000000)} sats), change address: ${output.scriptPubKey.address}`);
+        if (output.scriptPubKey.address) {
+          const timelock = federationTimelockByAddress.get(output.scriptPubKey.address);
+          if (timelock !== undefined) {
+            // Check that the UTXO was not already added in the DB by previous scans
+            const [rows_check] = await DB.query(`SELECT txid FROM federation_txos WHERE txid = ? AND txindex = ?`, [tx.txid, output.n]) as any[];
+            if (rows_check.length === 0) {
+              const query_utxos = `INSERT INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate, timelock, expiredAt, emergencyKey, pegtxid, pegindex, pegblocktime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
+              const params_utxos: (string | number)[] = [tx.txid, output.n, output.scriptPubKey.address, output.value * 100000000, block.height, block.time, 1, block.height, 0, timelock, 0, 0, '', 0, 0];
+              await DB.query(query_utxos, params_utxos);
+              // Add the UTXO to the map
+              spentAsTip.set(`${tx.txid}:${output.n}`, {
+                txid: tx.txid,
+                txindex: output.n,
+                bitcoinaddress: output.scriptPubKey.address,
+                amount: output.value * 100000000,
+                blocknumber: block.height,
+                timelock: timelock,
+                expiredAt: 0,
+              });
+              logger.debug(`Added new Federation UTXO ${tx.txid}:${output.n} (${Math.round(output.value * 100000000)} sats), change address: ${output.scriptPubKey.address}`);
+            }
           }
-        }
-        if (mightRedeemInThisTx && output.scriptPubKey.address && redeemAddresses.includes(output.scriptPubKey.address)) {
-          // Find the number of times output.scriptPubKey.address appears in redeemAddresses. There can be address reuse for peg-outs...
-          const matchingAddress: any[] = redeemAddressesData.filter(redeemAddress => redeemAddress.bitcoinaddress === output.scriptPubKey.address && -redeemAddress.amount === Math.round(output.value * 100000000));
+
+          if (!mightRedeemInThisTx) {
+            continue;
+          }
+          const redeemCandidates = redeemAddressesByAddress.get(output.scriptPubKey.address);
+          if (!redeemCandidates) {
+            continue;
+          }
+          // Find the number of times output.scriptPubKey.address appears in the candidates. There can be address reuse for peg-outs...
+          const matchingAddress: any[] = redeemCandidates.filter(redeemAddress => -redeemAddress.amount === Math.round(output.value * 100000000));
           if (matchingAddress.length > 0) {
             if (matchingAddress.length > 1) {
               // If there are more than one peg out address with the same amount, we can't know which one redeemed the UTXO: we take the oldest one
@@ -286,9 +445,13 @@ class ElementsParser {
             const query_add_redeem = `UPDATE elements_pegs SET bitcointxid = ?, bitcoinindex = ? WHERE bitcoinaddress = ? AND amount = ? AND datetime = ?`;
             const params_add_redeem: (string | number)[] = [tx.txid, output.n, matchingAddress[0].bitcoinaddress, matchingAddress[0].amount, matchingAddress[0].datetime];
             await DB.query(query_add_redeem, params_add_redeem);
-            const index = redeemAddressesData.indexOf(matchingAddress[0]);
-            redeemAddressesData.splice(index, 1);
-            redeemAddresses.splice(index, 1);
+            const index = redeemCandidates.indexOf(matchingAddress[0]);
+            if (index !== -1) {
+              redeemCandidates.splice(index, 1);
+              if (redeemCandidates.length === 0) {
+                redeemAddressesByAddress.delete(output.scriptPubKey.address);
+              }
+            }
           } else { // The output amount does not match the peg-out amount... log it
             logger.debug(`Found redeem txid ${tx.txid}:${output.n} to peg-out address ${output.scriptPubKey.address} but output amount ${Math.round(output.value * 100000000)} does not match the peg-out amount!`);
           }
@@ -296,7 +459,7 @@ class ElementsParser {
       }
     }
 
-    for (const utxo of spentAsTip) {   
+    for (const utxo of spentAsTip.values()) {
       if (utxo.expiredAt === 0 && block.height >= utxo.blocknumber + utxo.timelock) { // The UTXO is expiring in this block
         await DB.query(`UPDATE federation_txos SET lastblockupdate = ?, expiredAt = ? WHERE txid = ? AND txindex = ?`, [block.height, block.time, utxo.txid, utxo.txindex]);
       } else {
@@ -304,23 +467,25 @@ class ElementsParser {
       }
     }
 
-    for (const utxo of unspentAsTip) {
+    for (const utxo of unspentAsTip.values()) {
       if (utxo.expiredAt === 0 && block.height >= utxo.blocknumber + utxo.timelock) { // The UTXO is expiring in this block
-        await DB.query(`UPDATE federation_txos SET unspent = 0, lastblockupdate = ?, expiredAt = ? WHERE txid = ? AND txindex = ?`, [confirmedTip, block.time, utxo.txid, utxo.txindex]);
+        await DB.query(`UPDATE federation_txos SET lastblockupdate = ?, expiredAt = ? WHERE txid = ? AND txindex = ?`, [confirmedTip, block.time, utxo.txid, utxo.txindex]);
       } else if (utxo.expiredAt === 0 && confirmedTip >= utxo.blocknumber + utxo.timelock) { // The UTXO is expiring before the tip: we need to keep track of it
-        await DB.query(`UPDATE federation_txos SET lastblockupdate = ? WHERE txid = ? AND txindex = ?`, [utxo.blocknumber + utxo.timelock - 1, utxo.txid, utxo.txindex]); 
+        await DB.query(`UPDATE federation_txos SET lastblockupdate = ? WHERE txid = ? AND txindex = ?`, [utxo.blocknumber + utxo.timelock - 1, utxo.txid, utxo.txindex]);
       } else {
       await DB.query(`UPDATE federation_txos SET lastblockupdate = ? WHERE txid = ? AND txindex = ?`, [confirmedTip, utxo.txid, utxo.txindex]);
       }
     }
   }
 
+  /** @asyncUnsafe */
   protected async $saveLastBlockAuditToDatabase(blockHeight: number) {
     const query = `UPDATE state SET number = ? WHERE name = 'last_bitcoin_block_audit'`;
     await DB.query(query, [blockHeight]);
   }
 
   // Get the bitcoin block where the audit process was last updated
+  /** @asyncUnsafe */
   protected async $getAuditProgress(): Promise<any> {
     const lastblockaudit = await this.$getLastBlockAudit();
     const bitcoinBlocksToSync = await this.$getBitcoinBlockchainState();
@@ -331,20 +496,23 @@ class ElementsParser {
   }
 
   // Get the bitcoin blocks remaining to be synced
+  /** @asyncUnsafe */
   protected async $getBitcoinBlockchainState(): Promise<any> {
     const result = await bitcoinSecondClient.getBlockchainInfo();
     return {
       bitcoinBlocks: result.blocks,
       bitcoinHeaders: result.headers,
-    }
+    };
   }
 
+  /** @asyncUnsafe */
   protected async $getLastBlockAudit(): Promise<number> {
     const query = `SELECT number FROM state WHERE name = 'last_bitcoin_block_audit'`;
     const [rows] = await DB.query(query);
     return rows[0]['number'];
   }
 
+  /** @asyncUnsafe */
   protected async $getRedeemAddressesToScan(): Promise<any[]> {
     const query = `SELECT datetime, amount, bitcoinaddress FROM elements_pegs where amount < 0 AND bitcoinaddress != '' AND bitcointxid = '';`;
     const [rows]: any[] = await DB.query(query);
@@ -357,6 +525,7 @@ class ElementsParser {
 
   ///////////// DATA QUERY //////////////
 
+  /** @asyncUnsafe */
   public async $getAuditStatus(): Promise<any> {
     const lastBlockAudit = await this.$getLastBlockAudit();
     const bitcoinBlocksToSync = await this.$getBitcoinBlockchainState();
@@ -364,16 +533,18 @@ class ElementsParser {
       bitcoinBlocks: bitcoinBlocksToSync.bitcoinBlocks,
       bitcoinHeaders: bitcoinBlocksToSync.bitcoinHeaders,
       lastBlockAudit: lastBlockAudit,
-      isAuditSynced: bitcoinBlocksToSync.bitcoinHeaders - bitcoinBlocksToSync.bitcoinBlocks <= 2 && bitcoinBlocksToSync.bitcoinBlocks - lastBlockAudit <= 3,
+      isAuditSynced: bitcoinBlocksToSync.bitcoinHeaders - bitcoinBlocksToSync.bitcoinBlocks <= 3 && bitcoinBlocksToSync.bitcoinBlocks - lastBlockAudit <= auditSyncToleranceBlocks,
     };
   }
 
+  /** @asyncUnsafe */
   public async $getPegDataByMonth(): Promise<any> {
     const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`;
     const [rows] = await DB.query(query);
     return rows;
   }
 
+  /** @asyncUnsafe */
   public async $getFederationReservesByMonth(): Promise<any> {
     const query = `
     SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(blocktime), '%Y-%m-01') AS date FROM federation_txos 
@@ -384,12 +555,13 @@ class ElementsParser {
       AND 
         (expiredAt = 0 OR expiredAt > UNIX_TIMESTAMP(LAST_DAY(FROM_UNIXTIME(blocktime)) + INTERVAL 1 DAY))
     GROUP BY 
-        date;`;          
+        date;`;
     const [rows] = await DB.query(query);
     return rows;
   }
 
   // Get the current L-BTC pegs and the last Liquid block it was updated
+  /** @asyncUnsafe */
   public async $getCurrentLbtcSupply(): Promise<any> {
     const [rows] = await DB.query(`SELECT SUM(amount) AS LBTC_supply FROM elements_pegs;`);
     const lastblockupdate = await this.$getLatestBlockHeightFromDatabase();
@@ -402,6 +574,7 @@ class ElementsParser {
   }
 
   // Get the current reserves of the federation and the last Bitcoin block it was updated
+  /** @asyncUnsafe */
   public async $getCurrentFederationReserves(): Promise<any> {
     const [rows] = await DB.query(`SELECT SUM(amount) AS total_balance FROM federation_txos WHERE unspent = 1 AND expiredAt = 0;`);
     const lastblockaudit = await this.$getLastBlockAudit();
@@ -414,6 +587,7 @@ class ElementsParser {
   }
 
   // Get all of the federation addresses, most balances first
+  /** @asyncUnsafe */
   public async $getFederationAddresses(): Promise<any> {
     const query = `SELECT bitcoinaddress, SUM(amount) AS balance FROM federation_txos WHERE unspent = 1 AND expiredAt = 0 GROUP BY bitcoinaddress ORDER BY balance DESC;`;
     const [rows] = await DB.query(query);
@@ -421,6 +595,7 @@ class ElementsParser {
   }
 
   // Get all of the UTXOs held by the federation, most recent first
+  /** @asyncUnsafe */
   public async $getFederationUtxos(): Promise<any> {
     const query = `SELECT txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, pegtxid, pegindex, pegblocktime, timelock, expiredAt FROM federation_txos WHERE unspent = 1 AND expiredAt = 0 ORDER BY blocktime DESC;`;
     const [rows] = await DB.query(query);
@@ -428,6 +603,7 @@ class ElementsParser {
   }
 
   // Get expired UTXOs, most recent first
+  /** @asyncUnsafe */
   public async $getExpiredUtxos(): Promise<any> {
     const query = `SELECT txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, pegtxid, pegindex, pegblocktime, timelock, expiredAt FROM federation_txos WHERE unspent = 1 AND expiredAt > 0 ORDER BY blocktime DESC;`;
     const [rows]: any[] = await DB.query(query);
@@ -439,13 +615,15 @@ class ElementsParser {
   }
 
     // Get utxos that were spent using emergency keys
+    /** @asyncUnsafe */
     public async $getEmergencySpentUtxos(): Promise<any> {
       const query = `SELECT txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, pegtxid, pegindex, pegblocktime, timelock, expiredAt FROM federation_txos WHERE emergencyKey = 1 ORDER BY blocktime DESC;`;
       const [rows] = await DB.query(query);
       return rows;
     }
-  
+
   // Get the total number of federation addresses
+  /** @asyncUnsafe */
   public async $getFederationAddressesNumber(): Promise<any> {
     const query = `SELECT COUNT(DISTINCT bitcoinaddress) AS address_count FROM federation_txos WHERE unspent = 1 AND expiredAt = 0;`;
     const [rows] = await DB.query(query);
@@ -453,6 +631,7 @@ class ElementsParser {
   }
 
   // Get the total number of federation utxos
+  /** @asyncUnsafe */
   public async $getFederationUtxosNumber(): Promise<any> {
     const query = `SELECT COUNT(*) AS utxo_count FROM federation_txos WHERE unspent = 1 AND expiredAt = 0;`;
     const [rows] = await DB.query(query);
@@ -460,6 +639,7 @@ class ElementsParser {
   }
 
   // Get the total number of emergency spent utxos and their total amount
+  /** @asyncUnsafe */
   public async $getEmergencySpentUtxosStats(): Promise<any> {
     const query = `SELECT COUNT(*) AS utxo_count, SUM(amount) AS total_amount FROM federation_txos WHERE emergencyKey = 1;`;
     const [rows] = await DB.query(query);
@@ -467,6 +647,7 @@ class ElementsParser {
   }
 
   // Get recent pegs in / out
+  /** @asyncUnsafe */
   public async $getPegsList(count: number = 0): Promise<any> {
     const query = `SELECT txid, txindex, amount, bitcoinaddress, bitcointxid, bitcoinindex, datetime AS blocktime FROM elements_pegs ORDER BY block DESC LIMIT 15 OFFSET ?;`;
     const [rows] = await DB.query(query, [count]);
@@ -474,6 +655,7 @@ class ElementsParser {
   }
 
   // Get all peg in / out from the last month
+  /** @asyncUnsafe */
   public async $getPegsVolumeDaily(): Promise<any> {
     const pegInQuery = await DB.query(`SELECT SUM(amount) AS volume, COUNT(*) AS number FROM elements_pegs WHERE amount > 0 and datetime > UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -1, CURRENT_TIMESTAMP()));`);
     const pegOutQuery = await DB.query(`SELECT SUM(amount) AS volume, COUNT(*) AS number FROM elements_pegs WHERE amount < 0 and datetime > UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -1, CURRENT_TIMESTAMP()));`);
@@ -484,6 +666,7 @@ class ElementsParser {
   }
 
   // Get the total pegs number
+  /** @asyncUnsafe */
   public async $getPegsCount(): Promise<any> {
     const [rows] = await DB.query(`SELECT COUNT(*) AS pegs_count FROM elements_pegs;`);
     return rows[0];

backend/src/api/liquid/liquid.routes.ts

@@ -6,6 +6,8 @@ import icons from './icons';
 import { handleError } from '../../utils/api';
 import PricesRepository from '../../repositories/PricesRepository';
 
+const PROXY_PATH_SEGMENT_REGEX = /^(?!\.{1,2}$)[^\p{Cc}/?#\\]{1,256}$/u;
+
 class LiquidRoutes {
   public initRoutes(app: Application) {
     app
@@ -14,7 +16,7 @@ class LiquidRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'asset/:assetId/icon', this.getLiquidIcon)
       .get(config.MEMPOOL.API_URL_PREFIX + 'assets/group/:id', this.$getAssetGroup)
       ;
-    
+
     if (config.DATABASE.ENABLED) {
       app
         .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs', this.$getElementsPegs)
@@ -68,8 +70,13 @@ class LiquidRoutes {
   }
 
   private async $getAssetGroup(req: Request, res: Response) {
+    if (!PROXY_PATH_SEGMENT_REGEX.test(req.params.id)) {
+      handleError(req, res, 400, 'Invalid asset group id');
+      return;
+    }
+
     try {
-      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/group/${parseInt(req.params.id, 10)}`,
+      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/group/${encodeURIComponent(req.params.id)}`,
         { responseType: 'stream', timeout: 10000 });
       response.data.pipe(res);
     } catch (e) {
@@ -262,7 +269,7 @@ class LiquidRoutes {
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
-      if (['testnet', 'signet', 'liquidtestnet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
+      if (['testnet', 'signet', 'liquidtestnet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
         handleError(req, res, 400, 'Prices are not available on testnets.');
         return;
       }

backend/src/api/mempool-blocks.ts

@@ -8,6 +8,7 @@ import path from 'path';
 import mempool from './mempool';
 import { Acceleration } from './services/acceleration';
 import PoolsRepository from '../repositories/PoolsRepository';
+import { ProjectedBlock } from '../cluster-mempool/cluster-mempool';
 
 const MAX_UINT32 = Math.pow(2, 32) - 1;
 
@@ -45,8 +46,9 @@ class MempoolBlocks {
     return this.mempoolBlockDeltas;
   }
 
+  /** @asyncUnsafe */
   public async updatePools$(): Promise<void> {
-    if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) === false) {
+    if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) === false) {
       this.pools = {};
       return;
     }
@@ -98,6 +100,7 @@ class MempoolBlocks {
     return mempoolBlockDeltas;
   }
 
+  /** @asyncSafe */
   public async $makeBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
     const start = Date.now();
 
@@ -172,6 +175,7 @@ class MempoolBlocks {
     return this.mempoolBlocks;
   }
 
+  /** @asyncSafe */
   public async $updateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise<void> {
     if (!this.txSelectionWorker) {
       // need to reset the worker
@@ -228,12 +232,14 @@ class MempoolBlocks {
     }
   }
 
+  /** @asyncSafe */
   private resetRustGbt(): void {
     this.rustInitialized = false;
     this.rustGbtGenerator = new GbtGenerator(config.MEMPOOL.BLOCK_WEIGHT_UNITS, config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT);
   }
 
-  public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+  /** @asyncSafe */
+  public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number, dryRun = false): Promise<MempoolBlockWithTransactions[]> {
     const start = Date.now();
 
     // reset mempool short ids
@@ -273,7 +279,7 @@ class MempoolBlocks {
       const expectedSize = transactions.length;
       const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length;
       logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${expectedSize} in the mempool, ${overflow.length} were unmineable`);
-      const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults);
+      const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults, dryRun);
       logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
       return processed;
     } catch (e) {
@@ -285,20 +291,21 @@ class MempoolBlocks {
     return this.mempoolBlocks;
   }
 
+  /** @asyncSafe */
   public async $oneOffRustBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
     return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, false, useAccelerations, accelerationPool);
   }
 
-  public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+  /** @asyncSafe */
+  public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number, dryRun = false): Promise<MempoolBlockWithTransactions[]> {
     // GBT optimization requires that uids never get too sparse
     // as a sanity check, we should also explicitly prevent uint32 uid overflow
     if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * transactions.length), MAX_UINT32)) {
       this.resetRustGbt();
     }
 
-    if (!this.rustInitialized) {
-      // need to reset the worker
-      return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, true, useAccelerations, accelerationPool);
+    if (!this.rustInitialized || dryRun) {
+      return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, !dryRun, useAccelerations, accelerationPool, dryRun);
     }
 
     const start = Date.now();
@@ -337,7 +344,7 @@ class MempoolBlocks {
       if (transactions.length !== resultMempoolSize) {
         throw new Error(`GBT returned wrong number of transactions ${transactions.length} vs ${resultMempoolSize}, cache is probably out of sync`);
       } else {
-        const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, true);
+        const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, !dryRun, dryRun);
         this.removeUids(removedTxs);
         logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
         return processed;
@@ -349,20 +356,22 @@ class MempoolBlocks {
     }
   }
 
-  private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations: { [txid: string]: Acceleration }, accelerationPool, saveResults): MempoolBlockWithTransactions[] {
-    for (const txid of Object.keys(candidates?.txs ?? mempool)) {
-      if (txid in mempool) {
-        mempool[txid].cpfpDirty = false;
-        mempool[txid].ancestors = [];
-        mempool[txid].descendants = [];
-        mempool[txid].bestDescendant = null;
+  private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations: { [txid: string]: Acceleration }, accelerationPool, saveResults, dryRun = false): MempoolBlockWithTransactions[] {
+    if (!dryRun) {
+      for (const txid of Object.keys(candidates?.txs ?? mempool)) {
+        if (txid in mempool) {
+          mempool[txid].cpfpDirty = false;
+          mempool[txid].ancestors = [];
+          mempool[txid].descendants = [];
+          mempool[txid].bestDescendant = null;
+        }
       }
-    }
-    for (const [txid, rate] of rates) {
-      if (txid in mempool) {
-        mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize);
-        mempool[txid].effectiveFeePerVsize = rate;
-        mempool[txid].cpfpChecked = true;
+      for (const [txid, rate] of rates) {
+        if (txid in mempool) {
+          mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize);
+          mempool[txid].effectiveFeePerVsize = rate;
+          mempool[txid].cpfpChecked = true;
+        }
       }
     }
 
@@ -380,58 +389,60 @@ class MempoolBlocks {
       feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
     }
 
-    const ancestors: Ancestor[] = [];
-    const descendants: Ancestor[] = [];
-    let ancestor: MempoolTransactionExtended;
-    for (const cluster of clusters) {
-      for (const memberTxid of cluster) {
-        const mempoolTx = mempool[memberTxid];
-        if (mempoolTx) {
-          // ugly micro-optimization to avoid allocating new arrays
-          ancestors.length = 0;
-          descendants.length = 0;
-          let matched = false;
-          cluster.forEach(txid => {
-            ancestor = mempool[txid];
-            if (txid === memberTxid) {
-              matched = true;
-            } else {
-              if (!ancestor) {
-                console.log('txid missing from mempool! ', txid, candidates?.txs[txid]);
-                return;
-              }
-              const relative = {
-                txid: txid,
-                fee: ancestor.fee,
-                weight: (ancestor.adjustedVsize * 4),
-              };
-              if (matched) {
-                descendants.push(relative);
-                if (!mempoolTx.lastBoosted || (ancestor.firstSeen && ancestor.firstSeen > mempoolTx.lastBoosted)) {
-                  mempoolTx.lastBoosted = ancestor.firstSeen;
-                }
+    if (!dryRun) {
+      const ancestors: Ancestor[] = [];
+      const descendants: Ancestor[] = [];
+      let ancestor: MempoolTransactionExtended;
+      for (const cluster of clusters) {
+        for (const memberTxid of cluster) {
+          const mempoolTx = mempool[memberTxid];
+          if (mempoolTx) {
+            // ugly micro-optimization to avoid allocating new arrays
+            ancestors.length = 0;
+            descendants.length = 0;
+            let matched = false;
+            cluster.forEach(txid => {
+              ancestor = mempool[txid];
+              if (txid === memberTxid) {
+                matched = true;
               } else {
-                ancestors.push(relative);
+                if (!ancestor) {
+                  console.log('txid missing from mempool! ', txid, candidates?.txs[txid]);
+                  return;
+                }
+                const relative = {
+                  txid: txid,
+                  fee: ancestor.fee,
+                  weight: (ancestor.adjustedVsize * 4),
+                };
+                if (matched) {
+                  descendants.push(relative);
+                  if (!mempoolTx.lastBoosted || (ancestor.firstSeen && ancestor.firstSeen > mempoolTx.lastBoosted)) {
+                    mempoolTx.lastBoosted = ancestor.firstSeen;
+                  }
+                } else {
+                  ancestors.push(relative);
+                }
               }
+            });
+            if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) {
+              mempoolTx.cpfpDirty = true;
             }
-          });
-          if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) {
-            mempoolTx.cpfpDirty = true;
+            // ugly micro-optimization to avoid allocating new arrays or objects
+            if (mempoolTx.ancestors) {
+              mempoolTx.ancestors.length = 0;
+            } else {
+              mempoolTx.ancestors = [];
+            }
+            if (mempoolTx.descendants) {
+              mempoolTx.descendants.length = 0;
+            } else {
+              mempoolTx.descendants = [];
+            }
+            mempoolTx.ancestors.push(...ancestors);
+            mempoolTx.descendants.push(...descendants);
+            mempoolTx.cpfpChecked = true;
           }
-          // ugly micro-optimization to avoid allocating new arrays or objects
-          if (mempoolTx.ancestors) {
-            mempoolTx.ancestors.length = 0;
-          } else {
-            mempoolTx.ancestors = [];
-          }
-          if (mempoolTx.descendants) {
-            mempoolTx.descendants.length = 0;
-          } else {
-            mempoolTx.descendants = [];
-          }
-          mempoolTx.ancestors.push(...ancestors);
-          mempoolTx.descendants.push(...descendants);
-          mempoolTx.cpfpChecked = true;
         }
       }
     }
@@ -464,34 +475,42 @@ class MempoolBlocks {
         const txid = block[i];
         if (txid in mempool) {
           mempoolTx = mempool[txid];
-          // save position in projected blocks
-          mempoolTx.position = {
-            block: blockIndex,
-            vsize: totalVsize + (mempoolTx.vsize / 2),
-          };
 
-          if (txid in accelerations) {
-            acceleration = accelerations[txid];
-            if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
-              if (!mempoolTx.acceleration) {
-                mempoolTx.cpfpDirty = true;
-              }
-              mempoolTx.acceleration = true;
-              mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools;
-              mempoolTx.acceleratedAt = acceleration?.added;
-              mempoolTx.feeDelta = acceleration?.feeDelta;
-              for (const ancestor of mempoolTx.ancestors || []) {
-                if (!(ancestor.txid in mempool)) {
-                  continue;
+          if (!dryRun) {
+            // save position in projected blocks
+            mempoolTx.position = {
+              block: blockIndex,
+              vsize: totalVsize + (mempoolTx.vsize / 2),
+            };
+
+            if (txid in accelerations) {
+              acceleration = accelerations[txid];
+              if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
+                if (!mempoolTx.acceleration) {
+                  mempoolTx.cpfpDirty = true;
                 }
-                if (!mempool[ancestor.txid].acceleration) {
-                  mempool[ancestor.txid].cpfpDirty = true;
+                mempoolTx.acceleration = true;
+                mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools;
+                mempoolTx.acceleratedAt = acceleration?.added;
+                mempoolTx.feeDelta = acceleration?.feeDelta;
+                for (const ancestor of mempoolTx.ancestors || []) {
+                  if (!(ancestor.txid in mempool)) {
+                    continue;
+                  }
+                  if (!mempool[ancestor.txid].acceleration) {
+                    mempool[ancestor.txid].cpfpDirty = true;
+                  }
+                  mempool[ancestor.txid].acceleration = true;
+                  mempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy;
+                  mempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt;
+                  mempool[ancestor.txid].feeDelta = mempoolTx.feeDelta;
+                  isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy;
+                }
+              } else {
+                if (mempoolTx.acceleration) {
+                  mempoolTx.cpfpDirty = true;
+                  delete mempoolTx.acceleration;
                 }
-                mempool[ancestor.txid].acceleration = true;
-                mempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy;
-                mempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt;
-                mempool[ancestor.txid].feeDelta = mempoolTx.feeDelta;
-                isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy;
               }
             } else {
               if (mempoolTx.acceleration) {
@@ -499,11 +518,6 @@ class MempoolBlocks {
                 delete mempoolTx.acceleration;
               }
             }
-          } else {
-            if (mempoolTx.acceleration) {
-              mempoolTx.cpfpDirty = true;
-              delete mempoolTx.acceleration;
-            }
           }
 
           // online calculation of stack-of-blocks fee stats
@@ -541,6 +555,115 @@ class MempoolBlocks {
     return mempoolBlocks;
   }
 
+  public processClusterMempoolBlocks(projectedBlocks: ProjectedBlock[], newMempool: { [txid: string]: MempoolTransactionExtended }, accelerations: { [txid: string]: Acceleration }, saveResults = true, accelerationPool?: number): MempoolBlockWithTransactions[] {
+    const lastBlockIndex = projectedBlocks.length - 1;
+    let hasBlockStack = projectedBlocks.length >= 8;
+    let stackWeight = 0;
+    let feeStatsCalculator: OnlineFeeStatsCalculator | null = null;
+    if (hasBlockStack) {
+      stackWeight = projectedBlocks[lastBlockIndex].weight;
+      hasBlockStack = stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS;
+      feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
+    }
+
+    const isAcceleratedBy: { [txid: string]: number[] | false } = {};
+
+    const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
+    let mempoolTx: MempoolTransactionExtended;
+    let acceleration: Acceleration;
+    const mempoolBlocks: MempoolBlockWithTransactions[] = [];
+
+    for (let blockIndex = 0; blockIndex < projectedBlocks.length; blockIndex++) {
+      const projected = projectedBlocks[blockIndex];
+      let totalSize = 0;
+      let totalVsize = 0;
+      let totalWeight = 0;
+      let totalFees = 0;
+      const transactions: MempoolTransactionExtended[] = [];
+      const validTxids: string[] = [];
+
+      for (const txid of projected.txids) {
+        if (txid in newMempool) {
+          mempoolTx = newMempool[txid];
+          validTxids.push(txid);
+
+          // save position in projected blocks
+          mempoolTx.position = {
+            block: blockIndex,
+            vsize: totalVsize + (mempoolTx.vsize / 2),
+          };
+
+          if (txid in accelerations) {
+            acceleration = accelerations[txid];
+            if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
+              if (!mempoolTx.acceleration) {
+                mempoolTx.cpfpDirty = true;
+              }
+              mempoolTx.acceleration = true;
+              mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools;
+              mempoolTx.acceleratedAt = acceleration?.added;
+              mempoolTx.feeDelta = acceleration?.feeDelta;
+              for (const ancestor of mempoolTx.ancestors || []) {
+                if (!(ancestor.txid in newMempool)) {
+                  continue;
+                }
+                if (!newMempool[ancestor.txid].acceleration) {
+                  newMempool[ancestor.txid].cpfpDirty = true;
+                }
+                newMempool[ancestor.txid].acceleration = true;
+                newMempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy;
+                newMempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt;
+                newMempool[ancestor.txid].feeDelta = mempoolTx.feeDelta;
+                isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy;
+              }
+            } else {
+              if (mempoolTx.acceleration) {
+                mempoolTx.cpfpDirty = true;
+                delete mempoolTx.acceleration;
+              }
+            }
+          } else {
+            if (mempoolTx.acceleration) {
+              mempoolTx.cpfpDirty = true;
+              delete mempoolTx.acceleration;
+            }
+          }
+
+          if (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) {
+            feeStatsCalculator.processNext(mempoolTx);
+          }
+
+          totalSize += mempoolTx.size;
+          totalVsize += mempoolTx.vsize;
+          totalWeight += mempoolTx.weight;
+          totalFees += mempoolTx.fee;
+
+          if (totalVsize <= sizeLimit) {
+            transactions.push(mempoolTx);
+          }
+        }
+      }
+
+      mempoolBlocks[blockIndex] = this.dataToMempoolBlocks(
+        validTxids,
+        transactions,
+        totalSize,
+        totalWeight,
+        totalFees,
+        (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined,
+      );
+    }
+
+    if (saveResults) {
+      const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
+      this.mempoolBlocks = mempoolBlocks;
+      this.mempoolBlockDeltas = deltas;
+      this.updateAccelerationPositions(newMempool, accelerations, mempoolBlocks);
+    }
+
+    return mempoolBlocks;
+  }
+
   private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions {
     if (!feeStats) {
       feeStats = Common.calcEffectiveFeeStatistics(transactions);

backend/src/api/mempool.ts

@@ -13,6 +13,7 @@ import { Acceleration } from './services/acceleration';
 import accelerationApi from './services/acceleration';
 import redisCache from './redis-cache';
 import blocks from './blocks';
+import { ClusterMempool } from '../cluster-mempool/cluster-mempool';
 
 class Mempool {
   private inSync: boolean = false;
@@ -22,6 +23,7 @@ class Mempool {
   private spendMap = new Map<string, MempoolTransactionExtended>();
   private recentlyDeleted: MempoolTransactionExtended[][] = []; // buffer of transactions deleted in recent mempool updates
   private mempoolInfo: IBitcoinApi.MempoolInfo;
+  public clusterMempool: ClusterMempool | null = null;
   private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[],
     deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[]) => void) | undefined;
   private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[],
@@ -38,7 +40,7 @@ class Mempool {
   private mempoolProtection = 0;
   private latestTransactions: any[] = [];
 
-  private ESPLORA_MISSING_TX_WARNING_THRESHOLD = 100; 
+  private ESPLORA_MISSING_TX_WARNING_THRESHOLD = 100;
   private SAMPLE_TIME = 10000; // In ms
   private timer = new Date().getTime();
   private missingTxCount = 0;
@@ -51,17 +53,20 @@ class Mempool {
     // Initialize mempoolInfo here to avoid circular dependency issues
     // Use config directly instead of Common.isLiquid() to break circular dependency
     const isLiquid = config.MEMPOOL.NETWORK === 'liquid' || config.MEMPOOL.NETWORK === 'liquidtestnet';
-    this.mempoolInfo = { 
-      loaded: false, 
-      size: 0, 
-      bytes: 0, 
-      usage: 0, 
+    this.mempoolInfo = {
+      loaded: false,
+      size: 0,
+      bytes: 0,
+      usage: 0,
       total_fee: 0,
-      maxmempool: 300000000, 
-      mempoolminfee: isLiquid ? 0.00000100 : 0.00001000, 
-      minrelaytxfee: isLiquid ? 0.00000100 : 0.00001000 
+      maxmempool: 300000000,
+      mempoolminfee: isLiquid ? 0.00000100 : 0.00001000,
+      minrelaytxfee: isLiquid ? 0.00000100 : 0.00001000
     };
     this.txPerSecondInterval = setInterval(this.updateTxPerSecond.bind(this), 1000);
+    if (config.MEMPOOL.CLUSTER_MEMPOOL) {
+      this.clusterMempool = new ClusterMempool(this.mempoolCache, this.accelerations);
+    }
   }
 
   /**
@@ -122,6 +127,7 @@ class Mempool {
     return this.spendMap.get(`${txid}:${index}`);
   }
 
+  /** @asyncUnsafe */
   public async $setMempool(mempoolData: { [txId: string]: MempoolTransactionExtended }) {
     this.mempoolCache = mempoolData;
     let count = 0;
@@ -152,6 +158,9 @@ class Mempool {
       await redisCache.$flushTransactions();
       logger.debug(`Finished migrating cache transactions in ${((Date.now() - redisTimer) / 1000).toFixed(2)} seconds`);
     }
+    if (config.MEMPOOL.CLUSTER_MEMPOOL) {
+      this.clusterMempool = new ClusterMempool(this.mempoolCache, this.accelerations);
+    }
     if (this.mempoolChangedCallback) {
       this.mempoolChangedCallback(this.mempoolCache, [], [], []);
     }
@@ -203,6 +212,7 @@ class Mempool {
     return this.mempoolCandidates;
   }
 
+  /** @asyncUnsafe */
   public async $updateMemPoolInfo() {
     this.mempoolInfo = await this.$getMempoolInfo();
   }
@@ -232,6 +242,7 @@ class Mempool {
     return txTimes;
   }
 
+  /** @asyncUnsafe */
   public async $updateMempool(transactions: string[], accelerations: Record<string, Acceleration> | null, minFeeMempool: string[], minFeeTip: number, pollRate: number): Promise<void> {
     logger.debug(`Updating mempool...`);
 
@@ -371,6 +382,7 @@ class Mempool {
       for (const tx of deletedTransactions) {
         delete this.mempoolCache[tx.txid];
       }
+      redisCache.queueTransactionsForRemoval(deletedTransactions.map(tx => tx.txid));
     }
 
     const candidates = await this.getNextCandidates(minFeeMempool, minFeeTip, deletedTransactions);
@@ -384,6 +396,14 @@ class Mempool {
       hasChange = true;
     }
 
+    if (config.MEMPOOL.CLUSTER_MEMPOOL && (newTransactions.length || deletedTransactions.length || accelerationDelta.length)) {
+      this.clusterMempool?.applyMempoolChange({
+        added: newTransactions,
+        removed: deletedTransactions,
+        accelerations: this.getAccelerations(),
+      });
+    }
+
     this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize);
 
     const candidatesChanged = candidates?.added?.length || candidates?.removed?.length;
@@ -409,7 +429,7 @@ class Mempool {
     // Update Redis cache
     if (config.REDIS.ENABLED) {
       await redisCache.$flushTransactions();
-      await redisCache.$removeTransactions(deletedTransactions.map(tx => tx.txid));
+      await redisCache.$removeTransactions();
       await rbfCache.updateCache();
     }
 

backend/src/api/mining/mining-routes.ts

@@ -1,12 +1,12 @@
 import { Application, Request, Response } from 'express';
-import config from "../../config";
+import config from '../../config';
 import logger from '../../logger';
 import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
 import BlocksRepository from '../../repositories/BlocksRepository';
 import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
 import HashratesRepository from '../../repositories/HashratesRepository';
 import bitcoinClient from '../bitcoin/bitcoin-client';
-import mining from "./mining";
+import mining from './mining';
 import PricesRepository from '../../repositories/PricesRepository';
 import AccelerationRepository from '../../repositories/AccelerationRepository';
 import accelerationApi from '../services/acceleration';
@@ -53,7 +53,7 @@ class MiningRoutes {
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
-      if (['testnet', 'signet', 'liquidtestnet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
+      if (['testnet', 'signet', 'liquidtestnet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
         handleError(req, res, 400, 'Prices are not available on testnets.');
         return;
       }
@@ -394,7 +394,7 @@ class MiningRoutes {
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-      if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
+      if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
         handleError(req, res, 400, 'Acceleration data is not available.');
         return;
       }
@@ -409,7 +409,7 @@ class MiningRoutes {
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
-      if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
+      if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
         handleError(req, res, 400, 'Acceleration data is not available.');
         return;
       }
@@ -425,7 +425,7 @@ class MiningRoutes {
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-      if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
+      if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
         handleError(req, res, 400, 'Acceleration data is not available.');
         return;
       }
@@ -440,7 +440,7 @@ class MiningRoutes {
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-      if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
+      if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
         handleError(req, res, 400, 'Acceleration data is not available.');
         return;
       }
@@ -455,7 +455,7 @@ class MiningRoutes {
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-      if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
+      if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
         handleError(req, res, 400, 'Acceleration data is not available.');
         return;
       }

backend/src/api/mining/mining.ts

@@ -26,7 +26,7 @@ class Mining {
   private blocksPriceIndexingRunning = false;
   public lastHashrateIndexingDate: number | null = null;
   public lastWeeklyHashrateIndexingDate: number | null = null;
-  
+
   public reindexHashrateRequested = false;
   public reindexDifficultyAdjustmentRequested = false;
 
@@ -66,7 +66,7 @@ class Mining {
       {from, to}
     );
   }
-  
+
   /**
    * Get historical block rewards
    */
@@ -114,31 +114,29 @@ class Mining {
     const poolsStatistics = {};
 
     const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(interval);
-    const emptyBlocks: any[] = await BlocksRepository.$countEmptyBlocks(null, interval);
 
     const poolsStats: PoolStats[] = [];
     let rank = 1;
+    let blockCount = 0;
 
     poolsInfo.forEach((poolInfo: PoolInfo) => {
-      const emptyBlocksCount = emptyBlocks.filter((emptyCount) => emptyCount.poolId === poolInfo.poolId);
       const poolStat: PoolStats = {
         poolId: poolInfo.poolId, // mysql row id
         name: poolInfo.name,
         link: poolInfo.link,
         blockCount: poolInfo.blockCount,
         rank: rank++,
-        emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0,
+        emptyBlocks: poolInfo.emptyBlocks,
         slug: poolInfo.slug,
         avgMatchRate: poolInfo.avgMatchRate !== null ? Math.round(100 * poolInfo.avgMatchRate) / 100 : null,
         avgFeeDelta: poolInfo.avgFeeDelta,
         poolUniqueId: poolInfo.poolUniqueId
       };
       poolsStats.push(poolStat);
+      blockCount += poolInfo.blockCount;
     });
 
     poolsStatistics['pools'] = poolsStats;
-
-    const blockCount: number = await BlocksRepository.$blockCount(null, interval);
     poolsStatistics['blockCount'] = blockCount;
 
     const totalBlock24h: number = await BlocksRepository.$blockCount(null, '24h');
@@ -151,6 +149,8 @@ class Mining {
       poolsStatistics['lastEstimatedHashrate1w'] = await bitcoinClient.getNetworkHashPs(totalBlock1w);
     } catch (e) {
       poolsStatistics['lastEstimatedHashrate'] = 0;
+      poolsStatistics['lastEstimatedHashrate3d'] = 0;
+      poolsStatistics['lastEstimatedHashrate1w'] = 0;
       logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate', logger.tags.mining);
     }
 
@@ -175,8 +175,8 @@ class Mining {
     const blockCount1w: number = await BlocksRepository.$blockCount(pool.id, '1w');
     const totalBlock1w: number = await BlocksRepository.$blockCount(null, '1w');
 
-    const avgHealth = await BlocksRepository.$getAvgBlockHealthPerPoolId(pool.id);    
-    const totalReward = await BlocksRepository.$getTotalRewardForPoolId(pool.id);    
+    const avgHealth = await BlocksRepository.$getAvgBlockHealthPerPoolId(pool.id);
+    const totalReward = await BlocksRepository.$getTotalRewardForPoolId(pool.id);
 
     let currentEstimatedHashrate = 0;
     try {
@@ -218,7 +218,7 @@ class Mining {
     const now = new Date();
 
     // Run only if:
-    // * this.lastWeeklyHashrateIndexingDate is set to null (node backend restart, reorg)
+    // * this.lastWeeklyHashrateIndexingDate is set to null (node backend restart, reorg, or re-indexing was requested after mining pools update)
     // * we started a new week (around Monday midnight)
     const runIndexing = this.lastWeeklyHashrateIndexingDate === null ||
       now.getUTCDay() === 1 && this.lastWeeklyHashrateIndexingDate !== now.getUTCDate();
@@ -235,7 +235,7 @@ class Mining {
 
       const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
       const hashrates: any[] = [];
- 
+
       const lastMonday = new Date(now.setDate(now.getDate() - (now.getDay() + 6) % 7));
       const lastMondayMidnight = this.getDateMidnight(lastMonday);
       let toTimestamp = lastMondayMidnight.getTime();
@@ -326,6 +326,7 @@ class Mining {
 
   /**
    * Generate daily hashrate data
+   * @asyncUnsafe
    */
   public async $generateNetworkHashrateHistory(): Promise<void> {
     // If a re-index was requested, truncate first
@@ -333,6 +334,7 @@ class Mining {
       logger.notice(`hashrates will now be re-indexed`);
       await database.query(`TRUNCATE hashrates`);
       this.lastHashrateIndexingDate = 0;
+      this.lastWeeklyHashrateIndexingDate = null;
       this.reindexHashrateRequested = false;
     }
 
@@ -439,6 +441,7 @@ class Mining {
 
   /**
    * Index difficulty adjustments
+   * @asyncUnsafe
    */
   public async $indexDifficultyAdjustments(): Promise<void> {
     // If a re-index was requested, truncate first
@@ -528,6 +531,8 @@ class Mining {
 
   /**
    * Create a link between blocks and the latest price at when they were mined
+   *
+   * @asyncSafe
    */
   public async $indexBlockPrices(): Promise<void> {
     if (this.blocksPriceIndexingRunning === true) {
@@ -537,7 +542,7 @@ class Mining {
 
     let totalInserted = 0;
     try {
-      const prices: any[] = await PricesRepository.$getPricesTimesAndId();    
+      const prices: any[] = await PricesRepository.$getPricesTimesAndId();
       const blocksWithoutPrices: any[] = await BlocksRepository.$getBlocksWithoutPrice();
 
       const blocksPrices: BlockPrice[] = [];
@@ -598,6 +603,8 @@ class Mining {
 
   /**
    * Index core coinstatsindex
+   *
+   * @asyncUnsafe
    */
   public async $indexCoinStatsIndex(): Promise<void> {
     let timer = new Date().getTime() / 1000;
@@ -609,11 +616,11 @@ class Mining {
     while (currentBlockHeight > 0) {
       const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex(
         currentBlockHeight, currentBlockHeight - 10000);
-        
+
       for (const block of indexedBlocks) {
         const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
         await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts,
-          Math.round(txoutset.block_info.prevout_spent * 100000000));        
+          Math.round(txoutset.block_info.prevout_spent * 100000000));
         ++totalIndexed;
 
         const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
@@ -635,6 +642,7 @@ class Mining {
 
   /**
    * List existing mining pools
+   * @asyncUnsafe
    */
   public async $listPools(): Promise<{name: string, slug: string, unique_id: number}[] | null> {
     const [rows] = await database.query(`
@@ -688,7 +696,7 @@ class Mining {
       default: return 1 * scale;
     }
   }
-  
+
 
   // Finds the oldest block in a consecutive chain back from the tip
   // assumes `blocks` is sorted in ascending height order
@@ -701,6 +709,7 @@ class Mining {
     return blocks[0];
   }
 
+  /** @asyncUnsafe */
   private async getGenesisData(): Promise<{timestamp: number, bits: number, difficulty: number}> {
     if (this.genesisData == null) {
       const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0));

backend/src/api/pools-parser.ts

@@ -33,6 +33,7 @@ class PoolsParser {
   /**
    * Populate our db with updated mining pool definition
    * @param pools
+   * @asyncUnsafe
    */
   public async migratePoolsJson(): Promise<void> {
     // We also need to wipe the backend cache to make sure we don't serve blocks with
@@ -126,8 +127,8 @@ class PoolsParser {
         block.extras.pool = reindexedBlock.extras.pool;
       }
       // update persistent cache with the reindexed data
-      diskCache.$saveCacheToDisk();
-      redisCache.$updateBlocks(blocks.getBlocks());
+      void diskCache.$saveCacheToDisk();
+      void redisCache.$updateBlocks(blocks.getBlocks());
     }
   }
 
@@ -159,6 +160,7 @@ class PoolsParser {
 
   /**
    * Manually add the 'unknown pool'
+   * @asyncSafe
    */
   public async $insertUnknownPool(): Promise<void> {
     if (!config.DATABASE.ENABLED) {
@@ -190,12 +192,13 @@ class PoolsParser {
    * re-index pool assignment for blocks previously associated with pool
    *
    * @param pool local id of existing pool to reindex
+   * @asyncUnsafe
    */
   private async $reindexBlocksForPool(poolId: number): Promise<void> {
     let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52
     if (config.MEMPOOL.NETWORK === 'testnet') {
       firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581
-    } else if (['signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
+    } else if (['signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
       firstKnownBlockPool = 0;
     }
 

backend/src/api/rbf-cache.ts

@@ -1,10 +1,10 @@
-import config from "../config";
-import logger from "../logger";
-import { MempoolTransactionExtended, TransactionStripped } from "../mempool.interfaces";
+import config from '../config';
+import logger from '../logger';
+import { MempoolTransactionExtended, TransactionStripped } from '../mempool.interfaces';
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
-import { IEsploraApi } from "./bitcoin/esplora-api.interface";
-import { Common } from "./common";
-import redisCache from "./redis-cache";
+import { IEsploraApi } from './bitcoin/esplora-api.interface';
+import { Common } from './common';
+import redisCache from './redis-cache';
 
 export interface RbfTransaction extends TransactionStripped {
   rbf?: boolean;
@@ -407,6 +407,7 @@ class RbfCache {
     };
   }
 
+  /** @asyncSafe */
   public async load({ txs, trees, expiring, mempool, spendMap }): Promise<void> {
     try {
       txs.forEach(txEntry => {

backend/src/api/redis-cache.ts

@@ -23,11 +23,14 @@ class RedisCache {
 
   private pauseFlush: boolean = false;
   private cacheQueue: MempoolTransactionExtended[] = [];
-  private removeQueue: string[] = [];
+  private removeQueue = new Set<string>();
+  private removeQueueFlushInProgress: boolean = false;
   private rbfCacheQueue: { type: string, txid: string, value: any }[] = [];
   private rbfRemoveQueue: { type: string, txid: string }[] = [];
   private txFlushLimit: number = 10000;
   private ignoreBlocksCache = false;
+  private reconciliationInProgress: boolean = false;
+  private reconciliationCursor: string = '0';
 
   constructor() {
     if (config.REDIS.ENABLED) {
@@ -37,11 +40,13 @@ class RedisCache {
         },
         database: NetworkDB[config.MEMPOOL.NETWORK],
       };
-      this.$ensureConnected();
-      setInterval(() => { this.$ensureConnected(); }, 10000);
+      void this.$ensureConnected();
+      setInterval(() => { void this.$ensureConnected(); }, 10000);
+      setInterval(() => { void this.$reconcileMempoolTransactions(); }, 30000);
     }
   }
 
+  /** @asyncSafe */
   private async $ensureConnected(): Promise<boolean> {
     if (!this.connected && config.REDIS.ENABLED) {
       try {
@@ -91,10 +96,11 @@ class RedisCache {
 
   private async $onConnected(): Promise<void> {
     await this.$flushTransactions();
-    await this.$removeTransactions([]);
+    await this.$removeTransactions();
     await this.$flushRbfQueues();
   }
 
+  /** @asyncSafe */
   async $updateBlocks(blocks: BlockExtended[]): Promise<void> {
     if (!config.REDIS.ENABLED) {
       return;
@@ -127,10 +133,12 @@ class RedisCache {
     }
   }
 
+  /** @asyncSafe */
   async $addTransaction(tx: MempoolTransactionExtended): Promise<void> {
     if (!config.REDIS.ENABLED) {
       return;
     }
+    this.removeQueue.delete(tx.txid);
     this.cacheQueue.push(tx);
     if (this.cacheQueue.length >= this.txFlushLimit) {
       if (!this.pauseFlush) {
@@ -139,6 +147,7 @@ class RedisCache {
     }
   }
 
+  /** @asyncSafe */
   async $flushTransactions(): Promise<void> {
     if (!config.REDIS.ENABLED) {
       return;
@@ -178,34 +187,92 @@ class RedisCache {
     }
   }
 
-  async $removeTransactions(transactions: string[]): Promise<void> {
+  queueTransactionsForRemoval(transactions: string[]): void {
     if (!config.REDIS.ENABLED) {
       return;
     }
-    const toRemove = this.removeQueue.concat(transactions);
-    this.removeQueue = [];
-    let failed: string[] = [];
-    let numRemoved = 0;
-    if (this.connected) {
+
+    for (const txid of transactions) {
+      this.removeQueue.add(txid);
+    }
+  }
+
+  /** @asyncSafe */
+  async $removeTransactions(transactions: string[] = []): Promise<void> {
+    if (!config.REDIS.ENABLED) {
+      return;
+    }
+    for (const txid of transactions) {
+      this.removeQueue.add(txid);
+    }
+    await this.$flushQueuedMempoolTxRemovals();
+  }
+
+  // incrementally reconcile the redis cache with the in-memory mempool
+  // by scanning for cached txs no longer in the mempool and marking for deletion
+  // each invocation scans 1000 keys, cursor loops back to the start after completing a full scan
+  /** @asyncSafe */
+  private async $reconcileMempoolTransactions(): Promise<void> {
+    if (!config.REDIS.ENABLED || !this.connected || this.reconciliationInProgress || !memPool.isInSync()) {
+      return;
+    }
+
+    this.reconciliationInProgress = true;
+    try {
+      const result = await this.client.scan(this.reconciliationCursor, {
+        MATCH: 'mempool:tx:*',
+        COUNT: 1000
+      });
+      const mempool = memPool.getMempool();
+      let staleCount = 0;
+      this.reconciliationCursor = result.cursor.toString();
+
+      for (const key of result.keys) {
+        const txid = key.slice('mempool:tx:'.length);
+        if (!mempool[txid]) {
+          this.removeQueue.add(txid);
+          staleCount++;
+        }
+      }
+
+      if (staleCount) {
+        logger.debug(`Removing ${staleCount} stale transactions from the redis cache`);
+        void this.$removeTransactions();
+      }
+    } catch (e) {
+      logger.warn(`Failed to reconcile Redis mempool cache: ${e instanceof Error ? e.message : e}`);
+    } finally {
+      this.reconciliationInProgress = false;
+    }
+  }
+
+  /** @asyncSafe */
+  private async $flushQueuedMempoolTxRemovals(): Promise<void> {
+    if (!this.connected || !this.removeQueue.size || this.removeQueueFlushInProgress) {
+      return;
+    }
+
+    this.removeQueueFlushInProgress = true;
+    const toRemove = Array.from(this.removeQueue);
+    this.removeQueue.clear();
+    try {
       const sliceLength = config.REDIS.BATCH_QUERY_BASE_SIZE;
       for (let i = 0; i < Math.ceil(toRemove.length / sliceLength); i++) {
         const slice = toRemove.slice(i * sliceLength, (i + 1) * sliceLength);
         try {
           await this.client.unlink(slice.map(txid => `mempool:tx:${txid}`));
-          numRemoved+= sliceLength;
           logger.debug(`Deleted ${slice.length} transactions from the Redis cache`);
         } catch (e) {
           logger.warn(`Failed to remove ${slice.length} transactions from Redis cache: ${e instanceof Error ? e.message : e}`);
-          failed = failed.concat(slice);
+          this.queueTransactionsForRemoval(slice);
         }
       }
-      // concat instead of replace, in case more txs have been added in the meantime
-      this.removeQueue = this.removeQueue.concat(failed);
-    } else {
-      this.removeQueue = this.removeQueue.concat(toRemove);
+    } finally {
+      this.removeQueueFlushInProgress = false;
     }
   }
 
+  /** @asyncSafe */
   async $setRbfEntry(type: string, txid: string, value: any): Promise<void> {
     if (!config.REDIS.ENABLED) {
       return;
@@ -222,6 +289,7 @@ class RedisCache {
     }
   }
 
+  /** @asyncSafe */
   async $removeRbfEntry(type: string, txid: string): Promise<void> {
     if (!config.REDIS.ENABLED) {
       return;
@@ -238,6 +306,7 @@ class RedisCache {
     }
   }
 
+  /** @asyncSafe */
   private async $flushRbfQueues(): Promise<void> {
     if (!config.REDIS.ENABLED) {
       return;
@@ -263,6 +332,7 @@ class RedisCache {
     }
   }
 
+  /** @asyncSafe */
   async $getBlocks(): Promise<BlockExtended[]> {
     if (!config.REDIS.ENABLED) {
       return [];
@@ -280,6 +350,7 @@ class RedisCache {
     }
   }
 
+  /** @asyncSafe */
   async $getBlockSummaries(): Promise<BlockSummary[]> {
     if (!config.REDIS.ENABLED) {
       return [];
@@ -297,7 +368,8 @@ class RedisCache {
     }
   }
 
-  async $getMempool(): Promise<{ [txid: string]: MempoolTransactionExtended }> {
+  /** @asyncSafe */
+  async $getMempool(validTxids?: Set<string>): Promise<{ [txid: string]: MempoolTransactionExtended }> {
     if (!config.REDIS.ENABLED) {
       return {};
     }
@@ -308,7 +380,9 @@ class RedisCache {
     const start = Date.now();
     const mempool = {};
     try {
-      const mempoolList = await this.scanKeys<MempoolTransactionExtended>('mempool:tx:*');
+      const mempoolList = validTxids?.size
+        ? await this.loadKeys<MempoolTransactionExtended>('mempool:tx:*', Array.from(validTxids))
+        : await this.scanKeys<MempoolTransactionExtended>('mempool:tx:*');
       for (const tx of mempoolList) {
         mempool[tx.key] = tx.value;
       }
@@ -320,6 +394,7 @@ class RedisCache {
     return {};
   }
 
+  /** @asyncSafe */
   async $getRbfEntries(type: string): Promise<any[]> {
     if (!config.REDIS.ENABLED) {
       return [];
@@ -337,14 +412,15 @@ class RedisCache {
     }
   }
 
-  async $loadCache(): Promise<void> {
+  /** @asyncUnsafe */
+  async $loadCache(validTxids?: Set<string>): Promise<void> {
     if (!config.REDIS.ENABLED) {
       return;
     }
     logger.info('Restoring mempool and blocks data from Redis cache');
 
     // Load mempool
-    const loadedMempool = await this.$getMempool();
+    const loadedMempool = await this.$getMempool(validTxids);
     this.inflateLoadedTxs(loadedMempool);
     // Load rbf data
     const rbfTxs = await this.$getRbfEntries('tx');
@@ -385,12 +461,14 @@ class RedisCache {
     }
   }
 
+  /** @asyncUnsafe */
   private async scanKeys<T>(pattern): Promise<{ key: string, value: T }[]> {
     logger.info(`loading Redis entries for ${pattern}`);
     let keys: string[] = [];
     const result: { key: string, value: T }[] = [];
     const patternLength = pattern.length - 1;
     let count = 0;
+    /** @asyncUnsafe */
     const processValues = async (keys): Promise<void> => {
       const values = await this.client.MGET(keys);
       for (let i = 0; i < values.length; i++) {
@@ -417,6 +495,29 @@ class RedisCache {
     return result;
   }
 
+  /** @asyncUnsafe */
+  private async loadKeys<T>(pattern, keys: string[]): Promise<{ key: string, value: T }[]> {
+    const prefix = pattern.slice(0, -1);
+    const result: { key: string, value: T }[] = [];
+    let count = 0;
+    /** @asyncUnsafe */
+    const processValues = async (slice: string[]): Promise<void> => {
+      const values = await this.client.MGET(slice.map(key => `${prefix}${key}`));
+      for (let i = 0; i < values.length; i++) {
+        if (values[i]) {
+          result.push({ key: slice[i], value: JSON.parse(values[i]) });
+          count++;
+        }
+      }
+      logger.info(`loaded ${count} entries from Redis cache`);
+    };
+    for (let i = 0; i < Math.ceil(keys.length / 10000); i++) {
+      const slice = keys.slice(i * 10000, (i + 1) * 10000);
+      await processValues(slice);
+    }
+    return result;
+  }
+
   public setIgnoreBlocksCache(): void {
     this.ignoreBlocksCache = true;
   }

backend/src/api/services/acceleration.ts

@@ -5,6 +5,7 @@ import { BlockExtended } from '../../mempool.interfaces';
 import axios from 'axios';
 import mempool from '../mempool';
 import websocketHandler from '../websocket-handler';
+import { Common } from '../common';
 
 type MyAccelerationStatus = 'requested' | 'accelerating' | 'done';
 
@@ -74,6 +75,7 @@ class AccelerationApi {
     this.forcePoll = true;
   }
 
+  /** @asyncSafe */
   private async $fetchAccelerations(): Promise<Acceleration[] | null> {
     try {
       const response = await axios.get(this.apiPath, { responseType: 'json', timeout: 10000 });
@@ -238,6 +240,7 @@ class AccelerationApi {
     }
   }
 
+  /** @asyncSafe */
   public async connectWebsocket(): Promise<void> {
     if (this.startedWebsocketLoop) {
       return;
@@ -314,7 +317,7 @@ class AccelerationApi {
           }
         }
       }
-      await new Promise(resolve => setTimeout(resolve, 5000));
+      await Common.sleep$(5000);
     }
   }
 }

backend/src/api/services/stratum.ts

@@ -2,6 +2,7 @@ import { WebSocket } from 'ws';
 import logger from '../../logger';
 import config from '../../config';
 import websocketHandler from '../websocket-handler';
+import { Common } from '../common';
 
 export interface StratumJob {
   pool: number;
@@ -58,6 +59,7 @@ class StratumApi {
     }
   }
 
+  /** @asyncSafe */
   public async connectWebsocket(): Promise<void> {
     if (!config.STRATUM.ENABLED) {
       return;
@@ -97,7 +99,7 @@ class StratumApi {
           }
         });
       }
-      await new Promise(resolve => setTimeout(resolve, 5000));
+      await Common.sleep$(5000);
     }
   }
 }

backend/src/api/services/wallets.ts

@@ -8,7 +8,6 @@ import { promises as fsPromises } from 'fs';
 
 interface WalletAddress {
   address: string;
-  active: boolean;
   stats: {
     funded_txo_count: number;
     funded_txo_sum: number;
@@ -56,10 +55,11 @@ class WalletApi {
 
     // Load cache on startup
     if (config.WALLETS.ENABLED) {
-      this.$loadCache();
+      void this.$loadCache();
     }
   }
 
+  /** @asyncSafe */
   private async $loadCache(): Promise<void> {
     try {
       const cacheData = await fsPromises.readFile(WalletApi.FILE_NAME, 'utf8');
@@ -148,6 +148,7 @@ class WalletApi {
   }
 
   // resync wallet addresses from the services backend
+  /** @asyncSafe */
   async $syncWallets(): Promise<void> {
     if (!config.WALLETS.ENABLED || this.syncing) {
       return;
@@ -242,15 +243,18 @@ class WalletApi {
 
   // resync address transactions from esplora
   async $syncWalletAddress(wallet: Wallet, address: WalletAddress): Promise<void> {
-    // fetch full transaction data if the address is new or still active and hasn't been synced in the last hour
-    const refreshTransactions = !wallet.addresses[address.address] || (address.active && (Date.now() - wallet.addresses[address.address].lastSync) > 60 * 60 * 1000);
+    if (address.address === 'private') {
+      // skip pseudo-address for private balances
+      return;
+    }
+    // fetch full transaction data if the address is new or hasn't been synced in the last hour
+    const refreshTransactions = !wallet.addresses[address.address] || (Date.now() - wallet.addresses[address.address].lastSync) > 60 * 60 * 1000;
     if (refreshTransactions) {
       try {
         const summary = await bitcoinApi.$getAddressTransactionSummary(address.address);
         const addressInfo = await bitcoinApi.$getAddress(address.address);
         const walletAddress: WalletAddress = {
           address: address.address,
-          active: address.active,
           transactions: summary,
           stats: addressInfo.chain_stats,
           lastSync: Date.now(),
@@ -320,7 +324,6 @@ function convertBalancesToWalletAddress(wallet: string, balances: { balance: num
   const sortedBalances = balances.sort((a, b) => a.time - b.time);
   const walletAddress: WalletAddress = {
     address: 'private',
-    active: false,
     stats: {
       funded_txo_count: 0,
       funded_txo_sum: sortedBalances[sortedBalances.length - 1].balance,

backend/src/api/statistics/statistics-api.ts

@@ -514,7 +514,7 @@ class StatisticsApi {
         vsize_1600: completeVsizes[36],
         vsize_1800: completeVsizes[37],
         vsize_2000: completeVsizes[38],
-      }
+      };
     });
   }
 }

backend/src/api/statistics/statistics.ts

@@ -22,13 +22,14 @@ class Statistics {
     const difference = nextInterval.getTime() - now.getTime();
 
     setTimeout(() => {
-      this.runStatistics();
+      void this.runStatistics();
       this.intervalTimer = setInterval(() => {
-        this.runStatistics(true);
+        void this.runStatistics(true);
       }, 1 * 60 * 1000);
     }, difference);
   }
 
+  /** @asyncSafe */
   public async runStatistics(skipIfRecent = false): Promise<void> {
     if (!memPool.isInSync()) {
       return;

backend/src/api/transaction-utils.ts

@@ -47,6 +47,7 @@ class TransactionUtils {
    * @param addPrevouts
    * @param lazyPrevouts
    * @param forceCore - See https://github.com/mempool/mempool/issues/2904
+   * @asyncUnsafe
    */
   public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false, forceCore = false, addMempoolData = false): Promise<TransactionExtended> {
     let transaction: IEsploraApi.Transaction;
@@ -69,10 +70,12 @@ class TransactionUtils {
     }
   }
 
+  /** @asyncUnsafe */
   public async $getMempoolTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise<MempoolTransactionExtended> {
     return (await this.$getTransactionExtended(txId, addPrevouts, lazyPrevouts, forceCore, true)) as MempoolTransactionExtended;
   }
 
+  /** @asyncUnsafe */
   public async $getMempoolTransactionsExtended(txids: string[], addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise<MempoolTransactionExtended[]> {
     if (forceCore || config.MEMPOOL.BACKEND !== 'esplora') {
       const limiter = pLimit(8); // Run 8 requests at a time
@@ -116,7 +119,7 @@ class TransactionUtils {
   public extendMempoolTransaction(transaction: IEsploraApi.Transaction): MempoolTransactionExtended {
     const vsize = Math.ceil(transaction.weight / 4);
     const fractionalVsize = (transaction.weight / 4);
-    let sigops = Common.isLiquid() ? 0 : (transaction.sigops != null ? transaction.sigops : this.countSigops(transaction));
+    const sigops = Common.isLiquid() ? 0 : (transaction.sigops != null ? transaction.sigops : this.countSigops(transaction));
     // https://github.com/bitcoin/bitcoin/blob/e9262ea32a6e1d364fb7974844fadc36f931f8c6/src/policy/policy.cpp#L295-L298
     const adjustedVsize = Math.max(fractionalVsize, sigops *  5); // adjusted vsize = Max(weight, sigops * bytes_per_sigop) / witness_scale_factor
     const feePerVbytes = (transaction.fee || 0) / fractionalVsize;
@@ -253,13 +256,14 @@ class TransactionUtils {
 
   // returns the most significant 4 bytes of the txid as an integer
   public txidToOrdering(txid: string): number {
-    return parseInt(
-      txid.substr(62, 2) +
-        txid.substr(60, 2) +
-        txid.substr(58, 2) +
-        txid.substr(56, 2),
-      16
-    );
+    // Parse last 4 bytes of txid as little-endian uint32, without string allocation
+    let result = 0;
+    for (let i = 62; i >= 56; i -= 2) {
+      const hi = txid.charCodeAt(i);
+      const lo = txid.charCodeAt(i + 1);
+      result = result * 256 + (hi < 58 ? hi - 48 : hi - 87) * 16 + (lo < 58 ? lo - 48 : lo - 87);
+    }
+    return result;
   }
 
   public addInnerScriptsToVin(vin: IEsploraApi.Vin): void {
@@ -365,7 +369,7 @@ class TransactionUtils {
    *          the script item if it is a script spend.
    */
   public witnessToP2TRScript(witness: string[]): string | null {
-    if (witness.length < 2) return null;
+    if (witness.length < 2) {return null;}
     // Note: see BIP341 for parsing details of witness stack
 
     // If there are at least two witness elements, and the first byte of the
@@ -375,7 +379,7 @@ class TransactionUtils {
     // If there are at least two witness elements left, script path spending is used.
     // Call the second-to-last stack element s, the script.
     // (Note: this phrasing from BIP341 assumes we've *removed* the annex from the stack)
-    if (hasAnnex && witness.length < 3) return null;
+    if (hasAnnex && witness.length < 3) {return null;}
     const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2;
     return witness[positionOfScript];
   }
@@ -482,7 +486,7 @@ class TransactionUtils {
       return 'unknown';
     }
   }
-  
+
 }
 
 export default new TransactionUtils();

backend/src/api/tx-selection-worker.ts

@@ -18,7 +18,7 @@ if (parentPort) {
         mempool.delete(uid);
       });
     }
-    
+
     const { blocks, rates, clusters } = makeBlockTemplates(mempool);
 
     // return the result to main thread.
@@ -38,7 +38,7 @@ function makeBlockTemplates(mempool: Map<number, CompactThreadTransaction>)
   const auditPool: Map<number, AuditTransaction> = new Map();
   const mempoolArray: AuditTransaction[] = [];
   const cpfpClusters: Map<number, number[]> = new Map();
-  
+
   mempool.forEach(tx => {
     tx.dirty = false;
     // initializing everything up front helps V8 optimize property access later
@@ -85,7 +85,7 @@ function makeBlockTemplates(mempool: Map<number, CompactThreadTransaction>)
   // (i.e. the package rooted in the transaction with the best ancestor score)
   const blocks: number[][] = [];
   let blockWeight = 4000;
-  let blockSigops = 0;
+  const blockSigops = 0;
   let transactions: AuditTransaction[] = [];
   const modified: PairingHeap<AuditTransaction> = new PairingHeap((a, b): boolean => {
     if (a.score === b.score) {

backend/src/api/websocket-handler.ts

@@ -3,7 +3,7 @@ import * as WebSocket from 'ws';
 import {
   BlockExtended, TransactionExtended, MempoolTransactionExtended, WebsocketResponse,
   OptimizedStatistic, ILoadingIndicators, GbtCandidates, TxTrackingInfo,
-  MempoolDelta, MempoolDeltaTxids
+  MempoolDelta, MempoolDeltaTxids, CpfpInfo
 } from '../mempool.interfaces';
 import blocks from './blocks';
 import memPool from './mempool';
@@ -16,17 +16,12 @@ import transactionUtils from './transaction-utils';
 import rbfCache, { ReplacementInfo } from './rbf-cache';
 import difficultyAdjustment from './difficulty-adjustment';
 import feeApi from './fee-api';
-import BlocksRepository from '../repositories/BlocksRepository';
-import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
-import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
-import Audit from './audit';
 import priceUpdater from '../tasks/price-updater';
 import { ApiPrice } from '../repositories/PricesRepository';
 import { Acceleration } from './services/acceleration';
 import accelerationApi from './services/acceleration';
 import mempool from './mempool';
 import statistics from './statistics/statistics';
-import accelerationRepository from '../repositories/AccelerationRepository';
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import walletApi from './services/wallets';
 
@@ -37,7 +32,6 @@ interface AddressTransactions {
 }
 import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
 import { calculateMempoolTxCpfp } from './cpfp';
-import { getRecentFirstSeen } from '../utils/file-read';
 import stratumApi, { StratumJob } from './services/stratum';
 
 // valid 'want' subscriptions
@@ -64,6 +58,12 @@ class WebsocketHandler {
 
   private accelerations: Record<string, Acceleration> = {};
 
+  private MAX_BUFFERED_AMOUNT = 10_000_000;
+  public MAX_MESSAGE_SIZE = 50_000;
+  private MAX_TRACKED_TXS = 100;
+  private MSG_RATE_LIMIT = 100;
+  private MSG_RATE_WINDOW = 10_000;
+
   constructor() { }
 
   addWebsocketServer(wss: WebSocket.Server) {
@@ -120,6 +120,7 @@ class WebsocketHandler {
     server.on('connection', (client: WebSocket, req) => {
       this.numConnected++;
       client['remoteAddress'] = req.headers['x-forwarded-for'] || req.socket?.remoteAddress || 'unknown';
+      client['msgTimestamps'] = [];
       client.on('error', (e) => {
         logger.info(`websocket client error from ${client['remoteAddress']}: ` + (e instanceof Error ? e.message : e));
         client.close();
@@ -127,9 +128,30 @@ class WebsocketHandler {
       client.on('close', () => {
         this.numDisconnected++;
       });
-      client.on('message', async (message: string) => {
+      client.on('message', async (message) => {
         try {
-          const parsedMessage: WebsocketResponse = JSON.parse(message);
+          const msgLength = Buffer.isBuffer(message) ? message.byteLength
+            : message instanceof ArrayBuffer ? message.byteLength
+            : message.reduce((sum, buf) => sum + buf.byteLength, 0);
+          if (msgLength > this.MAX_MESSAGE_SIZE) {
+            logger.debug(`Dropping oversized websocket message from ${client['remoteAddress']}: ${msgLength} bytes`);
+            client.terminate();
+            return;
+          }
+
+          const now = Date.now();
+          const timestamps: number[] = client['msgTimestamps'];
+          timestamps.push(now);
+          while (timestamps.length && timestamps[0] <= now - this.MSG_RATE_WINDOW) {
+            timestamps.shift();
+          }
+          if (timestamps.length > this.MSG_RATE_LIMIT) {
+            logger.debug(`Rate limiting websocket client ${client['remoteAddress']}`);
+            client.close();
+            return;
+          }
+
+          const parsedMessage: WebsocketResponse = JSON.parse(message as any);
           const response = {};
 
           const wantNow = {};
@@ -225,45 +247,60 @@ class WebsocketHandler {
           if (parsedMessage && parsedMessage['track-txs']) {
             const txids: string[] = [];
             if (Array.isArray(parsedMessage['track-txs'])) {
+              if (parsedMessage['track-txs'].length > this.MAX_TRACKED_TXS) {
+                response['track-txs-error'] = `"too many txids requested, this connection supports tracking a maximum of ${this.MAX_TRACKED_TXS} transactions"`;
+                this.send(client, this.serializeResponse(response));
+                client['track-txs'] = null;
+                client.close();
+                return;
+              }
               for (const txid of parsedMessage['track-txs']) {
                 if (/^[a-fA-F0-9]{64}$/.test(txid)) {
                   txids.push(txid);
                 }
               }
+            } else {
+              response['track-txs-error'] = `"incorrect track-txs format"`;
+              this.send(client, this.serializeResponse(response));
+              client['track-txs'] = null;
+              client.close();
+              return;
             }
 
             const txs: { [txid: string]: TxTrackingInfo } = {};
             for (const txid of txids) {
-              const txInfo: TxTrackingInfo = {
-                confirmed: true,
-              };
+              const txInfo: TxTrackingInfo = {};
               const rbfCacheTxid = rbfCache.getReplacedBy(txid);
               if (rbfCacheTxid) {
                 txInfo.replacedBy = rbfCacheTxid;
                 txInfo.confirmed = false;
+                txs[txid] = txInfo;
               }
               const tx = memPool.getMempool()[txid];
-              if (tx && tx.position) {
-                txInfo.position = {
-                  ...tx.position
-                };
-                if (tx.acceleration) {
-                  txInfo.accelerated = tx.acceleration;
-                }
-              }
               if (tx) {
+                if (tx.position) {
+                  txInfo.position = {
+                    ...tx.position
+                  };
+                  if (tx.acceleration) {
+                    txInfo.accelerated = tx.acceleration;
+                  }
+                }
                 txInfo.confirmed = false;
+                txs[txid] = txInfo;
               }
-              txs[txid] = txInfo;
             }
 
             if (txids.length) {
               client['track-txs'] = txids;
+              client['track-txs-updates'] = 0;
             } else {
               client['track-txs'] = null;
+              client['track-txs-updates'] = 0;
             }
 
             if (Object.keys(txs).length) {
+              client['track-txs-updates'] = (client['track-txs-updates'] || 0) + Object.keys(txs).length;
               response['tracked-txs'] = JSON.stringify(txs);
             }
           }
@@ -288,10 +325,13 @@ class WebsocketHandler {
             if (Object.keys(addressMap).length > config.MEMPOOL.MAX_TRACKED_ADDRESSES) {
               response['track-addresses-error'] = `"too many addresses requested, this connection supports tracking a maximum of ${config.MEMPOOL.MAX_TRACKED_ADDRESSES} addresses"`;
               client['track-addresses'] = null;
+              client['track-addresses-updates'] = 0;
             } else if (Object.keys(addressMap).length > 0) {
               client['track-addresses'] = addressMap;
+              client['track-addresses-updates'] = 0;
             } else {
               client['track-addresses'] = null;
+              client['track-addresses-updates'] = 0;
             }
           }
 
@@ -315,8 +355,10 @@ class WebsocketHandler {
           if (parsedMessage && parsedMessage['track-wallet']) {
             if (parsedMessage['track-wallet'] === 'stop') {
               client['track-wallet'] = null;
-            } else {
+            } else if (typeof parsedMessage['track-wallet'] === 'string' && walletApi.getWallets().includes(parsedMessage['track-wallet'])) {
               client['track-wallet'] = parsedMessage['track-wallet'];
+            } else {
+              client['track-wallet'] = null;
             }
           }
 
@@ -329,8 +371,8 @@ class WebsocketHandler {
           }
 
           if (parsedMessage && parsedMessage['track-mempool-block'] !== undefined) {
-            if (Number.isInteger(parsedMessage['track-mempool-block']) && parsedMessage['track-mempool-block'] >= 0) {
-              const index = parsedMessage['track-mempool-block'];
+            const index = parsedMessage['track-mempool-block'];
+            if (Number.isInteger(index) && index >= 0 && index < config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT) {
               client['track-mempool-block'] = index;
               const mBlocksWithTransactions = mempoolBlocks.getMempoolBlocksWithTransactions();
               response['projected-block-transactions'] = JSON.stringify({
@@ -381,14 +423,14 @@ class WebsocketHandler {
             if (!this.socketData['blocks']?.length) {
               return;
             }
-            client.send(this.serializedInitData);
+            this.send(client, this.serializedInitData);
           }
 
           if (parsedMessage.action === 'ping') {
             response['pong'] = JSON.stringify(true);
           }
 
-          if (parsedMessage['track-donation'] && parsedMessage['track-donation'].length === 22) {
+          if (typeof parsedMessage['track-donation'] === 'string' && parsedMessage['track-donation'].length === 22) {
             client['track-donation'] = parsedMessage['track-donation'];
           }
 
@@ -404,18 +446,18 @@ class WebsocketHandler {
             delete client['track-mempool'];
           }
 
-          if (parsedMessage && parsedMessage['track-stratum'] != null) {
-            if (parsedMessage['track-stratum']) {
+          if (parsedMessage && parsedMessage['track-stratum'] !== undefined) {
+            if (parsedMessage['track-stratum'] === 'all' || typeof parsedMessage['track-stratum'] === 'number') {
               const sub = parsedMessage['track-stratum'];
               client['track-stratum'] = sub;
-              response['stratumJobs'] = this.socketData['stratumJobs'];
+              response['stratumJobs'] = JSON.stringify(stratumApi.getJobs());
             } else {
               client['track-stratum'] = false;
             }
           }
 
           if (Object.keys(response).length) {
-            client.send(this.serializeResponse(response));
+            this.send(client, this.serializeResponse(response));
           }
         } catch (e) {
           logger.debug(`Error parsing websocket message from ${client['remoteAddress']}: ` + (e instanceof Error ? e.message : e));
@@ -438,7 +480,7 @@ class WebsocketHandler {
         return;
       }
       if (client['track-donation'] === id) {
-        client.send(JSON.stringify({ donationConfirmed: true }));
+        this.send(client, JSON.stringify({ donationConfirmed: true }));
       }
     });
     }
@@ -458,7 +500,7 @@ class WebsocketHandler {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
-      client.send(response);
+      this.send(client, response);
     });
     }
   }
@@ -477,7 +519,7 @@ class WebsocketHandler {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
-      client.send(response);
+      this.send(client, response);
     });
     }
   }
@@ -504,7 +546,7 @@ class WebsocketHandler {
         return;
       }
 
-      client.send(response);
+      this.send(client, response);
     });
     }
   }
@@ -537,7 +579,7 @@ class WebsocketHandler {
           if (client.readyState !== WebSocket.OPEN) {
             return;
           }
-          client.send(response);
+          this.send(client, response);
         });
       }
     } catch (e) {
@@ -575,7 +617,7 @@ class WebsocketHandler {
       }
 
       if (Object.keys(response).length) {
-        client.send(this.serializeResponse(response));
+        this.send(client, this.serializeResponse(response));
       }
     });
     }
@@ -609,7 +651,10 @@ class WebsocketHandler {
       removed = candidates?.removed || [];
     }
 
-    if (config.MEMPOOL.RUST_GBT) {
+    if (config.MEMPOOL.CLUSTER_MEMPOOL) {
+      const cmBlocks = mempool.clusterMempool?.getBlocks(config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT) ?? [];
+      mempoolBlocks.processClusterMempoolBlocks(cmBlocks, newMempool, mempool.getAccelerations());
+    } else if (config.MEMPOOL.RUST_GBT) {
       await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, newMempool, added, removed, candidates, true);
     } else {
       await mempoolBlocks.$updateBlockTemplates(transactionIds, newMempool, added, removed, candidates, accelerationDelta, true, true);
@@ -738,6 +783,8 @@ class WebsocketHandler {
       removed: websocketAccelerationDelta.filter(txid => !accelerations[txid]),
     };
 
+    const cpfpUpdatesSent = new Set<string>();
+
     // TODO - Fix indentation after PR is merged
     for (const server of this.webSocketServers) {
     server.clients.forEach(async (client) => {
@@ -815,6 +862,8 @@ class WebsocketHandler {
         }
 
         if (Object.keys(addressMap).length > 0) {
+          client['track-addresses-updates'] =
+            (client['track-addresses-updates'] || 0) + this.countAddressTransactions(addressMap);
           response['multi-address-transactions'] = JSON.stringify(addressMap);
         }
       }
@@ -906,15 +955,23 @@ class WebsocketHandler {
             calculateMempoolTxCpfp(mempoolTx, newMempool);
           }
           if (mempoolTx.cpfpDirty) {
-            positionData['cpfp'] = {
-              ancestors: mempoolTx.ancestors,
+            const cpfp: CpfpInfo = {
+              ancestors: mempoolTx.ancestors || [],
               bestDescendant: mempoolTx.bestDescendant || null,
-              descendants: mempoolTx.descendants || null,
-              effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize || null,
+              descendants: mempoolTx.descendants,
+              effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize,
               sigops: mempoolTx.sigops,
               adjustedVsize: mempoolTx.adjustedVsize,
               acceleration: mempoolTx.acceleration,
             };
+            if (config.MEMPOOL.CLUSTER_MEMPOOL && mempoolTx.clusterId != null) {
+              const cluster = mempool.clusterMempool?.getClusterForApi(mempoolTx.txid);
+              if (cluster) {
+                cpfp.cluster = cluster;
+              }
+            }
+            positionData['cpfp'] = cpfp;
+            cpfpUpdatesSent.add(trackTxid);
           }
           response['txPosition'] = JSON.stringify(positionData);
         }
@@ -925,13 +982,16 @@ class WebsocketHandler {
         const txs: { [txid: string]: TxTrackingInfo } = {};
         for (const txid of txids) {
           const txInfo: TxTrackingInfo = {};
+          let txHasInfo = false;
           const outspends = outspendCache[txid];
           if (outspends && Object.keys(outspends).length) {
             txInfo.utxoSpent = outspends;
+            txHasInfo = true;
           }
           const replacedBy = rbfChanges.map[txid] ? rbfCache.getReplacedBy(txid) : false;
           if (replacedBy) {
             txInfo.replacedBy = replacedBy;
+            txHasInfo = true;
           }
           const mempoolTx = newMempool[txid];
           if (mempoolTx && mempoolTx.position) {
@@ -949,16 +1009,27 @@ class WebsocketHandler {
               txInfo.cpfp = {
                 ancestors: mempoolTx.ancestors,
                 bestDescendant: mempoolTx.bestDescendant || null,
-                descendants: mempoolTx.descendants || null,
-                effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize || null,
+                descendants: mempoolTx.descendants,
+                effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize,
                 sigops: mempoolTx.sigops,
                 adjustedVsize: mempoolTx.adjustedVsize,
               };
+              if (config.MEMPOOL.CLUSTER_MEMPOOL && mempoolTx.clusterId != null) {
+                const cluster = mempool.clusterMempool?.getClusterForApi(mempoolTx.txid);
+                if (cluster) {
+                  (txInfo.cpfp as CpfpInfo).cluster = cluster;
+                }
+              }
+              cpfpUpdatesSent.add(txid);
             }
+            txHasInfo = true;
+          }
+          if (txHasInfo) {
+            txs[txid] = txInfo;
           }
-          txs[txid] = txInfo;
         }
         if (Object.keys(txs).length) {
+          client['track-txs-updates'] = (client['track-txs-updates'] || 0) + Object.keys(txs).length;
           response['tracked-txs'] = JSON.stringify(txs);
         }
       }
@@ -997,142 +1068,37 @@ class WebsocketHandler {
       }
 
       if (Object.keys(response).length) {
-        client.send(this.serializeResponse(response));
+        this.send(client, this.serializeResponse(response));
       }
     });
     }
+
+    for (const txid of cpfpUpdatesSent) {
+      if (newMempool[txid]) {
+        newMempool[txid].cpfpDirty = false;
+      }
+    }
   }
- 
-  async handleNewBlock(block: BlockExtended, txIds: string[], transactions: MempoolTransactionExtended[]): Promise<void> {
+
+  /** @asyncSafe */
+  async handleNewBlock(
+    block: BlockExtended,
+    txIds: string[],
+    transactions: MempoolTransactionExtended[],
+    rbfTransactions: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }}
+  ): Promise<void> {
     if (!this.webSocketServers.length) {
       throw new Error('No WebSocket.Server have been set');
     }
 
-    const blockTransactions = structuredClone(transactions);
-
     this.printLogs();
-    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
-      await statistics.runStatistics();
-    }
 
     const _memPool = memPool.getMempool();
-    const candidateTxs = memPool.getMempoolCandidates();
-    let candidates: GbtCandidates | undefined = (memPool.limitGBT && candidateTxs) ? { txs: candidateTxs, added: [], removed: [] } : undefined;
-    let transactionIds: string[] = (memPool.limitGBT) ? Object.keys(candidates?.txs || {}) : Object.keys(_memPool);
-
-    if (config.DATABASE.ENABLED) {
-      const accelerations = Object.values(mempool.getAccelerations());
-      await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, structuredClone(transactions));
-    }
-
-    const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
-    memPool.handleRbfTransactions(rbfTransactions);
-    memPool.removeFromSpendMap(transactions);
-
-    if (config.MEMPOOL.AUDIT && memPool.isInSync()) {
-      let projectedBlocks;
-      const auditMempool = _memPool;
-      const isAccelerated = accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations()));
-
-      if (config.MEMPOOL.RUST_GBT) {
-        const added = memPool.limitGBT ? (candidates?.added || []) : [];
-        const removed = memPool.limitGBT ? (candidates?.removed || []) : [];
-        projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, auditMempool, added, removed, candidates, isAccelerated, block.extras.pool.id);
-      } else {
-        projectedBlocks = await mempoolBlocks.$makeBlockTemplates(transactionIds, auditMempool, candidates, false, isAccelerated, block.extras.pool.id);
-      }
-
-      if (Common.indexingEnabled()) {
-        const { unseen, censored, added, prioritized, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(block.height, blockTransactions, projectedBlocks, auditMempool);
-        const matchRate = Math.round(score * 100 * 100) / 100;
-
-        const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
-
-        let totalFees = 0;
-        let totalWeight = 0;
-        for (const tx of stripped) {
-          totalFees += tx.fee;
-          totalWeight += (tx.vsize * 4);
-        }
-
-        BlocksSummariesRepository.$saveTemplate({
-          height: block.height,
-          template: {
-            id: block.id,
-            transactions: stripped,
-          },
-          version: 1,
-        });
-
-        BlocksAuditsRepository.$saveAudit({
-          version: 1,
-          time: block.timestamp,
-          height: block.height,
-          hash: block.id,
-          unseenTxs: unseen,
-          addedTxs: added,
-          prioritizedTxs: prioritized,
-          missingTxs: censored,
-          freshTxs: fresh,
-          sigopTxs: sigop,
-          fullrbfTxs: fullrbf,
-          acceleratedTxs: accelerated,
-          matchRate: matchRate,
-          expectedFees: totalFees,
-          expectedWeight: totalWeight,
-        });
-
-        if (block.extras) {
-          block.extras.matchRate = matchRate;
-          block.extras.expectedFees = totalFees;
-          block.extras.expectedWeight = totalWeight;
-          block.extras.similarity = similarity;
-        }
-      }
-    } else if (block.extras) {
-      const mBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
-      if (mBlocks?.length && mBlocks[0].transactions) {
-        block.extras.similarity = Common.getSimilarity(mBlocks[0], transactions);
-      }
-    }
-
-    if (config.CORE_RPC.DEBUG_LOG_PATH && block.extras) {
-      const firstSeen = getRecentFirstSeen(block.id);
-      if (firstSeen) {
-        if (config.DATABASE.ENABLED) {
-          BlocksRepository.$saveFirstSeenTime(block.id, firstSeen);
-        }
-        block.extras.firstSeen = firstSeen;
-      }
-    }
-
     const confirmedTxids: { [txid: string]: boolean } = {};
-
-    // Update mempool to remove transactions included in the new block
     for (const txId of txIds) {
-      delete _memPool[txId];
-      rbfCache.mined(txId);
       confirmedTxids[txId] = true;
     }
 
-    if (memPool.limitGBT) {
-      const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null;
-      const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1;
-      candidates = memPool.getNextCandidates(minFeeMempool, minFeeTip, transactions);
-      transactionIds = Object.keys(candidates?.txs || {});
-    } else {
-      candidates = undefined;
-      transactionIds = Object.keys(memPool.getMempool());
-    }
-
-
-    if (config.MEMPOOL.RUST_GBT) {
-      const added = memPool.limitGBT ? (candidates?.added || []) : [];
-      const removed = memPool.limitGBT ? (candidates?.removed || []) : transactions;
-      await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, _memPool, added, removed, candidates, true);
-    } else {
-      await mempoolBlocks.$makeBlockTemplates(transactionIds, _memPool, candidates, true, true);
-    }
     const mBlocks = mempoolBlocks.getMempoolBlocks();
     const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
 
@@ -1265,6 +1231,7 @@ class WebsocketHandler {
           }
         }
         if (Object.keys(txs).length) {
+          client['track-txs-updates'] = (client['track-txs-updates'] || 0) + Object.keys(txs).length;
           response['tracked-txs'] = JSON.stringify(txs);
         }
       }
@@ -1300,6 +1267,8 @@ class WebsocketHandler {
         }
 
         if (Object.keys(addressMap).length > 0) {
+          client['track-addresses-updates'] =
+            (client['track-addresses-updates'] || 0) + this.countAddressTransactions(addressMap);
           response['multi-address-transactions'] = JSON.stringify(addressMap);
         }
       }
@@ -1393,26 +1362,20 @@ class WebsocketHandler {
       }
 
       if (Object.keys(response).length) {
-        client.send(this.serializeResponse(response));
+        this.send(client, this.serializeResponse(response));
       }
     });
     }
-
-    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
-      await statistics.runStatistics();
-    }
   }
 
   public handleNewStratumJob(job: StratumJob): void {
-    this.updateSocketDataFields({ 'stratumJobs': stratumApi.getJobs() });
-
     for (const server of this.webSocketServers) {
       server.clients.forEach((client) => {
         if (client.readyState !== WebSocket.OPEN) {
           return;
         }
         if (client['track-stratum'] && (client['track-stratum'] === 'all' || client['track-stratum'] === job.pool)) {
-          client.send(JSON.stringify({
+          this.send(client, JSON.stringify({
             'stratumJob': job
         }));
         }
@@ -1420,6 +1383,14 @@ class WebsocketHandler {
     }
   }
 
+  private send(client: WebSocket.WebSocket, data: string): void {
+    if (client.bufferedAmount > this.MAX_BUFFERED_AMOUNT) {
+      client.terminate();
+      return;
+    }
+    client.send(data);
+  }
+
   // takes a dictionary of JSON serialized values
   // and zips it together into a valid JSON object
   private serializeResponse(response): string {
@@ -1485,6 +1456,7 @@ class WebsocketHandler {
     return addressCache;
   }
 
+  /** @asyncSafe */
   private async getFullTransactions(transactions: MempoolTransactionExtended[]): Promise<MempoolTransactionExtended[]> {
     for (let i = 0; i < transactions.length; i++) {
       try {
@@ -1500,25 +1472,72 @@ class WebsocketHandler {
     if (this.webSocketServers.length) {
       let numTxSubs = 0;
       let numTxsSubs = 0;
+      let numAddressSubs = 0;
+      let numAddressesSubs = 0;
       let numProjectedSubs = 0;
       let numRbfSubs = 0;
+      let trackedTxsTotal = 0;
+      let trackedAddressesTotal = 0;
+      let trackedTxsMax = 0;
+      let trackedAddressesMax = 0;
+      let trackTxsTrackedTotal = 0;
+      let trackTxsTrackedMax = 0;
+      let trackAddressesTrackedTotal = 0;
+      let trackAddressesTrackedMax = 0;
+      let trackTxsUpdatesTotal = 0;
+      let trackTxsUpdatesMax = 0;
+      let trackAddressesUpdatesTotal = 0;
+      let trackAddressesUpdatesMax = 0;
 
-      // TODO - Fix indentation after PR is merged
       for (const server of this.webSocketServers) {
-      server.clients.forEach((client) => {
-        if (client['track-tx']) {
-          numTxSubs++;
-        }
-        if (client['track-txs']) {
-          numTxsSubs++;
-        }
-        if (client['track-mempool-block'] != null && client['track-mempool-block'] >= 0) {
-          numProjectedSubs++;
-        }
-        if (client['track-rbf']) {
-          numRbfSubs++;
-        }
-      })
+        server.clients.forEach((client) => {
+          let trackedTxCount = 0;
+          let trackedAddressCount = 0;
+
+          if (client['track-tx']) {
+            numTxSubs++;
+            trackedTxCount += 1;
+          }
+          if (client['track-txs']) {
+            numTxsSubs++;
+            trackedTxCount += client['track-txs'].length;
+          }
+          if (client['track-address']) {
+            numAddressSubs++;
+            trackedAddressCount += 1;
+          }
+          if (client['track-addresses']) {
+            numAddressesSubs++;
+            const addressCount = Object.keys(client['track-addresses']).length;
+            trackedAddressCount += addressCount;
+            trackAddressesTrackedTotal += addressCount;
+            trackAddressesTrackedMax = Math.max(trackAddressesTrackedMax, addressCount);
+            const updates = client['track-addresses-updates'] || 0;
+            trackAddressesUpdatesTotal += updates;
+            trackAddressesUpdatesMax = Math.max(trackAddressesUpdatesMax, updates);
+            client['track-addresses-updates'] = 0;
+          }
+          if (client['track-mempool-block'] != null && client['track-mempool-block'] >= 0) {
+            numProjectedSubs++;
+          }
+          if (client['track-rbf']) {
+            numRbfSubs++;
+          }
+          if (client['track-txs']) {
+            const txCount = client['track-txs'].length;
+            trackTxsTrackedTotal += txCount;
+            trackTxsTrackedMax = Math.max(trackTxsTrackedMax, txCount);
+            const updates = client['track-txs-updates'] || 0;
+            trackTxsUpdatesTotal += updates;
+            trackTxsUpdatesMax = Math.max(trackTxsUpdatesMax, updates);
+            client['track-txs-updates'] = 0;
+          }
+
+          trackedTxsTotal += trackedTxCount;
+          trackedAddressesTotal += trackedAddressCount;
+          trackedTxsMax = Math.max(trackedTxsMax, trackedTxCount);
+          trackedAddressesMax = Math.max(trackedAddressesMax, trackedAddressCount);
+        });
       }
 
       let count = 0;
@@ -1527,12 +1546,33 @@ class WebsocketHandler {
       }
       const diff = count - this.numClients;
       this.numClients = count;
-      logger.debug(`${count} websocket clients | ${this.numConnected} connected | ${this.numDisconnected} disconnected | (${diff >= 0 ? '+' : ''}${diff})`);
-      logger.debug(`websocket subscriptions: track-tx: ${numTxSubs}, track-txs: ${numTxsSubs}, track-mempool-block: ${numProjectedSubs} track-rbf: ${numRbfSubs}`);
+      const trackedTxsAvg = count > 0 ? trackedTxsTotal / count : 0;
+      const trackedAddressesAvg = count > 0 ? trackedAddressesTotal / count : 0;
+      const trackTxsTrackedAvg = numTxsSubs > 0 ? trackTxsTrackedTotal / numTxsSubs : 0;
+      const trackAddressesTrackedAvg =
+        numAddressesSubs > 0 ? trackAddressesTrackedTotal / numAddressesSubs : 0;
+      const trackTxsUpdatesAvg = numTxsSubs > 0 ? trackTxsUpdatesTotal / numTxsSubs : 0;
+      const trackAddressesUpdatesAvg =
+        numAddressesSubs > 0 ? trackAddressesUpdatesTotal / numAddressesSubs : 0;
+      logger.debug(
+        `${count} websocket clients | ${this.numConnected} connected | ${this.numDisconnected} disconnected | (${diff >= 0 ? '+' : ''}${diff}) | tracked txs: total=${trackedTxsTotal}, avg=${trackedTxsAvg.toFixed(2)}, max=${trackedTxsMax} | tracked addresses: total=${trackedAddressesTotal}, avg=${trackedAddressesAvg.toFixed(2)}, max=${trackedAddressesMax} | ws-subscriptions: tx=${numTxSubs},txs=${numTxsSubs},address=${numAddressSubs},addresses=${numAddressesSubs},txs-tracked-avg=${trackTxsTrackedAvg.toFixed(2)},txs-tracked-max=${trackTxsTrackedMax},addresses-tracked-avg=${trackAddressesTrackedAvg.toFixed(2)},addresses-tracked-max=${trackAddressesTrackedMax},txs-updates-avg=${trackTxsUpdatesAvg.toFixed(2)},txs-updates-max=${trackTxsUpdatesMax},addresses-updates-avg=${trackAddressesUpdatesAvg.toFixed(2)},addresses-updates-max=${trackAddressesUpdatesMax}`
+      );
+      logger.debug(`websocket subscriptions: track-tx: ${numTxSubs}, track-txs: ${numTxsSubs}, track-address: ${numAddressSubs}, track-addresses: ${numAddressesSubs}, track-mempool-block: ${numProjectedSubs} track-rbf: ${numRbfSubs}`);
       this.numConnected = 0;
       this.numDisconnected = 0;
     }
   }
+
+  private countAddressTransactions(addressMap: { [address: string]: AddressTransactions }): number {
+    return Object.values(addressMap).reduce(
+      (total, transactions) =>
+        total
+        + transactions.mempool.length
+        + transactions.confirmed.length
+        + transactions.removed.length,
+      0,
+    );
+  }
 }
 
 export default new WebsocketHandler();

backend/src/cluster-mempool/block-builder.ts

@@ -0,0 +1,165 @@
+import { PairingHeap } from '../utils/pairing-heap';
+import { LinearizationChunk } from './linearize';
+import { MempoolTransactionExtended } from '../mempool.interfaces';
+import { Cluster } from './cluster-mempool';
+
+export interface ProjectedBlock {
+  txids: string[];
+  weight: number;
+  sigops: number;
+}
+
+interface ChunkHeapEntry {
+  fee: number;
+  weight: number;
+  sigops: number;
+  equalFeeratePrefixWeight: number;
+  maxOrder: number;
+  clusterId: number;
+  chunkIndex: number;
+}
+
+const BLOCK_WEIGHT_UNITS = 4_000_000;
+const MAX_BLOCK_SIGOPS_COST = 80_000;
+const COINBASE_RESERVED_WEIGHT = 8000;
+const MAX_CONSECUTIVE_FAILURES = 1000;
+const MAX_WASTED_WEIGHT = 4000;
+
+function chunkHeapHigherPriority(a: ChunkHeapEntry, b: ChunkHeapEntry): boolean {
+  const feerateDiff = a.fee * b.weight - b.fee * a.weight;
+  if (feerateDiff !== 0) {
+    return feerateDiff > 0;
+  }
+  if (a.equalFeeratePrefixWeight !== b.equalFeeratePrefixWeight) {
+    return a.equalFeeratePrefixWeight < b.equalFeeratePrefixWeight;
+  }
+  return a.maxOrder < b.maxOrder;
+}
+
+function equalFeerate(a: LinearizationChunk, b: LinearizationChunk): boolean {
+  return a.fee * b.weight === b.fee * a.weight;
+}
+
+function makeChunkHeapEntry(cluster: Cluster, chunkIndex: number, mempool: { [txid: string]: MempoolTransactionExtended }): ChunkHeapEntry {
+  const chunk = cluster.chunks[chunkIndex];
+  let maxOrder = 0;
+  let sigops = 0;
+  for (const tx of chunk.txs) {
+    if (tx.order > maxOrder) {
+      maxOrder = tx.order;
+    }
+    const mempoolTx = mempool[tx.txid];
+    if (mempoolTx) {
+      sigops += mempoolTx.sigops || 0;
+    }
+  }
+  let prefixWeight = chunk.weight;
+  for (let i = chunkIndex - 1; i >= 0; i--) {
+    if (equalFeerate(cluster.chunks[i], chunk)) {
+      prefixWeight += cluster.chunks[i].weight;
+    } else {
+      break;
+    }
+  }
+  return {
+    fee: chunk.fee,
+    weight: chunk.weight,
+    sigops,
+    equalFeeratePrefixWeight: prefixWeight,
+    maxOrder,
+    clusterId: cluster.id,
+    chunkIndex,
+  };
+}
+
+function buildChunkHeap(clusters: Map<number, Cluster>, mempool: { [txid: string]: MempoolTransactionExtended }): PairingHeap<ChunkHeapEntry> {
+  const heap = new PairingHeap<ChunkHeapEntry>(chunkHeapHigherPriority);
+  for (const cluster of clusters.values()) {
+    if (cluster.chunks.length > 0) {
+      heap.add(makeChunkHeapEntry(cluster, 0, mempool));
+    }
+  }
+  return heap;
+}
+
+export function assembleBlocks(
+  n: number,
+  clusters: Map<number, Cluster>,
+  mempool: { [txid: string]: MempoolTransactionExtended },
+  enforceLimit: boolean,
+): ProjectedBlock[] {
+  const heap = buildChunkHeap(clusters, mempool);
+  const blocks: ProjectedBlock[] = [];
+  for (let blockIdx = 0; blockIdx < n; blockIdx++) {
+    const limited = enforceLimit || blockIdx < n - 1;
+    const maxWeight = limited ? BLOCK_WEIGHT_UNITS : Infinity;
+    const maxSigops = limited ? MAX_BLOCK_SIGOPS_COST : Infinity;
+    const block = fillBlock(heap, clusters, mempool, maxWeight, maxSigops);
+    if (block.txids.length === 0) {
+      break;
+    }
+    blocks.push(block);
+  }
+  return blocks;
+}
+
+function fillBlock(
+  heap: PairingHeap<ChunkHeapEntry>,
+  clusters: Map<number, Cluster>,
+  mempool: { [txid: string]: MempoolTransactionExtended },
+  maxWeight: number,
+  maxSigops: number,
+): ProjectedBlock {
+  const block: ProjectedBlock = { txids: [], weight: COINBASE_RESERVED_WEIGHT, sigops: 0 };
+  const deferred: ChunkHeapEntry[] = [];
+  let consecutiveFailed = 0;
+  let full = false;
+
+  while (!heap.isEmpty() && !full) {
+    const entry = heap.pop() as ChunkHeapEntry;
+    const cluster = clusters.get(entry.clusterId);
+    const chunk = cluster?.chunks[entry.chunkIndex];
+
+    if (!cluster || !chunk) {
+      // stale entry
+    } else if (block.weight + entry.weight < maxWeight
+      && block.sigops + entry.sigops < maxSigops) {
+      consecutiveFailed = 0;
+      block.weight += chunkWeight(chunk, mempool);
+      block.sigops += entry.sigops;
+      for (const tx of chunk.txs) {
+        block.txids.push(tx.txid);
+      }
+      if (entry.chunkIndex + 1 < cluster.chunks.length) {
+        heap.add(makeChunkHeapEntry(cluster, entry.chunkIndex + 1, mempool));
+      }
+    } else {
+      deferred.push(entry);
+      consecutiveFailed++;
+      if (consecutiveFailed > MAX_CONSECUTIVE_FAILURES
+        && block.weight + MAX_WASTED_WEIGHT > maxWeight) {
+        full = true;
+      }
+    }
+  }
+
+  for (const entry of deferred) {
+    heap.add(entry);
+  }
+
+  return block;
+}
+
+function chunkWeight(
+  chunk: LinearizationChunk,
+  mempool: { [txid: string]: MempoolTransactionExtended },
+): number {
+  let weight = 0;
+  for (const clusterTx of chunk.txs) {
+    const mempoolTx = mempool[clusterTx.txid];
+    if (mempoolTx) {
+      weight += mempoolTx.weight;
+    }
+  }
+  return weight;
+}

backend/src/cluster-mempool/cluster-mempool.ts

@@ -0,0 +1,709 @@
+import { ClusterTx, DepGraph, sortTopological, subgraph } from './depgraph';
+import { linearizeCluster, LinearizationChunk } from './linearize';
+import { ProjectedBlock, assembleBlocks } from './block-builder';
+import { Ancestor, CpfpClusterData, CpfpClusterTx, MempoolTransactionExtended } from '../mempool.interfaces';
+import logger from '../logger';
+
+export interface MempoolDiff {
+  added: MempoolTransactionExtended[];
+  removed: MempoolTransactionExtended[];
+  accelerations: { [txid: string]: { feeDelta: number } };
+}
+
+export interface ClusterInfo {
+  clusterId: number;
+  chunkIndex: number;
+  chunkFeerate: number;
+}
+
+export { ProjectedBlock };
+
+export interface Cluster {
+  id: number;
+  depgraph: DepGraph;
+  txs: Map<string, ClusterTx>;
+  linearization: ClusterTx[];
+  chunks: LinearizationChunk[];
+  dirty: boolean;
+}
+
+export interface TxCpfpData {
+  effectiveFeePerVsize: number;
+  clusterId: number;
+  chunkIndex: number;
+  cpfpDirty: boolean;
+  cpfpChecked: boolean;
+  ancestors: Ancestor[];
+  descendants: Ancestor[];
+}
+
+const DEFAULT_COST_BUDGET = 75000;
+
+export class ClusterMempool {
+  private clusters = new Map<number, Cluster>();
+  private txToCluster = new Map<string, number>();
+  private spentBy = new Map<string, string>();
+  private mempool: Readonly<{ [txid: string]: MempoolTransactionExtended }>;
+  private accelerations: { [txid: string]: { feeDelta: number } } = {};
+  private nextClusterId = 0;
+  private modifyTxs: boolean;
+  private costBudget: number = DEFAULT_COST_BUDGET;
+
+  constructor(mempool: { [txid: string]: MempoolTransactionExtended }, accelerations?: { [txid: string]: { feeDelta: number } }, modifyTxs: boolean = true, costBudget: number = DEFAULT_COST_BUDGET) {
+    this.mempool = mempool;
+    if (accelerations) {
+      this.accelerations = accelerations;
+    }
+    this.modifyTxs = modifyTxs;
+    this.costBudget = costBudget;
+    this.buildFromMempool();
+  }
+
+  applyMempoolChange(diff: MempoolDiff): void {
+    this.processRemovals(diff.removed);
+    this.splitDisconnectedClusters();
+    this.processAccelerationChanges(diff.accelerations);
+    this.processAdditions(diff.added);
+    this.relinearizeDirtyClusters();
+  }
+
+  getBlocks(n: number, enforceLimit = false): ProjectedBlock[] {
+    return assembleBlocks(n, this.clusters, this.mempool, enforceLimit);
+  }
+
+  getCluster(clusterId: number): CpfpClusterData | null {
+    const cluster = this.clusters.get(clusterId);
+    if (!cluster) {
+      return null;
+    }
+    return this.buildClusterData(cluster);
+  }
+
+  getClusterInfo(txid: string): ClusterInfo | null {
+    const match = this.getClusterForTx(txid);
+    if (!match) {
+      return null;
+    }
+    return this.findChunkInfo(match.cluster, match.clusterTx);
+  }
+
+  getClusterForApi(txid: string): (CpfpClusterData & { chunkIndex: number }) | null {
+    const info = this.getClusterInfo(txid);
+    if (!info) {
+      return null;
+    }
+    const cluster = this.getCluster(info.clusterId);
+    if (!cluster || cluster.txs.length <= 1) {
+      return null;
+    }
+    return { ...cluster, chunkIndex: info.chunkIndex };
+  }
+
+  getCpfpDataForTx(txid: string): TxCpfpData | null {
+    const clusterInfo = this.getClusterForTx(txid);
+    if (!clusterInfo) {
+      return null;
+    }
+    const chunkInfo = this.findChunkInfo(clusterInfo.cluster, clusterInfo.clusterTx);
+    if (!chunkInfo) {
+      return null;
+    }
+    const chunk = clusterInfo.cluster.chunks[chunkInfo?.chunkIndex];
+    const chunkSet = chunk.txs.length > 1 ? new Set(chunk.txs) : null;
+    return {
+      effectiveFeePerVsize: chunkInfo.chunkFeerate,
+      clusterId: clusterInfo.cluster.id,
+      chunkIndex: chunkInfo.chunkIndex,
+      cpfpDirty: true,
+      cpfpChecked: true,
+      ancestors: chunkSet ? this.getChunkRelatives(clusterInfo.clusterTx, chunkSet, 'ancestors') : [],
+      descendants: chunkSet ? this.getChunkRelatives(clusterInfo.clusterTx, chunkSet, 'descendants') : [],
+    };
+  }
+
+  getClusterCount(): number {
+    return this.clusters.size;
+  }
+
+  getTxCount(): number {
+    return this.txToCluster.size;
+  }
+
+  private getClusterForTx(txid: string): { cluster: Cluster; clusterTx: ClusterTx } | null {
+    const clusterId = this.txToCluster.get(txid);
+    if (clusterId === undefined) {
+      return null;
+    }
+    const cluster = this.clusters.get(clusterId);
+    if (!cluster) {
+      return null;
+    }
+    const clusterTx = cluster.txs.get(txid);
+    if (!clusterTx) {
+      return null;
+    }
+    return { cluster, clusterTx };
+  }
+
+  private buildFromMempool(): void {
+    const parentMap = this.buildRelativeMaps();
+    const components = this.findMempoolComponents(parentMap);
+    for (const component of components) {
+      this.createClusterFromTxids(component, parentMap);
+    }
+  }
+
+  private buildRelativeMaps(): Map<string, Set<string>> {
+    const parentMap = new Map<string, Set<string>>();
+    this.spentBy.clear();
+    for (const txid in this.mempool) {
+      const tx = this.mempool[txid];
+      const txParents = new Set<string>();
+      for (const vin of tx.vin) {
+        if (!vin.is_coinbase && this.mempool[vin.txid]) {
+          txParents.add(vin.txid);
+          this.spentBy.set(`${vin.txid}:${vin.vout}`, txid);
+        }
+      }
+      if (txParents.size > 0) {
+        parentMap.set(txid, txParents);
+      }
+    }
+    return parentMap;
+  }
+
+  private findMempoolComponents(parentMap: Map<string, Set<string>>): Set<string>[] {
+    const visited = new Set<string>();
+    const components: Set<string>[] = [];
+
+    for (const txid in this.mempool) {
+      if (!visited.has(txid)) {
+        const component = this.dfsComponent(txid, visited, parentMap);
+        components.push(component);
+      }
+    }
+    return components;
+  }
+
+  private dfsComponent(
+    startTxid: string,
+    visited: Set<string>,
+    parentMap: Map<string, Set<string>>
+  ): Set<string> {
+    const component = new Set<string>();
+    const stack = [startTxid];
+    while (stack.length > 0) {
+      const current = stack.pop();
+      if (current !== undefined && !visited.has(current)) {
+        visited.add(current);
+        component.add(current);
+
+        const txParents = parentMap.get(current);
+        if (txParents) {
+          for (const p of txParents) {
+            if (!visited.has(p)) {
+              stack.push(p);
+            }
+          }
+        }
+
+        const tx = this.mempool[current];
+        if (tx) {
+          for (let vout = 0; vout < tx.vout.length; vout++) {
+            const child = this.spentBy.get(`${current}:${vout}`);
+            if (child && !visited.has(child)) {
+              stack.push(child);
+            }
+          }
+        }
+      }
+    }
+    return component;
+  }
+
+  private effectiveFee(txid: string, tx: MempoolTransactionExtended): number {
+    return tx.fee + (this.accelerations[txid]?.feeDelta || 0);
+  }
+
+  private adjustedWeight(tx: MempoolTransactionExtended): number {
+    return Math.max(tx.weight, (tx.sigops || 0) * 20);
+  }
+
+  private createClusterFromTxids(
+    txids: Set<string>,
+    parentMap: Map<string, Set<string>>
+  ): Cluster | null {
+    const clusterId = this.nextClusterId++;
+    const depgraph = new DepGraph();
+    const txMap = new Map<string, ClusterTx>();
+
+    for (const txid of txids) {
+      const tx = this.mempool[txid];
+      if (!tx) {
+        logger.warn(`Warning: missing mempool tx ${txid} during cluster creation, skipping`);
+        return null;
+      }
+      const clusterTx = depgraph.addTransaction(txid, this.effectiveFee(txid, tx), this.adjustedWeight(tx), tx.order ?? 0);
+      txMap.set(txid, clusterTx);
+    }
+
+    for (const txid of txids) {
+      const txParents = parentMap.get(txid);
+      if (txParents) {
+        for (const parentTxid of txParents) {
+          if (txids.has(parentTxid)) {
+            const parentTx = txMap.get(parentTxid);
+            const childTx = txMap.get(txid);
+            if (parentTx && childTx) {
+              depgraph.addDependency(parentTx, childTx);
+            }
+          }
+        }
+      }
+    }
+
+    const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), this.costBudget);
+
+    const cluster: Cluster = {
+      id: clusterId,
+      depgraph,
+      txs: txMap,
+      linearization,
+      chunks,
+      dirty: false,
+    };
+
+    this.clusters.set(clusterId, cluster);
+    for (const txid of txids) {
+      this.txToCluster.set(txid, clusterId);
+    }
+
+    if (this.modifyTxs) {
+      this.writeBackCluster(cluster);
+    }
+    return cluster;
+  }
+
+  public writeBackClusters(): void {
+    for (const cluster of this.clusters.values()) {
+      this.writeBackCluster(cluster);
+    }
+  }
+
+  private writeBackCluster(cluster: Cluster): void {
+    for (let chunkIdx = 0; chunkIdx < cluster.chunks.length; chunkIdx++) {
+      const chunk = cluster.chunks[chunkIdx];
+      const chunkFeerate = chunk.weight > 0 ? (chunk.fee * 4) / chunk.weight : 0;
+      const chunkSet = chunk.txs.length > 1 ? new Set(chunk.txs) : null;
+      for (const clusterTx of chunk.txs) {
+        this.writeBackTx(cluster, clusterTx, chunkIdx, chunkFeerate, chunkSet);
+      }
+    }
+  }
+
+  private writeBackTx(
+    cluster: Cluster,
+    clusterTx: ClusterTx,
+    chunkIdx: number,
+    chunkFeerate: number,
+    chunkSet: Set<ClusterTx> | null
+  ): void {
+    const txid = clusterTx.txid;
+    if (!this.mempool[txid]) {
+      logger.warn(`ClusterMempool.writeBackTx: ${txid} missing from mempool (cluster ${cluster.id})`);
+      return;
+    }
+    const tx = this.mempool[txid];
+    if (tx.effectiveFeePerVsize !== chunkFeerate || tx.clusterId !== cluster.id) {
+      tx.cpfpDirty = true;
+    }
+    tx.effectiveFeePerVsize = chunkFeerate;
+    tx.clusterId = cluster.id;
+    tx.chunkIndex = chunkIdx;
+    tx.cpfpChecked = true;
+
+    if (chunkSet) {
+      tx.ancestors = this.getChunkRelatives(clusterTx, chunkSet, 'ancestors');
+      tx.descendants = this.getChunkRelatives(clusterTx, chunkSet, 'descendants');
+    } else {
+      tx.ancestors = [];
+      tx.descendants = [];
+    }
+  }
+
+  private getChunkRelatives(
+    clusterTx: ClusterTx,
+    chunkSet: Set<ClusterTx>,
+    direction: 'ancestors' | 'descendants'
+  ): { txid: string; fee: number; weight: number }[] {
+    const relatives: { txid: string; fee: number; weight: number }[] = [];
+    const related = direction === 'ancestors' ? clusterTx.ancestors : clusterTx.descendants;
+    for (const rel of related) {
+      if (rel !== clusterTx && chunkSet.has(rel)) {
+        const mempoolTx = this.mempool[rel.txid];
+        if (mempoolTx) {
+          relatives.push({ txid: rel.txid, fee: mempoolTx.fee, weight: mempoolTx.weight });
+        } else {
+          logger.warn(`ClusterMempool.getChunkRelatives: ${rel.txid} missing from mempool`);
+        }
+      }
+    }
+    return relatives;
+  }
+
+  private processRemovals(removed: MempoolTransactionExtended[]): void {
+    for (const tx of removed) {
+      for (const vin of tx.vin) {
+        if (!vin.is_coinbase) {
+          const spentOutpoint = `${vin.txid}:${vin.vout}`;
+          if (this.spentBy.get(spentOutpoint) === tx.txid) {
+            this.spentBy.delete(spentOutpoint);
+          }
+        }
+      }
+    }
+
+    for (const tx of removed) {
+      const match = this.getClusterForTx(tx.txid);
+      if (match) {
+        match.cluster.depgraph.removeTransactions(new Set([match.clusterTx]));
+        match.cluster.txs.delete(tx.txid);
+        match.cluster.linearization = match.cluster.linearization.filter(t => t !== match.clusterTx);
+        this.txToCluster.delete(tx.txid);
+        match.cluster.dirty = true;
+      }
+    }
+  }
+
+  private splitDisconnectedClusters(): void {
+    for (const [clusterId, cluster] of this.clusters.entries()) {
+      if (cluster.dirty) {
+        if (cluster.depgraph.size === 0) {
+          this.clusters.delete(clusterId);
+        } else {
+          const components = cluster.depgraph.findConnectedComponents();
+          if (components.length > 1) {
+            this.clusters.delete(clusterId);
+            for (const component of components) {
+              this.splitComponentToCluster(cluster, component);
+            }
+          }
+        }
+      }
+    }
+  }
+
+  private splitComponentToCluster(sourceCluster: Cluster, component: Set<ClusterTx>): void {
+    const newClusterId = this.nextClusterId++;
+    const { depgraph: newDepgraph, txMap } = subgraph(component);
+
+    const newTxs = new Map<string, ClusterTx>();
+    for (const oldTx of component) {
+      const newTx = txMap.get(oldTx);
+      if (newTx) {
+        newTxs.set(oldTx.txid, newTx);
+        this.txToCluster.set(oldTx.txid, newClusterId);
+      }
+    }
+
+    const newLinearization: ClusterTx[] = [];
+    for (const oldTx of sourceCluster.linearization) {
+      if (component.has(oldTx)) {
+        const newTx = txMap.get(oldTx);
+        if (newTx) {
+          newLinearization.push(newTx);
+        }
+      }
+    }
+
+    const newCluster: Cluster = {
+      id: newClusterId,
+      dirty: true,
+      depgraph: newDepgraph,
+      txs: newTxs,
+      linearization: newLinearization,
+      chunks: [],
+    };
+    this.clusters.set(newClusterId, newCluster);
+  }
+
+  private processAdditions(added: MempoolTransactionExtended[]): void {
+    for (const tx of added) {
+      const txid = tx.txid;
+
+      // sanity check for duplicate transactions
+      if (this.getClusterForTx(txid) !== null) {
+        logger.warn(`ClusterMempool.processAdditions: ${txid} was already added, skipping`);
+        continue;
+      }
+
+      for (const vin of tx.vin) {
+        if (!vin.is_coinbase) {
+          this.spentBy.set(`${vin.txid}:${vin.vout}`, txid);
+        }
+      }
+
+      const { relatedClusterIds, parentTxids, childTxids } = this.findRelatedClusters(tx);
+
+      if (relatedClusterIds.size === 0) {
+        this.addSingletonCluster(tx);
+      } else if (relatedClusterIds.size === 1) {
+        this.addToExistingCluster(tx, relatedClusterIds, parentTxids, childTxids);
+      } else {
+        this.mergeAndAddToCluster(tx, relatedClusterIds, parentTxids, childTxids);
+      }
+    }
+  }
+
+  private findRelatedClusters(tx: MempoolTransactionExtended): {
+    relatedClusterIds: Set<number>;
+    parentTxids: string[];
+    childTxids: string[];
+  } {
+    const relatedClusterIds = new Set<number>();
+    const parentTxids: string[] = [];
+    const childTxids: string[] = [];
+
+    for (const vin of tx.vin) {
+      if (!vin.is_coinbase && this.mempool[vin.txid]) {
+        const parentCluster = this.txToCluster.get(vin.txid);
+        if (parentCluster !== undefined) {
+          relatedClusterIds.add(parentCluster);
+          parentTxids.push(vin.txid);
+        }
+      }
+    }
+
+    for (let vout = 0; vout < tx.vout.length; vout++) {
+      const childTxid = this.spentBy.get(`${tx.txid}:${vout}`);
+      if (childTxid && this.mempool[childTxid]) {
+        const childCluster = this.txToCluster.get(childTxid);
+        if (childCluster !== undefined) {
+          relatedClusterIds.add(childCluster);
+          childTxids.push(childTxid);
+        }
+      }
+    }
+
+    return { relatedClusterIds, parentTxids, childTxids };
+  }
+
+  private addSingletonCluster(tx: MempoolTransactionExtended): void {
+    const txid = tx.txid;
+    const clusterId = this.nextClusterId++;
+    const depgraph = new DepGraph();
+    const clusterTx = depgraph.addTransaction(txid, this.effectiveFee(txid, tx), this.adjustedWeight(tx), tx.order ?? 0);
+
+    const cluster: Cluster = {
+      id: clusterId,
+      dirty: true,
+      depgraph,
+      txs: new Map([[txid, clusterTx]]),
+      linearization: [clusterTx],
+      chunks: [],
+    };
+    this.clusters.set(clusterId, cluster);
+    this.txToCluster.set(txid, clusterId);
+  }
+
+  private addToExistingCluster(
+    tx: MempoolTransactionExtended,
+    relatedClusterIds: Set<number>,
+    parentTxids: string[],
+    childTxids: string[],
+  ): void {
+    const txid = tx.txid;
+    const clusterId = relatedClusterIds.values().next().value;
+    const cluster = this.clusters.get(clusterId);
+    if (!cluster) {
+      return;
+    }
+    const clusterTx = cluster.depgraph.addTransaction(txid, this.effectiveFee(txid, tx), this.adjustedWeight(tx), tx.order ?? 0);
+    cluster.txs.set(txid, clusterTx);
+    cluster.linearization.push(clusterTx);
+    this.txToCluster.set(txid, clusterId);
+
+    this.addParentDeps(cluster, clusterTx, parentTxids);
+    this.addChildDeps(cluster, clusterTx, childTxids);
+    cluster.dirty = true;
+  }
+
+  private mergeAndAddToCluster(
+    tx: MempoolTransactionExtended,
+    relatedClusterIds: Set<number>,
+    parentTxids: string[],
+    childTxids: string[],
+  ): void {
+    const clusterIterator = relatedClusterIds.values();
+    const primaryId: number = clusterIterator.next().value;
+    const primary = this.clusters.get(primaryId);
+    if (!primary) {
+      return;
+    }
+
+    for (const clusterId of clusterIterator) {
+      const other = this.clusters.get(clusterId);
+      if (other) {
+        this.mergeClusterInto(primary, other);
+        this.clusters.delete(clusterId);
+      }
+    }
+
+    const clusterTx = primary.depgraph.addTransaction(tx.txid, this.effectiveFee(tx.txid, tx), this.adjustedWeight(tx), tx.order ?? 0);
+    primary.txs.set(tx.txid, clusterTx);
+    primary.linearization.push(clusterTx);
+    this.txToCluster.set(tx.txid, primaryId);
+
+    this.addParentDeps(primary, clusterTx, parentTxids);
+    this.addChildDeps(primary, clusterTx, childTxids);
+    primary.dirty = true;
+  }
+
+  private addParentDeps(cluster: Cluster, childTx: ClusterTx, parentTxids: string[]): void {
+    for (const parentTxid of parentTxids) {
+      const parentTx = cluster.txs.get(parentTxid);
+      if (parentTx) {
+        cluster.depgraph.addDependency(parentTx, childTx);
+      }
+    }
+  }
+
+  private addChildDeps(cluster: Cluster, parentTx: ClusterTx, childTxids: string[]): void {
+    for (const childTxid of childTxids) {
+      const childTx = cluster.txs.get(childTxid);
+      if (childTx) {
+        cluster.depgraph.addDependency(parentTx, childTx);
+      }
+    }
+  }
+
+  private mergeClusterInto(primary: Cluster, other: Cluster): void {
+    for (const [txid, otherTx] of other.txs) {
+      const newTx = primary.depgraph.addTransaction(txid, otherTx.effectiveFee, otherTx.weight, otherTx.order);
+      primary.txs.set(txid, newTx);
+      this.txToCluster.set(txid, primary.id);
+    }
+
+    for (const otherTx of other.depgraph.getTxs()) {
+      for (const parent of otherTx.parents) {
+        const newChild = primary.txs.get(otherTx.txid);
+        const newParent = primary.txs.get(parent.txid);
+        if (newChild && newParent) {
+          primary.depgraph.addDependency(newParent, newChild);
+        }
+      }
+    }
+
+    for (const otherTx of other.linearization) {
+      const newTx = primary.txs.get(otherTx.txid);
+      if (newTx) {
+        primary.linearization.push(newTx);
+      }
+    }
+  }
+
+  private processAccelerationChanges(newAccelerations: { [txid: string]: { feeDelta: number } }): void {
+    const changed = new Set<string>();
+    for (const txid in newAccelerations) {
+      if ((newAccelerations[txid]?.feeDelta || 0) !== (this.accelerations[txid]?.feeDelta || 0)) {
+        changed.add(txid);
+      }
+    }
+    for (const txid in this.accelerations) {
+      if (!newAccelerations[txid]) {
+        changed.add(txid);
+      }
+    }
+    this.accelerations = newAccelerations;
+    for (const txid of changed) {
+      const tx = this.mempool[txid];
+      if (!tx) {
+        continue;
+      }
+      const match = this.getClusterForTx(txid);
+      if (match) {
+        match.clusterTx.effectiveFee = this.effectiveFee(txid, tx);
+        match.cluster.dirty = true;
+      }
+    }
+  }
+
+  private relinearizeDirtyClusters(): void {
+    for (const [clusterId, cluster] of this.clusters.entries()) {
+      if (cluster.dirty) {
+        cluster.dirty = false;
+        const newId = this.nextClusterId++;
+        this.clusters.delete(clusterId);
+        cluster.id = newId;
+        this.clusters.set(newId, cluster);
+        for (const txid of cluster.txs.keys()) {
+          this.txToCluster.set(txid, newId);
+        }
+
+        const { linearization, chunks } = linearizeCluster(
+          cluster.depgraph.getTxs(),
+          this.costBudget,
+          cluster.linearization,
+        );
+        cluster.linearization = linearization;
+        cluster.chunks = chunks;
+
+        if (this.modifyTxs) {
+          this.writeBackCluster(cluster);
+        }
+      }
+    }
+  }
+
+  private buildClusterData(cluster: Cluster): CpfpClusterData {
+    const txs: CpfpClusterTx[] = [];
+    const txToFlatIdx = new Map<ClusterTx, number>();
+
+    for (const chunk of cluster.chunks) {
+      const ordered = sortTopological(new Set(chunk.txs));
+      for (const clusterTx of ordered) {
+        if (this.mempool[clusterTx.txid]) {
+          txToFlatIdx.set(clusterTx, txs.length);
+          const parents: number[] = [];
+          for (const parentTx of clusterTx.parents) {
+            const flatIdx = txToFlatIdx.get(parentTx);
+            if (flatIdx !== undefined) {
+              parents.push(flatIdx);
+            }
+          }
+          const mempoolTx = this.mempool[clusterTx.txid];
+          txs.push({ txid: clusterTx.txid, fee: mempoolTx.fee, weight: mempoolTx.weight, parents });
+        } else {
+          logger.warn(`ClusterMempool.buildClusterData: ${clusterTx.txid} missing from mempool (cluster ${cluster.id})`);
+        }
+      }
+    }
+
+    let offset = 0;
+    const chunks = cluster.chunks.map(chunk => {
+      const count = chunk.txs.length;
+      const chunkEntry = {
+        txs: Array.from({ length: count }, (_, i) => offset + i),
+        feerate: chunk.weight > 0 ? (chunk.fee * 4) / chunk.weight : 0,
+      };
+      offset += count;
+      return chunkEntry;
+    });
+
+    return { txs, chunks };
+  }
+
+  private findChunkInfo(cluster: Cluster, tx: ClusterTx): ClusterInfo | null {
+    for (let chunkIdx = 0; chunkIdx < cluster.chunks.length; chunkIdx++) {
+      const chunk = cluster.chunks[chunkIdx];
+      if (chunk.txs.includes(tx)) {
+        return {
+          clusterId: cluster.id,
+          chunkIndex: chunkIdx,
+          chunkFeerate: chunk.weight > 0 ? (chunk.fee * 4) / chunk.weight : 0,
+        };
+      }
+    }
+    return null;
+  }
+}

backend/src/cluster-mempool/depgraph.ts

@@ -0,0 +1,165 @@
+import logger from '../logger';
+
+export class ClusterTx {
+  txid: string;
+  effectiveFee: number;
+  weight: number;
+  order: number;
+  ancestors: Set<ClusterTx>;
+  descendants: Set<ClusterTx>;
+  parents: Set<ClusterTx>;
+  children: Set<ClusterTx>;
+
+  constructor(txid: string, effectiveFee: number, weight: number, order: number) {
+    this.txid = txid;
+    this.effectiveFee = effectiveFee;
+    this.weight = weight;
+    this.order = order;
+    this.ancestors = new Set([this]);
+    this.descendants = new Set([this]);
+    this.parents = new Set();
+    this.children = new Set();
+  }
+}
+
+export class DepGraph {
+  private txs: Set<ClusterTx> = new Set();
+
+  get size(): number {
+    return this.txs.size;
+  }
+
+  addTransaction(txid: string, fee: number, weight: number, order: number = 0): ClusterTx {
+    const tx = new ClusterTx(txid, fee, weight, order);
+    this.txs.add(tx);
+    return tx;
+  }
+
+  addDependency(parent: ClusterTx, child: ClusterTx): void {
+    if (!this.txs.has(parent) || !this.txs.has(child)) {
+      logger.warn(`Warning: invalid dependency, skipping`);
+      return;
+    }
+
+    parent.children.add(child);
+    child.parents.add(parent);
+
+    if (child.ancestors.has(parent)) {
+      return;
+    }
+
+    for (const descendant of child.descendants) {
+      for (const ancestor of parent.ancestors) {
+        descendant.ancestors.add(ancestor);
+        ancestor.descendants.add(descendant);
+      }
+    }
+  }
+
+  removeTransactions(toRemove: Set<ClusterTx>): void {
+    for (const tx of toRemove) {
+      for (const parent of tx.parents) {
+        parent.children.delete(tx);
+      }
+      for (const child of tx.children) {
+        child.parents.delete(tx);
+      }
+      this.txs.delete(tx);
+    }
+
+    for (const tx of this.txs) {
+      for (const removed of toRemove) {
+        tx.ancestors.delete(removed);
+        tx.descendants.delete(removed);
+      }
+    }
+
+    this.rederiveAncestorsDescendants();
+  }
+
+  private rederiveAncestorsDescendants(): void {
+    const ordered = [...this.txs].sort((a, b) => a.ancestors.size - b.ancestors.size);
+    for (const tx of this.txs) {
+      tx.ancestors = new Set([tx]);
+      tx.descendants = new Set([tx]);
+    }
+    for (const tx of ordered) {
+      for (const parent of tx.parents) {
+        for (const ancestor of parent.ancestors) {
+          tx.ancestors.add(ancestor);
+        }
+      }
+      for (const ancestor of tx.ancestors) {
+        ancestor.descendants.add(tx);
+      }
+    }
+  }
+
+  hasTx(tx: ClusterTx): boolean {
+    return this.txs.has(tx);
+  }
+
+  getTxs(): Set<ClusterTx> {
+    return this.txs;
+  }
+
+  findConnectedComponents(): Set<ClusterTx>[] {
+    const visited = new Set<ClusterTx>();
+    const components: Set<ClusterTx>[] = [];
+
+    for (const tx of this.txs) {
+      if (!visited.has(tx)) {
+        const component = new Set<ClusterTx>();
+        const stack: ClusterTx[] = [tx];
+        while (stack.length > 0) {
+          const node = stack.pop();
+          if (node && !visited.has(node)) {
+            visited.add(node);
+            component.add(node);
+            for (const a of node.ancestors) {
+              if (!visited.has(a) && this.txs.has(a)) {
+                stack.push(a);
+              }
+            }
+            for (const d of node.descendants) {
+              if (!visited.has(d) && this.txs.has(d)) {
+                stack.push(d);
+              }
+            }
+          }
+        }
+        components.push(component);
+      }
+    }
+    return components;
+  }
+
+}
+
+export function sortTopological(subset: Set<ClusterTx>): ClusterTx[] {
+  return [...subset].sort((a, b) => a.ancestors.size - b.ancestors.size);
+}
+
+export function subgraph(txSubset: Set<ClusterTx>): { depgraph: DepGraph; txMap: Map<ClusterTx, ClusterTx> } {
+  const newGraph = new DepGraph();
+  const txMap = new Map<ClusterTx, ClusterTx>();
+
+  for (const oldTx of txSubset) {
+    const newTx = newGraph.addTransaction(oldTx.txid, oldTx.effectiveFee, oldTx.weight, oldTx.order);
+    txMap.set(oldTx, newTx);
+  }
+
+  for (const oldTx of txSubset) {
+    for (const parent of oldTx.parents) {
+      if (txSubset.has(parent)) {
+        const newChild = txMap.get(oldTx);
+        const newParent = txMap.get(parent);
+        if (newChild && newParent) {
+          newGraph.addDependency(newParent, newChild);
+        }
+      }
+    }
+  }
+
+  return { depgraph: newGraph, txMap };
+}

backend/src/config.ts

@@ -6,7 +6,7 @@ interface IConfig {
   MEMPOOL: {
     ENABLED: boolean;
     OFFICIAL: boolean;
-    NETWORK: 'mainnet' | 'testnet' | 'signet' | 'liquid' | 'liquidtestnet';
+    NETWORK: 'mainnet' | 'testnet' | 'testnet4' | 'signet' | 'liquid' | 'liquidtestnet' | 'regtest';
     BACKEND: 'esplora' | 'electrum' | 'none';
     HTTP_PORT: number;
     UNIX_SOCKET_PATH: string;
@@ -34,6 +34,8 @@ interface IConfig {
     POOLS_JSON_TREE_URL: string,
     POOLS_UPDATE_DELAY: number,
     AUDIT: boolean;
+    CLUSTER_MEMPOOL: boolean;
+    CLUSTER_MEMPOOL_INDEXING: boolean;
     RUST_GBT: boolean;
     LIMIT_GBT: boolean;
     CPFP_INDEXING: boolean;
@@ -205,6 +207,8 @@ const defaults: IConfig = {
     'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
     'POOLS_UPDATE_DELAY': 604800, // in seconds, default is one week
     'AUDIT': false,
+    'CLUSTER_MEMPOOL': false,
+    'CLUSTER_MEMPOOL_INDEXING': false,
     'RUST_GBT': true,
     'LIMIT_GBT': false,
     'CPFP_INDEXING': false,
@@ -398,7 +402,7 @@ class Config implements IConfig {
       });
       return next;
     });
-  }
+  };
 }
 
 export default new Config();

backend/src/database.ts

@@ -25,6 +25,7 @@ import { execSync } from 'child_process';
     timezone: '+00:00',
   };
 
+  /** @asyncUnsafe */
   private checkDBFlag() {
     if (config.DATABASE.ENABLED === false) {
       const stack = new Error().stack;
@@ -32,6 +33,7 @@ import { execSync } from 'child_process';
     }
   }
 
+  /** @asyncUnsafe */
   public async query<T extends RowDataPacket[][] | RowDataPacket[] | OkPacket |
     OkPacket[] | ResultSetHeader>(query, params?, errorLogLevel: LogLevel | 'silent' = 'debug', connection?: PoolConnection): Promise<[T, FieldPacket[]]>
   {
@@ -76,6 +78,7 @@ import { execSync } from 'child_process';
     }
   }
 
+  /** @asyncSafe */
   private async $rollbackAtomic(connection: PoolConnection): Promise<void> {
     try {
       await connection.rollback();
@@ -85,6 +88,7 @@ import { execSync } from 'child_process';
     }
   }
 
+  /** @asyncSafe */
   public async $atomicQuery<T extends RowDataPacket[][] | RowDataPacket[] | OkPacket |
     OkPacket[] | ResultSetHeader>(queries: { query, params }[], errorLogLevel: LogLevel | 'silent' = 'debug'): Promise<[T, FieldPacket[]][]>
   {
@@ -116,6 +120,8 @@ import { execSync } from 'child_process';
     }
   }
 
+
+  /** @asyncSafe */
   public async checkDbConnection() {
     this.checkDBFlag();
     try {
@@ -171,10 +177,12 @@ import { execSync } from 'child_process';
     }
   }
 
+  /** @asyncSafe */
   private async getPool(): Promise<Pool> {
     if (this.pool === null) {
       this.pool = createPool(this.poolConfig);
       this.pool.on('connection', function (newConnection: PoolConnection) {
+        // eslint-disable-next-line @typescript-eslint/no-floating-promises -- callback API, not a promise despite types
         newConnection.query(`SET time_zone='+00:00'`);
       });
     }
@@ -188,7 +196,11 @@ import { execSync } from 'child_process';
    */
   public async close(): Promise<void> {
     if (this.pool !== null) {
-      await this.pool.end();
+      try {
+        await this.pool.end();
+      } catch (e) {
+        logger.err(`Exception in close. Reason: ${(e instanceof Error ? e.message : e)}`);
+      }
       this.pool = null;
       logger.debug('Database connection pool closed');
     }

backend/src/index.ts

@@ -1,5 +1,6 @@
 import express from 'express';
 import { Application, Request, Response, NextFunction } from 'express';
+import * as fs from 'fs';
 import * as http from 'http';
 import * as WebSocket from 'ws';
 import bitcoinApi from './api/bitcoin/bitcoin-api-factory';
@@ -67,8 +68,12 @@ class Server {
   constructor() {
     this.app = express();
 
+    if (cluster.isPrimary && config.MEMPOOL.UNIX_SOCKET_PATH) {
+      this.clearStaleUnixSocket(config.MEMPOOL.UNIX_SOCKET_PATH);
+    }
+
     if (!config.MEMPOOL.SPAWN_CLUSTER_PROCS) {
-      this.startServer();
+      void this.startServer();
       return;
     }
 
@@ -92,10 +97,11 @@ class Server {
         }, 10000);
       });
     } else {
-      this.startServer(true);
+      void this.startServer(true);
     }
   }
 
+  /** @asyncSafe */
   async startServer(worker = false): Promise<void> {
     logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
 
@@ -148,33 +154,42 @@ class Server {
       ;
 
     if (config.DATABASE.ENABLED && config.FIAT_PRICE.ENABLED) {
+      /** @asyncUnsafe */
       await priceUpdater.$initializeLatestPriceWithDb();
     }
 
     this.server = http.createServer(this.app);
-    this.wss = new WebSocket.Server({ server: this.server });
+    this.wss = new WebSocket.Server({ server: this.server, maxPayload: websocketHandler.MAX_MESSAGE_SIZE });
     if (config.MEMPOOL.UNIX_SOCKET_PATH) {
       this.serverUnixSocket = http.createServer(this.app);
-      this.wssUnixSocket = new WebSocket.Server({ server: this.serverUnixSocket });
+      this.wssUnixSocket = new WebSocket.Server({ server: this.serverUnixSocket, maxPayload: websocketHandler.MAX_MESSAGE_SIZE });
     }
 
     this.setUpWebsocketHandling();
 
     await poolsUpdater.updatePoolsJson(); // Needs to be done before loading the disk cache because we sometimes wipe it
-    if (config.DATABASE.ENABLED === true && config.MEMPOOL.ENABLED && ['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) && !poolsUpdater.currentSha) {
-      logger.err(`Failed to retreive pools-v2.json sha, cannot run block indexing. Please make sure you've set valid urls in your mempool-config.json::MEMPOOL::POOLS_JSON_URL and mempool-config.json::MEMPOOL::POOLS_JSON_TREE_UR, aborting now`);
+    if (config.DATABASE.ENABLED === true && config.MEMPOOL.ENABLED && ['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) && !poolsUpdater.currentSha) {
+      logger.err(`Failed to retrieve pools-v2.json sha, cannot run block indexing. Please make sure you've set valid urls in your mempool-config.json::MEMPOOL::POOLS_JSON_URL and mempool-config.json::MEMPOOL::POOLS_JSON_TREE_UR, aborting now`);
       return process.exit(1);
     }
 
     await syncAssets.syncAssets$();
     if (config.DATABASE.ENABLED) {
+      /** @asyncUnsafe */
       await mempoolBlocks.updatePools$();
     }
     if (config.MEMPOOL.ENABLED) {
       if (config.MEMPOOL.CACHE_ENABLED) {
         await diskCache.$loadMempoolCache();
       } else if (config.REDIS.ENABLED) {
-        await redisCache.$loadCache();
+        let currentMempoolTxids: Set<string> | undefined;
+        try {
+          currentMempoolTxids = new Set(await bitcoinApi.$getRawMempool());
+        } catch (e) {
+          logger.warn(`Failed to fetch raw mempool before loading Redis cache. Reason: ${e instanceof Error ? e.message : e}`);
+        }
+        /** @asyncUnsafe */
+        await redisCache.$loadCache(currentMempoolTxids);
       }
     }
 
@@ -197,20 +212,20 @@ class Server {
     }
 
     if (config.FIAT_PRICE.ENABLED) {
-      priceUpdater.$run();
+      void priceUpdater.$run();
     }
     await chainTips.updateOrphanedBlocks();
 
     this.setUpHttpApiRoutes();
 
     if (config.MEMPOOL.ENABLED) {
-      this.runMainUpdateLoop();
+      void this.runMainUpdateLoop();
     }
 
     setInterval(() => { this.healthCheck(); }, 2500);
 
     if (config.LIGHTNING.ENABLED) {
-      this.$runLightningBackend();
+      void this.$runLightningBackend();
     }
 
     this.server.listen(config.MEMPOOL.HTTP_PORT, () => {
@@ -220,6 +235,8 @@ class Server {
         logger.notice(`Mempool Server is running on port ${config.MEMPOOL.HTTP_PORT}`);
       }
     });
+    this.server.keepAliveTimeout = 70 * 1000;
+    this.server.headersTimeout = 71 * 1000;
 
     if (this.serverUnixSocket) {
       this.serverUnixSocket.listen(config.MEMPOOL.UNIX_SOCKET_PATH, () => {
@@ -229,11 +246,26 @@ class Server {
           logger.notice(`Mempool Server is listening on ${config.MEMPOOL.UNIX_SOCKET_PATH}`);
         }
       });
+
+      this.serverUnixSocket.keepAliveTimeout = 70 * 1000;
+      this.serverUnixSocket.headersTimeout = 71 * 1000;
     }
 
-    poolsUpdater.$startService();
+    void poolsUpdater.$startService();
   }
 
+  clearStaleUnixSocket(path: string): void {
+    try {
+      if (fs.existsSync(path)) {
+        fs.unlinkSync(path);
+        logger.notice(`Removed stale unix socket ${path}`);
+      }
+    } catch (e) {
+      logger.err(`Failed to remove stale unix socket ${path}. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  /** @asyncSafe */
   async runMainUpdateLoop(): Promise<void> {
     const start = Date.now();
     try {
@@ -256,13 +288,13 @@ class Server {
       if (numHandledBlocks === 0) {
         await memPool.$updateMempool(newMempool, latestAccelerations, minFeeMempool, minFeeTip, pollRate);
       }
-      indexer.$run();
+      void indexer.$run();
       if (config.WALLETS.ENABLED) {
         // might take a while, so run in the background
-        walletApi.$syncWallets();
+        void walletApi.$syncWallets();
       }
       if (config.FIAT_PRICE.ENABLED) {
-        priceUpdater.$run();
+        void priceUpdater.$run();
       }
 
       // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS
@@ -294,6 +326,7 @@ class Server {
     }
   }
 
+  /** @asyncSafe */
   async $runLightningBackend(): Promise<void> {
     try {
       await fundingTxFetcher.$init();
@@ -303,7 +336,7 @@ class Server {
     } catch(e) {
       logger.err(`Exception in $runLightningBackend. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
       await Common.sleep$(1000 * 60);
-      this.$runLightningBackend();
+      void this.$runLightningBackend();
     };
   }
 
@@ -319,7 +352,6 @@ class Server {
       blocks.setNewBlockCallback(async () => {
         try {
           await elementsParser.$parse();
-          await elementsParser.$updateFederationUtxos();
         } catch (e) {
           logger.warn('Elements parsing error: ' + (e instanceof Error ? e.message : e));
         }
@@ -329,16 +361,15 @@ class Server {
     if (config.MEMPOOL.ENABLED) {
       statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler));
       memPool.setAsyncMempoolChangedCallback(websocketHandler.$handleMempoolChange.bind(websocketHandler));
-      blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
     }
     if (config.FIAT_PRICE.ENABLED) {
       priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
     }
     loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler));
 
-    accelerationApi.connectWebsocket();
+    void accelerationApi.connectWebsocket();
     if (config.STRATUM.ENABLED) {
-      stratumApi.connectWebsocket();
+      void stratumApi.connectWebsocket();
     }
   }
 
@@ -390,6 +421,16 @@ class Server {
       this.warnedHeapCritical = false;
       this.maxHeapSize = 0;
       this.lastHeapLogTime = now;
+      this.server?.getConnections((error, count) => {
+        if (!error) {
+          logger.debug(`${count} open TCP sockets`);
+        }
+      });
+      this.serverUnixSocket?.getConnections((error, count) => {
+        if (!error) {
+          logger.debug(`${count} open unix sockets`);
+        }
+      });
     }
   }
 

backend/src/indexer.ts

@@ -35,6 +35,8 @@ class Indexer {
 
   /**
    * Check which core index is available for indexing
+   * 
+   * @asyncUnsafe
    */
   public async checkAvailableCoreIndexes(): Promise<void> {
     const updatedCoreIndexes: CoreIndex[] = [];
@@ -46,13 +48,13 @@ class Indexer {
         synced: indexes[indexName].synced,
         best_block_height: indexes[indexName].best_block_height,
       };
-      logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`);      
+      logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`);
       updatedCoreIndexes.push(newState);
 
       if (indexName === 'coinstatsindex' && newState.synced === true) {
         const previousState = this.isCoreIndexReady('coinstatsindex');
         // if (!previousState || previousState.synced === false) {
-          this.runSingleTask('coinStatsIndex');
+          void this.runSingleTask('coinStatsIndex');
         // }
       }
     }
@@ -62,9 +64,9 @@ class Indexer {
 
   /**
    * Return the best block height if a core index is available, or 0 if not
-   * 
-   * @param name 
-   * @returns 
+   *
+   * @param name
+   * @returns
    */
   public isCoreIndexReady(name: string): CoreIndex | null {
     for (const index of this.coreIndexes) {
@@ -111,12 +113,11 @@ class Indexer {
       }
     }
     this.tasksScheduled[task] = setTimeout(async () => {
+      delete this.tasksScheduled[task];
       try {
         await this.runSingleTask(task);
       } catch (e) {
         logger.err(`Unexpected error in scheduled task ${task}: ` + (e instanceof Error ? e.message : e));
-      } finally {
-        clearTimeout(this.tasksScheduled[task]);
       }
     }, timeout);
   }
@@ -125,6 +126,8 @@ class Indexer {
    * Runs a single task immediately
    *
    * (use `scheduleSingleTask` instead to queue a task to run after some timeout)
+   *
+   * @asyncSafe
    */
   public async runSingleTask(task: TaskName): Promise<void> {
     if (!Common.indexingEnabled() || this.tasksRunning[task]) {
@@ -134,13 +137,14 @@ class Indexer {
 
     switch (task) {
       case 'blocksPrices': {
-        if (!['testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) && config.FIAT_PRICE.ENABLED) {
-          let lastestPriceId;
+        if (!['testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) && config.FIAT_PRICE.ENABLED) {
+          let latestPriceId;
           try {
-            lastestPriceId = await PricesRepository.$getLatestPriceId();
+            latestPriceId = await PricesRepository.$getLatestPriceId();
           } catch (e) {
             logger.debug('failed to fetch latest price id from db: ' + (e instanceof Error ? e.message : e));
-          }          if (priceUpdater.historyInserted === false || lastestPriceId === null) {
+          }
+          if (priceUpdater.historyInserted === false || latestPriceId === null) {
             logger.debug(`Blocks prices indexer is waiting for the price updater to complete`, logger.tags.mining);
             this.scheduleSingleTask(task, 10000);
           } else {
@@ -163,6 +167,7 @@ class Indexer {
     this.tasksRunning[task] = false;
   }
 
+  /** @asyncSafe */
   public async $run(): Promise<void> {
     if (!Common.indexingEnabled() || this.runIndexer === false ||
       this.indexerRunning === true || mempool.hasPriority()
@@ -207,7 +212,7 @@ class Indexer {
         return;
       }
 
-      this.runSingleTask('blocksPrices');
+      void this.runSingleTask('blocksPrices');
       await blocks.$indexCoinbaseAddresses();
       await mining.$indexDifficultyAdjustments();
       await mining.$generateNetworkHashrateHistory();
@@ -215,13 +220,16 @@ class Indexer {
       await blocks.$generateBlocksSummariesDatabase();
       await blocks.$generateCPFPDatabase();
       await blocks.$generateAuditStats();
+      await blocks.$indexBlocksFirstSeen();
       await auditReplicator.$sync();
       await statisticsReplicator.$sync();
       await AccelerationRepository.$indexPastAccelerations();
       await BlocksAuditsRepository.$migrateAuditsV0toV1();
       await BlocksRepository.$migrateBlocks();
+
+      void blocks.$generateFlagValuesDatabase();
       // do not wait for classify blocks to finish
-      blocks.$classifyBlocks();
+      void blocks.$classifyBlocks();
       runSuccessful = true;
     } catch (e) {
       nextRunDelay = retryDelay;

backend/src/logger.ts

@@ -36,7 +36,7 @@ class Logger {
     mining: 'Mining',
     ln: 'Lightning',
     goggles: 'Goggles',
-  };  
+  };
 
   // @ts-ignore
   public emerg: ((msg: string, tag?: string) => void);
@@ -86,7 +86,7 @@ class Logger {
 
   private getNetwork(): string {
     if (config.LIGHTNING.ENABLED) {
-      return config.MEMPOOL.NETWORK === 'mainnet' ? 'lightning' : `${config.MEMPOOL.NETWORK}-lightning`; 
+      return config.MEMPOOL.NETWORK === 'mainnet' ? 'lightning' : `${config.MEMPOOL.NETWORK}-lightning`;
     }
     if (config.MEMPOOL.NETWORK && config.MEMPOOL.NETWORK !== 'mainnet') {
       return config.MEMPOOL.NETWORK;

backend/src/mempool.interfaces.ts

@@ -17,6 +17,7 @@ export interface PoolInfo {
   name: string;
   link: string;
   blockCount: number;
+  emptyBlocks: number;
   slug: string;
   avgMatchRate: number | null;
   avgFeeDelta: number | null;
@@ -25,7 +26,11 @@ export interface PoolInfo {
 
 export interface PoolStats extends PoolInfo {
   rank: number;
-  emptyBlocks: number;
+}
+
+export enum TemplateAlgorithm {
+  legacy = 0,
+  clusterMempool = 1,
 }
 
 export interface BlockAudit {
@@ -45,6 +50,7 @@ export interface BlockAudit {
   expectedFees?: number,
   expectedWeight?: number,
   template?: any[];
+  templateAlgorithm?: TemplateAlgorithm,
 }
 
 export interface TransactionAudit {
@@ -132,6 +138,8 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
   replacement?: boolean;
   uid?: number;
   flags?: number;
+  clusterId?: number;
+  chunkIndex?: number;
 }
 
 export interface MempoolTransactionExtended extends TransactionExtended {
@@ -227,6 +235,7 @@ export interface CpfpInfo {
   adjustedVsize?: number,
   acceleration?: boolean,
   fee?: number;
+  cluster?: CpfpClusterData & { chunkIndex: number };
 }
 
 export interface TransactionStripped {
@@ -388,6 +397,25 @@ export interface CpfpCluster {
   height: number,
   txs: Ancestor[],
   effectiveFeePerVsize: number,
+  templateAlgorithm?: TemplateAlgorithm,
+  clusterData?: CpfpClusterData,
+}
+
+export interface CpfpClusterTx {
+  txid: string;
+  fee: number;
+  weight: number;
+  parents: number[];
+}
+
+export interface CpfpClusterChunk {
+  txs: number[];
+  feerate: number;
+}
+
+export interface CpfpClusterData {
+  txs: CpfpClusterTx[];
+  chunks: CpfpClusterChunk[];
 }
 
 export interface CpfpSummary {
@@ -504,9 +532,36 @@ export interface IBackendInfo {
   gitCommit: string;
   version: string;
   lightning: boolean;
+  coreVersion: string;
+  osVersion: string;
   backend: 'esplora' | 'electrum' | 'none';
 }
 
+export interface INetworkInfo {
+    version: number;
+    subversion: string;
+    protocolversion: number;
+    localservices: string;
+    localrelay: boolean;
+    timeoffset: number;
+    networkactive: boolean;
+    networks: {
+      name: string;
+      limited: boolean;
+      reachable: boolean;
+      proxy: string;
+      proxy_randomize_credentials: boolean;
+    }[];
+    relayfee: number;
+    incrementalfee: number;
+    localaddresses: {
+      address: string;
+      port: number;
+      score: number;
+    }[];
+    warnings: string;
+}
+
 export interface IDifficultyAdjustment {
   progressPercent: number;
   difficultyChange: number;

backend/src/replication/AuditReplication.ts

@@ -17,6 +17,7 @@ class AuditReplication {
   inProgress: boolean = false;
   skip: Set<string> = new Set();
 
+  /** @asyncUnsafe */
   public async $sync(): Promise<void> {
     if (!config.REPLICATION.ENABLED || !config.REPLICATION.AUDIT) {
       // replication not enabled
@@ -54,6 +55,7 @@ class AuditReplication {
     this.inProgress = false;
   }
 
+  /** @asyncUnsafe */
   private async $syncAudit(hash: string): Promise<boolean> {
     if (this.skip.has(hash)) {
       // we already know none of our trusted servers have this audit
@@ -77,6 +79,8 @@ class AuditReplication {
     return success;
   }
 
+  
+  /** @asyncSafe */
   private async $getMissingAuditBlocks(): Promise<string[]> {
     try {
       const startHeight = config.REPLICATION.AUDIT_START_HEIGHT || 0;
@@ -110,6 +114,7 @@ class AuditReplication {
     });
     await blocksAuditsRepository.$saveAudit({
       version: auditSummary.version || 0,
+      templateAlgorithm: auditSummary.templateAlgorithm ?? 0,
       hash: blockHash,
       height: auditSummary.height,
       time: auditSummary.timestamp || auditSummary.time,

backend/src/replication/StatisticsReplication.ts

@@ -31,6 +31,7 @@ const steps = {
 class StatisticsReplication {
   inProgress: boolean = false;
 
+  /** @asyncUnsafe */
   public async $sync(): Promise<void> {
     if (!config.REPLICATION.ENABLED || !config.REPLICATION.STATISTICS || !config.STATISTICS.ENABLED) {
       // replication not enabled, or statistics not enabled
@@ -51,12 +52,12 @@ class StatisticsReplication {
       logger.info(`Statistics table is complete, no replication needed`, 'Replication');
       return;
     }
-    
+
     for (const interval of missingIntervals) {
       logger.debug(`Missing ${missingStatistics[interval].size} statistics rows in '${interval}' timespan`, 'Replication');
     }
     logger.debug(`Fetching ${missingIntervals.join(', ')} statistics endpoints from trusted servers to fill ${totalMissing} rows missing in statistics`, 'Replication');
-    
+
     let totalSynced = 0;
     let totalMissed = 0;
 
@@ -74,16 +75,17 @@ class StatisticsReplication {
     this.inProgress = false;
   }
 
+  /** @asyncUnsafe */
   private async $syncStatistics(interval: string, missingTimes: Set<number>): Promise<any> {
-  
+
     let success = false;
     let synced = 0;
-    let missed = new Set(missingTimes);
+    const missed = new Set(missingTimes);
     const syncResult = await $sync(`/api/v1/statistics/${interval}`);
     if (syncResult && syncResult.data?.length) {
       success = true;
-      logger.info(`Fetched /api/v1/statistics/${interval} from ${syncResult.server}`);     
-    
+      logger.info(`Fetched /api/v1/statistics/${interval} from ${syncResult.server}`);
+
       for (const stat of syncResult.data) {
         const time = this.roundToNearestStep(stat.added, steps[interval]);
         if (missingTimes.has(time)) {
@@ -105,6 +107,8 @@ class StatisticsReplication {
     return { success, synced, missed: missed.size };
   }
 
+  
+  /** @asyncUnsafe */
   private async $getMissingStatistics(): Promise<MissingStatistics> {
     try {
       const now = Math.floor(Date.now() / 1000);
@@ -129,7 +133,7 @@ class StatisticsReplication {
         startTime < now - day * 30 ?      [now - day * 90,      now - day * 30,      '3m' ] : null, // from 3 months ago to 1 month ago = 2 hours granularity
         startTime < now - day * 90 ?      [now - day * 180,     now - day * 90,      '6m' ] : null, // from 6 months ago to 3 months ago = 3 hours granularity
         startTime < now - day * 180 ?     [now - day * 365 * 2, now - day * 180,     '2y' ] : null, // from 2 years ago to 6 months ago = 8 hours granularity
-        startTime < now - day * 365 * 2 ? [startTime,           now - day * 365 * 2, 'all'] : null, // from start of statistics to 2 years ago = 12 hours granularity   
+        startTime < now - day * 365 * 2 ? [startTime,           now - day * 365 * 2, 'all'] : null, // from start of statistics to 2 years ago = 12 hours granularity
       ];
 
       for (const interval of intervals) {
@@ -138,7 +142,7 @@ class StatisticsReplication {
         }
         missingStatistics[interval[2] as string] = await this.$getMissingStatisticsInterval(interval, startTime);
       }
-      
+
       return missingStatistics;
     } catch (e: any) {
       logger.err(`Cannot fetch missing statistics times from db. Reason: ` + (e instanceof Error ? e.message : e));
@@ -146,6 +150,7 @@ class StatisticsReplication {
     }
   }
 
+  /** @asyncUnsafe */
   private async $getMissingStatisticsInterval(interval: any, startTime: number): Promise<Set<number>> {
     try {
       const start = interval[0];
@@ -169,17 +174,17 @@ class StatisticsReplication {
       if (timeSteps.length === 0) {
         return new Set<number>();
       }
-      
+
       const roundedTimesAlreadyHere: number[] = Array.from(new Set(rows.map(row => this.roundToNearestStep(row.added, step))));
 
       const missingTimes = timeSteps.filter(time => !roundedTimesAlreadyHere.includes(time)).filter((time, i, arr) => {
         // Remove outsiders
         if (i === 0) {
-          return arr[i + 1] === time + step
+          return arr[i + 1] === time + step;
         } else if (i === arr.length - 1) {
           return arr[i - 1] === time - step;
         }
-        return (arr[i + 1] === time + step) && (arr[i - 1] === time - step)
+        return (arr[i + 1] === time + step) && (arr[i - 1] === time - step);
       });
 
       // Don't bother fetching if very few rows are missing

backend/src/replication/replicator.ts

@@ -4,6 +4,7 @@ import axios, { AxiosResponse } from 'axios';
 import { SocksProxyAgent } from 'socks-proxy-agent';
 import * as https from 'https';
 
+/** @asyncSafe */
 export async function $sync(path): Promise<{ data?: any, exists: boolean, server?: string }> {
   // start with a random server so load is uniformly spread
   let allMissing = true;
@@ -14,7 +15,7 @@ export async function $sync(path): Promise<{ data?: any, exists: boolean, server
     if (server === backendInfo.getBackendInfo().hostname) {
       continue;
     }
-    
+
     try {
       const result = await query(`https://${server}${path}`);
       if (result) {
@@ -33,6 +34,7 @@ export async function $sync(path): Promise<{ data?: any, exists: boolean, server
   return { exists: !allMissing };
 }
 
+/** @asyncUnsafe */
 export async function query(path): Promise<object> {
   type axiosOptions = {
     headers: {

backend/src/repositories/AccelerationRepository.ts

@@ -31,6 +31,7 @@ export interface PublicAcceleration {
 class AccelerationRepository {
   private bidBoostV2Activated = 831580;
 
+  /** @asyncSafe */
   public async $saveAcceleration(acceleration: AccelerationInfo, block: IEsploraApi.Block, pool_id: number, accelerationData: Acceleration[]): Promise<void> {
     const accelerationMap: { [txid: string]: Acceleration } = {};
     for (const acc of accelerationData) {
@@ -60,28 +61,34 @@ class AccelerationRepository {
     }
   }
 
+  /** @asyncSafe */
   public async $getAccelerationInfoForTxid(txid: string): Promise<PublicAcceleration | null> {
-    const [rows] = await DB.query(`
-      SELECT *, UNIX_TIMESTAMP(requested) as requested_timestamp, UNIX_TIMESTAMP(added) as block_timestamp FROM accelerations
-      JOIN pools on pools.unique_id = accelerations.pool
-      WHERE txid = ?
-    `, [txid]) as RowDataPacket[][];
-    if (rows?.length) {
-      const row = rows[0];
-      return {
-        txid: row.txid,
-        height: row.height,
-        added: row.requested_timestamp || row.block_timestamp,
-        pool: {
-          id: row.id,
-          slug: row.slug,
-          name: row.name,
-        },
-        effective_vsize: row.effective_vsize,
-        effective_fee: row.effective_fee,
-        boost_rate: row.boost_rate,
-        boost_cost: row.boost_cost,
-      };
+    try {
+      const [rows] = await DB.query(`
+        SELECT *, UNIX_TIMESTAMP(requested) as requested_timestamp, UNIX_TIMESTAMP(added) as block_timestamp FROM accelerations
+        JOIN pools on pools.unique_id = accelerations.pool
+        WHERE txid = ?
+      `, [txid]) as RowDataPacket[][];
+      if (rows?.length) {
+        const row = rows[0];
+        return {
+          txid: row.txid,
+          height: row.height,
+          added: row.requested_timestamp || row.block_timestamp,
+          pool: {
+            id: row.id,
+            slug: row.slug,
+            name: row.name,
+          },
+          effective_vsize: row.effective_vsize,
+          effective_fee: row.effective_fee,
+          boost_rate: row.boost_rate,
+          boost_cost: row.boost_cost,
+        };
+      }
+    } catch (e: any) {
+      logger.err(`Cannot get acceleration info for txid ${txid}. Reason: ` + (e instanceof Error ? e.message : e));
+      return null;
     }
     return null;
   }
@@ -100,7 +107,7 @@ class AccelerationRepository {
       SELECT *, UNIX_TIMESTAMP(requested) as requested_timestamp, UNIX_TIMESTAMP(added) as block_timestamp FROM accelerations
       JOIN pools on pools.unique_id = accelerations.pool
     `;
-    let params: any[] = [];
+    const params: any[] = [];
     let hasFilter = false;
 
     if (interval && height === null) {
@@ -163,7 +170,7 @@ class AccelerationRepository {
       SELECT SUM(boost_cost) as total_cost, COUNT(txid) as count FROM accelerations
       JOIN pools on pools.unique_id = accelerations.pool
     `;
-    let params: any[] = [];
+    const params: any[] = [];
     let hasFilter = false;
 
     if (interval) {
@@ -191,6 +198,7 @@ class AccelerationRepository {
     }
   }
 
+  /** @asyncSafe */
   public async $getLastSyncedHeight(): Promise<number> {
     try {
       const [rows] = await DB.query(`
@@ -206,6 +214,7 @@ class AccelerationRepository {
     return 0;
   }
 
+  /** @asyncSafe */
   private async $setLastSyncedHeight(height: number): Promise<void> {
     try {
       await DB.query(`
@@ -219,6 +228,7 @@ class AccelerationRepository {
   }
 
   // modifies block transactions
+  /** @asyncSafe */
   public async $indexAccelerationsForBlock(block: BlockExtended, accelerations: Acceleration[], transactions: MempoolTransactionExtended[]): Promise<void> {
     const blockTxs: { [txid: string]: MempoolTransactionExtended } = {};
     for (const tx of transactions) {
@@ -237,7 +247,7 @@ class AccelerationRepository {
         const tx = blockTxs[acc.txid];
         const accelerationInfo = accelerationCosts.getAccelerationInfo(tx, boostRate, transactions);
         accelerationInfo.cost = Math.max(0, Math.min(acc.feeDelta, accelerationInfo.cost));
-        this.$saveAcceleration(accelerationInfo, block, block.extras.pool.id, successfulAccelerations);
+        void this.$saveAcceleration(accelerationInfo, block, block.extras.pool.id, successfulAccelerations);
       }
     }
     let anyConfirmed = false;
@@ -346,7 +356,7 @@ class AccelerationRepository {
         const accelerationSummaries = accelerations.map(acc => ({
           ...acc,
           pools: acc.pools,
-        }))
+        }));
         for (const acc of accelerations) {
           if (blockTxs[acc.txid] && acc.pools.includes(block.extras.pool.id)) {
             const tx = blockTxs[acc.txid];