Merge pull request #6693 from mempool/jramos/x-viewing

fix: viewing link in x app

.github/dependabot.yml

@@ -6,6 +6,11 @@ updates:
     schedule:
       interval: daily
     open-pull-requests-limit: 10
+    cooldown:
+      default-days: 7
+      semver-major-days: 30
+      semver-minor-days: 14
+      semver-patch-days: 7
     ignore:
       - dependency-name: "*"
         update-types:
@@ -33,6 +38,11 @@ updates:
     schedule:
       interval: daily
     open-pull-requests-limit: 10
+    cooldown:
+      default-days: 7
+      semver-major-days: 30
+      semver-minor-days: 14
+      semver-patch-days: 7
     ignore:
       - dependency-name: "*"
         update-types:
@@ -44,6 +54,8 @@ updates:
     directory: "/docker/backend"
     schedule:
       interval: weekly
+    cooldown:
+      default-days: 7
     ignore:
       - dependency-name: "*"
         update-types:
@@ -53,6 +65,8 @@ updates:
     directory: "/docker/frontend"
     schedule:
       interval: weekly
+    cooldown:
+      default-days: 7
     ignore:
       - dependency-name: "*"
         update-types:
@@ -62,6 +76,8 @@ updates:
     directory: "/"
     schedule:
       interval: weekly
+    cooldown:
+      default-days: 7
     ignore:
       - dependency-name: "*"
         update-types:

.github/nginx-check/Dockerfile.docker

@@ -0,0 +1,17 @@
+FROM nginx:1.30.1-alpine
+
+# Validate the config set loaded by the mempool/frontend docker image.
+# Build context: repo root
+COPY nginx.conf http-basic.conf /etc/nginx/
+COPY nginx-mempool.conf /etc/nginx/conf.d/
+
+# Replay the transforms applied by docker/init.sh (with the default values
+# docker/frontend/entrypoint.sh substitutes at container start)
+RUN sed -i \
+      -e "s!127.0.0.1:80!0.0.0.0:8080!g" \
+      -e "s!127.0.0.1!0.0.0.0!g" \
+      -e "s!user nobody;!!g" \
+      -e "s!/etc/nginx/nginx-mempool.conf!/etc/nginx/conf.d/nginx-mempool.conf!g" \
+      /etc/nginx/nginx.conf
+
+CMD ["nginx", "-t"]

.github/nginx-check/Dockerfile.production

@@ -0,0 +1,13 @@
+FROM nginx:1.30.1-alpine
+
+# Stage the config the way production/install does: the repo is available at
+# ${NGINX_ETC_FOLDER}/mempool and production/nginx/nginx.conf becomes the main
+# config, with __NGINX_USER__ and __NGINX_ETC_FOLDER__ substituted.
+# Build context: production/
+COPY nginx /etc/nginx/mempool/production/nginx
+
+RUN cp /etc/nginx/mempool/production/nginx/nginx.conf /etc/nginx/nginx.conf && \
+    sed -i "s!__NGINX_USER__!nginx!" /etc/nginx/nginx.conf && \
+    sed -i "s!__NGINX_ETC_FOLDER__!/etc/nginx!" /etc/nginx/nginx.conf
+
+CMD ["nginx", "-t"]

.github/workflows/backend-integration.yml

@@ -2,11 +2,14 @@ name: Backend Integration Tests with MariaDB
 
 on:
   pull_request:
-    types: [opened, review_requested, synchronize]
+    types: [opened, synchronize]
   push:
     branches:
       - master
 
+permissions:
+  contents: read
+
 jobs:
   backend-integration:
     if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
@@ -20,12 +23,12 @@ jobs:
     
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: ${{ matrix.node }}/integration
 
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@3235b876344d2a9aa001b8d1453c930bba69e610 # v3
         with:
           node-version: ${{ matrix.node }}
           registry-url: "https://registry.npmjs.org"
@@ -33,7 +36,7 @@ jobs:
           cache-dependency-path: '${{ matrix.node }}/integration/backend/package-lock.json'
 
       - name: Cache node modules
-        uses: actions/cache@v4
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: ${{ matrix.node }}/integration/backend/node_modules
           key: ${{ runner.os }}-backend-integration-node-${{ matrix.node }}-${{ hashFiles('${{ matrix.node }}/integration/backend/package-lock.json') }}
@@ -47,7 +50,7 @@ jobs:
         working-directory: ${{ matrix.node }}/integration
 
       - name: Cache Rust dependencies
-        uses: actions/cache@v4
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             ~/.cargo/bin/
@@ -66,7 +69,7 @@ jobs:
           toolchain: ${{ steps.gettoolchain.outputs.toolchain }}
 
       - name: Install dependencies
-        run: npm ci
+        run: bash meta/scripts/check-install-scripts.sh && npm ci
         working-directory: ${{ matrix.node }}/integration/backend
 
       - name: Build backend

.github/workflows/ci.yml

@@ -2,11 +2,14 @@ name: CI Pipeline for the Backend and Frontend
 
 on:
   pull_request:
-    types: [opened, review_requested, synchronize]
+    types: [opened, synchronize]
   push:
     branches:
       - master
 
+permissions:
+  contents: read
+
 jobs:
   backend:
     if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
@@ -20,20 +23,23 @@ jobs:
     name: Backend (${{ matrix.flavor }}) - node ${{ matrix.node }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: ${{ matrix.node }}/${{ matrix.flavor }}
 
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
         with:
           node-version: ${{ matrix.node }}
           registry-url: "https://registry.npmjs.org"
           cache: 'npm'
           cache-dependency-path: '${{ matrix.node }}/${{ matrix.flavor }}/backend/package-lock.json'
 
+      - name: Install npm 11.12.0
+        run: npm install -g npm@11.12.0
+
       - name: Cache node modules
-        uses: actions/cache@v4
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: ${{ matrix.node }}/${{ matrix.flavor }}/backend/node_modules
           key: ${{ runner.os }}-backend-${{ matrix.flavor }}-node-${{ matrix.node }}-${{ hashFiles('${{ matrix.node }}/${{ matrix.flavor }}/backend/package-lock.json') }}
@@ -47,7 +53,7 @@ jobs:
         working-directory: ${{ matrix.node }}/${{ matrix.flavor }}
 
       - name: Cache Rust dependencies
-        uses: actions/cache@v4
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             ~/.cargo/bin/
@@ -69,12 +75,12 @@ jobs:
 
       - name: Install
         if: ${{ matrix.flavor == 'dev'}}
-        run: npm ci
+        run: bash meta/scripts/check-install-scripts.sh && npm ci
         working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
 
       - name: Install (Prod dependencies only)
         if: ${{ matrix.flavor == 'prod'}}
-        run: npm ci --omit=dev --omit=optional
+        run: bash meta/scripts/check-install-scripts.sh && npm ci --omit=dev --omit=optional
         working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
 
       - name: Lint
@@ -100,20 +106,23 @@ jobs:
     runs-on: mempool-ci
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: assets
 
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
         with:
           node-version: ${{ matrix.node }}
           registry-url: "https://registry.npmjs.org"
           cache: 'npm'
           cache-dependency-path: 'assets/frontend/package-lock.json'
 
+      - name: Install npm 11.12.0
+        run: npm install -g npm@11.12.0
+
       - name: Cache node modules for frontend
-        uses: actions/cache@v4
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: assets/frontend/node_modules
           key: ${{ runner.os }}-cache-frontend-node-${{ matrix.node }}-${{ hashFiles('assets/frontend/package-lock.json') }}
@@ -122,13 +131,13 @@ jobs:
             ${{ runner.os }}-cache-frontend-
 
       - name: Install (Prod dependencies only)
-        run: npm ci --omit=dev --omit=optional
+        run: bash meta/scripts/check-install-scripts.sh && npm ci --omit=dev --omit=optional
         working-directory: assets/frontend
 
       - name: Restore cached mining pool assets
         continue-on-error: true
         id: cache-mining-pool-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             mining-pool-assets.zip
@@ -137,7 +146,7 @@ jobs:
       - name: Restore promo video assets
         continue-on-error: true
         id: cache-promo-video-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             promo-video-assets.zip
@@ -170,20 +179,20 @@ jobs:
         run: zip -jrq promo-video-assets.zip assets/frontend/src/resources/promo-video/*
 
       - name: Upload mining pool assets as artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: mining-pool-assets
           path: mining-pool-assets.zip
 
       - name: Upload promo video assets as artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: promo-video-assets
           path: promo-video-assets.zip
 
       - name: Save mining pool assets cache
         id: cache-mining-pool-save
-        uses: actions/cache/save@v4
+        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             mining-pool-assets.zip
@@ -191,7 +200,7 @@ jobs:
 
       - name: Save promo video assets cache
         id: cache-promo-video-save
-        uses: actions/cache/save@v4
+        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             promo-video-assets.zip
@@ -210,20 +219,23 @@ jobs:
     name: Frontend (${{ matrix.flavor }}) - node ${{ matrix.node }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: ${{ matrix.node }}/${{ matrix.flavor }}
 
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
         with:
           node-version: ${{ matrix.node }}
           registry-url: "https://registry.npmjs.org"
           cache: 'npm'
           cache-dependency-path: '${{ matrix.node }}/${{ matrix.flavor }}/frontend/package-lock.json'
 
+      - name: Install npm 11.12.0
+        run: npm install -g npm@11.12.0
+
       - name: Cache node modules
-        uses: actions/cache@v4
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: ${{ matrix.node }}/${{ matrix.flavor }}/frontend/node_modules
           key: ${{ runner.os }}-frontend-${{ matrix.flavor }}-node-${{ matrix.node }}-${{ hashFiles('${{ matrix.node }}/${{ matrix.flavor }}/frontend/package-lock.json') }}
@@ -232,13 +244,13 @@ jobs:
             ${{ runner.os }}-frontend-${{ matrix.flavor }}-
 
       - name: Install (Prod dependencies only)
-        run: npm ci --omit=dev --omit=optional
+        run: bash meta/scripts/check-install-scripts.sh && npm ci --omit=dev --omit=optional
         if: ${{ matrix.flavor == 'prod'}}
         working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
 
       - name: Install
         if: ${{ matrix.flavor == 'dev'}}
-        run: npm ci
+        run: bash meta/scripts/check-install-scripts.sh && npm ci
         working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
 
       - name: Lint
@@ -252,7 +264,7 @@ jobs:
       - name: Restore cached mining pool assets
         continue-on-error: true
         id: cache-mining-pool-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             mining-pool-assets.zip
@@ -261,14 +273,14 @@ jobs:
       - name: Restore promo video assets
         continue-on-error: true
         id: cache-promo-video-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             promo-video-assets.zip
           key: promo-video-assets-cache
 
       - name: Download artifact
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
           name: mining-pool-assets
 
@@ -276,7 +288,7 @@ jobs:
         run: unzip -o mining-pool-assets.zip -d ${{ matrix.node }}/${{ matrix.flavor }}/frontend/src/resources/mining-pools
 
       - name: Download artifact
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
           name: promo-video-assets
 
@@ -310,19 +322,22 @@ jobs:
     name: E2E tests for ${{ matrix.module }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: ${{ matrix.module }}
 
       - name: Setup node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
         with:
           node-version: ${{ matrix.node }}
           cache: "npm"
           cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json
 
+      - name: Install npm 11.12.0
+        run: npm install -g npm@11.12.0
+
       - name: Cache node modules for e2e
-        uses: actions/cache@v4
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: ${{ matrix.module }}/frontend/node_modules
           key: ${{ runner.os }}-e2e-${{ matrix.module }}-node-${{ matrix.node }}-${{ hashFiles('${{ matrix.module }}/frontend/package-lock.json') }}
@@ -333,7 +348,7 @@ jobs:
       - name: Restore cached mining pool assets
         continue-on-error: true
         id: cache-mining-pool-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             mining-pool-assets.zip
@@ -342,14 +357,14 @@ jobs:
       - name: Restore cached promo video assets
         continue-on-error: true
         id: cache-promo-video-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             promo-video-assets.zip
           key: promo-video-assets-cache
 
       - name: Download artifact
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
           name: mining-pool-assets
 
@@ -357,7 +372,7 @@ jobs:
         run: unzip -o mining-pool-assets.zip -d ${{ matrix.module }}/frontend/src/resources/mining-pools
 
       - name: Download artifact
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
           name: promo-video-assets
 
@@ -367,7 +382,7 @@ jobs:
       # mempool
       - name: Chrome browser tests (${{ matrix.module }})
         if: ${{ matrix.module == 'mempool' }}
-        uses: cypress-io/github-action@v5
+        uses: cypress-io/github-action@248bde77443c376edc45906ede03a1aba9da0462 # v5
         with:
           tag: ${{ github.event_name }}
           working-directory: ${{ matrix.module }}/frontend
@@ -392,7 +407,7 @@ jobs:
       # liquid
       - name: Chrome browser tests (${{ matrix.module }})
         if: ${{ matrix.module == 'liquid' }}
-        uses: cypress-io/github-action@v5
+        uses: cypress-io/github-action@248bde77443c376edc45906ede03a1aba9da0462 # v5
         with:
           tag: ${{ github.event_name }}
           working-directory: ${{ matrix.module }}/frontend
@@ -417,7 +432,7 @@ jobs:
       # testnet
       - name: Chrome browser tests (${{ matrix.module }})
         if: ${{ matrix.module == 'testnet4' }}
-        uses: cypress-io/github-action@v5
+        uses: cypress-io/github-action@248bde77443c376edc45906ede03a1aba9da0462 # v5
         with:
           tag: ${{ github.event_name }}
           working-directory: ${{ matrix.module }}/frontend
@@ -445,7 +460,7 @@ jobs:
 
     steps: 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: docker
       

.github/workflows/dependabot-provenance-check.yml

@@ -0,0 +1,102 @@
+name: Dependabot Provenance Check
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions:
+  contents: read
+  pull-requests: write
+  issues: write
+
+jobs:
+  check-provenance:
+    if: >-
+      github.event.pull_request.user.login == 'dependabot[bot]' &&
+      !contains(github.event.pull_request.labels.*.name, 'provenance-exception')
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - name: Fetch Dependabot metadata
+        id: metadata
+        uses: dependabot/fetch-metadata@21025c705c08248db411dc16f3619e6b5f9ea21a # v2
+        with:
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+
+      - name: Setup Node
+        if: steps.metadata.outputs.package-ecosystem == 'npm_and_yarn'
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: "24.13.0"
+
+      - name: Check npm provenance attestation
+        if: steps.metadata.outputs.package-ecosystem == 'npm_and_yarn'
+        id: npm-provenance
+        run: |
+          IFS=',' read -ra DEPS <<< "${{ steps.metadata.outputs.dependency-names }}"
+          UPDATED_DEPS_JSON='${{ steps.metadata.outputs.updated-dependencies-json }}'
+
+          FAILED_DEPS=""
+          for DEP in "${DEPS[@]}"; do
+            DEP=$(echo "$DEP" | xargs)
+
+            NEW_VERSION=$(echo "$UPDATED_DEPS_JSON" | jq -r --arg name "$DEP" '
+              .[] | select(.dependencyName == $name) | .newVersion
+            ' | head -n 1)
+
+            if [ -z "$NEW_VERSION" ] || [ "$NEW_VERSION" = "null" ]; then
+              echo "::error::Could not determine updated version for dependency $DEP from Dependabot metadata"
+              FAILED_DEPS="${FAILED_DEPS:+$FAILED_DEPS, }$DEP"
+              continue
+            fi
+            echo "::group::Checking provenance for $DEP@$NEW_VERSION"
+
+            ATTESTATIONS=$(npm view "$DEP@$NEW_VERSION" --json 2>/dev/null | jq -r '.dist.attestations // empty')
+
+            if [ -z "$ATTESTATIONS" ]; then
+              echo "::error::No provenance attestation found for $DEP@$NEW_VERSION"
+              FAILED_DEPS="${FAILED_DEPS:+$FAILED_DEPS, }$DEP"
+            else
+              echo "Provenance attestation found for $DEP@$NEW_VERSION"
+            fi
+            echo "::endgroup::"
+          done
+
+          if [ -n "$FAILED_DEPS" ]; then
+            echo "failed=true" >> "$GITHUB_OUTPUT"
+            echo "failed-deps=$FAILED_DEPS" >> "$GITHUB_OUTPUT"
+          else
+            echo "failed=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Close PR if provenance check failed
+        if: steps.npm-provenance.outputs.failed == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          FAILED_DEPS: ${{ steps.npm-provenance.outputs.failed-deps }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          COMMENT_BODY_FILE="$(mktemp)"
+          cat > "$COMMENT_BODY_FILE" <<EOF
+          ## Provenance Attestation Check Failed
+          
+          This pull request has been automatically closed because the following dependencies do not have [provenance attestation](https://docs.npmjs.com/generating-provenance-statements):
+          
+          **Failed:** $FAILED_DEPS
+          
+          ### What is provenance attestation?
+          
+          Provenance attestation cryptographically proves that a package was built from a specific source repository using a specific build process. This helps protect against supply chain attacks by verifying the link between published packages and their source code.
+          
+          ### What to do
+          
+          - Check if a newer version of the dependency publishes provenance
+          - Contact the package maintainer to request [npm provenance](https://docs.npmjs.com/generating-provenance-statements) support
+          - If this dependency is trusted and an exception is warranted, a maintainer can reopen this PR and add the \`provenance-exception\` label
+          EOF
+          
+          gh pr comment "$PR_NUMBER" --body-file "$COMMENT_BODY_FILE"
+          gh pr close "$PR_NUMBER"

.github/workflows/docker.yml

@@ -24,7 +24,7 @@ jobs:
     name: Test built Docker images
     steps:
       - name: Checkout project
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
 
       - name: Add SHORT_SHA env property with commit short sha
         run: |
@@ -59,7 +59,7 @@ jobs:
         run: docker/init.sh "$TAG"
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
 
       - name: Build frontend image locally
         run: |
@@ -288,13 +288,13 @@ jobs:
 
 
       - name: Login to Docker for building
-        uses: docker/login-action@v3
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Checkout project
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
 
       # For PRs: use package.json version + short sha as TAG
       - name: Set TAG from service package.json for pull requests
@@ -319,13 +319,13 @@ jobs:
         run: docker/init.sh "$TAG"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
         with:
           platforms: linux/amd64,linux/arm64
         id: qemu
 
       - name: Setup Docker buildx action
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
         with:
           platforms: linux/amd64,linux/arm64
           driver-opts: |
@@ -336,7 +336,7 @@ jobs:
         run: echo ${{ steps.buildx.outputs.platforms }}
 
       - name: Cache Docker layers
-        uses: actions/cache@v3
+        uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3
         id: cache
         with:
           path: /tmp/.buildx-cache
@@ -375,19 +375,19 @@ jobs:
         run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
         with:
           platforms: linux/amd64,linux/arm64
 
       - name: Setup Docker buildx action
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
         with:
           platforms: linux/amd64,linux/arm64
           driver-opts: |
             network=host
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/e2e_parameterized.yml

@@ -17,7 +17,10 @@ on:
         description: 'Liquid Hostname'
         required: true
         default: 'liquid.network'
-        type: string 
+        type: string
+
+permissions:
+  contents: read
 
 jobs:
   cache:
@@ -35,25 +38,25 @@ jobs:
           fi
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           ref: ${{ steps.determine-ref.outputs.ref }}
           path: assets
 
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@3235b876344d2a9aa001b8d1453c930bba69e610 # v3
         with:
           node-version: 24.13.0
           registry-url: "https://registry.npmjs.org"
 
       - name: Install (Prod dependencies only)
-        run: npm ci --omit=dev --omit=optional
+        run: bash meta/scripts/check-install-scripts.sh && npm ci --omit=dev --omit=optional
         working-directory: assets/frontend
 
       - name: Restore cached mining pool assets
         continue-on-error: true
         id: cache-mining-pool-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             mining-pool-assets.zip
@@ -62,7 +65,7 @@ jobs:
       - name: Restore promo video assets
         continue-on-error: true
         id: cache-promo-video-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             promo-video-assets.zip
@@ -95,20 +98,20 @@ jobs:
         run: zip -jrq promo-video-assets.zip assets/frontend/src/resources/promo-video/*
 
       - name: Upload mining pool assets as artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: mining-pool-assets
           path: mining-pool-assets.zip
 
       - name: Upload promo video assets as artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: promo-video-assets
           path: promo-video-assets.zip
 
       - name: Save mining pool assets cache
         id: cache-mining-pool-save
-        uses: actions/cache/save@v4
+        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             mining-pool-assets.zip
@@ -116,7 +119,7 @@ jobs:
 
       - name: Save promo video assets cache
         id: cache-promo-video-save
-        uses: actions/cache/save@v4
+        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             promo-video-assets.zip
@@ -143,13 +146,13 @@ jobs:
           fi
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           ref: ${{ steps.determine-ref.outputs.ref }}
           path: ${{ matrix.module }}
 
       - name: Setup node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@3235b876344d2a9aa001b8d1453c930bba69e610 # v3
         with:
           node-version: 24.13.0
           cache: "npm"
@@ -158,7 +161,7 @@ jobs:
       - name: Restore cached mining pool assets
         continue-on-error: true
         id: cache-mining-pool-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             mining-pool-assets.zip
@@ -167,14 +170,14 @@ jobs:
       - name: Restore cached promo video assets
         continue-on-error: true
         id: cache-promo-video-restore
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
             promo-video-assets.zip
           key: promo-video-assets-cache
 
       - name: Download artifact
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
           name: mining-pool-assets
 
@@ -182,7 +185,7 @@ jobs:
         run: unzip -o mining-pool-assets.zip -d ${{ matrix.module }}/frontend/src/resources/mining-pools
 
       - name: Download artifact
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
           name: promo-video-assets
 
@@ -192,7 +195,7 @@ jobs:
     # mempool
       - name: Chrome browser tests (${{ matrix.module }})
         if: ${{ matrix.module == 'mempool' }}
-        uses: cypress-io/github-action@v5
+        uses: cypress-io/github-action@248bde77443c376edc45906ede03a1aba9da0462 # v5
         with:
           tag: ${{ github.event_name }}
           working-directory: ${{ matrix.module }}/frontend
@@ -220,7 +223,7 @@ jobs:
       # liquid
       - name: Chrome browser tests (${{ matrix.module }})
         if: ${{ matrix.module == 'liquid' }}
-        uses: cypress-io/github-action@v5
+        uses: cypress-io/github-action@248bde77443c376edc45906ede03a1aba9da0462 # v5
         with:
           tag: ${{ github.event_name }}
           working-directory: ${{ matrix.module }}/frontend
@@ -248,7 +251,7 @@ jobs:
       # testnet
       - name: Chrome browser tests (${{ matrix.module }})
         if: ${{ matrix.module == 'testnet4' }}
-        uses: cypress-io/github-action@v5
+        uses: cypress-io/github-action@248bde77443c376edc45906ede03a1aba9da0462 # v5
         with:
           tag: ${{ github.event_name }}
           working-directory: ${{ matrix.module }}/frontend

.github/workflows/get_backend_block_height.yml

@@ -2,13 +2,16 @@ name: 'Check if servers are in sync'
 
 on: [workflow_dispatch]
 
+permissions:
+  contents: read
+
 jobs:
   print-backend-sha:
     runs-on: mempool-ci
     name: Get block height
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: repo
 

.github/workflows/get_backend_hash.yml

@@ -2,13 +2,16 @@ name: 'Print backend hashes'
 
 on: [workflow_dispatch]
 
+permissions:
+  contents: read
+
 jobs:
   print-backend-sha:
     runs-on: mempool-ci
     name: Print backend hashes
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: repo
 

.github/workflows/get_image_digest.yml

@@ -7,14 +7,18 @@ on:
         description: 'Image Version'
         required: false
         default: 'latest'
-        type: string 
+        type: string
+
+permissions:
+  contents: read
+
 jobs:
   print-images-sha:
     runs-on: mempool-ci
     name: Print digest for images
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           path: digest
 

.github/workflows/project-review-status.yml

@@ -1,131 +1,21 @@
-# Workflow: Automate project board management
-# - Add newly created issues to project #8
-# - Set status to "Review Needed" when a reviewer is requested on a non-draft PR
 name: Project Board Automation
 
-# Triggers: Review requested on PRs, or new issues opened
 on:
   pull_request:
     types: [review_requested]
   issues:
     types: [opened]
 
+permissions:
+  contents: read
+
 jobs:
-  manage-project-board:
-    runs-on: mempool-ci
-    steps:
-      - name: Update Project Board
-        uses: actions/github-script@v7
-        with:
-          # Use the PAT stored in repository secrets (has project write access)
-          github-token: ${{ secrets.PROJECT_TOKEN }}
-          script: |
-            // Skip draft PRs
-            if (context.eventName === 'pull_request' && context.payload.pull_request.draft) {
-              console.log('PR is a draft, skipping Review Needed status...');
-              return;
-            }
-
-            // Handle new issues - add to project
-            if (context.eventName === 'issues') {
-              const addMutation = `
-                mutation($projectId: ID!, $contentId: ID!) {
-                  addProjectV2ItemById(input: {projectId: $projectId, contentId: $contentId}) {
-                    item { id }
-                  }
-                }
-              `;
-
-              try {
-                await github.graphql(addMutation, {
-                  projectId: "${{ secrets.PROJECT_ID }}",
-                  contentId: context.payload.issue.node_id
-                });
-
-                console.log('Successfully added issue to project #8');
-              } catch (error) {
-                // Handle case where the issue is already in the project, or log other failures
-                const errors = error && error.errors ? error.errors : [];
-                const alreadyInProject = errors.some(e =>
-                  typeof e.message === 'string' &&
-                  e.message.toLowerCase().includes('already') &&
-                  e.message.toLowerCase().includes('project')
-                );
-
-                if (alreadyInProject) {
-                  console.log('Issue is already in project #8, skipping add.');
-                } else {
-                  console.error('Failed to add issue to project #8:', error);
-                  throw error;
-                }
-              }
-              return;
-            }
-
-            // Handle PR review_requested - update status to "Review Needed"
-            // GraphQL query to find the PR's project items
-            // This fetches all projects the PR is linked to
-            const query = `
-              query($owner: String!, $repo: String!, $pr: Int!) {
-                repository(owner: $owner, name: $repo) {
-                  pullRequest(number: $pr) {
-                    projectItems(first: 10) {
-                      nodes {
-                        id
-                        project {
-                          number
-                        }
-                      }
-                    }
-                  }
-                }
-              }
-            `;
-
-            // Execute the query with current repo/PR context
-            const result = await github.graphql(query, {
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              pr: context.payload.pull_request.number
-            });
-
-            // Find the project item that belongs to project #8
-            const projectItems = result.repository.pullRequest.projectItems.nodes;
-            const projectItem = projectItems.find(item => item.project.number === 8);
-
-            // Exit early if PR isn't in project #8
-            if (!projectItem) {
-              console.log('PR is not in project #8, skipping...');
-              return;
-            }
-
-            // GraphQL mutation to update the Status field
-            const mutation = `
-              mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!) {
-                updateProjectV2ItemFieldValue(
-                  input: {
-                    projectId: $projectId
-                    itemId: $itemId
-                    fieldId: $fieldId
-                    value: { singleSelectOptionId: $optionId }
-                  }
-                ) {
-                  projectV2Item {
-                    id
-                  }
-                }
-              }
-            `;
-
-            // Execute the mutation using IDs stored in repository variables
-            // PROJECT_ID: The project's unique identifier
-            // STATUS_FIELD_ID: The "Status" field's unique identifier
-            // REVIEW_NEEDED_OPTION_ID: The "Review Needed" option's unique identifier
-            await github.graphql(mutation, {
-              projectId: "${{ secrets.PROJECT_ID }}",
-              itemId: projectItem.id,
-              fieldId: "${{ secrets.STATUS_FIELD_ID }}",
-              optionId: "${{ secrets.REVIEW_NEEDED_OPTION_ID }}"
-            });
-
-            console.log('Successfully updated project status to Review Needed');
+  project-automation:
+    uses: mempool/.github/.github/workflows/project-board-automation.yml@267e0f38846bfb8cf1b118798dfb12def19ccb35 # master
+    with:
+      project-number: 8
+    secrets:
+      PROJECT_TOKEN: ${{ secrets.PROJECT_TOKEN }}
+      PROJECT_ID: ${{ secrets.PROJECT_ID }}
+      STATUS_FIELD_ID: ${{ secrets.STATUS_FIELD_ID }}
+      REVIEW_NEEDED_OPTION_ID: ${{ secrets.REVIEW_NEEDED_OPTION_ID }}

.github/workflows/server-config.yml

@@ -0,0 +1,32 @@
+name: Validate server config
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+      - "nginx.conf"
+      - "http-basic.conf"
+      - "nginx-mempool.conf"
+      - "production/nginx/**"
+      - ".github/nginx-check/**"
+      - ".github/workflows/server-config.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  check_config:
+    runs-on: "ubuntu-latest"
+
+    name: Validate nginx config
+    steps:
+      - name: Checkout
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
+
+      - name: Validate the production nginx config
+        run: |
+          docker run --rm $(docker build -q -f .github/nginx-check/Dockerfile.production production)
+
+      - name: Validate the docker frontend nginx config
+        run: |
+          docker run --rm $(docker build -q -f .github/nginx-check/Dockerfile.docker .)

.github/workflows/supply-chain-audit.yml

@@ -0,0 +1,58 @@
+name: Supply Chain Audit
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+  push:
+    branches:
+      - master
+
+permissions:
+  contents: read
+
+jobs:
+  backend:
+    if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
+    runs-on: mempool-ci
+    name: Backend install-script audit
+    steps:
+      - name: Checkout
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
+
+      - name: Setup Node
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: "24.13.0"
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Install npm 11.12.0
+        run: npm install -g npm@11.12.0
+
+      - name: Audit backend install scripts
+        run: bash backend/meta/scripts/check-install-scripts.sh
+
+      - name: Run backend safe install
+        run: bash backend/meta/scripts/safe-install.sh
+
+  frontend:
+    if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
+    runs-on: mempool-ci
+    name: Frontend install-script audit
+    steps:
+      - name: Checkout
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
+
+      - name: Setup Node
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: "24.13.0"
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Install npm 11.12.0
+        run: npm install -g npm@11.12.0
+
+      - name: Audit frontend install scripts
+        run: bash frontend/meta/scripts/check-install-scripts.sh
+
+      - name: Run frontend safe install
+        run: bash frontend/meta/scripts/safe-install.sh

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 # Contributing to The Mempool Open Source Project
 
-Thank you for contributing to The Mempool Open Source Project managed by Mempool Space K.K. (“Mempool”).
+Thank you for contributing to The Mempool Open Source Project managed by Mempool Holdings S.A. de C.V. in El Salvador (“Mempool”).
 
 In order to clarify the intellectual property license granted with Contributions from any person or entity, Mempool must have a statement on file from each Contributor indicating their agreement to the Contributor License Agreement (“Agreement”). This license is for your protection as a Contributor as well as the protection of Mempool and its other contributors and users; it does not change your rights to use your own Contributions for any other purpose.
 

LICENSE

@@ -1,5 +1,5 @@
 The Mempool Open Source Project®
-Copyright (c) 2019-2025 Mempool Space K.K. and other shadowy super-coders
+Copyright (c) 2019-2026 Mempool Holdings S.A. de C.V. and other shadowy super-coders
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU Affero General Public License as published by the Free
@@ -7,17 +7,17 @@ Software Foundation, either version 3 of the License or any later version
 approved by a proxy statement published on <https://mempool.space/about>.
 
 However, this copyright license does not include an implied right or license
-to use any trademarks, service marks, logos, or trade names of Mempool Space K.K.
+to use any trademarks, service marks, logos, or trade names of Mempool Holdings
 or any other contributor to The Mempool Open Source Project.
 
 The Mempool Open Source Project®, Mempool Accelerator®, Mempool Enterprise®, 
 Mempool Wallet™, mempool.space®, Be your own explorer™, Explore the full 
-Bitcoin ecosystem™, Mempool Goggles™, the mempool Logo, the mempool Square Logo, 
+Bitcoin ecosystem™, Mempool Goggles®, the mempool Logo, the mempool Square Logo, 
 the mempool block visualization Logo, the mempool Blocks Logo, the mempool 
 transaction Logo, the mempool Blocks 3 | 2 Logo, the mempool research Logo, 
 the mempool.space Vertical Logo, and the mempool.space Horizontal Logo are 
-registered trademarks or trademarks of Mempool Space K.K in Japan, 
-the United States, and/or other countries.
+registered trademarks or trademarks of Mempool Holdings S.A. de C.V. in El
+Salvador, Japan, the United States, and/or other countries.
 
 See our full Trademark Policy and Guidelines for more details, published on 
 <https://mempool.space/trademark-policy>.

backend/.eslintrc.js

@@ -15,7 +15,7 @@ module.exports = {
     "plugin:@typescript-eslint/recommended",
     "prettier"
   ],
-  "ignorePatterns": ["dist", "eslint-local-rules", ".eslintrc.js", "testSetup*.ts", "jest.integration.*.ts", "__tests__", "*.config.ts"],
+  "ignorePatterns": ["dist", "eslint-local-rules", ".eslintrc.js", "testSetup*.ts", "jest.integration.*.ts", "__tests__", "__e2e__", "*.config.ts"],
   "overrides": [
     {
       "files": ["src/__integration_tests__/**/*"],

backend/.npmrc

@@ -0,0 +1 @@
+min-release-age=7

backend/jest.config.ts

@@ -17,8 +17,10 @@ const config: Config.InitialOptions = {
     './testSetup.ts',
   ],
   testPathIgnorePatterns: [
+    '/dist/',
     '/node_modules/',
     '/__integration_tests__/',
+    'test-utils\\.ts$',
   ],
 };
 export default config;

backend/mempool-config.sample.json

@@ -31,6 +31,8 @@
     "AUDIT": false,
     "RUST_GBT": true,
     "LIMIT_GBT": false,
+    "CLUSTER_MEMPOOL": false,
+    "CLUSTER_MEMPOOL_INDEXING": false,
     "CPFP_INDEXING": false,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,

backend/mempool-config.test.json

@@ -31,6 +31,8 @@
     "AUDIT": false,
     "RUST_GBT": true,
     "LIMIT_GBT": false,
+    "CLUSTER_MEMPOOL": false,
+    "CLUSTER_MEMPOOL_INDEXING": false,
     "CPFP_INDEXING": false,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,

backend/meta/scripts/check-install-scripts.sh

@@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+#
+# Audit backend install-script sources before running a full npm install.
+# Exits non-zero if any package not on the whitelist has hasInstallScript: true.
+#
+# Usage (from repo root):
+#   backend/meta/scripts/check-install-scripts.sh
+#
+
+set -euo pipefail
+
+BACKEND_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+LOCKFILE="${BACKEND_DIR}/package-lock.json"
+RUST_GBT_DIR="${BACKEND_DIR}/../rust/gbt"
+TMP_AUDIT_DIR=""
+
+BACKEND_ALLOWED=(
+  "mempool-backend"
+  "fsevents"
+  "unrs-resolver"
+)
+
+RUST_GBT_ALLOWED=()
+
+is_allowed() {
+  local pkg="$1"
+  shift
+  local allowed
+  for allowed in "$@"; do
+    if [[ "$pkg" == "$allowed" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+cleanup() {
+  if [[ -n "${TMP_AUDIT_DIR}" && -d "${TMP_AUDIT_DIR}" ]]; then
+    rm -rf "${TMP_AUDIT_DIR}"
+  fi
+}
+
+trap cleanup EXIT
+
+collect_has_install_script_packages() {
+  local lockfile="$1"
+  if [[ ! -f "$lockfile" ]]; then
+    echo "No package-lock.json found at ${lockfile}"
+    return 1
+  fi
+
+  LOCKFILE_PATH="$lockfile" node - <<'NODE'
+const fs = require('fs');
+
+const lock = JSON.parse(fs.readFileSync(process.env.LOCKFILE_PATH, 'utf8'));
+const pkgs = lock.packages || {};
+
+for (const [path, meta] of Object.entries(pkgs)) {
+  if (meta && meta.hasInstallScript) {
+    const name = path === '' ? (lock.name || '(root)') : path.replace(/^.*node_modules\//, '');
+    console.log(name);
+  }
+}
+NODE
+}
+
+audit_lockfile() {
+  local lockfile="$1"
+  local label="$2"
+  shift 2
+  local allowed=("$@")
+  local found_violations=0
+  local pkg
+
+  while IFS= read -r pkg; do
+    [[ -z "$pkg" ]] && continue
+    if ! is_allowed "$pkg" "${allowed[@]}"; then
+      echo "VIOLATION: unauthorized install script in '${pkg}' (${label}: ${lockfile})"
+      found_violations=1
+    fi
+  done < <(collect_has_install_script_packages "$lockfile")
+
+  if [[ "$found_violations" -eq 1 ]]; then
+    echo ""
+    echo "FAILED: Found packages with install scripts not on the whitelist."
+    return 1
+  fi
+
+  echo "OK: All ${label} install scripts are from whitelisted packages."
+}
+
+get_rust_gbt_napi_cli_version() {
+  local rust_gbt_package_json="${RUST_GBT_DIR}/package.json"
+  if [[ ! -f "$rust_gbt_package_json" ]]; then
+    echo "No rust/gbt package.json found at ${rust_gbt_package_json}"
+    return 1
+  fi
+
+  RUST_GBT_PACKAGE_JSON="$rust_gbt_package_json" node - <<'NODE'
+const fs = require('fs');
+
+const pkg = JSON.parse(fs.readFileSync(process.env.RUST_GBT_PACKAGE_JSON, 'utf8'));
+const buildScript = (pkg.scripts && pkg.scripts.build) || '';
+const match = buildScript.match(/npm install --no-save @napi-rs\/cli@([^\s]+)/);
+
+if (!match) {
+  console.error('Could not determine the @napi-rs/cli version from rust/gbt/package.json.');
+  process.exit(1);
+}
+
+process.stdout.write(match[1]);
+NODE
+}
+
+generate_rust_gbt_audit_lockfile() {
+  local napi_cli_version="$1"
+
+  TMP_AUDIT_DIR="$(mktemp -d "${TMPDIR:-/tmp}/backend-rust-gbt-audit.XXXXXX")"
+  cat > "${TMP_AUDIT_DIR}/package.json" <<EOF
+{
+  "name": "rust-gbt-install-audit",
+  "private": true,
+  "dependencies": {
+    "@napi-rs/cli": "${napi_cli_version}"
+  }
+}
+EOF
+
+  (
+    cd "${TMP_AUDIT_DIR}"
+    npm install --package-lock-only --ignore-scripts --no-audit --no-fund >/dev/null
+  )
+
+  printf '%s\n' "${TMP_AUDIT_DIR}/package-lock.json"
+}
+
+audit_rust_gbt_preinstall_chain() {
+  local napi_cli_version
+  local rust_gbt_lockfile
+
+  napi_cli_version="$(get_rust_gbt_napi_cli_version)"
+  rust_gbt_lockfile="$(generate_rust_gbt_audit_lockfile "${napi_cli_version}")"
+  if [[ "${#RUST_GBT_ALLOWED[@]}" -gt 0 ]]; then
+    audit_lockfile "${rust_gbt_lockfile}" "backend preinstall toolchain" "${RUST_GBT_ALLOWED[@]}"
+  else
+    audit_lockfile "${rust_gbt_lockfile}" "backend preinstall toolchain"
+  fi
+}
+
+audit_lockfile "${LOCKFILE}" "backend lockfile" "${BACKEND_ALLOWED[@]}"
+audit_rust_gbt_preinstall_chain
+
+echo "OK: Backend install-script audit passed."

backend/meta/scripts/safe-install.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+#
+# Safely install backend npm dependencies by refreshing the lockfile without
+# running install scripts, auditing it, then doing the real install.
+#
+# Usage (from repo root):
+#   backend/meta/scripts/safe-install.sh
+#
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BACKEND_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+LOCKFILE="${BACKEND_DIR}/package-lock.json"
+RESTORE_LOCKFILE_DONE=0
+
+# Back up the lockfile so we can restore on failure
+if [[ -f "$LOCKFILE" ]]; then
+  cp "$LOCKFILE" "${LOCKFILE}.bak"
+fi
+
+restore_lockfile() {
+  trap - ERR INT TERM
+  if [[ "${RESTORE_LOCKFILE_DONE}" -eq 1 ]]; then
+    return
+  fi
+  RESTORE_LOCKFILE_DONE=1
+
+  if [[ -f "${LOCKFILE}.bak" ]]; then
+    mv "${LOCKFILE}.bak" "$LOCKFILE"
+    echo "Restored original package-lock.json."
+  elif [[ -f "$LOCKFILE" ]]; then
+    rm "$LOCKFILE"
+    echo "Removed generated package-lock.json."
+  fi
+}
+
+trap restore_lockfile ERR INT TERM
+
+echo "==> Refreshing backend lockfile (--ignore-scripts --package-lock-only)..."
+(cd "$BACKEND_DIR" && npm install --ignore-scripts --package-lock-only --no-audit --no-fund)
+
+echo ""
+echo "==> Auditing lockfile for install scripts..."
+bash "${SCRIPT_DIR}/check-install-scripts.sh"
+
+trap - ERR INT TERM
+rm -f "${LOCKFILE}.bak"
+
+echo ""
+echo "==> Installing backend (npm ci)..."
+(cd "$BACKEND_DIR" && npm ci)
+
+echo ""
+echo "Done."

backend/package-lock.json

@@ -1,28 +1,28 @@
 {
   "name": "mempool-backend",
-  "version": "3.3-dev",
+  "version": "3.4-dev",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "mempool-backend",
-      "version": "3.3-dev",
+      "version": "3.4-dev",
       "hasInstallScript": true,
       "license": "GNU Affero General Public License v3.0",
       "dependencies": {
         "@mempool/electrum-client": "1.1.9",
         "@types/node": "^18.15.3",
-        "axios": "1.13.5",
+        "axios": "1.16.1",
         "bitcoinjs-lib": "~6.1.3",
         "crypto-js": "~4.2.0",
         "express": "~4.22.1",
         "maxmind": "~4.3.11",
-        "mysql2": "~3.17.1",
+        "mysql2": "~3.22.4",
         "redis": "^4.7.0",
         "rust-gbt": "file:./rust-gbt",
         "socks-proxy-agent": "~7.0.0",
         "typescript": "~4.9.3",
-        "ws": "~8.19.0"
+        "ws": "~8.21.0"
       },
       "devDependencies": {
         "@types/compression": "^1.7.2",
@@ -749,29 +749,6 @@
       "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==",
       "dev": true
     },
-    "node_modules/@isaacs/balanced-match": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz",
-      "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": "20 || >=22"
-      }
-    },
-    "node_modules/@isaacs/brace-expansion": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz",
-      "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@isaacs/balanced-match": "^4.0.1"
-      },
-      "engines": {
-        "node": "20 || >=22"
-      }
-    },
     "node_modules/@isaacs/cliui": {
       "version": "8.0.2",
       "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
@@ -2401,10 +2378,11 @@
       }
     },
     "node_modules/ajv": {
-      "version": "6.12.6",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
-      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+      "version": "6.14.0",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz",
+      "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "fast-deep-equal": "^3.1.1",
         "fast-json-stable-stringify": "^2.0.0",
@@ -2511,13 +2489,14 @@
       }
     },
     "node_modules/axios": {
-      "version": "1.13.5",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.5.tgz",
-      "integrity": "sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==",
+      "version": "1.16.1",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.16.1.tgz",
+      "integrity": "sha512-caYkukvroVPO8KrzuJEb50Hm07KwfBZPEC3VeFHTsqWHvKTsy54hjJz9BS/cdaypROE2rH6xvm9mHX4fgWkr3A==",
       "dependencies": {
-        "follow-redirects": "^1.15.11",
+        "follow-redirects": "^1.16.0",
         "form-data": "^4.0.5",
-        "proxy-from-env": "^1.1.0"
+        "https-proxy-agent": "^5.0.1",
+        "proxy-from-env": "^2.1.0"
       }
     },
     "node_modules/babel-jest": {
@@ -3199,10 +3178,11 @@
       }
     },
     "node_modules/diff": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
-      "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.4.tgz",
+      "integrity": "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ==",
       "dev": true,
+      "license": "BSD-3-Clause",
       "engines": {
         "node": ">=0.3.1"
       }
@@ -3874,15 +3854,16 @@
       }
     },
     "node_modules/flatted": {
-      "version": "3.2.7",
-      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz",
-      "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==",
-      "dev": true
+      "version": "3.4.2",
+      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz",
+      "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==",
+      "dev": true,
+      "license": "ISC"
     },
     "node_modules/follow-redirects": {
-      "version": "1.15.11",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz",
-      "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==",
+      "version": "1.16.0",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz",
+      "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==",
       "funding": [
         {
           "type": "individual",
@@ -4111,17 +4092,40 @@
         "node": ">=10.13.0"
       }
     },
+    "node_modules/glob/node_modules/balanced-match": {
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz",
+      "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": "18 || 20 || >=22"
+      }
+    },
+    "node_modules/glob/node_modules/brace-expansion": {
+      "version": "5.0.4",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz",
+      "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "balanced-match": "^4.0.2"
+      },
+      "engines": {
+        "node": "18 || 20 || >=22"
+      }
+    },
     "node_modules/glob/node_modules/minimatch": {
-      "version": "10.1.1",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz",
-      "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==",
+      "version": "10.2.4",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz",
+      "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==",
       "dev": true,
       "license": "BlueOak-1.0.0",
       "dependencies": {
-        "@isaacs/brace-expansion": "^5.0.0"
+        "brace-expansion": "^5.0.2"
       },
       "engines": {
-        "node": "20 || >=22"
+        "node": "18 || 20 || >=22"
       },
       "funding": {
         "url": "https://github.com/sponsors/isaacs"
@@ -4260,6 +4264,18 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/https-proxy-agent": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
+      "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
+      "dependencies": {
+        "agent-base": "6",
+        "debug": "4"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/human-signals": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz",
@@ -5475,10 +5491,11 @@
       }
     },
     "node_modules/minimatch": {
-      "version": "3.1.2",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
-      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "version": "3.1.5",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz",
+      "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==",
       "dev": true,
+      "license": "ISC",
       "dependencies": {
         "brace-expansion": "^1.1.7"
       },
@@ -5521,22 +5538,24 @@
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
     "node_modules/mysql2": {
-      "version": "3.17.1",
-      "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.17.1.tgz",
-      "integrity": "sha512-UzIzdVwPXPoZm+FaJ4lNsRt28HtUwt68gpLH7NP1oSjd91M5Qn1XJzbIsSRMRc5CV3pvktLNshmbaFfMYqPBhQ==",
+      "version": "3.22.4",
+      "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.22.4.tgz",
+      "integrity": "sha512-CtXYlmL7ZamiYKbmqkamQHWJROUHSfm+f3kByzGfknw7kW51mcB2ouMUqYq1XfYxbXmnWo6RhPydx6OCqdgcmQ==",
       "dependencies": {
         "aws-ssl-profiles": "^1.1.2",
         "denque": "^2.1.0",
         "generate-function": "^2.3.1",
         "iconv-lite": "^0.7.2",
         "long": "^5.3.2",
-        "lru.min": "^1.1.3",
+        "lru.min": "^1.1.4",
         "named-placeholders": "^1.1.6",
-        "seq-queue": "^0.0.5",
-        "sql-escaper": "^1.3.2"
+        "sql-escaper": "^1.3.3"
       },
       "engines": {
         "node": ">= 8.0"
+      },
+      "peerDependencies": {
+        "@types/node": ">= 8"
       }
     },
     "node_modules/mysql2/node_modules/iconv-lite": {
@@ -6009,9 +6028,12 @@
       }
     },
     "node_modules/proxy-from-env": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
-      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz",
+      "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==",
+      "engines": {
+        "node": ">=10"
+      }
     },
     "node_modules/punycode": {
       "version": "2.3.0",
@@ -6321,11 +6343,6 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
     },
-    "node_modules/seq-queue": {
-      "version": "0.0.5",
-      "resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz",
-      "integrity": "sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q=="
-    },
     "node_modules/serve-static": {
       "version": "1.16.2",
       "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz",
@@ -6507,9 +6524,9 @@
       }
     },
     "node_modules/sql-escaper": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/sql-escaper/-/sql-escaper-1.3.2.tgz",
-      "integrity": "sha512-lp+ZDVfSjHt+qAK1jXBTIXBNYnbo7gnaAGwoYTH9bE89kNkXwcu6g0WjJGRsdTKVpY1z70u3Y0IgmnBOoRybHw==",
+      "version": "1.3.3",
+      "resolved": "https://registry.npmjs.org/sql-escaper/-/sql-escaper-1.3.3.tgz",
+      "integrity": "sha512-BsTCV265VpTp8tm1wyIm1xqQCS+Q9NHx2Sr+WcnUrgLrQ6yiDIvHYJV5gHxsj1lMBy2zm5twLaZao8Jd+S8JJw==",
       "engines": {
         "bun": ">=1.0.0",
         "deno": ">=2.0.0",
@@ -7176,9 +7193,9 @@
       }
     },
     "node_modules/ws": {
-      "version": "8.19.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz",
-      "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==",
+      "version": "8.21.0",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz",
+      "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==",
       "engines": {
         "node": ">=10.0.0"
       },
@@ -7764,21 +7781,6 @@
       "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==",
       "dev": true
     },
-    "@isaacs/balanced-match": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz",
-      "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==",
-      "dev": true
-    },
-    "@isaacs/brace-expansion": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz",
-      "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==",
-      "dev": true,
-      "requires": {
-        "@isaacs/balanced-match": "^4.0.1"
-      }
-    },
     "@isaacs/cliui": {
       "version": "8.0.2",
       "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
@@ -8935,9 +8937,9 @@
       }
     },
     "ajv": {
-      "version": "6.12.6",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
-      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+      "version": "6.14.0",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz",
+      "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==",
       "dev": true,
       "requires": {
         "fast-deep-equal": "^3.1.1",
@@ -9014,13 +9016,14 @@
       "integrity": "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g=="
     },
     "axios": {
-      "version": "1.13.5",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.5.tgz",
-      "integrity": "sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==",
+      "version": "1.16.1",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.16.1.tgz",
+      "integrity": "sha512-caYkukvroVPO8KrzuJEb50Hm07KwfBZPEC3VeFHTsqWHvKTsy54hjJz9BS/cdaypROE2rH6xvm9mHX4fgWkr3A==",
       "requires": {
-        "follow-redirects": "^1.15.11",
+        "follow-redirects": "^1.16.0",
         "form-data": "^4.0.5",
-        "proxy-from-env": "^1.1.0"
+        "https-proxy-agent": "^5.0.1",
+        "proxy-from-env": "^2.1.0"
       }
     },
     "babel-jest": {
@@ -9488,9 +9491,9 @@
       "dev": true
     },
     "diff": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
-      "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.4.tgz",
+      "integrity": "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ==",
       "dev": true
     },
     "dir-glob": {
@@ -10006,15 +10009,15 @@
       }
     },
     "flatted": {
-      "version": "3.2.7",
-      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz",
-      "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==",
+      "version": "3.4.2",
+      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz",
+      "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==",
       "dev": true
     },
     "follow-redirects": {
-      "version": "1.15.11",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz",
-      "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ=="
+      "version": "1.16.0",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz",
+      "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw=="
     },
     "foreground-child": {
       "version": "3.3.1",
@@ -10145,13 +10148,28 @@
         "path-scurry": "^2.0.0"
       },
       "dependencies": {
-        "minimatch": {
-          "version": "10.1.1",
-          "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz",
-          "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==",
+        "balanced-match": {
+          "version": "4.0.4",
+          "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz",
+          "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==",
+          "dev": true
+        },
+        "brace-expansion": {
+          "version": "5.0.4",
+          "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz",
+          "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==",
           "dev": true,
           "requires": {
-            "@isaacs/brace-expansion": "^5.0.0"
+            "balanced-match": "^4.0.2"
+          }
+        },
+        "minimatch": {
+          "version": "10.2.4",
+          "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz",
+          "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==",
+          "dev": true,
+          "requires": {
+            "brace-expansion": "^5.0.2"
           }
         }
       }
@@ -10254,6 +10272,15 @@
         "toidentifier": "1.0.1"
       }
     },
+    "https-proxy-agent": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
+      "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
+      "requires": {
+        "agent-base": "6",
+        "debug": "4"
+      }
+    },
     "human-signals": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz",
@@ -11105,9 +11132,9 @@
       "dev": true
     },
     "minimatch": {
-      "version": "3.1.2",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
-      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "version": "3.1.5",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz",
+      "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==",
       "dev": true,
       "requires": {
         "brace-expansion": "^1.1.7"
@@ -11136,19 +11163,18 @@
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
     "mysql2": {
-      "version": "3.17.1",
-      "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.17.1.tgz",
-      "integrity": "sha512-UzIzdVwPXPoZm+FaJ4lNsRt28HtUwt68gpLH7NP1oSjd91M5Qn1XJzbIsSRMRc5CV3pvktLNshmbaFfMYqPBhQ==",
+      "version": "3.22.4",
+      "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.22.4.tgz",
+      "integrity": "sha512-CtXYlmL7ZamiYKbmqkamQHWJROUHSfm+f3kByzGfknw7kW51mcB2ouMUqYq1XfYxbXmnWo6RhPydx6OCqdgcmQ==",
       "requires": {
         "aws-ssl-profiles": "^1.1.2",
         "denque": "^2.1.0",
         "generate-function": "^2.3.1",
         "iconv-lite": "^0.7.2",
         "long": "^5.3.2",
-        "lru.min": "^1.1.3",
+        "lru.min": "^1.1.4",
         "named-placeholders": "^1.1.6",
-        "seq-queue": "^0.0.5",
-        "sql-escaper": "^1.3.2"
+        "sql-escaper": "^1.3.3"
       },
       "dependencies": {
         "iconv-lite": {
@@ -11465,9 +11491,9 @@
       }
     },
     "proxy-from-env": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
-      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz",
+      "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA=="
     },
     "punycode": {
       "version": "2.3.0",
@@ -11668,11 +11694,6 @@
         }
       }
     },
-    "seq-queue": {
-      "version": "0.0.5",
-      "resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz",
-      "integrity": "sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q=="
-    },
     "serve-static": {
       "version": "1.16.2",
       "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz",
@@ -11801,9 +11822,9 @@
       }
     },
     "sql-escaper": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/sql-escaper/-/sql-escaper-1.3.2.tgz",
-      "integrity": "sha512-lp+ZDVfSjHt+qAK1jXBTIXBNYnbo7gnaAGwoYTH9bE89kNkXwcu6g0WjJGRsdTKVpY1z70u3Y0IgmnBOoRybHw=="
+      "version": "1.3.3",
+      "resolved": "https://registry.npmjs.org/sql-escaper/-/sql-escaper-1.3.3.tgz",
+      "integrity": "sha512-BsTCV265VpTp8tm1wyIm1xqQCS+Q9NHx2Sr+WcnUrgLrQ6yiDIvHYJV5gHxsj1lMBy2zm5twLaZao8Jd+S8JJw=="
     },
     "stack-utils": {
       "version": "2.0.6",
@@ -12213,9 +12234,9 @@
       }
     },
     "ws": {
-      "version": "8.19.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz",
-      "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==",
+      "version": "8.21.0",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz",
+      "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==",
       "requires": {}
     },
     "y18n": {

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-backend",
-  "version": "3.3-dev",
+  "version": "3.4-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",
@@ -43,17 +43,17 @@
   "dependencies": {
     "@mempool/electrum-client": "1.1.9",
     "@types/node": "^18.15.3",
-    "axios": "1.13.5",
+    "axios": "1.16.1",
     "bitcoinjs-lib": "~6.1.3",
     "crypto-js": "~4.2.0",
     "express": "~4.22.1",
     "maxmind": "~4.3.11",
-    "mysql2": "~3.17.1",
+    "mysql2": "~3.22.4",
     "rust-gbt": "file:./rust-gbt",
     "redis": "^4.7.0",
     "socks-proxy-agent": "~7.0.0",
     "typescript": "~4.9.3",
-    "ws": "~8.19.0"
+    "ws": "~8.21.0"
   },
   "devDependencies": {
     "@types/compression": "^1.7.2",

backend/src/__e2e__/cluster-mempool/harness/rpc-client.ts

@@ -0,0 +1,137 @@
+import http from 'http';
+
+export interface RpcConfig {
+  host: string;
+  port: number;
+  user: string;
+  pass: string;
+}
+
+interface RpcResponse {
+  result: any;
+  error: { code: number; message: string } | null;
+  id: string;
+}
+
+export class RpcClient {
+  private config: RpcConfig;
+  private idCounter = 0;
+
+  constructor(config: RpcConfig) {
+    this.config = config;
+  }
+
+  private async call(method: string, params: any[] = []): Promise<any> {
+    const id = String(++this.idCounter);
+    const body = JSON.stringify({ jsonrpc: '2.0', id, method, params });
+    const auth = Buffer.from(`${this.config.user}:${this.config.pass}`).toString('base64');
+
+    return new Promise((resolve, reject) => {
+      const req = http.request(
+        {
+          hostname: this.config.host,
+          port: this.config.port,
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Basic ${auth}`,
+            'Content-Length': Buffer.byteLength(body),
+          },
+        },
+        (res) => {
+          let data = '';
+          res.on('data', (chunk) => (data += chunk));
+          res.on('end', () => {
+            try {
+              const parsed: RpcResponse = JSON.parse(data);
+              if (parsed.error) {
+                reject(new Error(`RPC error ${parsed.error.code}: ${parsed.error.message}`));
+              } else {
+                resolve(parsed.result);
+              }
+            } catch (e) {
+              reject(new Error(`Failed to parse RPC response: ${data.slice(0, 500)}`));
+            }
+          });
+        },
+      );
+      req.on('error', reject);
+      req.write(body);
+      req.end();
+    });
+  }
+
+  async getRawMempool(): Promise<string[]> {
+    return this.call('getrawmempool');
+  }
+
+  async getRawTransaction(txid: string, verbose = true): Promise<any> {
+    return this.call('getrawtransaction', [txid, verbose]);
+  }
+
+  async getBlockTemplate(rules: string[] = ['segwit']): Promise<any> {
+    return this.call('getblocktemplate', [{ rules }]);
+  }
+
+  async getMempoolEntry(txid: string): Promise<any> {
+    return this.call('getmempoolentry', [txid]);
+  }
+
+  async getMempoolCluster(txid: string): Promise<any> {
+    return this.call('getmempoolcluster', [txid]);
+  }
+
+  async getBlockCount(): Promise<number> {
+    return this.call('getblockcount');
+  }
+
+  async batch(calls: { method: string; params: any[] }[]): Promise<any[]> {
+    if (calls.length === 0) {
+      return [];
+    }
+    const bodies = calls.map((c) => ({
+      jsonrpc: '2.0',
+      id: String(++this.idCounter),
+      method: c.method,
+      params: c.params,
+    }));
+    const body = JSON.stringify(bodies);
+    const auth = Buffer.from(`${this.config.user}:${this.config.pass}`).toString('base64');
+
+    return new Promise((resolve, reject) => {
+      const req = http.request(
+        {
+          hostname: this.config.host,
+          port: this.config.port,
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Basic ${auth}`,
+            'Content-Length': Buffer.byteLength(body),
+          },
+        },
+        (res) => {
+          let data = '';
+          res.on('data', (chunk) => (data += chunk));
+          res.on('end', () => {
+            try {
+              const parsed: RpcResponse[] = JSON.parse(data);
+              const results = parsed.map((r) => {
+                if (r.error) {
+                  return { error: r.error };
+                }
+                return r.result;
+              });
+              resolve(results);
+            } catch (e) {
+              reject(new Error(`Failed to parse batch RPC response: ${data.slice(0, 500)}`));
+            }
+          });
+        },
+      );
+      req.on('error', reject);
+      req.write(body);
+      req.end();
+    });
+  }
+}

backend/src/__e2e__/cluster-mempool/harness/run-harness.ts

@@ -0,0 +1,463 @@
+/**
+ * Cluster Mempool Test Harness
+ *
+ * Standalone script that compares our ClusterMempool block template ordering
+ * against Bitcoin Core's getblocktemplate output.
+ *
+ * Uses the backend's own transactionUtils to fetch and convert transactions,
+ * ensuring fields (sigops, adjustedVsize, etc.) match exactly.
+ *
+ * Usage:
+ *   npx ts-node src/__tests__/cluster-mempool/harness/run-harness.ts [options]
+ *
+ * Options:
+ *   --host <host>       Override CORE_RPC host
+ *   --port <port>       Override CORE_RPC port
+ *   --user <user>       Override CORE_RPC username
+ *   --pass <pass>       Override CORE_RPC password
+ *   --interval <ms>     Comparison interval in ms (default: 30000)
+ *   --poll <ms>         Mempool poll interval in ms (default: 1000)
+ *   --max <n>           Max comparisons before exit (0 = unlimited, default: 0)
+ */
+
+import { RpcClient, RpcConfig } from './rpc-client';
+
+// ─── CLI Parsing (must happen before backend imports) ───────────────────────
+
+interface CliOptions {
+  rpcOverrides: Partial<RpcConfig>;
+  comparisonInterval: number;
+  pollInterval: number;
+  maxComparisons: number;
+}
+
+function parseArgs(): CliOptions {
+  const args = process.argv.slice(2);
+  const overrides: Partial<RpcConfig> = {};
+  let comparisonInterval = 30_000;
+  let pollInterval = 1000;
+  let maxComparisons = 0;
+
+  for (let i = 0; i < args.length; i++) {
+    switch (args[i]) {
+      case '--host':
+        overrides.host = args[++i];
+        break;
+      case '--port':
+        overrides.port = parseInt(args[++i], 10);
+        break;
+      case '--user':
+        overrides.user = args[++i];
+        break;
+      case '--pass':
+        overrides.pass = args[++i];
+        break;
+      case '--interval':
+        comparisonInterval = parseInt(args[++i], 10);
+        break;
+      case '--poll':
+        pollInterval = parseInt(args[++i], 10);
+        break;
+      case '--max':
+        maxComparisons = parseInt(args[++i], 10);
+        break;
+    }
+  }
+
+  return { rpcOverrides: overrides, comparisonInterval, pollInterval, maxComparisons };
+}
+
+const cliOptions = parseArgs();
+
+// ─── Backend Module Loading ─────────────────────────────────────────────────
+// Import config first, apply CLI overrides, then import modules that depend on it.
+// This works because Node's require() caches modules — when transactionUtils
+// (via bitcoinClient) reads config, it sees our modified values.
+
+const config = require('../../../config').default;
+
+if (cliOptions.rpcOverrides.host) { config.CORE_RPC.HOST = cliOptions.rpcOverrides.host; }
+if (cliOptions.rpcOverrides.port) { config.CORE_RPC.PORT = cliOptions.rpcOverrides.port; }
+if (cliOptions.rpcOverrides.user) { config.CORE_RPC.USERNAME = cliOptions.rpcOverrides.user; }
+if (cliOptions.rpcOverrides.pass) { config.CORE_RPC.PASSWORD = cliOptions.rpcOverrides.pass; }
+
+// Now load modules that depend on config
+const bitcoinApi = require('../../../api/bitcoin/bitcoin-api-factory').default;
+const transactionUtils = require('../../../api/transaction-utils').default;
+const { ClusterMempool } = require('../../../cluster-mempool/cluster-mempool');
+
+import { MempoolTransactionExtended } from '../../../mempool.interfaces';
+import { MempoolDiff } from '../../../cluster-mempool/cluster-mempool';
+
+// ─── Main Harness ───────────────────────────────────────────────────────────
+
+const TX_FETCH_BATCH_SIZE = 1000;
+
+class Harness {
+  private rpc: RpcClient;
+  private clusterMempool: InstanceType<typeof ClusterMempool> | null = null;
+  private mempool: { [txid: string]: MempoolTransactionExtended } = {};
+  private knownTxids = new Set<string>();
+  private lastBlockHeight = -1;
+  private comparisonInterval: number;
+  private pollInterval: number;
+  private maxComparisons: number;
+  private pollTimer: ReturnType<typeof setInterval> | null = null;
+  private polling = false;
+  private nextComparisonTime = 0;
+  private stats = {
+    templateMatches: 0,
+    templateMismatches: 0,
+    comparisons: 0,
+  };
+
+  constructor(rpcConfig: RpcConfig, opts: CliOptions) {
+    this.rpc = new RpcClient(rpcConfig);
+    this.comparisonInterval = opts.comparisonInterval;
+    this.pollInterval = opts.pollInterval;
+    this.maxComparisons = opts.maxComparisons;
+  }
+
+  async run(): Promise<void> {
+    console.log('=== Cluster Mempool Test Harness ===');
+    console.log(`RPC: ${config.CORE_RPC.HOST}:${config.CORE_RPC.PORT}`);
+    console.log(`Backend: ${config.MEMPOOL.BACKEND}`);
+    console.log(`Poll interval: ${this.pollInterval}ms`);
+    console.log(`Comparison interval: ${this.comparisonInterval}ms`);
+    console.log();
+
+    // Step 1: Fetch full mempool
+    console.log('Fetching full mempool...');
+    const txids: string[] = await bitcoinApi.$getRawMempool();
+    console.log(`Got ${txids.length} txids from getrawmempool`);
+
+    const fetchStart = Date.now();
+    await this.fetchTransactions(txids);
+    const fetchTime = Date.now() - fetchStart;
+    console.log(`Fetched ${Object.keys(this.mempool).length} transactions in ${fetchTime}ms`);
+
+    // Step 2: Initialize ClusterMempool
+    console.log('Building cluster mempool...');
+    const buildStart = Date.now();
+    this.clusterMempool = new ClusterMempool(this.mempool);
+    const buildTime = Date.now() - buildStart;
+    console.log(`Cluster mempool built in ${buildTime}ms`);
+    console.log(`  Clusters: ${this.clusterMempool.getClusterCount()}`);
+    console.log(`  Transactions: ${this.clusterMempool.getTxCount()}`);
+    console.log();
+
+    // Record initial block height
+    this.lastBlockHeight = await this.rpc.getBlockCount();
+
+    // Run initial comparison, then start polling
+    await this.runComparison();
+    this.nextComparisonTime = Date.now() + this.comparisonInterval;
+    this.pollTimer = setInterval(() => this.pollMempool(), this.pollInterval);
+
+    console.log('\nHarness running. Press Ctrl+C to stop.\n');
+
+    process.on('SIGINT', () => this.shutdown());
+    process.on('SIGTERM', () => this.shutdown());
+  }
+
+  private async fetchTransactions(txids: string[]): Promise<MempoolTransactionExtended[]> {
+    const added: MempoolTransactionExtended[] = [];
+
+    for (let offset = 0; offset < txids.length; offset += TX_FETCH_BATCH_SIZE) {
+      let batch = txids.slice(offset, offset + TX_FETCH_BATCH_SIZE);
+      let txs: MempoolTransactionExtended[] = [];
+      let tries = 0;
+      while (batch.length && tries < 20) {
+        try {
+          tries++;
+          txs = txs.concat(await transactionUtils.$getMempoolTransactionsExtended(batch, false, false, false));
+          let missing: string[] = [];
+          console.log(`txs: ${txs.length} of ${batch.length} fetched`);
+          for (const txid of batch) {
+            if (!txs.some(tx => tx.txid === txid)) {
+              console.log(`missing ${txid} at offset ${offset}, retrying`);
+              missing.push(txid);
+            }
+          }
+          batch = missing;
+          if (batch.length) {
+            await new Promise(resolve => setTimeout(resolve, 500));
+          }
+        } catch (err: any) {
+          console.log(`  Fetch batch failed at offset ${offset}: ${err.message}, retrying`);
+        }
+      }
+
+      for (const tx of txs) {
+        this.mempool[tx.txid] = tx;
+        this.knownTxids.add(tx.txid);
+        added.push(tx);
+      }
+
+      if (txids.length > TX_FETCH_BATCH_SIZE && (offset + batch.length) % 5000 < TX_FETCH_BATCH_SIZE) {
+        console.log(`  ${offset + batch.length} / ${txids.length} fetched...`);
+      }
+    }
+
+    return added;
+  }
+
+  private async pollMempool(): Promise<void> {
+    if (this.polling || !this.clusterMempool) {
+      return;
+    }
+    this.polling = true;
+
+    try {
+      // Check for new block
+      const height = await this.rpc.getBlockCount();
+      const newBlock = height > this.lastBlockHeight;
+      if (newBlock) {
+        console.log(`\n--- New block detected (height ${height}) ---`);
+        this.lastBlockHeight = height;
+      }
+
+      // Get current mempool
+      const currentTxids = await bitcoinApi.$getRawMempool();
+      const currentSet = new Set(currentTxids);
+
+      // Find added and removed
+      const addedTxids: string[] = [];
+      for (const txid of currentTxids) {
+        if (!this.knownTxids.has(txid)) {
+          addedTxids.push(txid);
+        }
+      }
+      const removedTxids: string[] = [];
+      for (const txid of this.knownTxids) {
+        if (!currentSet.has(txid)) {
+          removedTxids.push(txid);
+        }
+      }
+
+      if (addedTxids.length === 0 && removedTxids.length === 0) {
+        if (newBlock) {
+          await this.runComparison();
+        }
+        return;
+      }
+
+      const added = await this.fetchTransactions(addedTxids);
+
+      const removed = removedTxids
+        .map(txid => this.mempool[txid])
+        .filter((tx): tx is MempoolTransactionExtended => !!tx);
+      const diff: MempoolDiff = { added, removed, accelerations: {} };
+      const t0 = Date.now();
+      this.clusterMempool.applyMempoolChange(diff);
+      const dt = Date.now() - t0;
+      for (const txid of removedTxids) {
+        delete this.mempool[txid];
+        this.knownTxids.delete(txid);
+      }
+      console.log(
+        `Applied${newBlock ? ' block' : ''} diff: +${added.length} -${removedTxids.length} txs in ${dt}ms` +
+        ` (${this.clusterMempool.getTxCount()} txs, ${this.clusterMempool.getClusterCount()} clusters)`
+      );
+
+      if (newBlock) {
+        await this.runComparison();
+      } else if (Date.now() >= this.nextComparisonTime) {
+        await this.runComparison();
+        this.nextComparisonTime = Date.now() + this.comparisonInterval;
+      }
+    } catch (err: any) {
+      console.error(`Poll error: ${err.message}`);
+    } finally {
+      this.polling = false;
+    }
+  }
+
+  private async runComparison(): Promise<void> {
+    if (!this.clusterMempool) {
+      return;
+    }
+    this.stats.comparisons++;
+    console.log(`\n=== Comparison #${this.stats.comparisons} ===`);
+
+    // Step 1: Get Core's block template
+    let template: any;
+    try {
+      template = await this.rpc.getBlockTemplate(['segwit']);
+    } catch (err: any) {
+      console.log(`getblocktemplate failed: ${err.message}`);
+      return;
+    }
+    const coreTxids: string[] = template.transactions.map((t: any) => t.txid);
+    const coreSet = new Set(coreTxids);
+
+    const missingTxids: string[] = [];
+    for (const txid of coreSet) {
+      if (!this.mempool[txid]) {
+        missingTxids.push(txid);
+      }
+    }
+    if (missingTxids.length) {
+      const added = await this.fetchTransactions(missingTxids);
+      if (added.length) {
+        console.log(`Reconciled: +${added.length} missing txs`);
+        this.clusterMempool.applyMempoolChange({ added, removed: [], accelerations: {} });
+      }
+    }
+
+    const ourBlocks = this.clusterMempool.getBlocks(1, true);
+    const ourTxids: string[] = ourBlocks[0]?.txids || [];
+
+    await this.compareTemplateOrdering(coreTxids, ourTxids);
+
+    this.printStats();
+
+    if (this.maxComparisons > 0 && this.stats.comparisons >= this.maxComparisons) {
+      console.log(`\nReached ${this.maxComparisons} comparisons, stopping.`);
+      this.shutdown();
+    }
+  }
+
+  private async compareTemplateOrdering(coreTxids: string[], ourTxids: string[]): Promise<void> {
+    const coreSet = new Set(coreTxids);
+    const ourSet = new Set(ourTxids);
+
+    let inBoth = 0;
+    let onlyCore = 0;
+    let onlyOurs = 0;
+    for (const txid of coreSet) {
+      if (ourSet.has(txid)) {
+        inBoth++;
+      } else {
+        onlyCore++;
+      }
+    }
+    for (const txid of ourSet) {
+      if (!coreSet.has(txid)) {
+        onlyOurs++;
+      }
+    }
+
+    console.log(`Template sets: Core ${coreTxids.length} txs, Ours ${ourTxids.length} txs`);
+    console.log(`  In both: ${inBoth}, Only Core: ${onlyCore}, Only ours: ${onlyOurs}`);
+
+    // Build position maps for the intersection
+    const corePos = new Map<string, number>();
+    for (let i = 0; i < coreTxids.length; i++) {
+      if (ourSet.has(coreTxids[i])) {
+        corePos.set(coreTxids[i], i);
+      }
+    }
+
+    // Filter to shared txids in each order
+    const sharedInCoreOrder = coreTxids.filter(t => ourSet.has(t));
+    const sharedInOurOrder = ourTxids.filter(t => coreSet.has(t));
+
+    let exactMatches = 0;
+    let firstDivergence = -1;
+    for (let i = 0; i < sharedInCoreOrder.length; i++) {
+      if (sharedInCoreOrder[i] === sharedInOurOrder[i]) {
+        exactMatches++;
+      } else if (firstDivergence === -1) {
+        firstDivergence = i;
+      }
+    }
+
+    const orderMatchRate = sharedInCoreOrder.length > 0
+      ? (exactMatches / sharedInCoreOrder.length * 100).toFixed(1)
+      : 'N/A';
+    console.log(`  Ordering: ${orderMatchRate}% exact position match (${exactMatches}/${sharedInCoreOrder.length} shared txs)`);
+
+    if (firstDivergence >= 0) {
+      const coreTx = sharedInCoreOrder[firstDivergence];
+      const ourTx = sharedInOurOrder[firstDivergence];
+      console.log(`\n  First divergence at position ${firstDivergence}:`);
+      console.log(`    Core wants: ${coreTx}`);
+      console.log(`    We placed:  ${ourTx} (Core has this at position ${corePos.get(ourTx)})`);
+
+      for (const [label, txid] of [['Core tx', coreTx], ['Our tx', ourTx]] as const) {
+        const inOurMempool = !!this.mempool[txid];
+        const info = this.clusterMempool?.getClusterInfo(txid);
+        console.log(`\n    --- ${label}: ${txid} ---`);
+        console.log(`    In our mempool: ${inOurMempool}`);
+        if (inOurMempool) {
+          const tx = this.mempool[txid];
+          console.log(`    fee=${tx.fee} weight=${tx.weight} vsize=${tx.vsize} sigops=${tx.sigops} adjustedVsize=${tx.adjustedVsize}`);
+        }
+        if (info) {
+          console.log(`    clusterId=${info.clusterId} chunkIndex=${info.chunkIndex} chunkFeerate=${info.chunkFeerate.toFixed(6)}`);
+          const cluster = this.clusterMempool?.getCluster(info.clusterId);
+          if (cluster) {
+            console.log(`    Cluster has ${cluster.chunks.length} chunk(s):`);
+            for (let ci = 0; ci < cluster.chunks.length; ci++) {
+              const c = cluster.chunks[ci];
+              console.log(`      chunk[${ci}]: ${c.txs.length} txs, feerate=${c.feerate.toFixed(6)}`);
+              if (ci === info.chunkIndex) {
+                const chunkTxids = c.txs.map((idx: number) => cluster.txs[idx]?.txid).filter(Boolean);
+                for (const tid of chunkTxids) {
+                  const t = this.mempool[tid];
+                  const parents = t?.vin
+                    ?.filter(v => !v.is_coinbase && this.mempool[v.txid] && chunkTxids.includes(v.txid))
+                    .map(v => v.txid.substring(0, 12)) || [];
+                  console.log(`        ${tid.substring(0, 12)}  fee=${t?.fee} size=${(t?.weight || 0) / 4} sigops=${t?.sigops} parents=[${parents.join(', ')}]`);
+                }
+              }
+            }
+          }
+        } else {
+          console.log(`    NOT in our cluster mempool`);
+        }
+      }
+
+      for (const [label, txid] of [['Core', coreTx], ['Our', ourTx]] as const) {
+        try {
+          const coreCluster = await this.rpc.getMempoolCluster(txid);
+          console.log(`\n    --- Core cluster for ${label} tx ${txid.substring(0, 12)} ---`);
+          console.log(`    Raw response: ${JSON.stringify(coreCluster).substring(0, 2000)}`);
+        } catch (e: any) {
+          console.log(`    Failed to get Core cluster for ${label} tx: ${e.message}`);
+        }
+      }
+
+      this.stats.templateMismatches++;
+    } else if (sharedInCoreOrder.length === coreTxids.length && sharedInCoreOrder.length === ourTxids.length) {
+      console.log(`  PERFECT MATCH`);
+      this.stats.templateMatches++;
+    } else {
+      console.log(`  Ordering matches for shared txs, but sets differ`);
+      this.stats.templateMatches++;
+    }
+  }
+
+  private printStats(): void {
+    console.log('\n--- Cumulative Stats ---');
+    const total = this.stats.templateMatches + this.stats.templateMismatches;
+    console.log(`Template comparisons: ${total} (${this.stats.templateMatches} perfect, ${this.stats.templateMismatches} divergent)`);
+  }
+
+  private shutdown(): void {
+    console.log('\nShutting down...');
+    if (this.pollTimer) {
+      clearInterval(this.pollTimer);
+    }
+    this.printStats();
+    process.exit(0);
+  }
+}
+
+// ─── Entry Point ────────────────────────────────────────────────────────────
+
+const rpcConfig: RpcConfig = {
+  host: config.CORE_RPC.HOST,
+  port: config.CORE_RPC.PORT,
+  user: config.CORE_RPC.USERNAME,
+  pass: config.CORE_RPC.PASSWORD,
+};
+
+console.log(`Connecting to Bitcoin Core at ${rpcConfig.host}:${rpcConfig.port}`);
+
+const harness = new Harness(rpcConfig, cliOptions);
+harness.run().catch((err) => {
+  console.error('Harness failed:', err);
+  process.exit(1);
+});

backend/src/__fixtures__/mempool-config.template.json

@@ -32,6 +32,8 @@
     "AUDIT": true,
     "RUST_GBT": false,
     "LIMIT_GBT": false,
+    "CLUSTER_MEMPOOL": false,
+    "CLUSTER_MEMPOOL_INDEXING": false,
     "CPFP_INDEXING": true,
     "MAX_BLOCKS_BULK_QUERY": 999,
     "DISK_CACHE_BLOCK_INTERVAL": 999,

backend/src/__tests__/cluster-mempool/cluster-mempool.test.ts

@@ -0,0 +1,544 @@
+import { ClusterMempool } from '../../cluster-mempool/cluster-mempool';
+import { makeTx, txid } from './test-utils';
+import { MempoolTransactionExtended } from '../../mempool.interfaces';
+
+function buildMempool(txs: MempoolTransactionExtended[]): { [txid: string]: MempoolTransactionExtended } {
+  const mempool: { [txid: string]: MempoolTransactionExtended } = {};
+  for (const tx of txs) {
+    mempool[tx.txid] = tx;
+  }
+  return mempool;
+}
+
+describe('ClusterMempool', () => {
+  describe('constructor', () => {
+    it('should build clusters from mempool', () => {
+      const parentId = txid('a1');
+      const childId = txid('a2');
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        makeTx(childId, 5000, 100, [parentId]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(2);
+    });
+
+    it('should create separate clusters for unrelated txs', () => {
+      const mempool = buildMempool([
+        makeTx(txid('a1'), 100, 100),
+        makeTx(txid('b1'), 200, 100),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(2);
+    });
+  });
+
+  describe('getClusterInfo', () => {
+    it('should return cluster info for a known tx', () => {
+      const parentId = txid('a1');
+      const childId = txid('a2');
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        makeTx(childId, 5000, 100, [parentId]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const info = cm.getClusterInfo(parentId);
+      expect(info).not.toBeNull();
+      expect(info?.chunkFeerate).toBeGreaterThan(0);
+    });
+
+    it('should return null for unknown tx', () => {
+      const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterInfo(txid('zz'))).toBeNull();
+    });
+  });
+
+  describe('getCluster', () => {
+    it('should return cluster data with correct topology', () => {
+      const parentId = txid('a1');
+      const childId = txid('a2');
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        makeTx(childId, 5000, 100, [parentId]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const info = cm.getClusterInfo(parentId);
+      expect(info).not.toBeNull();
+      const data = cm.getCluster(info!.clusterId);
+      expect(data).not.toBeNull();
+      expect(data!.txs.length).toBe(2);
+      expect(data!.chunks.length).toBeGreaterThan(0);
+    });
+
+    it('should return null for unknown cluster id', () => {
+      const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getCluster(9999)).toBeNull();
+    });
+  });
+
+  describe('applyMempoolChange', () => {
+    it('should handle adding a new singleton tx', () => {
+      const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+      const initialCount = cm.getClusterCount();
+
+      cm.applyMempoolChange({
+        added: [makeTx(txid('b1'), 200, 100)],
+        removed: [],
+        accelerations: {},
+      });
+
+      expect(cm.getClusterCount()).toBe(initialCount + 1);
+      expect(cm.getTxCount()).toBe(2);
+    });
+
+    it('should skip duplicate tx additions', () => {
+      const mempool = buildMempool([]);
+      const cm = new ClusterMempool(mempool);
+      const tx = makeTx(txid('a1'), 100, 100);
+      mempool[tx.txid] = tx;
+
+      cm.applyMempoolChange({
+        added: [tx, tx],
+        removed: [],
+        accelerations: {},
+      });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(1);
+      expect(cm.getBlocks(1)[0].txids).toEqual([tx.txid]);
+    });
+
+    it('should handle removing a tx', () => {
+      const parentId = txid('a1');
+      const childId = txid('a2');
+      const childTx = makeTx(childId, 5000, 100, [parentId]);
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        childTx,
+      ]);
+      const cm = new ClusterMempool(mempool);
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [childTx],
+        accelerations: {},
+      });
+
+      expect(cm.getTxCount()).toBe(1);
+      expect(cm.getClusterInfo(childId)).toBeNull();
+      expect(cm.getClusterInfo(parentId)).not.toBeNull();
+    });
+
+    it('should clean spentBy when removed tx is already missing from mempool', () => {
+      const parentId = txid('a1');
+      const childId = txid('a2');
+      const childTx = makeTx(childId, 5000, 100, [parentId]);
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        childTx,
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const spentBy = (cm as unknown as { spentBy: Map<string, string> }).spentBy;
+
+      expect(spentBy.get(`${parentId}:0`)).toBe(childId);
+
+      delete mempool[childId];
+      cm.applyMempoolChange({
+        added: [],
+        removed: [childTx],
+        accelerations: {},
+      });
+
+      expect(spentBy.has(`${parentId}:0`)).toBe(false);
+    });
+
+    it('should not delete spentBy for a replacement transaction', () => {
+      const parentId = txid('a1');
+      const replacedId = txid('a2');
+      const replacementId = txid('a3');
+      const replacedTx = makeTx(replacedId, 5000, 100, [parentId]);
+      const replacementTx = makeTx(replacementId, 6000, 100, [parentId]);
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        replacementTx,
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const spentBy = (cm as unknown as { spentBy: Map<string, string> }).spentBy;
+
+      expect(spentBy.get(`${parentId}:0`)).toBe(replacementId);
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [replacedTx],
+        accelerations: {},
+      });
+
+      expect(spentBy.get(`${parentId}:0`)).toBe(replacementId);
+    });
+
+    it('should split cluster when middle tx is removed', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const bTx = makeTx(b, 200, 100, [a]);
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        bTx,
+        makeTx(c, 300, 100, [b]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [bTx],
+        accelerations: {},
+      });
+
+      expect(cm.getTxCount()).toBe(2);
+      const infoA = cm.getClusterInfo(a);
+      const infoC = cm.getClusterInfo(c);
+      expect(infoA).not.toBeNull();
+      expect(infoC).not.toBeNull();
+      expect(infoA!.clusterId).not.toBe(infoC!.clusterId);
+    });
+
+    it('should handle fee changes via acceleration', () => {
+      const parentId = txid('a1');
+      const childId = txid('a2');
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        makeTx(childId, 100, 100, [parentId]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const infoBefore = cm.getClusterInfo(childId);
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [],
+        accelerations: { [childId]: { feeDelta: 49900 } },
+      });
+
+      const infoAfter = cm.getClusterInfo(childId);
+      expect(infoAfter).not.toBeNull();
+      expect(infoAfter!.clusterId).not.toBe(infoBefore!.clusterId);
+    });
+
+    it('should merge clusters when new tx connects them', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        makeTx(b, 200, 100),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(2);
+
+      const bridgeTx = makeTx(txid('c1'), 300, 100, [a, b]);
+      cm.applyMempoolChange({
+        added: [bridgeTx],
+        removed: [],
+        accelerations: {},
+      });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(3);
+    });
+  });
+
+  describe('cluster merging', () => {
+    it('should merge 3 separate clusters when new tx bridges them', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        makeTx(b, 200, 100),
+        makeTx(c, 300, 100),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(3);
+
+      const bridge = makeTx(txid('d1'), 400, 100, [a, b, c]);
+      mempool[bridge.txid] = bridge;
+      cm.applyMempoolChange({ added: [bridge], removed: [], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(4);
+    });
+
+    it('should grow cluster by 1 when new tx has parents in same cluster', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        makeTx(b, 200, 100, [a]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+
+      const c = makeTx(txid('c1'), 300, 100, [a]);
+      mempool[c.txid] = c;
+      cm.applyMempoolChange({ added: [c], removed: [], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(3);
+    });
+
+    it('should grow chain incrementally', () => {
+      const a = txid('a1');
+      const mempool = buildMempool([makeTx(a, 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+
+      const b = makeTx(txid('b1'), 200, 100, [a]);
+      mempool[b.txid] = b;
+      cm.applyMempoolChange({ added: [b], removed: [], accelerations: {} });
+
+      const c = makeTx(txid('c1'), 300, 100, [txid('b1')]);
+      mempool[c.txid] = c;
+      cm.applyMempoolChange({ added: [c], removed: [], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(3);
+    });
+  });
+
+  describe('cluster splitting', () => {
+    it('should split star into singletons when center is removed', () => {
+      const center = txid('center');
+      const leaves = Array.from({ length: 5 }, (_, i) => txid(`leaf${i}`));
+      const centerTx = makeTx(center, 100, 100);
+      for (let i = 1; i < 5; i++) {
+        centerTx.vout.push({
+          scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000,
+        });
+      }
+      const leafTxs = leaves.map((l, i) => {
+        const tx = makeTx(l, 200, 100, [center]);
+        tx.vin[0].vout = i;
+        return tx;
+      });
+      const mempool = buildMempool([centerTx, ...leafTxs]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+
+      cm.applyMempoolChange({ added: [], removed: [centerTx], accelerations: {} });
+
+      expect(cm.getTxCount()).toBe(5);
+      expect(cm.getClusterCount()).toBe(5);
+    });
+
+    it('should shrink cluster without splitting when leaf is removed', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const aTx = makeTx(a, 100, 100);
+      aTx.vout.push({ scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000 });
+      const bTx = makeTx(b, 200, 100, [a]);
+      bTx.vin[0].vout = 0;
+      const cTx = makeTx(c, 300, 100, [a]);
+      cTx.vin[0].vout = 1;
+      const mempool = buildMempool([aTx, bTx, cTx]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+
+      cm.applyMempoolChange({ added: [], removed: [cTx], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(2);
+    });
+
+    it('should produce singleton when tx is removed from 2-tx cluster', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const bTx = makeTx(b, 200, 100, [a]);
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        bTx,
+      ]);
+      const cm = new ClusterMempool(mempool);
+
+      cm.applyMempoolChange({ added: [], removed: [bTx], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(1);
+      expect(cm.getClusterInfo(a)).not.toBeNull();
+    });
+
+    it('should create 3+ components when removing a tx that bridges multiple subgraphs', () => {
+      const hub = txid('hub');
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const hubTx = makeTx(hub, 100, 100);
+      hubTx.vout.push(
+        { scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000 },
+        { scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000 },
+      );
+      const txA = makeTx(a, 200, 100, [hub]);
+      txA.vin[0].vout = 0;
+      const txB = makeTx(b, 300, 100, [hub]);
+      txB.vin[0].vout = 1;
+      const txC = makeTx(c, 400, 100, [hub]);
+      txC.vin[0].vout = 2;
+      const mempool = buildMempool([hubTx, txA, txB, txC]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+
+      cm.applyMempoolChange({ added: [], removed: [hubTx], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(3);
+      expect(cm.getTxCount()).toBe(3);
+    });
+  });
+
+  describe('fee changes via accelerations', () => {
+    it('should increase chunk feerate when acceleration added', () => {
+      const a = txid('a1');
+      const mempool = buildMempool([makeTx(a, 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+      const infoBefore = cm.getClusterInfo(a);
+      expect(infoBefore).not.toBeNull();
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [],
+        accelerations: { [a]: { feeDelta: 9900 } },
+      });
+
+      const infoAfter = cm.getClusterInfo(a);
+      expect(infoAfter).not.toBeNull();
+      expect(infoAfter!.chunkFeerate).toBeGreaterThan(infoBefore!.chunkFeerate);
+    });
+
+    it('should decrease chunk feerate when acceleration removed', () => {
+      const a = txid('a1');
+      const mempool = buildMempool([makeTx(a, 100, 100)]);
+      const cm = new ClusterMempool(mempool, { [a]: { feeDelta: 9900 } });
+      const infoBefore = cm.getClusterInfo(a);
+      expect(infoBefore).not.toBeNull();
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [],
+        accelerations: {},
+      });
+
+      const infoAfter = cm.getClusterInfo(a);
+      expect(infoAfter).not.toBeNull();
+      expect(infoAfter!.chunkFeerate).toBeLessThan(infoBefore!.chunkFeerate);
+    });
+
+    it('should reorder chunks when acceleration shifts priorities', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        makeTx(b, 200, 100, [a]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const infoBBefore = cm.getClusterInfo(b);
+      expect(infoBBefore).not.toBeNull();
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [],
+        accelerations: { [b]: { feeDelta: 49800 } },
+      });
+
+      const infoBAfter = cm.getClusterInfo(b);
+      expect(infoBAfter).not.toBeNull();
+      expect(infoBAfter!.chunkFeerate).toBeGreaterThan(infoBBefore!.chunkFeerate);
+    });
+  });
+
+  describe('getBlocks', () => {
+    it('should return projected blocks', () => {
+      const txs: MempoolTransactionExtended[] = [];
+      for (let i = 0; i < 10; i++) {
+        txs.push(makeTx(txid(`t${i}`), 1000 * (i + 1), 100));
+      }
+      const mempool = buildMempool(txs);
+      const cm = new ClusterMempool(mempool);
+      const blocks = cm.getBlocks(3);
+      expect(blocks.length).toBeGreaterThan(0);
+      expect(blocks[0].txids.length).toBeGreaterThan(0);
+    });
+
+    it('should return empty array for empty mempool', () => {
+      const cm = new ClusterMempool({});
+      const blocks = cm.getBlocks(3);
+      expect(blocks.length).toBe(0);
+    });
+
+    it('should respect chunk ordering for single-cluster mempool', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const mempool = buildMempool([
+        makeTx(a, 3000, 100),
+        makeTx(b, 200, 100, [a]),
+        makeTx(c, 100, 100, [b]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const blocks = cm.getBlocks(1);
+      expect(blocks.length).toBe(1);
+      const txids = blocks[0].txids;
+      expect(txids.indexOf(a)).toBeLessThan(txids.indexOf(b));
+      expect(txids.indexOf(b)).toBeLessThan(txids.indexOf(c));
+    });
+
+    it('should maintain topological validity within blocks', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const d = txid('d1');
+      const aTx = makeTx(a, 1000, 100);
+      aTx.vout.push({ scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000 });
+      const bTx = makeTx(b, 500, 100, [a]);
+      bTx.vin[0].vout = 0;
+      const cTx = makeTx(c, 500, 100, [a]);
+      cTx.vin[0].vout = 1;
+      const mempool = buildMempool([aTx, bTx, cTx, makeTx(d, 200, 100, [b, c])]);
+      const cm = new ClusterMempool(mempool);
+      const blocks = cm.getBlocks(1);
+      const txids = blocks[0].txids;
+
+      expect(txids.indexOf(a)).toBeLessThan(txids.indexOf(b));
+      expect(txids.indexOf(a)).toBeLessThan(txids.indexOf(c));
+      expect(txids.indexOf(b)).toBeLessThan(txids.indexOf(d));
+      expect(txids.indexOf(c)).toBeLessThan(txids.indexOf(d));
+    });
+  });
+
+  describe('empty and degenerate cases', () => {
+    it('should handle empty diff with no changes', () => {
+      const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+      const countBefore = cm.getClusterCount();
+      const txCountBefore = cm.getTxCount();
+
+      cm.applyMempoolChange({ added: [], removed: [], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(countBefore);
+      expect(cm.getTxCount()).toBe(txCountBefore);
+    });
+
+    it('should not crash when removing nonexistent tx', () => {
+      const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [makeTx(txid('nonexistent'), 100, 100)],
+        accelerations: {},
+      });
+
+      expect(cm.getTxCount()).toBe(1);
+    });
+  });
+});

backend/src/__tests__/cluster-mempool/depgraph.test.ts

@@ -0,0 +1,494 @@
+import { DepGraph, sortTopological, subgraph } from '../../cluster-mempool/depgraph';
+import { buildChain, buildFanOut, buildDiamond, buildStar } from './test-utils';
+
+describe('DepGraph', () => {
+  describe('addTransaction', () => {
+    it('should add a transaction and return a ClusterTx', () => {
+      const dg = new DepGraph();
+      const tx = dg.addTransaction('tx0', 1000, 100);
+      expect(dg.size).toBe(1);
+      expect(tx.effectiveFee).toBe(1000);
+      expect(tx.weight).toBe(100);
+      expect(tx.txid).toBe('tx0');
+    });
+
+    it('should assign distinct ClusterTx objects', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      expect(a).not.toBe(b);
+      expect(b).not.toBe(c);
+      expect(dg.size).toBe(3);
+    });
+
+    it('should include self in ancestors and descendants', () => {
+      const dg = new DepGraph();
+      const tx = dg.addTransaction('tx0', 100, 10);
+      expect(tx.ancestors.has(tx)).toBe(true);
+      expect(tx.descendants.has(tx)).toBe(true);
+    });
+
+    it('should handle large clusters', () => {
+      const dg = new DepGraph();
+      for (let i = 0; i < 100; i++) {
+        dg.addTransaction(`tx${i}`, 100, 10);
+      }
+      expect(dg.size).toBe(100);
+    });
+  });
+
+  describe('addDependency', () => {
+    it('should establish parent-child relationship', () => {
+      const dg = new DepGraph();
+      const parent = dg.addTransaction('parent', 100, 10);
+      const child = dg.addTransaction('child', 200, 20);
+      dg.addDependency(parent, child);
+
+      expect(child.ancestors.has(parent)).toBe(true);
+      expect(parent.descendants.has(child)).toBe(true);
+      expect(child.parents.has(parent)).toBe(true);
+      expect(parent.children.has(child)).toBe(true);
+    });
+
+    it('should propagate ancestors transitively', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+
+      expect(c.ancestors.has(a)).toBe(true);
+      expect(c.ancestors.has(b)).toBe(true);
+      expect(c.ancestors.has(c)).toBe(true);
+
+      expect(a.descendants.has(a)).toBe(true);
+      expect(a.descendants.has(b)).toBe(true);
+      expect(a.descendants.has(c)).toBe(true);
+    });
+
+    it('should handle diamond dependencies', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      const d = dg.addTransaction('d', 400, 40);
+      dg.addDependency(a, b);
+      dg.addDependency(a, c);
+      dg.addDependency(b, d);
+      dg.addDependency(c, d);
+
+      expect(d.ancestors.size).toBe(4);
+      expect(a.descendants.size).toBe(4);
+    });
+  });
+
+  describe('removeTransactions', () => {
+    it('should remove transactions and update sets', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+
+      dg.removeTransactions(new Set([b]));
+      expect(dg.size).toBe(2);
+      expect(dg.hasTx(b)).toBe(false);
+      expect(c.ancestors.has(b)).toBe(false);
+      expect(a.descendants.has(b)).toBe(false);
+    });
+
+    it('should handle slot reuse after removal', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      dg.addTransaction('b', 200, 20);
+      dg.removeTransactions(new Set([a]));
+      const c = dg.addTransaction('c', 300, 30);
+      expect(dg.size).toBe(2);
+      expect(c.txid).toBe('c');
+    });
+  });
+
+  describe('dependsOn (via ancestors)', () => {
+    it('should correctly identify dependencies', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+
+      expect(b.ancestors.has(a)).toBe(true);
+      expect(a.ancestors.has(b)).toBe(false);
+      expect(c.ancestors.has(a)).toBe(false);
+    });
+  });
+
+  describe('findConnectedComponents', () => {
+    it('should find a single component', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addDependency(a, b);
+
+      const components = dg.findConnectedComponents();
+      expect(components.length).toBe(1);
+      expect(components[0].size).toBe(2);
+    });
+
+    it('should find multiple disconnected components', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      const d = dg.addTransaction('d', 400, 40);
+      dg.addDependency(a, b);
+      dg.addDependency(c, d);
+
+      const components = dg.findConnectedComponents();
+      expect(components.length).toBe(2);
+    });
+
+    it('should handle isolated transactions', () => {
+      const dg = new DepGraph();
+      dg.addTransaction('a', 100, 10);
+      dg.addTransaction('b', 200, 20);
+      dg.addTransaction('c', 300, 30);
+
+      const components = dg.findConnectedComponents();
+      expect(components.length).toBe(3);
+    });
+  });
+
+  describe('parents / children (direct)', () => {
+    it('should return only direct parents, not transitive', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+
+      expect(c.parents.has(b)).toBe(true);
+      expect(c.parents.has(a)).toBe(false);
+    });
+
+    it('should return only direct children', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+
+      expect(a.children.has(b)).toBe(true);
+      expect(a.children.has(c)).toBe(false);
+    });
+  });
+
+  describe('appendTopo', () => {
+    it('should output in topological order', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+
+      const output = sortTopological(new Set([c, a, b]));
+      expect(output.indexOf(a)).toBeLessThan(output.indexOf(b));
+      expect(output.indexOf(b)).toBeLessThan(output.indexOf(c));
+    });
+  });
+
+  describe('restrict', () => {
+    it('should create a subgraph with correct deps', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+
+      const { depgraph: sub, txMap } = subgraph(new Set([a, b]));
+      expect(sub.size).toBe(2);
+      const newA = txMap.get(a)!;
+      const newB = txMap.get(b)!;
+      expect(newB.ancestors.has(newA)).toBe(true);
+    });
+  });
+
+  describe('graceful error handling', () => {
+    it('should no-op addDependency with non-member txs', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const dg2 = new DepGraph();
+      const foreign = dg2.addTransaction('foreign', 200, 20);
+      dg.addDependency(a, foreign);
+      dg.addDependency(foreign, a);
+      expect(a.ancestors.size).toBe(1);
+    });
+  });
+
+  describe('deep chain topology', () => {
+    it('should track ancestors and descendants at each depth', () => {
+      const { depgraph, txs } = buildChain(20, 100, 10);
+      expect(depgraph.size).toBe(20);
+
+      expect(txs[19].ancestors.size).toBe(20);
+      expect(txs[0].descendants.size).toBe(20);
+
+      expect(txs[10].ancestors.size).toBe(11);
+      expect(txs[10].descendants.size).toBe(10);
+
+      expect(txs[10].parents.size).toBe(1);
+      expect(txs[10].parents.has(txs[9])).toBe(true);
+      expect(txs[10].children.size).toBe(1);
+      expect(txs[10].children.has(txs[11])).toBe(true);
+    });
+  });
+
+  describe('wide fan-out topology', () => {
+    it('should track parent-children relationships for 1 parent with 10 children', () => {
+      const { depgraph, parent, children } = buildFanOut(10, 100, 10, 50, 10);
+      expect(depgraph.size).toBe(11);
+      expect(parent.children.size).toBe(10);
+      expect(parent.descendants.size).toBe(11);
+
+      for (const child of children) {
+        expect(child.ancestors.size).toBe(2);
+        expect(child.ancestors.has(parent)).toBe(true);
+        expect(child.parents.size).toBe(1);
+        expect(child.parents.has(parent)).toBe(true);
+      }
+    });
+  });
+
+  describe('wide fan-in topology', () => {
+    it('should track many parents converging to one child', () => {
+      const dg = new DepGraph();
+      const parents: any[] = [];
+      for (let i = 0; i < 10; i++) {
+        parents.push(dg.addTransaction(`p${i}`, 100, 10));
+      }
+      const child = dg.addTransaction('child', 500, 50);
+      for (const p of parents) {
+        dg.addDependency(p, child);
+      }
+
+      expect(child.parents.size).toBe(10);
+      expect(child.ancestors.size).toBe(11);
+      for (const p of parents) {
+        expect(p.descendants.has(child)).toBe(true);
+      }
+    });
+  });
+
+  describe('multiple diamonds in sequence', () => {
+    it('should handle A→B,C→D→E,F→G topology', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 100, 10);
+      const c = dg.addTransaction('c', 100, 10);
+      const d = dg.addTransaction('d', 100, 10);
+      const e = dg.addTransaction('e', 100, 10);
+      const f = dg.addTransaction('f', 100, 10);
+      const g = dg.addTransaction('g', 100, 10);
+      dg.addDependency(a, b);
+      dg.addDependency(a, c);
+      dg.addDependency(b, d);
+      dg.addDependency(c, d);
+      dg.addDependency(d, e);
+      dg.addDependency(d, f);
+      dg.addDependency(e, g);
+      dg.addDependency(f, g);
+
+      expect(g.ancestors.size).toBe(7);
+      expect(a.descendants.size).toBe(7);
+      expect(d.parents.size).toBe(2);
+      expect(d.children.size).toBe(2);
+    });
+  });
+
+  describe('disconnected subgraphs', () => {
+    it('should coexist in one DepGraph', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addDependency(a, b);
+
+      const c = dg.addTransaction('c', 300, 30);
+      const d = dg.addTransaction('d', 400, 40);
+      dg.addDependency(c, d);
+
+      const e = dg.addTransaction('e', 500, 50);
+
+      expect(dg.size).toBe(5);
+      expect(b.ancestors.has(c)).toBe(false);
+      expect(a.descendants.has(d)).toBe(false);
+      expect(e.ancestors.size).toBe(1);
+      expect(dg.findConnectedComponents().length).toBe(3);
+    });
+  });
+
+  describe('removeTransactions edge cases', () => {
+    it('should remove a leaf without affecting siblings', () => {
+      const { depgraph, parent, children } = buildFanOut(3, 100, 10, 50, 10);
+      depgraph.removeTransactions(new Set([children[2]]));
+      expect(depgraph.size).toBe(3);
+      expect(parent.children.size).toBe(2);
+      expect(depgraph.hasTx(children[0])).toBe(true);
+      expect(depgraph.hasTx(children[1])).toBe(true);
+    });
+
+    it('should remove a root without affecting unrelated txs', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addDependency(a, b);
+      dg.removeTransactions(new Set([a]));
+      expect(dg.size).toBe(1);
+      expect(dg.hasTx(a)).toBe(false);
+      expect(b.ancestors.size).toBe(1);
+      expect(b.ancestors.has(b)).toBe(true);
+    });
+
+    it('should break transitive edges when middle of chain is removed', () => {
+      const { depgraph, txs } = buildChain(5, 100, 10);
+      depgraph.removeTransactions(new Set([txs[2]]));
+      expect(depgraph.size).toBe(4);
+      expect(txs[0].descendants.has(txs[3])).toBe(false);
+      expect(txs[0].descendants.has(txs[1])).toBe(true);
+      expect(txs[3].ancestors.has(txs[0])).toBe(false);
+      expect(txs[3].descendants.has(txs[4])).toBe(true);
+    });
+
+    it('should handle batch removal of multiple txs', () => {
+      const { depgraph, txs } = buildChain(5, 100, 10);
+      depgraph.removeTransactions(new Set([txs[1], txs[3]]));
+      expect(depgraph.size).toBe(3);
+      expect(depgraph.hasTx(txs[0])).toBe(true);
+      expect(depgraph.hasTx(txs[2])).toBe(true);
+      expect(depgraph.hasTx(txs[4])).toBe(true);
+    });
+
+    it('should result in empty graph when all txs removed', () => {
+      const { depgraph, txs } = buildChain(3, 100, 10);
+      depgraph.removeTransactions(new Set(txs));
+      expect(depgraph.size).toBe(0);
+      expect(depgraph.getTxs().size).toBe(0);
+    });
+
+    it('should produce clean state when new tx is added after removal', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addDependency(a, b);
+      dg.removeTransactions(new Set([a]));
+
+      const c = dg.addTransaction('c', 300, 30);
+      expect(c.ancestors.size).toBe(1);
+      expect(c.descendants.size).toBe(1);
+      expect(c.ancestors.has(c)).toBe(true);
+      expect(b.ancestors.has(c)).toBe(false);
+    });
+  });
+
+  describe('findConnectedComponents after removal', () => {
+    it('should split into 2 components when bridge tx is removed', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      const d = dg.addTransaction('d', 400, 40);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+      dg.addDependency(d, c);
+
+      dg.removeTransactions(new Set([b]));
+      const components = dg.findConnectedComponents();
+      expect(components.length).toBe(2);
+      const sizes = components.map(comp => comp.size).sort((x, y) => x - y);
+      expect(sizes).toEqual([1, 2]);
+    });
+
+    it('should produce N singletons when center of star is removed', () => {
+      const { depgraph, center, leaves } = buildStar(5, 100, 10, 50, 10);
+      depgraph.removeTransactions(new Set([center]));
+      const components = depgraph.findConnectedComponents();
+      expect(components.length).toBe(5);
+      for (const comp of components) {
+        expect(comp.size).toBe(1);
+      }
+    });
+
+    it('should remain 1 component when non-bridge tx is removed', () => {
+      const { depgraph, txs } = buildDiamond([100, 200, 300, 400], [10, 20, 30, 40]);
+      depgraph.removeTransactions(new Set([txs[1]]));
+      const components = depgraph.findConnectedComponents();
+      expect(components.length).toBe(1);
+    });
+  });
+
+  describe('restrict edge cases', () => {
+    it('should restrict to a single tx', () => {
+      const { depgraph, txs } = buildChain(3, 100, 10);
+      const { depgraph: sub } = subgraph(new Set([txs[1]]));
+      expect(sub.size).toBe(1);
+    });
+
+    it('should preserve edges within partial chain subset', () => {
+      const { depgraph, txs } = buildChain(5, 100, 10);
+      const subset = new Set([txs[0], txs[1], txs[2]]);
+      const { depgraph: sub, txMap } = subgraph(subset);
+      expect(sub.size).toBe(3);
+
+      const newA = txMap.get(txs[0]);
+      const newB = txMap.get(txs[1]);
+      const newC = txMap.get(txs[2]);
+      if (!newA || !newB || !newC) {
+        throw new Error('txMap missing entries');
+      }
+      expect(newB.ancestors.has(newA)).toBe(true);
+      expect(newC.ancestors.has(newB)).toBe(true);
+    });
+  });
+
+  describe('sortTopological edge cases', () => {
+    it('should handle subset with no internal dependencies', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+
+      const output = sortTopological(new Set([a, b, c]));
+      expect(output.length).toBe(3);
+      expect(new Set(output).size).toBe(3);
+    });
+
+    it('should handle single-tx subset', () => {
+      const { txs } = buildChain(3, 100, 10);
+      const output = sortTopological(new Set([txs[1]]));
+      expect(output).toEqual([txs[1]]);
+    });
+  });
+
+  describe('addDependency edge cases', () => {
+    it('should be idempotent when adding the same dependency twice', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addDependency(a, b);
+      dg.addDependency(a, b);
+      expect(b.ancestors.size).toBe(2);
+      expect(a.descendants.size).toBe(2);
+    });
+
+    it('should handle redundant edge when parent is already transitive ancestor', () => {
+      const { depgraph, txs } = buildChain(3, 100, 10);
+      depgraph.addDependency(txs[0], txs[2]);
+      expect(txs[2].ancestors.size).toBe(3);
+      expect(txs[2].parents.size).toBe(2);
+    });
+  });
+});

backend/src/__tests__/cluster-mempool/linearize.test.ts

@@ -0,0 +1,485 @@
+import { DepGraph } from '../../cluster-mempool/depgraph';
+import { chunkify, postLinearize, spanningForestLinearize, linearizeCluster } from '../../cluster-mempool/linearize';
+import { buildChain, buildFanOut, buildStar, verifyLinearization, verifyTopologicalOrder } from './test-utils';
+
+describe('chunkify', () => {
+  it('should create one chunk per tx when feerates are decreasing', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 300, 10);
+    const b = dg.addTransaction('b', 200, 10);
+    const c = dg.addTransaction('c', 100, 10);
+
+    const chunks = chunkify([a, b, c]);
+    expect(chunks.length).toBe(3);
+    expect(chunks[0].txs).toEqual([a]);
+    expect(chunks[1].txs).toEqual([b]);
+    expect(chunks[2].txs).toEqual([c]);
+  });
+
+  it('should merge all into one chunk when feerates are increasing', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 10);
+    const b = dg.addTransaction('b', 200, 10);
+    const c = dg.addTransaction('c', 300, 10);
+
+    const chunks = chunkify([a, b, c]);
+    expect(chunks.length).toBe(1);
+    expect(chunks[0].txs).toEqual([a, b, c]);
+    expect(chunks[0].fee).toBe(600);
+    expect(chunks[0].weight).toBe(30);
+  });
+
+  it('should NOT merge equal feerates (matching Core behavior)', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 10);
+    const b = dg.addTransaction('b', 100, 10);
+
+    const chunks = chunkify([a, b]);
+    expect(chunks.length).toBe(2);
+  });
+
+  it('should handle single transaction', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 500, 50);
+
+    const chunks = chunkify([a]);
+    expect(chunks.length).toBe(1);
+    expect(chunks[0].fee).toBe(500);
+    expect(chunks[0].weight).toBe(50);
+  });
+
+  it('should handle empty linearization', () => {
+    const chunks = chunkify([]);
+    expect(chunks.length).toBe(0);
+  });
+
+  it('should produce decreasing chunk feerates', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 10);
+    const b = dg.addTransaction('b', 50, 10);
+    const c = dg.addTransaction('c', 200, 10);
+    const d = dg.addTransaction('d', 30, 10);
+
+    const chunks = chunkify([a, c, b, d]);
+    for (let i = 1; i < chunks.length; i++) {
+      const prevRate = chunks[i - 1].fee / chunks[i - 1].weight;
+      const curRate = chunks[i].fee / chunks[i].weight;
+      expect(prevRate).toBeGreaterThanOrEqual(curRate);
+    }
+  });
+});
+
+describe('postLinearize', () => {
+  it('should improve a bad linearization', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 500, 100);
+
+    const result = postLinearize([a, b]);
+    expect(result[0]).toBe(b);
+    expect(result[1]).toBe(a);
+  });
+
+  it('should not violate dependencies', () => {
+    const dg = new DepGraph();
+    const parent = dg.addTransaction('parent', 100, 100);
+    const child = dg.addTransaction('child', 500, 100);
+    dg.addDependency(parent, child);
+
+    const result = postLinearize([parent, child]);
+    expect(result.indexOf(parent)).toBeLessThan(result.indexOf(child));
+  });
+
+  it('should handle already-optimal ordering', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 500, 100);
+    const b = dg.addTransaction('b', 100, 100);
+
+    const result = postLinearize([a, b]);
+    expect(result).toEqual([a, b]);
+  });
+});
+
+describe('spanningForestLinearize', () => {
+  it('should sort independent transactions by feerate', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 500, 100);
+    const c = dg.addTransaction('c', 300, 100);
+
+    const result = spanningForestLinearize(dg.getTxs(), 75000);
+    expect(result[0]).toBe(b);
+    expect(result[1]).toBe(c);
+    expect(result[2]).toBe(a);
+  });
+
+  it('should respect dependencies', () => {
+    const dg = new DepGraph();
+    const parent = dg.addTransaction('parent', 100, 100);
+    const child = dg.addTransaction('child', 500, 100);
+    dg.addDependency(parent, child);
+
+    const result = spanningForestLinearize(dg.getTxs(), 75000);
+    expect(result.indexOf(parent)).toBeLessThan(result.indexOf(child));
+  });
+
+  it('should handle CPFP pattern', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 10000, 100);
+    dg.addDependency(a, b);
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(1);
+    expect(chunks[0].txs.length).toBe(2);
+  });
+
+  it('should separate high and low feerate independent txs', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 100);
+    const b = dg.addTransaction('b', 100, 100);
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(2);
+    expect(chunks[0].txs).toContain(a);
+    expect(chunks[1].txs).toContain(b);
+  });
+
+  it('should handle single transaction', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 100);
+
+    const result = spanningForestLinearize(dg.getTxs(), 75000);
+    expect(result).toEqual([a]);
+  });
+
+  it('should handle empty graph', () => {
+    const dg = new DepGraph();
+    const result = spanningForestLinearize(dg.getTxs(), 75000);
+    expect(result).toEqual([]);
+  });
+});
+
+describe('minimize', () => {
+  it('should keep equal-feerate chain as individual chunks', () => {
+    const dg = new DepGraph();
+    const txs: any[] = [];
+    for (let i = 0; i < 5; i++) {
+      txs.push(dg.addTransaction(`tx${i}`, 19, 140));
+    }
+    for (let i = 1; i < 5; i++) {
+      dg.addDependency(txs[i - 1], txs[i]);
+    }
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(5);
+    for (const chunk of chunks) {
+      expect(chunk.txs.length).toBe(1);
+    }
+  });
+
+  it('should merge chain where child has strictly higher feerate', () => {
+    const dg = new DepGraph();
+    const parent = dg.addTransaction('parent', 100, 200);
+    const child = dg.addTransaction('child', 900, 100);
+    dg.addDependency(parent, child);
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(1);
+    expect(chunks[0].txs.length).toBe(2);
+  });
+
+  it('should split parent-child with equal feerate', () => {
+    const dg = new DepGraph();
+    const parent = dg.addTransaction('parent', 1720, 344);
+    const child = dg.addTransaction('child', 1240, 248);
+    dg.addDependency(parent, child);
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(2);
+  });
+
+  it('should split disconnected equal-feerate components', () => {
+    const dg = new DepGraph();
+    dg.addTransaction('a', 100, 100);
+    dg.addTransaction('b', 100, 100);
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(2);
+    expect(chunks[0].txs.length).toBe(1);
+    expect(chunks[1].txs.length).toBe(1);
+  });
+});
+
+describe('chunkify edge cases', () => {
+  it('should produce N separate chunks when all feerates are equal', () => {
+    const dg = new DepGraph();
+    const txs: any[] = [];
+    for (let i = 0; i < 5; i++) {
+      txs.push(dg.addTransaction(`tx${i}`, 100, 10));
+    }
+    const chunks = chunkify(txs);
+    expect(chunks.length).toBe(5);
+  });
+
+  it('should handle alternating high/low feerates', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 10);
+    const b = dg.addTransaction('b', 100, 10);
+    const c = dg.addTransaction('c', 1000, 10);
+    const d = dg.addTransaction('d', 100, 10);
+
+    const chunks = chunkify([a, b, c, d]);
+    for (let i = 1; i < chunks.length; i++) {
+      const prevRate = chunks[i - 1].fee * chunks[i].weight;
+      const curRate = chunks[i].fee * chunks[i - 1].weight;
+      expect(prevRate).toBeGreaterThanOrEqual(curRate);
+    }
+  });
+
+  it('should merge all when single very high feerate tx is at the end', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 10, 10);
+    const b = dg.addTransaction('b', 10, 10);
+    const c = dg.addTransaction('c', 10, 10);
+    const d = dg.addTransaction('d', 10000, 10);
+
+    const chunks = chunkify([a, b, c, d]);
+    expect(chunks.length).toBe(1);
+    expect(chunks[0].txs.length).toBe(4);
+  });
+
+  it('should maintain non-increasing feerates for 50+ tx linearization', () => {
+    const dg = new DepGraph();
+    const txs: any[] = [];
+    for (let i = 0; i < 50; i++) {
+      txs.push(dg.addTransaction(`tx${i}`, 5000 - i * 100, 100));
+    }
+    const chunks = chunkify(txs);
+    for (let i = 1; i < chunks.length; i++) {
+      const prevRate = chunks[i - 1].fee * chunks[i].weight;
+      const curRate = chunks[i].fee * chunks[i - 1].weight;
+      expect(prevRate).toBeGreaterThanOrEqual(curRate);
+    }
+  });
+
+  it('should handle zero-fee transaction', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 100);
+    const b = dg.addTransaction('b', 0, 100);
+
+    const chunks = chunkify([a, b]);
+    expect(chunks.length).toBe(2);
+    expect(chunks[1].fee).toBe(0);
+  });
+});
+
+describe('postLinearize edge cases', () => {
+  it('should sort three independent txs by feerate', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 500, 100);
+    const c = dg.addTransaction('c', 300, 100);
+
+    const result = postLinearize([a, c, b]);
+    expect(result[0]).toBe(b);
+    expect(result[2]).toBe(a);
+  });
+
+  it('should respect parent-child dependency even when child has higher feerate', () => {
+    const dg = new DepGraph();
+    const parent = dg.addTransaction('parent', 100, 100);
+    const child = dg.addTransaction('child', 1000, 100);
+    dg.addDependency(parent, child);
+
+    const result = postLinearize([parent, child]);
+    expect(result[0]).toBe(parent);
+    expect(result[1]).toBe(child);
+  });
+
+  it('should handle chain A→B→C with CPFP-like feerates', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 200, 100);
+    const c = dg.addTransaction('c', 10000, 100);
+    dg.addDependency(a, b);
+    dg.addDependency(b, c);
+
+    const result = postLinearize([a, b, c]);
+    verifyTopologicalOrder(result);
+  });
+});
+
+describe('SFL adversarial topologies', () => {
+  it('should handle comb pattern: one root with many children at different feerates', () => {
+    const dg = new DepGraph();
+    const root = dg.addTransaction('root', 100, 100);
+    for (let i = 0; i < 8; i++) {
+      const child = dg.addTransaction(`child${i}`, (i + 1) * 500, 100);
+      dg.addDependency(root, child);
+    }
+
+    const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000);
+    verifyLinearization(dg.getTxs(), linearization, chunks);
+  });
+
+  it('should handle inverted tree: many leaves → intermediates → root', () => {
+    const dg = new DepGraph();
+    const root = dg.addTransaction('root', 100, 100);
+    const mid1 = dg.addTransaction('mid1', 200, 100);
+    const mid2 = dg.addTransaction('mid2', 300, 100);
+    dg.addDependency(root, mid1);
+    dg.addDependency(root, mid2);
+
+    for (let i = 0; i < 4; i++) {
+      const leaf = dg.addTransaction(`leaf${i}`, 5000, 100);
+      dg.addDependency(i < 2 ? mid1 : mid2, leaf);
+    }
+
+    const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000);
+    verifyLinearization(dg.getTxs(), linearization, chunks);
+  });
+
+  it('should handle two parallel chains with shared root', () => {
+    const dg = new DepGraph();
+    const root = dg.addTransaction('root', 100, 100);
+
+    let prev1: any = root;
+    for (let i = 0; i < 5; i++) {
+      const tx = dg.addTransaction(`chain1_${i}`, 200, 100);
+      dg.addDependency(prev1, tx);
+      prev1 = tx;
+    }
+
+    let prev2: any = root;
+    for (let i = 0; i < 3; i++) {
+      const tx = dg.addTransaction(`chain2_${i}`, 300, 100);
+      dg.addDependency(prev2, tx);
+      prev2 = tx;
+    }
+
+    const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000);
+    verifyLinearization(dg.getTxs(), linearization, chunks);
+  });
+
+  it('should handle deep CPFP: low-fee chain with high-fee tip', () => {
+    const { depgraph, txs } = buildChain(6, 10, 100);
+    txs[5].effectiveFee = 50000;
+
+    const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), 75000);
+    verifyLinearization(depgraph.getTxs(), linearization, chunks);
+    expect(chunks[0].txs.length).toBeGreaterThan(1);
+  });
+
+  it('should find better result than ancestor-feerate for overlapping high-feerate subsets', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 100, 100);
+    const c = dg.addTransaction('c', 10000, 100);
+    dg.addDependency(a, c);
+    dg.addDependency(b, c);
+
+    const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000 );
+    verifyLinearization(dg.getTxs(), linearization, chunks);
+    const firstChunkFee = chunks[0].fee;
+    const firstChunkSize = chunks[0].weight;
+    expect(firstChunkFee / firstChunkSize).toBeGreaterThan(100 / 100);
+  });
+});
+
+describe('linearizeCluster', () => {
+  it('should produce valid topological linearization', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 10);
+    const b = dg.addTransaction('b', 200, 20);
+    const c = dg.addTransaction('c', 300, 30);
+    dg.addDependency(a, b);
+    dg.addDependency(b, c);
+
+    const { linearization } = linearizeCluster(dg.getTxs(), 75000);
+    expect(linearization.indexOf(a)).toBeLessThan(linearization.indexOf(b));
+    expect(linearization.indexOf(b)).toBeLessThan(linearization.indexOf(c));
+  });
+
+  it('should produce monotonically decreasing chunk feerates', () => {
+    const dg = new DepGraph();
+    for (let i = 0; i < 10; i++) {
+      dg.addTransaction(`tx${i}`, Math.floor(Math.random() * 10000) + 100, Math.floor(Math.random() * 500) + 50);
+    }
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    for (let i = 1; i < chunks.length; i++) {
+      const prevRate = chunks[i - 1].fee * chunks[i].weight;
+      const curRate = chunks[i].fee * chunks[i - 1].weight;
+      expect(prevRate).toBeGreaterThanOrEqual(curRate);
+    }
+  });
+
+  it('should handle complex diamond dependency graph', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 100);
+    const b = dg.addTransaction('b', 500, 100);
+    const c = dg.addTransaction('c', 100, 100);
+    const d = dg.addTransaction('d', 300, 100);
+
+    dg.addDependency(a, b);
+    dg.addDependency(a, c);
+    dg.addDependency(b, d);
+    dg.addDependency(c, d);
+
+    const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000);
+    verifyLinearization(dg.getTxs(), linearization, chunks);
+  });
+
+  it('should produce at-least-as-good result when given suboptimal hint', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 500, 100);
+    const b = dg.addTransaction('b', 100, 100);
+    const c = dg.addTransaction('c', 1000, 100);
+
+    const suboptimal = [b, a, c];
+    const { chunks: hintChunks } = linearizeCluster(dg.getTxs(), 75000, suboptimal);
+    const { chunks: freshChunks } = linearizeCluster(dg.getTxs(), 75000);
+
+    const hintFirstFeerate = hintChunks[0].fee * freshChunks[0].weight;
+    const freshFirstFeerate = freshChunks[0].fee * hintChunks[0].weight;
+    expect(hintFirstFeerate).toBeGreaterThanOrEqual(freshFirstFeerate - 1);
+  });
+
+  it('should preserve an already-optimal linearization', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 100);
+    const b = dg.addTransaction('b', 500, 100);
+    const c = dg.addTransaction('c', 100, 100);
+
+    const optimal = [a, b, c];
+    const { linearization } = linearizeCluster(dg.getTxs(), 75000, optimal);
+    expect(linearization).toEqual(optimal);
+  });
+
+  it('should produce valid linearizations on repeated calls', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 100, 100);
+    const c = dg.addTransaction('c', 100, 100);
+    dg.addDependency(a, c);
+    dg.addDependency(b, c);
+
+    const result1 = linearizeCluster(dg.getTxs(), 75000);
+    const result2 = linearizeCluster(dg.getTxs(), 75000);
+    verifyLinearization(dg.getTxs(), result1.linearization, result1.chunks);
+    verifyLinearization(dg.getTxs(), result2.linearization, result2.chunks);
+  });
+
+  it('should pass invariant checks for fan-out topology', () => {
+    const { depgraph } = buildFanOut(6, 100, 100, 500, 100);
+    const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), 75000);
+    verifyLinearization(depgraph.getTxs(), linearization, chunks);
+  });
+
+  it('should pass invariant checks for star topology', () => {
+    const { depgraph } = buildStar(5, 100, 100, 300, 100);
+    const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), 75000);
+    verifyLinearization(depgraph.getTxs(), linearization, chunks);
+  });
+});

backend/src/__tests__/cluster-mempool/test-utils.ts

@@ -0,0 +1,175 @@
+import { MempoolTransactionExtended } from '../../mempool.interfaces';
+import { ClusterTx, DepGraph } from '../../cluster-mempool/depgraph';
+import { LinearizationChunk } from '../../cluster-mempool/linearize';
+
+export function makeTx(
+  txid: string,
+  fee: number,
+  vsize: number,
+  parentTxids: string[] = [],
+): MempoolTransactionExtended {
+  const vin = parentTxids.length > 0
+    ? parentTxids.map(ptxid => ({
+        txid: ptxid,
+        vout: 0,
+        is_coinbase: false,
+        scriptsig: '',
+        scriptsig_asm: '',
+        inner_redeemscript_asm: '',
+        inner_witnessscript_asm: '',
+        sequence: 0,
+        witness: [] as string[],
+        prevout: null,
+      }))
+    : [{
+        txid: '0000000000000000000000000000000000000000000000000000000000000000',
+        vout: 0,
+        is_coinbase: false,
+        scriptsig: '',
+        scriptsig_asm: '',
+        inner_redeemscript_asm: '',
+        inner_witnessscript_asm: '',
+        sequence: 0,
+        witness: [] as string[],
+        prevout: null,
+      }];
+
+  return {
+    txid,
+    version: 2,
+    locktime: 0,
+    size: vsize,
+    weight: vsize * 4,
+    fee,
+    vin,
+    vout: [{
+      scriptpubkey: '',
+      scriptpubkey_asm: '',
+      scriptpubkey_type: 'v0_p2wpkh',
+      value: 50000,
+    }],
+    status: { confirmed: false },
+    vsize,
+    feePerVsize: fee / vsize,
+    effectiveFeePerVsize: fee / vsize,
+    order: 0,
+    sigops: 0,
+    adjustedVsize: vsize,
+    adjustedFeePerVsize: fee / vsize,
+  } as MempoolTransactionExtended;
+}
+
+export function txid(short: string): string {
+  return short.padStart(64, '0');
+}
+
+export function buildChain(
+  n: number,
+  baseFee: number,
+  baseSize: number,
+): { depgraph: DepGraph; txs: ClusterTx[] } {
+  const depgraph = new DepGraph();
+  const txs: ClusterTx[] = [];
+  for (let i = 0; i < n; i++) {
+    txs.push(depgraph.addTransaction(`chain_${i}`, baseFee, baseSize));
+  }
+  for (let i = 1; i < n; i++) {
+    depgraph.addDependency(txs[i - 1], txs[i]);
+  }
+  return { depgraph, txs };
+}
+
+export function buildFanOut(
+  nChildren: number,
+  parentFee: number,
+  parentSize: number,
+  childFee: number,
+  childSize: number,
+): { depgraph: DepGraph; parent: ClusterTx; children: ClusterTx[] } {
+  const depgraph = new DepGraph();
+  const parent = depgraph.addTransaction('fanout_parent', parentFee, parentSize);
+  const children: ClusterTx[] = [];
+  for (let i = 0; i < nChildren; i++) {
+    const child = depgraph.addTransaction(`fanout_child_${i}`, childFee, childSize);
+    depgraph.addDependency(parent, child);
+    children.push(child);
+  }
+  return { depgraph, parent, children };
+}
+
+export function buildDiamond(
+  fees: [number, number, number, number],
+  sizes: [number, number, number, number],
+): { depgraph: DepGraph; txs: [ClusterTx, ClusterTx, ClusterTx, ClusterTx] } {
+  const depgraph = new DepGraph();
+  const a = depgraph.addTransaction('diamond_a', fees[0], sizes[0]);
+  const b = depgraph.addTransaction('diamond_b', fees[1], sizes[1]);
+  const c = depgraph.addTransaction('diamond_c', fees[2], sizes[2]);
+  const d = depgraph.addTransaction('diamond_d', fees[3], sizes[3]);
+  depgraph.addDependency(a, b);
+  depgraph.addDependency(a, c);
+  depgraph.addDependency(b, d);
+  depgraph.addDependency(c, d);
+  return { depgraph, txs: [a, b, c, d] };
+}
+
+export function buildStar(
+  nLeaves: number,
+  centerFee: number,
+  centerSize: number,
+  leafFee: number,
+  leafSize: number,
+): { depgraph: DepGraph; center: ClusterTx; leaves: ClusterTx[] } {
+  const depgraph = new DepGraph();
+  const center = depgraph.addTransaction('star_center', centerFee, centerSize);
+  const leaves: ClusterTx[] = [];
+  for (let i = 0; i < nLeaves; i++) {
+    const leaf = depgraph.addTransaction(`star_leaf_${i}`, leafFee, leafSize);
+    depgraph.addDependency(center, leaf);
+    leaves.push(leaf);
+  }
+  return { depgraph, center, leaves };
+}
+
+export function verifyTopologicalOrder(ordering: ClusterTx[]): void {
+  const positionMap = new Map<ClusterTx, number>();
+  for (let i = 0; i < ordering.length; i++) {
+    positionMap.set(ordering[i], i);
+  }
+  for (const tx of ordering) {
+    for (const parent of tx.parents) {
+      const parentPos = positionMap.get(parent);
+      const childPos = positionMap.get(tx);
+      if (parentPos !== undefined && childPos !== undefined) {
+        expect(parentPos).toBeLessThan(childPos);
+      }
+    }
+  }
+}
+
+export function verifyLinearization(
+  txs: Set<ClusterTx>,
+  linearization: ClusterTx[],
+  chunks: LinearizationChunk[],
+): void {
+  expect(linearization.length).toBe(txs.size);
+  const linSet = new Set(linearization);
+  expect(linSet.size).toBe(linearization.length);
+  for (const tx of txs) {
+    expect(linSet.has(tx)).toBe(true);
+  }
+
+  verifyTopologicalOrder(linearization);
+
+  for (let i = 1; i < chunks.length; i++) {
+    const prevRate = chunks[i - 1].fee * chunks[i].weight;
+    const curRate = chunks[i].fee * chunks[i - 1].weight;
+    expect(prevRate).toBeGreaterThanOrEqual(curRate);
+  }
+
+  const chunkTxs = chunks.flatMap(c => c.txs);
+  expect(chunkTxs.length).toBe(linearization.length);
+  for (let i = 0; i < chunkTxs.length; i++) {
+    expect(chunkTxs[i]).toBe(linearization[i]);
+  }
+}

backend/src/__tests__/config.test.ts

@@ -45,6 +45,8 @@ describe('Mempool Backend Config', () => {
         AUDIT: false,
         RUST_GBT: true,
         LIMIT_GBT: false,
+        CLUSTER_MEMPOOL: false,
+        CLUSTER_MEMPOOL_INDEXING: false,
         CPFP_INDEXING: false,
         MAX_BLOCKS_BULK_QUERY: 0,
         DISK_CACHE_BLOCK_INTERVAL: 6,

backend/src/api/about.routes.ts

@@ -3,6 +3,8 @@ import config from '../config';
 import axios from 'axios';
 import logger from '../logger';
 
+const PROXY_PATH_SEGMENT_REGEX = /^(?!\.{1,2}$)[^\p{Cc}/?#\\]{1,256}$/u;
+
 class AboutRoutes {
   public initRoutes(app: Application) {
     app
@@ -15,8 +17,13 @@ class AboutRoutes {
         }
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => {
+        if (!PROXY_PATH_SEGMENT_REGEX.test(req.params.id)) {
+          res.status(400).end();
+          return;
+        }
+
         try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${req.params.id}`, {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${encodeURIComponent(req.params.id)}`, {
             responseType: 'stream', timeout: 10000
           });
           response.data.pipe(res);
@@ -33,8 +40,13 @@ class AboutRoutes {
         }
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => {
+        if (!PROXY_PATH_SEGMENT_REGEX.test(req.params.id)) {
+          res.status(400).end();
+          return;
+        }
+
         try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${req.params.id}`, {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${encodeURIComponent(req.params.id)}`, {
             responseType: 'stream', timeout: 10000
           });
           response.data.pipe(res);
@@ -51,8 +63,13 @@ class AboutRoutes {
         }
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => {
+        if (!PROXY_PATH_SEGMENT_REGEX.test(req.params.id)) {
+          res.status(400).end();
+          return;
+        }
+
         try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${req.params.id}`, {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${encodeURIComponent(req.params.id)}`, {
             responseType: 'stream', timeout: 10000
           });
           response.data.pipe(res);
@@ -61,7 +78,7 @@ class AboutRoutes {
         }
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'services/sponsors', async (req, res) => {
-        const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+        const url = `${config.MEMPOOL_SERVICES.API}/sponsors`;
         try {
           const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
           response.data.pipe(res);
@@ -71,7 +88,12 @@ class AboutRoutes {
         }
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'services/account/images/:username/:md5', async (req, res) => {
-        const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+        if (!PROXY_PATH_SEGMENT_REGEX.test(req.params.username) || !PROXY_PATH_SEGMENT_REGEX.test(req.params.md5)) {
+          res.status(400).end();
+          return;
+        }
+
+        const url = `${config.MEMPOOL_SERVICES.API}/account/images/${encodeURIComponent(req.params.username)}/${encodeURIComponent(req.params.md5)}`;
         try {
           const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
           response.data.pipe(res);
@@ -84,4 +106,4 @@ class AboutRoutes {
   }
 }
 
-export default new AboutRoutes();
+export default new AboutRoutes();

backend/src/api/acceleration/acceleration.routes.ts

@@ -5,16 +5,18 @@ import logger from '../../logger';
 import mempool from '../mempool';
 import AccelerationRepository from '../../repositories/AccelerationRepository';
 
+const TXID_REGEX = /^[a-f0-9]{64}$/i;
+
 class AccelerationRoutes {
   private tag = 'Accelerator';
 
   public initRoutes(app: Application): void {
     app
       .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations', this.$getAcceleratorAccelerations.bind(this))
-      .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/:txid', this.$getAcceleratorAcceleration.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/history', this.$getAcceleratorAccelerationsHistory.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/history/aggregated', this.$getAcceleratorAccelerationsHistoryAggregated.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/stats', this.$getAcceleratorAccelerationsStats.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/:txid', this.$getAcceleratorAcceleration.bind(this))
       .post(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/estimate', this.$getAcceleratorEstimate.bind(this))
     ;
   }
@@ -26,7 +28,7 @@ class AccelerationRoutes {
 
   /** @asyncUnsafe */
   private async $getAcceleratorAcceleration(req: Request, res: Response): Promise<void> {
-    if (req.params.txid) {
+    if (req.params.txid && TXID_REGEX.test(req.params.txid)) {
       const acceleration = await AccelerationRepository.$getAccelerationInfoForTxid(req.params.txid);
       if (acceleration) {
         res.status(200).send(acceleration);
@@ -34,7 +36,7 @@ class AccelerationRoutes {
         res.status(404).send('Acceleration not found');
       }
     } else {
-      res.status(400).send('txid is required');
+      res.status(400).send('invalid txid');
     }
   }
 
@@ -55,9 +57,9 @@ class AccelerationRoutes {
   }
 
   private async $getAcceleratorAccelerationsHistoryAggregated(req: Request, res: Response): Promise<void> {
-    const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+    const url = `${config.MEMPOOL_SERVICES.API}/accelerator/accelerations/history/aggregated`;
     try {
-      const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
+      const response = await axios.get(url, { params: req.query, responseType: 'stream', timeout: 10000 });
       for (const key in response.headers) {
         res.setHeader(key, response.headers[key]);
       }
@@ -69,9 +71,9 @@ class AccelerationRoutes {
   }
 
   private async $getAcceleratorAccelerationsStats(req: Request, res: Response): Promise<void> {
-    const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+    const url = `${config.MEMPOOL_SERVICES.API}/accelerator/accelerations/stats`;
     try {
-      const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
+      const response = await axios.get(url, { params: req.query, responseType: 'stream', timeout: 10000 });
       for (const key in response.headers) {
         res.setHeader(key, response.headers[key]);
       }
@@ -83,7 +85,7 @@ class AccelerationRoutes {
   }
 
   private async $getAcceleratorEstimate(req: Request, res: Response): Promise<void> {
-    const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+    const url = `${config.MEMPOOL_SERVICES.API}/accelerator/estimate`;
     try {
       const response = await axios.post(url, req.body, { responseType: 'stream', timeout: 10000 });
       for (const key in response.headers) {
@@ -97,4 +99,4 @@ class AccelerationRoutes {
   }
 }
 
-export default new AccelerationRoutes();
+export default new AccelerationRoutes();

backend/src/api/audit.ts

@@ -6,11 +6,28 @@ import transactionUtils from './transaction-utils';
 
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
 
+export interface AuditResult {
+  unseen: string[];
+  censored: string[];
+  added: string[];
+  prioritized: string[];
+  fresh: string[];
+  sigop: string[];
+  fullrbf: string[];
+  accelerated: string[];
+  matchRate: number;
+  similarity: number;
+}
+
 class Audit {
-  auditBlock(height: number, transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended })
-   : { unseen: string[], censored: string[], added: string[], prioritized: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } {
+  auditBlock(
+    height: number,
+    transactions: MempoolTransactionExtended[],
+    projectedBlocks: MempoolBlockWithTransactions[],
+    mempool: { [txId: string]: MempoolTransactionExtended }
+  ): AuditResult {
     if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
-      return { unseen: [], censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 1, similarity: 1 };
+      return { unseen: [], censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], matchRate: 100, similarity: 1 };
     }
 
     const matches: string[] = []; // present in both mined block and template
@@ -176,6 +193,8 @@ class Audit {
     }
     const similarity = projectedWeight ? matchedWeight / projectedWeight : 1;
 
+    const matchRate = Math.round(score * 100 * 100) / 100;
+
     return {
       unseen,
       censored: Object.keys(isCensored),
@@ -185,7 +204,7 @@ class Audit {
       sigop: [],
       fullrbf: rbf,
       accelerated,
-      score,
+      matchRate,
       similarity,
     };
   }

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -57,4 +57,19 @@ export interface HealthCheckHost {
   unreachable: boolean;
   checked: boolean;
   lastChecked: number;
+  hashes?: {
+    frontend?: string;
+    hybrid?: string;
+    backend?: string;
+    electrs?: string;
+    ssr?: string;
+    core?: string;
+    os?: string;
+    lastUpdated?: number;
+  };
+  liquidAudit?: {
+    pegRatio: number;
+    bitcoinLastBlockUpdate: number;
+    liquidLastBlockUpdate: number;
+  };
 }

backend/src/api/bitcoin/bitcoin-api.interface.ts

@@ -53,6 +53,7 @@ export namespace IBitcoinApi {
     nTx: number;                     //  (numeric) The number of transactions in the block
     previousblockhash: string;       //  (string) The hash of the previous block
     nextblockhash: string;           //  (string) The hash of the next block
+    dynamic_parameters?: any;        //  (object) Elements only: dynamic parameters at this block
   }
 
   export interface Transaction {
@@ -90,6 +91,7 @@ export namespace IBitcoinApi {
     };
     sequence: number;                //  (numeric) The script sequence number
     txinwitness?: string[];          //  (string) hex-encoded witness data
+    pegin_witness?: string[];        //  (string) Elements peg-in witness
     coinbase?: string;
     is_pegin?: boolean;              //  (boolean) Elements peg-in
   }

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -12,7 +12,7 @@ import backendInfo from '../backend-info';
 import transactionUtils from '../transaction-utils';
 import { IEsploraApi } from './esplora-api.interface';
 import loadingIndicators from '../loading-indicators';
-import { TransactionExtended } from '../../mempool.interfaces';
+import { CpfpInfo, TransactionExtended } from '../../mempool.interfaces';
 import logger from '../../logger';
 import blocks from '../blocks';
 import bitcoinClient from './bitcoin-client';
@@ -23,11 +23,14 @@ import { calculateMempoolTxCpfp } from '../cpfp';
 import { handleError } from '../../utils/api';
 import poolsUpdater from '../../tasks/pools-updater';
 import chainTips from '../chain-tips';
+import FlagValueRepository, { INTERVAL_PRESETS } from '../../repositories/FlagValueRepository';
 
 const TXID_REGEX = /^[a-f0-9]{64}$/i;
 const BLOCK_HASH_REGEX = /^[a-f0-9]{64}$/i;
 const ADDRESS_REGEX = /^[a-z0-9]{2,120}$/i;
 const SCRIPT_HASH_REGEX = /^([a-f0-9]{2})+$/i;
+const MAX_TRANSACTION_TIMES = 100;
+const JUST_NUMBERS_REGEX = /^[1-9]\d*$/;
 
 class BitcoinRoutes {
   public initRoutes(app: Application) {
@@ -37,6 +40,7 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', this.getDifficultyChange)
       .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', this.getRecommendedFees)
       .get(config.MEMPOOL.API_URL_PREFIX + 'fees/precise', this.getPreciseRecommendedFees)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'fee-estimates', this.getPreciseRecommendedFeesEsploraTransformed)
       .get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', this.getMempoolBlocks)
       .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', this.getBackendInfo)
       .get(config.MEMPOOL.API_URL_PREFIX + 'init-data', this.getInitData)
@@ -59,6 +63,7 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'chain-tips', this.getChainTips.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'stale-tips', this.getStaleTips.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'stale-tips/:height', this.getStaleTips.bind(this))
       .post(config.MEMPOOL.API_URL_PREFIX + 'prevouts', this.$getPrevouts)
       .post(config.MEMPOOL.API_URL_PREFIX + 'cpfp', this.getCpfpLocalTxs)
       // Temporarily add txs/package endpoint for all backends until esplora supports it
@@ -67,6 +72,10 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/definition/list', this.getBlockDefinitionHashes)
       .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/definition/current', this.getCurrentBlockDefinitionHash)
       .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/:definitionHash', this.getBlocksByDefinitionHash)
+
+      .get(config.MEMPOOL.API_URL_PREFIX + 'goggles/:interval/', this.getTxCountPerFlagValue)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'goggles/:interval/:bucketSize', this.getTxCountPerFlagValue)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'goggles/:interval/:bucketSize/:op/:mask', this.getTxCountPerFlagValue)
       ;
 
       if (config.MEMPOOL.BACKEND !== 'esplora') {
@@ -133,6 +142,58 @@ class BitcoinRoutes {
     res.json(result);
   }
 
+  private getPreciseRecommendedFeesEsploraTransformed(req: Request, res: Response) {
+    if (!mempool.isInSync()) {
+      res.statusCode = 503;
+      res.send('Service Unavailable');
+      return;
+    }
+    const result = feeApi.getPreciseRecommendedFee();
+
+    // Note: Manually tested against rust-esplora-client 0.12.3
+    // to make sure the status code and extra header didn't blow everything up.
+    // 3xx redirects with a Location header were silently followed and caused the client to
+    // blow up, 4xx and 5xx caused errors in the client as well, so only 2xx was acceptable for this endpoint.
+
+    // HTTP 203 Non-Authoritative Information is used to indicate
+    // that the response is not the original from the server, but a transformed version of it.
+    res.statusCode = 203;
+    res.setHeader(
+      'x-warning-from-mempool',
+      `This endpoint is deprecated and will be removed in a future release. Please use /api/v1/fees/precise instead.`,
+    );
+    res.json({
+      '1': result.fastestFee,
+      '2': result.fastestFee,
+      '3': result.halfHourFee,
+      '4': result.halfHourFee,
+      '5': result.halfHourFee,
+      '6': result.hourFee,
+      '7': result.hourFee,
+      '8': result.hourFee,
+      '9': result.hourFee,
+      '10': result.hourFee,
+      '11': result.hourFee,
+      '12': result.hourFee,
+      '13': result.hourFee,
+      '14': result.hourFee,
+      '15': result.hourFee,
+      '16': result.hourFee,
+      '17': result.hourFee,
+      '18': result.hourFee,
+      '19': result.hourFee,
+      '20': result.hourFee,
+      '21': result.hourFee,
+      '22': result.hourFee,
+      '23': result.hourFee,
+      '24': result.hourFee,
+      '25': result.hourFee,
+      '144': result.economyFee,
+      '504': result.economyFee,
+      '1008': result.minimumFee,
+    });
+  }
+
   private getMempoolBlocks(req: Request, res: Response) {
     try {
       const result = mempoolBlocks.getMempoolBlocks();
@@ -144,11 +205,18 @@ class BitcoinRoutes {
 
   private getTransactionTimes(req: Request, res: Response) {
     if (!req.query.txId || typeof req.query.txId !== 'object') {
-      handleError(req, res, 500, 'invalid txId format');
+      handleError(req, res, 400, 'invalid txId format');
       return;
     }
+
+    const requestedTxIds = Object.values(req.query.txId);
+    if (requestedTxIds.length > MAX_TRANSACTION_TIMES) {
+      handleError(req, res, 400, 'Too many txids requested');
+      return;
+    }
+
     const txIds: string[] = [];
-    for (const txid of Object.values(req.query.txId)) {
+    for (const txid of requestedTxIds) {
       if (typeof txid === 'string' && TXID_REGEX.test(txid)) {
         txIds.push(txid);
       }
@@ -161,7 +229,7 @@ class BitcoinRoutes {
   private async $getBatchedOutspends(req: Request, res: Response): Promise<IEsploraApi.Outspend[][] | void> {
     const txids_csv = req.query.txids;
     if (!txids_csv || typeof txids_csv !== 'string') {
-      handleError(req, res, 500, 'Invalid txids format');
+      handleError(req, res, 400, 'Invalid txids format');
       return;
     }
     const txids = txids_csv.split(',');
@@ -184,18 +252,18 @@ class BitcoinRoutes {
 
   private async $getCpfpInfo(req: Request, res: Response) {
     if (!TXID_REGEX.test(req.params.txId)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
 
     const tx = mempool.getMempool()[req.params.txId];
     if (tx) {
       if (tx?.cpfpChecked) {
-        res.json({
-          ancestors: tx.ancestors,
+        const response: CpfpInfo & { acceleratedBy?: number[], acceleratedAt?: number, feeDelta?: number } = {
+          ancestors: tx.ancestors || [],
           bestDescendant: tx.bestDescendant || null,
-          descendants: tx.descendants || null,
-          effectiveFeePerVsize: tx.effectiveFeePerVsize || null,
+          descendants: tx.descendants,
+          effectiveFeePerVsize: tx.effectiveFeePerVsize,
           sigops: tx.sigops,
           fee: tx.fee,
           adjustedVsize: tx.adjustedVsize,
@@ -203,7 +271,14 @@ class BitcoinRoutes {
           acceleratedBy: tx.acceleratedBy || undefined,
           acceleratedAt: tx.acceleratedAt || undefined,
           feeDelta: tx.feeDelta || undefined,
-        });
+        };
+        if (config.MEMPOOL.CLUSTER_MEMPOOL && tx.clusterId != null) {
+          const cluster = mempool.clusterMempool?.getClusterForApi(req.params.txId);
+          if (cluster) {
+            response.cluster = cluster;
+          }
+        }
+        res.json(response);
         return;
       }
 
@@ -239,7 +314,7 @@ class BitcoinRoutes {
 
   private async getTransaction(req: Request, res: Response) {
     if (!TXID_REGEX.test(req.params.txId)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -258,7 +333,7 @@ class BitcoinRoutes {
 
   private async getRawTransaction(req: Request, res: Response) {
     if (!TXID_REGEX.test(req.params.txId)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -346,7 +421,7 @@ class BitcoinRoutes {
 
   private async getTransactionStatus(req: Request, res: Response) {
     if (!TXID_REGEX.test(req.params.txId)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -365,7 +440,7 @@ class BitcoinRoutes {
 
   private async getStrippedBlockTransactions(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     try {
@@ -379,11 +454,11 @@ class BitcoinRoutes {
 
   private async getStrippedBlockTransaction(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     if (!TXID_REGEX.test(req.params.txid)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -401,7 +476,7 @@ class BitcoinRoutes {
 
   private async getBlock(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     try {
@@ -427,7 +502,7 @@ class BitcoinRoutes {
 
   private async getBlockHeader(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     try {
@@ -441,7 +516,7 @@ class BitcoinRoutes {
 
   private async getBlockAuditSummary(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     try {
@@ -460,11 +535,11 @@ class BitcoinRoutes {
 
   private async $getBlockTxAuditSummary(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     if (!TXID_REGEX.test(req.params.txid)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -560,14 +635,18 @@ class BitcoinRoutes {
   private async getStaleTips(req: Request, res: Response) {
     try {
       if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
-        res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-        const tips = await chainTips.getStaleTips();
-        if (tips.length > 0) {
-          res.json(tips);
-        } else {
-          handleError(req, res, 503, `Temporarily unavailable`);
-          return;
+        let fromHeight: number | undefined;
+        if (req.params.height !== undefined) {
+          fromHeight = parseInt(req.params.height, 10);
+          if (isNaN(fromHeight) || fromHeight < 0) {
+            handleError(req, res, 400, `Parameter 'height' must be a block height (integer)`);
+            return;
+          }
         }
+
+        res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+        const tips = await chainTips.$getStaleTipsPage(fromHeight, 25);
+        res.json(tips);
       } else { // Liquid
         handleError(req, res, 404, `This API is only available for Bitcoin networks`);
         return;
@@ -614,7 +693,7 @@ class BitcoinRoutes {
 
   private async getBlockTransactions(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     try {
@@ -656,7 +735,7 @@ class BitcoinRoutes {
       return;
     }
     if (!ADDRESS_REGEX.test(req.params.address)) {
-      handleError(req, res, 501, `Invalid address`);
+      handleError(req, res, 400, `Invalid address`);
       return;
     }
 
@@ -664,6 +743,10 @@ class BitcoinRoutes {
       const addressData = await bitcoinApi.$getAddress(req.params.address);
       res.json(addressData);
     } catch (e) {
+      if (e instanceof Error && e.message === 'Invalid Bitcoin address') {
+        res.status(400).send(e.message);
+        return;
+      }
       if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
         handleError(req, res, 413, e.message);
         return;
@@ -678,7 +761,7 @@ class BitcoinRoutes {
       return;
     }
     if (!ADDRESS_REGEX.test(req.params.address)) {
-      handleError(req, res, 501, `Invalid address`);
+      handleError(req, res, 400, `Invalid address`);
       return;
     }
 
@@ -690,6 +773,10 @@ class BitcoinRoutes {
       const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, lastTxId);
       res.json(transactions);
     } catch (e) {
+      if (e instanceof Error && e.message === 'Invalid Bitcoin address') {
+        res.status(400).send(e.message);
+        return;
+      }
       if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
         handleError(req, res, 413, e.message);
         return;
@@ -704,7 +791,7 @@ class BitcoinRoutes {
       return;
     }
     if (!ADDRESS_REGEX.test(req.params.address)) {
-      handleError(req, res, 501, `Invalid address`);
+      handleError(req, res, 400, `Invalid address`);
       return;
     }
 
@@ -712,6 +799,10 @@ class BitcoinRoutes {
       const addressData = await bitcoinApi.$getAddressUtxos(req.params.address);
       res.json(addressData);
     } catch (e) {
+      if (e instanceof Error && e.message === 'Invalid Bitcoin address') {
+        res.status(400).send(e.message);
+        return;
+      }
       if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
         handleError(req, res, 413, e.message);
         return;
@@ -733,7 +824,7 @@ class BitcoinRoutes {
       return;
     }
     if (!SCRIPT_HASH_REGEX.test(req.params.scripthash)) {
-      handleError(req, res, 501, `Invalid scripthash`);
+      handleError(req, res, 400, `Invalid scripthash`);
       return;
     }
 
@@ -757,7 +848,7 @@ class BitcoinRoutes {
       return;
     }
     if (!SCRIPT_HASH_REGEX.test(req.params.scripthash)) {
-      handleError(req, res, 501, `Invalid scripthash`);
+      handleError(req, res, 400, `Invalid scripthash`);
       return;
     }
 
@@ -785,7 +876,7 @@ class BitcoinRoutes {
       return;
     }
     if (!SCRIPT_HASH_REGEX.test(req.params.scripthash)) {
-      handleError(req, res, 501, `Invalid scripthash`);
+      handleError(req, res, 400, `Invalid scripthash`);
       return;
     }
 
@@ -918,7 +1009,7 @@ class BitcoinRoutes {
 
   private async getRawBlock(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     try {
@@ -932,7 +1023,7 @@ class BitcoinRoutes {
 
   private async getTxIdsForBlock(req: Request, res: Response) {
     if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
-      handleError(req, res, 501, `Invalid block hash`);
+      handleError(req, res, 400, `Invalid block hash`);
       return;
     }
     try {
@@ -945,7 +1036,7 @@ class BitcoinRoutes {
 
   private async validateAddress(req: Request, res: Response) {
     if (!ADDRESS_REGEX.test(req.params.address)) {
-      handleError(req, res, 501, `Invalid address`);
+      handleError(req, res, 400, `Invalid address`);
       return;
     }
     try {
@@ -958,7 +1049,7 @@ class BitcoinRoutes {
 
   private async getRbfHistory(req: Request, res: Response) {
     if (!TXID_REGEX.test(req.params.txId)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -993,7 +1084,7 @@ class BitcoinRoutes {
 
   private async getCachedTx(req: Request, res: Response) {
     if (!TXID_REGEX.test(req.params.txId)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -1010,7 +1101,7 @@ class BitcoinRoutes {
 
   private async getTransactionOutspends(req: Request, res: Response) {
     if (!TXID_REGEX.test(req.params.txId)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -1023,7 +1114,7 @@ class BitcoinRoutes {
 
   private async getTransactionMerkleProof(req: Request, res: Response): Promise<void> {
     if (!TXID_REGEX.test(req.params.txId)) {
-      handleError(req, res, 501, `Invalid transaction ID`);
+      handleError(req, res, 400, `Invalid transaction ID`);
       return;
     }
     try {
@@ -1047,6 +1138,63 @@ class BitcoinRoutes {
     }
   }
 
+  private async getTxCountPerFlagValue(req: Request, res: Response) {
+    try {
+      if (!Common.blocksSummariesIndexingEnabled()) {
+        handleError(req, res, 404, `Block summaries indexing is required for this API`);
+        return;
+      }
+
+      const presets = INTERVAL_PRESETS;
+      const operations = ['and', 'or', 'nor', undefined];
+      const intervals = Object.keys(presets);
+      const interval = req.params.interval;
+
+      if (!intervals.includes(interval)) {
+        handleError(req, res, 400, `Invalid interval, must be one of ${intervals.toString()}`);
+        return;
+      }
+
+      const validBucketSizes = presets[interval].bucketSizes;
+      const rawBucketSize = req.params.bucketSize;
+      const bucketSize: number = rawBucketSize === undefined ? validBucketSizes[0] : Number(rawBucketSize);
+      if (!Number.isInteger(bucketSize) || !validBucketSizes.includes(bucketSize)) {
+        handleError(req, res, 400, `Invalid bucket size, must be ${validBucketSizes.toString()}`);
+        return;
+      }
+
+      if (!operations.includes(req.params.op)) {
+        handleError(req, res, 400, `Invalid operation, must be 'and', 'or', 'nor' or undefined.`);
+        return;
+      }
+
+      if (req.params.mask && !JUST_NUMBERS_REGEX.test(req.params.mask)) {
+        handleError(req, res, 400, `Invalid mask value, must be a positive integer`);
+        return;
+      }
+
+      const op = (req.params.op) as 'and' | 'or' | 'nor' | undefined;
+      const mask = BigInt(req.params.mask ?? 0n);
+
+      const { tip, tail }  = await FlagValueRepository.$getTipAndTailIndexedByBucketSize(bucketSize) || { tip: undefined, tail: undefined };
+
+      if (tip === undefined || tail === undefined) {
+        handleError(req, res, 400, `Failed to get latest indexed flag values for ${interval}`);
+        return;
+      }
+
+      const totalCount = await FlagValueRepository.$getTotalBlocksIndexedByBucketSize(bucketSize === 1 ? 1008 : bucketSize) ?? tip - tail;
+
+      const startHeight = presets[interval].retentionSpan !== -1 ? (tip - presets[interval].retentionSpan) : -1;
+      const txsCount = await FlagValueRepository.$queryTxCountBasedOnMask(mask, bucketSize, op, startHeight);
+      res.header('X-total-count', totalCount.toString());
+      res.header('Expires', new Date(Date.now() + 1000 * 3600 * 24 * (presets[interval].bucketSizes[0] / 144)).toUTCString());
+      res.send(txsCount);
+    } catch (e: any) {
+      handleError(req, res, 400, e instanceof Error ? e.message : 'Failed to get flag values');
+    }
+  }
+
   private async $postTransaction(req: Request, res: Response) {
     res.setHeader('content-type', 'text/plain');
     try {

backend/src/api/bitcoin/electrum-api.ts

@@ -44,23 +44,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
   async $getAddress(address: string): Promise<IEsploraApi.Address> {
     const addressInfo = await this.bitcoindClient.validateAddress(address);
     if (!addressInfo || !addressInfo.isvalid) {
-      return ({
-        'address': address,
-        'chain_stats': {
-          'funded_txo_count': 0,
-          'funded_txo_sum': 0,
-          'spent_txo_count': 0,
-          'spent_txo_sum': 0,
-          'tx_count': 0
-        },
-        'mempool_stats': {
-          'funded_txo_count': 0,
-          'funded_txo_sum': 0,
-          'spent_txo_count': 0,
-          'spent_txo_sum': 0,
-          'tx_count': 0
-        }
-      });
+      throw new Error('Invalid Bitcoin address');
     }
 
     try {
@@ -96,7 +80,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
   async $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]> {
     const addressInfo = await this.bitcoindClient.validateAddress(address);
     if (!addressInfo || !addressInfo.isvalid) {
-      return [];
+      throw new Error('Invalid Bitcoin address');
     }
 
     try {
@@ -166,7 +150,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
   async $getAddressUtxos(address: string): Promise<IEsploraApi.UTXO[]> {
     const addressInfo = await this.bitcoindClient.validateAddress(address);
     if (!addressInfo || !addressInfo.isvalid) {
-      return [];
+      throw new Error('Invalid Bitcoin address');
     }
     const scripthash = this.encodeScriptHash(addressInfo.scriptPubKey);
     return this.$getScriptHashUtxos(scripthash);

backend/src/api/bitcoin/esplora-api.ts

@@ -32,6 +32,11 @@ interface FailoverHost {
     core?: string,
     os?: string,
     lastUpdated: number,
+  },
+  liquidAudit?: {
+    pegRatio: number,
+    bitcoinLastBlockUpdate: number,
+    liquidLastBlockUpdate: number,
   }
 }
 
@@ -145,6 +150,8 @@ class FailoverRouter {
             }
           }
 
+          await this.$updateLiquidAudit(host);
+
           // Check front and backend git hashes less often
           if (Date.now() - host.hashes.lastUpdated > this.gitHashInterval) {
             await Promise.all([
@@ -351,6 +358,43 @@ class FailoverRouter {
     }
   }
 
+  private async $updateLiquidAudit(host: FailoverHost): Promise<void> {
+    if (config.MEMPOOL.NETWORK !== 'liquid') {
+      return;
+    }
+    try {
+      const [reservesResponse, pegsResponse] = await Promise.all([
+        this.pollConnection.get<any>(
+          `${host.publicDomain}/api/v1/liquid/reserves`, {
+            timeout: config.ESPLORA.FALLBACK_TIMEOUT,
+            headers: { 'Host': 'liquid.network' }
+          }
+        ),
+        this.pollConnection.get<any>(
+          `${host.publicDomain}/api/v1/liquid/pegs`, {
+            timeout: config.ESPLORA.FALLBACK_TIMEOUT,
+            headers: { 'Host': 'liquid.network' }
+          }
+        ),
+      ]);
+
+      const reservesAmount = Number(reservesResponse.data?.amount);
+      const pegsAmount = Number(pegsResponse.data?.amount);
+      const bitcoinLastBlockUpdate = Number(reservesResponse.data?.lastBlockUpdate);
+      const liquidLastBlockUpdate = Number(pegsResponse.data?.lastBlockUpdate);
+
+      if (Number.isFinite(reservesAmount) && Number.isFinite(pegsAmount) && Number.isFinite(bitcoinLastBlockUpdate) && Number.isFinite(liquidLastBlockUpdate) && pegsAmount > 0) {
+        host.liquidAudit = {
+          pegRatio: (reservesAmount / pegsAmount) * 100,
+          bitcoinLastBlockUpdate,
+          liquidLastBlockUpdate,
+        };
+      }
+    } catch (e) {
+      // failed to get liquid audit values - do nothing
+    }
+  }
+
   // returns the public mempool domain corresponding to an esplora server url
   // (a bit of a hack to avoid manually specifying frontend & backend URLs for each esplora server)
   private extractPublicDomain(url: string): string {
@@ -585,6 +629,7 @@ class ElectrsApi implements AbstractBitcoinApi {
         checked: !!host.checked,
         lastChecked: host.lastChecked || 0,
         hashes: host.hashes,
+        ...(config.MEMPOOL.NETWORK === 'liquid' ? { liquidAudit: host.liquidAudit } : {}),
       }));
     } else {
       return [];

backend/src/api/block-processor.ts

@@ -0,0 +1,262 @@
+import config from '../config';
+import logger from '../logger';
+import {
+  BlockExtended,
+  BlockSummary,
+  PoolTag,
+  MempoolTransactionExtended,
+  CpfpSummary,
+  TemplateAlgorithm,
+  MempoolBlockWithTransactions,
+} from '../mempool.interfaces';
+import { IEsploraApi } from './bitcoin/esplora-api.interface';
+import { Acceleration } from './services/acceleration';
+import { calculateGoodBlockCpfp, calculateClusterMempoolBlockCpfp, calculateFastBlockCpfp, BlockCpfpData } from './cpfp';
+import mempoolBlocks from './mempool-blocks';
+import memPool from './mempool';
+import Audit, { AuditResult } from './audit';
+import blocks from './blocks';
+import transactionUtils from './transaction-utils';
+import { ClusterMempool } from '../cluster-mempool/cluster-mempool';
+import { Common } from './common';
+import accelerationApi from './services/acceleration';
+
+interface ProcessedAudit extends AuditResult {
+  expectedFees: number;
+  expectedWeight: number;
+  projectedBlocks: MempoolBlockWithTransactions[];
+}
+
+export interface BlockProcessingResult {
+  templateAlgorithm: TemplateAlgorithm;
+  cpfpSummary: CpfpSummary;
+  blockExtended: BlockExtended;
+  blockSummary: BlockSummary;
+  auditResult?: ProcessedAudit;
+}
+
+const CM_ACTIVATION_HEIGHT: { [network: string]: number } = {
+  'mainnet': 940000,
+  'testnet': 4860000,
+  'testnet4': 125000,
+  'signet': 294000,
+  'regtest': 0,
+};
+
+class BlockProcessor {
+
+  /** @asyncUnsafe */
+  public async $processNewBlock(
+    block: IEsploraApi.Block,
+    transactions: MempoolTransactionExtended[],
+    pool: PoolTag,
+    accelerations: Record<string, Acceleration>
+  ): Promise<BlockProcessingResult> {
+    const poolAccelerations = Object.values(accelerations)
+      .filter(a => a.pools.includes(pool.uniqueId))
+      .map(a => ({ txid: a.txid, max_bid: a.feeDelta }));
+
+    const { templateAlgorithm, cpfpSummary } = detectTemplateAlgorithm(
+      block.height,
+      transactions,
+      poolAccelerations
+    );
+
+
+    const blockExtended = await blocks.$getBlockExtended(block, cpfpSummary.transactions, pool);
+    const blockSummary = blocks.summarizeBlockTransactions(block.id, block.height, cpfpSummary.transactions);
+
+    let auditResult: ProcessedAudit | undefined;
+    if (config.MEMPOOL.AUDIT && memPool.isInSync()) {
+      auditResult = await this.$runAudit(
+        blockExtended,
+        transactions,
+        templateAlgorithm,
+        pool,
+        accelerations
+      );
+
+      if (blockExtended.extras) {
+        blockExtended.extras.matchRate = auditResult.matchRate;
+        blockExtended.extras.expectedFees = auditResult.expectedFees;
+        blockExtended.extras.expectedWeight = auditResult.expectedWeight;
+        blockExtended.extras.similarity = auditResult.similarity;
+      }
+    } else if (blockExtended.extras) {
+      const mBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
+      if (mBlocks?.length && mBlocks[0].transactions) {
+        blockExtended.extras.similarity = Common.getSimilarity(mBlocks[0], transactions);
+      }
+    }
+
+    return {
+      templateAlgorithm,
+      cpfpSummary,
+      blockExtended,
+      blockSummary,
+      auditResult,
+    };
+  }
+
+  private async $runAudit(
+    block: BlockExtended,
+    transactions: MempoolTransactionExtended[],
+    templateAlgorithm: TemplateAlgorithm,
+    pool: PoolTag,
+    accelerations: Record<string, Acceleration>
+  ): Promise<ProcessedAudit> {
+    const auditMempool = memPool.getMempool();
+    const isAccelerated = accelerationApi.isAcceleratedBlock(block, Object.values(accelerations));
+
+    const candidateTxs = memPool.getMempoolCandidates();
+    const candidates = (memPool.limitGBT && candidateTxs)
+      ? { txs: candidateTxs, added: [], removed: [] }
+      : undefined;
+    const transactionIds = (memPool.limitGBT)
+      ? Object.keys(candidates?.txs || {})
+      : Object.keys(auditMempool);
+
+    let projectedBlocks: MempoolBlockWithTransactions[];
+
+    if (templateAlgorithm === TemplateAlgorithm.clusterMempool) {
+      const clusterMempool = memPool.clusterMempool ?? new ClusterMempool(auditMempool, accelerations, true, 75000);
+      const cmBlocks = clusterMempool.getBlocks(config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT) ?? [];
+      projectedBlocks = mempoolBlocks.processClusterMempoolBlocks(
+        cmBlocks,
+        auditMempool,
+        accelerations,
+        false,
+        pool.uniqueId
+      );
+    } else if (config.MEMPOOL.RUST_GBT) {
+      const added = memPool.limitGBT ? (candidates?.added || []) : [];
+      const removed = memPool.limitGBT ? (candidates?.removed || []) : [];
+      projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(
+        transactionIds,
+        auditMempool,
+        added,
+        removed,
+        candidates,
+        isAccelerated,
+        pool.uniqueId,
+        true
+      );
+    } else {
+      projectedBlocks = await mempoolBlocks.$makeBlockTemplates(
+        transactionIds,
+        auditMempool,
+        candidates,
+        false,
+        isAccelerated,
+        pool.uniqueId
+      );
+    }
+
+    const auditResult = Audit.auditBlock(block.height, transactions, projectedBlocks, auditMempool);
+
+    const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
+
+    let totalFees = 0;
+    let totalWeight = 0;
+    for (const tx of stripped) {
+      totalFees += tx.fee;
+      totalWeight += (tx.vsize * 4);
+    }
+
+    return {
+      ...auditResult,
+      expectedFees: totalFees,
+      expectedWeight: totalWeight,
+      projectedBlocks,
+    };
+  }
+}
+
+function saveCpfpDataToTransactions(transactions: MempoolTransactionExtended[], cpfpData: BlockCpfpData): void {
+  for (const tx of transactions) {
+    if (cpfpData.txs[tx.txid]) {
+      Object.assign(tx, cpfpData.txs[tx.txid]);
+    }
+  }
+}
+
+export function saveCpfpDataToCpfpSummary(transactions: MempoolTransactionExtended[], cpfpData: BlockCpfpData): CpfpSummary {
+  saveCpfpDataToTransactions(transactions, cpfpData);
+  return {
+    transactions,
+    clusters: cpfpData.clusters,
+    version: cpfpData.version,
+  };
+}
+
+/**
+ *
+ * @param height
+ * @param blockTransactions
+ * @param poolAccelerations
+ * @param fast
+ *
+ * saves effective fee rates from detected algorithm to blockTransactions
+ */
+export function detectTemplateAlgorithm(
+  height: number,
+  blockTransactions: MempoolTransactionExtended[],
+  poolAccelerations: { txid: string; max_bid: number }[],
+  fast: boolean = false
+): { templateAlgorithm: TemplateAlgorithm; cpfpSummary: CpfpSummary } {
+
+  const legacyCpfpData = fast ? calculateFastBlockCpfp(
+    height,
+    blockTransactions,
+  ) : calculateGoodBlockCpfp(
+    height,
+    blockTransactions,
+    poolAccelerations
+  );
+
+  if (!config.MEMPOOL.CLUSTER_MEMPOOL_INDEXING) {
+    return {
+      templateAlgorithm: TemplateAlgorithm.legacy,
+      cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, legacyCpfpData),
+    };
+  }
+
+  const network = config.MEMPOOL.NETWORK || 'mainnet';
+  const activationHeight = CM_ACTIVATION_HEIGHT[network] ?? Infinity;
+
+  if (height < activationHeight) {
+    return {
+      templateAlgorithm: TemplateAlgorithm.legacy,
+      cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, legacyCpfpData),
+    };
+  }
+
+  const clusterCpfpData = calculateClusterMempoolBlockCpfp(
+    height,
+    blockTransactions,
+    poolAccelerations
+  );
+
+  const clusterTxs = blockTransactions.map(tx => ({ txid: tx.txid, rate: clusterCpfpData.txs[tx.txid].effectiveFeePerVsize ?? tx.effectiveFeePerVsize }));
+  const legacyTxs = blockTransactions.map(tx => ({ txid: tx.txid, rate: legacyCpfpData.txs[tx.txid].effectiveFeePerVsize ?? tx.effectiveFeePerVsize }));
+  const clusterPrioritization = transactionUtils.identifyPrioritizedTransactions(clusterTxs, 'rate');
+  const legacyPrioritization = transactionUtils.identifyPrioritizedTransactions(legacyTxs, 'rate');
+
+  const clusterCount = clusterPrioritization.prioritized.length + clusterPrioritization.deprioritized.length;
+  const legacyCount = legacyPrioritization.prioritized.length + legacyPrioritization.deprioritized.length;
+
+  if (clusterCount < legacyCount) {
+    saveCpfpDataToTransactions(blockTransactions, clusterCpfpData);
+    return {
+      templateAlgorithm: TemplateAlgorithm.clusterMempool,
+      cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, clusterCpfpData),
+    };
+  } else {
+    return {
+      templateAlgorithm: TemplateAlgorithm.legacy,
+      cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, legacyCpfpData),
+    };
+  }
+}
+
+export default new BlockProcessor();

backend/src/api/blocks.ts

@@ -2,7 +2,7 @@ import config from '../config';
 import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 import logger from '../logger';
 import memPool from './mempool';
-import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified, BlockAudit, TransactionAudit } from '../mempool.interfaces';
+import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified, BlockAudit, TransactionAudit, TemplateAlgorithm } from '../mempool.interfaces';
 import { Common } from './common';
 import diskCache from './disk-cache';
 import transactionUtils from './transaction-utils';
@@ -28,14 +28,19 @@ import chainTips from './chain-tips';
 import websocketHandler from './websocket-handler';
 import redisCache from './redis-cache';
 import rbfCache from './rbf-cache';
+import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
+import mempoolBlocks from './mempool-blocks';
+import statistics from './statistics/statistics';
 import { calcBitsDifference } from './difficulty-adjustment';
 import AccelerationRepository from '../repositories/AccelerationRepository';
-import { calculateFastBlockCpfp, calculateGoodBlockCpfp } from './cpfp';
+import { calculateGoodBlockCpfp } from './cpfp';
+import blockProcessor, { BlockProcessingResult, detectTemplateAlgorithm, saveCpfpDataToCpfpSummary } from './block-processor';
 import mempool from './mempool';
 import CpfpRepository from '../repositories/CpfpRepository';
-import { parseDATUMTemplateCreator } from '../utils/bitcoin-script';
+import { parseDATUMTemplateCreator, parseDMNDTemplateCreator } from '../utils/bitcoin-script';
 import database from '../database';
 import { getBlockFirstSeenFromLogs, getOldestLogTimestampFromLogs, scanLogsForBlocksFirstSeen } from '../utils/file-read';
+import FlagValueRepository, { INDEXING_PRESETS } from '../repositories/FlagValueRepository';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -46,11 +51,12 @@ class Blocks {
   private previousDifficultyRetarget = 0;
   private quarterEpochBlockTime: number | null = null;
   private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
-  private newAsyncBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: MempoolTransactionExtended[]) => Promise<void>)[] = [];
   private classifyingBlocks: boolean = false;
   private oldestCoreLogTimestamp: number | undefined | null = undefined;
 
   private mainLoopTimeout: number = 120000;
+  private indexingFlagValues: boolean = false;
+  private flagValuesDeleteQueue: number[]= [];
 
   constructor() { }
 
@@ -74,10 +80,6 @@ class Blocks {
     this.newBlockCallbacks.push(fn);
   }
 
-  public setNewAsyncBlockCallback(fn: (block: BlockExtended, txIds: string[], transactions: MempoolTransactionExtended[]) => Promise<void>) {
-    this.newAsyncBlockCallbacks.push(fn);
-  }
-
   /**
    * Return the list of transaction for a block
    * @param blockHash
@@ -252,7 +254,7 @@ class Blocks {
    *
    * @asyncUnsafe
    */
-  private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
+  public async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[], providedPool?: PoolTag): Promise<BlockExtended> {
     const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
 
     const blk: Partial<BlockExtended> = Object.assign({}, block);
@@ -335,7 +337,9 @@ class Blocks {
 
     if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
       let pool: PoolTag;
-      if (coinbaseTx !== undefined) {
+      if (providedPool) {
+        pool = providedPool;
+      } else if (coinbaseTx !== undefined) {
         pool = await this.$findBlockMiner(coinbaseTx);
       } else {
         if (config.DATABASE.ENABLED === true) {
@@ -358,6 +362,8 @@ class Blocks {
 
         if (extras.pool.name === 'OCEAN') {
           extras.pool.minerNames = parseDATUMTemplateCreator(extras.coinbaseRaw);
+        } else if (extras.pool.name === 'DMND') {
+          extras.pool.minerNames = parseDMNDTemplateCreator(extras.coinbaseRaw);
         }
       }
 
@@ -373,6 +379,7 @@ class Blocks {
         }
       }
 
+      extras.firstSeen = null;
       if (config.CORE_RPC.DEBUG_LOG_PATH) {
         const oldestLog = this.getOldestCoreLogTimestamp();
         if (oldestLog) {
@@ -385,7 +392,7 @@ class Blocks {
     return <BlockExtended>blk;
   }
 
-  private async $getBlockStats(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<IBitcoinApi.BlockStats> {
+  public async $getBlockStats(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<IBitcoinApi.BlockStats> {
     if (!block.stale) {
       return bitcoinClient.getBlockStats(block.id);
     }
@@ -495,6 +502,129 @@ class Blocks {
     }
   }
 
+  /** @asyncUnsafe */
+  private async $applyBlockTransactionsToMempool(
+    txIds: string[],
+    transactions: MempoolTransactionExtended[]
+  ): Promise<{ rbfTransactions: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }}}> {
+    const _memPool = memPool.getMempool();
+
+    const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
+    memPool.handleRbfTransactions(rbfTransactions);
+    memPool.removeFromSpendMap(transactions);
+
+    if (config.MEMPOOL.CLUSTER_MEMPOOL) {
+      memPool.clusterMempool?.applyMempoolChange({
+        added: [],
+        removed: transactions,
+        accelerations: mempool.getAccelerations(),
+      });
+    }
+
+    for (const txId of txIds) {
+      delete _memPool[txId];
+      rbfCache.mined(txId);
+    }
+    redisCache.queueTransactionsForRemoval(txIds);
+
+    let candidates;
+    let transactionIds: string[];
+
+    if (memPool.limitGBT) {
+      const minFeeMempool = await bitcoinSecondClient.getRawMemPool();
+      const minFeeTip = await bitcoinSecondClient.getBlockCount();
+      candidates = memPool.getNextCandidates(minFeeMempool, minFeeTip, transactions);
+      transactionIds = Object.keys(candidates?.txs || {});
+    } else {
+      candidates = undefined;
+      transactionIds = Object.keys(memPool.getMempool());
+    }
+
+    if (config.MEMPOOL.CLUSTER_MEMPOOL) {
+      const cmBlocks = mempool.clusterMempool?.getBlocks(config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT) ?? [];
+      mempoolBlocks.processClusterMempoolBlocks(cmBlocks, _memPool, mempool.getAccelerations());
+    } else if (config.MEMPOOL.RUST_GBT) {
+      const added = memPool.limitGBT ? (candidates?.added || []) : [];
+      const removed = memPool.limitGBT ? (candidates?.removed || []) : transactions;
+      await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, _memPool, added, removed, candidates, true);
+    } else {
+      await mempoolBlocks.$makeBlockTemplates(transactionIds, _memPool, candidates, true, true);
+    }
+
+    return { rbfTransactions };
+  }
+
+  /** @asyncUnsafe */
+  private async $saveBlockData(
+    processingResult: BlockProcessingResult,
+    timer: number
+  ): Promise<void> {
+    const blockExtended = processingResult.blockExtended;
+    const cpfpSummary = processingResult.cpfpSummary;
+
+    let latestPriceId;
+    try {
+      latestPriceId = await PricesRepository.$getLatestPriceId();
+      this.updateTimerProgress(timer, `got latest price id ${this.currentBlockHeight}`);
+    } catch (e) {
+      logger.debug('failed to fetch latest price id from db: ' + (e instanceof Error ? e.message : e));
+    }
+    if (priceUpdater.historyInserted === true && latestPriceId !== null) {
+      await blocksRepository.$saveBlockPrices([{
+        height: blockExtended.height,
+        priceId: latestPriceId,
+      }]);
+      this.updateTimerProgress(timer, `saved prices for ${this.currentBlockHeight}`);
+    } else {
+      logger.debug(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining);
+      indexer.scheduleSingleTask('blocksPrices', 10000);
+    }
+
+    if (Common.blocksSummariesIndexingEnabled() === true) {
+      // indexes the summary as a side effect
+      await this.$getStrippedBlockTransactions(blockExtended.id, true, false, cpfpSummary, blockExtended.height);
+      this.updateTimerProgress(timer, `saved block summary for ${this.currentBlockHeight}`);
+    }
+
+    if (config.MEMPOOL.CPFP_INDEXING) {
+      // can be slow, and isn't critical, so don't await
+      void this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary);
+      this.updateTimerProgress(timer, `saved cpfp for ${this.currentBlockHeight}`);
+    }
+
+    if (processingResult.auditResult) {
+      void BlocksSummariesRepository.$saveTemplate({
+        height: blockExtended.height,
+        template: {
+          id: blockExtended.id,
+          transactions: processingResult.auditResult.projectedBlocks[0].transactions,
+        },
+        version: 1,
+      });
+      this.updateTimerProgress(timer, `saved audit template for ${this.currentBlockHeight}`);
+
+      void BlocksAuditsRepository.$saveAudit({
+        version: 1,
+        templateAlgorithm: processingResult.templateAlgorithm,
+        time: blockExtended.timestamp,
+        height: blockExtended.height,
+        hash: blockExtended.id,
+        unseenTxs: processingResult.auditResult.unseen,
+        addedTxs: processingResult.auditResult.added,
+        prioritizedTxs: processingResult.auditResult.prioritized,
+        missingTxs: processingResult.auditResult.censored,
+        freshTxs: processingResult.auditResult.fresh,
+        sigopTxs: processingResult.auditResult.sigop,
+        fullrbfTxs: processingResult.auditResult.fullrbf,
+        acceleratedTxs: processingResult.auditResult.accelerated,
+        matchRate: processingResult.auditResult.matchRate,
+        expectedFees: processingResult.auditResult.expectedFees,
+        expectedWeight: processingResult.auditResult.expectedWeight,
+      });
+      this.updateTimerProgress(timer, `saved audit results for ${this.currentBlockHeight}`);
+    }
+  }
+
   /**
    * [INDEXING] Index all blocks summaries for the block txs visualization
    */
@@ -562,6 +692,155 @@ class Blocks {
     }
   }
 
+  /**
+   * [INDEXING] Index all blocks flag values for the goggles graph rendering
+   *
+   *  @asyncSafe
+   */
+  public async $generateFlagValuesDatabase(): Promise<void> {
+    const MAX_BLOCKS_PERQUERY = 144;
+    if (this.indexingFlagValues) {
+      return;
+    }
+
+    if (Common.blocksSummariesIndexingEnabled() === false || Common.isLiquid()) {
+      return;
+    }
+
+    this.indexingFlagValues = true;
+
+    const tipOfSummaries = await BlocksSummariesRepository.$getTipIndexed();
+    if (!tipOfSummaries) {
+      this.indexingFlagValues = false;
+      return;
+    }
+
+    let newlyIndexedBuckets = 0;
+
+    while (this.flagValuesDeleteQueue.length > 0) { // Deletion of in-queue heights due to reorg
+      const deletionHeight = this.flagValuesDeleteQueue.shift();
+      if (deletionHeight === undefined) {
+        continue;
+      }
+      await FlagValueRepository.$deleteFlagValuesFromHeight(deletionHeight);
+    }
+
+    for (const preset of INDEXING_PRESETS) {
+      let seedHeight = preset.retentionSpan > -1 ? tipOfSummaries - preset.retentionSpan : 0;
+      if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT > 0) {
+        seedHeight = Math.max(seedHeight, tipOfSummaries - config.MEMPOOL.INDEXING_BLOCKS_AMOUNT + 1);
+      }
+      const firstBucket = Math.floor((tipOfSummaries + 1) / preset.bucketSize) * preset.bucketSize - preset.bucketSize;
+      const lastBucket = Math.max(0, Math.floor(seedHeight / preset.bucketSize) * preset.bucketSize);
+
+      // Deletion of flag values out of retention span
+      const tipAndTailOfFlagValues = await FlagValueRepository.$getTipAndTailIndexedByBucketSize(preset.bucketSize);
+      if (tipAndTailOfFlagValues && lastBucket > tipAndTailOfFlagValues.tail) { // Drop buckets that fell out of block span
+        logger.debug(`Deleting all the flag values ${preset.name} below height #${lastBucket}`, logger.tags.goggles);
+        await FlagValueRepository.$deleteFlagValuesBelowHeight(lastBucket, preset.bucketSize);
+      }
+
+      if (firstBucket < lastBucket) {
+        continue; // no complete bucket in range
+      }
+
+      const indexedBuckets = await FlagValueRepository.$getIndexedStartHeights(preset.bucketSize, firstBucket, lastBucket);
+      const isBucketIndexed = {};
+      // We map the buckets that are already indexed to skip them
+      for (const startHeight of indexedBuckets) {
+        isBucketIndexed[startHeight] = true;
+      }
+
+      logger.debug(`Processing and indexing flag values from #${firstBucket} to #${lastBucket} ${preset.name}`, logger.tags.goggles);
+
+      let timer = Date.now() / 1000;
+      const startedAt = Date.now() / 1000;
+      let blocksComputedInTotal = 0;
+      let blocksComputedThisRun = 0;
+      const blocksToCompute = firstBucket + preset.bucketSize - lastBucket - (indexedBuckets.length * preset.bucketSize);
+      for (let bucketStart = firstBucket; bucketStart >= lastBucket; bucketStart -= preset.bucketSize) {
+        if (isBucketIndexed[bucketStart]) {
+          continue; // already indexed
+        }
+        try {
+          const bucketFirstHeight = bucketStart + preset.bucketSize - 1;
+          const bucketLastHeight = bucketStart - 1;
+
+          let step = bucketFirstHeight;
+
+          const dataPerFlag: Record<string, Record<string, number>> = {};
+          let sumTimestamps = 0;
+          let nBlocks = 0;
+          let incomplete = false;
+
+          // Incrementalized logic capped by max blocks per query, not bucket size
+          while (step > bucketLastHeight) {
+            const blocksPerQuery = Math.min(step - bucketLastHeight, MAX_BLOCKS_PERQUERY);
+            const cappedLastHeight = step - blocksPerQuery;
+
+            const blocks = await BlocksSummariesRepository.$getSummariesBetweenHeights(step, cappedLastHeight);
+            await Common.sleep$(250); // Don't query/index flag values too fast
+
+            if (!blocks || blocks.length < blocksPerQuery) {
+              incomplete = true;
+              break; // Incomplete bucket
+            }
+
+            // Flag values processing
+            for (const block of blocks) {
+              const txData = JSON.parse(block.transactions).map((tx) => ({flags: tx.flags, vsize: tx.vsize}));
+              for (const data of txData) {
+                if (dataPerFlag[data.flags] === undefined || Object.keys(dataPerFlag[data.flags]).length === 0) {
+                  dataPerFlag[data.flags] = {
+                    txCount: 0,
+                    vSizeTotal: 0
+                  };
+                }
+                dataPerFlag[data.flags].txCount = dataPerFlag[data.flags].txCount + 1;
+                dataPerFlag[data.flags].vSizeTotal = dataPerFlag[data.flags].vSizeTotal + data.vsize;
+              }
+              sumTimestamps += block.timestamp;
+              blocksComputedInTotal++;
+              blocksComputedThisRun++;
+              nBlocks++;
+            }
+
+            // Logging
+            const elapsedSeconds = (Date.now() / 1000) - timer;
+            if (elapsedSeconds > 5) {
+              const runningFor = (Date.now() / 1000) - startedAt;
+              const blocksPerSecond = blocksComputedThisRun / elapsedSeconds;
+              const completion = (blocksComputedInTotal / blocksToCompute) * 100;
+              logger.debug(`Indexing flag values ${preset.name} | ${blocksComputedInTotal}/${blocksToCompute} (${completion.toFixed(2)}%) | ~${blocksPerSecond.toFixed(2)} blocks/sec | elapsed: ${runningFor.toFixed(2)} seconds`,logger.tags.goggles);
+              timer = Date.now() / 1000;
+              blocksComputedThisRun = 0;
+            }
+
+            step -= blocksPerQuery;
+          }
+
+          if (incomplete) {
+            continue;
+          }
+
+          const avgTimestamp = sumTimestamps / nBlocks;
+          await FlagValueRepository.$saveBatchFlagValues(preset.bucketSize, bucketStart, dataPerFlag, avgTimestamp);
+          nBlocks = 0;
+          newlyIndexedBuckets++;
+        } catch (e) {
+          logger.err(`Failed to index flag values between #${bucketStart} and #${bucketStart + preset.bucketSize - 1}. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.goggles);
+        }
+      }
+      logger.debug(`Successfully indexed #${blocksComputedInTotal} blocks ${preset.name} in ${((Date.now() / 1000) - startedAt).toFixed(2)} seconds`, logger.tags.goggles);
+    }
+    if (newlyIndexedBuckets > 0) {
+      logger.notice(`Flag values indexing completed: indexed ${newlyIndexedBuckets} buckets`, logger.tags.goggles);
+    } else {
+      logger.debug(`Flag values indexing completed: indexed ${newlyIndexedBuckets} buckets`, logger.tags.goggles);
+    }
+    this.indexingFlagValues = false;
+  }
+
   /** @asyncUnsafe */
   public async $indexBlockSummary(hash: string, height: number, stale?: boolean): Promise<void> {
     if (config.MEMPOOL.BACKEND === 'esplora') {
@@ -727,7 +1006,8 @@ class Blocks {
           // fetch transactions
           txs = (await bitcoinApi.$getTxsForBlock(blockHash, true)).map(tx => transactionUtils.extendMempoolTransaction(tx)) || [];
           // add CPFP
-          const cpfpSummary = calculateGoodBlockCpfp(height, txs, []);
+          const blockCpfpData = calculateGoodBlockCpfp(height, txs, []);
+          const cpfpSummary = saveCpfpDataToCpfpSummary(txs, blockCpfpData);
           // classify
           const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, height, cpfpSummary.transactions);
           await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 2);
@@ -764,7 +1044,8 @@ class Blocks {
               }
               templateTxs.push(tx || templateTx);
             }
-            const cpfpSummary = calculateGoodBlockCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as MempoolTransactionExtended[], []);
+            const blockCpfpData = calculateGoodBlockCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as MempoolTransactionExtended[], []);
+            const cpfpSummary = saveCpfpDataToCpfpSummary(templateTxs as MempoolTransactionExtended[], blockCpfpData);
             // classify
             const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, height, cpfpSummary.transactions);
             const classifiedTxMap: { [txid: string]: TransactionClassified } = {};
@@ -955,12 +1236,12 @@ class Blocks {
     logger.debug(`Found first seen times of ${foundCount} / ${results.length} blocks in Core logs, saving to database...`);
     await BlocksRepository.$saveFirstSeenTimes(results);
 
+    const blocksByHash = new Map<string, BlockExtended>(this.blocks.map<[string, BlockExtended]>(block => [block.id, block]));
+
     for (const { hash, firstSeen } of results) {
-      if (firstSeen !== null) {
-        const cachedBlock = this.blocks.find(blk => blk.id === hash);
-        if (cachedBlock) {
-          cachedBlock.extras.firstSeen = firstSeen;
-        }
+      const cachedBlock = blocksByHash.get(hash);
+      if (cachedBlock?.extras) {
+        cachedBlock.extras.firstSeen = firstSeen;
       }
     }
 
@@ -1053,59 +1334,55 @@ class Blocks {
         }
       }
 
-      let accelerations = Object.values(mempool.getAccelerations());
-      if (accelerations?.length > 0) {
-        const pool = await this.$findBlockMiner(transactionUtils.stripCoinbaseTransaction(transactions[0]));
-        accelerations = accelerations.filter(a => a.pools.includes(pool.uniqueId));
-      }
-      const cpfpSummary: CpfpSummary = calculateGoodBlockCpfp(block.height, transactions, accelerations.map(a => ({ txid: a.txid, max_bid: a.feeDelta })));
-      const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions);
-      const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, block.height, cpfpSummary.transactions);
+      const pool = await this.$findBlockMiner(transactionUtils.stripCoinbaseTransaction(transactions[0]));
+      const accelerations = mempool.getAccelerations();
+
+      const processingResult = await blockProcessor.$processNewBlock(
+        block,
+        transactions,
+        pool,
+        accelerations
+      );
+
+      const blockExtended = processingResult.blockExtended;
+      const blockSummary = processingResult.blockSummary;
+      const cpfpSummary = processingResult.cpfpSummary;
       this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`);
 
-      if (Common.indexingEnabled()) {
-        if (!fastForwarded) {
-          await this.$handleReorgs(blockExtended, timer);
-        }
+      if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
+        await statistics.runStatistics();
+      }
 
+      const { rbfTransactions } = await this.$applyBlockTransactionsToMempool(txIds, cpfpSummary.transactions);
+      this.updateTimerProgress(timer, `applied mempool changes for ${this.currentBlockHeight}`);
+
+      if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
+        await statistics.runStatistics();
+      }
+
+      if (Common.indexingEnabled() && !fastForwarded) {
+        await this.$handleReorgs(blockExtended, timer);
+      }
+
+      await websocketHandler.handleNewBlock(blockExtended, txIds, cpfpSummary.transactions, rbfTransactions);
+      this.updateTimerProgress(timer, `sent websocket updates for ${this.currentBlockHeight}`);
+
+      if (Common.indexingEnabled()) {
         await blocksRepository.$saveBlockInDatabase(blockExtended);
         this.updateTimerProgress(timer, `saved ${this.currentBlockHeight} to database`);
 
-        if (!fastForwarded) {
-          let lastestPriceId;
-          try {
-            lastestPriceId = await PricesRepository.$getLatestPriceId();
-            this.updateTimerProgress(timer, `got latest price id ${this.currentBlockHeight}`);
-          } catch (e) {
-            logger.debug('failed to fetch latest price id from db: ' + (e instanceof Error ? e.message : e));
-          }
-          if (priceUpdater.historyInserted === true && lastestPriceId !== null) {
-            await blocksRepository.$saveBlockPrices([{
-              height: blockExtended.height,
-              priceId: lastestPriceId,
-            }]);
-            this.updateTimerProgress(timer, `saved prices for ${this.currentBlockHeight}`);
-          } else {
-            logger.debug(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining);
-            indexer.scheduleSingleTask('blocksPrices', 10000);
-          }
+        await AccelerationRepository.$indexAccelerationsForBlock(
+          blockExtended,
+          Object.values(accelerations),
+          cpfpSummary.transactions
+        );
+        this.updateTimerProgress(timer, `indexed accelerations for ${this.currentBlockHeight}`);
 
-          // Save blocks summary for visualization if it's enabled
-          if (Common.blocksSummariesIndexingEnabled() === true) {
-            await this.$getStrippedBlockTransactions(blockExtended.id, true, false, cpfpSummary, blockExtended.height);
-            this.updateTimerProgress(timer, `saved block summary for ${this.currentBlockHeight}`);
-          }
-          if (config.MEMPOOL.CPFP_INDEXING) {
-            void this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary);
-            this.updateTimerProgress(timer, `saved cpfp for ${this.currentBlockHeight}`);
-          }
+        if (!fastForwarded) {
+          await this.$saveBlockData(processingResult, timer);
         }
       }
 
-      // start async callbacks
-      this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
-      const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, cpfpSummary.transactions));
-
       if (block.height % 2016 === 0) {
         if (Common.indexingEnabled()) {
           let adjustment;
@@ -1143,11 +1420,6 @@ class Blocks {
         await chainTips.updateOrphanedBlocks();
       }
 
-      // wait for pending async callbacks to finish
-      this.updateTimerProgress(timer, `waiting for async callbacks to complete for ${this.currentBlockHeight}`);
-      await Promise.all(callbackPromises);
-      this.updateTimerProgress(timer, `async callbacks completed for ${this.currentBlockHeight}`);
-
       this.blocks.push(blockExtended);
       if (this.blocks.length > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4) {
         this.blocks = this.blocks.slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4);
@@ -1171,7 +1443,7 @@ class Blocks {
       if (config.REDIS.ENABLED) {
         await redisCache.$updateBlocks(this.blocks);
         await redisCache.$updateBlockSummaries(this.blockSummaries);
-        await redisCache.$removeTransactions(txIds);
+        await redisCache.$removeTransactions();
         await rbfCache.updateCache();
       }
 
@@ -1289,6 +1561,7 @@ class Blocks {
       await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(forkTail.height);
       await cpfpRepository.$deleteClustersFrom(forkTail.height);
       await AccelerationRepository.$deleteAccelerationsFrom(forkTail.height);
+      this.flagValuesDeleteQueue.push(forkTail.height);
       chainTips.clearOrphanCacheAboveHeight(forkTail.height);
       this.updateTimerProgress(timer, `deleted stale block data`);
 
@@ -1647,16 +1920,16 @@ class Blocks {
     }
 
     if (transactions?.length != null) {
-      const summary = calculateFastBlockCpfp(height, transactions);
+      const { cpfpSummary } = await detectTemplateAlgorithm(height, transactions, [], true);
 
-      if (!stale) {
-        await this.$saveCpfp(hash, height, summary);
+      if (!stale && Common.cpfpIndexingEnabled() === true) {
+        await this.$saveCpfp(hash, height, cpfpSummary);
       }
 
-      const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions);
+      const effectiveFeeStats = Common.calcEffectiveFeeStatistics(cpfpSummary.transactions);
       await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats);
 
-      return summary;
+      return cpfpSummary;
     } else {
       logger.err(`Cannot index CPFP for block ${height} - missing transaction data`);
       return null;

backend/src/api/chain-tips.ts

@@ -2,6 +2,7 @@ import config from '../config';
 import logger from '../logger';
 import { BlockExtended } from '../mempool.interfaces';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
+import BlocksRepository from '../repositories/BlocksRepository';
 import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 import bitcoinClient from './bitcoin/bitcoin-client';
 import { IEsploraApi } from './bitcoin/esplora-api.interface';
@@ -23,26 +24,29 @@ export interface StaleTip extends ChainTip {
 export interface OrphanedBlock {
   height: number;
   hash: string;
+  branchlen: number;
   status: 'valid-fork' | 'valid-headers' | 'headers-only';
   prevhash: string;
 }
 
 class ChainTips {
   private chainTips: ChainTip[] = [];
-  private staleTips: Record<number, StaleTip> = {};
+  private validChainTips: ChainTip[] = []; // 'valid-fork' and 'valid-headers' only, in descending height order
+  private staleBlocks: Record<string, BlockExtended> = {};
   private orphanedBlocks: { [hash: string]: OrphanedBlock } = {};
   private blockCache: { [hash: string]: OrphanedBlock } = {};
   private orphansByHeight: { [height: number]: OrphanedBlock[] } = {};
   private indexingOrphanedBlocks = false;
   private indexingQueue: { blockhash?: string, block?: IEsploraApi.Block, tip: OrphanedBlock }[] = [];
 
-  private staleTipsCacheSize = 50;
+  private staleBlocksCacheSize = 50;
   private maxIndexingQueueSize = 100;
 
   /** @asyncSafe */
   public async updateOrphanedBlocks(): Promise<void> {
     try {
       this.chainTips = await bitcoinClient.getChainTips();
+      this.validChainTips = this.chainTips.filter(tip => tip.status === 'valid-fork' || tip.status === 'valid-headers').sort((a, b) => b.height - a.height);
 
       const activeTipHeight = this.chainTips.find(tip => tip.status === 'active')?.height || (await bitcoinApi.$getBlockHeightTip());
       let minIndexHeight = 0;
@@ -69,6 +73,7 @@ class ChainTips {
                 orphan = {
                   height: block.height,
                   hash: block.id,
+                  branchlen: chain.branchlen,
                   status: chain.status,
                   prevhash: block.previousblockhash,
                 };
@@ -119,13 +124,7 @@ class ChainTips {
         this.orphansByHeight[orphan.height].push(orphan);
       }
 
-      const heightsToKeep = new Set(this.chainTips.filter(tip => tip.status !== 'active').map(tip => tip.height));
-      const heightsToRemove: number[] = Object.keys(this.staleTips).map(Number).filter(height => !heightsToKeep.has(height));
-      for (const height of heightsToRemove) {
-        delete this.staleTips[height];
-      }
-
-      this.trimStaleTipsCache();
+      this.trimStaleBlocksCache();
 
       // index new orphaned blocks in the background
       void this.$indexOrphanedBlocks();
@@ -157,7 +156,7 @@ class ChainTips {
         }
         let staleBlock: BlockExtended | undefined;
         const alreadyIndexed = await BlocksSummariesRepository.$isSummaryIndexed(block.id);
-        const needToCache = Object.keys(this.staleTips).length < this.staleTipsCacheSize || block.height > Object.keys(this.staleTips).map(Number).sort((a, b) => b - a)[this.staleTipsCacheSize - 1];
+        const needToCache = this.shouldCacheStaleBlock(block.id, block.height);
         if (!alreadyIndexed) {
           staleBlock = await blocks.$indexBlock(block.id, block, true);
           await blocks.$indexBlockSummary(block.id, block.height, true);
@@ -168,16 +167,9 @@ class ChainTips {
         }
 
         if (staleBlock && needToCache) {
-          const canonicalBlock = await blocks.$indexBlockByHeight(staleBlock.height);
-          this.staleTips[staleBlock.height] = {
-            height: staleBlock.height,
-            hash: staleBlock.id,
-            branchlen: tip.height - staleBlock.height,
-            status: tip.status,
-            stale: staleBlock,
-            canonical: canonicalBlock,
-          };
-          this.trimStaleTipsCache();
+          // ensure the canonical block is correctly indexed
+          await blocks.$indexBlockByHeight(staleBlock.height);
+          this.cacheStaleBlock(staleBlock);
         }
       } catch (e) {
         logger.err(`Failed to index orphaned block ${block?.id} at height ${block?.height}. Reason: ${e instanceof Error ? e.message : e}`);
@@ -186,12 +178,42 @@ class ChainTips {
     this.indexingOrphanedBlocks = false;
   }
 
-  private trimStaleTipsCache(): void {
-    const staleTipHeights = Object.keys(this.staleTips).map(Number).sort((a, b) => b - a);
-    if (staleTipHeights.length > this.staleTipsCacheSize) {
-      const heightsToDiscard = staleTipHeights.slice(this.staleTipsCacheSize);
-      for (const height of heightsToDiscard) {
-        delete this.staleTips[height];
+  private shouldCacheStaleBlock(hash: string, height: number): boolean {
+    // already cached
+    if (this.staleBlocks[hash]) {
+      return false;
+    }
+
+    // cache is not full
+    const cachedBlocks = Object.values(this.staleBlocks);
+    if (cachedBlocks.length < this.staleBlocksCacheSize) {
+      return true;
+    }
+
+    // otherwise cache if this block is newer than the oldest in the cache
+    const oldestCachedHeight = cachedBlocks.reduce((min, block) => Math.min(min, block.height), Infinity);
+    return height >= oldestCachedHeight;
+  }
+
+  private cacheStaleBlock(block: BlockExtended): void {
+    this.staleBlocks[block.id] = block;
+    this.trimStaleBlocksCache();
+  }
+
+  // evict the oldest stale blocks until the cache is within the size limit
+  private trimStaleBlocksCache(): void {
+    // sort by height
+    const cachedBlocks = Object.values(this.staleBlocks).sort((a, b) => {
+      if (b.height !== a.height) {
+        return b.height - a.height;
+      }
+      // tie-break by hash
+      return a.id.localeCompare(b.id);
+    });
+    // delete everything beyond the size limit
+    if (cachedBlocks.length > this.staleBlocksCacheSize) {
+      for (const block of cachedBlocks.slice(this.staleBlocksCacheSize)) {
+        delete this.staleBlocks[block.id];
       }
     }
   }
@@ -208,8 +230,51 @@ class ChainTips {
     return this.chainTips;
   }
 
-  public getStaleTips(): StaleTip[] {
-    return Object.values(this.staleTips).sort((a, b) => b.height - a.height);
+  /**
+   * get paginated stale chain tips
+   * @param fromHeight - start height (exclusive)
+   * @param count - requested page size (target, but not strictly enforced)
+   *
+   * @asyncSafe
+   */
+  public async $getStaleTipsPage(fromHeight: number | undefined, count: number): Promise<StaleTip[]> {
+    const start = fromHeight === undefined ? 0 : this.validChainTips.findIndex(tip => tip.height < fromHeight);
+    // no tips beyond the requested height, we can return early
+    if (start === -1) {
+      return [];
+    }
+
+    // fill the response array with hydrated tip data
+    const tips: StaleTip[] = [];
+    let lastHeight;
+    for (let index = start; index < this.validChainTips.length; index++) {
+      const staleTip = this.validChainTips[index];
+      // stretch the page to include any remaining blocks at the last included height to avoid pagination gaps with a height-based cursor
+      if (tips.length >= count) {
+        if (staleTip.height !== lastHeight) {
+          break;
+        }
+      }
+      // fetch blocks from caches if available, or DB otherwise
+      const canonical = blocks.getBlocks().find(block => block.height === staleTip.height) || await BlocksRepository.$getBlockByHeight(staleTip.height);
+      let stale: BlockExtended | null | undefined = this.staleBlocks[staleTip.hash];
+      if (!stale) {
+        stale = await BlocksRepository.$getBlockByHash(staleTip.hash);
+      }
+      // skip tips with missing block data
+      if (!canonical || !stale) {
+        continue;
+      }
+
+      tips.push({
+        ...staleTip,
+        stale,
+        canonical,
+      });
+      lastHeight = staleTip.height;
+    }
+
+    return tips;
   }
 
   clearOrphanCacheAboveHeight(height: number): void {
@@ -234,4 +299,4 @@ class ChainTips {
   }
 }
 
-export default new ChainTips();
+export default new ChainTips();

backend/src/api/common.ts

@@ -749,7 +749,15 @@ export class Common {
     // fast but bad heuristic to detect possible coinjoins
     // (at least 5 inputs and 5 outputs, less than half of which are unique amounts, with no address reuse)
     const addressReuse = Object.keys(reusedOutputAddresses).reduce((acc, key) => Math.max(acc, (reusedInputAddresses[key] || 0) + (reusedOutputAddresses[key] || 0)), 0) > 1;
-    if (!addressReuse && tx.vin.length >= 5 && tx.vout.length >= 5 && (Object.keys(inValues).length + Object.keys(outValues).length) <= (tx.vin.length + tx.vout.length) / 2 ) {
+    const tokenRelated = (flags & (TransactionFlags.inscription | TransactionFlags.op_return)) !== 0n;
+    if (!addressReuse &&
+        tx.vin.length >= 5 &&
+        tx.vout.length >= 5 &&
+        (Object.keys(inValues).length + Object.keys(outValues).length) <= (tx.vin.length + tx.vout.length) / 2 &&
+        !tokenRelated &&
+        tx.vin.length / tx.vout.length < 5 &&
+        tx.vin.length / tx.vout.length > 0.2
+      ) {
       flags |= TransactionFlags.coinjoin;
     }
     // more than 5:1 input:output ratio

backend/src/api/cpfp.ts

@@ -1,20 +1,30 @@
-import { Ancestor, CpfpCluster, CpfpInfo, CpfpSummary, MempoolTransactionExtended, TransactionExtended } from '../mempool.interfaces';
+import { Ancestor, CpfpCluster, CpfpInfo, MempoolTransactionExtended, TemplateAlgorithm, TransactionExtended } from '../mempool.interfaces';
 import { GraphTx, convertToGraphTx, expandRelativesGraph, initializeRelatives, makeBlockTemplate, mempoolComparator, removeAncestors, setAncestorScores } from './mini-miner';
 import memPool from './mempool';
 import { Acceleration } from './acceleration/acceleration';
+import { ClusterMempool } from '../cluster-mempool/cluster-mempool';
 
 const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction
 const MAX_CLUSTER_ITERATIONS = 100;
 
-export function calculateFastBlockCpfp(height: number, transactions: MempoolTransactionExtended[], saveRelatives: boolean = false): CpfpSummary {
+type TransactionCpfpData = Partial<CpfpInfo & { cpfpDirty?: boolean, clusterId?: number, chunkIndex?: number }>;
+export interface BlockCpfpData {
+  txs: Record<string, TransactionCpfpData>,
+  clusters: CpfpCluster[];
+  version: number;
+}
+
+export function calculateFastBlockCpfp(height: number, transactions: MempoolTransactionExtended[], saveRelatives: boolean = false): BlockCpfpData {
   const clusters: CpfpCluster[] = []; // list of all cpfp clusters in this block
   const clusterMap: { [txid: string]: CpfpCluster } = {}; // map transactions to their cpfp cluster
   let clusterTxs: TransactionExtended[] = []; // working list of elements of the current cluster
   let ancestors: { [txid: string]: boolean } = {}; // working set of ancestors of the current cluster root
   const txMap: { [txid: string]: TransactionExtended } = {};
+  const cpfpData: Record<string, TransactionCpfpData> = {};
   // initialize the txMap
   for (const tx of transactions) {
     txMap[tx.txid] = tx;
+    cpfpData[tx.txid] = {};
   }
   // reverse pass to identify CPFP clusters
   for (let i = transactions.length - 1; i >= 0; i--) {
@@ -38,7 +48,9 @@ export function calculateFastBlockCpfp(height: number, transactions: MempoolTran
         clusters.push(cluster);
       }
       clusterTxs.forEach(tx => {
-        txMap[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize;
+        cpfpData[tx.txid] = {
+          effectiveFeePerVsize
+        };
         if (cluster) {
           clusterMap[tx.txid] = cluster;
         }
@@ -54,17 +66,19 @@ export function calculateFastBlockCpfp(height: number, transactions: MempoolTran
   }
   // forward pass to enforce ancestor rate caps
   for (const tx of transactions) {
-    let minAncestorRate = tx.effectiveFeePerVsize;
+    const txRate = cpfpData[tx.txid]?.effectiveFeePerVsize ?? tx.effectiveFeePerVsize;
+    let minAncestorRate = txRate;
     for (const vin of tx.vin) {
-      if (txMap[vin.txid]?.effectiveFeePerVsize) {
-        minAncestorRate = Math.min(minAncestorRate, txMap[vin.txid].effectiveFeePerVsize);
+      const vinRate = cpfpData[vin.txid]?.effectiveFeePerVsize ?? txMap[vin.txid]?.effectiveFeePerVsize;
+      if (vinRate) {
+        minAncestorRate = Math.min(minAncestorRate, vinRate);
       }
     }
     // check rounded values to skip cases with almost identical fees
     const roundedMinAncestorRate = Math.ceil(minAncestorRate);
-    const roundedEffectiveFeeRate = Math.floor(tx.effectiveFeePerVsize);
+    const roundedEffectiveFeeRate = Math.floor(txRate);
     if (roundedMinAncestorRate < roundedEffectiveFeeRate) {
-      tx.effectiveFeePerVsize = minAncestorRate;
+      cpfpData[tx.txid].effectiveFeePerVsize = minAncestorRate;
       if (!clusterMap[tx.txid]) {
         // add a single-tx cluster to record the dependent rate
         const cluster = {
@@ -84,23 +98,25 @@ export function calculateFastBlockCpfp(height: number, transactions: MempoolTran
   if (saveRelatives) {
     for (const cluster of clusters) {
       cluster.txs.forEach((member, index) => {
-        txMap[member.txid].descendants = cluster.txs.slice(0, index).reverse();
-        txMap[member.txid].ancestors = cluster.txs.slice(index + 1).reverse();
-        txMap[member.txid].effectiveFeePerVsize = cluster.effectiveFeePerVsize;
+        cpfpData[member.txid].descendants = cluster.txs.slice(0, index).reverse();
+        cpfpData[member.txid].ancestors = cluster.txs.slice(index + 1).reverse();
+        cpfpData[member.txid].effectiveFeePerVsize = cluster.effectiveFeePerVsize;
       });
     }
   }
   return {
-    transactions,
+    txs: cpfpData,
     clusters,
     version: 1,
   };
 }
 
-export function calculateGoodBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): CpfpSummary {
+export function calculateGoodBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): BlockCpfpData {
   const txMap: { [txid: string]: MempoolTransactionExtended } = {};
+  const cpfpData: Record<string, TransactionCpfpData> = {};
   for (const tx of transactions) {
     txMap[tx.txid] = tx;
+    cpfpData[tx.txid] = {};
   }
   const template = makeBlockTemplate(transactions, accelerations, 1, Infinity, Infinity);
   const clusters = new Map<string, string[]>();
@@ -110,15 +126,15 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran
     if (cluster.length > 1 && root && !clusters.has(root)) {
       clusters.set(root, cluster);
     }
-    txMap[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize;
+    cpfpData[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize;
   }
 
   const clusterArray: CpfpCluster[] = [];
 
   for (const cluster of clusters.values()) {
     for (const txid of cluster) {
-      const mempoolTx = txMap[txid];
-      if (mempoolTx) {
+      const mempoolTxCpfpData = cpfpData[txid];
+      if (mempoolTxCpfpData) {
         const ancestors: Ancestor[] = [];
         const descendants: Ancestor[] = [];
         let matched = false;
@@ -138,10 +154,10 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran
             }
           }
         });
-        if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) {
-          mempoolTx.cpfpDirty = true;
+        if (mempoolTxCpfpData.ancestors?.length !== ancestors.length || mempoolTxCpfpData.descendants?.length !== descendants.length) {
+          mempoolTxCpfpData.cpfpDirty = true;
         }
-        Object.assign(mempoolTx, { ancestors, descendants, bestDescendant: null, cpfpChecked: true });
+        Object.assign(mempoolTxCpfpData, { ancestors, descendants, bestDescendant: null, cpfpChecked: true });
       }
     }
     const root = cluster[cluster.length - 1];
@@ -153,17 +169,75 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran
         fee: txMap[txid].fee,
         weight: (txMap[txid].adjustedVsize * 4) || txMap[txid].weight,
       })),
-      effectiveFeePerVsize: txMap[root].effectiveFeePerVsize,
+      effectiveFeePerVsize: cpfpData[root].effectiveFeePerVsize ?? txMap[root].effectiveFeePerVsize,
     });
   }
 
   return {
-    transactions: transactions.map(tx => txMap[tx.txid]),
+    txs: cpfpData,
     clusters: clusterArray,
     version: 2,
   };
 }
 
+export function calculateClusterMempoolBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): BlockCpfpData {
+  const txMap: { [txid: string]: MempoolTransactionExtended } = {};
+  const cpfpData: Record<string, TransactionCpfpData> = {};
+  for (const tx of transactions) {
+    txMap[tx.txid] = tx;
+    cpfpData[tx.txid] = {};
+  }
+
+  const accelMap: { [txid: string]: { feeDelta: number } } = {};
+  for (const acc of accelerations) {
+    accelMap[acc.txid] = { feeDelta: acc.max_bid };
+  }
+
+  const cm = new ClusterMempool(txMap, accelMap, false, 25000);
+
+  const seenClusters = new Set<number>();
+  const clusters: CpfpCluster[] = [];
+
+  for (const txid in cpfpData) {
+    const txCpfpData = cm.getCpfpDataForTx(txid);
+    if (!txCpfpData) {
+      continue;
+    }
+    cpfpData[txid].effectiveFeePerVsize = txCpfpData.effectiveFeePerVsize;
+    cpfpData[txid].clusterId = txCpfpData.clusterId;
+    cpfpData[txid].chunkIndex = txCpfpData.chunkIndex;
+    cpfpData[txid].ancestors = txCpfpData.ancestors;
+    cpfpData[txid].descendants = txCpfpData.descendants;
+    if (txCpfpData.clusterId !== undefined && !seenClusters.has(txCpfpData.clusterId)) {
+      seenClusters.add(txCpfpData.clusterId);
+
+      const clusterData = cm.getCluster(txCpfpData.clusterId);
+      if (clusterData && clusterData.txs.length > 1) {
+        let totalFee = 0;
+        let totalWeight = 0;
+        for (const t of clusterData.txs) {
+          totalFee += t.fee;
+          totalWeight += t.weight;
+        }
+        clusters.push({
+          root: clusterData.txs[0].txid,
+          height,
+          txs: clusterData.txs.map(t => ({ txid: t.txid, weight: t.weight, fee: t.fee })),
+          effectiveFeePerVsize: totalFee / (totalWeight / 4),
+          templateAlgorithm: TemplateAlgorithm.clusterMempool,
+          clusterData,
+        });
+      }
+    }
+  }
+
+  return {
+    txs: cpfpData,
+    clusters,
+    version: 3,
+  };
+}
+
 /**
  * Takes a mempool transaction and a copy of the current mempool, and calculates the CPFP data for
  * that transaction (and all others in the same cluster)

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 106;
+  private static currentVersion = 112;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -1222,6 +1222,44 @@ class DatabaseMigration {
       }
       await this.updateToSchemaVersion(106);
     }
+
+    if (databaseSchemaVersion < 107) {
+      await this.$executeQuery('ALTER TABLE `federation_txos` DROP FOREIGN KEY IF EXISTS `federation_txos_ibfk_1`');
+      await this.$executeQuery('DROP TABLE IF EXISTS `federation_addresses`');
+      await this.updateToSchemaVersion(107);
+    }
+
+    if (databaseSchemaVersion < 108) {
+      await this.$executeQuery(this.getCreateFederationPegScriptsTableQuery(), await this.$checkIfTableExists('federation_peg_scripts'),);
+      await this.updateToSchemaVersion(108);
+    }
+
+    // safe to make this conditional on the network since it doesn't change the database schema
+    if (databaseSchemaVersion < 109 && config.MEMPOOL.NETWORK === 'liquid') {
+      // hardcode current and past federation peg scripts to avoid reindexing existing instances
+      await this.$executeQuery(`
+          INSERT IGNORE INTO federation_peg_scripts (address, fedpegscript, timelock, blocknumber) VALUES
+            ('3G6neksSBMp51kHJ2if8SeDUrzT8iVETWT', '745c87635b21020e0338c96a8870479f2396c373cc7696ba124e8635d41b0ea581112b678172612102675333a4e4b8fb51d9d4e22fa5a8eaced3fdac8a8cbf9be8c030f75712e6af992102896807d54bc55c24981f24a453c60ad3e8993d693732288068a23df3d9f50d4821029e51a5ef5db3137051de8323b001749932f2ff0d34c82e96a2c2461de96ae56c2102a4e1a9638d46923272c266631d94d36bdb03a64ee0e14c7518e49d2f29bc40102102f8a00b269f8c5e59c67d36db3cdc11b11b21f64b4bffb2815e9100d9aa8daf072103079e252e85abffd3c401a69b087e590a9b86f33f574f08129ccbd3521ecf516b2103111cf405b627e22135b3b3733a4a34aa5723fb0f58379a16d32861bf576b0ec2210318f331b3e5d38156da6633b31929c5b220349859cc9ca3d33fb4e68aa08401742103230dae6b4ac93480aeab26d000841298e3b8f6157028e47b0897c1e025165de121035abff4281ff00660f99ab27bb53e6b33689c2cd8dcd364bc3c90ca5aea0d71a62103bd45cddfacf2083b14310ae4a84e25de61e451637346325222747b157446614c2103cc297026b06c71cbfa52089149157b5ff23de027ac5ab781800a578192d175462103d3bde5d63bdb3a6379b461be64dad45eabff42f758543a9645afd42f6d4248282103ed1e8d5109c9ed66f7941bc53cc71137baa76d50d274bda8d5e8ffbd6e61fe9a5f6702c00fb275522103aab896d53a8e7d6433137bbba940f9c521e085dd07e60994579b64a6d992cf79210291b7d0b1b692f8f524516ed950872e5da10fb1b808b5a526dedc6fed1cf29807210386aa9372fbab374593466bc5451dc59954e90787f08060964d95c87ef34ca5bb5368ae', 4032, 0),
+            ('3EiAcrzq1cELXScc98KeCswGWZaPGceT1d', '745c87635b21020e0338c96a8870479f2396c373cc7696ba124e8635d41b0ea581112b678172612102675333a4e4b8fb51d9d4e22fa5a8eaced3fdac8a8cbf9be8c030f75712e6af992102896807d54bc55c24981f24a453c60ad3e8993d693732288068a23df3d9f50d4821029e51a5ef5db3137051de8323b001749932f2ff0d34c82e96a2c2461de96ae56c2102a4e1a9638d46923272c266631d94d36bdb03a64ee0e14c7518e49d2f29bc40102102f8a00b269f8c5e59c67d36db3cdc11b11b21f64b4bffb2815e9100d9aa8daf072103079e252e85abffd3c401a69b087e590a9b86f33f574f08129ccbd3521ecf516b2103111cf405b627e22135b3b3733a4a34aa5723fb0f58379a16d32861bf576b0ec2210318f331b3e5d38156da6633b31929c5b220349859cc9ca3d33fb4e68aa08401742103230dae6b4ac93480aeab26d000841298e3b8f6157028e47b0897c1e025165de121035abff4281ff00660f99ab27bb53e6b33689c2cd8dcd364bc3c90ca5aea0d71a62103bd45cddfacf2083b14310ae4a84e25de61e451637346325222747b157446614c2103cc297026b06c71cbfa52089149157b5ff23de027ac5ab781800a578192d175462103d3bde5d63bdb3a6379b461be64dad45eabff42f758543a9645afd42f6d4248282103ed1e8d5109c9ed66f7941bc53cc71137baa76d50d274bda8d5e8ffbd6e61fe9a5f6702e007b275522103aab896d53a8e7d6433137bbba940f9c521e085dd07e60994579b64a6d992cf79210291b7d0b1b692f8f524516ed950872e5da10fb1b808b5a526dedc6fed1cf29807210386aa9372fbab374593466bc5451dc59954e90787f08060964d95c87ef34ca5bb5368ae', 2016, 0),
+            ('bc1qxvay4an52gcghxq5lavact7r6qe9l4laedsazz8fj2ee2cy47tlqff4aj4', '5b21020e0338c96a8870479f2396c373cc7696ba124e8635d41b0ea581112b678172612102675333a4e4b8fb51d9d4e22fa5a8eaced3fdac8a8cbf9be8c030f75712e6af992102896807d54bc55c24981f24a453c60ad3e8993d693732288068a23df3d9f50d4821029e51a5ef5db3137051de8323b001749932f2ff0d34c82e96a2c2461de96ae56c2102a4e1a9638d46923272c266631d94d36bdb03a64ee0e14c7518e49d2f29bc401021031c41fdbcebe17bec8d49816e00ca1b5ac34766b91c9f2ac37d39c63e5e008afb2103079e252e85abffd3c401a69b087e590a9b86f33f574f08129ccbd3521ecf516b2103111cf405b627e22135b3b3733a4a34aa5723fb0f58379a16d32861bf576b0ec2210318f331b3e5d38156da6633b31929c5b220349859cc9ca3d33fb4e68aa08401742103230dae6b4ac93480aeab26d000841298e3b8f6157028e47b0897c1e025165de121035abff4281ff00660f99ab27bb53e6b33689c2cd8dcd364bc3c90ca5aea0d71a62103bd45cddfacf2083b14310ae4a84e25de61e451637346325222747b157446614c2103cc297026b06c71cbfa52089149157b5ff23de027ac5ab781800a578192d175462103d3bde5d63bdb3a6379b461be64dad45eabff42f758543a9645afd42f6d4248282103ed1e8d5109c9ed66f7941bc53cc71137baa76d50d274bda8d5e8ffbd6e61fe9a5fae736402c00fb269522103aab896d53a8e7d6433137bbba940f9c521e085dd07e60994579b64a6d992cf79210291b7d0b1b692f8f524516ed950872e5da10fb1b808b5a526dedc6fed1cf29807210386aa9372fbab374593466bc5451dc59954e90787f08060964d95c87ef34ca5bb53ae68', 4032, 2197440),
+            ('bc1qwnevjp8nsq7adu3hxlvdvslrf242q4vuavfg0y929jp2zntp3vgq7cq6z2', '5b21020e0338c96a8870479f2396c373cc7696ba124e8635d41b0ea581112b678172612102675333a4e4b8fb51d9d4e22fa5a8eaced3fdac8a8cbf9be8c030f75712e6af9921021fad939f956beae2e6765b450552f1a75236c8b7314ec30474d8f4978b252ab221029e51a5ef5db3137051de8323b001749932f2ff0d34c82e96a2c2461de96ae56c2102a4e1a9638d46923272c266631d94d36bdb03a64ee0e14c7518e49d2f29bc401021031c41fdbcebe17bec8d49816e00ca1b5ac34766b91c9f2ac37d39c63e5e008afb2102e323a079079582cf6bb68583df94bd68c9590e5904062b503ef5476109ad739e2103111cf405b627e22135b3b3733a4a34aa5723fb0f58379a16d32861bf576b0ec2210318f331b3e5d38156da6633b31929c5b220349859cc9ca3d33fb4e68aa08401742103230dae6b4ac93480aeab26d000841298e3b8f6157028e47b0897c1e025165de121035abff4281ff00660f99ab27bb53e6b33689c2cd8dcd364bc3c90ca5aea0d71a62103bd45cddfacf2083b14310ae4a84e25de61e451637346325222747b157446614c2103cc297026b06c71cbfa52089149157b5ff23de027ac5ab781800a578192d175462103d3bde5d63bdb3a6379b461be64dad45eabff42f758543a9645afd42f6d4248282103ed1e8d5109c9ed66f7941bc53cc71137baa76d50d274bda8d5e8ffbd6e61fe9a5fae736402c00fb269522103aab896d53a8e7d6433137bbba940f9c521e085dd07e60994579b64a6d992cf79210291b7d0b1b692f8f524516ed950872e5da10fb1b808b5a526dedc6fed1cf29807210386aa9372fbab374593466bc5451dc59954e90787f08060964d95c87ef34ca5bb53ae68', 4032, 3729600);
+        `);
+      await this.updateToSchemaVersion(109);
+    }
+    if (databaseSchemaVersion < 110 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD template_algo TINYINT UNSIGNED NOT NULL DEFAULT 0');
+      await this.updateToSchemaVersion(110);
+    }
+
+    if (databaseSchemaVersion < 111 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `compact_cpfp_clusters` ADD template_algo TINYINT UNSIGNED NOT NULL DEFAULT 0');
+      await this.updateToSchemaVersion(111);
+    }
+
+    if (databaseSchemaVersion < 112) {
+      await this.$executeQuery(this.getCreateFlagsValuesTableQuery(), await this.$checkIfTableExists('flag_values'));
+      await this.updateToSchemaVersion(112);
+    }
   }
 
   /**
@@ -1511,6 +1549,16 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateFederationPegScriptsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS federation_peg_scripts (
+      address varchar(100) NOT NULL,
+      fedpegscript text NOT NULL,
+      timelock int(11) unsigned NOT NULL,
+      blocknumber int(11) unsigned NOT NULL,
+      PRIMARY KEY (address)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
   private getCreatePoolsTableQuery(): string {
     return `CREATE TABLE IF NOT EXISTS pools (
       id int(11) NOT NULL AUTO_INCREMENT,
@@ -1798,6 +1846,18 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateFlagsValuesTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS flag_values (
+      bucket_size enum('1', '1008', '4032') NOT NULL,
+      start_height int unsigned NOT NULL,
+      avg_timestamp timestamp NOT NULL,
+      flag_value bigint unsigned NOT NULL,
+      tx_count int unsigned NOT NULL,
+      vsize_total int unsigned NOT NULL,
+      PRIMARY KEY (bucket_size, start_height, flag_value)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8`;
+  }
+
   /** @asyncUnsafe */
   public async $blocksReindexingTruncate(): Promise<void> {
     logger.warn(`Truncating pools, blocks, hashrates and difficulty_adjustments tables for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);

backend/src/api/liquid/elements-parser.ts

@@ -4,13 +4,23 @@ import bitcoinSecondClient from '../bitcoin/bitcoin-second-client';
 import { Common } from '../common';
 import DB from '../../database';
 import logger from '../../logger';
+import * as bitcoinjs from 'bitcoinjs-lib';
 
-const federationChangeAddresses = ['bc1qxvay4an52gcghxq5lavact7r6qe9l4laedsazz8fj2ee2cy47tlqff4aj4', '3EiAcrzq1cELXScc98KeCswGWZaPGceT1d', '3G6neksSBMp51kHJ2if8SeDUrzT8iVETWT', 'bc1qwnevjp8nsq7adu3hxlvdvslrf242q4vuavfg0y929jp2zntp3vgq7cq6z2'];
 const auditBlockOffsetWithTip = 1; // Wait for 1 block confirmation before processing the block in the audit process to reduce the risk of reorgs
+const auditSyncToleranceBlocks = 105; // Audit lag from bitcoin tip to avoid potential unsynced state on peg-ins, which require 102 confirmations
+const DYNAFED_ACTIVATION_HEIGHT = 1517040;
+const DYNAFED_CHECK_INTERVAL = 20160;
+const FEDERATION_TIMELOCK_FALLBACK = 0x0000ffff; // Preventive timelock in case an error occurs while extracting it from the fedpegscript
+interface FederationAddress {
+  fedpegscript: string;
+  address: string;
+  timelock: number;
+  blocknumber: number;
+}
 
 class ElementsParser {
   private isRunning = false;
-  private isUtxosUpdatingRunning = false;
+  private federationPegScripts: FederationAddress[] = [];
 
   constructor() { }
 
@@ -20,15 +30,23 @@ class ElementsParser {
     }
     try {
       this.isRunning = true;
+      const startedAt = Date.now() / 1000;
+      const stopAt = startedAt + 3600; // Limit one parse run to 1 hour to keep the tip sufficiently up to date
       const result = await bitcoinClient.getChainTips();
       const tip = result[0].height;
       const latestBlockHeight = await this.$getLatestBlockHeightFromDatabase();
       for (let height = latestBlockHeight + 1; height <= tip; height++) {
-        const blockHash: IBitcoinApi.ChainTips = await bitcoinClient.getBlockHash(height);
+        if ((Date.now() / 1000) >= stopAt) {
+          logger.debug(`Reached max Elements parsing runtime of one hour, pausing parse to resume later`);
+          break;
+        }
+        const blockHash: string = await bitcoinClient.getBlockHash(height);
         const block: IBitcoinApi.Block = await bitcoinClient.getBlock(blockHash, 2);
+        await this.$updateFederationPegScripts(block);
         await this.$parseBlock(block);
         await this.$saveLatestBlockToDatabase(block.height);
       }
+      await this.$updateFederationUtxos(stopAt);
       this.isRunning = false;
     } catch (e) {
       this.isRunning = false;
@@ -59,8 +77,9 @@ class ElementsParser {
     const bitcoinBlock: IBitcoinApi.Block = await bitcoinSecondClient.getBlock(bitcoinTx.blockhash);
     const prevout = bitcoinTx.vout[input.vout || 0];
     const outputAddress = prevout.scriptPubKey.address || (prevout.scriptPubKey.addresses && prevout.scriptPubKey.addresses[0]) || '';
+    const timelock = await this.$resolvePegInTimelock(input, outputAddress);
     await this.$savePegToDatabase(block.height, block.time, prevout.value * 100000000, txid, vindex,
-      outputAddress, bitcoinTx.txid, prevout.n, bitcoinBlock.height, bitcoinBlock.time, 1);
+      outputAddress, bitcoinTx.txid, prevout.n, bitcoinBlock.height, bitcoinBlock.time, 1, timelock);
   }
 
   /** @asyncUnsafe */
@@ -80,7 +99,7 @@ class ElementsParser {
 
   /** @asyncUnsafe */
   protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string,
-    txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, bitcoinblock: number, bitcoinBlockTime: number, final_tx: number): Promise<void> {
+    txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, bitcoinblock: number, bitcoinBlockTime: number, final_tx: number, pegInTimelock: number = FEDERATION_TIMELOCK_FALLBACK): Promise<void> {
     const query = `INSERT IGNORE INTO elements_pegs(
         block, datetime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
       ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`;
@@ -92,13 +111,9 @@ class ElementsParser {
     logger.debug(`Saved L-BTC peg from Liquid block height #${height} with TXID ${txid}.`);
 
     if (amount > 0) { // Peg-in
-
-      // Add the address to the federation addresses table
-      await DB.query(`INSERT IGNORE INTO federation_addresses (bitcoinaddress) VALUES (?)`, [bitcoinaddress]);
-
       // Add the UTXO to the federation txos table
       const query_utxos = `INSERT IGNORE INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate, timelock, expiredAt, emergencyKey, pegtxid, pegindex, pegblocktime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
-      const params_utxos: (string | number)[] = [bitcointxid, bitcoinindex, bitcoinaddress, amount, bitcoinblock, bitcoinBlockTime, 1, bitcoinblock - 1, 0, 4032, 0, 0, txid, txindex, blockTime];
+      const params_utxos: (string | number)[] = [bitcointxid, bitcoinindex, bitcoinaddress, amount, bitcoinblock, bitcoinBlockTime, 1, bitcoinblock - 1, 0, pegInTimelock, 0, 0, txid, txindex, blockTime];
       await DB.query(query_utxos, params_utxos);
       const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`);
       await this.$saveLastBlockAuditToDatabase(minBlockUpdate[0]['lastblockupdate']);
@@ -107,6 +122,127 @@ class ElementsParser {
     }
   }
 
+  /** @asyncUnsafe */
+  private async $getFederationPegScripts(): Promise<FederationAddress[]> {
+    if (this.federationPegScripts.length === 0) {
+      const [rows] = await DB.query(`SELECT fedpegscript, address, timelock, blocknumber FROM federation_peg_scripts ORDER BY blocknumber DESC`);
+      this.federationPegScripts = rows as FederationAddress[];
+    }
+    return this.federationPegScripts;
+  }
+
+  /** @asyncUnsafe */
+  private async $updateFederationPegScripts(block: IBitcoinApi.Block): Promise<void> {
+    const height = block.height;
+    if (height < DYNAFED_ACTIVATION_HEIGHT || height % DYNAFED_CHECK_INTERVAL !== 0) {
+      return;
+    }
+
+    const fedpegscript = block.dynamic_parameters?.current?.fedpegscript as string | undefined;
+    const fedpegProgram = block.dynamic_parameters?.current?.fedpeg_program as string | undefined;
+    if (!fedpegscript || !fedpegProgram) {
+      logger.err(`Missing fedpeg fields at height ${height}, skipping dynamic federation address update.`);
+      return;
+    }
+
+    if ((await this.$getFederationPegScripts()).some(entry => entry.fedpegscript === fedpegscript)) {
+      logger.debug(`Fedpegscript at height ${height} is already known, skipping.`);
+      return;
+    }
+
+    logger.debug(`New fedpegscript found at height ${height}, deriving federation address.`);
+    let address: string;
+    try {
+      address = bitcoinjs.address.fromOutputScript(Buffer.from(fedpegProgram, 'hex'), bitcoinjs.networks.bitcoin);
+    } catch (e) {
+      logger.err(`Unable to derive federation address from fedpeg program at height ${height}: ${e instanceof Error ? e.message : 'Error'}`);
+      return;
+    }
+
+    let timelock = this.extractTimelockFromFedpegscript(fedpegscript);
+    if (timelock === null) {
+      timelock = FEDERATION_TIMELOCK_FALLBACK;
+      logger.err(`Unable to extract federation address timelock from fedpegscript at height ${height}, using fallback timelock ${FEDERATION_TIMELOCK_FALLBACK}.`);
+    }
+
+    this.federationPegScripts.unshift({ fedpegscript, address, timelock, blocknumber: height });
+    logger.debug(`Derived new federation address ${address} with timelock ${timelock} from fedpegscript at height ${height}. Adding to database.`);
+    try {
+      await DB.query(
+        `INSERT INTO federation_peg_scripts (fedpegscript, address, timelock, blocknumber) VALUES (?, ?, ?, ?)`,
+        [fedpegscript, address, timelock, height]
+      );
+      logger.debug(`Added new federation address ${address} to the database.`);
+    } catch (e) {
+      logger.err(`Error while inserting new federation address ${address} into the database: ${e instanceof Error ? e.message : 'Error'}`);
+    }
+  }
+
+  private extractTimelockFromFedpegscript(fedpegscript: string): number | null {
+    const chunks = bitcoinjs.script.decompile(Buffer.from(fedpegscript, 'hex'));
+    if (!chunks) {
+      return null;
+    }
+
+    for (let i = 0; i < chunks.length; i++) {
+      if (chunks[i] !== bitcoinjs.opcodes.OP_CHECKSEQUENCEVERIFY) {
+        continue;
+      }
+
+      const previous = chunks[i - 1];
+      if (previous === undefined) {
+        return null;
+      }
+
+      if (Buffer.isBuffer(previous)) {
+        try {
+          return bitcoinjs.script.number.decode(previous, 5, true);
+        } catch (e) {
+          return null;
+        }
+      }
+
+      if (typeof previous === 'number') {
+        if (previous === bitcoinjs.opcodes.OP_0) {
+          return 0;
+        }
+        if (previous >= bitcoinjs.opcodes.OP_1 && previous <= bitcoinjs.opcodes.OP_16) {
+          return previous - bitcoinjs.opcodes.OP_1 + 1;
+        }
+      }
+
+      return null;
+    }
+
+    return null;
+  }
+
+  /** @asyncUnsafe */
+  private async $resolvePegInTimelock(input: IBitcoinApi.Vin, bitcoinaddress: string): Promise<number> {
+    const claimScript = input.pegin_witness?.[3];
+    if (!claimScript) {
+      logger.err(`Missing claim_script for peg-in input to address ${bitcoinaddress}, using fallback timelock ${FEDERATION_TIMELOCK_FALLBACK}.`);
+      return FEDERATION_TIMELOCK_FALLBACK;
+    }
+
+    // Check claim script against each fedpegscript most recent first
+    for (const pegScript of (await this.$getFederationPegScripts())) {
+      try {
+        const tweakResult = await bitcoinClient.tweakFedPegScript(claimScript, pegScript.fedpegscript);
+        const matches = tweakResult?.p2wsh === bitcoinaddress || tweakResult?.p2shwsh === bitcoinaddress;
+        if (matches) {
+          return pegScript.timelock;
+        }
+      } catch (e) {
+        const message = e instanceof Error ? e.message : String(e);
+        logger.err(`tweakfedpegscript failed for address ${bitcoinaddress}: ${message}`);
+      }
+    }
+
+    logger.err(`No matching fedpegscript found for address ${bitcoinaddress}, using fallback timelock ${FEDERATION_TIMELOCK_FALLBACK}.`);
+    return FEDERATION_TIMELOCK_FALLBACK;
+  }
+
   /** @asyncUnsafe */
   protected async $getLatestBlockHeightFromDatabase(): Promise<number> {
     const query = `SELECT number FROM state WHERE name = 'last_elements_block'`;
@@ -122,19 +258,12 @@ class ElementsParser {
 
   ///////////// FEDERATION AUDIT //////////////
 
-  public async $updateFederationUtxos() {
-    if (this.isUtxosUpdatingRunning) {
-      return;
-    }
-
-    this.isUtxosUpdatingRunning = true;
-
+  public async $updateFederationUtxos(stopAt: number) {
     try {
       let auditProgress = await this.$getAuditProgress();
       // If no peg in transaction was found in the database, return
       if (!auditProgress.lastBlockAudit) {
         logger.debug(`No Federation UTXOs found in the database. Waiting for some to be confirmed before starting the Federation UTXOs audit`);
-        this.isUtxosUpdatingRunning = false;
         return;
       }
 
@@ -142,7 +271,6 @@ class ElementsParser {
       // If the bitcoin blockchain is not synced yet, return
       if (bitcoinBlocksToSync.bitcoinHeaders > bitcoinBlocksToSync.bitcoinBlocks + 1) {
         logger.debug(`Bitcoin client is not synced yet. ${bitcoinBlocksToSync.bitcoinHeaders - bitcoinBlocksToSync.bitcoinBlocks} blocks remaining to sync before the Federation audit process can start`);
-        this.isUtxosUpdatingRunning = false;
         return;
       }
 
@@ -161,19 +289,28 @@ class ElementsParser {
 
         // Get the peg-out addresses that need to be scanned
         const redeemAddresses = await this.$getRedeemAddressesToScan();
+        const redeemAddressesByAddress = new Map<string, any[]>();
+        for (const redeemAddress of redeemAddresses) {
+          const entries = redeemAddressesByAddress.get(redeemAddress.bitcoinaddress);
+          if (entries) {
+            entries.push(redeemAddress);
+          } else {
+            redeemAddressesByAddress.set(redeemAddress.bitcoinaddress, [redeemAddress]);
+          }
+        }
 
         // The fast way: check if these UTXOs are still unspent as of the current block with gettxout
-        let spentAsTip: any[];
-        let unspentAsTip: any[];
+        let spentAsTip: Map<string, any>;
+        let unspentAsTip: Map<string, any>;
         if (auditProgress.confirmedTip - auditProgress.lastBlockAudit <= 150) { // If the audit status is not too far in the past, we can use gettxout (fast way)
           const utxosToParse = await this.$getFederationUtxosToParse(utxos);
           spentAsTip = utxosToParse.spentAsTip;
           unspentAsTip = utxosToParse.unspentAsTip;
           logger.debug(`Found ${utxos.length} Federation UTXOs and ${redeemAddresses.length} Peg-Out Addresses to scan in Bitcoin block height #${auditProgress.lastBlockAudit} / #${auditProgress.confirmedTip}`);
-          logger.debug(`${unspentAsTip.length} / ${utxos.length} Federation UTXOs are unspent as of tip`);
+          logger.debug(`${unspentAsTip.size} / ${utxos.length} Federation UTXOs are unspent as of tip`);
         } else { // If the audit status is too far in the past, it is useless and wasteful to look for still unspent txos since they will all be spent as of the tip
-          spentAsTip = utxos;
-          unspentAsTip = [];
+          spentAsTip = new Map<string, any>(utxos.map(utxo => [`${utxo.txid}:${utxo.txindex}`, utxo]));
+          unspentAsTip = new Map<string, any>();
 
           // Logging
           const elapsedSeconds = (Date.now() / 1000) - timer;
@@ -193,7 +330,7 @@ class ElementsParser {
         // The slow way: parse the block to look for the spending tx
         const blockHash: IBitcoinApi.ChainTips = await bitcoinSecondClient.getBlockHash(auditProgress.lastBlockAudit);
         const block: IBitcoinApi.Block = await bitcoinSecondClient.getBlock(blockHash, 2);
-        await this.$parseBitcoinBlock(block, spentAsTip, unspentAsTip, auditProgress.confirmedTip, redeemAddresses);
+        await this.$parseBitcoinBlock(block, spentAsTip, unspentAsTip, auditProgress.confirmedTip, redeemAddressesByAddress);
 
         // Finally, update the lastblockupdate of the remaining UTXOs and save to the database
         const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`);
@@ -202,11 +339,14 @@ class ElementsParser {
         auditProgress = await this.$getAuditProgress();
         auditProgress.lastBlockAudit++;
         indexedThisRun++;
+
+        if ((Date.now() / 1000) >= stopAt) {
+          logger.debug(`Reached max federation audit runtime, pausing audit to let Liquid parsing run and resuming later`);
+          break;
+        }
       }
 
-      this.isUtxosUpdatingRunning = false;
     } catch (e) {
-      this.isUtxosUpdatingRunning = false;
       throw new Error(e instanceof Error ? e.message : 'Error');
     }
   }
@@ -221,28 +361,29 @@ class ElementsParser {
 
   // Returns the UTXOs that are spent as of tip and need to be scanned
   /** @asyncUnsafe */
-  protected async $getFederationUtxosToParse(utxos: any[]): Promise<any> {
-    const spentAsTip: any[] = [];
-    const unspentAsTip: any[] = [];
+  protected async $getFederationUtxosToParse(utxos: any[]): Promise<{ spentAsTip: Map<string, any>; unspentAsTip: Map<string, any> }> {
+    const spentAsTip = new Map<string, any>();
+    const unspentAsTip = new Map<string, any>();
 
     for (const utxo of utxos) {
       const result = await bitcoinSecondClient.getTxOut(utxo.txid, utxo.txindex, false);
-      result ? unspentAsTip.push(utxo) : spentAsTip.push(utxo);
+      const key = `${utxo.txid}:${utxo.txindex}`;
+      result ? unspentAsTip.set(key, utxo) : spentAsTip.set(key, utxo);
     }
 
     return {spentAsTip, unspentAsTip};
   }
 
   /** @asyncUnsafe */
-  protected async $parseBitcoinBlock(block: IBitcoinApi.Block, spentAsTip: any[], unspentAsTip: any[], confirmedTip: number, redeemAddressesData: any[] = []) {
-    const redeemAddresses: string[] = redeemAddressesData.map(redeemAddress => redeemAddress.bitcoinaddress);
+  protected async $parseBitcoinBlock(block: IBitcoinApi.Block, spentAsTip: Map<string, any>, unspentAsTip: Map<string, any>, confirmedTip: number, redeemAddressesByAddress: Map<string, any[]>) {
+    const federationTimelockByAddress = new Map<string, number>((await this.$getFederationPegScripts()).map(fedAddress => [fedAddress.address, fedAddress.timelock]));
     for (const tx of block.tx) {
       let mightRedeemInThisTx = false;
       // Check if the Federation UTXOs that was spent as of tip are spent in this block
       for (const input of tx.vin) {
-        const txo = spentAsTip.find(txo => txo.txid === input.txid && txo.txindex === input.vout);
+        const txo = spentAsTip.get(`${input.txid!}:${input.vout!}`);
         if (txo) {
-          mightRedeemInThisTx = true; // A Federation UTXO is spent in this block: we might find a peg-out address in the outputs
+          mightRedeemInThisTx = true; // A Federation UTXO is spent in this tx: a peg-out redeem output may be present
           if (txo.expiredAt > 0 ) {
             if (input.txinwitness?.length !== 13) { // Check if the witness data of the input contains the 11 signatures: if it doesn't, emergency keys are being used
               await DB.query(`UPDATE federation_txos SET unspent = 0, lastblockupdate = ?, lasttimeupdate = ?, emergencyKey = 1 WHERE txid = ? AND txindex = ?`, [block.height, block.time, txo.txid, txo.txindex]);
@@ -255,36 +396,44 @@ class ElementsParser {
             await DB.query(`UPDATE federation_txos SET unspent = 0, lastblockupdate = ?, lasttimeupdate = ? WHERE txid = ? AND txindex = ?`, [block.height, block.time, txo.txid, txo.txindex]);
             logger.debug(`Federation UTXO ${txo.txid}:${txo.txindex} (${txo.amount} sats) was spent in block ${block.height}`);
           }
-          // Remove the TXO from the utxo array
-          spentAsTip.splice(spentAsTip.indexOf(txo), 1);
+          // Remove the TXO from the map
+          spentAsTip.delete(`${txo.txid}:${txo.txindex}`);
         }
       }
       // Check if an output is sent to a change address of the federation
       for (const output of tx.vout) {
-        if (output.scriptPubKey.address && federationChangeAddresses.includes(output.scriptPubKey.address)) {
-          // Check that the UTXO was not already added in the DB by previous scans
-          const [rows_check] = await DB.query(`SELECT txid FROM federation_txos WHERE txid = ? AND txindex = ?`, [tx.txid, output.n]) as any[];
-          if (rows_check.length === 0) {
-            const timelock = output.scriptPubKey.address === federationChangeAddresses[1] ? 2016 : 4032; // hardcode timelock for 3EiAcrzq... This will be addressed better in the future
-            const query_utxos = `INSERT INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate, timelock, expiredAt, emergencyKey, pegtxid, pegindex, pegblocktime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
-            const params_utxos: (string | number)[] = [tx.txid, output.n, output.scriptPubKey.address, output.value * 100000000, block.height, block.time, 1, block.height, 0, timelock, 0, 0, '', 0, 0];
-            await DB.query(query_utxos, params_utxos);
-            // Add the UTXO to the utxo array
-            spentAsTip.push({
-              txid: tx.txid,
-              txindex: output.n,
-              bitcoinaddress: output.scriptPubKey.address,
-              amount: output.value * 100000000,
-              blocknumber: block.height,
-              timelock: timelock,
-              expiredAt: 0,
-            });
-            logger.debug(`Added new Federation UTXO ${tx.txid}:${output.n} (${Math.round(output.value * 100000000)} sats), change address: ${output.scriptPubKey.address}`);
+        if (output.scriptPubKey.address) {
+          const timelock = federationTimelockByAddress.get(output.scriptPubKey.address);
+          if (timelock !== undefined) {
+            // Check that the UTXO was not already added in the DB by previous scans
+            const [rows_check] = await DB.query(`SELECT txid FROM federation_txos WHERE txid = ? AND txindex = ?`, [tx.txid, output.n]) as any[];
+            if (rows_check.length === 0) {
+              const query_utxos = `INSERT INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate, timelock, expiredAt, emergencyKey, pegtxid, pegindex, pegblocktime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
+              const params_utxos: (string | number)[] = [tx.txid, output.n, output.scriptPubKey.address, output.value * 100000000, block.height, block.time, 1, block.height, 0, timelock, 0, 0, '', 0, 0];
+              await DB.query(query_utxos, params_utxos);
+              // Add the UTXO to the map
+              spentAsTip.set(`${tx.txid}:${output.n}`, {
+                txid: tx.txid,
+                txindex: output.n,
+                bitcoinaddress: output.scriptPubKey.address,
+                amount: output.value * 100000000,
+                blocknumber: block.height,
+                timelock: timelock,
+                expiredAt: 0,
+              });
+              logger.debug(`Added new Federation UTXO ${tx.txid}:${output.n} (${Math.round(output.value * 100000000)} sats), change address: ${output.scriptPubKey.address}`);
+            }
           }
-        }
-        if (mightRedeemInThisTx && output.scriptPubKey.address && redeemAddresses.includes(output.scriptPubKey.address)) {
-          // Find the number of times output.scriptPubKey.address appears in redeemAddresses. There can be address reuse for peg-outs...
-          const matchingAddress: any[] = redeemAddressesData.filter(redeemAddress => redeemAddress.bitcoinaddress === output.scriptPubKey.address && -redeemAddress.amount === Math.round(output.value * 100000000));
+
+          if (!mightRedeemInThisTx) {
+            continue;
+          }
+          const redeemCandidates = redeemAddressesByAddress.get(output.scriptPubKey.address);
+          if (!redeemCandidates) {
+            continue;
+          }
+          // Find the number of times output.scriptPubKey.address appears in the candidates. There can be address reuse for peg-outs...
+          const matchingAddress: any[] = redeemCandidates.filter(redeemAddress => -redeemAddress.amount === Math.round(output.value * 100000000));
           if (matchingAddress.length > 0) {
             if (matchingAddress.length > 1) {
               // If there are more than one peg out address with the same amount, we can't know which one redeemed the UTXO: we take the oldest one
@@ -296,9 +445,13 @@ class ElementsParser {
             const query_add_redeem = `UPDATE elements_pegs SET bitcointxid = ?, bitcoinindex = ? WHERE bitcoinaddress = ? AND amount = ? AND datetime = ?`;
             const params_add_redeem: (string | number)[] = [tx.txid, output.n, matchingAddress[0].bitcoinaddress, matchingAddress[0].amount, matchingAddress[0].datetime];
             await DB.query(query_add_redeem, params_add_redeem);
-            const index = redeemAddressesData.indexOf(matchingAddress[0]);
-            redeemAddressesData.splice(index, 1);
-            redeemAddresses.splice(index, 1);
+            const index = redeemCandidates.indexOf(matchingAddress[0]);
+            if (index !== -1) {
+              redeemCandidates.splice(index, 1);
+              if (redeemCandidates.length === 0) {
+                redeemAddressesByAddress.delete(output.scriptPubKey.address);
+              }
+            }
           } else { // The output amount does not match the peg-out amount... log it
             logger.debug(`Found redeem txid ${tx.txid}:${output.n} to peg-out address ${output.scriptPubKey.address} but output amount ${Math.round(output.value * 100000000)} does not match the peg-out amount!`);
           }
@@ -306,7 +459,7 @@ class ElementsParser {
       }
     }
 
-    for (const utxo of spentAsTip) {
+    for (const utxo of spentAsTip.values()) {
       if (utxo.expiredAt === 0 && block.height >= utxo.blocknumber + utxo.timelock) { // The UTXO is expiring in this block
         await DB.query(`UPDATE federation_txos SET lastblockupdate = ?, expiredAt = ? WHERE txid = ? AND txindex = ?`, [block.height, block.time, utxo.txid, utxo.txindex]);
       } else {
@@ -314,7 +467,7 @@ class ElementsParser {
       }
     }
 
-    for (const utxo of unspentAsTip) {
+    for (const utxo of unspentAsTip.values()) {
       if (utxo.expiredAt === 0 && block.height >= utxo.blocknumber + utxo.timelock) { // The UTXO is expiring in this block
         await DB.query(`UPDATE federation_txos SET lastblockupdate = ?, expiredAt = ? WHERE txid = ? AND txindex = ?`, [confirmedTip, block.time, utxo.txid, utxo.txindex]);
       } else if (utxo.expiredAt === 0 && confirmedTip >= utxo.blocknumber + utxo.timelock) { // The UTXO is expiring before the tip: we need to keep track of it
@@ -380,7 +533,7 @@ class ElementsParser {
       bitcoinBlocks: bitcoinBlocksToSync.bitcoinBlocks,
       bitcoinHeaders: bitcoinBlocksToSync.bitcoinHeaders,
       lastBlockAudit: lastBlockAudit,
-      isAuditSynced: bitcoinBlocksToSync.bitcoinHeaders - bitcoinBlocksToSync.bitcoinBlocks <= 2 && bitcoinBlocksToSync.bitcoinBlocks - lastBlockAudit <= 3,
+      isAuditSynced: bitcoinBlocksToSync.bitcoinHeaders - bitcoinBlocksToSync.bitcoinBlocks <= 3 && bitcoinBlocksToSync.bitcoinBlocks - lastBlockAudit <= auditSyncToleranceBlocks,
     };
   }
 

backend/src/api/liquid/liquid.routes.ts

@@ -6,6 +6,8 @@ import icons from './icons';
 import { handleError } from '../../utils/api';
 import PricesRepository from '../../repositories/PricesRepository';
 
+const PROXY_PATH_SEGMENT_REGEX = /^(?!\.{1,2}$)[^\p{Cc}/?#\\]{1,256}$/u;
+
 class LiquidRoutes {
   public initRoutes(app: Application) {
     app
@@ -68,8 +70,13 @@ class LiquidRoutes {
   }
 
   private async $getAssetGroup(req: Request, res: Response) {
+    if (!PROXY_PATH_SEGMENT_REGEX.test(req.params.id)) {
+      handleError(req, res, 400, 'Invalid asset group id');
+      return;
+    }
+
     try {
-      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/group/${parseInt(req.params.id, 10)}`,
+      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/group/${encodeURIComponent(req.params.id)}`,
         { responseType: 'stream', timeout: 10000 });
       response.data.pipe(res);
     } catch (e) {

backend/src/api/mempool-blocks.ts

@@ -8,6 +8,7 @@ import path from 'path';
 import mempool from './mempool';
 import { Acceleration } from './services/acceleration';
 import PoolsRepository from '../repositories/PoolsRepository';
+import { ProjectedBlock } from '../cluster-mempool/cluster-mempool';
 
 const MAX_UINT32 = Math.pow(2, 32) - 1;
 
@@ -238,7 +239,7 @@ class MempoolBlocks {
   }
 
   /** @asyncSafe */
-  public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+  public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number, dryRun = false): Promise<MempoolBlockWithTransactions[]> {
     const start = Date.now();
 
     // reset mempool short ids
@@ -278,7 +279,7 @@ class MempoolBlocks {
       const expectedSize = transactions.length;
       const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length;
       logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${expectedSize} in the mempool, ${overflow.length} were unmineable`);
-      const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults);
+      const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults, dryRun);
       logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
       return processed;
     } catch (e) {
@@ -296,16 +297,15 @@ class MempoolBlocks {
   }
 
   /** @asyncSafe */
-  public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+  public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number, dryRun = false): Promise<MempoolBlockWithTransactions[]> {
     // GBT optimization requires that uids never get too sparse
     // as a sanity check, we should also explicitly prevent uint32 uid overflow
     if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * transactions.length), MAX_UINT32)) {
       this.resetRustGbt();
     }
 
-    if (!this.rustInitialized) {
-      // need to reset the worker
-      return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, true, useAccelerations, accelerationPool);
+    if (!this.rustInitialized || dryRun) {
+      return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, !dryRun, useAccelerations, accelerationPool, dryRun);
     }
 
     const start = Date.now();
@@ -344,7 +344,7 @@ class MempoolBlocks {
       if (transactions.length !== resultMempoolSize) {
         throw new Error(`GBT returned wrong number of transactions ${transactions.length} vs ${resultMempoolSize}, cache is probably out of sync`);
       } else {
-        const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, true);
+        const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, !dryRun, dryRun);
         this.removeUids(removedTxs);
         logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
         return processed;
@@ -356,20 +356,22 @@ class MempoolBlocks {
     }
   }
 
-  private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations: { [txid: string]: Acceleration }, accelerationPool, saveResults): MempoolBlockWithTransactions[] {
-    for (const txid of Object.keys(candidates?.txs ?? mempool)) {
-      if (txid in mempool) {
-        mempool[txid].cpfpDirty = false;
-        mempool[txid].ancestors = [];
-        mempool[txid].descendants = [];
-        mempool[txid].bestDescendant = null;
+  private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations: { [txid: string]: Acceleration }, accelerationPool, saveResults, dryRun = false): MempoolBlockWithTransactions[] {
+    if (!dryRun) {
+      for (const txid of Object.keys(candidates?.txs ?? mempool)) {
+        if (txid in mempool) {
+          mempool[txid].cpfpDirty = false;
+          mempool[txid].ancestors = [];
+          mempool[txid].descendants = [];
+          mempool[txid].bestDescendant = null;
+        }
       }
-    }
-    for (const [txid, rate] of rates) {
-      if (txid in mempool) {
-        mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize);
-        mempool[txid].effectiveFeePerVsize = rate;
-        mempool[txid].cpfpChecked = true;
+      for (const [txid, rate] of rates) {
+        if (txid in mempool) {
+          mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize);
+          mempool[txid].effectiveFeePerVsize = rate;
+          mempool[txid].cpfpChecked = true;
+        }
       }
     }
 
@@ -387,58 +389,60 @@ class MempoolBlocks {
       feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
     }
 
-    const ancestors: Ancestor[] = [];
-    const descendants: Ancestor[] = [];
-    let ancestor: MempoolTransactionExtended;
-    for (const cluster of clusters) {
-      for (const memberTxid of cluster) {
-        const mempoolTx = mempool[memberTxid];
-        if (mempoolTx) {
-          // ugly micro-optimization to avoid allocating new arrays
-          ancestors.length = 0;
-          descendants.length = 0;
-          let matched = false;
-          cluster.forEach(txid => {
-            ancestor = mempool[txid];
-            if (txid === memberTxid) {
-              matched = true;
-            } else {
-              if (!ancestor) {
-                console.log('txid missing from mempool! ', txid, candidates?.txs[txid]);
-                return;
-              }
-              const relative = {
-                txid: txid,
-                fee: ancestor.fee,
-                weight: (ancestor.adjustedVsize * 4),
-              };
-              if (matched) {
-                descendants.push(relative);
-                if (!mempoolTx.lastBoosted || (ancestor.firstSeen && ancestor.firstSeen > mempoolTx.lastBoosted)) {
-                  mempoolTx.lastBoosted = ancestor.firstSeen;
-                }
+    if (!dryRun) {
+      const ancestors: Ancestor[] = [];
+      const descendants: Ancestor[] = [];
+      let ancestor: MempoolTransactionExtended;
+      for (const cluster of clusters) {
+        for (const memberTxid of cluster) {
+          const mempoolTx = mempool[memberTxid];
+          if (mempoolTx) {
+            // ugly micro-optimization to avoid allocating new arrays
+            ancestors.length = 0;
+            descendants.length = 0;
+            let matched = false;
+            cluster.forEach(txid => {
+              ancestor = mempool[txid];
+              if (txid === memberTxid) {
+                matched = true;
               } else {
-                ancestors.push(relative);
+                if (!ancestor) {
+                  console.log('txid missing from mempool! ', txid, candidates?.txs[txid]);
+                  return;
+                }
+                const relative = {
+                  txid: txid,
+                  fee: ancestor.fee,
+                  weight: (ancestor.adjustedVsize * 4),
+                };
+                if (matched) {
+                  descendants.push(relative);
+                  if (!mempoolTx.lastBoosted || (ancestor.firstSeen && ancestor.firstSeen > mempoolTx.lastBoosted)) {
+                    mempoolTx.lastBoosted = ancestor.firstSeen;
+                  }
+                } else {
+                  ancestors.push(relative);
+                }
               }
+            });
+            if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) {
+              mempoolTx.cpfpDirty = true;
             }
-          });
-          if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) {
-            mempoolTx.cpfpDirty = true;
+            // ugly micro-optimization to avoid allocating new arrays or objects
+            if (mempoolTx.ancestors) {
+              mempoolTx.ancestors.length = 0;
+            } else {
+              mempoolTx.ancestors = [];
+            }
+            if (mempoolTx.descendants) {
+              mempoolTx.descendants.length = 0;
+            } else {
+              mempoolTx.descendants = [];
+            }
+            mempoolTx.ancestors.push(...ancestors);
+            mempoolTx.descendants.push(...descendants);
+            mempoolTx.cpfpChecked = true;
           }
-          // ugly micro-optimization to avoid allocating new arrays or objects
-          if (mempoolTx.ancestors) {
-            mempoolTx.ancestors.length = 0;
-          } else {
-            mempoolTx.ancestors = [];
-          }
-          if (mempoolTx.descendants) {
-            mempoolTx.descendants.length = 0;
-          } else {
-            mempoolTx.descendants = [];
-          }
-          mempoolTx.ancestors.push(...ancestors);
-          mempoolTx.descendants.push(...descendants);
-          mempoolTx.cpfpChecked = true;
         }
       }
     }
@@ -471,34 +475,42 @@ class MempoolBlocks {
         const txid = block[i];
         if (txid in mempool) {
           mempoolTx = mempool[txid];
-          // save position in projected blocks
-          mempoolTx.position = {
-            block: blockIndex,
-            vsize: totalVsize + (mempoolTx.vsize / 2),
-          };
 
-          if (txid in accelerations) {
-            acceleration = accelerations[txid];
-            if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
-              if (!mempoolTx.acceleration) {
-                mempoolTx.cpfpDirty = true;
-              }
-              mempoolTx.acceleration = true;
-              mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools;
-              mempoolTx.acceleratedAt = acceleration?.added;
-              mempoolTx.feeDelta = acceleration?.feeDelta;
-              for (const ancestor of mempoolTx.ancestors || []) {
-                if (!(ancestor.txid in mempool)) {
-                  continue;
+          if (!dryRun) {
+            // save position in projected blocks
+            mempoolTx.position = {
+              block: blockIndex,
+              vsize: totalVsize + (mempoolTx.vsize / 2),
+            };
+
+            if (txid in accelerations) {
+              acceleration = accelerations[txid];
+              if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
+                if (!mempoolTx.acceleration) {
+                  mempoolTx.cpfpDirty = true;
                 }
-                if (!mempool[ancestor.txid].acceleration) {
-                  mempool[ancestor.txid].cpfpDirty = true;
+                mempoolTx.acceleration = true;
+                mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools;
+                mempoolTx.acceleratedAt = acceleration?.added;
+                mempoolTx.feeDelta = acceleration?.feeDelta;
+                for (const ancestor of mempoolTx.ancestors || []) {
+                  if (!(ancestor.txid in mempool)) {
+                    continue;
+                  }
+                  if (!mempool[ancestor.txid].acceleration) {
+                    mempool[ancestor.txid].cpfpDirty = true;
+                  }
+                  mempool[ancestor.txid].acceleration = true;
+                  mempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy;
+                  mempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt;
+                  mempool[ancestor.txid].feeDelta = mempoolTx.feeDelta;
+                  isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy;
+                }
+              } else {
+                if (mempoolTx.acceleration) {
+                  mempoolTx.cpfpDirty = true;
+                  delete mempoolTx.acceleration;
                 }
-                mempool[ancestor.txid].acceleration = true;
-                mempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy;
-                mempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt;
-                mempool[ancestor.txid].feeDelta = mempoolTx.feeDelta;
-                isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy;
               }
             } else {
               if (mempoolTx.acceleration) {
@@ -506,11 +518,6 @@ class MempoolBlocks {
                 delete mempoolTx.acceleration;
               }
             }
-          } else {
-            if (mempoolTx.acceleration) {
-              mempoolTx.cpfpDirty = true;
-              delete mempoolTx.acceleration;
-            }
           }
 
           // online calculation of stack-of-blocks fee stats
@@ -548,6 +555,115 @@ class MempoolBlocks {
     return mempoolBlocks;
   }
 
+  public processClusterMempoolBlocks(projectedBlocks: ProjectedBlock[], newMempool: { [txid: string]: MempoolTransactionExtended }, accelerations: { [txid: string]: Acceleration }, saveResults = true, accelerationPool?: number): MempoolBlockWithTransactions[] {
+    const lastBlockIndex = projectedBlocks.length - 1;
+    let hasBlockStack = projectedBlocks.length >= 8;
+    let stackWeight = 0;
+    let feeStatsCalculator: OnlineFeeStatsCalculator | null = null;
+    if (hasBlockStack) {
+      stackWeight = projectedBlocks[lastBlockIndex].weight;
+      hasBlockStack = stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS;
+      feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
+    }
+
+    const isAcceleratedBy: { [txid: string]: number[] | false } = {};
+
+    const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
+    let mempoolTx: MempoolTransactionExtended;
+    let acceleration: Acceleration;
+    const mempoolBlocks: MempoolBlockWithTransactions[] = [];
+
+    for (let blockIndex = 0; blockIndex < projectedBlocks.length; blockIndex++) {
+      const projected = projectedBlocks[blockIndex];
+      let totalSize = 0;
+      let totalVsize = 0;
+      let totalWeight = 0;
+      let totalFees = 0;
+      const transactions: MempoolTransactionExtended[] = [];
+      const validTxids: string[] = [];
+
+      for (const txid of projected.txids) {
+        if (txid in newMempool) {
+          mempoolTx = newMempool[txid];
+          validTxids.push(txid);
+
+          // save position in projected blocks
+          mempoolTx.position = {
+            block: blockIndex,
+            vsize: totalVsize + (mempoolTx.vsize / 2),
+          };
+
+          if (txid in accelerations) {
+            acceleration = accelerations[txid];
+            if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
+              if (!mempoolTx.acceleration) {
+                mempoolTx.cpfpDirty = true;
+              }
+              mempoolTx.acceleration = true;
+              mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools;
+              mempoolTx.acceleratedAt = acceleration?.added;
+              mempoolTx.feeDelta = acceleration?.feeDelta;
+              for (const ancestor of mempoolTx.ancestors || []) {
+                if (!(ancestor.txid in newMempool)) {
+                  continue;
+                }
+                if (!newMempool[ancestor.txid].acceleration) {
+                  newMempool[ancestor.txid].cpfpDirty = true;
+                }
+                newMempool[ancestor.txid].acceleration = true;
+                newMempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy;
+                newMempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt;
+                newMempool[ancestor.txid].feeDelta = mempoolTx.feeDelta;
+                isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy;
+              }
+            } else {
+              if (mempoolTx.acceleration) {
+                mempoolTx.cpfpDirty = true;
+                delete mempoolTx.acceleration;
+              }
+            }
+          } else {
+            if (mempoolTx.acceleration) {
+              mempoolTx.cpfpDirty = true;
+              delete mempoolTx.acceleration;
+            }
+          }
+
+          if (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) {
+            feeStatsCalculator.processNext(mempoolTx);
+          }
+
+          totalSize += mempoolTx.size;
+          totalVsize += mempoolTx.vsize;
+          totalWeight += mempoolTx.weight;
+          totalFees += mempoolTx.fee;
+
+          if (totalVsize <= sizeLimit) {
+            transactions.push(mempoolTx);
+          }
+        }
+      }
+
+      mempoolBlocks[blockIndex] = this.dataToMempoolBlocks(
+        validTxids,
+        transactions,
+        totalSize,
+        totalWeight,
+        totalFees,
+        (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined,
+      );
+    }
+
+    if (saveResults) {
+      const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
+      this.mempoolBlocks = mempoolBlocks;
+      this.mempoolBlockDeltas = deltas;
+      this.updateAccelerationPositions(newMempool, accelerations, mempoolBlocks);
+    }
+
+    return mempoolBlocks;
+  }
+
   private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions {
     if (!feeStats) {
       feeStats = Common.calcEffectiveFeeStatistics(transactions);

backend/src/api/mempool.ts

@@ -13,6 +13,7 @@ import { Acceleration } from './services/acceleration';
 import accelerationApi from './services/acceleration';
 import redisCache from './redis-cache';
 import blocks from './blocks';
+import { ClusterMempool } from '../cluster-mempool/cluster-mempool';
 
 class Mempool {
   private inSync: boolean = false;
@@ -22,6 +23,7 @@ class Mempool {
   private spendMap = new Map<string, MempoolTransactionExtended>();
   private recentlyDeleted: MempoolTransactionExtended[][] = []; // buffer of transactions deleted in recent mempool updates
   private mempoolInfo: IBitcoinApi.MempoolInfo;
+  public clusterMempool: ClusterMempool | null = null;
   private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[],
     deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[]) => void) | undefined;
   private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[],
@@ -62,6 +64,9 @@ class Mempool {
       minrelaytxfee: isLiquid ? 0.00000100 : 0.00001000
     };
     this.txPerSecondInterval = setInterval(this.updateTxPerSecond.bind(this), 1000);
+    if (config.MEMPOOL.CLUSTER_MEMPOOL) {
+      this.clusterMempool = new ClusterMempool(this.mempoolCache, this.accelerations);
+    }
   }
 
   /**
@@ -153,6 +158,9 @@ class Mempool {
       await redisCache.$flushTransactions();
       logger.debug(`Finished migrating cache transactions in ${((Date.now() - redisTimer) / 1000).toFixed(2)} seconds`);
     }
+    if (config.MEMPOOL.CLUSTER_MEMPOOL) {
+      this.clusterMempool = new ClusterMempool(this.mempoolCache, this.accelerations);
+    }
     if (this.mempoolChangedCallback) {
       this.mempoolChangedCallback(this.mempoolCache, [], [], []);
     }
@@ -374,6 +382,7 @@ class Mempool {
       for (const tx of deletedTransactions) {
         delete this.mempoolCache[tx.txid];
       }
+      redisCache.queueTransactionsForRemoval(deletedTransactions.map(tx => tx.txid));
     }
 
     const candidates = await this.getNextCandidates(minFeeMempool, minFeeTip, deletedTransactions);
@@ -387,6 +396,14 @@ class Mempool {
       hasChange = true;
     }
 
+    if (config.MEMPOOL.CLUSTER_MEMPOOL && (newTransactions.length || deletedTransactions.length || accelerationDelta.length)) {
+      this.clusterMempool?.applyMempoolChange({
+        added: newTransactions,
+        removed: deletedTransactions,
+        accelerations: this.getAccelerations(),
+      });
+    }
+
     this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize);
 
     const candidatesChanged = candidates?.added?.length || candidates?.removed?.length;
@@ -412,7 +429,7 @@ class Mempool {
     // Update Redis cache
     if (config.REDIS.ENABLED) {
       await redisCache.$flushTransactions();
-      await redisCache.$removeTransactions(deletedTransactions.map(tx => tx.txid));
+      await redisCache.$removeTransactions();
       await rbfCache.updateCache();
     }
 

backend/src/api/mining/mining.ts

@@ -114,31 +114,29 @@ class Mining {
     const poolsStatistics = {};
 
     const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(interval);
-    const emptyBlocks: any[] = await BlocksRepository.$countEmptyBlocks(null, interval);
 
     const poolsStats: PoolStats[] = [];
     let rank = 1;
+    let blockCount = 0;
 
     poolsInfo.forEach((poolInfo: PoolInfo) => {
-      const emptyBlocksCount = emptyBlocks.filter((emptyCount) => emptyCount.poolId === poolInfo.poolId);
       const poolStat: PoolStats = {
         poolId: poolInfo.poolId, // mysql row id
         name: poolInfo.name,
         link: poolInfo.link,
         blockCount: poolInfo.blockCount,
         rank: rank++,
-        emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0,
+        emptyBlocks: poolInfo.emptyBlocks,
         slug: poolInfo.slug,
         avgMatchRate: poolInfo.avgMatchRate !== null ? Math.round(100 * poolInfo.avgMatchRate) / 100 : null,
         avgFeeDelta: poolInfo.avgFeeDelta,
         poolUniqueId: poolInfo.poolUniqueId
       };
       poolsStats.push(poolStat);
+      blockCount += poolInfo.blockCount;
     });
 
     poolsStatistics['pools'] = poolsStats;
-
-    const blockCount: number = await BlocksRepository.$blockCount(null, interval);
     poolsStatistics['blockCount'] = blockCount;
 
     const totalBlock24h: number = await BlocksRepository.$blockCount(null, '24h');
@@ -151,6 +149,8 @@ class Mining {
       poolsStatistics['lastEstimatedHashrate1w'] = await bitcoinClient.getNetworkHashPs(totalBlock1w);
     } catch (e) {
       poolsStatistics['lastEstimatedHashrate'] = 0;
+      poolsStatistics['lastEstimatedHashrate3d'] = 0;
+      poolsStatistics['lastEstimatedHashrate1w'] = 0;
       logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate', logger.tags.mining);
     }
 

backend/src/api/redis-cache.ts

@@ -23,11 +23,14 @@ class RedisCache {
 
   private pauseFlush: boolean = false;
   private cacheQueue: MempoolTransactionExtended[] = [];
-  private removeQueue: string[] = [];
+  private removeQueue = new Set<string>();
+  private removeQueueFlushInProgress: boolean = false;
   private rbfCacheQueue: { type: string, txid: string, value: any }[] = [];
   private rbfRemoveQueue: { type: string, txid: string }[] = [];
   private txFlushLimit: number = 10000;
   private ignoreBlocksCache = false;
+  private reconciliationInProgress: boolean = false;
+  private reconciliationCursor: string = '0';
 
   constructor() {
     if (config.REDIS.ENABLED) {
@@ -39,6 +42,7 @@ class RedisCache {
       };
       void this.$ensureConnected();
       setInterval(() => { void this.$ensureConnected(); }, 10000);
+      setInterval(() => { void this.$reconcileMempoolTransactions(); }, 30000);
     }
   }
 
@@ -92,7 +96,7 @@ class RedisCache {
 
   private async $onConnected(): Promise<void> {
     await this.$flushTransactions();
-    await this.$removeTransactions([]);
+    await this.$removeTransactions();
     await this.$flushRbfQueues();
   }
 
@@ -134,6 +138,7 @@ class RedisCache {
     if (!config.REDIS.ENABLED) {
       return;
     }
+    this.removeQueue.delete(tx.txid);
     this.cacheQueue.push(tx);
     if (this.cacheQueue.length >= this.txFlushLimit) {
       if (!this.pauseFlush) {
@@ -182,32 +187,88 @@ class RedisCache {
     }
   }
 
-  /** @asyncSafe */
-  async $removeTransactions(transactions: string[]): Promise<void> {
+  queueTransactionsForRemoval(transactions: string[]): void {
     if (!config.REDIS.ENABLED) {
       return;
     }
-    const toRemove = this.removeQueue.concat(transactions);
-    this.removeQueue = [];
-    let failed: string[] = [];
-    let numRemoved = 0;
-    if (this.connected) {
+
+    for (const txid of transactions) {
+      this.removeQueue.add(txid);
+    }
+  }
+
+  /** @asyncSafe */
+  async $removeTransactions(transactions: string[] = []): Promise<void> {
+    if (!config.REDIS.ENABLED) {
+      return;
+    }
+    for (const txid of transactions) {
+      this.removeQueue.add(txid);
+    }
+    await this.$flushQueuedMempoolTxRemovals();
+  }
+
+  // incrementally reconcile the redis cache with the in-memory mempool
+  // by scanning for cached txs no longer in the mempool and marking for deletion
+  // each invocation scans 1000 keys, cursor loops back to the start after completing a full scan
+  /** @asyncSafe */
+  private async $reconcileMempoolTransactions(): Promise<void> {
+    if (!config.REDIS.ENABLED || !this.connected || this.reconciliationInProgress || !memPool.isInSync()) {
+      return;
+    }
+
+    this.reconciliationInProgress = true;
+    try {
+      const result = await this.client.scan(this.reconciliationCursor, {
+        MATCH: 'mempool:tx:*',
+        COUNT: 1000
+      });
+      const mempool = memPool.getMempool();
+      let staleCount = 0;
+      this.reconciliationCursor = result.cursor.toString();
+
+      for (const key of result.keys) {
+        const txid = key.slice('mempool:tx:'.length);
+        if (!mempool[txid]) {
+          this.removeQueue.add(txid);
+          staleCount++;
+        }
+      }
+
+      if (staleCount) {
+        logger.debug(`Removing ${staleCount} stale transactions from the redis cache`);
+        void this.$removeTransactions();
+      }
+    } catch (e) {
+      logger.warn(`Failed to reconcile Redis mempool cache: ${e instanceof Error ? e.message : e}`);
+    } finally {
+      this.reconciliationInProgress = false;
+    }
+  }
+
+  /** @asyncSafe */
+  private async $flushQueuedMempoolTxRemovals(): Promise<void> {
+    if (!this.connected || !this.removeQueue.size || this.removeQueueFlushInProgress) {
+      return;
+    }
+
+    this.removeQueueFlushInProgress = true;
+    const toRemove = Array.from(this.removeQueue);
+    this.removeQueue.clear();
+    try {
       const sliceLength = config.REDIS.BATCH_QUERY_BASE_SIZE;
       for (let i = 0; i < Math.ceil(toRemove.length / sliceLength); i++) {
         const slice = toRemove.slice(i * sliceLength, (i + 1) * sliceLength);
         try {
           await this.client.unlink(slice.map(txid => `mempool:tx:${txid}`));
-          numRemoved+= sliceLength;
           logger.debug(`Deleted ${slice.length} transactions from the Redis cache`);
         } catch (e) {
           logger.warn(`Failed to remove ${slice.length} transactions from Redis cache: ${e instanceof Error ? e.message : e}`);
-          failed = failed.concat(slice);
+          this.queueTransactionsForRemoval(slice);
         }
       }
-      // concat instead of replace, in case more txs have been added in the meantime
-      this.removeQueue = this.removeQueue.concat(failed);
-    } else {
-      this.removeQueue = this.removeQueue.concat(toRemove);
+    } finally {
+      this.removeQueueFlushInProgress = false;
     }
   }
 
@@ -308,7 +369,7 @@ class RedisCache {
   }
 
   /** @asyncSafe */
-  async $getMempool(): Promise<{ [txid: string]: MempoolTransactionExtended }> {
+  async $getMempool(validTxids?: Set<string>): Promise<{ [txid: string]: MempoolTransactionExtended }> {
     if (!config.REDIS.ENABLED) {
       return {};
     }
@@ -319,7 +380,9 @@ class RedisCache {
     const start = Date.now();
     const mempool = {};
     try {
-      const mempoolList = await this.scanKeys<MempoolTransactionExtended>('mempool:tx:*');
+      const mempoolList = validTxids?.size
+        ? await this.loadKeys<MempoolTransactionExtended>('mempool:tx:*', Array.from(validTxids))
+        : await this.scanKeys<MempoolTransactionExtended>('mempool:tx:*');
       for (const tx of mempoolList) {
         mempool[tx.key] = tx.value;
       }
@@ -350,14 +413,14 @@ class RedisCache {
   }
 
   /** @asyncUnsafe */
-  async $loadCache(): Promise<void> {
+  async $loadCache(validTxids?: Set<string>): Promise<void> {
     if (!config.REDIS.ENABLED) {
       return;
     }
     logger.info('Restoring mempool and blocks data from Redis cache');
 
     // Load mempool
-    const loadedMempool = await this.$getMempool();
+    const loadedMempool = await this.$getMempool(validTxids);
     this.inflateLoadedTxs(loadedMempool);
     // Load rbf data
     const rbfTxs = await this.$getRbfEntries('tx');
@@ -432,6 +495,29 @@ class RedisCache {
     return result;
   }
 
+  /** @asyncUnsafe */
+  private async loadKeys<T>(pattern, keys: string[]): Promise<{ key: string, value: T }[]> {
+    const prefix = pattern.slice(0, -1);
+    const result: { key: string, value: T }[] = [];
+    let count = 0;
+    /** @asyncUnsafe */
+    const processValues = async (slice: string[]): Promise<void> => {
+      const values = await this.client.MGET(slice.map(key => `${prefix}${key}`));
+      for (let i = 0; i < values.length; i++) {
+        if (values[i]) {
+          result.push({ key: slice[i], value: JSON.parse(values[i]) });
+          count++;
+        }
+      }
+      logger.info(`loaded ${count} entries from Redis cache`);
+    };
+    for (let i = 0; i < Math.ceil(keys.length / 10000); i++) {
+      const slice = keys.slice(i * 10000, (i + 1) * 10000);
+      await processValues(slice);
+    }
+    return result;
+  }
+
   public setIgnoreBlocksCache(): void {
     this.ignoreBlocksCache = true;
   }

backend/src/api/services/wallets.ts

@@ -8,7 +8,6 @@ import { promises as fsPromises } from 'fs';
 
 interface WalletAddress {
   address: string;
-  active: boolean;
   stats: {
     funded_txo_count: number;
     funded_txo_sum: number;
@@ -244,15 +243,18 @@ class WalletApi {
 
   // resync address transactions from esplora
   async $syncWalletAddress(wallet: Wallet, address: WalletAddress): Promise<void> {
-    // fetch full transaction data if the address is new or still active and hasn't been synced in the last hour
-    const refreshTransactions = !wallet.addresses[address.address] || (address.active && (Date.now() - wallet.addresses[address.address].lastSync) > 60 * 60 * 1000);
+    if (address.address === 'private') {
+      // skip pseudo-address for private balances
+      return;
+    }
+    // fetch full transaction data if the address is new or hasn't been synced in the last hour
+    const refreshTransactions = !wallet.addresses[address.address] || (Date.now() - wallet.addresses[address.address].lastSync) > 60 * 60 * 1000;
     if (refreshTransactions) {
       try {
         const summary = await bitcoinApi.$getAddressTransactionSummary(address.address);
         const addressInfo = await bitcoinApi.$getAddress(address.address);
         const walletAddress: WalletAddress = {
           address: address.address,
-          active: address.active,
           transactions: summary,
           stats: addressInfo.chain_stats,
           lastSync: Date.now(),
@@ -322,7 +324,6 @@ function convertBalancesToWalletAddress(wallet: string, balances: { balance: num
   const sortedBalances = balances.sort((a, b) => a.time - b.time);
   const walletAddress: WalletAddress = {
     address: 'private',
-    active: false,
     stats: {
       funded_txo_count: 0,
       funded_txo_sum: sortedBalances[sortedBalances.length - 1].balance,

backend/src/api/transaction-utils.ts

@@ -256,13 +256,14 @@ class TransactionUtils {
 
   // returns the most significant 4 bytes of the txid as an integer
   public txidToOrdering(txid: string): number {
-    return parseInt(
-      txid.substr(62, 2) +
-        txid.substr(60, 2) +
-        txid.substr(58, 2) +
-        txid.substr(56, 2),
-      16
-    );
+    // Parse last 4 bytes of txid as little-endian uint32, without string allocation
+    let result = 0;
+    for (let i = 62; i >= 56; i -= 2) {
+      const hi = txid.charCodeAt(i);
+      const lo = txid.charCodeAt(i + 1);
+      result = result * 256 + (hi < 58 ? hi - 48 : hi - 87) * 16 + (lo < 58 ? lo - 48 : lo - 87);
+    }
+    return result;
   }
 
   public addInnerScriptsToVin(vin: IEsploraApi.Vin): void {

backend/src/api/websocket-handler.ts

@@ -3,7 +3,7 @@ import * as WebSocket from 'ws';
 import {
   BlockExtended, TransactionExtended, MempoolTransactionExtended, WebsocketResponse,
   OptimizedStatistic, ILoadingIndicators, GbtCandidates, TxTrackingInfo,
-  MempoolDelta, MempoolDeltaTxids
+  MempoolDelta, MempoolDeltaTxids, CpfpInfo
 } from '../mempool.interfaces';
 import blocks from './blocks';
 import memPool from './mempool';
@@ -16,16 +16,12 @@ import transactionUtils from './transaction-utils';
 import rbfCache, { ReplacementInfo } from './rbf-cache';
 import difficultyAdjustment from './difficulty-adjustment';
 import feeApi from './fee-api';
-import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
-import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
-import Audit from './audit';
 import priceUpdater from '../tasks/price-updater';
 import { ApiPrice } from '../repositories/PricesRepository';
 import { Acceleration } from './services/acceleration';
 import accelerationApi from './services/acceleration';
 import mempool from './mempool';
 import statistics from './statistics/statistics';
-import accelerationRepository from '../repositories/AccelerationRepository';
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import walletApi from './services/wallets';
 
@@ -62,6 +58,12 @@ class WebsocketHandler {
 
   private accelerations: Record<string, Acceleration> = {};
 
+  private MAX_BUFFERED_AMOUNT = 10_000_000;
+  public MAX_MESSAGE_SIZE = 50_000;
+  private MAX_TRACKED_TXS = 100;
+  private MSG_RATE_LIMIT = 100;
+  private MSG_RATE_WINDOW = 10_000;
+
   constructor() { }
 
   addWebsocketServer(wss: WebSocket.Server) {
@@ -118,6 +120,7 @@ class WebsocketHandler {
     server.on('connection', (client: WebSocket, req) => {
       this.numConnected++;
       client['remoteAddress'] = req.headers['x-forwarded-for'] || req.socket?.remoteAddress || 'unknown';
+      client['msgTimestamps'] = [];
       client.on('error', (e) => {
         logger.info(`websocket client error from ${client['remoteAddress']}: ` + (e instanceof Error ? e.message : e));
         client.close();
@@ -125,9 +128,30 @@ class WebsocketHandler {
       client.on('close', () => {
         this.numDisconnected++;
       });
-      client.on('message', async (message: string) => {
+      client.on('message', async (message) => {
         try {
-          const parsedMessage: WebsocketResponse = JSON.parse(message);
+          const msgLength = Buffer.isBuffer(message) ? message.byteLength
+            : message instanceof ArrayBuffer ? message.byteLength
+            : message.reduce((sum, buf) => sum + buf.byteLength, 0);
+          if (msgLength > this.MAX_MESSAGE_SIZE) {
+            logger.debug(`Dropping oversized websocket message from ${client['remoteAddress']}: ${msgLength} bytes`);
+            client.terminate();
+            return;
+          }
+
+          const now = Date.now();
+          const timestamps: number[] = client['msgTimestamps'];
+          timestamps.push(now);
+          while (timestamps.length && timestamps[0] <= now - this.MSG_RATE_WINDOW) {
+            timestamps.shift();
+          }
+          if (timestamps.length > this.MSG_RATE_LIMIT) {
+            logger.debug(`Rate limiting websocket client ${client['remoteAddress']}`);
+            client.close();
+            return;
+          }
+
+          const parsedMessage: WebsocketResponse = JSON.parse(message as any);
           const response = {};
 
           const wantNow = {};
@@ -223,45 +247,60 @@ class WebsocketHandler {
           if (parsedMessage && parsedMessage['track-txs']) {
             const txids: string[] = [];
             if (Array.isArray(parsedMessage['track-txs'])) {
+              if (parsedMessage['track-txs'].length > this.MAX_TRACKED_TXS) {
+                response['track-txs-error'] = `"too many txids requested, this connection supports tracking a maximum of ${this.MAX_TRACKED_TXS} transactions"`;
+                this.send(client, this.serializeResponse(response));
+                client['track-txs'] = null;
+                client.close();
+                return;
+              }
               for (const txid of parsedMessage['track-txs']) {
                 if (/^[a-fA-F0-9]{64}$/.test(txid)) {
                   txids.push(txid);
                 }
               }
+            } else {
+              response['track-txs-error'] = `"incorrect track-txs format"`;
+              this.send(client, this.serializeResponse(response));
+              client['track-txs'] = null;
+              client.close();
+              return;
             }
 
             const txs: { [txid: string]: TxTrackingInfo } = {};
             for (const txid of txids) {
-              const txInfo: TxTrackingInfo = {
-                confirmed: true,
-              };
+              const txInfo: TxTrackingInfo = {};
               const rbfCacheTxid = rbfCache.getReplacedBy(txid);
               if (rbfCacheTxid) {
                 txInfo.replacedBy = rbfCacheTxid;
                 txInfo.confirmed = false;
+                txs[txid] = txInfo;
               }
               const tx = memPool.getMempool()[txid];
-              if (tx && tx.position) {
-                txInfo.position = {
-                  ...tx.position
-                };
-                if (tx.acceleration) {
-                  txInfo.accelerated = tx.acceleration;
-                }
-              }
               if (tx) {
+                if (tx.position) {
+                  txInfo.position = {
+                    ...tx.position
+                  };
+                  if (tx.acceleration) {
+                    txInfo.accelerated = tx.acceleration;
+                  }
+                }
                 txInfo.confirmed = false;
+                txs[txid] = txInfo;
               }
-              txs[txid] = txInfo;
             }
 
             if (txids.length) {
               client['track-txs'] = txids;
+              client['track-txs-updates'] = 0;
             } else {
               client['track-txs'] = null;
+              client['track-txs-updates'] = 0;
             }
 
             if (Object.keys(txs).length) {
+              client['track-txs-updates'] = (client['track-txs-updates'] || 0) + Object.keys(txs).length;
               response['tracked-txs'] = JSON.stringify(txs);
             }
           }
@@ -286,10 +325,13 @@ class WebsocketHandler {
             if (Object.keys(addressMap).length > config.MEMPOOL.MAX_TRACKED_ADDRESSES) {
               response['track-addresses-error'] = `"too many addresses requested, this connection supports tracking a maximum of ${config.MEMPOOL.MAX_TRACKED_ADDRESSES} addresses"`;
               client['track-addresses'] = null;
+              client['track-addresses-updates'] = 0;
             } else if (Object.keys(addressMap).length > 0) {
               client['track-addresses'] = addressMap;
+              client['track-addresses-updates'] = 0;
             } else {
               client['track-addresses'] = null;
+              client['track-addresses-updates'] = 0;
             }
           }
 
@@ -313,8 +355,10 @@ class WebsocketHandler {
           if (parsedMessage && parsedMessage['track-wallet']) {
             if (parsedMessage['track-wallet'] === 'stop') {
               client['track-wallet'] = null;
-            } else {
+            } else if (typeof parsedMessage['track-wallet'] === 'string' && walletApi.getWallets().includes(parsedMessage['track-wallet'])) {
               client['track-wallet'] = parsedMessage['track-wallet'];
+            } else {
+              client['track-wallet'] = null;
             }
           }
 
@@ -327,8 +371,8 @@ class WebsocketHandler {
           }
 
           if (parsedMessage && parsedMessage['track-mempool-block'] !== undefined) {
-            if (Number.isInteger(parsedMessage['track-mempool-block']) && parsedMessage['track-mempool-block'] >= 0) {
-              const index = parsedMessage['track-mempool-block'];
+            const index = parsedMessage['track-mempool-block'];
+            if (Number.isInteger(index) && index >= 0 && index < config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT) {
               client['track-mempool-block'] = index;
               const mBlocksWithTransactions = mempoolBlocks.getMempoolBlocksWithTransactions();
               response['projected-block-transactions'] = JSON.stringify({
@@ -379,14 +423,14 @@ class WebsocketHandler {
             if (!this.socketData['blocks']?.length) {
               return;
             }
-            client.send(this.serializedInitData);
+            this.send(client, this.serializedInitData);
           }
 
           if (parsedMessage.action === 'ping') {
             response['pong'] = JSON.stringify(true);
           }
 
-          if (parsedMessage['track-donation'] && parsedMessage['track-donation'].length === 22) {
+          if (typeof parsedMessage['track-donation'] === 'string' && parsedMessage['track-donation'].length === 22) {
             client['track-donation'] = parsedMessage['track-donation'];
           }
 
@@ -402,18 +446,18 @@ class WebsocketHandler {
             delete client['track-mempool'];
           }
 
-          if (parsedMessage && parsedMessage['track-stratum'] != null) {
-            if (parsedMessage['track-stratum']) {
+          if (parsedMessage && parsedMessage['track-stratum'] !== undefined) {
+            if (parsedMessage['track-stratum'] === 'all' || typeof parsedMessage['track-stratum'] === 'number') {
               const sub = parsedMessage['track-stratum'];
               client['track-stratum'] = sub;
-              response['stratumJobs'] = this.socketData['stratumJobs'];
+              response['stratumJobs'] = JSON.stringify(stratumApi.getJobs());
             } else {
               client['track-stratum'] = false;
             }
           }
 
           if (Object.keys(response).length) {
-            client.send(this.serializeResponse(response));
+            this.send(client, this.serializeResponse(response));
           }
         } catch (e) {
           logger.debug(`Error parsing websocket message from ${client['remoteAddress']}: ` + (e instanceof Error ? e.message : e));
@@ -436,7 +480,7 @@ class WebsocketHandler {
         return;
       }
       if (client['track-donation'] === id) {
-        client.send(JSON.stringify({ donationConfirmed: true }));
+        this.send(client, JSON.stringify({ donationConfirmed: true }));
       }
     });
     }
@@ -456,7 +500,7 @@ class WebsocketHandler {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
-      client.send(response);
+      this.send(client, response);
     });
     }
   }
@@ -475,7 +519,7 @@ class WebsocketHandler {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
-      client.send(response);
+      this.send(client, response);
     });
     }
   }
@@ -502,7 +546,7 @@ class WebsocketHandler {
         return;
       }
 
-      client.send(response);
+      this.send(client, response);
     });
     }
   }
@@ -535,7 +579,7 @@ class WebsocketHandler {
           if (client.readyState !== WebSocket.OPEN) {
             return;
           }
-          client.send(response);
+          this.send(client, response);
         });
       }
     } catch (e) {
@@ -573,7 +617,7 @@ class WebsocketHandler {
       }
 
       if (Object.keys(response).length) {
-        client.send(this.serializeResponse(response));
+        this.send(client, this.serializeResponse(response));
       }
     });
     }
@@ -607,7 +651,10 @@ class WebsocketHandler {
       removed = candidates?.removed || [];
     }
 
-    if (config.MEMPOOL.RUST_GBT) {
+    if (config.MEMPOOL.CLUSTER_MEMPOOL) {
+      const cmBlocks = mempool.clusterMempool?.getBlocks(config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT) ?? [];
+      mempoolBlocks.processClusterMempoolBlocks(cmBlocks, newMempool, mempool.getAccelerations());
+    } else if (config.MEMPOOL.RUST_GBT) {
       await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, newMempool, added, removed, candidates, true);
     } else {
       await mempoolBlocks.$updateBlockTemplates(transactionIds, newMempool, added, removed, candidates, accelerationDelta, true, true);
@@ -736,6 +783,8 @@ class WebsocketHandler {
       removed: websocketAccelerationDelta.filter(txid => !accelerations[txid]),
     };
 
+    const cpfpUpdatesSent = new Set<string>();
+
     // TODO - Fix indentation after PR is merged
     for (const server of this.webSocketServers) {
     server.clients.forEach(async (client) => {
@@ -813,6 +862,8 @@ class WebsocketHandler {
         }
 
         if (Object.keys(addressMap).length > 0) {
+          client['track-addresses-updates'] =
+            (client['track-addresses-updates'] || 0) + this.countAddressTransactions(addressMap);
           response['multi-address-transactions'] = JSON.stringify(addressMap);
         }
       }
@@ -904,15 +955,23 @@ class WebsocketHandler {
             calculateMempoolTxCpfp(mempoolTx, newMempool);
           }
           if (mempoolTx.cpfpDirty) {
-            positionData['cpfp'] = {
-              ancestors: mempoolTx.ancestors,
+            const cpfp: CpfpInfo = {
+              ancestors: mempoolTx.ancestors || [],
               bestDescendant: mempoolTx.bestDescendant || null,
-              descendants: mempoolTx.descendants || null,
-              effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize || null,
+              descendants: mempoolTx.descendants,
+              effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize,
               sigops: mempoolTx.sigops,
               adjustedVsize: mempoolTx.adjustedVsize,
               acceleration: mempoolTx.acceleration,
             };
+            if (config.MEMPOOL.CLUSTER_MEMPOOL && mempoolTx.clusterId != null) {
+              const cluster = mempool.clusterMempool?.getClusterForApi(mempoolTx.txid);
+              if (cluster) {
+                cpfp.cluster = cluster;
+              }
+            }
+            positionData['cpfp'] = cpfp;
+            cpfpUpdatesSent.add(trackTxid);
           }
           response['txPosition'] = JSON.stringify(positionData);
         }
@@ -923,13 +982,16 @@ class WebsocketHandler {
         const txs: { [txid: string]: TxTrackingInfo } = {};
         for (const txid of txids) {
           const txInfo: TxTrackingInfo = {};
+          let txHasInfo = false;
           const outspends = outspendCache[txid];
           if (outspends && Object.keys(outspends).length) {
             txInfo.utxoSpent = outspends;
+            txHasInfo = true;
           }
           const replacedBy = rbfChanges.map[txid] ? rbfCache.getReplacedBy(txid) : false;
           if (replacedBy) {
             txInfo.replacedBy = replacedBy;
+            txHasInfo = true;
           }
           const mempoolTx = newMempool[txid];
           if (mempoolTx && mempoolTx.position) {
@@ -947,16 +1009,27 @@ class WebsocketHandler {
               txInfo.cpfp = {
                 ancestors: mempoolTx.ancestors,
                 bestDescendant: mempoolTx.bestDescendant || null,
-                descendants: mempoolTx.descendants || null,
-                effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize || null,
+                descendants: mempoolTx.descendants,
+                effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize,
                 sigops: mempoolTx.sigops,
                 adjustedVsize: mempoolTx.adjustedVsize,
               };
+              if (config.MEMPOOL.CLUSTER_MEMPOOL && mempoolTx.clusterId != null) {
+                const cluster = mempool.clusterMempool?.getClusterForApi(mempoolTx.txid);
+                if (cluster) {
+                  (txInfo.cpfp as CpfpInfo).cluster = cluster;
+                }
+              }
+              cpfpUpdatesSent.add(txid);
             }
+            txHasInfo = true;
+          }
+          if (txHasInfo) {
+            txs[txid] = txInfo;
           }
-          txs[txid] = txInfo;
         }
         if (Object.keys(txs).length) {
+          client['track-txs-updates'] = (client['track-txs-updates'] || 0) + Object.keys(txs).length;
           response['tracked-txs'] = JSON.stringify(txs);
         }
       }
@@ -995,133 +1068,37 @@ class WebsocketHandler {
       }
 
       if (Object.keys(response).length) {
-        client.send(this.serializeResponse(response));
+        this.send(client, this.serializeResponse(response));
       }
     });
     }
+
+    for (const txid of cpfpUpdatesSent) {
+      if (newMempool[txid]) {
+        newMempool[txid].cpfpDirty = false;
+      }
+    }
   }
 
-  /** @asyncUnsafe */
-  async handleNewBlock(block: BlockExtended, txIds: string[], transactions: MempoolTransactionExtended[]): Promise<void> {
+  /** @asyncSafe */
+  async handleNewBlock(
+    block: BlockExtended,
+    txIds: string[],
+    transactions: MempoolTransactionExtended[],
+    rbfTransactions: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }}
+  ): Promise<void> {
     if (!this.webSocketServers.length) {
       throw new Error('No WebSocket.Server have been set');
     }
 
-    const blockTransactions = structuredClone(transactions);
-
     this.printLogs();
-    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
-      await statistics.runStatistics();
-    }
 
     const _memPool = memPool.getMempool();
-    const candidateTxs = memPool.getMempoolCandidates();
-    let candidates: GbtCandidates | undefined = (memPool.limitGBT && candidateTxs) ? { txs: candidateTxs, added: [], removed: [] } : undefined;
-    let transactionIds: string[] = (memPool.limitGBT) ? Object.keys(candidates?.txs || {}) : Object.keys(_memPool);
-
-    if (config.DATABASE.ENABLED) {
-      const accelerations = Object.values(mempool.getAccelerations());
-      await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, structuredClone(transactions));
-    }
-
-    const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
-    memPool.handleRbfTransactions(rbfTransactions);
-    memPool.removeFromSpendMap(transactions);
-
-    if (config.MEMPOOL.AUDIT && memPool.isInSync()) {
-      let projectedBlocks;
-      const auditMempool = _memPool;
-      const isAccelerated = accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations()));
-
-      if (config.MEMPOOL.RUST_GBT) {
-        const added = memPool.limitGBT ? (candidates?.added || []) : [];
-        const removed = memPool.limitGBT ? (candidates?.removed || []) : [];
-        projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, auditMempool, added, removed, candidates, isAccelerated, block.extras.pool.id);
-      } else {
-        projectedBlocks = await mempoolBlocks.$makeBlockTemplates(transactionIds, auditMempool, candidates, false, isAccelerated, block.extras.pool.id);
-      }
-
-      if (Common.indexingEnabled()) {
-        const { unseen, censored, added, prioritized, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(block.height, blockTransactions, projectedBlocks, auditMempool);
-        const matchRate = Math.round(score * 100 * 100) / 100;
-
-        const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
-
-        let totalFees = 0;
-        let totalWeight = 0;
-        for (const tx of stripped) {
-          totalFees += tx.fee;
-          totalWeight += (tx.vsize * 4);
-        }
-
-        void BlocksSummariesRepository.$saveTemplate({
-          height: block.height,
-          template: {
-            id: block.id,
-            transactions: stripped,
-          },
-          version: 1,
-        });
-
-        void BlocksAuditsRepository.$saveAudit({
-          version: 1,
-          time: block.timestamp,
-          height: block.height,
-          hash: block.id,
-          unseenTxs: unseen,
-          addedTxs: added,
-          prioritizedTxs: prioritized,
-          missingTxs: censored,
-          freshTxs: fresh,
-          sigopTxs: sigop,
-          fullrbfTxs: fullrbf,
-          acceleratedTxs: accelerated,
-          matchRate: matchRate,
-          expectedFees: totalFees,
-          expectedWeight: totalWeight,
-        });
-
-        if (block.extras) {
-          block.extras.matchRate = matchRate;
-          block.extras.expectedFees = totalFees;
-          block.extras.expectedWeight = totalWeight;
-          block.extras.similarity = similarity;
-        }
-      }
-    } else if (block.extras) {
-      const mBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
-      if (mBlocks?.length && mBlocks[0].transactions) {
-        block.extras.similarity = Common.getSimilarity(mBlocks[0], transactions);
-      }
-    }
-
     const confirmedTxids: { [txid: string]: boolean } = {};
-
-    // Update mempool to remove transactions included in the new block
     for (const txId of txIds) {
-      delete _memPool[txId];
-      rbfCache.mined(txId);
       confirmedTxids[txId] = true;
     }
 
-    if (memPool.limitGBT) {
-      const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null;
-      const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1;
-      candidates = memPool.getNextCandidates(minFeeMempool, minFeeTip, transactions);
-      transactionIds = Object.keys(candidates?.txs || {});
-    } else {
-      candidates = undefined;
-      transactionIds = Object.keys(memPool.getMempool());
-    }
-
-
-    if (config.MEMPOOL.RUST_GBT) {
-      const added = memPool.limitGBT ? (candidates?.added || []) : [];
-      const removed = memPool.limitGBT ? (candidates?.removed || []) : transactions;
-      await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, _memPool, added, removed, candidates, true);
-    } else {
-      await mempoolBlocks.$makeBlockTemplates(transactionIds, _memPool, candidates, true, true);
-    }
     const mBlocks = mempoolBlocks.getMempoolBlocks();
     const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
 
@@ -1254,6 +1231,7 @@ class WebsocketHandler {
           }
         }
         if (Object.keys(txs).length) {
+          client['track-txs-updates'] = (client['track-txs-updates'] || 0) + Object.keys(txs).length;
           response['tracked-txs'] = JSON.stringify(txs);
         }
       }
@@ -1289,6 +1267,8 @@ class WebsocketHandler {
         }
 
         if (Object.keys(addressMap).length > 0) {
+          client['track-addresses-updates'] =
+            (client['track-addresses-updates'] || 0) + this.countAddressTransactions(addressMap);
           response['multi-address-transactions'] = JSON.stringify(addressMap);
         }
       }
@@ -1382,26 +1362,20 @@ class WebsocketHandler {
       }
 
       if (Object.keys(response).length) {
-        client.send(this.serializeResponse(response));
+        this.send(client, this.serializeResponse(response));
       }
     });
     }
-
-    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
-      await statistics.runStatistics();
-    }
   }
 
   public handleNewStratumJob(job: StratumJob): void {
-    this.updateSocketDataFields({ 'stratumJobs': stratumApi.getJobs() });
-
     for (const server of this.webSocketServers) {
       server.clients.forEach((client) => {
         if (client.readyState !== WebSocket.OPEN) {
           return;
         }
         if (client['track-stratum'] && (client['track-stratum'] === 'all' || client['track-stratum'] === job.pool)) {
-          client.send(JSON.stringify({
+          this.send(client, JSON.stringify({
             'stratumJob': job
         }));
         }
@@ -1409,6 +1383,14 @@ class WebsocketHandler {
     }
   }
 
+  private send(client: WebSocket.WebSocket, data: string): void {
+    if (client.bufferedAmount > this.MAX_BUFFERED_AMOUNT) {
+      client.terminate();
+      return;
+    }
+    client.send(data);
+  }
+
   // takes a dictionary of JSON serialized values
   // and zips it together into a valid JSON object
   private serializeResponse(response): string {
@@ -1490,25 +1472,72 @@ class WebsocketHandler {
     if (this.webSocketServers.length) {
       let numTxSubs = 0;
       let numTxsSubs = 0;
+      let numAddressSubs = 0;
+      let numAddressesSubs = 0;
       let numProjectedSubs = 0;
       let numRbfSubs = 0;
+      let trackedTxsTotal = 0;
+      let trackedAddressesTotal = 0;
+      let trackedTxsMax = 0;
+      let trackedAddressesMax = 0;
+      let trackTxsTrackedTotal = 0;
+      let trackTxsTrackedMax = 0;
+      let trackAddressesTrackedTotal = 0;
+      let trackAddressesTrackedMax = 0;
+      let trackTxsUpdatesTotal = 0;
+      let trackTxsUpdatesMax = 0;
+      let trackAddressesUpdatesTotal = 0;
+      let trackAddressesUpdatesMax = 0;
 
-      // TODO - Fix indentation after PR is merged
       for (const server of this.webSocketServers) {
-      server.clients.forEach((client) => {
-        if (client['track-tx']) {
-          numTxSubs++;
-        }
-        if (client['track-txs']) {
-          numTxsSubs++;
-        }
-        if (client['track-mempool-block'] != null && client['track-mempool-block'] >= 0) {
-          numProjectedSubs++;
-        }
-        if (client['track-rbf']) {
-          numRbfSubs++;
-        }
-      });
+        server.clients.forEach((client) => {
+          let trackedTxCount = 0;
+          let trackedAddressCount = 0;
+
+          if (client['track-tx']) {
+            numTxSubs++;
+            trackedTxCount += 1;
+          }
+          if (client['track-txs']) {
+            numTxsSubs++;
+            trackedTxCount += client['track-txs'].length;
+          }
+          if (client['track-address']) {
+            numAddressSubs++;
+            trackedAddressCount += 1;
+          }
+          if (client['track-addresses']) {
+            numAddressesSubs++;
+            const addressCount = Object.keys(client['track-addresses']).length;
+            trackedAddressCount += addressCount;
+            trackAddressesTrackedTotal += addressCount;
+            trackAddressesTrackedMax = Math.max(trackAddressesTrackedMax, addressCount);
+            const updates = client['track-addresses-updates'] || 0;
+            trackAddressesUpdatesTotal += updates;
+            trackAddressesUpdatesMax = Math.max(trackAddressesUpdatesMax, updates);
+            client['track-addresses-updates'] = 0;
+          }
+          if (client['track-mempool-block'] != null && client['track-mempool-block'] >= 0) {
+            numProjectedSubs++;
+          }
+          if (client['track-rbf']) {
+            numRbfSubs++;
+          }
+          if (client['track-txs']) {
+            const txCount = client['track-txs'].length;
+            trackTxsTrackedTotal += txCount;
+            trackTxsTrackedMax = Math.max(trackTxsTrackedMax, txCount);
+            const updates = client['track-txs-updates'] || 0;
+            trackTxsUpdatesTotal += updates;
+            trackTxsUpdatesMax = Math.max(trackTxsUpdatesMax, updates);
+            client['track-txs-updates'] = 0;
+          }
+
+          trackedTxsTotal += trackedTxCount;
+          trackedAddressesTotal += trackedAddressCount;
+          trackedTxsMax = Math.max(trackedTxsMax, trackedTxCount);
+          trackedAddressesMax = Math.max(trackedAddressesMax, trackedAddressCount);
+        });
       }
 
       let count = 0;
@@ -1517,12 +1546,33 @@ class WebsocketHandler {
       }
       const diff = count - this.numClients;
       this.numClients = count;
-      logger.debug(`${count} websocket clients | ${this.numConnected} connected | ${this.numDisconnected} disconnected | (${diff >= 0 ? '+' : ''}${diff})`);
-      logger.debug(`websocket subscriptions: track-tx: ${numTxSubs}, track-txs: ${numTxsSubs}, track-mempool-block: ${numProjectedSubs} track-rbf: ${numRbfSubs}`);
+      const trackedTxsAvg = count > 0 ? trackedTxsTotal / count : 0;
+      const trackedAddressesAvg = count > 0 ? trackedAddressesTotal / count : 0;
+      const trackTxsTrackedAvg = numTxsSubs > 0 ? trackTxsTrackedTotal / numTxsSubs : 0;
+      const trackAddressesTrackedAvg =
+        numAddressesSubs > 0 ? trackAddressesTrackedTotal / numAddressesSubs : 0;
+      const trackTxsUpdatesAvg = numTxsSubs > 0 ? trackTxsUpdatesTotal / numTxsSubs : 0;
+      const trackAddressesUpdatesAvg =
+        numAddressesSubs > 0 ? trackAddressesUpdatesTotal / numAddressesSubs : 0;
+      logger.debug(
+        `${count} websocket clients | ${this.numConnected} connected | ${this.numDisconnected} disconnected | (${diff >= 0 ? '+' : ''}${diff}) | tracked txs: total=${trackedTxsTotal}, avg=${trackedTxsAvg.toFixed(2)}, max=${trackedTxsMax} | tracked addresses: total=${trackedAddressesTotal}, avg=${trackedAddressesAvg.toFixed(2)}, max=${trackedAddressesMax} | ws-subscriptions: tx=${numTxSubs},txs=${numTxsSubs},address=${numAddressSubs},addresses=${numAddressesSubs},txs-tracked-avg=${trackTxsTrackedAvg.toFixed(2)},txs-tracked-max=${trackTxsTrackedMax},addresses-tracked-avg=${trackAddressesTrackedAvg.toFixed(2)},addresses-tracked-max=${trackAddressesTrackedMax},txs-updates-avg=${trackTxsUpdatesAvg.toFixed(2)},txs-updates-max=${trackTxsUpdatesMax},addresses-updates-avg=${trackAddressesUpdatesAvg.toFixed(2)},addresses-updates-max=${trackAddressesUpdatesMax}`
+      );
+      logger.debug(`websocket subscriptions: track-tx: ${numTxSubs}, track-txs: ${numTxsSubs}, track-address: ${numAddressSubs}, track-addresses: ${numAddressesSubs}, track-mempool-block: ${numProjectedSubs} track-rbf: ${numRbfSubs}`);
       this.numConnected = 0;
       this.numDisconnected = 0;
     }
   }
+
+  private countAddressTransactions(addressMap: { [address: string]: AddressTransactions }): number {
+    return Object.values(addressMap).reduce(
+      (total, transactions) =>
+        total
+        + transactions.mempool.length
+        + transactions.confirmed.length
+        + transactions.removed.length,
+      0,
+    );
+  }
 }
 
 export default new WebsocketHandler();

backend/src/cluster-mempool/block-builder.ts

@@ -0,0 +1,165 @@
+import { PairingHeap } from '../utils/pairing-heap';
+import { LinearizationChunk } from './linearize';
+import { MempoolTransactionExtended } from '../mempool.interfaces';
+import { Cluster } from './cluster-mempool';
+
+export interface ProjectedBlock {
+  txids: string[];
+  weight: number;
+  sigops: number;
+}
+
+interface ChunkHeapEntry {
+  fee: number;
+  weight: number;
+  sigops: number;
+  equalFeeratePrefixWeight: number;
+  maxOrder: number;
+  clusterId: number;
+  chunkIndex: number;
+}
+
+const BLOCK_WEIGHT_UNITS = 4_000_000;
+const MAX_BLOCK_SIGOPS_COST = 80_000;
+const COINBASE_RESERVED_WEIGHT = 8000;
+const MAX_CONSECUTIVE_FAILURES = 1000;
+const MAX_WASTED_WEIGHT = 4000;
+
+function chunkHeapHigherPriority(a: ChunkHeapEntry, b: ChunkHeapEntry): boolean {
+  const feerateDiff = a.fee * b.weight - b.fee * a.weight;
+  if (feerateDiff !== 0) {
+    return feerateDiff > 0;
+  }
+  if (a.equalFeeratePrefixWeight !== b.equalFeeratePrefixWeight) {
+    return a.equalFeeratePrefixWeight < b.equalFeeratePrefixWeight;
+  }
+  return a.maxOrder < b.maxOrder;
+}
+
+function equalFeerate(a: LinearizationChunk, b: LinearizationChunk): boolean {
+  return a.fee * b.weight === b.fee * a.weight;
+}
+
+function makeChunkHeapEntry(cluster: Cluster, chunkIndex: number, mempool: { [txid: string]: MempoolTransactionExtended }): ChunkHeapEntry {
+  const chunk = cluster.chunks[chunkIndex];
+  let maxOrder = 0;
+  let sigops = 0;
+  for (const tx of chunk.txs) {
+    if (tx.order > maxOrder) {
+      maxOrder = tx.order;
+    }
+    const mempoolTx = mempool[tx.txid];
+    if (mempoolTx) {
+      sigops += mempoolTx.sigops || 0;
+    }
+  }
+  let prefixWeight = chunk.weight;
+  for (let i = chunkIndex - 1; i >= 0; i--) {
+    if (equalFeerate(cluster.chunks[i], chunk)) {
+      prefixWeight += cluster.chunks[i].weight;
+    } else {
+      break;
+    }
+  }
+  return {
+    fee: chunk.fee,
+    weight: chunk.weight,
+    sigops,
+    equalFeeratePrefixWeight: prefixWeight,
+    maxOrder,
+    clusterId: cluster.id,
+    chunkIndex,
+  };
+}
+
+function buildChunkHeap(clusters: Map<number, Cluster>, mempool: { [txid: string]: MempoolTransactionExtended }): PairingHeap<ChunkHeapEntry> {
+  const heap = new PairingHeap<ChunkHeapEntry>(chunkHeapHigherPriority);
+  for (const cluster of clusters.values()) {
+    if (cluster.chunks.length > 0) {
+      heap.add(makeChunkHeapEntry(cluster, 0, mempool));
+    }
+  }
+  return heap;
+}
+
+export function assembleBlocks(
+  n: number,
+  clusters: Map<number, Cluster>,
+  mempool: { [txid: string]: MempoolTransactionExtended },
+  enforceLimit: boolean,
+): ProjectedBlock[] {
+  const heap = buildChunkHeap(clusters, mempool);
+  const blocks: ProjectedBlock[] = [];
+  for (let blockIdx = 0; blockIdx < n; blockIdx++) {
+    const limited = enforceLimit || blockIdx < n - 1;
+    const maxWeight = limited ? BLOCK_WEIGHT_UNITS : Infinity;
+    const maxSigops = limited ? MAX_BLOCK_SIGOPS_COST : Infinity;
+    const block = fillBlock(heap, clusters, mempool, maxWeight, maxSigops);
+    if (block.txids.length === 0) {
+      break;
+    }
+    blocks.push(block);
+  }
+  return blocks;
+}
+
+function fillBlock(
+  heap: PairingHeap<ChunkHeapEntry>,
+  clusters: Map<number, Cluster>,
+  mempool: { [txid: string]: MempoolTransactionExtended },
+  maxWeight: number,
+  maxSigops: number,
+): ProjectedBlock {
+  const block: ProjectedBlock = { txids: [], weight: COINBASE_RESERVED_WEIGHT, sigops: 0 };
+  const deferred: ChunkHeapEntry[] = [];
+  let consecutiveFailed = 0;
+  let full = false;
+
+  while (!heap.isEmpty() && !full) {
+    const entry = heap.pop() as ChunkHeapEntry;
+    const cluster = clusters.get(entry.clusterId);
+    const chunk = cluster?.chunks[entry.chunkIndex];
+
+    if (!cluster || !chunk) {
+      // stale entry
+    } else if (block.weight + entry.weight < maxWeight
+      && block.sigops + entry.sigops < maxSigops) {
+      consecutiveFailed = 0;
+      block.weight += chunkWeight(chunk, mempool);
+      block.sigops += entry.sigops;
+      for (const tx of chunk.txs) {
+        block.txids.push(tx.txid);
+      }
+      if (entry.chunkIndex + 1 < cluster.chunks.length) {
+        heap.add(makeChunkHeapEntry(cluster, entry.chunkIndex + 1, mempool));
+      }
+    } else {
+      deferred.push(entry);
+      consecutiveFailed++;
+      if (consecutiveFailed > MAX_CONSECUTIVE_FAILURES
+        && block.weight + MAX_WASTED_WEIGHT > maxWeight) {
+        full = true;
+      }
+    }
+  }
+
+  for (const entry of deferred) {
+    heap.add(entry);
+  }
+
+  return block;
+}
+
+function chunkWeight(
+  chunk: LinearizationChunk,
+  mempool: { [txid: string]: MempoolTransactionExtended },
+): number {
+  let weight = 0;
+  for (const clusterTx of chunk.txs) {
+    const mempoolTx = mempool[clusterTx.txid];
+    if (mempoolTx) {
+      weight += mempoolTx.weight;
+    }
+  }
+  return weight;
+}

backend/src/cluster-mempool/cluster-mempool.ts

@@ -0,0 +1,709 @@
+import { ClusterTx, DepGraph, sortTopological, subgraph } from './depgraph';
+import { linearizeCluster, LinearizationChunk } from './linearize';
+import { ProjectedBlock, assembleBlocks } from './block-builder';
+import { Ancestor, CpfpClusterData, CpfpClusterTx, MempoolTransactionExtended } from '../mempool.interfaces';
+import logger from '../logger';
+
+export interface MempoolDiff {
+  added: MempoolTransactionExtended[];
+  removed: MempoolTransactionExtended[];
+  accelerations: { [txid: string]: { feeDelta: number } };
+}
+
+export interface ClusterInfo {
+  clusterId: number;
+  chunkIndex: number;
+  chunkFeerate: number;
+}
+
+export { ProjectedBlock };
+
+export interface Cluster {
+  id: number;
+  depgraph: DepGraph;
+  txs: Map<string, ClusterTx>;
+  linearization: ClusterTx[];
+  chunks: LinearizationChunk[];
+  dirty: boolean;
+}
+
+export interface TxCpfpData {
+  effectiveFeePerVsize: number;
+  clusterId: number;
+  chunkIndex: number;
+  cpfpDirty: boolean;
+  cpfpChecked: boolean;
+  ancestors: Ancestor[];
+  descendants: Ancestor[];
+}
+
+const DEFAULT_COST_BUDGET = 75000;
+
+export class ClusterMempool {
+  private clusters = new Map<number, Cluster>();
+  private txToCluster = new Map<string, number>();
+  private spentBy = new Map<string, string>();
+  private mempool: Readonly<{ [txid: string]: MempoolTransactionExtended }>;
+  private accelerations: { [txid: string]: { feeDelta: number } } = {};
+  private nextClusterId = 0;
+  private modifyTxs: boolean;
+  private costBudget: number = DEFAULT_COST_BUDGET;
+
+  constructor(mempool: { [txid: string]: MempoolTransactionExtended }, accelerations?: { [txid: string]: { feeDelta: number } }, modifyTxs: boolean = true, costBudget: number = DEFAULT_COST_BUDGET) {
+    this.mempool = mempool;
+    if (accelerations) {
+      this.accelerations = accelerations;
+    }
+    this.modifyTxs = modifyTxs;
+    this.costBudget = costBudget;
+    this.buildFromMempool();
+  }
+
+  applyMempoolChange(diff: MempoolDiff): void {
+    this.processRemovals(diff.removed);
+    this.splitDisconnectedClusters();
+    this.processAccelerationChanges(diff.accelerations);
+    this.processAdditions(diff.added);
+    this.relinearizeDirtyClusters();
+  }
+
+  getBlocks(n: number, enforceLimit = false): ProjectedBlock[] {
+    return assembleBlocks(n, this.clusters, this.mempool, enforceLimit);
+  }
+
+  getCluster(clusterId: number): CpfpClusterData | null {
+    const cluster = this.clusters.get(clusterId);
+    if (!cluster) {
+      return null;
+    }
+    return this.buildClusterData(cluster);
+  }
+
+  getClusterInfo(txid: string): ClusterInfo | null {
+    const match = this.getClusterForTx(txid);
+    if (!match) {
+      return null;
+    }
+    return this.findChunkInfo(match.cluster, match.clusterTx);
+  }
+
+  getClusterForApi(txid: string): (CpfpClusterData & { chunkIndex: number }) | null {
+    const info = this.getClusterInfo(txid);
+    if (!info) {
+      return null;
+    }
+    const cluster = this.getCluster(info.clusterId);
+    if (!cluster || cluster.txs.length <= 1) {
+      return null;
+    }
+    return { ...cluster, chunkIndex: info.chunkIndex };
+  }
+
+  getCpfpDataForTx(txid: string): TxCpfpData | null {
+    const clusterInfo = this.getClusterForTx(txid);
+    if (!clusterInfo) {
+      return null;
+    }
+    const chunkInfo = this.findChunkInfo(clusterInfo.cluster, clusterInfo.clusterTx);
+    if (!chunkInfo) {
+      return null;
+    }
+    const chunk = clusterInfo.cluster.chunks[chunkInfo?.chunkIndex];
+    const chunkSet = chunk.txs.length > 1 ? new Set(chunk.txs) : null;
+    return {
+      effectiveFeePerVsize: chunkInfo.chunkFeerate,
+      clusterId: clusterInfo.cluster.id,
+      chunkIndex: chunkInfo.chunkIndex,
+      cpfpDirty: true,
+      cpfpChecked: true,
+      ancestors: chunkSet ? this.getChunkRelatives(clusterInfo.clusterTx, chunkSet, 'ancestors') : [],
+      descendants: chunkSet ? this.getChunkRelatives(clusterInfo.clusterTx, chunkSet, 'descendants') : [],
+    };
+  }
+
+  getClusterCount(): number {
+    return this.clusters.size;
+  }
+
+  getTxCount(): number {
+    return this.txToCluster.size;
+  }
+
+  private getClusterForTx(txid: string): { cluster: Cluster; clusterTx: ClusterTx } | null {
+    const clusterId = this.txToCluster.get(txid);
+    if (clusterId === undefined) {
+      return null;
+    }
+    const cluster = this.clusters.get(clusterId);
+    if (!cluster) {
+      return null;
+    }
+    const clusterTx = cluster.txs.get(txid);
+    if (!clusterTx) {
+      return null;
+    }
+    return { cluster, clusterTx };
+  }
+
+  private buildFromMempool(): void {
+    const parentMap = this.buildRelativeMaps();
+    const components = this.findMempoolComponents(parentMap);
+    for (const component of components) {
+      this.createClusterFromTxids(component, parentMap);
+    }
+  }
+
+  private buildRelativeMaps(): Map<string, Set<string>> {
+    const parentMap = new Map<string, Set<string>>();
+    this.spentBy.clear();
+    for (const txid in this.mempool) {
+      const tx = this.mempool[txid];
+      const txParents = new Set<string>();
+      for (const vin of tx.vin) {
+        if (!vin.is_coinbase && this.mempool[vin.txid]) {
+          txParents.add(vin.txid);
+          this.spentBy.set(`${vin.txid}:${vin.vout}`, txid);
+        }
+      }
+      if (txParents.size > 0) {
+        parentMap.set(txid, txParents);
+      }
+    }
+    return parentMap;
+  }
+
+  private findMempoolComponents(parentMap: Map<string, Set<string>>): Set<string>[] {
+    const visited = new Set<string>();
+    const components: Set<string>[] = [];
+
+    for (const txid in this.mempool) {
+      if (!visited.has(txid)) {
+        const component = this.dfsComponent(txid, visited, parentMap);
+        components.push(component);
+      }
+    }
+    return components;
+  }
+
+  private dfsComponent(
+    startTxid: string,
+    visited: Set<string>,
+    parentMap: Map<string, Set<string>>
+  ): Set<string> {
+    const component = new Set<string>();
+    const stack = [startTxid];
+    while (stack.length > 0) {
+      const current = stack.pop();
+      if (current !== undefined && !visited.has(current)) {
+        visited.add(current);
+        component.add(current);
+
+        const txParents = parentMap.get(current);
+        if (txParents) {
+          for (const p of txParents) {
+            if (!visited.has(p)) {
+              stack.push(p);
+            }
+          }
+        }
+
+        const tx = this.mempool[current];
+        if (tx) {
+          for (let vout = 0; vout < tx.vout.length; vout++) {
+            const child = this.spentBy.get(`${current}:${vout}`);
+            if (child && !visited.has(child)) {
+              stack.push(child);
+            }
+          }
+        }
+      }
+    }
+    return component;
+  }
+
+  private effectiveFee(txid: string, tx: MempoolTransactionExtended): number {
+    return tx.fee + (this.accelerations[txid]?.feeDelta || 0);
+  }
+
+  private adjustedWeight(tx: MempoolTransactionExtended): number {
+    return Math.max(tx.weight, (tx.sigops || 0) * 20);
+  }
+
+  private createClusterFromTxids(
+    txids: Set<string>,
+    parentMap: Map<string, Set<string>>
+  ): Cluster | null {
+    const clusterId = this.nextClusterId++;
+    const depgraph = new DepGraph();
+    const txMap = new Map<string, ClusterTx>();
+
+    for (const txid of txids) {
+      const tx = this.mempool[txid];
+      if (!tx) {
+        logger.warn(`Warning: missing mempool tx ${txid} during cluster creation, skipping`);
+        return null;
+      }
+      const clusterTx = depgraph.addTransaction(txid, this.effectiveFee(txid, tx), this.adjustedWeight(tx), tx.order ?? 0);
+      txMap.set(txid, clusterTx);
+    }
+
+    for (const txid of txids) {
+      const txParents = parentMap.get(txid);
+      if (txParents) {
+        for (const parentTxid of txParents) {
+          if (txids.has(parentTxid)) {
+            const parentTx = txMap.get(parentTxid);
+            const childTx = txMap.get(txid);
+            if (parentTx && childTx) {
+              depgraph.addDependency(parentTx, childTx);
+            }
+          }
+        }
+      }
+    }
+
+    const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), this.costBudget);
+
+    const cluster: Cluster = {
+      id: clusterId,
+      depgraph,
+      txs: txMap,
+      linearization,
+      chunks,
+      dirty: false,
+    };
+
+    this.clusters.set(clusterId, cluster);
+    for (const txid of txids) {
+      this.txToCluster.set(txid, clusterId);
+    }
+
+    if (this.modifyTxs) {
+      this.writeBackCluster(cluster);
+    }
+    return cluster;
+  }
+
+  public writeBackClusters(): void {
+    for (const cluster of this.clusters.values()) {
+      this.writeBackCluster(cluster);
+    }
+  }
+
+  private writeBackCluster(cluster: Cluster): void {
+    for (let chunkIdx = 0; chunkIdx < cluster.chunks.length; chunkIdx++) {
+      const chunk = cluster.chunks[chunkIdx];
+      const chunkFeerate = chunk.weight > 0 ? (chunk.fee * 4) / chunk.weight : 0;
+      const chunkSet = chunk.txs.length > 1 ? new Set(chunk.txs) : null;
+      for (const clusterTx of chunk.txs) {
+        this.writeBackTx(cluster, clusterTx, chunkIdx, chunkFeerate, chunkSet);
+      }
+    }
+  }
+
+  private writeBackTx(
+    cluster: Cluster,
+    clusterTx: ClusterTx,
+    chunkIdx: number,
+    chunkFeerate: number,
+    chunkSet: Set<ClusterTx> | null
+  ): void {
+    const txid = clusterTx.txid;
+    if (!this.mempool[txid]) {
+      logger.warn(`ClusterMempool.writeBackTx: ${txid} missing from mempool (cluster ${cluster.id})`);
+      return;
+    }
+    const tx = this.mempool[txid];
+    if (tx.effectiveFeePerVsize !== chunkFeerate || tx.clusterId !== cluster.id) {
+      tx.cpfpDirty = true;
+    }
+    tx.effectiveFeePerVsize = chunkFeerate;
+    tx.clusterId = cluster.id;
+    tx.chunkIndex = chunkIdx;
+    tx.cpfpChecked = true;
+
+    if (chunkSet) {
+      tx.ancestors = this.getChunkRelatives(clusterTx, chunkSet, 'ancestors');
+      tx.descendants = this.getChunkRelatives(clusterTx, chunkSet, 'descendants');
+    } else {
+      tx.ancestors = [];
+      tx.descendants = [];
+    }
+  }
+
+  private getChunkRelatives(
+    clusterTx: ClusterTx,
+    chunkSet: Set<ClusterTx>,
+    direction: 'ancestors' | 'descendants'
+  ): { txid: string; fee: number; weight: number }[] {
+    const relatives: { txid: string; fee: number; weight: number }[] = [];
+    const related = direction === 'ancestors' ? clusterTx.ancestors : clusterTx.descendants;
+    for (const rel of related) {
+      if (rel !== clusterTx && chunkSet.has(rel)) {
+        const mempoolTx = this.mempool[rel.txid];
+        if (mempoolTx) {
+          relatives.push({ txid: rel.txid, fee: mempoolTx.fee, weight: mempoolTx.weight });
+        } else {
+          logger.warn(`ClusterMempool.getChunkRelatives: ${rel.txid} missing from mempool`);
+        }
+      }
+    }
+    return relatives;
+  }
+
+  private processRemovals(removed: MempoolTransactionExtended[]): void {
+    for (const tx of removed) {
+      for (const vin of tx.vin) {
+        if (!vin.is_coinbase) {
+          const spentOutpoint = `${vin.txid}:${vin.vout}`;
+          if (this.spentBy.get(spentOutpoint) === tx.txid) {
+            this.spentBy.delete(spentOutpoint);
+          }
+        }
+      }
+    }
+
+    for (const tx of removed) {
+      const match = this.getClusterForTx(tx.txid);
+      if (match) {
+        match.cluster.depgraph.removeTransactions(new Set([match.clusterTx]));
+        match.cluster.txs.delete(tx.txid);
+        match.cluster.linearization = match.cluster.linearization.filter(t => t !== match.clusterTx);
+        this.txToCluster.delete(tx.txid);
+        match.cluster.dirty = true;
+      }
+    }
+  }
+
+  private splitDisconnectedClusters(): void {
+    for (const [clusterId, cluster] of this.clusters.entries()) {
+      if (cluster.dirty) {
+        if (cluster.depgraph.size === 0) {
+          this.clusters.delete(clusterId);
+        } else {
+          const components = cluster.depgraph.findConnectedComponents();
+          if (components.length > 1) {
+            this.clusters.delete(clusterId);
+            for (const component of components) {
+              this.splitComponentToCluster(cluster, component);
+            }
+          }
+        }
+      }
+    }
+  }
+
+  private splitComponentToCluster(sourceCluster: Cluster, component: Set<ClusterTx>): void {
+    const newClusterId = this.nextClusterId++;
+    const { depgraph: newDepgraph, txMap } = subgraph(component);
+
+    const newTxs = new Map<string, ClusterTx>();
+    for (const oldTx of component) {
+      const newTx = txMap.get(oldTx);
+      if (newTx) {
+        newTxs.set(oldTx.txid, newTx);
+        this.txToCluster.set(oldTx.txid, newClusterId);
+      }
+    }
+
+    const newLinearization: ClusterTx[] = [];
+    for (const oldTx of sourceCluster.linearization) {
+      if (component.has(oldTx)) {
+        const newTx = txMap.get(oldTx);
+        if (newTx) {
+          newLinearization.push(newTx);
+        }
+      }
+    }
+
+    const newCluster: Cluster = {
+      id: newClusterId,
+      dirty: true,
+      depgraph: newDepgraph,
+      txs: newTxs,
+      linearization: newLinearization,
+      chunks: [],
+    };
+    this.clusters.set(newClusterId, newCluster);
+  }
+
+  private processAdditions(added: MempoolTransactionExtended[]): void {
+    for (const tx of added) {
+      const txid = tx.txid;
+
+      // sanity check for duplicate transactions
+      if (this.getClusterForTx(txid) !== null) {
+        logger.warn(`ClusterMempool.processAdditions: ${txid} was already added, skipping`);
+        continue;
+      }
+
+      for (const vin of tx.vin) {
+        if (!vin.is_coinbase) {
+          this.spentBy.set(`${vin.txid}:${vin.vout}`, txid);
+        }
+      }
+
+      const { relatedClusterIds, parentTxids, childTxids } = this.findRelatedClusters(tx);
+
+      if (relatedClusterIds.size === 0) {
+        this.addSingletonCluster(tx);
+      } else if (relatedClusterIds.size === 1) {
+        this.addToExistingCluster(tx, relatedClusterIds, parentTxids, childTxids);
+      } else {
+        this.mergeAndAddToCluster(tx, relatedClusterIds, parentTxids, childTxids);
+      }
+    }
+  }
+
+  private findRelatedClusters(tx: MempoolTransactionExtended): {
+    relatedClusterIds: Set<number>;
+    parentTxids: string[];
+    childTxids: string[];
+  } {
+    const relatedClusterIds = new Set<number>();
+    const parentTxids: string[] = [];
+    const childTxids: string[] = [];
+
+    for (const vin of tx.vin) {
+      if (!vin.is_coinbase && this.mempool[vin.txid]) {
+        const parentCluster = this.txToCluster.get(vin.txid);
+        if (parentCluster !== undefined) {
+          relatedClusterIds.add(parentCluster);
+          parentTxids.push(vin.txid);
+        }
+      }
+    }
+
+    for (let vout = 0; vout < tx.vout.length; vout++) {
+      const childTxid = this.spentBy.get(`${tx.txid}:${vout}`);
+      if (childTxid && this.mempool[childTxid]) {
+        const childCluster = this.txToCluster.get(childTxid);
+        if (childCluster !== undefined) {
+          relatedClusterIds.add(childCluster);
+          childTxids.push(childTxid);
+        }
+      }
+    }
+
+    return { relatedClusterIds, parentTxids, childTxids };
+  }
+
+  private addSingletonCluster(tx: MempoolTransactionExtended): void {
+    const txid = tx.txid;
+    const clusterId = this.nextClusterId++;
+    const depgraph = new DepGraph();
+    const clusterTx = depgraph.addTransaction(txid, this.effectiveFee(txid, tx), this.adjustedWeight(tx), tx.order ?? 0);
+
+    const cluster: Cluster = {
+      id: clusterId,
+      dirty: true,
+      depgraph,
+      txs: new Map([[txid, clusterTx]]),
+      linearization: [clusterTx],
+      chunks: [],
+    };
+    this.clusters.set(clusterId, cluster);
+    this.txToCluster.set(txid, clusterId);
+  }
+
+  private addToExistingCluster(
+    tx: MempoolTransactionExtended,
+    relatedClusterIds: Set<number>,
+    parentTxids: string[],
+    childTxids: string[],
+  ): void {
+    const txid = tx.txid;
+    const clusterId = relatedClusterIds.values().next().value;
+    const cluster = this.clusters.get(clusterId);
+    if (!cluster) {
+      return;
+    }
+    const clusterTx = cluster.depgraph.addTransaction(txid, this.effectiveFee(txid, tx), this.adjustedWeight(tx), tx.order ?? 0);
+    cluster.txs.set(txid, clusterTx);
+    cluster.linearization.push(clusterTx);
+    this.txToCluster.set(txid, clusterId);
+
+    this.addParentDeps(cluster, clusterTx, parentTxids);
+    this.addChildDeps(cluster, clusterTx, childTxids);
+    cluster.dirty = true;
+  }
+
+  private mergeAndAddToCluster(
+    tx: MempoolTransactionExtended,
+    relatedClusterIds: Set<number>,
+    parentTxids: string[],
+    childTxids: string[],
+  ): void {
+    const clusterIterator = relatedClusterIds.values();
+    const primaryId: number = clusterIterator.next().value;
+    const primary = this.clusters.get(primaryId);
+    if (!primary) {
+      return;
+    }
+
+    for (const clusterId of clusterIterator) {
+      const other = this.clusters.get(clusterId);
+      if (other) {
+        this.mergeClusterInto(primary, other);
+        this.clusters.delete(clusterId);
+      }
+    }
+
+    const clusterTx = primary.depgraph.addTransaction(tx.txid, this.effectiveFee(tx.txid, tx), this.adjustedWeight(tx), tx.order ?? 0);
+    primary.txs.set(tx.txid, clusterTx);
+    primary.linearization.push(clusterTx);
+    this.txToCluster.set(tx.txid, primaryId);
+
+    this.addParentDeps(primary, clusterTx, parentTxids);
+    this.addChildDeps(primary, clusterTx, childTxids);
+    primary.dirty = true;
+  }
+
+  private addParentDeps(cluster: Cluster, childTx: ClusterTx, parentTxids: string[]): void {
+    for (const parentTxid of parentTxids) {
+      const parentTx = cluster.txs.get(parentTxid);
+      if (parentTx) {
+        cluster.depgraph.addDependency(parentTx, childTx);
+      }
+    }
+  }
+
+  private addChildDeps(cluster: Cluster, parentTx: ClusterTx, childTxids: string[]): void {
+    for (const childTxid of childTxids) {
+      const childTx = cluster.txs.get(childTxid);
+      if (childTx) {
+        cluster.depgraph.addDependency(parentTx, childTx);
+      }
+    }
+  }
+
+  private mergeClusterInto(primary: Cluster, other: Cluster): void {
+    for (const [txid, otherTx] of other.txs) {
+      const newTx = primary.depgraph.addTransaction(txid, otherTx.effectiveFee, otherTx.weight, otherTx.order);
+      primary.txs.set(txid, newTx);
+      this.txToCluster.set(txid, primary.id);
+    }
+
+    for (const otherTx of other.depgraph.getTxs()) {
+      for (const parent of otherTx.parents) {
+        const newChild = primary.txs.get(otherTx.txid);
+        const newParent = primary.txs.get(parent.txid);
+        if (newChild && newParent) {
+          primary.depgraph.addDependency(newParent, newChild);
+        }
+      }
+    }
+
+    for (const otherTx of other.linearization) {
+      const newTx = primary.txs.get(otherTx.txid);
+      if (newTx) {
+        primary.linearization.push(newTx);
+      }
+    }
+  }
+
+  private processAccelerationChanges(newAccelerations: { [txid: string]: { feeDelta: number } }): void {
+    const changed = new Set<string>();
+    for (const txid in newAccelerations) {
+      if ((newAccelerations[txid]?.feeDelta || 0) !== (this.accelerations[txid]?.feeDelta || 0)) {
+        changed.add(txid);
+      }
+    }
+    for (const txid in this.accelerations) {
+      if (!newAccelerations[txid]) {
+        changed.add(txid);
+      }
+    }
+    this.accelerations = newAccelerations;
+    for (const txid of changed) {
+      const tx = this.mempool[txid];
+      if (!tx) {
+        continue;
+      }
+      const match = this.getClusterForTx(txid);
+      if (match) {
+        match.clusterTx.effectiveFee = this.effectiveFee(txid, tx);
+        match.cluster.dirty = true;
+      }
+    }
+  }
+
+  private relinearizeDirtyClusters(): void {
+    for (const [clusterId, cluster] of this.clusters.entries()) {
+      if (cluster.dirty) {
+        cluster.dirty = false;
+        const newId = this.nextClusterId++;
+        this.clusters.delete(clusterId);
+        cluster.id = newId;
+        this.clusters.set(newId, cluster);
+        for (const txid of cluster.txs.keys()) {
+          this.txToCluster.set(txid, newId);
+        }
+
+        const { linearization, chunks } = linearizeCluster(
+          cluster.depgraph.getTxs(),
+          this.costBudget,
+          cluster.linearization,
+        );
+        cluster.linearization = linearization;
+        cluster.chunks = chunks;
+
+        if (this.modifyTxs) {
+          this.writeBackCluster(cluster);
+        }
+      }
+    }
+  }
+
+  private buildClusterData(cluster: Cluster): CpfpClusterData {
+    const txs: CpfpClusterTx[] = [];
+    const txToFlatIdx = new Map<ClusterTx, number>();
+
+    for (const chunk of cluster.chunks) {
+      const ordered = sortTopological(new Set(chunk.txs));
+      for (const clusterTx of ordered) {
+        if (this.mempool[clusterTx.txid]) {
+          txToFlatIdx.set(clusterTx, txs.length);
+          const parents: number[] = [];
+          for (const parentTx of clusterTx.parents) {
+            const flatIdx = txToFlatIdx.get(parentTx);
+            if (flatIdx !== undefined) {
+              parents.push(flatIdx);
+            }
+          }
+          const mempoolTx = this.mempool[clusterTx.txid];
+          txs.push({ txid: clusterTx.txid, fee: mempoolTx.fee, weight: mempoolTx.weight, parents });
+        } else {
+          logger.warn(`ClusterMempool.buildClusterData: ${clusterTx.txid} missing from mempool (cluster ${cluster.id})`);
+        }
+      }
+    }
+
+    let offset = 0;
+    const chunks = cluster.chunks.map(chunk => {
+      const count = chunk.txs.length;
+      const chunkEntry = {
+        txs: Array.from({ length: count }, (_, i) => offset + i),
+        feerate: chunk.weight > 0 ? (chunk.fee * 4) / chunk.weight : 0,
+      };
+      offset += count;
+      return chunkEntry;
+    });
+
+    return { txs, chunks };
+  }
+
+  private findChunkInfo(cluster: Cluster, tx: ClusterTx): ClusterInfo | null {
+    for (let chunkIdx = 0; chunkIdx < cluster.chunks.length; chunkIdx++) {
+      const chunk = cluster.chunks[chunkIdx];
+      if (chunk.txs.includes(tx)) {
+        return {
+          clusterId: cluster.id,
+          chunkIndex: chunkIdx,
+          chunkFeerate: chunk.weight > 0 ? (chunk.fee * 4) / chunk.weight : 0,
+        };
+      }
+    }
+    return null;
+  }
+}

backend/src/cluster-mempool/depgraph.ts

@@ -0,0 +1,165 @@
+import logger from '../logger';
+
+export class ClusterTx {
+  txid: string;
+  effectiveFee: number;
+  weight: number;
+  order: number;
+  ancestors: Set<ClusterTx>;
+  descendants: Set<ClusterTx>;
+  parents: Set<ClusterTx>;
+  children: Set<ClusterTx>;
+
+  constructor(txid: string, effectiveFee: number, weight: number, order: number) {
+    this.txid = txid;
+    this.effectiveFee = effectiveFee;
+    this.weight = weight;
+    this.order = order;
+    this.ancestors = new Set([this]);
+    this.descendants = new Set([this]);
+    this.parents = new Set();
+    this.children = new Set();
+  }
+}
+
+export class DepGraph {
+  private txs: Set<ClusterTx> = new Set();
+
+  get size(): number {
+    return this.txs.size;
+  }
+
+  addTransaction(txid: string, fee: number, weight: number, order: number = 0): ClusterTx {
+    const tx = new ClusterTx(txid, fee, weight, order);
+    this.txs.add(tx);
+    return tx;
+  }
+
+  addDependency(parent: ClusterTx, child: ClusterTx): void {
+    if (!this.txs.has(parent) || !this.txs.has(child)) {
+      logger.warn(`Warning: invalid dependency, skipping`);
+      return;
+    }
+
+    parent.children.add(child);
+    child.parents.add(parent);
+
+    if (child.ancestors.has(parent)) {
+      return;
+    }
+
+    for (const descendant of child.descendants) {
+      for (const ancestor of parent.ancestors) {
+        descendant.ancestors.add(ancestor);
+        ancestor.descendants.add(descendant);
+      }
+    }
+  }
+
+  removeTransactions(toRemove: Set<ClusterTx>): void {
+    for (const tx of toRemove) {
+      for (const parent of tx.parents) {
+        parent.children.delete(tx);
+      }
+      for (const child of tx.children) {
+        child.parents.delete(tx);
+      }
+      this.txs.delete(tx);
+    }
+
+    for (const tx of this.txs) {
+      for (const removed of toRemove) {
+        tx.ancestors.delete(removed);
+        tx.descendants.delete(removed);
+      }
+    }
+
+    this.rederiveAncestorsDescendants();
+  }
+
+  private rederiveAncestorsDescendants(): void {
+    const ordered = [...this.txs].sort((a, b) => a.ancestors.size - b.ancestors.size);
+    for (const tx of this.txs) {
+      tx.ancestors = new Set([tx]);
+      tx.descendants = new Set([tx]);
+    }
+    for (const tx of ordered) {
+      for (const parent of tx.parents) {
+        for (const ancestor of parent.ancestors) {
+          tx.ancestors.add(ancestor);
+        }
+      }
+      for (const ancestor of tx.ancestors) {
+        ancestor.descendants.add(tx);
+      }
+    }
+  }
+
+  hasTx(tx: ClusterTx): boolean {
+    return this.txs.has(tx);
+  }
+
+  getTxs(): Set<ClusterTx> {
+    return this.txs;
+  }
+
+  findConnectedComponents(): Set<ClusterTx>[] {
+    const visited = new Set<ClusterTx>();
+    const components: Set<ClusterTx>[] = [];
+
+    for (const tx of this.txs) {
+      if (!visited.has(tx)) {
+        const component = new Set<ClusterTx>();
+        const stack: ClusterTx[] = [tx];
+        while (stack.length > 0) {
+          const node = stack.pop();
+          if (node && !visited.has(node)) {
+            visited.add(node);
+            component.add(node);
+            for (const a of node.ancestors) {
+              if (!visited.has(a) && this.txs.has(a)) {
+                stack.push(a);
+              }
+            }
+            for (const d of node.descendants) {
+              if (!visited.has(d) && this.txs.has(d)) {
+                stack.push(d);
+              }
+            }
+          }
+        }
+        components.push(component);
+      }
+    }
+    return components;
+  }
+
+}
+
+export function sortTopological(subset: Set<ClusterTx>): ClusterTx[] {
+  return [...subset].sort((a, b) => a.ancestors.size - b.ancestors.size);
+}
+
+export function subgraph(txSubset: Set<ClusterTx>): { depgraph: DepGraph; txMap: Map<ClusterTx, ClusterTx> } {
+  const newGraph = new DepGraph();
+  const txMap = new Map<ClusterTx, ClusterTx>();
+
+  for (const oldTx of txSubset) {
+    const newTx = newGraph.addTransaction(oldTx.txid, oldTx.effectiveFee, oldTx.weight, oldTx.order);
+    txMap.set(oldTx, newTx);
+  }
+
+  for (const oldTx of txSubset) {
+    for (const parent of oldTx.parents) {
+      if (txSubset.has(parent)) {
+        const newChild = txMap.get(oldTx);
+        const newParent = txMap.get(parent);
+        if (newChild && newParent) {
+          newGraph.addDependency(newParent, newChild);
+        }
+      }
+    }
+  }
+
+  return { depgraph: newGraph, txMap };
+}

backend/src/config.ts

@@ -34,6 +34,8 @@ interface IConfig {
     POOLS_JSON_TREE_URL: string,
     POOLS_UPDATE_DELAY: number,
     AUDIT: boolean;
+    CLUSTER_MEMPOOL: boolean;
+    CLUSTER_MEMPOOL_INDEXING: boolean;
     RUST_GBT: boolean;
     LIMIT_GBT: boolean;
     CPFP_INDEXING: boolean;
@@ -205,6 +207,8 @@ const defaults: IConfig = {
     'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
     'POOLS_UPDATE_DELAY': 604800, // in seconds, default is one week
     'AUDIT': false,
+    'CLUSTER_MEMPOOL': false,
+    'CLUSTER_MEMPOOL_INDEXING': false,
     'RUST_GBT': true,
     'LIMIT_GBT': false,
     'CPFP_INDEXING': false,

backend/src/index.ts

@@ -1,5 +1,6 @@
 import express from 'express';
 import { Application, Request, Response, NextFunction } from 'express';
+import * as fs from 'fs';
 import * as http from 'http';
 import * as WebSocket from 'ws';
 import bitcoinApi from './api/bitcoin/bitcoin-api-factory';
@@ -67,6 +68,10 @@ class Server {
   constructor() {
     this.app = express();
 
+    if (cluster.isPrimary && config.MEMPOOL.UNIX_SOCKET_PATH) {
+      this.clearStaleUnixSocket(config.MEMPOOL.UNIX_SOCKET_PATH);
+    }
+
     if (!config.MEMPOOL.SPAWN_CLUSTER_PROCS) {
       void this.startServer();
       return;
@@ -154,17 +159,17 @@ class Server {
     }
 
     this.server = http.createServer(this.app);
-    this.wss = new WebSocket.Server({ server: this.server });
+    this.wss = new WebSocket.Server({ server: this.server, maxPayload: websocketHandler.MAX_MESSAGE_SIZE });
     if (config.MEMPOOL.UNIX_SOCKET_PATH) {
       this.serverUnixSocket = http.createServer(this.app);
-      this.wssUnixSocket = new WebSocket.Server({ server: this.serverUnixSocket });
+      this.wssUnixSocket = new WebSocket.Server({ server: this.serverUnixSocket, maxPayload: websocketHandler.MAX_MESSAGE_SIZE });
     }
 
     this.setUpWebsocketHandling();
 
     await poolsUpdater.updatePoolsJson(); // Needs to be done before loading the disk cache because we sometimes wipe it
     if (config.DATABASE.ENABLED === true && config.MEMPOOL.ENABLED && ['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) && !poolsUpdater.currentSha) {
-      logger.err(`Failed to retreive pools-v2.json sha, cannot run block indexing. Please make sure you've set valid urls in your mempool-config.json::MEMPOOL::POOLS_JSON_URL and mempool-config.json::MEMPOOL::POOLS_JSON_TREE_UR, aborting now`);
+      logger.err(`Failed to retrieve pools-v2.json sha, cannot run block indexing. Please make sure you've set valid urls in your mempool-config.json::MEMPOOL::POOLS_JSON_URL and mempool-config.json::MEMPOOL::POOLS_JSON_TREE_UR, aborting now`);
       return process.exit(1);
     }
 
@@ -177,8 +182,14 @@ class Server {
       if (config.MEMPOOL.CACHE_ENABLED) {
         await diskCache.$loadMempoolCache();
       } else if (config.REDIS.ENABLED) {
+        let currentMempoolTxids: Set<string> | undefined;
+        try {
+          currentMempoolTxids = new Set(await bitcoinApi.$getRawMempool());
+        } catch (e) {
+          logger.warn(`Failed to fetch raw mempool before loading Redis cache. Reason: ${e instanceof Error ? e.message : e}`);
+        }
         /** @asyncUnsafe */
-        await redisCache.$loadCache();
+        await redisCache.$loadCache(currentMempoolTxids);
       }
     }
 
@@ -224,6 +235,8 @@ class Server {
         logger.notice(`Mempool Server is running on port ${config.MEMPOOL.HTTP_PORT}`);
       }
     });
+    this.server.keepAliveTimeout = 70 * 1000;
+    this.server.headersTimeout = 71 * 1000;
 
     if (this.serverUnixSocket) {
       this.serverUnixSocket.listen(config.MEMPOOL.UNIX_SOCKET_PATH, () => {
@@ -233,11 +246,25 @@ class Server {
           logger.notice(`Mempool Server is listening on ${config.MEMPOOL.UNIX_SOCKET_PATH}`);
         }
       });
+
+      this.serverUnixSocket.keepAliveTimeout = 70 * 1000;
+      this.serverUnixSocket.headersTimeout = 71 * 1000;
     }
 
     void poolsUpdater.$startService();
   }
 
+  clearStaleUnixSocket(path: string): void {
+    try {
+      if (fs.existsSync(path)) {
+        fs.unlinkSync(path);
+        logger.notice(`Removed stale unix socket ${path}`);
+      }
+    } catch (e) {
+      logger.err(`Failed to remove stale unix socket ${path}. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
   /** @asyncSafe */
   async runMainUpdateLoop(): Promise<void> {
     const start = Date.now();
@@ -325,7 +352,6 @@ class Server {
       blocks.setNewBlockCallback(async () => {
         try {
           await elementsParser.$parse();
-          await elementsParser.$updateFederationUtxos();
         } catch (e) {
           logger.warn('Elements parsing error: ' + (e instanceof Error ? e.message : e));
         }
@@ -335,7 +361,6 @@ class Server {
     if (config.MEMPOOL.ENABLED) {
       statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler));
       memPool.setAsyncMempoolChangedCallback(websocketHandler.$handleMempoolChange.bind(websocketHandler));
-      blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
     }
     if (config.FIAT_PRICE.ENABLED) {
       priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
@@ -396,6 +421,16 @@ class Server {
       this.warnedHeapCritical = false;
       this.maxHeapSize = 0;
       this.lastHeapLogTime = now;
+      this.server?.getConnections((error, count) => {
+        if (!error) {
+          logger.debug(`${count} open TCP sockets`);
+        }
+      });
+      this.serverUnixSocket?.getConnections((error, count) => {
+        if (!error) {
+          logger.debug(`${count} open unix sockets`);
+        }
+      });
     }
   }
 

backend/src/indexer.ts

@@ -113,12 +113,11 @@ class Indexer {
       }
     }
     this.tasksScheduled[task] = setTimeout(async () => {
+      delete this.tasksScheduled[task];
       try {
         await this.runSingleTask(task);
       } catch (e) {
         logger.err(`Unexpected error in scheduled task ${task}: ` + (e instanceof Error ? e.message : e));
-      } finally {
-        clearTimeout(this.tasksScheduled[task]);
       }
     }, timeout);
   }
@@ -139,12 +138,13 @@ class Indexer {
     switch (task) {
       case 'blocksPrices': {
         if (!['testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) && config.FIAT_PRICE.ENABLED) {
-          let lastestPriceId;
+          let latestPriceId;
           try {
-            lastestPriceId = await PricesRepository.$getLatestPriceId();
+            latestPriceId = await PricesRepository.$getLatestPriceId();
           } catch (e) {
             logger.debug('failed to fetch latest price id from db: ' + (e instanceof Error ? e.message : e));
-          }          if (priceUpdater.historyInserted === false || lastestPriceId === null) {
+          }
+          if (priceUpdater.historyInserted === false || latestPriceId === null) {
             logger.debug(`Blocks prices indexer is waiting for the price updater to complete`, logger.tags.mining);
             this.scheduleSingleTask(task, 10000);
           } else {
@@ -226,6 +226,8 @@ class Indexer {
       await AccelerationRepository.$indexPastAccelerations();
       await BlocksAuditsRepository.$migrateAuditsV0toV1();
       await BlocksRepository.$migrateBlocks();
+
+      void blocks.$generateFlagValuesDatabase();
       // do not wait for classify blocks to finish
       void blocks.$classifyBlocks();
       runSuccessful = true;

backend/src/mempool.interfaces.ts

@@ -17,6 +17,7 @@ export interface PoolInfo {
   name: string;
   link: string;
   blockCount: number;
+  emptyBlocks: number;
   slug: string;
   avgMatchRate: number | null;
   avgFeeDelta: number | null;
@@ -25,7 +26,11 @@ export interface PoolInfo {
 
 export interface PoolStats extends PoolInfo {
   rank: number;
-  emptyBlocks: number;
+}
+
+export enum TemplateAlgorithm {
+  legacy = 0,
+  clusterMempool = 1,
 }
 
 export interface BlockAudit {
@@ -45,6 +50,7 @@ export interface BlockAudit {
   expectedFees?: number,
   expectedWeight?: number,
   template?: any[];
+  templateAlgorithm?: TemplateAlgorithm,
 }
 
 export interface TransactionAudit {
@@ -132,6 +138,8 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
   replacement?: boolean;
   uid?: number;
   flags?: number;
+  clusterId?: number;
+  chunkIndex?: number;
 }
 
 export interface MempoolTransactionExtended extends TransactionExtended {
@@ -227,6 +235,7 @@ export interface CpfpInfo {
   adjustedVsize?: number,
   acceleration?: boolean,
   fee?: number;
+  cluster?: CpfpClusterData & { chunkIndex: number };
 }
 
 export interface TransactionStripped {
@@ -388,6 +397,25 @@ export interface CpfpCluster {
   height: number,
   txs: Ancestor[],
   effectiveFeePerVsize: number,
+  templateAlgorithm?: TemplateAlgorithm,
+  clusterData?: CpfpClusterData,
+}
+
+export interface CpfpClusterTx {
+  txid: string;
+  fee: number;
+  weight: number;
+  parents: number[];
+}
+
+export interface CpfpClusterChunk {
+  txs: number[];
+  feerate: number;
+}
+
+export interface CpfpClusterData {
+  txs: CpfpClusterTx[];
+  chunks: CpfpClusterChunk[];
 }
 
 export interface CpfpSummary {

backend/src/replication/AuditReplication.ts

@@ -114,6 +114,7 @@ class AuditReplication {
     });
     await blocksAuditsRepository.$saveAudit({
       version: auditSummary.version || 0,
+      templateAlgorithm: auditSummary.templateAlgorithm ?? 0,
       hash: blockHash,
       height: auditSummary.height,
       time: auditSummary.timestamp || auditSummary.time,

backend/src/repositories/BlocksAuditsRepository.ts

@@ -1,7 +1,7 @@
 import DB from '../database';
 import logger from '../logger';
 import bitcoinApi from '../api/bitcoin/bitcoin-api-factory';
-import { BlockAudit, AuditScore, TransactionAudit, TransactionStripped } from '../mempool.interfaces';
+import { BlockAudit, AuditScore, TransactionAudit, TransactionStripped, TemplateAlgorithm } from '../mempool.interfaces';
 
 interface MigrationAudit {
   version: number,
@@ -18,8 +18,8 @@ class BlocksAuditRepositories {
   /** @asyncSafe */
   public async $saveAudit(audit: BlockAudit): Promise<void> {
     try {
-      await DB.query(`INSERT INTO blocks_audits(version, time, height, hash, unseen_txs, missing_txs, added_txs, prioritized_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight)
-        VALUE (?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.version, audit.time, audit.height, audit.hash, JSON.stringify(audit.unseenTxs), JSON.stringify(audit.missingTxs),
+      await DB.query(`INSERT INTO blocks_audits(version, template_algo, time, height, hash, unseen_txs, missing_txs, added_txs, prioritized_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight)
+        VALUE (?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.version, audit.templateAlgorithm ?? 0, audit.time, audit.height, audit.hash, JSON.stringify(audit.unseenTxs), JSON.stringify(audit.missingTxs),
           JSON.stringify(audit.addedTxs), JSON.stringify(audit.prioritizedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), JSON.stringify(audit.acceleratedTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]);
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
@@ -80,6 +80,7 @@ class BlocksAuditRepositories {
       const [rows]: any[] = await DB.query(
         `SELECT
           blocks_audits.version,
+          blocks_audits.template_algo as templateAlgorithm,
           blocks_audits.height,
           blocks_audits.hash as id,
           UNIX_TIMESTAMP(blocks_audits.time) as timestamp,
@@ -120,6 +121,23 @@ class BlocksAuditRepositories {
     }
   }
 
+  /** @asyncSafe */
+  public async $getBlockTemplateAlgo(hash: string): Promise<TemplateAlgorithm | null> {
+    try {
+      const [rows]: any[] = await DB.query(
+        `SELECT template_algo FROM blocks_audits WHERE hash = ?`,
+        [hash]
+      );
+      if (rows.length) {
+        return rows[0].template_algo as TemplateAlgorithm;
+      }
+      return null;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block template algo from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return null;
+    }
+  }
+
   /** @asyncSafe */
   public async $getBlockTxAudit(hash: string, txid: string): Promise<TransactionAudit | null> {
     try {

backend/src/repositories/BlocksRepository.ts

@@ -14,7 +14,7 @@ import chainTips from '../api/chain-tips';
 import blocks from '../api/blocks';
 import BlocksAuditsRepository from './BlocksAuditsRepository';
 import transactionUtils from '../api/transaction-utils';
-import { parseDATUMTemplateCreator } from '../utils/bitcoin-script';
+import { parseDATUMTemplateCreator, parseDMNDTemplateCreator } from '../utils/bitcoin-script';
 import poolsUpdater from '../tasks/pools-updater';
 
 interface DatabaseBlock {
@@ -305,39 +305,6 @@ class BlocksRepository {
     }
   }
 
-  /**
-   * Get empty blocks for one or all pools
-   * @asyncSafe
-   */
-  public async $countEmptyBlocks(poolId: number | null, interval: string | null = null): Promise<any> {
-    interval = Common.getSqlInterval(interval);
-
-    const params: any[] = [];
-    let query = `SELECT count(height) as count, pools.id as poolId
-      FROM blocks
-      JOIN pools on pools.id = blocks.pool_id
-      WHERE tx_count = 1 AND stale = 0`;
-
-    if (poolId) {
-      query += ` AND pool_id = ?`;
-      params.push(poolId);
-    }
-
-    if (interval) {
-      query += ` AND blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
-    }
-
-    query += ` GROUP by pools.id`;
-
-    try {
-      const [rows] = await DB.query(query, params);
-      return rows;
-    } catch (e) {
-      logger.err('Cannot count empty blocks. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
   /**
    * Return most recent block height
    * @asyncSafe
@@ -360,17 +327,26 @@ class BlocksRepository {
     interval = Common.getSqlInterval(interval);
 
     const params: any[] = [];
-    let query = `SELECT count(height) as blockCount
-      FROM blocks 
-      WHERE stale = 0`;
-
-    if (poolId) {
-      query += ` AND pool_id = ?`;
+    let query;
+    if (!poolId && !interval) {
+      // optimized query to get full indexed block count
+      query = `SELECT CAST(COALESCE(MAX(height) - MIN(height) + 1, 0) AS SIGNED) as blockCount FROM blocks`;
+    } else if (!poolId) {
+      // optimized query for indexed count within an interval
+      query = `SELECT GREATEST(0, COALESCE(
+        CAST((SELECT height FROM blocks WHERE stale = 0 AND blockTimestamp <= NOW() ORDER BY blockTimestamp DESC LIMIT 1) AS SIGNED) -
+        CAST((SELECT height FROM blocks WHERE stale = 0 AND blockTimestamp >= DATE_SUB(NOW(), INTERVAL ${interval}) ORDER BY blockTimestamp ASC LIMIT 1) AS SIGNED) + 1
+      , 0)) as blockCount`;
+    } else {
+      // for specific pools, we do still have to actually count the blocks
+      query = `SELECT count(*) as blockCount
+        FROM blocks
+        WHERE stale = 0 AND pool_id = ?`;
       params.push(poolId);
-    }
 
-    if (interval) {
-      query += ` AND blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      if (interval) {
+        query += ` AND blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      }
     }
 
     try {
@@ -575,8 +551,34 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Get the canonical block hash at a given height
+   * @asyncSafe
+   */
+  public async $getCanonicalBlockHashByHeight(height: number): Promise<string | null> {
+    try {
+      const [rows]: any[] = await DB.query(`
+        SELECT hash
+        FROM blocks
+        WHERE height = ? AND stale = 0
+        LIMIT 1`,
+        [height]
+      );
+
+      if (rows.length <= 0) {
+        return null;
+      }
+
+      return rows[0].hash;
+    } catch (e) {
+      logger.err(`Cannot get canonical block hash at height ${height}. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   /**
    * Get one block by hash
+   * @asyncSafe
    */
   public async $getBlockByHash(hash: string): Promise<BlockExtended | null> {
     try {
@@ -677,6 +679,7 @@ class BlocksRepository {
       const start = new Date().getTime();
       const tip = await bitcoinApi.$getBlockHashTip();
       let firstBadBlockHeight: number | null = null;
+      let firstBadBlockTimestamp: number | null = null;
       const [blocks]: any[] = await DB.query(`
         SELECT
           height,
@@ -687,6 +690,9 @@ class BlocksRepository {
         FROM blocks
         ORDER BY height DESC
       `);
+      if (!blocks || blocks.length === 0) {
+        throw new Error('Cannot validate chain: no indexed blocks in database'); 
+      }
       const blocksByHash = {};
       const blocksByHeight = {};
       let minHeight = Infinity;
@@ -703,7 +709,11 @@ class BlocksRepository {
       // ensure that indexed blocks are correctly classified as stale or canonical
       // iterate back to genesis, resetting canonical status where necessary
       let hash = tip;
-      const tipHeight = blocksByHash[hash].height || (await bitcoinApi.$getBlock(hash))?.height;
+      const indexedTip = blocksByHash[hash];
+      const tipHeight = indexedTip?.height ?? (await bitcoinApi.$getBlock(hash))?.height;
+      if (typeof tipHeight !== 'number') {
+         throw new Error(`Cannot validate chain: could not resolve tip block height for ${hash} from index or node`);
+      }
 
       // stop at the last canonical block we're supposed to have indexed already
       let lastIndexedBlockHeight = minHeight;
@@ -725,6 +735,7 @@ class BlocksRepository {
           // block is marked stale, but shouldn't be
           await this.$setCanonicalBlockAtHeight(block.hash, height);
           firstBadBlockHeight = height;
+          firstBadBlockTimestamp = block.timestamp;
         }
         hash = block?.previous_block_hash;
         if (!hash) {
@@ -741,7 +752,9 @@ class BlocksRepository {
 
       if (firstBadBlockHeight != null) {
         logger.warn(`Chain divergence detected at block ${firstBadBlockHeight}`);
-        await HashratesRepository.$deleteHashratesFromTimestamp(blocksByHash[firstBadBlockHeight].timestamp - 604800);
+        if (firstBadBlockTimestamp != null) {
+          await HashratesRepository.$deleteHashratesFromTimestamp(firstBadBlockTimestamp - 604800);
+        }
         await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(firstBadBlockHeight);
         return false;
       }
@@ -1245,6 +1258,10 @@ class BlocksRepository {
     blk.previousblockhash = dbBlk.previousblockhash;
     blk.mediantime = dbBlk.mediantime;
     blk.indexVersion = dbBlk.index_version;
+    blk.stale = dbBlk.stale;
+    if (dbBlk.stale) {
+      blk.canonical = await this.$getCanonicalBlockHashByHeight(dbBlk.height) || undefined;
+    }
     // BlockExtension
     extras.totalFees = dbBlk.totalFees;
     extras.medianFee = dbBlk.medianFee;
@@ -1333,6 +1350,8 @@ class BlocksRepository {
 
     if (extras.pool.name === 'OCEAN') {
       extras.pool.minerNames = parseDATUMTemplateCreator(extras.coinbaseRaw);
+    } else if (extras.pool.name === 'DMND') {
+      extras.pool.minerNames = parseDMNDTemplateCreator(extras.coinbaseRaw);
     }
 
     blk.extras = <BlockExtension>extras;

backend/src/repositories/BlocksSummariesRepository.ts

@@ -216,6 +216,34 @@ class BlocksSummariesRepository {
     }
     return false;
   }
+
+  /** @asyncSafe */
+  public async $getTipIndexed(): Promise<number | null> {
+    if (!Common.blocksSummariesIndexingEnabled()) {
+      return null;
+    }
+    try {
+      const [row]: any[] = await DB.query('SELECT MAX(height) as tip FROM blocks_summaries WHERE version >= 1');
+
+      if (row !== null && row.length > 0) {
+        return row[0].tip;
+      }
+    } catch (e) {
+      logger.err(`Cannot get latest block summary. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+    return null;
+  }
+
+  public async $getSummariesBetweenHeights(startHeight: number, latestHeight: number): Promise<{height: number, transactions: string, timestamp: number}[]> {
+    try {
+      const [rows]: any[] = await DB.query(`SELECT bs.height, bs.transactions, UNIX_TIMESTAMP(b.blockTimestamp) as timestamp FROM blocks_summaries bs JOIN blocks b ON bs.id = b.hash WHERE bs.height <= ? AND bs.height > ? AND b.stale = 0 AND bs.version >= 1 ORDER BY height DESC`, [startHeight, latestHeight]);
+
+      return rows;
+    } catch (e) {
+      logger.err(`Cannot get blocks between ${startHeight} and ${latestHeight}. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
 }
 
 export default new BlocksSummariesRepository();

backend/src/repositories/CpfpRepository.ts

@@ -1,31 +1,47 @@
 import { RowDataPacket } from 'mysql2';
 import DB from '../database';
 import logger from '../logger';
-import { Ancestor, CpfpCluster } from '../mempool.interfaces';
+import { Ancestor, CpfpCluster, CpfpClusterData, CpfpClusterTx, TemplateAlgorithm } from '../mempool.interfaces';
 import transactionRepository from '../repositories/TransactionRepository';
 
 class CpfpRepository {
-  public async $batchSaveClusters(clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[]): Promise<boolean> {
+  public async $batchSaveClusters(clusters: CpfpCluster[]): Promise<boolean> {
     try {
-      const clusterValues: [string, number, Buffer, number][] = [];
+      const clusterValues: [string, number, Buffer, number, number][] = [];
       const txs: { txid: string, cluster: string }[] = [];
 
       for (const cluster of clusters) {
         if (cluster.txs?.length) {
-          const roundedEffectiveFee = Math.round(cluster.effectiveFeePerVsize * 100) / 100;
-          const equalFee = cluster.txs.length > 1 && cluster.txs.reduce((acc, tx) => {
-            return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee);
-          }, true);
-          if (!equalFee) {
+          const isCM = cluster.templateAlgorithm === TemplateAlgorithm.clusterMempool;
+
+          if (isCM && cluster.clusterData) {
             clusterValues.push([
               cluster.root,
               cluster.height,
-              Buffer.from(this.pack(cluster.txs)),
-              cluster.effectiveFeePerVsize
+              Buffer.from(this.packCM(cluster.clusterData)),
+              0,
+              TemplateAlgorithm.clusterMempool,
             ]);
-            for (const tx of cluster.txs) {
+            for (const tx of cluster.clusterData.txs) {
               txs.push({ txid: tx.txid, cluster: cluster.root });
             }
+          } else {
+            const roundedEffectiveFee = Math.round(cluster.effectiveFeePerVsize * 100) / 100;
+            const equalFee = cluster.txs.length > 1 && cluster.txs.reduce((acc, tx) => {
+              return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee);
+            }, true);
+            if (!equalFee) {
+              clusterValues.push([
+                cluster.root,
+                cluster.height,
+                Buffer.from(this.pack(cluster.txs)),
+                cluster.effectiveFeePerVsize,
+                TemplateAlgorithm.legacy,
+              ]);
+              for (const tx of cluster.txs) {
+                txs.push({ txid: tx.txid, cluster: cluster.root });
+              }
+            }
           }
         }
       }
@@ -42,11 +58,11 @@ class CpfpRepository {
       while (chunkIndex < clusterValues.length) {
         const chunk = clusterValues.slice(chunkIndex, chunkIndex + maxChunk);
         let query = `
-            INSERT IGNORE INTO compact_cpfp_clusters(root, height, txs, fee_rate)
+            INSERT IGNORE INTO compact_cpfp_clusters(root, height, txs, fee_rate, template_algo)
             VALUES
         `;
         query += chunk.map(chunk => {
-          return (' (UNHEX(?), ?, ?, ?)');
+          return (' (UNHEX(?), ?, ?, ?, ?)');
         }) + ';';
         const values = chunk.flat();
         queries.push({
@@ -86,8 +102,16 @@ class CpfpRepository {
     );
     const cluster = clusterRows[0];
     if (cluster?.txs) {
-      cluster.effectiveFeePerVsize = cluster.fee_rate;
-      cluster.txs = this.unpack(cluster.txs);
+      if (cluster.template_algo === TemplateAlgorithm.clusterMempool) {
+        cluster.templateAlgorithm = TemplateAlgorithm.clusterMempool;
+        cluster.clusterData = this.unpackCM(cluster.txs);
+        cluster.txs = cluster.clusterData.txs.map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee }));
+        cluster.effectiveFeePerVsize = 0;
+      } else {
+        cluster.templateAlgorithm = TemplateAlgorithm.legacy;
+        cluster.effectiveFeePerVsize = cluster.fee_rate;
+        cluster.txs = this.unpack(cluster.txs);
+      }
       return cluster;
     }
     return;
@@ -105,8 +129,16 @@ class CpfpRepository {
     );
     return clusterRows.map(cluster => {
       if (cluster?.txs) {
-        cluster.effectiveFeePerVsize = cluster.fee_rate;
-        cluster.txs = this.unpack(cluster.txs);
+        if (cluster.template_algo === TemplateAlgorithm.clusterMempool) {
+          cluster.templateAlgorithm = TemplateAlgorithm.clusterMempool;
+          cluster.clusterData = this.unpackCM(cluster.txs);
+          cluster.txs = cluster.clusterData.txs.map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee }));
+          cluster.effectiveFeePerVsize = 0;
+        } else {
+          cluster.templateAlgorithm = TemplateAlgorithm.legacy;
+          cluster.effectiveFeePerVsize = cluster.fee_rate;
+          cluster.txs = this.unpack(cluster.txs);
+        }
         return cluster;
       } else {
         return null;
@@ -119,16 +151,16 @@ class CpfpRepository {
     try {
       const [rows] = await DB.query(
         `
-          SELECT txs, height, root from compact_cpfp_clusters
+          SELECT txs, height, root, template_algo from compact_cpfp_clusters
           WHERE height >= ?
         `,
         [height]
       ) as RowDataPacket[][];
       if (rows?.length) {
         for (const clusterToDelete of rows) {
-          const txs = this.unpack(clusterToDelete?.txs);
-          for (const tx of txs) {
-            await transactionRepository.$removeTransaction(tx.txid);
+          const txids = this.extractTxids(clusterToDelete);
+          for (const txid of txids) {
+            await transactionRepository.$removeTransaction(txid);
           }
         }
       }
@@ -150,16 +182,16 @@ class CpfpRepository {
     try {
       const [rows] = await DB.query(
         `
-          SELECT txs, height, root from compact_cpfp_clusters
+          SELECT txs, height, root, template_algo from compact_cpfp_clusters
           WHERE height = ?
         `,
         [height]
       ) as RowDataPacket[][];
       if (rows?.length) {
         for (const clusterToDelete of rows) {
-          const txs = this.unpack(clusterToDelete?.txs);
-          for (const tx of txs) {
-            await transactionRepository.$removeTransaction(tx.txid);
+          const txids = this.extractTxids(clusterToDelete);
+          for (const txid of txids) {
+            await transactionRepository.$removeTransaction(txid);
           }
         }
       }
@@ -176,6 +208,13 @@ class CpfpRepository {
     }
   }
 
+  private extractTxids(row: any): string[] {
+    if (row.template_algo === TemplateAlgorithm.clusterMempool) {
+      return this.unpackCM(row.txs).txs.map(tx => tx.txid);
+    }
+    return this.unpack(row.txs).map(tx => tx.txid);
+  }
+
   // insert a dummy row to mark that we've indexed as far as this block
   public async $insertProgressMarker(height: number): Promise<void> {
     try {
@@ -245,6 +284,117 @@ class CpfpRepository {
     }
   }
 
+  /**
+   * Pack cluster mempool data into binary format:
+   * [num_chunks: uint16]
+   * Per chunk: [num_txs: uint16]
+   * Per tx (in linearization order, grouped by chunk):
+   *   [txid: 32 bytes LE] [weight: uint32] [fee: uint64] [num_parents: uint8] [parent_indices: uint8 each]
+   */
+  public packCM(clusterData: CpfpClusterData): ArrayBuffer {
+    const headerSize = 2;
+    const chunkHeadersSize = clusterData.chunks.length * 2;
+    let txDataSize = 0;
+    for (const tx of clusterData.txs) {
+      txDataSize += 32 + 4 + 8 + 1 + tx.parents.length;
+    }
+    const totalSize = headerSize + chunkHeadersSize + txDataSize;
+
+    const buf = new ArrayBuffer(totalSize);
+    const view = new DataView(buf);
+    let offset = 0;
+
+    view.setUint16(offset, clusterData.chunks.length);
+    offset += 2;
+
+    for (const chunk of clusterData.chunks) {
+      view.setUint16(offset, chunk.txs.length);
+      offset += 2;
+    }
+
+    for (const tx of clusterData.txs) {
+      for (let x = 0; x < 32; x++) {
+        view.setUint8(offset + (31 - x), parseInt(tx.txid.slice(x * 2, (x * 2) + 2), 16));
+      }
+      offset += 32;
+      view.setUint32(offset, tx.weight);
+      offset += 4;
+      view.setBigUint64(offset, BigInt(Math.round(tx.fee)));
+      offset += 8;
+      view.setUint8(offset, tx.parents.length);
+      offset += 1;
+      for (const parentIdx of tx.parents) {
+        view.setUint8(offset, parentIdx);
+        offset += 1;
+      }
+    }
+
+    return buf;
+  }
+
+  public unpackCM(buf: Buffer): CpfpClusterData {
+    if (!buf) {
+      return { txs: [], chunks: [] };
+    }
+
+    try {
+      const arrayBuffer = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
+      const view = new DataView(arrayBuffer);
+      let offset = 0;
+
+      const numChunks = view.getUint16(offset);
+      offset += 2;
+
+      const chunkSizes: number[] = [];
+      for (let i = 0; i < numChunks; i++) {
+        chunkSizes.push(view.getUint16(offset));
+        offset += 2;
+      }
+
+      const txs: CpfpClusterTx[] = [];
+      const chunks: { txs: number[], feerate: number }[] = [];
+      let txIndex = 0;
+
+      for (let chunkIdx = 0; chunkIdx < numChunks; chunkIdx++) {
+        const chunkTxIndices: number[] = [];
+        let chunkFee = 0;
+        let chunkWeight = 0;
+
+        for (let t = 0; t < chunkSizes[chunkIdx]; t++) {
+          const txid = Array.from(new Uint8Array(arrayBuffer, offset, 32)).reverse().map(b => b.toString(16).padStart(2, '0')).join('');
+          offset += 32;
+          const weight = view.getUint32(offset);
+          offset += 4;
+          const fee = Number(view.getBigUint64(offset));
+          offset += 8;
+          const numParents = view.getUint8(offset);
+          offset += 1;
+          const parents: number[] = [];
+          for (let p = 0; p < numParents; p++) {
+            parents.push(view.getUint8(offset));
+            offset += 1;
+          }
+
+          txs.push({ txid, fee, weight, parents });
+          chunkTxIndices.push(txIndex);
+          chunkFee += fee;
+          chunkWeight += weight;
+          txIndex++;
+        }
+
+        chunks.push({
+          txs: chunkTxIndices,
+          feerate: chunkWeight > 0 ? (chunkFee * 4) / chunkWeight : 0,
+        });
+      }
+
+      return { txs, chunks };
+    } catch (e) {
+      logger.warn(`Failed to unpack CM CPFP cluster. Reason: ` + (e instanceof Error ? e.message : e));
+      return { txs: [], chunks: [] };
+    }
+  }
+
   // returns `true` if two sets of CPFP clusters are deeply identical
   public compareClusters(clustersA: CpfpCluster[], clustersB: CpfpCluster[]): boolean {
     if (clustersA.length !== clustersB.length) {

backend/src/repositories/FlagValueRepository.ts

@@ -0,0 +1,145 @@
+import DB from '../database';
+import logger from '../logger';
+
+export const INDEXING_PRESETS = [
+  {name: 'per block', bucketSize: 1,  retentionSpan: 144}, // block span of ~1 day
+  {name: 'per week', bucketSize: 1008, retentionSpan: -1}, // all
+  {name: 'per month', bucketSize: 4032, retentionSpan: -1}, // all
+];
+
+export const INTERVAL_PRESETS = {
+  '24h': {retentionSpan: 144, bucketSizes: [1]},
+  '6m': {retentionSpan: 24192, bucketSizes: [1008, 4032]},
+  '1y': {retentionSpan: 48384, bucketSizes: [1008, 4032]},
+  '2y': {retentionSpan: 96768, bucketSizes: [1008, 4032]},
+  '3y': {retentionSpan: 145152, bucketSizes: [1008, 4032]},
+  'all': {retentionSpan: -1, bucketSizes: [1008, 4032]},
+};
+
+class FlagValuesRepository {
+  /**
+   * Get the latest indexed day from the database
+   *
+   * @asyncSafe */
+  public async $getTipAndTailIndexedByBucketSize(bucketSize: number): Promise<{tip: number, tail: number} | null> {
+    try {
+      const [rows]: any[] = await DB.query(`SELECT (MAX(start_height) + ?) as tip, MIN(start_height) as tail FROM flag_values WHERE bucket_size = ?`, [bucketSize, bucketSize.toString()]);
+      if (rows !== null && rows.length > 0 && rows[0].tip !== null && rows[0].tail !== null) {
+        return rows[0];
+      }
+    } catch (e) {
+      logger.err(`Cannot get tip and tail indexed from flag_values. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+    return null;
+  }
+
+  /**
+   * Get the set of bucket that area already indexed between heights by bucketSize
+   *
+   * @asyncSafe */
+  public async $getIndexedStartHeights(bucketSize: number, startHeight: number, latestHeight: number): Promise<number[]> {
+    try {
+      const [rows]: any[] = await DB.query(
+        `SELECT DISTINCT start_height FROM flag_values WHERE bucket_size = ? AND start_height <= ? AND start_height >= ?`,
+        [bucketSize.toString(), startHeight, latestHeight]
+      );
+      return rows.map(row => row.start_height);
+    } catch (e) {
+      logger.err(`Cannot get indexed start heights from flag_values. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+    return [];
+  }
+
+  public async $saveBatchFlagValues(bucketSize: number, startHeight: number, dataPerFlag: Record<string, Record<string, number>>, avgTimestamp: number): Promise<void> {
+    const params: any[] = [];
+    const distinctFlags = Object.keys(dataPerFlag);
+    const avgDate = new Date(Math.round(avgTimestamp) * 1000);
+    for (const flag of distinctFlags) {
+      params.push([bucketSize.toString(), startHeight, avgDate, BigInt(flag), dataPerFlag[flag].txCount, dataPerFlag[flag].vSizeTotal]);
+    }
+    try {
+      await DB.query(`
+        INSERT INTO flag_values (bucket_size, start_height, avg_timestamp, flag_value, tx_count, vsize_total) VALUES ?
+        ON DUPLICATE KEY UPDATE
+        avg_timestamp = VALUES(avg_timestamp), tx_count = VALUES(tx_count), vsize_total = VALUES(vsize_total)
+        `, [params]);
+    } catch (e) {
+      logger.debug(`Cannot save flag batched values. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $queryTxCountBasedOnMask(mask: bigint, bucketSize: number, op: 'and' | 'or' | 'nor' | undefined, startHeight: number): Promise<{bucketSize: string, startHeight: number, avgTimestamp: number, txCount: number, vSizeTotal: number}[]> {
+    let flagPredicate = '';
+    let params: any[]= [];
+    switch (op) {
+      case 'and': {
+        flagPredicate = 'AND (flag_value & ?) = ?';
+        params = [bucketSize.toString(), startHeight, mask, mask];
+      } break;
+      case 'or': {
+        flagPredicate = 'AND (flag_value & ?) > 0';
+        params = [bucketSize.toString(), startHeight, mask];
+      } break;
+      case 'nor': {
+        flagPredicate = 'AND (flag_value & ?) = 0';
+        params = [bucketSize.toString(), startHeight, mask];
+      } break;
+      case undefined: { // op not passed, no boolean operations
+        params = [bucketSize.toString(), startHeight];
+        break;
+      }
+      default: throw new Error(`Invalid op '${op}', expected 'and' | 'or' | 'nor' | undefined`);
+    }
+    try {
+      const [rows]: any[] = await DB.query(`
+        SELECT bucket_size as bucketSize, start_height as startHeight, UNIX_TIMESTAMP(avg_timestamp) as avgTimestamp,
+          SUM(tx_count) as txCount, SUM(vsize_total) as vSizeTotal
+        FROM flag_values
+        WHERE bucket_size = ? AND start_height >= ? ${flagPredicate}
+        GROUP BY start_height ORDER BY start_height DESC
+        `, params);
+      if (rows !== null && rows.length > 0) {
+        return rows;
+      }
+    } catch (e) {
+      logger.debug(`Cannot get tx counts. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+    return [];
+  }
+
+  /** @asyncSafe */
+  public async $deleteFlagValuesBelowHeight(height: number, bucketSize: number):  Promise<void> {
+    try {
+      await DB.query(`DELETE FROM flag_values WHERE start_height < ? AND bucket_size = ?`, [height, bucketSize.toString()]);
+    } catch(e) {
+      logger.err(`Cannot delete flag values below block #${height}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /** @asyncSafe */
+  public async $deleteFlagValuesFromHeight(height: number): Promise<void> {
+    try {
+      for (const preset of INDEXING_PRESETS) {
+        const startHeight = Math.floor(height / preset.bucketSize) * preset.bucketSize;
+        await DB.query(`DELETE FROM flag_values WHERE start_height >= ? AND bucket_size = ?`, [startHeight, preset.bucketSize.toString()]);
+      }
+    } catch (e) {
+      logger.err(`Cannot delete flag values above ${height}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  public async $getTotalBlocksIndexedByBucketSize(bucketSize: number): Promise<number | null> {
+    try {
+      const [rows]: any[] = await DB.query(`SELECT (count(distinct start_height) * ?) as total FROM flag_values WHERE bucket_size = ?`, [bucketSize, bucketSize.toString()]);
+      if (rows !== null && rows.length > 0) {
+        return rows[0].total;
+      }
+    } catch (e) {
+      logger.err(`Cannot get total blocks indexed in flag_values. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+    return null;
+  }
+}
+
+export default new FlagValuesRepository();

backend/src/repositories/NodesSocketsRepository.ts

@@ -30,12 +30,13 @@ class NodesSocketsRepository {
       return 0;
     }
     try {
+      const placeholders = addresses.map(() => '?').join(',');
       const query = `
         DELETE FROM nodes_sockets
         WHERE public_key = ?
-        AND socket NOT IN (${addresses.map(id => `"${id}"`).join(',')})
+        AND socket NOT IN (${placeholders})
       `;
-      const [result] = await DB.query<ResultSetHeader>(query, [publicKey]);
+      const [result] = await DB.query<ResultSetHeader>(query, [publicKey, ...addresses]);
       return result.affectedRows;
     } catch (e) {
       logger.err(`Cannot delete unused sockets for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));

backend/src/repositories/PoolsRepository.ts

@@ -38,6 +38,7 @@ class PoolsRepository {
     let query = `
       SELECT
         COUNT(blocks.height) As blockCount,
+          COUNT(CASE WHEN blocks.tx_count = 1 THEN 1 END) AS emptyBlocks,
           pool_id AS poolId,
           pools.name AS name,
           pools.link AS link,
@@ -47,7 +48,7 @@ class PoolsRepository {
           unique_id as poolUniqueId
       FROM blocks
       JOIN pools on pools.id = pool_id
-      LEFT JOIN blocks_audits ON blocks_audits.height = blocks.height
+      LEFT JOIN blocks_audits ON blocks_audits.hash = blocks.hash
       WHERE blocks.stale = 0
     `;
 

backend/src/repositories/PricesRepository.ts

@@ -330,7 +330,7 @@ class PricesRepository {
       const [rates] = await DB.query(`
         SELECT ${ApiPriceFields}
         FROM prices
-        WHERE UNIX_TIMESTAMP(time) < ?
+        WHERE UNIX_TIMESTAMP(time) <= ?
           AND USD >= 0
         ORDER BY time DESC
         LIMIT 1`,

backend/src/repositories/TransactionRepository.ts

@@ -1,6 +1,6 @@
 import DB from '../database';
 import logger from '../logger';
-import { Ancestor, CpfpInfo } from '../mempool.interfaces';
+import { Ancestor, CpfpCluster, CpfpInfo, TemplateAlgorithm } from '../mempool.interfaces';
 import cpfpRepository from './CpfpRepository';
 
 class TransactionRepository {
@@ -72,6 +72,9 @@ class TransactionRepository {
         const clusterId = txRows[0].root.toLowerCase();
         const cluster = await cpfpRepository.$getCluster(clusterId);
         if (cluster) {
+          if (cluster.templateAlgorithm === TemplateAlgorithm.clusterMempool && cluster.clusterData) {
+            return this.convertCpfpCM(txid, cluster);
+          }
           return this.convertCpfp(txid, cluster);
         }
       }
@@ -116,6 +119,95 @@ class TransactionRepository {
       effectiveFeePerVsize: cluster.effectiveFeePerVsize,
     };
   }
+
+  private convertCpfpCM(txid: string, cluster: CpfpCluster): CpfpInfo {
+    const clusterData = cluster.clusterData;
+    if (!clusterData) {
+      return { ancestors: [], descendants: [], effectiveFeePerVsize: 0 };
+    }
+
+    // Find which chunk this tx belongs to
+    let txFlatIdx = -1;
+    let txChunkIndex = -1;
+    for (let i = 0; i < clusterData.txs.length; i++) {
+      if (clusterData.txs[i].txid === txid) {
+        txFlatIdx = i;
+        break;
+      }
+    }
+
+    // Find the chunk containing this tx
+    for (let chunkIdx = 0; chunkIdx < clusterData.chunks.length; chunkIdx++) {
+      if (clusterData.chunks[chunkIdx].txs.includes(txFlatIdx)) {
+        txChunkIndex = chunkIdx;
+        break;
+      }
+    }
+
+    // Derive ancestors/descendants from in-chunk depgraph parents
+    // For CM, ancestors are the tx's depgraph parents within the cluster,
+    // descendants are txs that depend on this tx
+    const ancestors: Ancestor[] = [];
+    const descendants: Ancestor[] = [];
+
+    if (txFlatIdx >= 0) {
+      // Build child map
+      const childMap = new Map<number, number[]>();
+      for (let i = 0; i < clusterData.txs.length; i++) {
+        for (const parentIdx of clusterData.txs[i].parents) {
+          let children = childMap.get(parentIdx);
+          if (!children) {
+            children = [];
+            childMap.set(parentIdx, children);
+          }
+          children.push(i);
+        }
+      }
+
+      const ancestorSet = new Set<number>();
+      const stack = [...clusterData.txs[txFlatIdx].parents];
+      while (stack.length) {
+        const idx = stack.pop();
+        if (idx === undefined || ancestorSet.has(idx)) {
+          continue;
+        }
+        ancestorSet.add(idx);
+        stack.push(...clusterData.txs[idx].parents);
+      }
+
+      const descendantSet = new Set<number>();
+      const dStack = [...(childMap.get(txFlatIdx) || [])];
+      while (dStack.length) {
+        const idx = dStack.pop();
+        if (idx === undefined || descendantSet.has(idx)) {
+          continue;
+        }
+        descendantSet.add(idx);
+        dStack.push(...(childMap.get(idx) || []));
+      }
+
+      for (const idx of ancestorSet) {
+        const tx = clusterData.txs[idx];
+        ancestors.push({ txid: tx.txid, weight: tx.weight, fee: tx.fee });
+      }
+      for (const idx of descendantSet) {
+        const tx = clusterData.txs[idx];
+        descendants.push({ txid: tx.txid, weight: tx.weight, fee: tx.fee });
+      }
+    }
+
+    const effectiveFeePerVsize = txChunkIndex >= 0 ? clusterData.chunks[txChunkIndex].feerate : 0;
+
+    return {
+      ancestors,
+      descendants,
+      effectiveFeePerVsize,
+      cluster: {
+        ...clusterData,
+        chunkIndex: txChunkIndex,
+      },
+    };
+  }
 }
 
 export default new TransactionRepository();

backend/src/rpc-api/commands.ts

@@ -94,4 +94,5 @@ module.exports = {
   getTxoutSetinfo: 'gettxoutsetinfo',
   getIndexInfo: 'getindexinfo',
   testMempoolAccept: 'testmempoolaccept',
+  tweakFedPegScript: 'tweakfedpegscript',
 };

backend/src/utils/bitcoin-script.ts

@@ -224,4 +224,27 @@ export function parseDATUMTemplateCreator(coinbaseRaw: string): string[] | null
   tagString = tagString.replace('\x00', '');
 
   return tagString.split('\x0f').map((name) => name.replace(/[^a-zA-Z0-9 ]/g, ''));
-}
+}
+
+/** Extracts miner names from a DMND coinbase transaction */
+export function parseDMNDTemplateCreator(coinbaseRaw: string): string[] {
+  try {
+    if (!coinbaseRaw || coinbaseRaw.length % 2 !== 0 || !/^[0-9a-fA-F]+$/.test(coinbaseRaw)) {
+      return [];
+    }
+
+    const bytes = Buffer.from(coinbaseRaw, 'hex');
+    const blockHeightLength = bytes[0];
+    const tagDelimiterIndex = 1 + blockHeightLength;
+    if (bytes.length <= tagDelimiterIndex || bytes[tagDelimiterIndex] !== 0x00) {
+      return [];
+    }
+
+    const tagStart = tagDelimiterIndex + 1;
+    const tagEnd = bytes.indexOf(0x00, tagStart);
+    const tags = bytes.subarray(tagStart, tagEnd === -1 ? undefined : tagEnd);
+    return tags.toString('utf8').split('/').slice(1, -1);
+  } catch {
+    return [];
+  }
+}

backend/tsconfig.build.json

@@ -1,6 +1,6 @@
 {
   "extends": "./tsconfig",
-  "exclude": ["**/*.test.*", "**/__mocks__/*", "**/__tests__/*"],
+  "exclude": ["**/*.test.*", "**/__mocks__/*", "**/__tests__/*", "**/__e2e__/*"],
   "compilerOptions": {
     "types": ["node"]
   },

docker/backend/Dockerfile

@@ -19,7 +19,7 @@ ENV PATH="/usr/local/cargo/bin:$PATH"
 COPY --from=backend . .
 COPY --from=rustgbt . ../rust/
 ENV FD=/build/rust-gbt
-RUN npm install --omit=dev --omit=optional
+RUN bash meta/scripts/check-install-scripts.sh && npm ci --omit=dev --omit=optional
 
 WORKDIR /build
 RUN npm run package

docker/backend/mempool-config.json

@@ -29,6 +29,8 @@
     "AUDIT": __MEMPOOL_AUDIT__,
     "RUST_GBT": __MEMPOOL_RUST_GBT__,
     "LIMIT_GBT": __MEMPOOL_LIMIT_GBT__,
+    "CLUSTER_MEMPOOL": __MEMPOOL_CLUSTER_MEMPOOL__,
+    "CLUSTER_MEMPOOL_INDEXING": __MEMPOOL_CLUSTER_MEMPOOL_INDEXING__,
     "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
     "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__,
     "DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__,

docker/backend/start.sh

@@ -33,6 +33,8 @@ __MEMPOOL_POOLS_UPDATE_DELAY__=${MEMPOOL_POOLS_UPDATE_DELAY:=604800}
 __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false}
 __MEMPOOL_RUST_GBT__=${MEMPOOL_RUST_GBT:=true}
 __MEMPOOL_LIMIT_GBT__=${MEMPOOL_LIMIT_GBT:=false}
+__MEMPOOL_CLUSTER_MEMPOOL__=${MEMPOOL_CLUSTER_MEMPOOL:=false}
+__MEMPOOL_CLUSTER_MEMPOOL_INDEXING__=${MEMPOOL_CLUSTER_MEMPOOL_INDEXING:=false}
 __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
 __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
 __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6}
@@ -197,6 +199,8 @@ sed -i "s!__MEMPOOL_POOLS_UPDATE_DELAY__!${__MEMPOOL_POOLS_UPDATE_DELAY__}!g" me
 sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_LIMIT_GBT__!${__MEMPOOL_LIMIT_GBT__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_CLUSTER_MEMPOOL__!${__MEMPOOL_CLUSTER_MEMPOOL__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_CLUSTER_MEMPOOL_INDEXING__!${__MEMPOOL_CLUSTER_MEMPOOL_INDEXING__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json

docker/frontend/Dockerfile

@@ -9,11 +9,11 @@ COPY . .
 RUN apt-get update
 RUN apt-get install -y build-essential rsync
 RUN cp mempool-frontend-config.sample.json mempool-frontend-config.json
-RUN npm install --omit=dev --omit=optional
+RUN bash meta/scripts/check-install-scripts.sh && npm ci --omit=dev --omit=optional
 
 RUN npm run build
 
-FROM nginx:1.27.0-alpine
+FROM nginx:1.30.1-alpine
 
 WORKDIR /patch
 
@@ -21,6 +21,7 @@ COPY --from=builder /build/entrypoint.sh .
 COPY --from=builder /build/wait-for .
 COPY --from=builder /build/dist/mempool /var/www/mempool
 COPY --from=builder /build/nginx.conf /etc/nginx/
+COPY --from=builder /build/http-basic.conf /etc/nginx/
 COPY --from=builder /build/nginx-mempool.conf /etc/nginx/conf.d/
 
 RUN chmod +x /patch/entrypoint.sh
@@ -30,6 +31,7 @@ RUN chown -R 1000:1000 /patch && chmod -R 755 /patch && \
         chown -R 1000:1000 /var/cache/nginx && \
         chown -R 1000:1000 /var/log/nginx && \
         chown -R 1000:1000 /etc/nginx/nginx.conf && \
+        chown -R 1000:1000 /etc/nginx/http-basic.conf && \
         chown -R 1000:1000 /etc/nginx/conf.d && \
         chown -R 1000:1000 /var/www/mempool
 

docker/frontend/entrypoint.sh

@@ -25,7 +25,7 @@ fi
 
 __MAINNET_ENABLED__=${MAINNET_ENABLED:=true}
 __TESTNET_ENABLED__=${TESTNET_ENABLED:=false}
-__TESTNET4_ENABLED__=${TESTNET_ENABLED:=false}
+__TESTNET4_ENABLED__=${TESTNET4_ENABLED:=false}
 __SIGNET_ENABLED__=${SIGNET_ENABLED:=false}
 __REGTEST_ENABLED__=${REGTEST_ENABLED:=false}
 __LIQUID_ENABLED__=${LIQUID_ENABLED:=false}
@@ -48,6 +48,12 @@ __MAINNET_BLOCK_AUDIT_START_HEIGHT__=${MAINNET_BLOCK_AUDIT_START_HEIGHT:=0}
 __TESTNET_BLOCK_AUDIT_START_HEIGHT__=${TESTNET_BLOCK_AUDIT_START_HEIGHT:=0}
 __SIGNET_BLOCK_AUDIT_START_HEIGHT__=${SIGNET_BLOCK_AUDIT_START_HEIGHT:=0}
 __REGTEST_BLOCK_AUDIT_START_HEIGHT__=${REGTEST_BLOCK_AUDIT_START_HEIGHT:=0}
+__TESTNET4_BLOCK_AUDIT_START_HEIGHT__=${TESTNET4_BLOCK_AUDIT_START_HEIGHT:=0}
+__MAINNET_TX_FIRST_SEEN_START_HEIGHT__=${MAINNET_TX_FIRST_SEEN_START_HEIGHT:=0}
+__TESTNET_TX_FIRST_SEEN_START_HEIGHT__=${TESTNET_TX_FIRST_SEEN_START_HEIGHT:=0}
+__TESTNET4_TX_FIRST_SEEN_START_HEIGHT__=${TESTNET4_TX_FIRST_SEEN_START_HEIGHT:=0}
+__SIGNET_TX_FIRST_SEEN_START_HEIGHT__=${SIGNET_TX_FIRST_SEEN_START_HEIGHT:=0}
+__REGTEST_TX_FIRST_SEEN_START_HEIGHT__=${REGTEST_TX_FIRST_SEEN_START_HEIGHT:=0}
 __ACCELERATOR__=${ACCELERATOR:=false}
 __ACCELERATOR_BUTTON__=${ACCELERATOR_BUTTON:=true}
 __SERVICES_API__=${SERVICES_API:=https://mempool.space/api/v1/services}
@@ -82,6 +88,12 @@ export __MAINNET_BLOCK_AUDIT_START_HEIGHT__
 export __TESTNET_BLOCK_AUDIT_START_HEIGHT__
 export __SIGNET_BLOCK_AUDIT_START_HEIGHT__
 export __REGTEST_BLOCK_AUDIT_START_HEIGHT__
+export __TESTNET4_BLOCK_AUDIT_START_HEIGHT__
+export __MAINNET_TX_FIRST_SEEN_START_HEIGHT__
+export __TESTNET_TX_FIRST_SEEN_START_HEIGHT__
+export __TESTNET4_TX_FIRST_SEEN_START_HEIGHT__
+export __SIGNET_TX_FIRST_SEEN_START_HEIGHT__
+export __REGTEST_TX_FIRST_SEEN_START_HEIGHT__
 export __ACCELERATOR__
 export __ACCELERATOR_BUTTON__
 export __SERVICES_API__

docker/init.sh

@@ -12,6 +12,7 @@ wget -O ./backend/GeoIP/GeoLite2-ASN.mmdb https://raw.githubusercontent.com/memp
 localhostIP="127.0.0.1"
 cp ./docker/frontend/* ./frontend
 cp ./nginx.conf ./frontend/
+cp ./http-basic.conf ./frontend/
 cp ./nginx-mempool.conf ./frontend/
 sed -i"" -e "s/${localhostIP}:80/0.0.0.0:__MEMPOOL_FRONTEND_HTTP_PORT__/g" ./frontend/nginx.conf
 sed -i"" -e "s/${localhostIP}/0.0.0.0/g" ./frontend/nginx.conf

frontend/.npmrc

@@ -0,0 +1 @@
+min-release-age=7

frontend/cypress/e2e/liquid/liquid.spec.ts

@@ -113,7 +113,7 @@ describe('Liquid', () => {
       it('allows searching assets', () => {
         cy.visit(`${basePath}/assets`);
         cy.waitForSkeletonGone();
-        cy.get('.container-xl input').click().type('Liquid Bitcoin').then(() => {
+        cy.get('.container-xl input').click().type('Tether USD').then(() => {
           cy.get('ngb-typeahead-window', { timeout: 30000 }).should('have.length', 1);
         });
       });

frontend/cypress/e2e/liquidtestnet/liquidtestnet.spec.ts

@@ -79,7 +79,7 @@ describe('Liquid Testnet', () => {
       it('allows searching assets', () => {
         cy.visit(`${basePath}/assets`);
         cy.waitForSkeletonGone();
-        cy.get('.container-xl input').click().type('Liquid Bitcoin').then(() => {
+        cy.get('.container-xl input').click().type('Tether USD').then(() => {
           cy.get('ngb-typeahead-window').should('have.length', 1);
         });
       });

frontend/cypress/e2e/mainnet/calculator.spec.ts

@@ -83,22 +83,28 @@ describe('Calculator', () => {
     });
 
     describe('bitcoin input updates fiat and sats', () => {
-      it('updates fiat and sats when entering 0.5 BTC', () => {
-        const expectedFiat = Math.round(MOCK_BTC_PRICE_USD * 0.5 * 100) / 100;
-        cy.get('input[formControlName="bitcoin"]').clear().type('0.5');
-        cy.get('input[formControlName="satoshis"]').invoke('val').should('equal', '50000000');
+      it('updates fiat and sats when entering 0.33 BTC', () => {
+        const expectedFiat = Math.round(MOCK_BTC_PRICE_USD * 0.33 * 100) / 100; // 40740.48 USD
+        cy.get('input[formControlName="bitcoin"]').clear().type('0.33');
+        cy.get('input[formControlName="satoshis"]').invoke('val').should('equal', '33000000');
         cy.get('input[formControlName="fiat"]').invoke('val').then((fiatVal) => {
           const fiat = parseFloat(String(fiatVal).replace(/,/g, ''));
           expect(fiat).to.equal(expectedFiat);
         });
+        cy.get('.fiat-text').invoke('text').then((text) => {
+          expect(text.trim()).to.equal('$40,740.48');
+        });
       });
 
       it('updates fiat and sats when entering 1 sat (0.00000001 BTC)', () => {
-        const expectedFiat = (MOCK_BTC_PRICE_USD / 100_000_000 * 100 / 100).toFixed(8);
+        const expectedFiat = (MOCK_BTC_PRICE_USD / 100_000_000 * 100 / 100);
         cy.get('input[formControlName="bitcoin"]').clear().type('0.00000001');
         cy.get('input[formControlName="satoshis"]').invoke('val').should('equal', '1');
         cy.get('input[formControlName="fiat"]').invoke('val').then((fiatVal) => {
-          expect(String(fiatVal)).to.equal(expectedFiat);
+          expect(String(fiatVal)).to.equal(String(expectedFiat));
+        });
+        cy.get('.fiat-text').invoke('text').then((text) => {
+          expect(text.trim()).to.equal('$0.00');
         });
       });
     });
@@ -220,6 +226,9 @@ describe('Calculator', () => {
         cy.get('.symbol').invoke('text').then((text) => {
           expect(text.replace(/,/g, '')).to.include(String(MOCK_BTC_PRICE_JPY));
         });
+        cy.get('.fiat-text').invoke('text').then((text) => {
+          expect(text.trim()).to.equal('¥11,057,757');
+        });
       });
 
       it('displays 1 BTC with correct sats and fiat in JPY', () => {
@@ -234,13 +243,17 @@ describe('Calculator', () => {
       });
 
       it('updates fiat and sats when entering 0.5 BTC in JPY', () => {
-        const expectedFiat = Math.round(MOCK_BTC_PRICE_JPY * 0.5 * 100) / 100;
+        const expectedFiat = Math.round(MOCK_BTC_PRICE_JPY * 0.5)
         cy.get('input[formControlName="bitcoin"]').clear().type('0.5');
         cy.get('input[formControlName="satoshis"]').invoke('val').should('equal', '50000000');
         cy.get('input[formControlName="fiat"]').invoke('val').then((fiatVal) => {
           const fiat = parseFloat(String(fiatVal).replace(/,/g, ''));
           expect(fiat).to.equal(expectedFiat);
         });
+        cy.get('.fiat-text').invoke('text').then((text) => {
+          expect(text.trim()).to.equal('¥5,528,879');
+        });
+
       });
 
       it('updates BTC and sats when entering fiat value in JPY', () => {
@@ -258,13 +271,16 @@ describe('Calculator', () => {
 
       it('updates BTC and fiat when entering 10000 sats in JPY', () => {
         const satsAmount = 10000;
-        const expectedFiat = Math.round((satsAmount / 100_000_000) * MOCK_BTC_PRICE_JPY * 100) / 100;
+        const expectedFiat = Math.round((satsAmount / 100_000_000) * MOCK_BTC_PRICE_JPY);
         cy.get('input[formControlName="satoshis"]').clear().type(String(satsAmount));
         cy.get('input[formControlName="bitcoin"]').invoke('val').should('equal', '0.00010000');
         cy.get('input[formControlName="fiat"]').invoke('val').then((fiatVal) => {
           const fiat = parseFloat(String(fiatVal).replace(/,/g, ''));
           expect(fiat).to.equal(expectedFiat);
         });
+        cy.get('.fiat-text').invoke('text').then((text) => {
+          expect(text.trim()).to.equal('¥1,106');
+        });
       });
     });
 

frontend/cypress/e2e/mainnet/fiat-currency-formatting.spec.ts

@@ -0,0 +1,275 @@
+const baseModule = Cypress.env('BASE_MODULE');
+
+// Helper to select "Fiat" from the BTC/sats/Fiat amount selector dropdown
+const selectFiatMode = () => {
+  cy.get('app-amount-selector').first().scrollIntoView();
+  cy.get('app-amount-selector select').first().select('fiat');
+};
+
+// Helper to select currency from the fiat selector dropdown
+const selectCurrency = (currency: 'USD' | 'JPY') => {
+  cy.get('app-fiat-selector').first().scrollIntoView();
+  cy.get('app-fiat-selector').first().click();
+  cy.get('app-fiat-selector select').first().select(currency);
+};
+
+describe('Fiat Currency Formatting', () => {
+
+  if (baseModule === 'mempool') {
+
+    describe('Dashboard', () => {
+      beforeEach(() => {
+        cy.visit('/');
+        cy.waitForSkeletonGone();
+        selectFiatMode();
+        cy.get('.latest-transactions', { timeout: 10000 }).should('exist');
+        cy.get('.table-cell-fiat', { timeout: 10000 }).should('have.length.at.least', 1);
+      });
+
+      describe('USD formatting', () => {
+        beforeEach(() => {
+          selectCurrency('USD');
+        });
+
+        it('displays USD values with correct currency symbol', () => {
+          cy.get('.table-cell-fiat').eq(1).invoke('text').then((text) => {
+            const trimmedText = text.trim();
+            expect(trimmedText).to.include('$');
+          });
+        });
+
+        it('displays USD values with proper decimal format', () => {
+          cy.get('.table-cell-fiat').eq(1).invoke('text').then((text) => {
+            const trimmedText = text.trim();
+            expect(trimmedText).to.match(/^\$[\d,]+(\.\d{2})?$/);
+          });
+        });
+      });
+
+      describe('JPY formatting', () => {
+        beforeEach(() => {
+          selectCurrency('JPY');
+          cy.get('.table-cell-fiat').eq(1).should(($el) => {
+            expect($el.text()).to.include('¥');
+          });
+        });
+
+        it('displays JPY values with yen symbol', () => {
+          cy.get('.table-cell-fiat').eq(1).invoke('text').then((text) => {
+            const trimmedText = text.trim();
+            expect(trimmedText).to.include('¥');
+          });
+        });
+
+        it('displays JPY values without decimal places', () => {
+          cy.get('.table-cell-fiat').eq(1).invoke('text').then((text) => {
+            const trimmedText = text.trim();
+            expect(trimmedText).to.not.match(/\.\d+$/);
+          });
+        });
+
+        it('formats all JPY values correctly without decimals', () => {
+          cy.get('.table-cell-fiat').each(($el) => {
+            const text = $el.text().trim();
+            if (text.includes('¥')) {
+              expect(text).to.not.match(/\.\d+/);
+            }
+          });
+        });
+      });
+
+      describe('currency switching', () => {
+        it('correctly formats when switching from USD to JPY', () => {
+          selectCurrency('USD');
+          cy.get('.table-cell-fiat').eq(1).invoke('text').then((usdText) => {
+            expect(usdText.trim()).to.include('$');
+          });
+
+          selectCurrency('JPY');
+          cy.get('.table-cell-fiat').eq(1).should(($el) => {
+            const text = $el.text().trim();
+            expect(text).to.include('¥');
+            expect(text).to.not.match(/\.\d+$/);
+          });
+        });
+
+        it('correctly formats when switching from JPY back to USD', () => {
+          selectCurrency('JPY');
+          cy.get('.table-cell-fiat').eq(1).should(($el) => {
+            expect($el.text()).to.include('¥');
+          });
+
+          selectCurrency('USD');
+          cy.get('.table-cell-fiat').eq(1).should(($el) => {
+            expect($el.text().trim()).to.include('$');
+          });
+        });
+      });
+    });
+
+    describe('Transaction Page', () => {
+      beforeEach(() => {
+        cy.visit('/tx/dd0faea1e9acd5bd812e5130c0912b62d6b63d04bac4558f2e07270ac613a8f2');
+        cy.waitForSkeletonGone();
+      });
+
+      it('displays USD fiat values with dollar symbol', () => {
+        selectCurrency('USD');
+        selectFiatMode();
+
+        cy.get('app-transaction-details .fiat').invoke('text').then((text) => {
+          expect(text.trim()).to.include('$');
+        });
+
+        cy.get('app-transactions-list .fiat', { timeout: 10000 }).first().invoke('text').then((text) => {
+          expect(text.trim()).to.include('$');
+        });
+      });
+
+      it('displays JPY fiat values without decimals', () => {
+        selectCurrency('JPY');
+        selectFiatMode();
+
+
+        cy.get('app-transaction-details .fiat').invoke('text').then((text) => {
+          const trimmedText = text.trim();
+          expect(trimmedText).to.include('¥');
+          expect(trimmedText).to.not.match(/\.\d+/);
+        });
+
+        cy.get('app-transactions-list .fiat').first().should(($el) => {
+          const text = $el.text().trim();
+          expect(text).to.include('¥');
+          expect(text).to.not.match(/\.\d+/);
+        });
+      });
+    });
+
+    describe('Block Page', () => {
+      beforeEach(() => {
+        cy.viewport(1080, 1920); // force taller screen to avoid scrollbars
+        cy.intercept('/api/v1/blocks/0').as('blocks-0');
+        cy.intercept('/api/v1/blocks/10').as('blocks-10');
+        cy.intercept('/api/v1/blocks/20').as('blocks-20');
+        cy.intercept('/api/txs/outspends*').as('outspends');
+        cy.visit('/block/100000');
+      });
+
+      it('displays USD fiat values in block reward with dollar symbol', () => {
+        cy.wait('@outspends').then(() => {
+          cy.waitUntil(() => cy.get('.fiat').each(($el) => $el.is(':visible') && $el.text().trim().includes('$')));
+          selectCurrency('USD');
+          selectFiatMode();
+          cy.get('.fiat').each(($el) => {
+            expect($el.text().trim()).to.include('$');
+          });
+        });
+      });
+    
+    it('displays JPY fiat values without decimals', () => {
+        cy.wait('@outspends').then(() => {
+          cy.waitUntil(() => cy.get('.fiat').each(($el) => $el.is(':visible') && $el.text().trim().includes('$')));
+          selectCurrency('JPY');
+          selectFiatMode();
+          cy.get('.fiat').each(($el) => {
+            expect($el.text().trim()).to.include('¥');
+            expect($el.text().trim()).to.not.match(/\.\d+/);
+          });
+        });        
+    })
+  });
+  
+    describe('Address Page', () => {
+      beforeEach(() => {
+        cy.visit('/address/1wizaAB16Wrua9V58uNvqktyq2LBLtYso');
+      });
+
+      it('displays USD fiat values with dollar symbol', () => {
+        selectCurrency('USD');
+        selectFiatMode();
+        cy.get('app-amount .fiat', { timeout: 10000 }).each(($el) => {
+          expect($el.text().trim()).to.include('$');
+        });
+      });
+
+      it('displays JPY fiat values without decimals', () => {
+        selectCurrency('JPY');
+        selectFiatMode();
+        cy.get('app-amount .fiat').first().should(($el) => {
+          const text = $el.text().trim();
+          expect(text).to.include('¥');
+          expect(text).to.not.match(/\.\d+/);
+        });
+      });
+    });
+
+    describe('Calculator Page', () => {
+      beforeEach(() => {
+        cy.visit('/tools/calculator');
+        cy.waitForSkeletonGone();
+        cy.get('input[formControlName="bitcoin"]', { timeout: 10000 }).should('be.visible');
+      });
+
+      it('displays USD price with dollar symbol', () => {
+        selectCurrency('USD');
+        cy.get('.symbol').invoke('text').then((text) => {
+          expect(text.trim()).to.include('$');
+        });
+      });
+
+      it('displays JPY price without decimals after switching currency', () => {
+        selectCurrency('JPY');
+        cy.get('.symbol').should(($el) => {
+          const text = $el.text().trim();
+          expect(text).to.include('¥');
+        });
+
+        // Enter a value in BTC to see the fiat conversion
+        cy.get('input[formControlName="bitcoin"]').clear().type('1');
+
+        // Check that the fiat output doesn't have decimals for JPY
+        cy.get('input[formControlName="fiat"]').should(($el) => {
+          const value = $el.val() as string;
+          // JPY should not have decimal places in large values
+          expect(value).to.not.match(/\.\d+$/);
+        });
+      });
+    });
+
+    describe('Mempool Block Tooltip', () => {
+      beforeEach(() => {
+        cy.visit('/');
+        cy.waitForSkeletonGone();
+        selectFiatMode();
+      });
+
+      it('displays USD fiat values in mempool block', () => {
+        selectCurrency('USD');
+        cy.get('#mempool-block-0').scrollIntoView();
+        cy.get('#mempool-block-0').click();
+
+        cy.waitForSkeletonGone();
+
+        cy.get('app-mempool-block .fiat').each(($el) => {
+          expect($el.text().trim()).to.include('$');
+        });
+      });
+
+      it('displays JPY fiat values in mempool block', () => {
+        selectCurrency('JPY');
+        cy.get('#mempool-block-0').scrollIntoView();
+        cy.get('#mempool-block-0').click();
+
+        cy.waitForSkeletonGone();
+
+        cy.get('app-mempool-block .fiat').invoke('text').then((text) => {
+          expect(text.trim()).to.include('¥');
+          expect(text).to.not.match(/\.\d+$/);
+        });
+      });
+    });
+
+  } else {
+    it.skip(`Tests cannot be run on the selected BASE_MODULE ${baseModule}`);
+  }
+});

frontend/cypress/e2e/mainnet/recent-transactions.spec.ts

@@ -0,0 +1,158 @@
+import { mockWebSocketV2, receiveWebSocketMessageFromServer } from '../../support/websocket';
+
+const baseModule = Cypress.env('BASE_MODULE');
+
+function randomHex(length: number): string {
+  let result = '';
+  const chars = '0123456789abcdef';
+  for (let i = 0; i < length; i++) {
+    result += chars[Math.floor(Math.random() * chars.length)];
+  }
+  return result;
+}
+
+function generateMockTransactions(count: number): any[] {
+  const txs = [];
+  for (let i = 0; i < count; i++) {
+    txs.push({
+      txid: randomHex(64),
+      fee: 1000 + Math.floor(Math.random() * 50000),
+      vsize: 140 + Math.floor(Math.random() * 500),
+      value: 10000 + Math.floor(Math.random() * 10000000),
+    });
+  }
+  return txs;
+}
+
+function sendMockTransactions(count: number): string[] {
+  const txs = generateMockTransactions(count);
+  receiveWebSocketMessageFromServer({
+    params: {
+      message: {
+        contents: JSON.stringify({ transactions: txs }),
+      },
+    },
+  });
+  return txs.map((tx) => tx.txid);
+}
+
+// Send init fixtures without waitForSkeletonGone (the /txs page has a
+// skeleton row that only disappears once transactions fill the limit,
+// which won't happen from mempool-info alone).
+function initMempoolData(): void {
+  cy.window({ timeout: 5000 })
+    .should((win) => {
+      expect(win.mockSocket).to.not.be.undefined;
+    })
+    .then((win) => {
+      cy.readFile('cypress/fixtures/mainnet_live2hchart.json', 'utf-8').then((fixture) => {
+        win.mockSocket.send(JSON.stringify(fixture));
+      });
+      cy.readFile('cypress/fixtures/mainnet_mempoolInfo.json', 'utf-8').then((fixture) => {
+        win.mockSocket.send(JSON.stringify(fixture));
+      });
+    });
+}
+
+describe('Recent Transactions Page', () => {
+  if (baseModule === 'mempool') {
+
+    it('updates the transaction list over time', () => {
+      mockWebSocketV2();
+      cy.visit('/txs');
+      initMempoolData();
+
+      sendMockTransactions(6);
+      cy.get('[data-cy="transactions-list"] tr').should('have.length.greaterThan', 0);
+
+      const newTxids = sendMockTransactions(6);
+      const markerTxid = newTxids[0].substring(0, 10);
+
+      cy.get('[data-cy="transactions-list"] tr .table-cell-txid a').should(($rows) => {
+        const visibleText = [...$rows].map((el) => el.textContent.trim()).join(' ');
+        expect(visibleText).to.include(markerTxid);
+      });
+    });
+
+    it('pauses updates when clicking the pause icon', () => {
+      mockWebSocketV2();
+      cy.visit('/txs');
+      initMempoolData();
+
+      sendMockTransactions(6);
+      cy.get('[data-cy="transactions-list"] tr').should('have.length.greaterThan', 0);
+
+      cy.get('[data-cy="btn-pause"]').click();
+
+      cy.get('[data-cy="transactions-list"] tr .table-cell-txid a').then(($rows) => {
+        const pausedTxids = [...$rows].map((el) => el.textContent.trim());
+
+        const newTxids = sendMockTransactions(6);
+        const markerTxid = newTxids[0].substring(0, 10);
+
+        // The new txid should NOT appear in the list while paused
+        cy.wait(1000);
+        cy.get('[data-cy="transactions-list"] tr .table-cell-txid a').should(($updatedRows) => {
+          const visibleText = [...$updatedRows].map((el) => el.textContent.trim()).join(' ');
+          expect(visibleText).to.not.include(markerTxid);
+          const updatedTxids = [...$updatedRows].map((el) => el.textContent.trim());
+          expect(updatedTxids).to.deep.equal(pausedTxids);
+        });
+      });
+    });
+
+    it('caps the list when changing the limit to 10', () => {
+      mockWebSocketV2();
+      cy.visit('/txs');
+      initMempoolData();
+
+      // Send enough batches to fill 50 transactions (6 per batch, need 9 batches)
+      for (let i = 0; i < 9; i++) {
+        sendMockTransactions(6);
+      }
+
+      cy.get('[data-cy="transactions-list"] tr').should('have.length', 50);
+      cy.get('[data-cy="limit-10"]').click();
+      cy.scrollTo('top');
+      cy.get('[data-cy="transactions-list"] tr').should('have.length', 10);
+    });
+
+    it('shows the new transaction pill when there are new transactions', () => {
+      mockWebSocketV2();
+      cy.visit('/txs');
+      initMempoolData();
+
+      sendMockTransactions(6);
+      cy.get('[data-cy="transactions-list"] tr').should('have.length.greaterThan', 0);
+      cy.scrollTo('bottom');
+      // Ensure scroll event has fired and auto-pause is active
+      cy.window().should((win) => {
+        expect(win.scrollY).to.be.greaterThan(0);
+      });
+
+      sendMockTransactions(6);
+      cy.get('[data-cy="new-tx-pill"]').should('be.visible');
+    });
+
+    it('shows the new transaction pill when there are new transactions and scrolls to the top when clicked', () => {
+      mockWebSocketV2();
+      cy.visit('/txs');
+      initMempoolData();
+
+      sendMockTransactions(6);
+      cy.get('[data-cy="transactions-list"] tr').should('have.length.greaterThan', 0);
+      cy.scrollTo('bottom');
+      cy.window().should((win) => {
+        expect(win.scrollY).to.be.greaterThan(0);
+      });
+
+      sendMockTransactions(6);
+      cy.get('[data-cy="new-tx-pill"]').should('be.visible');
+      cy.get('[data-cy="new-tx-pill"]').click();
+      cy.wait(1000);
+      cy.window().then((win) => {
+        expect(win.scrollY).to.be.eq(0);
+      });
+    });
+  }
+});

frontend/generate-themes.js

@@ -3,7 +3,7 @@ const fs = require('fs');
 const path = require('path');
 const crypto = require('crypto');
 
-const THEMES = ['contrast', 'softsimon', 'bukele'];
+const THEMES = ['contrast', 'softsimon', 'bukele', 'nymkappa'];
 const STAGING_DIR = path.join(__dirname, '.theme-build');
 const DIST_DIR = path.join(__dirname, 'dist/mempool/browser');
 const MANIFEST_FILE = path.join(__dirname, 'theme-manifest.json');

frontend/meta/scripts/check-install-scripts.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+#
+# Audit frontend package-lock.json for unexpected hasInstallScript entries.
+# Exits non-zero if any package not on the whitelist has hasInstallScript: true.
+#
+# Usage (from repo root):
+#   frontend/meta/scripts/check-install-scripts.sh
+#
+
+set -euo pipefail
+
+FRONTEND_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+LOCKFILE="${FRONTEND_DIR}/package-lock.json"
+
+ALLOWED=(
+  "mempool-frontend"
+  "@parcel/watcher"
+  "cypress"
+  "esbuild"
+  "fsevents"
+  "lmdb"
+  "msgpackr-extract"
+)
+
+is_allowed() {
+  local pkg="$1"
+  for allowed in "${ALLOWED[@]}"; do
+    if [[ "$pkg" == "$allowed" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+if [[ ! -f "$LOCKFILE" ]]; then
+  echo "No package-lock.json found at ${LOCKFILE}"
+  exit 1
+fi
+
+found_violations=0
+
+violations=$(
+  LOCKFILE_PATH="$LOCKFILE" node - <<'NODE'
+const fs = require('fs');
+
+const lock = JSON.parse(fs.readFileSync(process.env.LOCKFILE_PATH, 'utf8'));
+const pkgs = lock.packages || {};
+
+for (const [path, meta] of Object.entries(pkgs)) {
+  if (meta && meta.hasInstallScript) {
+    const name = path === '' ? (lock.name || '(root)') : path.replace(/^.*node_modules\//, '');
+    console.log(name);
+  }
+}
+NODE
+)
+
+while IFS= read -r pkg; do
+  [[ -z "$pkg" ]] && continue
+  if ! is_allowed "$pkg"; then
+    echo "VIOLATION: unauthorized install script in '${pkg}' (${LOCKFILE})"
+    found_violations=1
+  fi
+done <<< "$violations"
+
+if [[ "$found_violations" -eq 1 ]]; then
+  echo ""
+  echo "FAILED: Found packages with install scripts not on the whitelist."
+  echo "If this is a legitimate new dependency, add it to frontend/meta/scripts/check-install-scripts.sh"
+  exit 1
+else
+  echo "OK: All frontend install scripts are from whitelisted packages."
+fi

frontend/meta/scripts/safe-install.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+#
+# Safely install frontend npm dependencies by refreshing the lockfile without
+# running install scripts, auditing it, then doing the real install.
+#
+# Usage (from repo root):
+#   frontend/meta/scripts/safe-install.sh
+#
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+FRONTEND_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+LOCKFILE="${FRONTEND_DIR}/package-lock.json"
+RESTORE_LOCKFILE_DONE=0
+
+# Back up the lockfile so we can restore on failure
+if [[ -f "$LOCKFILE" ]]; then
+  cp "$LOCKFILE" "${LOCKFILE}.bak"
+fi
+
+restore_lockfile() {
+  trap - ERR INT TERM
+  if [[ "${RESTORE_LOCKFILE_DONE}" -eq 1 ]]; then
+    return
+  fi
+  RESTORE_LOCKFILE_DONE=1
+
+  if [[ -f "${LOCKFILE}.bak" ]]; then
+    mv "${LOCKFILE}.bak" "$LOCKFILE"
+    echo "Restored original package-lock.json."
+  elif [[ -f "$LOCKFILE" ]]; then
+    rm "$LOCKFILE"
+    echo "Removed generated package-lock.json."
+  fi
+}
+
+trap restore_lockfile ERR INT TERM
+
+echo "==> Refreshing frontend lockfile (--ignore-scripts --package-lock-only)..."
+(cd "$FRONTEND_DIR" && npm install --ignore-scripts --package-lock-only --no-audit --no-fund)
+
+echo ""
+echo "==> Auditing lockfile for install scripts..."
+bash "${SCRIPT_DIR}/check-install-scripts.sh"
+
+trap - ERR INT TERM
+rm -f "${LOCKFILE}.bak"
+
+echo ""
+echo "==> Installing frontend (npm ci)..."
+(cd "$FRONTEND_DIR" && npm ci)
+
+echo ""
+echo "Done."

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-frontend",
-  "version": "3.3-dev",
+  "version": "3.4-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",
@@ -59,30 +59,30 @@
     "cypress:run:ci:parameterized": "node update-config.js TESTNET_ENABLED=true TESTNET4_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true ITEMS_PER_PAGE=25 && npm run generate-config && start-server-and-test serve:parameterized 4200 cypress:run:record"
   },
   "dependencies": {
-    "@angular-devkit/build-angular": "^20.3.16",
-    "@angular/animations": "^20.3.16",
-    "@angular/cli": "^20.3.16",
-    "@angular/common": "^20.3.16",
-    "@angular/compiler": "^20.3.16",
-    "@angular/core": "^20.3.16",
-    "@angular/forms": "^20.3.16",
-    "@angular/localize": "^20.3.16",
-    "@angular/platform-browser": "^20.3.16",
-    "@angular/platform-browser-dynamic": "^20.3.16",
-    "@angular/platform-server": "^20.3.16",
-    "@angular/router": "^20.3.16",
-    "@angular/ssr": "^20.3.16",
+    "@angular-devkit/build-angular": "^20.3.25",
+    "@angular/animations": "^20.3.25",
+    "@angular/cli": "^20.3.25",
+    "@angular/common": "^20.3.25",
+    "@angular/compiler": "^20.3.25",
+    "@angular/core": "^20.3.25",
+    "@angular/forms": "^20.3.25",
+    "@angular/localize": "^20.3.25",
+    "@angular/platform-browser": "^20.3.25",
+    "@angular/platform-browser-dynamic": "^20.3.25",
+    "@angular/platform-server": "^20.3.25",
+    "@angular/router": "^20.3.25",
+    "@angular/ssr": "^20.3.25",
     "@fortawesome/angular-fontawesome": "^3.0.0",
     "@fortawesome/fontawesome-common-types": "~6.7.2",
     "@fortawesome/fontawesome-svg-core": "~6.7.2",
     "@fortawesome/free-solid-svg-icons": "~6.7.2",
     "@ng-bootstrap/ng-bootstrap": "^19.0.0",
     "@types/qrcode": "~1.5.0",
-    "@noble/secp256k1": "^3.0.0",
-    "bootstrap": "~4.6.2",
+    "@noble/secp256k1": "^3.1.0",
+    "bootstrap": "~5.3.8",
     "clipboard": "^2.0.11",
     "domino": "^2.1.6",
-    "echarts": "~5.4.0",
+    "echarts": "~6.1.0",
     "ngx-echarts": "~20.0.2",
     "ngx-infinite-scroll": "^20.0.0",
     "qrcode": "1.5.1",
@@ -91,8 +91,8 @@
     "zone.js": "~0.15.1"
   },
   "devDependencies": {
-    "@angular/compiler-cli": "^20.3.16",
-    "@angular/language-service": "^20.3.16",
+    "@angular/compiler-cli": "^20.3.25",
+    "@angular/language-service": "^20.3.25",
     "@types/node": "^24.9.2",
     "@typescript-eslint/eslint-plugin": "^8.46.2",
     "@typescript-eslint/parser": "^8.46.2",
@@ -106,7 +106,7 @@
   },
   "optionalDependencies": {
     "@cypress/schematic": "^2.5.0",
-    "cypress": "^15.9.0",
+    "cypress": "^15.16.0",
     "cypress-fail-on-console-error": "~5.1.1",
     "cypress-wait-until": "^3.0.1",
     "mock-socket": "~9.3.1",

frontend/src/app/app-routing.module.ts

@@ -9,11 +9,10 @@ import { StatusViewComponent } from '@components/status-view/status-view.compone
 import { AddressGroupComponent } from '@components/address-group/address-group.component';
 import { TrackerGuard } from '@app/route-guards';
 
-const browserWindow = window || {};
-// @ts-ignore
+const browserWindow = window as typeof window & { __env?: any };
 const browserWindowEnv = browserWindow.__env || {};
 
-let routes: Routes = [
+const testnetRoutes: Routes = browserWindowEnv.TESTNET_ENABLED ? [
   {
     path: 'testnet',
     children: [
@@ -52,6 +51,9 @@ let routes: Routes = [
       },
     ]
   },
+] : [];
+
+const testnet4Routes: Routes = browserWindowEnv.TESTNET4_ENABLED ? [
   {
     path: 'testnet4',
     children: [
@@ -90,6 +92,9 @@ let routes: Routes = [
       },
     ]
   },
+] : [];
+
+const signetRoutes: Routes = browserWindowEnv.SIGNET_ENABLED ? [
   {
     path: 'signet',
     children: [
@@ -133,6 +138,9 @@ let routes: Routes = [
       },
     ]
   },
+] : [];
+
+const regtestRoutes: Routes = browserWindowEnv.REGTEST_ENABLED ? [
   {
     path: 'regtest',
     children: [
@@ -176,6 +184,13 @@ let routes: Routes = [
       },
     ]
   },
+] : [];
+
+let routes: Routes = [
+  ...testnetRoutes,
+  ...testnet4Routes,
+  ...signetRoutes,
+  ...regtestRoutes,
   {
     path: '',
     pathMatch: 'full',
@@ -262,46 +277,50 @@ let routes: Routes = [
   },
 ];
 
+const liquidTestnetRoutes: Routes = browserWindowEnv.LIQUID_TESTNET_ENABLED ? [
+  {
+    path: 'testnet',
+    children: [
+      {
+        path: '',
+        pathMatch: 'full',
+        loadChildren: () => import('@app/liquid/liquid-graphs.module').then(m => m.LiquidGraphsModule),
+        data: { preload: true },
+      },
+      {
+        path: '',
+        loadChildren: () => import ('@app/liquid/liquid-master-page.module').then(m => m.LiquidMasterPageModule),
+        data: { preload: true },
+      },
+      {
+        path: 'widget/wallet',
+        children: [],
+        component: AddressGroupComponent,
+        data: {
+          networkSpecific: true,
+        }
+      },
+      {
+        path: 'status',
+        data: { networks: ['bitcoin', 'liquid'] },
+        component: StatusViewComponent
+      },
+      {
+        path: '',
+        loadChildren: () => import('@app/liquid/liquid-graphs.module').then(m => m.LiquidGraphsModule),
+        data: { preload: true },
+      },
+      {
+        path: '**',
+        redirectTo: '/testnet'
+      },
+    ]
+  },
+] : [];
+
 if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
   routes = [
-    {
-      path: 'testnet',
-      children: [
-        {
-          path: '',
-          pathMatch: 'full',
-          loadChildren: () => import('@app/liquid/liquid-graphs.module').then(m => m.LiquidGraphsModule),
-          data: { preload: true },
-        },
-        {
-          path: '',
-          loadChildren: () => import ('@app/liquid/liquid-master-page.module').then(m => m.LiquidMasterPageModule),
-          data: { preload: true },
-        },
-        {
-          path: 'widget/wallet',
-          children: [],
-          component: AddressGroupComponent,
-          data: {
-            networkSpecific: true,
-          }
-        },
-        {
-          path: 'status',
-          data: { networks: ['bitcoin', 'liquid'] },
-          component: StatusViewComponent
-        },
-        {
-          path: '',
-          loadChildren: () => import('@app/liquid/liquid-graphs.module').then(m => m.LiquidGraphsModule),
-          data: { preload: true },
-        },
-        {
-          path: '**',
-          redirectTo: '/signet'
-        },
-      ]
-    },
+    ...liquidTestnetRoutes,
     {
       path: '',
       pathMatch: 'full',
@@ -357,7 +376,7 @@ if (!window['isMempoolSpaceBuild']) {
 @NgModule({
   imports: [RouterModule.forRoot(routes, {
     initialNavigation: 'enabledBlocking',
-    scrollPositionRestoration: 'enabled',
+    scrollPositionRestoration: 'disabled',
     anchorScrolling: 'disabled',
     preloadingStrategy: AppPreloadingStrategy
   })],

frontend/src/app/app.constants.ts

@@ -82,6 +82,48 @@ export const contrastMempoolFeeColors = [
   'ffb700',
  ];
 
+ export const lightMempoolFeeColors = [
+  '49c284',
+  '99c246',
+  'a0c343',
+  'a7c240',
+  'b0c13c',
+  'b8c03b',
+  'bebe3a',
+  'beb238',
+  'bcac37',
+  'bba336',
+  'b99e35',
+  'b79334',
+  'b68f33',
+  'b68a32',
+  'b48430',
+  'b17f30',
+  'b07b2f',
+  'ae722e',
+  'ad6b2d',
+  'ab652b',
+  'aa5b2a',
+  'a95629',
+  'a74c28',
+  'a34126',
+  'a13a25',
+  'a02d24',
+  '9e2423',
+  '9c222a',
+  '9c2132',
+  '9b203a',
+  '99203e',
+  '972043',
+  '951f45',
+  '931e48',
+  '921d49',
+  '921c4e',
+  '901b50',
+  '8e1a53',
+  '8b1954',
+];
+
 export const chartColors = [
   '#A81524',
   '#D81B60',

frontend/src/app/components/about/about-sponsors.component.html

@@ -1,14 +1,14 @@
 <div id="become-sponsor-container" [ngClass]="context">
   <div class="become-sponsor community">
     <p style="font-weight: 700; font-size: 18px;">If you're an individual...</p>
-    <a [href]="host + '/sponsor'" class="btn" style="background-color: rgba(152, 88, 255, 0.75); box-shadow: 0px 0px 50px 5px rgba(152, 88, 255, 0.75)" i18n="about.community-sponsor-button" (click)="onSponsorClick($event)">Become a Community Sponsor</a>
+    <a [href]="host + '/sponsor'" class="btn" style="background-color: rgba(152, 88, 255, 0.75); box-shadow: 0px 0px 50px 5px rgba(152, 88, 255, 0.75)" i18n="about.community-sponsor-button">Become a Community Sponsor</a>
     <p class="sponsor-feature"><fa-icon [icon]="['fas', 'check']"></fa-icon> Exclusive swag</p>
     <p class="sponsor-feature"><fa-icon [icon]="['fas', 'check']"></fa-icon> Your avatar on the About page</p>
     <p class="sponsor-feature"><fa-icon [icon]="['fas', 'check']"></fa-icon> And more coming soon :)</p>
   </div>
   <div class="become-sponsor enterprise">
     <p style="font-weight: 700; font-size: 18px;">If you're a business...</p>
-    <a [href]="host + '/enterprise'" class="btn" style="background-color: rgba(152, 88, 255, 0.75); box-shadow: 0px 0px 50px 5px rgba(152, 88, 255, 0.75)" i18n="about.enterprise-sponsor-button" (click)="onEnterpriseClick($event)">Become an Enterprise Sponsor</a>
+    <a [href]="host + '/enterprise'" class="btn" style="background-color: rgba(152, 88, 255, 0.75); box-shadow: 0px 0px 50px 5px rgba(152, 88, 255, 0.75)" i18n="about.enterprise-sponsor-button">Become an Enterprise Sponsor</a>
     <p class="sponsor-feature"><fa-icon [icon]="['fas', 'check']"></fa-icon> Increased API limits</p>
     <p class="sponsor-feature"><fa-icon [icon]="['fas', 'check']"></fa-icon> Co-branded instance</p>
     <p class="sponsor-feature"><fa-icon [icon]="['fas', 'check']"></fa-icon> 99% service-level agreement</p>

frontend/src/app/components/about/about-sponsors.component.ts

@@ -1,5 +1,4 @@
 import { Component, Input } from '@angular/core';
-import { EnterpriseService } from '@app/services/enterprise.service';
 
 @Component({
   selector: 'app-about-sponsors',
@@ -10,17 +9,4 @@ import { EnterpriseService } from '@app/services/enterprise.service';
 export class AboutSponsorsComponent {
   @Input() host = 'https://mempool.space';
   @Input() context = 'about';
-
-  constructor(private enterpriseService: EnterpriseService) {
-  }
-
-  onSponsorClick(e): boolean {
-    this.enterpriseService.goal(5);
-    return true;
-  }
-
-  onEnterpriseClick(e): boolean {
-    this.enterpriseService.goal(6);
-    return true;
-  }
 }

frontend/src/app/components/about/about.component.html

@@ -2,7 +2,11 @@
 
   <div class="intro">
     <span style="margin-left: auto; margin-right: -20px; margin-bottom: -20px">&reg;</span>
-    <img class="logo" src="/resources/mempool-logo-bigger.png" />
+    @if (isLightMode) {
+      <img class="logo" src="/resources/mempool-logo-bigger-light.png" />
+    } @else {
+       <img class="logo" src="/resources/mempool-logo-bigger.png" />
+    }
     <div class="version">
       <span>v{{ packetJsonVersion }} [<a href="https://github.com/mempool/mempool/commit/{{ frontendGitCommitHash }}">{{ frontendGitCommitHash }}</a>]</span>
       <span *ngIf="stateService.env.GIT_COMMIT_HASH_MEMPOOL_SPACE">[{{ stateService.env.GIT_COMMIT_HASH_MEMPOOL_SPACE }}]</span>
@@ -41,15 +45,289 @@
     <h3 i18n="about.sponsors.enterprise.withRocket">Enterprise Sponsors 🚀</h3>
     <div class="wrapper">
       <a href="https://spiral.xyz/" target="_blank" title="Spiral">
-        <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="-115 -15 879 679" style="background-color: rgb(27,20,100)" class="image">
+        <svg class="image not-rounded" width="155" height="180" viewBox="0 0 155 180" fill="none" xmlns="http://www.w3.org/2000/svg">
+        <g clip-path="url(#clip0_441_342)">
+        <path d="M91.4267 0H78.4761C77.2348 0 76.2285 1.01085 76.2285 2.25779V15.267C76.2285 16.514 77.2348 17.5248 78.4761 17.5248H91.4267C92.6681 17.5248 93.6744 16.514 93.6744 15.267V2.25779C93.6744 1.01085 92.6681 0 91.4267 0Z" fill="url(#paint0_linear_441_342)"/>
+        <path d="M75.8447 2.78125H62.8941C61.6528 2.78125 60.6465 3.7921 60.6465 5.03904V18.0483C60.6465 19.2952 61.6528 20.3061 62.8941 20.3061H75.8447C77.086 20.3061 78.0923 19.2952 78.0923 18.0483V5.03904C78.0923 3.7921 77.086 2.78125 75.8447 2.78125Z" fill="url(#paint1_linear_441_342)"/>
+        <path d="M62.1494 7.43164H49.1988C47.9575 7.43164 46.9512 8.44249 46.9512 9.68943V22.6987C46.9512 23.9456 47.9575 24.9565 49.1988 24.9565H62.1494C63.3907 24.9565 64.397 23.9456 64.397 22.6987V9.68943C64.397 8.44249 63.3907 7.43164 62.1494 7.43164Z" fill="url(#paint2_linear_441_342)"/>
+        <path d="M50.2959 13.7324H37.3453C36.1039 13.7324 35.0977 14.7433 35.0977 15.9902V28.9995C35.0977 30.2464 36.1039 31.2573 37.3453 31.2573H50.2959C51.5372 31.2573 52.5435 30.2464 52.5435 28.9995V15.9902C52.5435 14.7433 51.5372 13.7324 50.2959 13.7324Z" fill="url(#paint3_linear_441_342)"/>
+        <path d="M40.2373 21.4707H27.2867C26.0454 21.4707 25.0391 22.4815 25.0391 23.7285V36.7377C25.0391 37.9847 26.0454 38.9955 27.2867 38.9955H40.2373C41.4786 38.9955 42.4849 37.9847 42.4849 36.7377V23.7285C42.4849 22.4815 41.4786 21.4707 40.2373 21.4707Z" fill="url(#paint4_linear_441_342)"/>
+        <path d="M31.9346 30.4238H18.9839C17.7426 30.4238 16.7363 31.4347 16.7363 32.6816V45.6909C16.7363 46.9378 17.7426 47.9487 18.9839 47.9487H31.9346C33.1759 47.9487 34.1822 46.9378 34.1822 45.6909V32.6816C34.1822 31.4347 33.1759 30.4238 31.9346 30.4238Z" fill="url(#paint5_linear_441_342)"/>
+        <path d="M25.3389 40.375H12.3882C11.1469 40.375 10.1406 41.3858 10.1406 42.6328V55.642C10.1406 56.889 11.1469 57.8998 12.3882 57.8998H25.3389C26.5802 57.8998 27.5865 56.889 27.5865 55.642V42.6328C27.5865 41.3858 26.5802 40.375 25.3389 40.375Z" fill="url(#paint6_linear_441_342)"/>
+        <path d="M20.415 51.1055H7.46441C6.22309 51.1055 5.2168 52.1163 5.2168 53.3633V66.3725C5.2168 67.6195 6.22309 68.6303 7.46441 68.6303H20.415C21.6563 68.6303 22.6626 67.6195 22.6626 66.3725V53.3633C22.6626 52.1163 21.6563 51.1055 20.415 51.1055Z" fill="url(#paint7_linear_441_342)"/>
+        <path d="M17.1103 62.4004H4.15972C2.9184 62.4004 1.91211 63.4112 1.91211 64.6582V77.6674C1.91211 78.9144 2.9184 79.9252 4.15972 79.9252H17.1103C18.3517 79.9252 19.3579 78.9144 19.3579 77.6674V64.6582C19.3579 63.4112 18.3517 62.4004 17.1103 62.4004Z" fill="url(#paint8_linear_441_342)"/>
+        <path d="M15.3857 74.0391H2.43511C1.19379 74.0391 0.1875 75.0499 0.1875 76.2969V89.3061C0.1875 90.5531 1.19379 91.5639 2.43511 91.5639H15.3857C16.6271 91.5639 17.6333 90.5531 17.6333 89.3061V76.2969C17.6333 75.0499 16.6271 74.0391 15.3857 74.0391Z" fill="url(#paint9_linear_441_342)"/>
+        <path d="M15.1982 85.8047H2.24761C1.00629 85.8047 0 86.8155 0 88.0625V101.072C0 102.319 1.00629 103.33 2.24761 103.33H15.1982C16.4396 103.33 17.4458 102.319 17.4458 101.072V88.0625C17.4458 86.8155 16.4396 85.8047 15.1982 85.8047Z" fill="url(#paint10_linear_441_342)"/>
+        <path d="M16.5029 97.4785H3.5523C2.31098 97.4785 1.30469 98.4894 1.30469 99.7363V112.746C1.30469 113.992 2.31098 115.003 3.5523 115.003H16.5029C17.7442 115.003 18.7505 113.992 18.7505 112.746V99.7363C18.7505 98.4894 17.7442 97.4785 16.5029 97.4785Z" fill="url(#paint11_linear_441_342)"/>
+        <path d="M19.2568 108.842H6.3062C5.06488 108.842 4.05859 109.853 4.05859 111.1V124.109C4.05859 125.356 5.06488 126.367 6.3062 126.367H19.2568C20.4981 126.367 21.5044 125.356 21.5044 124.109V111.1C21.5044 109.853 20.4981 108.842 19.2568 108.842Z" fill="url(#paint12_linear_441_342)"/>
+        <path d="M23.417 119.678H10.4664C9.22504 119.678 8.21875 120.689 8.21875 121.936V134.945C8.21875 136.192 9.22504 137.203 10.4664 137.203H23.417C24.6583 137.203 25.6646 136.192 25.6646 134.945V121.936C25.6646 120.689 24.6583 119.678 23.417 119.678Z" fill="url(#paint13_linear_441_342)"/>
+        <path d="M28.9385 129.771H15.9878C14.7465 129.771 13.7402 130.782 13.7402 132.029V145.039C13.7402 146.285 14.7465 147.296 15.9878 147.296H28.9385C30.1798 147.296 31.1861 146.285 31.1861 145.039V132.029C31.1861 130.782 30.1798 129.771 28.9385 129.771Z" fill="url(#paint14_linear_441_342)"/>
+        <path d="M35.7783 138.9H22.8277C21.5864 138.9 20.5801 139.911 20.5801 141.158V154.167C20.5801 155.414 21.5864 156.425 22.8277 156.425H35.7783C37.0196 156.425 38.0259 155.414 38.0259 154.167V141.158C38.0259 139.911 37.0196 138.9 35.7783 138.9Z" fill="url(#paint15_linear_441_342)"/>
+        <path d="M43.8955 146.848H30.9449C29.7036 146.848 28.6973 147.858 28.6973 149.105V162.115C28.6973 163.362 29.7036 164.372 30.9449 164.372H43.8955C45.1368 164.372 46.1431 163.362 46.1431 162.115V149.105C46.1431 147.858 45.1368 146.848 43.8955 146.848Z" fill="url(#paint16_linear_441_342)"/>
+        <path d="M53.2412 153.398H40.2906C39.0493 153.398 38.043 154.409 38.043 155.656V168.665C38.043 169.912 39.0493 170.923 40.2906 170.923H53.2412C54.4825 170.923 55.4888 169.912 55.4888 168.665V155.656C55.4888 154.409 54.4825 153.398 53.2412 153.398Z" fill="url(#paint17_linear_441_342)"/>
+        <path d="M63.7764 158.33H50.8257C49.5844 158.33 48.5781 159.341 48.5781 160.588V173.597C48.5781 174.844 49.5844 175.855 50.8257 175.855H63.7764C65.0177 175.855 66.024 174.844 66.024 173.597V160.588C66.024 159.341 65.0177 158.33 63.7764 158.33Z" fill="url(#paint18_linear_441_342)"/>
+        <path d="M75.456 161.43H62.5054C61.2641 161.43 60.2578 162.441 60.2578 163.687V176.697C60.2578 177.944 61.2641 178.955 62.5054 178.955H75.456C76.6974 178.955 77.7036 177.944 77.7036 176.697V163.687C77.7036 162.441 76.6974 161.43 75.456 161.43Z" fill="url(#paint19_linear_441_342)"/>
+        <path d="M88.3213 162.475H75.3707C74.1293 162.475 73.123 163.485 73.123 164.732V177.742C73.123 178.989 74.1293 179.999 75.3707 179.999H88.3213C89.5626 179.999 90.5689 178.989 90.5689 177.742V164.732C90.5689 163.485 89.5626 162.475 88.3213 162.475Z" fill="url(#paint20_linear_441_342)"/>
+        <path d="M103.837 160.953H90.8863C89.645 160.953 88.6387 161.964 88.6387 163.211V176.22C88.6387 177.467 89.645 178.478 90.8863 178.478H103.837C105.078 178.478 106.085 177.467 106.085 176.22V163.211C106.085 161.964 105.078 160.953 103.837 160.953Z" fill="url(#paint21_linear_441_342)"/>
+        <path d="M116.72 156.889H103.769C102.528 156.889 101.521 157.9 101.521 159.146V172.156C101.521 173.403 102.528 174.414 103.769 174.414H116.72C117.961 174.414 118.967 173.403 118.967 172.156V159.146C118.967 157.9 117.961 156.889 116.72 156.889Z" fill="url(#paint22_linear_441_342)"/>
+        <path d="M127.212 150.824H114.261C113.02 150.824 112.014 151.835 112.014 153.082V166.091C112.014 167.338 113.02 168.349 114.261 168.349H127.212C128.453 168.349 129.46 167.338 129.46 166.091V153.082C129.46 151.835 128.453 150.824 127.212 150.824Z" fill="url(#paint23_linear_441_342)"/>
+        <path d="M135.56 143.293H122.609C121.368 143.293 120.361 144.304 120.361 145.551V158.56C120.361 159.807 121.368 160.818 122.609 160.818H135.56C136.801 160.818 137.807 159.807 137.807 158.56V145.551C137.807 144.304 136.801 143.293 135.56 143.293Z" fill="url(#paint24_linear_441_342)"/>
+        <path d="M142.009 134.836H129.058C127.817 134.836 126.811 135.847 126.811 137.094V150.103C126.811 151.35 127.817 152.361 129.058 152.361H142.009C143.25 152.361 144.256 151.35 144.256 150.103V137.094C144.256 135.847 143.25 134.836 142.009 134.836Z" fill="url(#paint25_linear_441_342)"/>
+        <path d="M146.845 125.895H133.894C132.653 125.895 131.646 126.905 131.646 128.152V141.162C131.646 142.409 132.653 143.419 133.894 143.419H146.845C148.086 143.419 149.092 142.409 149.092 141.162V128.152C149.092 126.905 148.086 125.895 146.845 125.895Z" fill="url(#paint26_linear_441_342)"/>
+        <path d="M150.925 114.148H137.974C136.733 114.148 135.727 115.159 135.727 116.406V129.415C135.727 130.662 136.733 131.673 137.974 131.673H150.925C152.166 131.673 153.172 130.662 153.172 129.415V116.406C153.172 115.159 152.166 114.148 150.925 114.148Z" fill="url(#paint27_linear_441_342)"/>
+        <path d="M152.753 102.021H139.802C138.561 102.021 137.555 103.032 137.555 104.279V117.289C137.555 118.535 138.561 119.546 139.802 119.546H152.753C153.994 119.546 155.001 118.535 155.001 117.289V104.279C155.001 103.032 153.994 102.021 152.753 102.021Z" fill="url(#paint28_linear_441_342)"/>
+        <path d="M152.292 89.877H139.341C138.1 89.877 137.094 90.8878 137.094 92.1347V105.144C137.094 106.391 138.1 107.402 139.341 107.402H152.292C153.533 107.402 154.54 106.391 154.54 105.144V92.1347C154.54 90.8878 153.533 89.877 152.292 89.877Z" fill="url(#paint29_linear_441_342)"/>
+        <path d="M149.501 78.082H136.55C135.309 78.082 134.303 79.0929 134.303 80.3398V93.3491C134.303 94.596 135.309 95.6069 136.55 95.6069H149.501C150.742 95.6069 151.749 94.596 151.749 93.3491V80.3398C151.749 79.0929 150.742 78.082 149.501 78.082Z" fill="url(#paint30_linear_441_342)"/>
+        <path d="M144.347 66.9961H131.396C130.155 66.9961 129.148 68.0069 129.148 69.2539V82.2631C129.148 83.5101 130.155 84.5209 131.396 84.5209H144.347C145.588 84.5209 146.594 83.5101 146.594 82.2631V69.2539C146.594 68.0069 145.588 66.9961 144.347 66.9961Z" fill="url(#paint31_linear_441_342)"/>
+        <path d="M137.425 57.6875H124.474C123.233 57.6875 122.227 58.6983 122.227 59.9453V72.9545C122.227 74.2015 123.233 75.2123 124.474 75.2123H137.425C138.666 75.2123 139.672 74.2015 139.672 72.9545V59.9453C139.672 58.6983 138.666 57.6875 137.425 57.6875Z" fill="url(#paint32_linear_441_342)"/>
+        <path d="M128.974 50.3164H116.023C114.782 50.3164 113.775 51.3273 113.775 52.5742V65.5835C113.775 66.8304 114.782 67.8412 116.023 67.8412H128.974C130.215 67.8412 131.221 66.8304 131.221 65.5835V52.5742C131.221 51.3273 130.215 50.3164 128.974 50.3164Z" fill="url(#paint33_linear_441_342)"/>
+        <path d="M118.235 44.918H105.285C104.043 44.918 103.037 45.9288 103.037 47.1758V60.185C103.037 61.432 104.043 62.4428 105.285 62.4428H118.235C119.477 62.4428 120.483 61.432 120.483 60.185V47.1758C120.483 45.9288 119.477 44.918 118.235 44.918Z" fill="url(#paint34_linear_441_342)"/>
+        <path d="M104.95 42.3691H91.9996C90.7582 42.3691 89.752 43.38 89.752 44.6269V57.6362C89.752 58.8831 90.7582 59.894 91.9996 59.894H104.95C106.191 59.894 107.198 58.8831 107.198 57.6362V44.6269C107.198 43.38 106.191 42.3691 104.95 42.3691Z" fill="url(#paint35_linear_441_342)"/>
+        <path d="M90.5244 43.2051H77.5738C76.3325 43.2051 75.3262 44.2159 75.3262 45.4629V58.4721C75.3262 59.7191 76.3325 60.7299 77.5738 60.7299H90.5244C91.7657 60.7299 92.772 59.7191 92.772 58.4721V45.4629C92.772 44.2159 91.7657 43.2051 90.5244 43.2051Z" fill="url(#paint36_linear_441_342)"/>
+        <path d="M78.5244 46.7148H65.5738C64.3325 46.7148 63.3262 47.7257 63.3262 48.9726V61.9819C63.3262 63.2288 64.3325 64.2397 65.5738 64.2397H78.5244C79.7657 64.2397 80.772 63.2288 80.772 61.9819V48.9726C80.772 47.7257 79.7657 46.7148 78.5244 46.7148Z" fill="url(#paint37_linear_441_342)"/>
+        <path d="M68.8701 52.1504H55.9195C54.6782 52.1504 53.6719 53.1612 53.6719 54.4082V67.4174C53.6719 68.6644 54.6782 69.6752 55.9195 69.6752H68.8701C70.1114 69.6752 71.1177 68.6644 71.1177 67.4174V54.4082C71.1177 53.1612 70.1114 52.1504 68.8701 52.1504Z" fill="url(#paint38_linear_441_342)"/>
+        <path d="M60.3564 60.0605H47.4058C46.1645 60.0605 45.1582 61.0714 45.1582 62.3183V75.3276C45.1582 76.5745 46.1645 77.5854 47.4058 77.5854H60.3564C61.5977 77.5854 62.604 76.5745 62.604 75.3276V62.3183C62.604 61.0714 61.5977 60.0605 60.3564 60.0605Z" fill="url(#paint39_linear_441_342)"/>
+        <path d="M54.0986 70.5801H41.148C39.9067 70.5801 38.9004 71.5909 38.9004 72.8379V85.8471C38.9004 87.0941 39.9067 88.1049 41.148 88.1049H54.0986C55.3399 88.1049 56.3462 87.0941 56.3462 85.8471V72.8379C56.3462 71.5909 55.3399 70.5801 54.0986 70.5801Z" fill="url(#paint40_linear_441_342)"/>
+        <path d="M51.1377 82.2695H38.1871C36.9457 82.2695 35.9395 83.2804 35.9395 84.5273V97.5366C35.9395 98.7835 36.9457 99.7944 38.1871 99.7944H51.1377C52.379 99.7944 53.3853 98.7835 53.3853 97.5366V84.5273C53.3853 83.2804 52.379 82.2695 51.1377 82.2695Z" fill="url(#paint41_linear_441_342)"/>
+        <path d="M51.5322 94.291H38.5816C37.3403 94.291 36.334 95.3019 36.334 96.5488V109.558C36.334 110.805 37.3403 111.816 38.5816 111.816H51.5322C52.7735 111.816 53.7798 110.805 53.7798 109.558V96.5488C53.7798 95.3019 52.7735 94.291 51.5322 94.291Z" fill="url(#paint42_linear_441_342)"/>
+        <path d="M55.3349 105.816H42.3843C41.143 105.816 40.1367 106.827 40.1367 108.074V121.083C40.1367 122.33 41.143 123.341 42.3843 123.341H55.3349C56.5763 123.341 57.5826 122.33 57.5826 121.083V108.074C57.5826 106.827 56.5763 105.816 55.3349 105.816Z" fill="url(#paint43_linear_441_342)"/>
+        <path d="M62.5127 115.918H49.5621C48.3207 115.918 47.3145 116.929 47.3145 118.176V131.185C47.3145 132.432 48.3207 133.443 49.5621 133.443H62.5127C63.754 133.443 64.7603 132.432 64.7603 131.185V118.176C64.7603 116.929 63.754 115.918 62.5127 115.918Z" fill="url(#paint44_linear_441_342)"/>
+        <path d="M71.6064 122.869H58.6558C57.4145 122.869 56.4082 123.88 56.4082 125.127V138.136C56.4082 139.383 57.4145 140.394 58.6558 140.394H71.6064C72.8477 140.394 73.854 139.383 73.854 138.136V125.127C73.854 123.88 72.8477 122.869 71.6064 122.869Z" fill="url(#paint45_linear_441_342)"/>
+        <path d="M83.4931 126.576H70.5425C69.3012 126.576 68.2949 127.587 68.2949 128.834V141.843C68.2949 143.09 69.3012 144.101 70.5425 144.101H83.4931C84.7345 144.101 85.7408 143.09 85.7408 141.843V128.834C85.7408 127.587 84.7345 126.576 83.4931 126.576Z" fill="url(#paint46_linear_441_342)"/>
+        <path d="M99.9697 125.127H87.0191C85.7778 125.127 84.7715 126.138 84.7715 127.385V140.394C84.7715 141.641 85.7778 142.652 87.0191 142.652H99.9697C101.211 142.652 102.217 141.641 102.217 140.394V127.385C102.217 126.138 101.211 125.127 99.9697 125.127Z" fill="url(#paint47_linear_441_342)"/>
+        <path d="M111.558 117.775H98.607C97.3657 117.775 96.3594 118.786 96.3594 120.033V133.042C96.3594 134.289 97.3657 135.3 98.607 135.3H111.558C112.799 135.3 113.805 134.289 113.805 133.042V120.033C113.805 118.786 112.799 117.775 111.558 117.775Z" fill="url(#paint48_linear_441_342)"/>
+        <path d="M117.044 107.164H104.093C102.852 107.164 101.846 108.175 101.846 109.422V122.431C101.846 123.678 102.852 124.689 104.093 124.689H117.044C118.285 124.689 119.292 123.678 119.292 122.431V109.422C119.292 108.175 118.285 107.164 117.044 107.164Z" fill="url(#paint49_linear_441_342)"/>
+        <path d="M117.704 96.1035H104.753C103.512 96.1035 102.506 97.1144 102.506 98.3613V111.371C102.506 112.618 103.512 113.628 104.753 113.628H117.704C118.945 113.628 119.952 112.618 119.952 111.371V98.3613C119.952 97.1144 118.945 96.1035 117.704 96.1035Z" fill="url(#paint50_linear_441_342)"/>
+        <path d="M114.815 87.4043H101.865C100.623 87.4043 99.6172 88.4151 99.6172 89.6621V102.671C99.6172 103.918 100.623 104.929 101.865 104.929H114.815C116.057 104.929 117.063 103.918 117.063 102.671V89.6621C117.063 88.4151 116.057 87.4043 114.815 87.4043Z" fill="url(#paint51_linear_441_342)"/>
+        <path d="M106.394 80.1777H93.4429C92.2016 80.1777 91.1953 81.1886 91.1953 82.4355V95.4448C91.1953 96.6917 92.2016 97.7026 93.4429 97.7026H106.394C107.635 97.7026 108.641 96.6917 108.641 95.4448V82.4355C108.641 81.1886 107.635 80.1777 106.394 80.1777Z" fill="url(#paint52_linear_441_342)"/>
+        <path d="M95.6553 78.6895H82.7046C81.4633 78.6895 80.457 79.7003 80.457 80.9472V93.9565C80.457 95.2034 81.4633 96.2143 82.7046 96.2143H95.6553C96.8966 96.2143 97.9029 95.2034 97.9029 93.9565V80.9472C97.9029 79.7003 96.8966 78.6895 95.6553 78.6895Z" fill="url(#paint53_linear_441_342)"/>
+        <path d="M86.7842 87.9707H73.8335C72.5922 87.9707 71.5859 88.9816 71.5859 90.2285V103.238C71.5859 104.485 72.5922 105.496 73.8335 105.496H86.7842C88.0255 105.496 89.0318 104.485 89.0318 103.238V90.2285C89.0318 88.9816 88.0255 87.9707 86.7842 87.9707Z" fill="url(#paint54_linear_441_342)"/>
+        </g>
         <defs>
-        <style>.cls-1{fill:url(#linear-gradient);}</style>
-        <linearGradient id="linear-gradient" x1="81.36" y1="311.35" x2="541.35" y2="311.35" gradientUnits="userSpaceOnUse">
-          <stop offset="0.18" stop-color="blue"/>
-          <stop offset="1" stop-color="#f0f"/>
+        <linearGradient id="paint0_linear_441_342" x1="88.4919" y1="16.0496" x2="82.958" y2="4.76016" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
         </linearGradient>
+        <linearGradient id="paint1_linear_441_342" x1="72.9076" y1="18.8286" x2="67.376" y2="7.53915" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint2_linear_441_342" x1="59.2145" y1="23.479" x2="53.6807" y2="12.1895" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint3_linear_441_342" x1="47.3588" y1="29.782" x2="41.8249" y2="18.4926" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint4_linear_441_342" x1="37.3024" y1="37.5181" x2="31.7686" y2="26.2286" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint5_linear_441_342" x1="28.9997" y1="46.4712" x2="23.4658" y2="35.184" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint6_linear_441_342" x1="22.404" y1="56.4224" x2="16.8701" y2="45.1352" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint7_linear_441_342" x1="17.4779" y1="67.1528" x2="11.9463" y2="55.8656" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint8_linear_441_342" x1="14.1732" y1="78.45" x2="8.63936" y2="67.1605" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint9_linear_441_342" x1="12.4486" y1="90.0864" x2="6.917" y2="78.797" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint10_linear_441_342" x1="12.2611" y1="101.852" x2="6.72725" y2="90.5626" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint11_linear_441_342" x1="13.5658" y1="113.528" x2="8.03194" y2="102.239" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint12_linear_441_342" x1="16.3219" y1="124.891" x2="10.7881" y2="113.602" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint13_linear_441_342" x1="20.4799" y1="135.727" x2="14.9483" y2="124.44" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint14_linear_441_342" x1="26.0013" y1="145.821" x2="20.4697" y2="134.532" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint15_linear_441_342" x1="32.8412" y1="154.95" x2="27.3096" y2="143.661" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint16_linear_441_342" x1="40.9584" y1="162.897" x2="35.4245" y2="151.608" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint17_linear_441_342" x1="50.3063" y1="169.446" x2="44.7725" y2="158.159" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint18_linear_441_342" x1="60.8392" y1="174.38" x2="55.3054" y2="163.09" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint19_linear_441_342" x1="72.5212" y1="177.477" x2="66.9873" y2="166.19" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint20_linear_441_342" x1="85.3842" y1="178.522" x2="79.8525" y2="167.235" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint21_linear_441_342" x1="100.902" y1="177" x2="95.3682" y2="165.713" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint22_linear_441_342" x1="113.785" y1="172.938" x2="108.251" y2="161.649" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint23_linear_441_342" x1="124.275" y1="166.872" x2="118.743" y2="155.584" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint24_linear_441_342" x1="132.625" y1="159.34" x2="127.091" y2="148.053" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint25_linear_441_342" x1="139.072" y1="150.883" x2="133.538" y2="139.594" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint26_linear_441_342" x1="143.91" y1="141.944" x2="138.376" y2="130.655" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint27_linear_441_342" x1="147.988" y1="130.196" x2="142.454" y2="118.909" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint28_linear_441_342" x1="149.818" y1="118.071" x2="144.284" y2="106.782" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint29_linear_441_342" x1="149.357" y1="105.927" x2="143.823" y2="94.6394" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint30_linear_441_342" x1="146.566" y1="94.1317" x2="141.032" y2="82.8422" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint31_linear_441_342" x1="141.412" y1="83.0457" x2="135.878" y2="71.7563" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint32_linear_441_342" x1="134.488" y1="73.7349" x2="128.954" y2="62.4477" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint33_linear_441_342" x1="126.036" y1="66.3638" x2="120.503" y2="55.0766" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint34_linear_441_342" x1="115.3" y1="60.9653" x2="109.767" y2="49.6781" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint35_linear_441_342" x1="102.015" y1="58.4165" x2="96.4814" y2="47.127" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint36_linear_441_342" x1="87.5873" y1="59.2547" x2="82.0557" y2="47.9652" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint37_linear_441_342" x1="75.5873" y1="62.7622" x2="70.0534" y2="51.475" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint38_linear_441_342" x1="65.9352" y1="68.2" x2="60.4014" y2="56.9105" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint39_linear_441_342" x1="57.4193" y1="76.1079" x2="51.8855" y2="64.8185" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint40_linear_441_342" x1="51.1615" y1="86.6275" x2="45.6276" y2="75.3402" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint41_linear_441_342" x1="48.2028" y1="98.3192" x2="42.6689" y2="87.0297" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint42_linear_441_342" x1="48.5951" y1="110.341" x2="43.0612" y2="99.0512" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint43_linear_441_342" x1="52.3978" y1="121.866" x2="46.864" y2="110.577" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint44_linear_441_342" x1="59.5756" y1="131.968" x2="54.0417" y2="120.68" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint45_linear_441_342" x1="68.6693" y1="138.917" x2="63.1377" y2="127.629" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint46_linear_441_342" x1="80.5583" y1="142.626" x2="75.0244" y2="131.336" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint47_linear_441_342" x1="97.0326" y1="141.177" x2="91.4987" y2="129.887" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint48_linear_441_342" x1="108.623" y1="133.823" x2="103.089" y2="122.536" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint49_linear_441_342" x1="114.107" y1="123.214" x2="108.573" y2="111.924" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint50_linear_441_342" x1="114.767" y1="112.151" x2="109.235" y2="100.864" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint51_linear_441_342" x1="111.881" y1="103.454" x2="106.349" y2="92.1644" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint52_linear_441_342" x1="103.456" y1="96.2274" x2="97.9248" y2="84.9379" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint53_linear_441_342" x1="92.7204" y1="94.7368" x2="87.1865" y2="83.4474" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <linearGradient id="paint54_linear_441_342" x1="83.8471" y1="104.02" x2="78.3132" y2="92.7309" gradientUnits="userSpaceOnUse">
+        <stop stop-color="#1317F5"/>
+        <stop offset="1" stop-color="#FF00E1"/>
+        </linearGradient>
+        <clipPath id="clip0_441_342">
+        <rect width="155" height="180" fill="white"/>
+        </clipPath>
         </defs>
-        <path class="cls-1" d="M326.4,572.09C201.2,572.09,141,503,112.48,445,84.26,387.47,81.89,330.44,81.69,322.31c-4.85-77,41-231.78,249.58-271.2a28.05,28.05,0,0,1,10.41,55.13c-213.12,40.28-204.44,206-204,213,0,.53.06,1.06.07,1.6C137.9,328.74,142.85,516,326.4,516,394.74,516,443,486.6,470,428.63c24.48-52.74,19.29-112.45-13.52-155.83-22.89-30.27-52.46-45-90.38-45-34.46,0-63.47,9.88-86.21,29.37A91.5,91.5,0,0,0,248,322.3c-1.41,25.4,7.14,49.36,24.07,67.49C287.27,406,305,413.9,326.4,413.9c27.46,0,45.52-9,53.66-26.81,8.38-18.3,3.61-38.93-.19-43.33-9.11-10-18.69-13.68-22.48-13-2.53.43-5.78,4.61-8.48,10.92a28,28,0,0,1-51.58-22c14.28-33.44,37.94-42,50.76-44.2,24.78-4.18,52.17,7.3,73.34,30.65s25.51,68.55,10.15,103.22C421.54,432,394.52,470,326.4,470c-36.72,0-69.67-14.49-95.29-41.92C203.64,398.68,189.77,360,192,319.19a149.1,149.1,0,0,1,51.31-104.6c33.19-28.45,74.48-42.87,122.71-42.87,55.12,0,101.85,23.25,135.12,67.23,45.36,60,52.9,141.71,19.66,213.3C495.45,506.92,441.12,572.09,326.4,572.09Z"/>
         </svg>
         <span>Spiral</span>
       </a>
@@ -126,13 +404,23 @@
         <span>Blockstream</span>
       </a>
       <a href="https://unchained.com/" target="_blank" title="Unchained">
-        <svg id="Layer_1" width="78" height="78" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 156.68 156.68" class="image">
+        @if (isLightMode) {
+          <svg id="Layer_1" width="78" height="78" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 156.68 156.68" class="image">
+            <defs><style>.cls-bg{fill:#fff;}.cls-navy{fill:#002851;}.cls-cyan{fill:#0095d6;}.cls-star{fill:#fff;}</style></defs><ellipse class="cls-bg" cx="78.339596" cy="78.612736" rx="77.418334" ry="77.876421"/><path class="cls-cyan" d="M57.026351 111.292873C49.100201 123.36854 43.017302 135.026662 38.386035 145.479803C50.044884 152.548978 63.731367 156.604891 78.339435 156.604891C121.172597 156.604891 155.895792 121.676557 155.895792 78.589396C155.895792 77.175561 155.849758 75.761726 155.781109 74.347971H147.024797C118.454118 74.347971 81.726388 73.629424 57.026351 111.269452V111.292873Z"/><path class="cls-navy" d="M155.067165 67.255617H110.965783C57.671808 67.255617 27.211439 90.989083 9.976639 115.487697C4.101107 104.501522 0.78318 91.939662 0.78318 78.612575C0.78318 35.525575 35.506272 0.597046 78.339758 0.597046C117.325053 0.597046 149.605979 29.545729 155.067165 67.255617Z"/><path class="cls-star" d="M78.938938 20.552782L80.321437 24.840645C81.841879 29.49929 85.48267 33.161322 90.113614 34.691052L94.376279 36.081708C94.837435 36.243961 94.837435 36.89289 94.376279 37.031964L90.113614 38.42262C85.48267 39.95235 81.841879 43.614382 80.321437 48.273108L78.938938 52.56089C78.777654 53.024469 78.13252 53.024469 77.994254 52.56089L76.611836 48.273108C75.091071 43.614382 71.450603 39.95235 66.819336 38.42262L62.556671 37.031964C62.095838 36.869711 62.095838 36.220782 62.556671 36.081708L66.819336 34.691052C71.450603 33.161322 75.091071 29.49929 76.611836 24.840645L77.994254 20.552782C78.155538 20.089203 78.800752 20.089203 78.938938 20.552782V20.552782Z"/>
+          </svg>
+        } @else {
+           <svg id="Layer_1" width="78" height="78" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 156.68 156.68" class="image">
           <defs><style>.cls-unchained-1{fill:#fff;}</style></defs><path class="cls-unchained-1" d="m78.34,0C35.07,0,0,35.07,0,78.34s35.07,78.34,78.34,78.34,78.34-35.07,78.34-78.34S121.6,0,78.34,0ZM20.23,109.5c-4.99-9.28-7.81-19.89-7.81-31.16C12.42,41.93,41.93,12.42,78.34,12.42c33.15,0,60.58,24.46,65.23,56.32h-37.48c-45.29,0-71.19,20.05-85.85,40.76Zm58.11,34.76c-12.42,0-24.04-3.44-33.96-9.41,3.94-8.85,9.11-18.7,15.84-28.9,20.99-31.8,52.2-31.19,76.49-31.19h7.45c.06,1.18.1,2.38.1,3.58,0,36.41-29.51,65.92-65.92,65.92Z"/><path class="cls-unchained-1" d="m91.98,42.4l-3.62-1.18c-3.94-1.29-7.03-4.38-8.32-8.32l-1.18-3.63c-.13-.39-.68-.39-.81,0l-1.18,3.63c-1.29,3.94-4.38,7.03-8.32,8.32l-3.62,1.18c-.39.13-.39.68,0,.81l3.62,1.18c3.94,1.29,7.03,4.38,8.32,8.32l1.18,3.63c.13.39.68.39.81,0l1.18-3.63c1.29-3.94,4.38-7.03,8.32-8.32l3.62-1.18c.39-.13.39-.68,0-.81Z"/>
         </svg>
+        }
         <span>Unchained</span>
       </a>
       <a href="https://bitkey.world/" target="_blank" title="Bitkey">
-        <img class="image" src="/resources/profile/bitkey.svg" />
+        @if (isLightMode) {
+          <img class="image" src="/resources/profile/bitkey-light.svg" />
+        } @else {
+          <img class="image" src="/resources/profile/bitkey.svg" />
+        }
         <span>Bitkey</span>
       </a>
       <a href="https://metaplanet.jp" target="_blank" title="Metaplanet">
@@ -212,11 +500,19 @@
         <span>Onramp Bitcoin</span>
       </a>
       <a href="https://cakewallet.com/" target="_blank" title="Cake Wallet">
-        <img class="image not-rounded" src="/resources/profile/cake-wallet.svg" />
+        @if (isLightMode) {
+          <img class="image not-rounded" src="/resources/profile/cake-wallet-light.svg" />
+        } @else {
+          <img class="image not-rounded" src="/resources/profile/cake-wallet.svg" />
+        }
         <span>Cake Wallet</span>
       </a>
       <a href="https://leather.io/" target="_blank" title="Leather">
-        <img class="image" src="/resources/profile/leather.svg" />
+        @if (isLightMode) {
+          <img class="image" src="/resources/profile/leather-light.svg" />
+        } @else {
+          <img class="image" src="/resources/profile/leather.svg" />
+        }
         <span>Leather</span>
       </a>
 
@@ -294,7 +590,11 @@
         <span>NixOS</span>
       </a>
       <a href="https://github.com/Start9Labs/start-os" target="_blank" title="StartOS">
-        <img class="image" src="/resources/profile/start9.png" />
+        @if (isLightMode) {
+          <img class="image" src="/resources/profile/start9-light.png" />
+        } @else {
+          <img class="image" src="/resources/profile/start9.png" />
+        }
         <span>StartOS</span>
       </a>
       <a href="https://github.com/btcpayserver/btcpayserver" target="_blank" title="BTCPay Server">
@@ -382,17 +682,42 @@
 
   <div class="alliances" id="community-alliances">
     <h3 i18n="about.alliances">Community Alliances</h3>
-    <div class="wrapper">
-      <a href="https://liquid.net/" title="Liquid Network">
-        <img class="liquid" src="/resources/profile/liquid.svg" />
+    <div class="row alliances-logos">
+      <a target="_blank" class="d-block col-12 col-md-6 col-lg-3 pt-2" href="https://liquid.net/" title="Liquid Network">
+        <div class="p-2 alliance-logo">
+          @if (isLightMode) {
+            <img src="/resources/profile/liquid-light.svg" />
+          } @else {
+            <img src="/resources/profile/liquid.svg" />
+          }
+        </div>
       </a>
-      <a href="https://opencrypto.org/" title="Copa - Crypto Open Patent Alliance">
-        <img class="copa" src="/resources/profile/copa.png" />
+      <a target="_blank" class="d-block col-12 col-md-6 col-lg-4" href="https://opencrypto.org/" title="Copa - Crypto Open Patent Alliance" >
+        <div class="p-2 alliance-logo copa">
+          <img src="/resources/profile/copa.png" />
+        </div>
       </a>
-      <a href="https://bitcoin.gob.sv" title="Oficina Nacional del Bitcoin">
-        <img class="sv" src="/resources/profile/onbtc-full.svg" />
+      <a target="_blank" class="d-block col-12 col-md-6 col-lg-2" href="https://bitcoin.gob.sv" title="Oficina Nacional del Bitcoin" >
+        <div class="alliance-logo">
+          @if (isLightMode) {
+            <img src="/resources/profile/onbtc-full-light.svg" />
+          } @else {
+            <img src="/resources/profile/onbtc-full.svg" />
+          }
+        </div>
+      </a>
+      <a target="_blank" class="d-block col-12 col-md-6 col-lg-3" href="https://bitcoindevkit.org/foundation/about/" title="Bitcoin Dev Kit Foundation" >
+        <div class="p-2 alliance-logo">
+          <img src="/resources/profile/bdk.svg" />
+        </div>
+      </a>
+      <a target="_blank" class="d-block col-12 col-md-6 col-lg-4" href="https://stratumprotocol.org/" title="Stratum V2">
+        <div class="p-2 alliance-logo">
+          <img class="stratum-v2-logo" src="/resources/profile/stratum-v2.png" />
+        </div>
       </a>
     </div>
+  
   </div>
 
   <ng-container *ngIf="translators$ | async | keyvalue as translators else loadingSponsors">
@@ -439,7 +764,11 @@
     <h3 i18n="about.powered-by">Powered By</h3>
     <div class="wrapper">
         <a href="https://wiz.biz" target="_blank" title="wiz & associates">
-          <img class="wiz-logo" src="/resources/wiz.png" />
+          @if (isLightMode) {
+            <img class="wiz-logo" src="/resources/wiz-light.png" />
+          } @else {
+            <img class="wiz-logo" src="/resources/wiz.png" />
+          }
         </a>
     </div>
     <br>
@@ -447,7 +776,11 @@
     <h3 i18n="about.managed-by">Managed By</h3>
     <div class="wrapper">
         <a href="https://mempool.holdings" target="_blank" title="Mempool Holdings">
-          <img class="wiz-logo" src="/resources/mempool-holdings.png" />
+          @if (isLightMode) {
+            <img class="wiz-logo" src="/resources/mempool-holdings-light.png" />
+          } @else {
+            <img class="wiz-logo" src="/resources/mempool-holdings.png" />
+          }
         </a>
     </div>
   </div>
@@ -471,7 +804,7 @@
       Trademark Notice<br>
     </div>
     <p>
-      The Mempool Open Source Project&reg;, Mempool Accelerator&reg;, Mempool Enterprise&reg;, Mempool Wallet&trade;, mempool.space&reg;, Be your own explorer&trade;, Explore the full Bitcoin ecosystem&reg;, Mempool Goggles&trade;, the mempool Logo, the mempool Square Logo, the mempool block visualization Logo, the mempool Blocks Logo, the mempool transaction Logo, the mempool Blocks 3 | 2 Logo, the mempool research Logo, the mempool.space Vertical Logo, and the mempool.space Horizontal Logo are either registered trademarks or trademarks of Mempool Holdings S.A. de C.V. in Japan, the United States, and/or other countries.
+      The Mempool Open Source Project&reg;, Mempool Accelerator&reg;, Mempool Enterprise&reg;, Mempool Wallet&trade;, mempool.space&reg;, Be your own explorer&trade;, Explore the full Bitcoin ecosystem&reg;, Mempool Goggles&reg;, the mempool Logo, the mempool Square Logo, the mempool block visualization Logo, the mempool Blocks Logo, the mempool transaction Logo, the mempool Blocks 3 | 2 Logo, the mempool research Logo, the mempool.space Vertical Logo, and the mempool.space Horizontal Logo are either registered trademarks or trademarks of Mempool Holdings S.A. de C.V. in Japan, the United States, and/or other countries.
     </p>
     <p>
       While our software is available under an open source software license, the copyright license does not include an implied right or license to use our trademarks. See our <a href="https://mempool.space/trademark-policy">Trademark Policy and Guidelines</a> for more details, published on &lt;https://mempool.space/trademark-policy&gt;.

frontend/src/app/components/about/about.component.scss

@@ -101,42 +101,30 @@
     }
   }
 
-  .alliances {
-    margin-bottom: 100px;
-    a {
-      &:nth-child(3) {
-        position: relative;
-        top: 10px;
-      }
+  .alliances-logos {
+    align-items: center;
+    margin-bottom: 80px;
+    justify-content: center;
+  }
+  .alliance-logo {
+    height: 80px;
+    display: flex;
+    justify-content: center;
+    margin-top: 25px;
+    @media (min-width: 768px) {
+      margin-top: 45px;
     }
-    img {
-      display: inline-block;
-      margin: 15px auto;
-      height: 62px;
-      @media (min-width: 425px) {
-        margin: 15px 60px;
-      }
-      @media (min-width: 576px) {
-        margin: 15px 120px;
-      }
-      @media (min-width: 850px) {
-        margin: 50px 30px 0px;
-      }
-    }
-    .liquid {
-      top: 7px;
-      position: relative;
-    }
-    .copa {
-      height: auto;
-      top: 23px;
-      position: relative;
+  }
+  .stratum-v2-logo {
+    object-fit: contain;
+    @media (max-width: 767px) {
       width: 300px;
     }
-    .sv {
-      height: 85px;
-      width: auto;
+  }
+  .copa {
+    @media (min-width: 992px) {
       position: relative;
+      top: 15px;
     }
   }
 
@@ -147,6 +135,7 @@
   .community-integrations-sponsor,
   .maintainers {
     scroll-margin: 30px;
+    scroll-margin-top: 65px;
     .wrapper {
       display: inline-block;
       a {
@@ -198,7 +187,7 @@
     margin: auto;
     line-height: 1.8;
     font-size: 87.5%;
-    color: #e83e8c;
+    color: var(--pink);
     word-wrap: break-word;
     font-family: SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,monospace;
     ul {