Merge branch 'master' into knorrium/undefined_block_guard

backend/.eslintrc.js

@@ -15,7 +15,7 @@ module.exports = {
     "plugin:@typescript-eslint/recommended",
     "prettier"
   ],
-  "ignorePatterns": ["dist", "eslint-local-rules", ".eslintrc.js", "testSetup*.ts", "jest.integration.*.ts", "__tests__", "*.config.ts"],
+  "ignorePatterns": ["dist", "eslint-local-rules", ".eslintrc.js", "testSetup*.ts", "jest.integration.*.ts", "__tests__", "__e2e__", "*.config.ts"],
   "overrides": [
     {
       "files": ["src/__integration_tests__/**/*"],

backend/jest.config.ts

@@ -17,8 +17,10 @@ const config: Config.InitialOptions = {
     './testSetup.ts',
   ],
   testPathIgnorePatterns: [
+    '/dist/',
     '/node_modules/',
     '/__integration_tests__/',
+    'test-utils\\.ts$',
   ],
 };
 export default config;

backend/mempool-config.sample.json

@@ -31,6 +31,8 @@
     "AUDIT": false,
     "RUST_GBT": true,
     "LIMIT_GBT": false,
+    "CLUSTER_MEMPOOL": false,
+    "CLUSTER_MEMPOOL_INDEXING": false,
     "CPFP_INDEXING": false,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,

backend/mempool-config.test.json

@@ -31,6 +31,8 @@
     "AUDIT": false,
     "RUST_GBT": true,
     "LIMIT_GBT": false,
+    "CLUSTER_MEMPOOL": false,
+    "CLUSTER_MEMPOOL_INDEXING": false,
     "CPFP_INDEXING": false,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,

backend/src/__e2e__/cluster-mempool/harness/rpc-client.ts

@@ -0,0 +1,137 @@
+import http from 'http';
+
+export interface RpcConfig {
+  host: string;
+  port: number;
+  user: string;
+  pass: string;
+}
+
+interface RpcResponse {
+  result: any;
+  error: { code: number; message: string } | null;
+  id: string;
+}
+
+export class RpcClient {
+  private config: RpcConfig;
+  private idCounter = 0;
+
+  constructor(config: RpcConfig) {
+    this.config = config;
+  }
+
+  private async call(method: string, params: any[] = []): Promise<any> {
+    const id = String(++this.idCounter);
+    const body = JSON.stringify({ jsonrpc: '2.0', id, method, params });
+    const auth = Buffer.from(`${this.config.user}:${this.config.pass}`).toString('base64');
+
+    return new Promise((resolve, reject) => {
+      const req = http.request(
+        {
+          hostname: this.config.host,
+          port: this.config.port,
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Basic ${auth}`,
+            'Content-Length': Buffer.byteLength(body),
+          },
+        },
+        (res) => {
+          let data = '';
+          res.on('data', (chunk) => (data += chunk));
+          res.on('end', () => {
+            try {
+              const parsed: RpcResponse = JSON.parse(data);
+              if (parsed.error) {
+                reject(new Error(`RPC error ${parsed.error.code}: ${parsed.error.message}`));
+              } else {
+                resolve(parsed.result);
+              }
+            } catch (e) {
+              reject(new Error(`Failed to parse RPC response: ${data.slice(0, 500)}`));
+            }
+          });
+        },
+      );
+      req.on('error', reject);
+      req.write(body);
+      req.end();
+    });
+  }
+
+  async getRawMempool(): Promise<string[]> {
+    return this.call('getrawmempool');
+  }
+
+  async getRawTransaction(txid: string, verbose = true): Promise<any> {
+    return this.call('getrawtransaction', [txid, verbose]);
+  }
+
+  async getBlockTemplate(rules: string[] = ['segwit']): Promise<any> {
+    return this.call('getblocktemplate', [{ rules }]);
+  }
+
+  async getMempoolEntry(txid: string): Promise<any> {
+    return this.call('getmempoolentry', [txid]);
+  }
+
+  async getMempoolCluster(txid: string): Promise<any> {
+    return this.call('getmempoolcluster', [txid]);
+  }
+
+  async getBlockCount(): Promise<number> {
+    return this.call('getblockcount');
+  }
+
+  async batch(calls: { method: string; params: any[] }[]): Promise<any[]> {
+    if (calls.length === 0) {
+      return [];
+    }
+    const bodies = calls.map((c) => ({
+      jsonrpc: '2.0',
+      id: String(++this.idCounter),
+      method: c.method,
+      params: c.params,
+    }));
+    const body = JSON.stringify(bodies);
+    const auth = Buffer.from(`${this.config.user}:${this.config.pass}`).toString('base64');
+
+    return new Promise((resolve, reject) => {
+      const req = http.request(
+        {
+          hostname: this.config.host,
+          port: this.config.port,
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Basic ${auth}`,
+            'Content-Length': Buffer.byteLength(body),
+          },
+        },
+        (res) => {
+          let data = '';
+          res.on('data', (chunk) => (data += chunk));
+          res.on('end', () => {
+            try {
+              const parsed: RpcResponse[] = JSON.parse(data);
+              const results = parsed.map((r) => {
+                if (r.error) {
+                  return { error: r.error };
+                }
+                return r.result;
+              });
+              resolve(results);
+            } catch (e) {
+              reject(new Error(`Failed to parse batch RPC response: ${data.slice(0, 500)}`));
+            }
+          });
+        },
+      );
+      req.on('error', reject);
+      req.write(body);
+      req.end();
+    });
+  }
+}

backend/src/__e2e__/cluster-mempool/harness/run-harness.ts

@@ -0,0 +1,461 @@
+/**
+ * Cluster Mempool Test Harness
+ *
+ * Standalone script that compares our ClusterMempool block template ordering
+ * against Bitcoin Core's getblocktemplate output.
+ *
+ * Uses the backend's own transactionUtils to fetch and convert transactions,
+ * ensuring fields (sigops, adjustedVsize, etc.) match exactly.
+ *
+ * Usage:
+ *   npx ts-node src/__tests__/cluster-mempool/harness/run-harness.ts [options]
+ *
+ * Options:
+ *   --host <host>       Override CORE_RPC host
+ *   --port <port>       Override CORE_RPC port
+ *   --user <user>       Override CORE_RPC username
+ *   --pass <pass>       Override CORE_RPC password
+ *   --interval <ms>     Comparison interval in ms (default: 30000)
+ *   --poll <ms>         Mempool poll interval in ms (default: 1000)
+ *   --max <n>           Max comparisons before exit (0 = unlimited, default: 0)
+ */
+
+import { RpcClient, RpcConfig } from './rpc-client';
+
+// ─── CLI Parsing (must happen before backend imports) ───────────────────────
+
+interface CliOptions {
+  rpcOverrides: Partial<RpcConfig>;
+  comparisonInterval: number;
+  pollInterval: number;
+  maxComparisons: number;
+}
+
+function parseArgs(): CliOptions {
+  const args = process.argv.slice(2);
+  const overrides: Partial<RpcConfig> = {};
+  let comparisonInterval = 30_000;
+  let pollInterval = 1000;
+  let maxComparisons = 0;
+
+  for (let i = 0; i < args.length; i++) {
+    switch (args[i]) {
+      case '--host':
+        overrides.host = args[++i];
+        break;
+      case '--port':
+        overrides.port = parseInt(args[++i], 10);
+        break;
+      case '--user':
+        overrides.user = args[++i];
+        break;
+      case '--pass':
+        overrides.pass = args[++i];
+        break;
+      case '--interval':
+        comparisonInterval = parseInt(args[++i], 10);
+        break;
+      case '--poll':
+        pollInterval = parseInt(args[++i], 10);
+        break;
+      case '--max':
+        maxComparisons = parseInt(args[++i], 10);
+        break;
+    }
+  }
+
+  return { rpcOverrides: overrides, comparisonInterval, pollInterval, maxComparisons };
+}
+
+const cliOptions = parseArgs();
+
+// ─── Backend Module Loading ─────────────────────────────────────────────────
+// Import config first, apply CLI overrides, then import modules that depend on it.
+// This works because Node's require() caches modules — when transactionUtils
+// (via bitcoinClient) reads config, it sees our modified values.
+
+const config = require('../../../config').default;
+
+if (cliOptions.rpcOverrides.host) { config.CORE_RPC.HOST = cliOptions.rpcOverrides.host; }
+if (cliOptions.rpcOverrides.port) { config.CORE_RPC.PORT = cliOptions.rpcOverrides.port; }
+if (cliOptions.rpcOverrides.user) { config.CORE_RPC.USERNAME = cliOptions.rpcOverrides.user; }
+if (cliOptions.rpcOverrides.pass) { config.CORE_RPC.PASSWORD = cliOptions.rpcOverrides.pass; }
+
+// Now load modules that depend on config
+const bitcoinApi = require('../../../api/bitcoin/bitcoin-api-factory').default;
+const transactionUtils = require('../../../api/transaction-utils').default;
+const { ClusterMempool } = require('../../../cluster-mempool/cluster-mempool');
+
+import { MempoolTransactionExtended } from '../../../mempool.interfaces';
+import { MempoolDiff } from '../../../cluster-mempool/cluster-mempool';
+
+// ─── Main Harness ───────────────────────────────────────────────────────────
+
+const TX_FETCH_BATCH_SIZE = 1000;
+
+class Harness {
+  private rpc: RpcClient;
+  private clusterMempool: InstanceType<typeof ClusterMempool> | null = null;
+  private mempool: { [txid: string]: MempoolTransactionExtended } = {};
+  private knownTxids = new Set<string>();
+  private lastBlockHeight = -1;
+  private comparisonInterval: number;
+  private pollInterval: number;
+  private maxComparisons: number;
+  private pollTimer: ReturnType<typeof setInterval> | null = null;
+  private polling = false;
+  private nextComparisonTime = 0;
+  private stats = {
+    templateMatches: 0,
+    templateMismatches: 0,
+    comparisons: 0,
+  };
+
+  constructor(rpcConfig: RpcConfig, opts: CliOptions) {
+    this.rpc = new RpcClient(rpcConfig);
+    this.comparisonInterval = opts.comparisonInterval;
+    this.pollInterval = opts.pollInterval;
+    this.maxComparisons = opts.maxComparisons;
+  }
+
+  async run(): Promise<void> {
+    console.log('=== Cluster Mempool Test Harness ===');
+    console.log(`RPC: ${config.CORE_RPC.HOST}:${config.CORE_RPC.PORT}`);
+    console.log(`Backend: ${config.MEMPOOL.BACKEND}`);
+    console.log(`Poll interval: ${this.pollInterval}ms`);
+    console.log(`Comparison interval: ${this.comparisonInterval}ms`);
+    console.log();
+
+    // Step 1: Fetch full mempool
+    console.log('Fetching full mempool...');
+    const txids: string[] = await bitcoinApi.$getRawMempool();
+    console.log(`Got ${txids.length} txids from getrawmempool`);
+
+    const fetchStart = Date.now();
+    await this.fetchTransactions(txids);
+    const fetchTime = Date.now() - fetchStart;
+    console.log(`Fetched ${Object.keys(this.mempool).length} transactions in ${fetchTime}ms`);
+
+    // Step 2: Initialize ClusterMempool
+    console.log('Building cluster mempool...');
+    const buildStart = Date.now();
+    this.clusterMempool = new ClusterMempool(this.mempool);
+    const buildTime = Date.now() - buildStart;
+    console.log(`Cluster mempool built in ${buildTime}ms`);
+    console.log(`  Clusters: ${this.clusterMempool.getClusterCount()}`);
+    console.log(`  Transactions: ${this.clusterMempool.getTxCount()}`);
+    console.log();
+
+    // Record initial block height
+    this.lastBlockHeight = await this.rpc.getBlockCount();
+
+    // Run initial comparison, then start polling
+    await this.runComparison();
+    this.nextComparisonTime = Date.now() + this.comparisonInterval;
+    this.pollTimer = setInterval(() => this.pollMempool(), this.pollInterval);
+
+    console.log('\nHarness running. Press Ctrl+C to stop.\n');
+
+    process.on('SIGINT', () => this.shutdown());
+    process.on('SIGTERM', () => this.shutdown());
+  }
+
+  private async fetchTransactions(txids: string[]): Promise<MempoolTransactionExtended[]> {
+    const added: MempoolTransactionExtended[] = [];
+
+    for (let offset = 0; offset < txids.length; offset += TX_FETCH_BATCH_SIZE) {
+      let batch = txids.slice(offset, offset + TX_FETCH_BATCH_SIZE);
+      let txs: MempoolTransactionExtended[] = [];
+      let tries = 0;
+      while (batch.length && tries < 20) {
+        try {
+          tries++;
+          txs = txs.concat(await transactionUtils.$getMempoolTransactionsExtended(batch, false, false, false));
+          let missing: string[] = [];
+          console.log(`txs: ${txs.length} of ${batch.length} fetched`);
+          for (const txid of batch) {
+            if (!txs.some(tx => tx.txid === txid)) {
+              console.log(`missing ${txid} at offset ${offset}, retrying`);
+              missing.push(txid);
+            }
+          }
+          batch = missing;
+          if (batch.length) {
+            await new Promise(resolve => setTimeout(resolve, 500));
+          }
+        } catch (err: any) {
+          console.log(`  Fetch batch failed at offset ${offset}: ${err.message}, retrying`);
+        }
+      }
+
+      for (const tx of txs) {
+        this.mempool[tx.txid] = tx;
+        this.knownTxids.add(tx.txid);
+        added.push(tx);
+      }
+
+      if (txids.length > TX_FETCH_BATCH_SIZE && (offset + batch.length) % 5000 < TX_FETCH_BATCH_SIZE) {
+        console.log(`  ${offset + batch.length} / ${txids.length} fetched...`);
+      }
+    }
+
+    return added;
+  }
+
+  private async pollMempool(): Promise<void> {
+    if (this.polling || !this.clusterMempool) {
+      return;
+    }
+    this.polling = true;
+
+    try {
+      // Check for new block
+      const height = await this.rpc.getBlockCount();
+      const newBlock = height > this.lastBlockHeight;
+      if (newBlock) {
+        console.log(`\n--- New block detected (height ${height}) ---`);
+        this.lastBlockHeight = height;
+      }
+
+      // Get current mempool
+      const currentTxids = await bitcoinApi.$getRawMempool();
+      const currentSet = new Set(currentTxids);
+
+      // Find added and removed
+      const addedTxids: string[] = [];
+      for (const txid of currentTxids) {
+        if (!this.knownTxids.has(txid)) {
+          addedTxids.push(txid);
+        }
+      }
+      const removedTxids: string[] = [];
+      for (const txid of this.knownTxids) {
+        if (!currentSet.has(txid)) {
+          removedTxids.push(txid);
+        }
+      }
+
+      if (addedTxids.length === 0 && removedTxids.length === 0) {
+        if (newBlock) {
+          await this.runComparison();
+        }
+        return;
+      }
+
+      const added = await this.fetchTransactions(addedTxids);
+
+      // Apply diff before deleting from cache — processRemovals needs the tx data
+      const diff: MempoolDiff = { added, removed: removedTxids, accelerations: {} };
+      const t0 = Date.now();
+      this.clusterMempool.applyMempoolChange(diff);
+      const dt = Date.now() - t0;
+      for (const txid of removedTxids) {
+        delete this.mempool[txid];
+        this.knownTxids.delete(txid);
+      }
+      console.log(
+        `Applied${newBlock ? ' block' : ''} diff: +${added.length} -${removedTxids.length} txs in ${dt}ms` +
+        ` (${this.clusterMempool.getTxCount()} txs, ${this.clusterMempool.getClusterCount()} clusters)`
+      );
+
+      if (newBlock) {
+        await this.runComparison();
+      } else if (Date.now() >= this.nextComparisonTime) {
+        await this.runComparison();
+        this.nextComparisonTime = Date.now() + this.comparisonInterval;
+      }
+    } catch (err: any) {
+      console.error(`Poll error: ${err.message}`);
+    } finally {
+      this.polling = false;
+    }
+  }
+
+  private async runComparison(): Promise<void> {
+    if (!this.clusterMempool) {
+      return;
+    }
+    this.stats.comparisons++;
+    console.log(`\n=== Comparison #${this.stats.comparisons} ===`);
+
+    // Step 1: Get Core's block template
+    let template: any;
+    try {
+      template = await this.rpc.getBlockTemplate(['segwit']);
+    } catch (err: any) {
+      console.log(`getblocktemplate failed: ${err.message}`);
+      return;
+    }
+    const coreTxids: string[] = template.transactions.map((t: any) => t.txid);
+    const coreSet = new Set(coreTxids);
+
+    const missingTxids: string[] = [];
+    for (const txid of coreSet) {
+      if (!this.mempool[txid]) {
+        missingTxids.push(txid);
+      }
+    }
+    if (missingTxids.length) {
+      const added = await this.fetchTransactions(missingTxids);
+      if (added.length) {
+        console.log(`Reconciled: +${added.length} missing txs`);
+        this.clusterMempool.applyMempoolChange({ added, removed: [], accelerations: {} });
+      }
+    }
+
+    const ourBlocks = this.clusterMempool.getBlocks(1, true);
+    const ourTxids: string[] = ourBlocks[0]?.txids || [];
+
+    await this.compareTemplateOrdering(coreTxids, ourTxids);
+
+    this.printStats();
+
+    if (this.maxComparisons > 0 && this.stats.comparisons >= this.maxComparisons) {
+      console.log(`\nReached ${this.maxComparisons} comparisons, stopping.`);
+      this.shutdown();
+    }
+  }
+
+  private async compareTemplateOrdering(coreTxids: string[], ourTxids: string[]): Promise<void> {
+    const coreSet = new Set(coreTxids);
+    const ourSet = new Set(ourTxids);
+
+    let inBoth = 0;
+    let onlyCore = 0;
+    let onlyOurs = 0;
+    for (const txid of coreSet) {
+      if (ourSet.has(txid)) {
+        inBoth++;
+      } else {
+        onlyCore++;
+      }
+    }
+    for (const txid of ourSet) {
+      if (!coreSet.has(txid)) {
+        onlyOurs++;
+      }
+    }
+
+    console.log(`Template sets: Core ${coreTxids.length} txs, Ours ${ourTxids.length} txs`);
+    console.log(`  In both: ${inBoth}, Only Core: ${onlyCore}, Only ours: ${onlyOurs}`);
+
+    // Build position maps for the intersection
+    const corePos = new Map<string, number>();
+    for (let i = 0; i < coreTxids.length; i++) {
+      if (ourSet.has(coreTxids[i])) {
+        corePos.set(coreTxids[i], i);
+      }
+    }
+
+    // Filter to shared txids in each order
+    const sharedInCoreOrder = coreTxids.filter(t => ourSet.has(t));
+    const sharedInOurOrder = ourTxids.filter(t => coreSet.has(t));
+
+    let exactMatches = 0;
+    let firstDivergence = -1;
+    for (let i = 0; i < sharedInCoreOrder.length; i++) {
+      if (sharedInCoreOrder[i] === sharedInOurOrder[i]) {
+        exactMatches++;
+      } else if (firstDivergence === -1) {
+        firstDivergence = i;
+      }
+    }
+
+    const orderMatchRate = sharedInCoreOrder.length > 0
+      ? (exactMatches / sharedInCoreOrder.length * 100).toFixed(1)
+      : 'N/A';
+    console.log(`  Ordering: ${orderMatchRate}% exact position match (${exactMatches}/${sharedInCoreOrder.length} shared txs)`);
+
+    if (firstDivergence >= 0) {
+      const coreTx = sharedInCoreOrder[firstDivergence];
+      const ourTx = sharedInOurOrder[firstDivergence];
+      console.log(`\n  First divergence at position ${firstDivergence}:`);
+      console.log(`    Core wants: ${coreTx}`);
+      console.log(`    We placed:  ${ourTx} (Core has this at position ${corePos.get(ourTx)})`);
+
+      for (const [label, txid] of [['Core tx', coreTx], ['Our tx', ourTx]] as const) {
+        const inOurMempool = !!this.mempool[txid];
+        const info = this.clusterMempool?.getClusterInfo(txid);
+        console.log(`\n    --- ${label}: ${txid} ---`);
+        console.log(`    In our mempool: ${inOurMempool}`);
+        if (inOurMempool) {
+          const tx = this.mempool[txid];
+          console.log(`    fee=${tx.fee} weight=${tx.weight} vsize=${tx.vsize} sigops=${tx.sigops} adjustedVsize=${tx.adjustedVsize}`);
+        }
+        if (info) {
+          console.log(`    clusterId=${info.clusterId} chunkIndex=${info.chunkIndex} chunkFeerate=${info.chunkFeerate.toFixed(6)}`);
+          const cluster = this.clusterMempool?.getCluster(info.clusterId);
+          if (cluster) {
+            console.log(`    Cluster has ${cluster.chunks.length} chunk(s):`);
+            for (let ci = 0; ci < cluster.chunks.length; ci++) {
+              const c = cluster.chunks[ci];
+              console.log(`      chunk[${ci}]: ${c.txs.length} txs, feerate=${c.feerate.toFixed(6)}`);
+              if (ci === info.chunkIndex) {
+                const chunkTxids = c.txs.map((idx: number) => cluster.txs[idx]?.txid).filter(Boolean);
+                for (const tid of chunkTxids) {
+                  const t = this.mempool[tid];
+                  const parents = t?.vin
+                    ?.filter(v => !v.is_coinbase && this.mempool[v.txid] && chunkTxids.includes(v.txid))
+                    .map(v => v.txid.substring(0, 12)) || [];
+                  console.log(`        ${tid.substring(0, 12)}  fee=${t?.fee} size=${(t?.weight || 0) / 4} sigops=${t?.sigops} parents=[${parents.join(', ')}]`);
+                }
+              }
+            }
+          }
+        } else {
+          console.log(`    NOT in our cluster mempool`);
+        }
+      }
+
+      for (const [label, txid] of [['Core', coreTx], ['Our', ourTx]] as const) {
+        try {
+          const coreCluster = await this.rpc.getMempoolCluster(txid);
+          console.log(`\n    --- Core cluster for ${label} tx ${txid.substring(0, 12)} ---`);
+          console.log(`    Raw response: ${JSON.stringify(coreCluster).substring(0, 2000)}`);
+        } catch (e: any) {
+          console.log(`    Failed to get Core cluster for ${label} tx: ${e.message}`);
+        }
+      }
+
+      this.stats.templateMismatches++;
+    } else if (sharedInCoreOrder.length === coreTxids.length && sharedInCoreOrder.length === ourTxids.length) {
+      console.log(`  PERFECT MATCH`);
+      this.stats.templateMatches++;
+    } else {
+      console.log(`  Ordering matches for shared txs, but sets differ`);
+      this.stats.templateMatches++;
+    }
+  }
+
+  private printStats(): void {
+    console.log('\n--- Cumulative Stats ---');
+    const total = this.stats.templateMatches + this.stats.templateMismatches;
+    console.log(`Template comparisons: ${total} (${this.stats.templateMatches} perfect, ${this.stats.templateMismatches} divergent)`);
+  }
+
+  private shutdown(): void {
+    console.log('\nShutting down...');
+    if (this.pollTimer) {
+      clearInterval(this.pollTimer);
+    }
+    this.printStats();
+    process.exit(0);
+  }
+}
+
+// ─── Entry Point ────────────────────────────────────────────────────────────
+
+const rpcConfig: RpcConfig = {
+  host: config.CORE_RPC.HOST,
+  port: config.CORE_RPC.PORT,
+  user: config.CORE_RPC.USERNAME,
+  pass: config.CORE_RPC.PASSWORD,
+};
+
+console.log(`Connecting to Bitcoin Core at ${rpcConfig.host}:${rpcConfig.port}`);
+
+const harness = new Harness(rpcConfig, cliOptions);
+harness.run().catch((err) => {
+  console.error('Harness failed:', err);
+  process.exit(1);
+});

backend/src/__fixtures__/mempool-config.template.json

@@ -32,6 +32,8 @@
     "AUDIT": true,
     "RUST_GBT": false,
     "LIMIT_GBT": false,
+    "CLUSTER_MEMPOOL": false,
+    "CLUSTER_MEMPOOL_INDEXING": false,
     "CPFP_INDEXING": true,
     "MAX_BLOCKS_BULK_QUERY": 999,
     "DISK_CACHE_BLOCK_INTERVAL": 999,

backend/src/__tests__/cluster-mempool/cluster-mempool.test.ts

@@ -0,0 +1,477 @@
+import { ClusterMempool } from '../../cluster-mempool/cluster-mempool';
+import { makeTx, txid } from './test-utils';
+import { MempoolTransactionExtended } from '../../mempool.interfaces';
+
+function buildMempool(txs: MempoolTransactionExtended[]): { [txid: string]: MempoolTransactionExtended } {
+  const mempool: { [txid: string]: MempoolTransactionExtended } = {};
+  for (const tx of txs) {
+    mempool[tx.txid] = tx;
+  }
+  return mempool;
+}
+
+describe('ClusterMempool', () => {
+  describe('constructor', () => {
+    it('should build clusters from mempool', () => {
+      const parentId = txid('a1');
+      const childId = txid('a2');
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        makeTx(childId, 5000, 100, [parentId]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(2);
+    });
+
+    it('should create separate clusters for unrelated txs', () => {
+      const mempool = buildMempool([
+        makeTx(txid('a1'), 100, 100),
+        makeTx(txid('b1'), 200, 100),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(2);
+    });
+  });
+
+  describe('getClusterInfo', () => {
+    it('should return cluster info for a known tx', () => {
+      const parentId = txid('a1');
+      const childId = txid('a2');
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        makeTx(childId, 5000, 100, [parentId]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const info = cm.getClusterInfo(parentId);
+      expect(info).not.toBeNull();
+      expect(info?.chunkFeerate).toBeGreaterThan(0);
+    });
+
+    it('should return null for unknown tx', () => {
+      const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterInfo(txid('zz'))).toBeNull();
+    });
+  });
+
+  describe('getCluster', () => {
+    it('should return cluster data with correct topology', () => {
+      const parentId = txid('a1');
+      const childId = txid('a2');
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        makeTx(childId, 5000, 100, [parentId]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const info = cm.getClusterInfo(parentId);
+      expect(info).not.toBeNull();
+      const data = cm.getCluster(info!.clusterId);
+      expect(data).not.toBeNull();
+      expect(data!.txs.length).toBe(2);
+      expect(data!.chunks.length).toBeGreaterThan(0);
+    });
+
+    it('should return null for unknown cluster id', () => {
+      const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getCluster(9999)).toBeNull();
+    });
+  });
+
+  describe('applyMempoolChange', () => {
+    it('should handle adding a new singleton tx', () => {
+      const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+      const initialCount = cm.getClusterCount();
+
+      cm.applyMempoolChange({
+        added: [makeTx(txid('b1'), 200, 100)],
+        removed: [],
+        accelerations: {},
+      });
+
+      expect(cm.getClusterCount()).toBe(initialCount + 1);
+      expect(cm.getTxCount()).toBe(2);
+    });
+
+    it('should handle removing a tx', () => {
+      const parentId = txid('a1');
+      const childId = txid('a2');
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        makeTx(childId, 5000, 100, [parentId]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [childId],
+        accelerations: {},
+      });
+
+      expect(cm.getTxCount()).toBe(1);
+      expect(cm.getClusterInfo(childId)).toBeNull();
+      expect(cm.getClusterInfo(parentId)).not.toBeNull();
+    });
+
+    it('should split cluster when middle tx is removed', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        makeTx(b, 200, 100, [a]),
+        makeTx(c, 300, 100, [b]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [b],
+        accelerations: {},
+      });
+
+      expect(cm.getTxCount()).toBe(2);
+      const infoA = cm.getClusterInfo(a);
+      const infoC = cm.getClusterInfo(c);
+      expect(infoA).not.toBeNull();
+      expect(infoC).not.toBeNull();
+      expect(infoA!.clusterId).not.toBe(infoC!.clusterId);
+    });
+
+    it('should handle fee changes via acceleration', () => {
+      const parentId = txid('a1');
+      const childId = txid('a2');
+      const mempool = buildMempool([
+        makeTx(parentId, 100, 100),
+        makeTx(childId, 100, 100, [parentId]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const infoBefore = cm.getClusterInfo(childId);
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [],
+        accelerations: { [childId]: { feeDelta: 49900 } },
+      });
+
+      const infoAfter = cm.getClusterInfo(childId);
+      expect(infoAfter).not.toBeNull();
+      expect(infoAfter!.clusterId).not.toBe(infoBefore!.clusterId);
+    });
+
+    it('should merge clusters when new tx connects them', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        makeTx(b, 200, 100),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(2);
+
+      const bridgeTx = makeTx(txid('c1'), 300, 100, [a, b]);
+      cm.applyMempoolChange({
+        added: [bridgeTx],
+        removed: [],
+        accelerations: {},
+      });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(3);
+    });
+  });
+
+  describe('cluster merging', () => {
+    it('should merge 3 separate clusters when new tx bridges them', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        makeTx(b, 200, 100),
+        makeTx(c, 300, 100),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(3);
+
+      const bridge = makeTx(txid('d1'), 400, 100, [a, b, c]);
+      mempool[bridge.txid] = bridge;
+      cm.applyMempoolChange({ added: [bridge], removed: [], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(4);
+    });
+
+    it('should grow cluster by 1 when new tx has parents in same cluster', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        makeTx(b, 200, 100, [a]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+
+      const c = makeTx(txid('c1'), 300, 100, [a]);
+      mempool[c.txid] = c;
+      cm.applyMempoolChange({ added: [c], removed: [], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(3);
+    });
+
+    it('should grow chain incrementally', () => {
+      const a = txid('a1');
+      const mempool = buildMempool([makeTx(a, 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+
+      const b = makeTx(txid('b1'), 200, 100, [a]);
+      mempool[b.txid] = b;
+      cm.applyMempoolChange({ added: [b], removed: [], accelerations: {} });
+
+      const c = makeTx(txid('c1'), 300, 100, [txid('b1')]);
+      mempool[c.txid] = c;
+      cm.applyMempoolChange({ added: [c], removed: [], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(3);
+    });
+  });
+
+  describe('cluster splitting', () => {
+    it('should split star into singletons when center is removed', () => {
+      const center = txid('center');
+      const leaves = Array.from({ length: 5 }, (_, i) => txid(`leaf${i}`));
+      const centerTx = makeTx(center, 100, 100);
+      for (let i = 1; i < 5; i++) {
+        centerTx.vout.push({
+          scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000,
+        });
+      }
+      const leafTxs = leaves.map((l, i) => {
+        const tx = makeTx(l, 200, 100, [center]);
+        tx.vin[0].vout = i;
+        return tx;
+      });
+      const mempool = buildMempool([centerTx, ...leafTxs]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+
+      cm.applyMempoolChange({ added: [], removed: [center], accelerations: {} });
+
+      expect(cm.getTxCount()).toBe(5);
+      expect(cm.getClusterCount()).toBe(5);
+    });
+
+    it('should shrink cluster without splitting when leaf is removed', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const aTx = makeTx(a, 100, 100);
+      aTx.vout.push({ scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000 });
+      const bTx = makeTx(b, 200, 100, [a]);
+      bTx.vin[0].vout = 0;
+      const cTx = makeTx(c, 300, 100, [a]);
+      cTx.vin[0].vout = 1;
+      const mempool = buildMempool([aTx, bTx, cTx]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+
+      cm.applyMempoolChange({ added: [], removed: [c], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(2);
+    });
+
+    it('should produce singleton when tx is removed from 2-tx cluster', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        makeTx(b, 200, 100, [a]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+
+      cm.applyMempoolChange({ added: [], removed: [b], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(1);
+      expect(cm.getTxCount()).toBe(1);
+      expect(cm.getClusterInfo(a)).not.toBeNull();
+    });
+
+    it('should create 3+ components when removing a tx that bridges multiple subgraphs', () => {
+      const hub = txid('hub');
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const hubTx = makeTx(hub, 100, 100);
+      hubTx.vout.push(
+        { scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000 },
+        { scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000 },
+      );
+      const txA = makeTx(a, 200, 100, [hub]);
+      txA.vin[0].vout = 0;
+      const txB = makeTx(b, 300, 100, [hub]);
+      txB.vin[0].vout = 1;
+      const txC = makeTx(c, 400, 100, [hub]);
+      txC.vin[0].vout = 2;
+      const mempool = buildMempool([hubTx, txA, txB, txC]);
+      const cm = new ClusterMempool(mempool);
+      expect(cm.getClusterCount()).toBe(1);
+
+      cm.applyMempoolChange({ added: [], removed: [hub], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(3);
+      expect(cm.getTxCount()).toBe(3);
+    });
+  });
+
+  describe('fee changes via accelerations', () => {
+    it('should increase chunk feerate when acceleration added', () => {
+      const a = txid('a1');
+      const mempool = buildMempool([makeTx(a, 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+      const infoBefore = cm.getClusterInfo(a);
+      expect(infoBefore).not.toBeNull();
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [],
+        accelerations: { [a]: { feeDelta: 9900 } },
+      });
+
+      const infoAfter = cm.getClusterInfo(a);
+      expect(infoAfter).not.toBeNull();
+      expect(infoAfter!.chunkFeerate).toBeGreaterThan(infoBefore!.chunkFeerate);
+    });
+
+    it('should decrease chunk feerate when acceleration removed', () => {
+      const a = txid('a1');
+      const mempool = buildMempool([makeTx(a, 100, 100)]);
+      const cm = new ClusterMempool(mempool, { [a]: { feeDelta: 9900 } });
+      const infoBefore = cm.getClusterInfo(a);
+      expect(infoBefore).not.toBeNull();
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [],
+        accelerations: {},
+      });
+
+      const infoAfter = cm.getClusterInfo(a);
+      expect(infoAfter).not.toBeNull();
+      expect(infoAfter!.chunkFeerate).toBeLessThan(infoBefore!.chunkFeerate);
+    });
+
+    it('should reorder chunks when acceleration shifts priorities', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const mempool = buildMempool([
+        makeTx(a, 100, 100),
+        makeTx(b, 200, 100, [a]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const infoBBefore = cm.getClusterInfo(b);
+      expect(infoBBefore).not.toBeNull();
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [],
+        accelerations: { [b]: { feeDelta: 49800 } },
+      });
+
+      const infoBAfter = cm.getClusterInfo(b);
+      expect(infoBAfter).not.toBeNull();
+      expect(infoBAfter!.chunkFeerate).toBeGreaterThan(infoBBefore!.chunkFeerate);
+    });
+  });
+
+  describe('getBlocks', () => {
+    it('should return projected blocks', () => {
+      const txs: MempoolTransactionExtended[] = [];
+      for (let i = 0; i < 10; i++) {
+        txs.push(makeTx(txid(`t${i}`), 1000 * (i + 1), 100));
+      }
+      const mempool = buildMempool(txs);
+      const cm = new ClusterMempool(mempool);
+      const blocks = cm.getBlocks(3);
+      expect(blocks.length).toBeGreaterThan(0);
+      expect(blocks[0].txids.length).toBeGreaterThan(0);
+    });
+
+    it('should return empty array for empty mempool', () => {
+      const cm = new ClusterMempool({});
+      const blocks = cm.getBlocks(3);
+      expect(blocks.length).toBe(0);
+    });
+
+    it('should respect chunk ordering for single-cluster mempool', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const mempool = buildMempool([
+        makeTx(a, 3000, 100),
+        makeTx(b, 200, 100, [a]),
+        makeTx(c, 100, 100, [b]),
+      ]);
+      const cm = new ClusterMempool(mempool);
+      const blocks = cm.getBlocks(1);
+      expect(blocks.length).toBe(1);
+      const txids = blocks[0].txids;
+      expect(txids.indexOf(a)).toBeLessThan(txids.indexOf(b));
+      expect(txids.indexOf(b)).toBeLessThan(txids.indexOf(c));
+    });
+
+    it('should maintain topological validity within blocks', () => {
+      const a = txid('a1');
+      const b = txid('b1');
+      const c = txid('c1');
+      const d = txid('d1');
+      const aTx = makeTx(a, 1000, 100);
+      aTx.vout.push({ scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000 });
+      const bTx = makeTx(b, 500, 100, [a]);
+      bTx.vin[0].vout = 0;
+      const cTx = makeTx(c, 500, 100, [a]);
+      cTx.vin[0].vout = 1;
+      const mempool = buildMempool([aTx, bTx, cTx, makeTx(d, 200, 100, [b, c])]);
+      const cm = new ClusterMempool(mempool);
+      const blocks = cm.getBlocks(1);
+      const txids = blocks[0].txids;
+
+      expect(txids.indexOf(a)).toBeLessThan(txids.indexOf(b));
+      expect(txids.indexOf(a)).toBeLessThan(txids.indexOf(c));
+      expect(txids.indexOf(b)).toBeLessThan(txids.indexOf(d));
+      expect(txids.indexOf(c)).toBeLessThan(txids.indexOf(d));
+    });
+  });
+
+  describe('empty and degenerate cases', () => {
+    it('should handle empty diff with no changes', () => {
+      const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+      const countBefore = cm.getClusterCount();
+      const txCountBefore = cm.getTxCount();
+
+      cm.applyMempoolChange({ added: [], removed: [], accelerations: {} });
+
+      expect(cm.getClusterCount()).toBe(countBefore);
+      expect(cm.getTxCount()).toBe(txCountBefore);
+    });
+
+    it('should not crash when removing nonexistent tx', () => {
+      const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]);
+      const cm = new ClusterMempool(mempool);
+
+      cm.applyMempoolChange({
+        added: [],
+        removed: [txid('nonexistent')],
+        accelerations: {},
+      });
+
+      expect(cm.getTxCount()).toBe(1);
+    });
+  });
+});

backend/src/__tests__/cluster-mempool/depgraph.test.ts

@@ -0,0 +1,494 @@
+import { DepGraph, sortTopological, subgraph } from '../../cluster-mempool/depgraph';
+import { buildChain, buildFanOut, buildDiamond, buildStar } from './test-utils';
+
+describe('DepGraph', () => {
+  describe('addTransaction', () => {
+    it('should add a transaction and return a ClusterTx', () => {
+      const dg = new DepGraph();
+      const tx = dg.addTransaction('tx0', 1000, 100);
+      expect(dg.size).toBe(1);
+      expect(tx.effectiveFee).toBe(1000);
+      expect(tx.weight).toBe(100);
+      expect(tx.txid).toBe('tx0');
+    });
+
+    it('should assign distinct ClusterTx objects', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      expect(a).not.toBe(b);
+      expect(b).not.toBe(c);
+      expect(dg.size).toBe(3);
+    });
+
+    it('should include self in ancestors and descendants', () => {
+      const dg = new DepGraph();
+      const tx = dg.addTransaction('tx0', 100, 10);
+      expect(tx.ancestors.has(tx)).toBe(true);
+      expect(tx.descendants.has(tx)).toBe(true);
+    });
+
+    it('should handle large clusters', () => {
+      const dg = new DepGraph();
+      for (let i = 0; i < 100; i++) {
+        dg.addTransaction(`tx${i}`, 100, 10);
+      }
+      expect(dg.size).toBe(100);
+    });
+  });
+
+  describe('addDependency', () => {
+    it('should establish parent-child relationship', () => {
+      const dg = new DepGraph();
+      const parent = dg.addTransaction('parent', 100, 10);
+      const child = dg.addTransaction('child', 200, 20);
+      dg.addDependency(parent, child);
+
+      expect(child.ancestors.has(parent)).toBe(true);
+      expect(parent.descendants.has(child)).toBe(true);
+      expect(child.parents.has(parent)).toBe(true);
+      expect(parent.children.has(child)).toBe(true);
+    });
+
+    it('should propagate ancestors transitively', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+
+      expect(c.ancestors.has(a)).toBe(true);
+      expect(c.ancestors.has(b)).toBe(true);
+      expect(c.ancestors.has(c)).toBe(true);
+
+      expect(a.descendants.has(a)).toBe(true);
+      expect(a.descendants.has(b)).toBe(true);
+      expect(a.descendants.has(c)).toBe(true);
+    });
+
+    it('should handle diamond dependencies', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      const d = dg.addTransaction('d', 400, 40);
+      dg.addDependency(a, b);
+      dg.addDependency(a, c);
+      dg.addDependency(b, d);
+      dg.addDependency(c, d);
+
+      expect(d.ancestors.size).toBe(4);
+      expect(a.descendants.size).toBe(4);
+    });
+  });
+
+  describe('removeTransactions', () => {
+    it('should remove transactions and update sets', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+
+      dg.removeTransactions(new Set([b]));
+      expect(dg.size).toBe(2);
+      expect(dg.hasTx(b)).toBe(false);
+      expect(c.ancestors.has(b)).toBe(false);
+      expect(a.descendants.has(b)).toBe(false);
+    });
+
+    it('should handle slot reuse after removal', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      dg.addTransaction('b', 200, 20);
+      dg.removeTransactions(new Set([a]));
+      const c = dg.addTransaction('c', 300, 30);
+      expect(dg.size).toBe(2);
+      expect(c.txid).toBe('c');
+    });
+  });
+
+  describe('dependsOn (via ancestors)', () => {
+    it('should correctly identify dependencies', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+
+      expect(b.ancestors.has(a)).toBe(true);
+      expect(a.ancestors.has(b)).toBe(false);
+      expect(c.ancestors.has(a)).toBe(false);
+    });
+  });
+
+  describe('findConnectedComponents', () => {
+    it('should find a single component', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addDependency(a, b);
+
+      const components = dg.findConnectedComponents();
+      expect(components.length).toBe(1);
+      expect(components[0].size).toBe(2);
+    });
+
+    it('should find multiple disconnected components', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      const d = dg.addTransaction('d', 400, 40);
+      dg.addDependency(a, b);
+      dg.addDependency(c, d);
+
+      const components = dg.findConnectedComponents();
+      expect(components.length).toBe(2);
+    });
+
+    it('should handle isolated transactions', () => {
+      const dg = new DepGraph();
+      dg.addTransaction('a', 100, 10);
+      dg.addTransaction('b', 200, 20);
+      dg.addTransaction('c', 300, 30);
+
+      const components = dg.findConnectedComponents();
+      expect(components.length).toBe(3);
+    });
+  });
+
+  describe('parents / children (direct)', () => {
+    it('should return only direct parents, not transitive', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+
+      expect(c.parents.has(b)).toBe(true);
+      expect(c.parents.has(a)).toBe(false);
+    });
+
+    it('should return only direct children', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+
+      expect(a.children.has(b)).toBe(true);
+      expect(a.children.has(c)).toBe(false);
+    });
+  });
+
+  describe('appendTopo', () => {
+    it('should output in topological order', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+
+      const output = sortTopological(new Set([c, a, b]));
+      expect(output.indexOf(a)).toBeLessThan(output.indexOf(b));
+      expect(output.indexOf(b)).toBeLessThan(output.indexOf(c));
+    });
+  });
+
+  describe('restrict', () => {
+    it('should create a subgraph with correct deps', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addTransaction('c', 300, 30);
+      dg.addDependency(a, b);
+
+      const { depgraph: sub, txMap } = subgraph(new Set([a, b]));
+      expect(sub.size).toBe(2);
+      const newA = txMap.get(a)!;
+      const newB = txMap.get(b)!;
+      expect(newB.ancestors.has(newA)).toBe(true);
+    });
+  });
+
+  describe('graceful error handling', () => {
+    it('should no-op addDependency with non-member txs', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const dg2 = new DepGraph();
+      const foreign = dg2.addTransaction('foreign', 200, 20);
+      dg.addDependency(a, foreign);
+      dg.addDependency(foreign, a);
+      expect(a.ancestors.size).toBe(1);
+    });
+  });
+
+  describe('deep chain topology', () => {
+    it('should track ancestors and descendants at each depth', () => {
+      const { depgraph, txs } = buildChain(20, 100, 10);
+      expect(depgraph.size).toBe(20);
+
+      expect(txs[19].ancestors.size).toBe(20);
+      expect(txs[0].descendants.size).toBe(20);
+
+      expect(txs[10].ancestors.size).toBe(11);
+      expect(txs[10].descendants.size).toBe(10);
+
+      expect(txs[10].parents.size).toBe(1);
+      expect(txs[10].parents.has(txs[9])).toBe(true);
+      expect(txs[10].children.size).toBe(1);
+      expect(txs[10].children.has(txs[11])).toBe(true);
+    });
+  });
+
+  describe('wide fan-out topology', () => {
+    it('should track parent-children relationships for 1 parent with 10 children', () => {
+      const { depgraph, parent, children } = buildFanOut(10, 100, 10, 50, 10);
+      expect(depgraph.size).toBe(11);
+      expect(parent.children.size).toBe(10);
+      expect(parent.descendants.size).toBe(11);
+
+      for (const child of children) {
+        expect(child.ancestors.size).toBe(2);
+        expect(child.ancestors.has(parent)).toBe(true);
+        expect(child.parents.size).toBe(1);
+        expect(child.parents.has(parent)).toBe(true);
+      }
+    });
+  });
+
+  describe('wide fan-in topology', () => {
+    it('should track many parents converging to one child', () => {
+      const dg = new DepGraph();
+      const parents: any[] = [];
+      for (let i = 0; i < 10; i++) {
+        parents.push(dg.addTransaction(`p${i}`, 100, 10));
+      }
+      const child = dg.addTransaction('child', 500, 50);
+      for (const p of parents) {
+        dg.addDependency(p, child);
+      }
+
+      expect(child.parents.size).toBe(10);
+      expect(child.ancestors.size).toBe(11);
+      for (const p of parents) {
+        expect(p.descendants.has(child)).toBe(true);
+      }
+    });
+  });
+
+  describe('multiple diamonds in sequence', () => {
+    it('should handle A→B,C→D→E,F→G topology', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 100, 10);
+      const c = dg.addTransaction('c', 100, 10);
+      const d = dg.addTransaction('d', 100, 10);
+      const e = dg.addTransaction('e', 100, 10);
+      const f = dg.addTransaction('f', 100, 10);
+      const g = dg.addTransaction('g', 100, 10);
+      dg.addDependency(a, b);
+      dg.addDependency(a, c);
+      dg.addDependency(b, d);
+      dg.addDependency(c, d);
+      dg.addDependency(d, e);
+      dg.addDependency(d, f);
+      dg.addDependency(e, g);
+      dg.addDependency(f, g);
+
+      expect(g.ancestors.size).toBe(7);
+      expect(a.descendants.size).toBe(7);
+      expect(d.parents.size).toBe(2);
+      expect(d.children.size).toBe(2);
+    });
+  });
+
+  describe('disconnected subgraphs', () => {
+    it('should coexist in one DepGraph', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addDependency(a, b);
+
+      const c = dg.addTransaction('c', 300, 30);
+      const d = dg.addTransaction('d', 400, 40);
+      dg.addDependency(c, d);
+
+      const e = dg.addTransaction('e', 500, 50);
+
+      expect(dg.size).toBe(5);
+      expect(b.ancestors.has(c)).toBe(false);
+      expect(a.descendants.has(d)).toBe(false);
+      expect(e.ancestors.size).toBe(1);
+      expect(dg.findConnectedComponents().length).toBe(3);
+    });
+  });
+
+  describe('removeTransactions edge cases', () => {
+    it('should remove a leaf without affecting siblings', () => {
+      const { depgraph, parent, children } = buildFanOut(3, 100, 10, 50, 10);
+      depgraph.removeTransactions(new Set([children[2]]));
+      expect(depgraph.size).toBe(3);
+      expect(parent.children.size).toBe(2);
+      expect(depgraph.hasTx(children[0])).toBe(true);
+      expect(depgraph.hasTx(children[1])).toBe(true);
+    });
+
+    it('should remove a root without affecting unrelated txs', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addDependency(a, b);
+      dg.removeTransactions(new Set([a]));
+      expect(dg.size).toBe(1);
+      expect(dg.hasTx(a)).toBe(false);
+      expect(b.ancestors.size).toBe(1);
+      expect(b.ancestors.has(b)).toBe(true);
+    });
+
+    it('should break transitive edges when middle of chain is removed', () => {
+      const { depgraph, txs } = buildChain(5, 100, 10);
+      depgraph.removeTransactions(new Set([txs[2]]));
+      expect(depgraph.size).toBe(4);
+      expect(txs[0].descendants.has(txs[3])).toBe(false);
+      expect(txs[0].descendants.has(txs[1])).toBe(true);
+      expect(txs[3].ancestors.has(txs[0])).toBe(false);
+      expect(txs[3].descendants.has(txs[4])).toBe(true);
+    });
+
+    it('should handle batch removal of multiple txs', () => {
+      const { depgraph, txs } = buildChain(5, 100, 10);
+      depgraph.removeTransactions(new Set([txs[1], txs[3]]));
+      expect(depgraph.size).toBe(3);
+      expect(depgraph.hasTx(txs[0])).toBe(true);
+      expect(depgraph.hasTx(txs[2])).toBe(true);
+      expect(depgraph.hasTx(txs[4])).toBe(true);
+    });
+
+    it('should result in empty graph when all txs removed', () => {
+      const { depgraph, txs } = buildChain(3, 100, 10);
+      depgraph.removeTransactions(new Set(txs));
+      expect(depgraph.size).toBe(0);
+      expect(depgraph.getTxs().size).toBe(0);
+    });
+
+    it('should produce clean state when new tx is added after removal', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addDependency(a, b);
+      dg.removeTransactions(new Set([a]));
+
+      const c = dg.addTransaction('c', 300, 30);
+      expect(c.ancestors.size).toBe(1);
+      expect(c.descendants.size).toBe(1);
+      expect(c.ancestors.has(c)).toBe(true);
+      expect(b.ancestors.has(c)).toBe(false);
+    });
+  });
+
+  describe('findConnectedComponents after removal', () => {
+    it('should split into 2 components when bridge tx is removed', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+      const d = dg.addTransaction('d', 400, 40);
+      dg.addDependency(a, b);
+      dg.addDependency(b, c);
+      dg.addDependency(d, c);
+
+      dg.removeTransactions(new Set([b]));
+      const components = dg.findConnectedComponents();
+      expect(components.length).toBe(2);
+      const sizes = components.map(comp => comp.size).sort((x, y) => x - y);
+      expect(sizes).toEqual([1, 2]);
+    });
+
+    it('should produce N singletons when center of star is removed', () => {
+      const { depgraph, center, leaves } = buildStar(5, 100, 10, 50, 10);
+      depgraph.removeTransactions(new Set([center]));
+      const components = depgraph.findConnectedComponents();
+      expect(components.length).toBe(5);
+      for (const comp of components) {
+        expect(comp.size).toBe(1);
+      }
+    });
+
+    it('should remain 1 component when non-bridge tx is removed', () => {
+      const { depgraph, txs } = buildDiamond([100, 200, 300, 400], [10, 20, 30, 40]);
+      depgraph.removeTransactions(new Set([txs[1]]));
+      const components = depgraph.findConnectedComponents();
+      expect(components.length).toBe(1);
+    });
+  });
+
+  describe('restrict edge cases', () => {
+    it('should restrict to a single tx', () => {
+      const { depgraph, txs } = buildChain(3, 100, 10);
+      const { depgraph: sub } = subgraph(new Set([txs[1]]));
+      expect(sub.size).toBe(1);
+    });
+
+    it('should preserve edges within partial chain subset', () => {
+      const { depgraph, txs } = buildChain(5, 100, 10);
+      const subset = new Set([txs[0], txs[1], txs[2]]);
+      const { depgraph: sub, txMap } = subgraph(subset);
+      expect(sub.size).toBe(3);
+
+      const newA = txMap.get(txs[0]);
+      const newB = txMap.get(txs[1]);
+      const newC = txMap.get(txs[2]);
+      if (!newA || !newB || !newC) {
+        throw new Error('txMap missing entries');
+      }
+      expect(newB.ancestors.has(newA)).toBe(true);
+      expect(newC.ancestors.has(newB)).toBe(true);
+    });
+  });
+
+  describe('sortTopological edge cases', () => {
+    it('should handle subset with no internal dependencies', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      const c = dg.addTransaction('c', 300, 30);
+
+      const output = sortTopological(new Set([a, b, c]));
+      expect(output.length).toBe(3);
+      expect(new Set(output).size).toBe(3);
+    });
+
+    it('should handle single-tx subset', () => {
+      const { txs } = buildChain(3, 100, 10);
+      const output = sortTopological(new Set([txs[1]]));
+      expect(output).toEqual([txs[1]]);
+    });
+  });
+
+  describe('addDependency edge cases', () => {
+    it('should be idempotent when adding the same dependency twice', () => {
+      const dg = new DepGraph();
+      const a = dg.addTransaction('a', 100, 10);
+      const b = dg.addTransaction('b', 200, 20);
+      dg.addDependency(a, b);
+      dg.addDependency(a, b);
+      expect(b.ancestors.size).toBe(2);
+      expect(a.descendants.size).toBe(2);
+    });
+
+    it('should handle redundant edge when parent is already transitive ancestor', () => {
+      const { depgraph, txs } = buildChain(3, 100, 10);
+      depgraph.addDependency(txs[0], txs[2]);
+      expect(txs[2].ancestors.size).toBe(3);
+      expect(txs[2].parents.size).toBe(2);
+    });
+  });
+});

backend/src/__tests__/cluster-mempool/linearize.test.ts

@@ -0,0 +1,485 @@
+import { DepGraph } from '../../cluster-mempool/depgraph';
+import { chunkify, postLinearize, spanningForestLinearize, linearizeCluster } from '../../cluster-mempool/linearize';
+import { buildChain, buildFanOut, buildStar, verifyLinearization, verifyTopologicalOrder } from './test-utils';
+
+describe('chunkify', () => {
+  it('should create one chunk per tx when feerates are decreasing', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 300, 10);
+    const b = dg.addTransaction('b', 200, 10);
+    const c = dg.addTransaction('c', 100, 10);
+
+    const chunks = chunkify([a, b, c]);
+    expect(chunks.length).toBe(3);
+    expect(chunks[0].txs).toEqual([a]);
+    expect(chunks[1].txs).toEqual([b]);
+    expect(chunks[2].txs).toEqual([c]);
+  });
+
+  it('should merge all into one chunk when feerates are increasing', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 10);
+    const b = dg.addTransaction('b', 200, 10);
+    const c = dg.addTransaction('c', 300, 10);
+
+    const chunks = chunkify([a, b, c]);
+    expect(chunks.length).toBe(1);
+    expect(chunks[0].txs).toEqual([a, b, c]);
+    expect(chunks[0].fee).toBe(600);
+    expect(chunks[0].weight).toBe(30);
+  });
+
+  it('should NOT merge equal feerates (matching Core behavior)', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 10);
+    const b = dg.addTransaction('b', 100, 10);
+
+    const chunks = chunkify([a, b]);
+    expect(chunks.length).toBe(2);
+  });
+
+  it('should handle single transaction', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 500, 50);
+
+    const chunks = chunkify([a]);
+    expect(chunks.length).toBe(1);
+    expect(chunks[0].fee).toBe(500);
+    expect(chunks[0].weight).toBe(50);
+  });
+
+  it('should handle empty linearization', () => {
+    const chunks = chunkify([]);
+    expect(chunks.length).toBe(0);
+  });
+
+  it('should produce decreasing chunk feerates', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 10);
+    const b = dg.addTransaction('b', 50, 10);
+    const c = dg.addTransaction('c', 200, 10);
+    const d = dg.addTransaction('d', 30, 10);
+
+    const chunks = chunkify([a, c, b, d]);
+    for (let i = 1; i < chunks.length; i++) {
+      const prevRate = chunks[i - 1].fee / chunks[i - 1].weight;
+      const curRate = chunks[i].fee / chunks[i].weight;
+      expect(prevRate).toBeGreaterThanOrEqual(curRate);
+    }
+  });
+});
+
+describe('postLinearize', () => {
+  it('should improve a bad linearization', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 500, 100);
+
+    const result = postLinearize([a, b]);
+    expect(result[0]).toBe(b);
+    expect(result[1]).toBe(a);
+  });
+
+  it('should not violate dependencies', () => {
+    const dg = new DepGraph();
+    const parent = dg.addTransaction('parent', 100, 100);
+    const child = dg.addTransaction('child', 500, 100);
+    dg.addDependency(parent, child);
+
+    const result = postLinearize([parent, child]);
+    expect(result.indexOf(parent)).toBeLessThan(result.indexOf(child));
+  });
+
+  it('should handle already-optimal ordering', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 500, 100);
+    const b = dg.addTransaction('b', 100, 100);
+
+    const result = postLinearize([a, b]);
+    expect(result).toEqual([a, b]);
+  });
+});
+
+describe('spanningForestLinearize', () => {
+  it('should sort independent transactions by feerate', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 500, 100);
+    const c = dg.addTransaction('c', 300, 100);
+
+    const result = spanningForestLinearize(dg.getTxs(), 75000);
+    expect(result[0]).toBe(b);
+    expect(result[1]).toBe(c);
+    expect(result[2]).toBe(a);
+  });
+
+  it('should respect dependencies', () => {
+    const dg = new DepGraph();
+    const parent = dg.addTransaction('parent', 100, 100);
+    const child = dg.addTransaction('child', 500, 100);
+    dg.addDependency(parent, child);
+
+    const result = spanningForestLinearize(dg.getTxs(), 75000);
+    expect(result.indexOf(parent)).toBeLessThan(result.indexOf(child));
+  });
+
+  it('should handle CPFP pattern', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 10000, 100);
+    dg.addDependency(a, b);
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(1);
+    expect(chunks[0].txs.length).toBe(2);
+  });
+
+  it('should separate high and low feerate independent txs', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 100);
+    const b = dg.addTransaction('b', 100, 100);
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(2);
+    expect(chunks[0].txs).toContain(a);
+    expect(chunks[1].txs).toContain(b);
+  });
+
+  it('should handle single transaction', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 100);
+
+    const result = spanningForestLinearize(dg.getTxs(), 75000);
+    expect(result).toEqual([a]);
+  });
+
+  it('should handle empty graph', () => {
+    const dg = new DepGraph();
+    const result = spanningForestLinearize(dg.getTxs(), 75000);
+    expect(result).toEqual([]);
+  });
+});
+
+describe('minimize', () => {
+  it('should keep equal-feerate chain as individual chunks', () => {
+    const dg = new DepGraph();
+    const txs: any[] = [];
+    for (let i = 0; i < 5; i++) {
+      txs.push(dg.addTransaction(`tx${i}`, 19, 140));
+    }
+    for (let i = 1; i < 5; i++) {
+      dg.addDependency(txs[i - 1], txs[i]);
+    }
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(5);
+    for (const chunk of chunks) {
+      expect(chunk.txs.length).toBe(1);
+    }
+  });
+
+  it('should merge chain where child has strictly higher feerate', () => {
+    const dg = new DepGraph();
+    const parent = dg.addTransaction('parent', 100, 200);
+    const child = dg.addTransaction('child', 900, 100);
+    dg.addDependency(parent, child);
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(1);
+    expect(chunks[0].txs.length).toBe(2);
+  });
+
+  it('should split parent-child with equal feerate', () => {
+    const dg = new DepGraph();
+    const parent = dg.addTransaction('parent', 1720, 344);
+    const child = dg.addTransaction('child', 1240, 248);
+    dg.addDependency(parent, child);
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(2);
+  });
+
+  it('should split disconnected equal-feerate components', () => {
+    const dg = new DepGraph();
+    dg.addTransaction('a', 100, 100);
+    dg.addTransaction('b', 100, 100);
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    expect(chunks.length).toBe(2);
+    expect(chunks[0].txs.length).toBe(1);
+    expect(chunks[1].txs.length).toBe(1);
+  });
+});
+
+describe('chunkify edge cases', () => {
+  it('should produce N separate chunks when all feerates are equal', () => {
+    const dg = new DepGraph();
+    const txs: any[] = [];
+    for (let i = 0; i < 5; i++) {
+      txs.push(dg.addTransaction(`tx${i}`, 100, 10));
+    }
+    const chunks = chunkify(txs);
+    expect(chunks.length).toBe(5);
+  });
+
+  it('should handle alternating high/low feerates', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 10);
+    const b = dg.addTransaction('b', 100, 10);
+    const c = dg.addTransaction('c', 1000, 10);
+    const d = dg.addTransaction('d', 100, 10);
+
+    const chunks = chunkify([a, b, c, d]);
+    for (let i = 1; i < chunks.length; i++) {
+      const prevRate = chunks[i - 1].fee * chunks[i].weight;
+      const curRate = chunks[i].fee * chunks[i - 1].weight;
+      expect(prevRate).toBeGreaterThanOrEqual(curRate);
+    }
+  });
+
+  it('should merge all when single very high feerate tx is at the end', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 10, 10);
+    const b = dg.addTransaction('b', 10, 10);
+    const c = dg.addTransaction('c', 10, 10);
+    const d = dg.addTransaction('d', 10000, 10);
+
+    const chunks = chunkify([a, b, c, d]);
+    expect(chunks.length).toBe(1);
+    expect(chunks[0].txs.length).toBe(4);
+  });
+
+  it('should maintain non-increasing feerates for 50+ tx linearization', () => {
+    const dg = new DepGraph();
+    const txs: any[] = [];
+    for (let i = 0; i < 50; i++) {
+      txs.push(dg.addTransaction(`tx${i}`, 5000 - i * 100, 100));
+    }
+    const chunks = chunkify(txs);
+    for (let i = 1; i < chunks.length; i++) {
+      const prevRate = chunks[i - 1].fee * chunks[i].weight;
+      const curRate = chunks[i].fee * chunks[i - 1].weight;
+      expect(prevRate).toBeGreaterThanOrEqual(curRate);
+    }
+  });
+
+  it('should handle zero-fee transaction', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 100);
+    const b = dg.addTransaction('b', 0, 100);
+
+    const chunks = chunkify([a, b]);
+    expect(chunks.length).toBe(2);
+    expect(chunks[1].fee).toBe(0);
+  });
+});
+
+describe('postLinearize edge cases', () => {
+  it('should sort three independent txs by feerate', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 500, 100);
+    const c = dg.addTransaction('c', 300, 100);
+
+    const result = postLinearize([a, c, b]);
+    expect(result[0]).toBe(b);
+    expect(result[2]).toBe(a);
+  });
+
+  it('should respect parent-child dependency even when child has higher feerate', () => {
+    const dg = new DepGraph();
+    const parent = dg.addTransaction('parent', 100, 100);
+    const child = dg.addTransaction('child', 1000, 100);
+    dg.addDependency(parent, child);
+
+    const result = postLinearize([parent, child]);
+    expect(result[0]).toBe(parent);
+    expect(result[1]).toBe(child);
+  });
+
+  it('should handle chain A→B→C with CPFP-like feerates', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 200, 100);
+    const c = dg.addTransaction('c', 10000, 100);
+    dg.addDependency(a, b);
+    dg.addDependency(b, c);
+
+    const result = postLinearize([a, b, c]);
+    verifyTopologicalOrder(result);
+  });
+});
+
+describe('SFL adversarial topologies', () => {
+  it('should handle comb pattern: one root with many children at different feerates', () => {
+    const dg = new DepGraph();
+    const root = dg.addTransaction('root', 100, 100);
+    for (let i = 0; i < 8; i++) {
+      const child = dg.addTransaction(`child${i}`, (i + 1) * 500, 100);
+      dg.addDependency(root, child);
+    }
+
+    const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000);
+    verifyLinearization(dg.getTxs(), linearization, chunks);
+  });
+
+  it('should handle inverted tree: many leaves → intermediates → root', () => {
+    const dg = new DepGraph();
+    const root = dg.addTransaction('root', 100, 100);
+    const mid1 = dg.addTransaction('mid1', 200, 100);
+    const mid2 = dg.addTransaction('mid2', 300, 100);
+    dg.addDependency(root, mid1);
+    dg.addDependency(root, mid2);
+
+    for (let i = 0; i < 4; i++) {
+      const leaf = dg.addTransaction(`leaf${i}`, 5000, 100);
+      dg.addDependency(i < 2 ? mid1 : mid2, leaf);
+    }
+
+    const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000);
+    verifyLinearization(dg.getTxs(), linearization, chunks);
+  });
+
+  it('should handle two parallel chains with shared root', () => {
+    const dg = new DepGraph();
+    const root = dg.addTransaction('root', 100, 100);
+
+    let prev1: any = root;
+    for (let i = 0; i < 5; i++) {
+      const tx = dg.addTransaction(`chain1_${i}`, 200, 100);
+      dg.addDependency(prev1, tx);
+      prev1 = tx;
+    }
+
+    let prev2: any = root;
+    for (let i = 0; i < 3; i++) {
+      const tx = dg.addTransaction(`chain2_${i}`, 300, 100);
+      dg.addDependency(prev2, tx);
+      prev2 = tx;
+    }
+
+    const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000);
+    verifyLinearization(dg.getTxs(), linearization, chunks);
+  });
+
+  it('should handle deep CPFP: low-fee chain with high-fee tip', () => {
+    const { depgraph, txs } = buildChain(6, 10, 100);
+    txs[5].effectiveFee = 50000;
+
+    const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), 75000);
+    verifyLinearization(depgraph.getTxs(), linearization, chunks);
+    expect(chunks[0].txs.length).toBeGreaterThan(1);
+  });
+
+  it('should find better result than ancestor-feerate for overlapping high-feerate subsets', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 100, 100);
+    const c = dg.addTransaction('c', 10000, 100);
+    dg.addDependency(a, c);
+    dg.addDependency(b, c);
+
+    const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000 );
+    verifyLinearization(dg.getTxs(), linearization, chunks);
+    const firstChunkFee = chunks[0].fee;
+    const firstChunkSize = chunks[0].weight;
+    expect(firstChunkFee / firstChunkSize).toBeGreaterThan(100 / 100);
+  });
+});
+
+describe('linearizeCluster', () => {
+  it('should produce valid topological linearization', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 10);
+    const b = dg.addTransaction('b', 200, 20);
+    const c = dg.addTransaction('c', 300, 30);
+    dg.addDependency(a, b);
+    dg.addDependency(b, c);
+
+    const { linearization } = linearizeCluster(dg.getTxs(), 75000);
+    expect(linearization.indexOf(a)).toBeLessThan(linearization.indexOf(b));
+    expect(linearization.indexOf(b)).toBeLessThan(linearization.indexOf(c));
+  });
+
+  it('should produce monotonically decreasing chunk feerates', () => {
+    const dg = new DepGraph();
+    for (let i = 0; i < 10; i++) {
+      dg.addTransaction(`tx${i}`, Math.floor(Math.random() * 10000) + 100, Math.floor(Math.random() * 500) + 50);
+    }
+
+    const { chunks } = linearizeCluster(dg.getTxs(), 75000);
+    for (let i = 1; i < chunks.length; i++) {
+      const prevRate = chunks[i - 1].fee * chunks[i].weight;
+      const curRate = chunks[i].fee * chunks[i - 1].weight;
+      expect(prevRate).toBeGreaterThanOrEqual(curRate);
+    }
+  });
+
+  it('should handle complex diamond dependency graph', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 100);
+    const b = dg.addTransaction('b', 500, 100);
+    const c = dg.addTransaction('c', 100, 100);
+    const d = dg.addTransaction('d', 300, 100);
+
+    dg.addDependency(a, b);
+    dg.addDependency(a, c);
+    dg.addDependency(b, d);
+    dg.addDependency(c, d);
+
+    const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000);
+    verifyLinearization(dg.getTxs(), linearization, chunks);
+  });
+
+  it('should produce at-least-as-good result when given suboptimal hint', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 500, 100);
+    const b = dg.addTransaction('b', 100, 100);
+    const c = dg.addTransaction('c', 1000, 100);
+
+    const suboptimal = [b, a, c];
+    const { chunks: hintChunks } = linearizeCluster(dg.getTxs(), 75000, suboptimal);
+    const { chunks: freshChunks } = linearizeCluster(dg.getTxs(), 75000);
+
+    const hintFirstFeerate = hintChunks[0].fee * freshChunks[0].weight;
+    const freshFirstFeerate = freshChunks[0].fee * hintChunks[0].weight;
+    expect(hintFirstFeerate).toBeGreaterThanOrEqual(freshFirstFeerate - 1);
+  });
+
+  it('should preserve an already-optimal linearization', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 1000, 100);
+    const b = dg.addTransaction('b', 500, 100);
+    const c = dg.addTransaction('c', 100, 100);
+
+    const optimal = [a, b, c];
+    const { linearization } = linearizeCluster(dg.getTxs(), 75000, optimal);
+    expect(linearization).toEqual(optimal);
+  });
+
+  it('should produce valid linearizations on repeated calls', () => {
+    const dg = new DepGraph();
+    const a = dg.addTransaction('a', 100, 100);
+    const b = dg.addTransaction('b', 100, 100);
+    const c = dg.addTransaction('c', 100, 100);
+    dg.addDependency(a, c);
+    dg.addDependency(b, c);
+
+    const result1 = linearizeCluster(dg.getTxs(), 75000);
+    const result2 = linearizeCluster(dg.getTxs(), 75000);
+    verifyLinearization(dg.getTxs(), result1.linearization, result1.chunks);
+    verifyLinearization(dg.getTxs(), result2.linearization, result2.chunks);
+  });
+
+  it('should pass invariant checks for fan-out topology', () => {
+    const { depgraph } = buildFanOut(6, 100, 100, 500, 100);
+    const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), 75000);
+    verifyLinearization(depgraph.getTxs(), linearization, chunks);
+  });
+
+  it('should pass invariant checks for star topology', () => {
+    const { depgraph } = buildStar(5, 100, 100, 300, 100);
+    const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), 75000);
+    verifyLinearization(depgraph.getTxs(), linearization, chunks);
+  });
+});

backend/src/__tests__/cluster-mempool/test-utils.ts

@@ -0,0 +1,175 @@
+import { MempoolTransactionExtended } from '../../mempool.interfaces';
+import { ClusterTx, DepGraph } from '../../cluster-mempool/depgraph';
+import { LinearizationChunk } from '../../cluster-mempool/linearize';
+
+export function makeTx(
+  txid: string,
+  fee: number,
+  vsize: number,
+  parentTxids: string[] = [],
+): MempoolTransactionExtended {
+  const vin = parentTxids.length > 0
+    ? parentTxids.map(ptxid => ({
+        txid: ptxid,
+        vout: 0,
+        is_coinbase: false,
+        scriptsig: '',
+        scriptsig_asm: '',
+        inner_redeemscript_asm: '',
+        inner_witnessscript_asm: '',
+        sequence: 0,
+        witness: [] as string[],
+        prevout: null,
+      }))
+    : [{
+        txid: '0000000000000000000000000000000000000000000000000000000000000000',
+        vout: 0,
+        is_coinbase: false,
+        scriptsig: '',
+        scriptsig_asm: '',
+        inner_redeemscript_asm: '',
+        inner_witnessscript_asm: '',
+        sequence: 0,
+        witness: [] as string[],
+        prevout: null,
+      }];
+
+  return {
+    txid,
+    version: 2,
+    locktime: 0,
+    size: vsize,
+    weight: vsize * 4,
+    fee,
+    vin,
+    vout: [{
+      scriptpubkey: '',
+      scriptpubkey_asm: '',
+      scriptpubkey_type: 'v0_p2wpkh',
+      value: 50000,
+    }],
+    status: { confirmed: false },
+    vsize,
+    feePerVsize: fee / vsize,
+    effectiveFeePerVsize: fee / vsize,
+    order: 0,
+    sigops: 0,
+    adjustedVsize: vsize,
+    adjustedFeePerVsize: fee / vsize,
+  } as MempoolTransactionExtended;
+}
+
+export function txid(short: string): string {
+  return short.padStart(64, '0');
+}
+
+export function buildChain(
+  n: number,
+  baseFee: number,
+  baseSize: number,
+): { depgraph: DepGraph; txs: ClusterTx[] } {
+  const depgraph = new DepGraph();
+  const txs: ClusterTx[] = [];
+  for (let i = 0; i < n; i++) {
+    txs.push(depgraph.addTransaction(`chain_${i}`, baseFee, baseSize));
+  }
+  for (let i = 1; i < n; i++) {
+    depgraph.addDependency(txs[i - 1], txs[i]);
+  }
+  return { depgraph, txs };
+}
+
+export function buildFanOut(
+  nChildren: number,
+  parentFee: number,
+  parentSize: number,
+  childFee: number,
+  childSize: number,
+): { depgraph: DepGraph; parent: ClusterTx; children: ClusterTx[] } {
+  const depgraph = new DepGraph();
+  const parent = depgraph.addTransaction('fanout_parent', parentFee, parentSize);
+  const children: ClusterTx[] = [];
+  for (let i = 0; i < nChildren; i++) {
+    const child = depgraph.addTransaction(`fanout_child_${i}`, childFee, childSize);
+    depgraph.addDependency(parent, child);
+    children.push(child);
+  }
+  return { depgraph, parent, children };
+}
+
+export function buildDiamond(
+  fees: [number, number, number, number],
+  sizes: [number, number, number, number],
+): { depgraph: DepGraph; txs: [ClusterTx, ClusterTx, ClusterTx, ClusterTx] } {
+  const depgraph = new DepGraph();
+  const a = depgraph.addTransaction('diamond_a', fees[0], sizes[0]);
+  const b = depgraph.addTransaction('diamond_b', fees[1], sizes[1]);
+  const c = depgraph.addTransaction('diamond_c', fees[2], sizes[2]);
+  const d = depgraph.addTransaction('diamond_d', fees[3], sizes[3]);
+  depgraph.addDependency(a, b);
+  depgraph.addDependency(a, c);
+  depgraph.addDependency(b, d);
+  depgraph.addDependency(c, d);
+  return { depgraph, txs: [a, b, c, d] };
+}
+
+export function buildStar(
+  nLeaves: number,
+  centerFee: number,
+  centerSize: number,
+  leafFee: number,
+  leafSize: number,
+): { depgraph: DepGraph; center: ClusterTx; leaves: ClusterTx[] } {
+  const depgraph = new DepGraph();
+  const center = depgraph.addTransaction('star_center', centerFee, centerSize);
+  const leaves: ClusterTx[] = [];
+  for (let i = 0; i < nLeaves; i++) {
+    const leaf = depgraph.addTransaction(`star_leaf_${i}`, leafFee, leafSize);
+    depgraph.addDependency(center, leaf);
+    leaves.push(leaf);
+  }
+  return { depgraph, center, leaves };
+}
+
+export function verifyTopologicalOrder(ordering: ClusterTx[]): void {
+  const positionMap = new Map<ClusterTx, number>();
+  for (let i = 0; i < ordering.length; i++) {
+    positionMap.set(ordering[i], i);
+  }
+  for (const tx of ordering) {
+    for (const parent of tx.parents) {
+      const parentPos = positionMap.get(parent);
+      const childPos = positionMap.get(tx);
+      if (parentPos !== undefined && childPos !== undefined) {
+        expect(parentPos).toBeLessThan(childPos);
+      }
+    }
+  }
+}
+
+export function verifyLinearization(
+  txs: Set<ClusterTx>,
+  linearization: ClusterTx[],
+  chunks: LinearizationChunk[],
+): void {
+  expect(linearization.length).toBe(txs.size);
+  const linSet = new Set(linearization);
+  expect(linSet.size).toBe(linearization.length);
+  for (const tx of txs) {
+    expect(linSet.has(tx)).toBe(true);
+  }
+
+  verifyTopologicalOrder(linearization);
+
+  for (let i = 1; i < chunks.length; i++) {
+    const prevRate = chunks[i - 1].fee * chunks[i].weight;
+    const curRate = chunks[i].fee * chunks[i - 1].weight;
+    expect(prevRate).toBeGreaterThanOrEqual(curRate);
+  }
+
+  const chunkTxs = chunks.flatMap(c => c.txs);
+  expect(chunkTxs.length).toBe(linearization.length);
+  for (let i = 0; i < chunkTxs.length; i++) {
+    expect(chunkTxs[i]).toBe(linearization[i]);
+  }
+}

backend/src/__tests__/config.test.ts

@@ -45,6 +45,8 @@ describe('Mempool Backend Config', () => {
         AUDIT: false,
         RUST_GBT: true,
         LIMIT_GBT: false,
+        CLUSTER_MEMPOOL: false,
+        CLUSTER_MEMPOOL_INDEXING: false,
         CPFP_INDEXING: false,
         MAX_BLOCKS_BULK_QUERY: 0,
         DISK_CACHE_BLOCK_INTERVAL: 6,

backend/src/api/audit.ts

@@ -6,11 +6,28 @@ import transactionUtils from './transaction-utils';
 
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
 
+export interface AuditResult {
+  unseen: string[];
+  censored: string[];
+  added: string[];
+  prioritized: string[];
+  fresh: string[];
+  sigop: string[];
+  fullrbf: string[];
+  accelerated: string[];
+  matchRate: number;
+  similarity: number;
+}
+
 class Audit {
-  auditBlock(height: number, transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended })
-   : { unseen: string[], censored: string[], added: string[], prioritized: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } {
+  auditBlock(
+    height: number,
+    transactions: MempoolTransactionExtended[],
+    projectedBlocks: MempoolBlockWithTransactions[],
+    mempool: { [txId: string]: MempoolTransactionExtended }
+  ): AuditResult {
     if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
-      return { unseen: [], censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 1, similarity: 1 };
+      return { unseen: [], censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], matchRate: 100, similarity: 1 };
     }
 
     const matches: string[] = []; // present in both mined block and template
@@ -176,6 +193,8 @@ class Audit {
     }
     const similarity = projectedWeight ? matchedWeight / projectedWeight : 1;
 
+    const matchRate = Math.round(score * 100 * 100) / 100;
+
     return {
       unseen,
       censored: Object.keys(isCensored),
@@ -185,7 +204,7 @@ class Audit {
       sigop: [],
       fullrbf: rbf,
       accelerated,
-      score,
+      matchRate,
       similarity,
     };
   }

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -12,7 +12,7 @@ import backendInfo from '../backend-info';
 import transactionUtils from '../transaction-utils';
 import { IEsploraApi } from './esplora-api.interface';
 import loadingIndicators from '../loading-indicators';
-import { TransactionExtended } from '../../mempool.interfaces';
+import { CpfpInfo, TransactionExtended } from '../../mempool.interfaces';
 import logger from '../../logger';
 import blocks from '../blocks';
 import bitcoinClient from './bitcoin-client';
@@ -191,11 +191,11 @@ class BitcoinRoutes {
     const tx = mempool.getMempool()[req.params.txId];
     if (tx) {
       if (tx?.cpfpChecked) {
-        res.json({
-          ancestors: tx.ancestors,
+        const response: CpfpInfo & { acceleratedBy?: number[], acceleratedAt?: number, feeDelta?: number } = {
+          ancestors: tx.ancestors || [],
           bestDescendant: tx.bestDescendant || null,
-          descendants: tx.descendants || null,
-          effectiveFeePerVsize: tx.effectiveFeePerVsize || null,
+          descendants: tx.descendants,
+          effectiveFeePerVsize: tx.effectiveFeePerVsize,
           sigops: tx.sigops,
           fee: tx.fee,
           adjustedVsize: tx.adjustedVsize,
@@ -203,7 +203,14 @@ class BitcoinRoutes {
           acceleratedBy: tx.acceleratedBy || undefined,
           acceleratedAt: tx.acceleratedAt || undefined,
           feeDelta: tx.feeDelta || undefined,
-        });
+        };
+        if (config.MEMPOOL.CLUSTER_MEMPOOL && tx.clusterId != null) {
+          const cluster = mempool.clusterMempool?.getClusterForApi(req.params.txId);
+          if (cluster) {
+            response.cluster = cluster;
+          }
+        }
+        res.json(response);
         return;
       }
 

backend/src/api/block-processor.ts

@@ -0,0 +1,262 @@
+import config from '../config';
+import logger from '../logger';
+import {
+  BlockExtended,
+  BlockSummary,
+  PoolTag,
+  MempoolTransactionExtended,
+  CpfpSummary,
+  TemplateAlgorithm,
+  MempoolBlockWithTransactions,
+} from '../mempool.interfaces';
+import { IEsploraApi } from './bitcoin/esplora-api.interface';
+import { Acceleration } from './services/acceleration';
+import { calculateGoodBlockCpfp, calculateClusterMempoolBlockCpfp, calculateFastBlockCpfp, BlockCpfpData } from './cpfp';
+import mempoolBlocks from './mempool-blocks';
+import memPool from './mempool';
+import Audit, { AuditResult } from './audit';
+import blocks from './blocks';
+import transactionUtils from './transaction-utils';
+import { ClusterMempool } from '../cluster-mempool/cluster-mempool';
+import { Common } from './common';
+import accelerationApi from './services/acceleration';
+
+interface ProcessedAudit extends AuditResult {
+  expectedFees: number;
+  expectedWeight: number;
+  projectedBlocks: MempoolBlockWithTransactions[];
+}
+
+export interface BlockProcessingResult {
+  templateAlgorithm: TemplateAlgorithm;
+  cpfpSummary: CpfpSummary;
+  blockExtended: BlockExtended;
+  blockSummary: BlockSummary;
+  auditResult?: ProcessedAudit;
+}
+
+const CM_ACTIVATION_HEIGHT: { [network: string]: number } = {
+  'mainnet': 940000,
+  'testnet': 4860000,
+  'testnet4': 125000,
+  'signet': 294000,
+  'regtest': 0,
+};
+
+class BlockProcessor {
+
+  /** @asyncUnsafe */
+  public async $processNewBlock(
+    block: IEsploraApi.Block,
+    transactions: MempoolTransactionExtended[],
+    pool: PoolTag,
+    accelerations: Record<string, Acceleration>
+  ): Promise<BlockProcessingResult> {
+    const poolAccelerations = Object.values(accelerations)
+      .filter(a => a.pools.includes(pool.uniqueId))
+      .map(a => ({ txid: a.txid, max_bid: a.feeDelta }));
+
+    const { templateAlgorithm, cpfpSummary } = detectTemplateAlgorithm(
+      block.height,
+      transactions,
+      poolAccelerations
+    );
+
+
+    const blockExtended = await blocks.$getBlockExtended(block, cpfpSummary.transactions, pool);
+    const blockSummary = blocks.summarizeBlockTransactions(block.id, block.height, cpfpSummary.transactions);
+
+    let auditResult: ProcessedAudit | undefined;
+    if (config.MEMPOOL.AUDIT && memPool.isInSync()) {
+      auditResult = await this.$runAudit(
+        blockExtended,
+        transactions,
+        templateAlgorithm,
+        pool,
+        accelerations
+      );
+
+      if (blockExtended.extras) {
+        blockExtended.extras.matchRate = auditResult.matchRate;
+        blockExtended.extras.expectedFees = auditResult.expectedFees;
+        blockExtended.extras.expectedWeight = auditResult.expectedWeight;
+        blockExtended.extras.similarity = auditResult.similarity;
+      }
+    } else if (blockExtended.extras) {
+      const mBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
+      if (mBlocks?.length && mBlocks[0].transactions) {
+        blockExtended.extras.similarity = Common.getSimilarity(mBlocks[0], transactions);
+      }
+    }
+
+    return {
+      templateAlgorithm,
+      cpfpSummary,
+      blockExtended,
+      blockSummary,
+      auditResult,
+    };
+  }
+
+  private async $runAudit(
+    block: BlockExtended,
+    transactions: MempoolTransactionExtended[],
+    templateAlgorithm: TemplateAlgorithm,
+    pool: PoolTag,
+    accelerations: Record<string, Acceleration>
+  ): Promise<ProcessedAudit> {
+    const auditMempool = memPool.getMempool();
+    const isAccelerated = accelerationApi.isAcceleratedBlock(block, Object.values(accelerations));
+
+    const candidateTxs = memPool.getMempoolCandidates();
+    const candidates = (memPool.limitGBT && candidateTxs)
+      ? { txs: candidateTxs, added: [], removed: [] }
+      : undefined;
+    const transactionIds = (memPool.limitGBT)
+      ? Object.keys(candidates?.txs || {})
+      : Object.keys(auditMempool);
+
+    let projectedBlocks: MempoolBlockWithTransactions[];
+
+    if (templateAlgorithm === TemplateAlgorithm.clusterMempool) {
+      const clusterMempool = memPool.clusterMempool ?? new ClusterMempool(auditMempool, accelerations, true, 75000);
+      const cmBlocks = clusterMempool.getBlocks(config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT) ?? [];
+      projectedBlocks = mempoolBlocks.processClusterMempoolBlocks(
+        cmBlocks,
+        auditMempool,
+        accelerations,
+        false,
+        pool.uniqueId
+      );
+    } else if (config.MEMPOOL.RUST_GBT) {
+      const added = memPool.limitGBT ? (candidates?.added || []) : [];
+      const removed = memPool.limitGBT ? (candidates?.removed || []) : [];
+      projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(
+        transactionIds,
+        auditMempool,
+        added,
+        removed,
+        candidates,
+        isAccelerated,
+        pool.uniqueId,
+        true
+      );
+    } else {
+      projectedBlocks = await mempoolBlocks.$makeBlockTemplates(
+        transactionIds,
+        auditMempool,
+        candidates,
+        false,
+        isAccelerated,
+        pool.uniqueId
+      );
+    }
+
+    const auditResult = Audit.auditBlock(block.height, transactions, projectedBlocks, auditMempool);
+
+    const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
+
+    let totalFees = 0;
+    let totalWeight = 0;
+    for (const tx of stripped) {
+      totalFees += tx.fee;
+      totalWeight += (tx.vsize * 4);
+    }
+
+    return {
+      ...auditResult,
+      expectedFees: totalFees,
+      expectedWeight: totalWeight,
+      projectedBlocks,
+    };
+  }
+}
+
+function saveCpfpDataToTransactions(transactions: MempoolTransactionExtended[], cpfpData: BlockCpfpData): void {
+  for (const tx of transactions) {
+    if (cpfpData.txs[tx.txid]) {
+      Object.assign(tx, cpfpData.txs[tx.txid]);
+    }
+  }
+}
+
+export function saveCpfpDataToCpfpSummary(transactions: MempoolTransactionExtended[], cpfpData: BlockCpfpData): CpfpSummary {
+  saveCpfpDataToTransactions(transactions, cpfpData);
+  return {
+    transactions,
+    clusters: cpfpData.clusters,
+    version: cpfpData.version,
+  };
+}
+
+/**
+ *
+ * @param height
+ * @param blockTransactions
+ * @param poolAccelerations
+ * @param fast
+ *
+ * saves effective fee rates from detected algorithm to blockTransactions
+ */
+export function detectTemplateAlgorithm(
+  height: number,
+  blockTransactions: MempoolTransactionExtended[],
+  poolAccelerations: { txid: string; max_bid: number }[],
+  fast: boolean = false
+): { templateAlgorithm: TemplateAlgorithm; cpfpSummary: CpfpSummary } {
+
+  const legacyCpfpData = fast ? calculateFastBlockCpfp(
+    height,
+    blockTransactions,
+  ) : calculateGoodBlockCpfp(
+    height,
+    blockTransactions,
+    poolAccelerations
+  );
+
+  if (!config.MEMPOOL.CLUSTER_MEMPOOL_INDEXING) {
+    return {
+      templateAlgorithm: TemplateAlgorithm.legacy,
+      cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, legacyCpfpData),
+    };
+  }
+
+  const network = config.MEMPOOL.NETWORK || 'mainnet';
+  const activationHeight = CM_ACTIVATION_HEIGHT[network] ?? Infinity;
+
+  if (height < activationHeight) {
+    return {
+      templateAlgorithm: TemplateAlgorithm.legacy,
+      cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, legacyCpfpData),
+    };
+  }
+
+  const clusterCpfpData = calculateClusterMempoolBlockCpfp(
+    height,
+    blockTransactions,
+    poolAccelerations
+  );
+
+  const clusterTxs = blockTransactions.map(tx => ({ txid: tx.txid, rate: clusterCpfpData.txs[tx.txid].effectiveFeePerVsize ?? tx.effectiveFeePerVsize }));
+  const legacyTxs = blockTransactions.map(tx => ({ txid: tx.txid, rate: legacyCpfpData.txs[tx.txid].effectiveFeePerVsize ?? tx.effectiveFeePerVsize }));
+  const clusterPrioritization = transactionUtils.identifyPrioritizedTransactions(clusterTxs, 'rate');
+  const legacyPrioritization = transactionUtils.identifyPrioritizedTransactions(legacyTxs, 'rate');
+
+  const clusterCount = clusterPrioritization.prioritized.length + clusterPrioritization.deprioritized.length;
+  const legacyCount = legacyPrioritization.prioritized.length + legacyPrioritization.deprioritized.length;
+
+  if (clusterCount < legacyCount) {
+    saveCpfpDataToTransactions(blockTransactions, clusterCpfpData);
+    return {
+      templateAlgorithm: TemplateAlgorithm.clusterMempool,
+      cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, clusterCpfpData),
+    };
+  } else {
+    return {
+      templateAlgorithm: TemplateAlgorithm.legacy,
+      cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, legacyCpfpData),
+    };
+  }
+}
+
+export default new BlockProcessor();

backend/src/api/blocks.ts

@@ -2,7 +2,7 @@ import config from '../config';
 import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 import logger from '../logger';
 import memPool from './mempool';
-import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified, BlockAudit, TransactionAudit } from '../mempool.interfaces';
+import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified, BlockAudit, TransactionAudit, TemplateAlgorithm } from '../mempool.interfaces';
 import { Common } from './common';
 import diskCache from './disk-cache';
 import transactionUtils from './transaction-utils';
@@ -28,9 +28,13 @@ import chainTips from './chain-tips';
 import websocketHandler from './websocket-handler';
 import redisCache from './redis-cache';
 import rbfCache from './rbf-cache';
+import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
+import mempoolBlocks from './mempool-blocks';
+import statistics from './statistics/statistics';
 import { calcBitsDifference } from './difficulty-adjustment';
 import AccelerationRepository from '../repositories/AccelerationRepository';
-import { calculateFastBlockCpfp, calculateGoodBlockCpfp } from './cpfp';
+import { calculateGoodBlockCpfp } from './cpfp';
+import blockProcessor, { BlockProcessingResult, detectTemplateAlgorithm, saveCpfpDataToCpfpSummary } from './block-processor';
 import mempool from './mempool';
 import CpfpRepository from '../repositories/CpfpRepository';
 import { parseDATUMTemplateCreator } from '../utils/bitcoin-script';
@@ -46,7 +50,6 @@ class Blocks {
   private previousDifficultyRetarget = 0;
   private quarterEpochBlockTime: number | null = null;
   private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
-  private newAsyncBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: MempoolTransactionExtended[]) => Promise<void>)[] = [];
   private classifyingBlocks: boolean = false;
   private oldestCoreLogTimestamp: number | undefined | null = undefined;
 
@@ -74,10 +77,6 @@ class Blocks {
     this.newBlockCallbacks.push(fn);
   }
 
-  public setNewAsyncBlockCallback(fn: (block: BlockExtended, txIds: string[], transactions: MempoolTransactionExtended[]) => Promise<void>) {
-    this.newAsyncBlockCallbacks.push(fn);
-  }
-
   /**
    * Return the list of transaction for a block
    * @param blockHash
@@ -252,7 +251,7 @@ class Blocks {
    *
    * @asyncUnsafe
    */
-  private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
+  public async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[], providedPool?: PoolTag): Promise<BlockExtended> {
     const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
 
     const blk: Partial<BlockExtended> = Object.assign({}, block);
@@ -335,7 +334,9 @@ class Blocks {
 
     if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
       let pool: PoolTag;
-      if (coinbaseTx !== undefined) {
+      if (providedPool) {
+        pool = providedPool;
+      } else if (coinbaseTx !== undefined) {
         pool = await this.$findBlockMiner(coinbaseTx);
       } else {
         if (config.DATABASE.ENABLED === true) {
@@ -386,7 +387,7 @@ class Blocks {
     return <BlockExtended>blk;
   }
 
-  private async $getBlockStats(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<IBitcoinApi.BlockStats> {
+  public async $getBlockStats(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<IBitcoinApi.BlockStats> {
     if (!block.stale) {
       return bitcoinClient.getBlockStats(block.id);
     }
@@ -496,6 +497,128 @@ class Blocks {
     }
   }
 
+  /** @asyncUnsafe */
+  private async $applyBlockTransactionsToMempool(
+    txIds: string[],
+    transactions: MempoolTransactionExtended[]
+  ): Promise<{ rbfTransactions: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }}}> {
+    const _memPool = memPool.getMempool();
+
+    const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
+    memPool.handleRbfTransactions(rbfTransactions);
+    memPool.removeFromSpendMap(transactions);
+
+    if (config.MEMPOOL.CLUSTER_MEMPOOL) {
+      memPool.clusterMempool?.applyMempoolChange({
+        added: [],
+        removed: txIds,
+        accelerations: mempool.getAccelerations(),
+      });
+    }
+
+    for (const txId of txIds) {
+      delete _memPool[txId];
+      rbfCache.mined(txId);
+    }
+
+    let candidates;
+    let transactionIds: string[];
+
+    if (memPool.limitGBT) {
+      const minFeeMempool = await bitcoinSecondClient.getRawMemPool();
+      const minFeeTip = await bitcoinSecondClient.getBlockCount();
+      candidates = memPool.getNextCandidates(minFeeMempool, minFeeTip, transactions);
+      transactionIds = Object.keys(candidates?.txs || {});
+    } else {
+      candidates = undefined;
+      transactionIds = Object.keys(memPool.getMempool());
+    }
+
+    if (config.MEMPOOL.CLUSTER_MEMPOOL) {
+      const cmBlocks = mempool.clusterMempool?.getBlocks(config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT) ?? [];
+      mempoolBlocks.processClusterMempoolBlocks(cmBlocks, _memPool, mempool.getAccelerations());
+    } else if (config.MEMPOOL.RUST_GBT) {
+      const added = memPool.limitGBT ? (candidates?.added || []) : [];
+      const removed = memPool.limitGBT ? (candidates?.removed || []) : transactions;
+      await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, _memPool, added, removed, candidates, true);
+    } else {
+      await mempoolBlocks.$makeBlockTemplates(transactionIds, _memPool, candidates, true, true);
+    }
+
+    return { rbfTransactions };
+  }
+
+  /** @asyncUnsafe */
+  private async $saveBlockData(
+    processingResult: BlockProcessingResult,
+    timer: number
+  ): Promise<void> {
+    const blockExtended = processingResult.blockExtended;
+    const cpfpSummary = processingResult.cpfpSummary;
+
+    let latestPriceId;
+    try {
+      latestPriceId = await PricesRepository.$getLatestPriceId();
+      this.updateTimerProgress(timer, `got latest price id ${this.currentBlockHeight}`);
+    } catch (e) {
+      logger.debug('failed to fetch latest price id from db: ' + (e instanceof Error ? e.message : e));
+    }
+    if (priceUpdater.historyInserted === true && latestPriceId !== null) {
+      await blocksRepository.$saveBlockPrices([{
+        height: blockExtended.height,
+        priceId: latestPriceId,
+      }]);
+      this.updateTimerProgress(timer, `saved prices for ${this.currentBlockHeight}`);
+    } else {
+      logger.debug(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining);
+      indexer.scheduleSingleTask('blocksPrices', 10000);
+    }
+
+    if (Common.blocksSummariesIndexingEnabled() === true) {
+      // indexes the summary as a side effect
+      await this.$getStrippedBlockTransactions(blockExtended.id, true, false, cpfpSummary, blockExtended.height);
+      this.updateTimerProgress(timer, `saved block summary for ${this.currentBlockHeight}`);
+    }
+
+    if (config.MEMPOOL.CPFP_INDEXING) {
+      // can be slow, and isn't critical, so don't await
+      void this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary);
+      this.updateTimerProgress(timer, `saved cpfp for ${this.currentBlockHeight}`);
+    }
+
+    if (processingResult.auditResult) {
+      void BlocksSummariesRepository.$saveTemplate({
+        height: blockExtended.height,
+        template: {
+          id: blockExtended.id,
+          transactions: processingResult.auditResult.projectedBlocks[0].transactions,
+        },
+        version: 1,
+      });
+      this.updateTimerProgress(timer, `saved audit template for ${this.currentBlockHeight}`);
+
+      void BlocksAuditsRepository.$saveAudit({
+        version: 1,
+        templateAlgorithm: processingResult.templateAlgorithm,
+        time: blockExtended.timestamp,
+        height: blockExtended.height,
+        hash: blockExtended.id,
+        unseenTxs: processingResult.auditResult.unseen,
+        addedTxs: processingResult.auditResult.added,
+        prioritizedTxs: processingResult.auditResult.prioritized,
+        missingTxs: processingResult.auditResult.censored,
+        freshTxs: processingResult.auditResult.fresh,
+        sigopTxs: processingResult.auditResult.sigop,
+        fullrbfTxs: processingResult.auditResult.fullrbf,
+        acceleratedTxs: processingResult.auditResult.accelerated,
+        matchRate: processingResult.auditResult.matchRate,
+        expectedFees: processingResult.auditResult.expectedFees,
+        expectedWeight: processingResult.auditResult.expectedWeight,
+      });
+      this.updateTimerProgress(timer, `saved audit results for ${this.currentBlockHeight}`);
+    }
+  }
+
   /**
    * [INDEXING] Index all blocks summaries for the block txs visualization
    */
@@ -728,7 +851,8 @@ class Blocks {
           // fetch transactions
           txs = (await bitcoinApi.$getTxsForBlock(blockHash, true)).map(tx => transactionUtils.extendMempoolTransaction(tx)) || [];
           // add CPFP
-          const cpfpSummary = calculateGoodBlockCpfp(height, txs, []);
+          const blockCpfpData = calculateGoodBlockCpfp(height, txs, []);
+          const cpfpSummary = saveCpfpDataToCpfpSummary(txs, blockCpfpData);
           // classify
           const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, height, cpfpSummary.transactions);
           await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 2);
@@ -765,7 +889,8 @@ class Blocks {
               }
               templateTxs.push(tx || templateTx);
             }
-            const cpfpSummary = calculateGoodBlockCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as MempoolTransactionExtended[], []);
+            const blockCpfpData = calculateGoodBlockCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as MempoolTransactionExtended[], []);
+            const cpfpSummary = saveCpfpDataToCpfpSummary(templateTxs as MempoolTransactionExtended[], blockCpfpData);
             // classify
             const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, height, cpfpSummary.transactions);
             const classifiedTxMap: { [txid: string]: TransactionClassified } = {};
@@ -1054,59 +1179,55 @@ class Blocks {
         }
       }
 
-      let accelerations = Object.values(mempool.getAccelerations());
-      if (accelerations?.length > 0) {
-        const pool = await this.$findBlockMiner(transactionUtils.stripCoinbaseTransaction(transactions[0]));
-        accelerations = accelerations.filter(a => a.pools.includes(pool.uniqueId));
-      }
-      const cpfpSummary: CpfpSummary = calculateGoodBlockCpfp(block.height, transactions, accelerations.map(a => ({ txid: a.txid, max_bid: a.feeDelta })));
-      const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions);
-      const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, block.height, cpfpSummary.transactions);
+      const pool = await this.$findBlockMiner(transactionUtils.stripCoinbaseTransaction(transactions[0]));
+      const accelerations = mempool.getAccelerations();
+
+      const processingResult = await blockProcessor.$processNewBlock(
+        block,
+        transactions,
+        pool,
+        accelerations
+      );
+
+      const blockExtended = processingResult.blockExtended;
+      const blockSummary = processingResult.blockSummary;
+      const cpfpSummary = processingResult.cpfpSummary;
       this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`);
 
-      if (Common.indexingEnabled()) {
-        if (!fastForwarded) {
-          await this.$handleReorgs(blockExtended, timer);
-        }
+      if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
+        await statistics.runStatistics();
+      }
 
+      const { rbfTransactions } = await this.$applyBlockTransactionsToMempool(txIds, cpfpSummary.transactions);
+      this.updateTimerProgress(timer, `applied mempool changes for ${this.currentBlockHeight}`);
+
+      if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
+        await statistics.runStatistics();
+      }
+
+      if (Common.indexingEnabled() && !fastForwarded) {
+        await this.$handleReorgs(blockExtended, timer);
+      }
+
+      await websocketHandler.handleNewBlock(blockExtended, txIds, cpfpSummary.transactions, rbfTransactions);
+      this.updateTimerProgress(timer, `sent websocket updates for ${this.currentBlockHeight}`);
+
+      if (Common.indexingEnabled()) {
         await blocksRepository.$saveBlockInDatabase(blockExtended);
         this.updateTimerProgress(timer, `saved ${this.currentBlockHeight} to database`);
 
-        if (!fastForwarded) {
-          let lastestPriceId;
-          try {
-            lastestPriceId = await PricesRepository.$getLatestPriceId();
-            this.updateTimerProgress(timer, `got latest price id ${this.currentBlockHeight}`);
-          } catch (e) {
-            logger.debug('failed to fetch latest price id from db: ' + (e instanceof Error ? e.message : e));
-          }
-          if (priceUpdater.historyInserted === true && lastestPriceId !== null) {
-            await blocksRepository.$saveBlockPrices([{
-              height: blockExtended.height,
-              priceId: lastestPriceId,
-            }]);
-            this.updateTimerProgress(timer, `saved prices for ${this.currentBlockHeight}`);
-          } else {
-            logger.debug(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining);
-            indexer.scheduleSingleTask('blocksPrices', 10000);
-          }
+        await AccelerationRepository.$indexAccelerationsForBlock(
+          blockExtended,
+          Object.values(accelerations),
+          cpfpSummary.transactions
+        );
+        this.updateTimerProgress(timer, `indexed accelerations for ${this.currentBlockHeight}`);
 
-          // Save blocks summary for visualization if it's enabled
-          if (Common.blocksSummariesIndexingEnabled() === true) {
-            await this.$getStrippedBlockTransactions(blockExtended.id, true, false, cpfpSummary, blockExtended.height);
-            this.updateTimerProgress(timer, `saved block summary for ${this.currentBlockHeight}`);
-          }
-          if (config.MEMPOOL.CPFP_INDEXING) {
-            void this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary);
-            this.updateTimerProgress(timer, `saved cpfp for ${this.currentBlockHeight}`);
-          }
+        if (!fastForwarded) {
+          await this.$saveBlockData(processingResult, timer);
         }
       }
 
-      // start async callbacks
-      this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
-      const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, cpfpSummary.transactions));
-
       if (block.height % 2016 === 0) {
         if (Common.indexingEnabled()) {
           let adjustment;
@@ -1144,11 +1265,6 @@ class Blocks {
         await chainTips.updateOrphanedBlocks();
       }
 
-      // wait for pending async callbacks to finish
-      this.updateTimerProgress(timer, `waiting for async callbacks to complete for ${this.currentBlockHeight}`);
-      await Promise.all(callbackPromises);
-      this.updateTimerProgress(timer, `async callbacks completed for ${this.currentBlockHeight}`);
-
       this.blocks.push(blockExtended);
       if (this.blocks.length > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4) {
         this.blocks = this.blocks.slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4);
@@ -1648,16 +1764,16 @@ class Blocks {
     }
 
     if (transactions?.length != null) {
-      const summary = calculateFastBlockCpfp(height, transactions);
+      const { cpfpSummary } = await detectTemplateAlgorithm(height, transactions, [], true);
 
       if (!stale) {
-        await this.$saveCpfp(hash, height, summary);
+        await this.$saveCpfp(hash, height, cpfpSummary);
       }
 
-      const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions);
+      const effectiveFeeStats = Common.calcEffectiveFeeStatistics(cpfpSummary.transactions);
       await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats);
 
-      return summary;
+      return cpfpSummary;
     } else {
       logger.err(`Cannot index CPFP for block ${height} - missing transaction data`);
       return null;

backend/src/api/cpfp.ts

@@ -1,20 +1,30 @@
-import { Ancestor, CpfpCluster, CpfpInfo, CpfpSummary, MempoolTransactionExtended, TransactionExtended } from '../mempool.interfaces';
+import { Ancestor, CpfpCluster, CpfpInfo, MempoolTransactionExtended, TemplateAlgorithm, TransactionExtended } from '../mempool.interfaces';
 import { GraphTx, convertToGraphTx, expandRelativesGraph, initializeRelatives, makeBlockTemplate, mempoolComparator, removeAncestors, setAncestorScores } from './mini-miner';
 import memPool from './mempool';
 import { Acceleration } from './acceleration/acceleration';
+import { ClusterMempool } from '../cluster-mempool/cluster-mempool';
 
 const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction
 const MAX_CLUSTER_ITERATIONS = 100;
 
-export function calculateFastBlockCpfp(height: number, transactions: MempoolTransactionExtended[], saveRelatives: boolean = false): CpfpSummary {
+type TransactionCpfpData = Partial<CpfpInfo & { cpfpDirty?: boolean, clusterId?: number, chunkIndex?: number }>;
+export interface BlockCpfpData {
+  txs: Record<string, TransactionCpfpData>,
+  clusters: CpfpCluster[];
+  version: number;
+}
+
+export function calculateFastBlockCpfp(height: number, transactions: MempoolTransactionExtended[], saveRelatives: boolean = false): BlockCpfpData {
   const clusters: CpfpCluster[] = []; // list of all cpfp clusters in this block
   const clusterMap: { [txid: string]: CpfpCluster } = {}; // map transactions to their cpfp cluster
   let clusterTxs: TransactionExtended[] = []; // working list of elements of the current cluster
   let ancestors: { [txid: string]: boolean } = {}; // working set of ancestors of the current cluster root
   const txMap: { [txid: string]: TransactionExtended } = {};
+  const cpfpData: Record<string, TransactionCpfpData> = {};
   // initialize the txMap
   for (const tx of transactions) {
     txMap[tx.txid] = tx;
+    cpfpData[tx.txid] = {};
   }
   // reverse pass to identify CPFP clusters
   for (let i = transactions.length - 1; i >= 0; i--) {
@@ -38,7 +48,9 @@ export function calculateFastBlockCpfp(height: number, transactions: MempoolTran
         clusters.push(cluster);
       }
       clusterTxs.forEach(tx => {
-        txMap[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize;
+        cpfpData[tx.txid] = {
+          effectiveFeePerVsize
+        };
         if (cluster) {
           clusterMap[tx.txid] = cluster;
         }
@@ -54,17 +66,19 @@ export function calculateFastBlockCpfp(height: number, transactions: MempoolTran
   }
   // forward pass to enforce ancestor rate caps
   for (const tx of transactions) {
-    let minAncestorRate = tx.effectiveFeePerVsize;
+    const txRate = cpfpData[tx.txid]?.effectiveFeePerVsize ?? tx.effectiveFeePerVsize;
+    let minAncestorRate = txRate;
     for (const vin of tx.vin) {
-      if (txMap[vin.txid]?.effectiveFeePerVsize) {
-        minAncestorRate = Math.min(minAncestorRate, txMap[vin.txid].effectiveFeePerVsize);
+      const vinRate = cpfpData[vin.txid]?.effectiveFeePerVsize ?? txMap[vin.txid]?.effectiveFeePerVsize;
+      if (vinRate) {
+        minAncestorRate = Math.min(minAncestorRate, vinRate);
       }
     }
     // check rounded values to skip cases with almost identical fees
     const roundedMinAncestorRate = Math.ceil(minAncestorRate);
-    const roundedEffectiveFeeRate = Math.floor(tx.effectiveFeePerVsize);
+    const roundedEffectiveFeeRate = Math.floor(txRate);
     if (roundedMinAncestorRate < roundedEffectiveFeeRate) {
-      tx.effectiveFeePerVsize = minAncestorRate;
+      cpfpData[tx.txid].effectiveFeePerVsize = minAncestorRate;
       if (!clusterMap[tx.txid]) {
         // add a single-tx cluster to record the dependent rate
         const cluster = {
@@ -84,23 +98,25 @@ export function calculateFastBlockCpfp(height: number, transactions: MempoolTran
   if (saveRelatives) {
     for (const cluster of clusters) {
       cluster.txs.forEach((member, index) => {
-        txMap[member.txid].descendants = cluster.txs.slice(0, index).reverse();
-        txMap[member.txid].ancestors = cluster.txs.slice(index + 1).reverse();
-        txMap[member.txid].effectiveFeePerVsize = cluster.effectiveFeePerVsize;
+        cpfpData[member.txid].descendants = cluster.txs.slice(0, index).reverse();
+        cpfpData[member.txid].ancestors = cluster.txs.slice(index + 1).reverse();
+        cpfpData[member.txid].effectiveFeePerVsize = cluster.effectiveFeePerVsize;
       });
     }
   }
   return {
-    transactions,
+    txs: cpfpData,
     clusters,
     version: 1,
   };
 }
 
-export function calculateGoodBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): CpfpSummary {
+export function calculateGoodBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): BlockCpfpData {
   const txMap: { [txid: string]: MempoolTransactionExtended } = {};
+  const cpfpData: Record<string, TransactionCpfpData> = {};
   for (const tx of transactions) {
     txMap[tx.txid] = tx;
+    cpfpData[tx.txid] = {};
   }
   const template = makeBlockTemplate(transactions, accelerations, 1, Infinity, Infinity);
   const clusters = new Map<string, string[]>();
@@ -110,15 +126,15 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran
     if (cluster.length > 1 && root && !clusters.has(root)) {
       clusters.set(root, cluster);
     }
-    txMap[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize;
+    cpfpData[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize;
   }
 
   const clusterArray: CpfpCluster[] = [];
 
   for (const cluster of clusters.values()) {
     for (const txid of cluster) {
-      const mempoolTx = txMap[txid];
-      if (mempoolTx) {
+      const mempoolTxCpfpData = cpfpData[txid];
+      if (mempoolTxCpfpData) {
         const ancestors: Ancestor[] = [];
         const descendants: Ancestor[] = [];
         let matched = false;
@@ -138,10 +154,10 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran
             }
           }
         });
-        if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) {
-          mempoolTx.cpfpDirty = true;
+        if (mempoolTxCpfpData.ancestors?.length !== ancestors.length || mempoolTxCpfpData.descendants?.length !== descendants.length) {
+          mempoolTxCpfpData.cpfpDirty = true;
         }
-        Object.assign(mempoolTx, { ancestors, descendants, bestDescendant: null, cpfpChecked: true });
+        Object.assign(mempoolTxCpfpData, { ancestors, descendants, bestDescendant: null, cpfpChecked: true });
       }
     }
     const root = cluster[cluster.length - 1];
@@ -153,17 +169,75 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran
         fee: txMap[txid].fee,
         weight: (txMap[txid].adjustedVsize * 4) || txMap[txid].weight,
       })),
-      effectiveFeePerVsize: txMap[root].effectiveFeePerVsize,
+      effectiveFeePerVsize: cpfpData[root].effectiveFeePerVsize ?? txMap[root].effectiveFeePerVsize,
     });
   }
 
   return {
-    transactions: transactions.map(tx => txMap[tx.txid]),
+    txs: cpfpData,
     clusters: clusterArray,
     version: 2,
   };
 }
 
+export function calculateClusterMempoolBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): BlockCpfpData {
+  const txMap: { [txid: string]: MempoolTransactionExtended } = {};
+  const cpfpData: Record<string, TransactionCpfpData> = {};
+  for (const tx of transactions) {
+    txMap[tx.txid] = tx;
+    cpfpData[tx.txid] = {};
+  }
+
+  const accelMap: { [txid: string]: { feeDelta: number } } = {};
+  for (const acc of accelerations) {
+    accelMap[acc.txid] = { feeDelta: acc.max_bid };
+  }
+
+  const cm = new ClusterMempool(txMap, accelMap, false, 25000);
+
+  const seenClusters = new Set<number>();
+  const clusters: CpfpCluster[] = [];
+
+  for (const txid in cpfpData) {
+    const txCpfpData = cm.getCpfpDataForTx(txid);
+    if (!txCpfpData) {
+      continue;
+    }
+    cpfpData[txid].effectiveFeePerVsize = txCpfpData.effectiveFeePerVsize;
+    cpfpData[txid].clusterId = txCpfpData.clusterId;
+    cpfpData[txid].chunkIndex = txCpfpData.chunkIndex;
+    cpfpData[txid].ancestors = txCpfpData.ancestors;
+    cpfpData[txid].descendants = txCpfpData.descendants;
+    if (txCpfpData.clusterId !== undefined && !seenClusters.has(txCpfpData.clusterId)) {
+      seenClusters.add(txCpfpData.clusterId);
+
+      const clusterData = cm.getCluster(txCpfpData.clusterId);
+      if (clusterData && clusterData.txs.length > 1) {
+        let totalFee = 0;
+        let totalWeight = 0;
+        for (const t of clusterData.txs) {
+          totalFee += t.fee;
+          totalWeight += t.weight;
+        }
+        clusters.push({
+          root: clusterData.txs[0].txid,
+          height,
+          txs: clusterData.txs.map(t => ({ txid: t.txid, weight: t.weight, fee: t.fee })),
+          effectiveFeePerVsize: totalFee / (totalWeight / 4),
+          templateAlgorithm: TemplateAlgorithm.clusterMempool,
+          clusterData,
+        });
+      }
+    }
+  }
+
+  return {
+    txs: cpfpData,
+    clusters,
+    version: 3,
+  };
+}
+
 /**
  * Takes a mempool transaction and a copy of the current mempool, and calculates the CPFP data for
  * that transaction (and all others in the same cluster)

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 109;
+  private static currentVersion = 111;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -1246,6 +1246,15 @@ class DatabaseMigration {
         `);
       await this.updateToSchemaVersion(109);
     }
+    if (databaseSchemaVersion < 110 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD template_algo TINYINT UNSIGNED NOT NULL DEFAULT 0');
+      await this.updateToSchemaVersion(110);
+    }
+
+    if (databaseSchemaVersion < 111 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `compact_cpfp_clusters` ADD template_algo TINYINT UNSIGNED NOT NULL DEFAULT 0');
+      await this.updateToSchemaVersion(111);
+    }
   }
 
   /**

backend/src/api/mempool-blocks.ts

@@ -8,6 +8,7 @@ import path from 'path';
 import mempool from './mempool';
 import { Acceleration } from './services/acceleration';
 import PoolsRepository from '../repositories/PoolsRepository';
+import { ProjectedBlock } from '../cluster-mempool/cluster-mempool';
 
 const MAX_UINT32 = Math.pow(2, 32) - 1;
 
@@ -238,7 +239,7 @@ class MempoolBlocks {
   }
 
   /** @asyncSafe */
-  public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+  public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number, dryRun = false): Promise<MempoolBlockWithTransactions[]> {
     const start = Date.now();
 
     // reset mempool short ids
@@ -278,7 +279,7 @@ class MempoolBlocks {
       const expectedSize = transactions.length;
       const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length;
       logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${expectedSize} in the mempool, ${overflow.length} were unmineable`);
-      const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults);
+      const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults, dryRun);
       logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
       return processed;
     } catch (e) {
@@ -296,16 +297,15 @@ class MempoolBlocks {
   }
 
   /** @asyncSafe */
-  public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+  public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number, dryRun = false): Promise<MempoolBlockWithTransactions[]> {
     // GBT optimization requires that uids never get too sparse
     // as a sanity check, we should also explicitly prevent uint32 uid overflow
     if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * transactions.length), MAX_UINT32)) {
       this.resetRustGbt();
     }
 
-    if (!this.rustInitialized) {
-      // need to reset the worker
-      return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, true, useAccelerations, accelerationPool);
+    if (!this.rustInitialized || dryRun) {
+      return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, !dryRun, useAccelerations, accelerationPool, dryRun);
     }
 
     const start = Date.now();
@@ -344,7 +344,7 @@ class MempoolBlocks {
       if (transactions.length !== resultMempoolSize) {
         throw new Error(`GBT returned wrong number of transactions ${transactions.length} vs ${resultMempoolSize}, cache is probably out of sync`);
       } else {
-        const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, true);
+        const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, !dryRun, dryRun);
         this.removeUids(removedTxs);
         logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
         return processed;
@@ -356,20 +356,22 @@ class MempoolBlocks {
     }
   }
 
-  private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations: { [txid: string]: Acceleration }, accelerationPool, saveResults): MempoolBlockWithTransactions[] {
-    for (const txid of Object.keys(candidates?.txs ?? mempool)) {
-      if (txid in mempool) {
-        mempool[txid].cpfpDirty = false;
-        mempool[txid].ancestors = [];
-        mempool[txid].descendants = [];
-        mempool[txid].bestDescendant = null;
+  private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations: { [txid: string]: Acceleration }, accelerationPool, saveResults, dryRun = false): MempoolBlockWithTransactions[] {
+    if (!dryRun) {
+      for (const txid of Object.keys(candidates?.txs ?? mempool)) {
+        if (txid in mempool) {
+          mempool[txid].cpfpDirty = false;
+          mempool[txid].ancestors = [];
+          mempool[txid].descendants = [];
+          mempool[txid].bestDescendant = null;
+        }
       }
-    }
-    for (const [txid, rate] of rates) {
-      if (txid in mempool) {
-        mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize);
-        mempool[txid].effectiveFeePerVsize = rate;
-        mempool[txid].cpfpChecked = true;
+      for (const [txid, rate] of rates) {
+        if (txid in mempool) {
+          mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize);
+          mempool[txid].effectiveFeePerVsize = rate;
+          mempool[txid].cpfpChecked = true;
+        }
       }
     }
 
@@ -387,58 +389,60 @@ class MempoolBlocks {
       feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
     }
 
-    const ancestors: Ancestor[] = [];
-    const descendants: Ancestor[] = [];
-    let ancestor: MempoolTransactionExtended;
-    for (const cluster of clusters) {
-      for (const memberTxid of cluster) {
-        const mempoolTx = mempool[memberTxid];
-        if (mempoolTx) {
-          // ugly micro-optimization to avoid allocating new arrays
-          ancestors.length = 0;
-          descendants.length = 0;
-          let matched = false;
-          cluster.forEach(txid => {
-            ancestor = mempool[txid];
-            if (txid === memberTxid) {
-              matched = true;
-            } else {
-              if (!ancestor) {
-                console.log('txid missing from mempool! ', txid, candidates?.txs[txid]);
-                return;
-              }
-              const relative = {
-                txid: txid,
-                fee: ancestor.fee,
-                weight: (ancestor.adjustedVsize * 4),
-              };
-              if (matched) {
-                descendants.push(relative);
-                if (!mempoolTx.lastBoosted || (ancestor.firstSeen && ancestor.firstSeen > mempoolTx.lastBoosted)) {
-                  mempoolTx.lastBoosted = ancestor.firstSeen;
-                }
+    if (!dryRun) {
+      const ancestors: Ancestor[] = [];
+      const descendants: Ancestor[] = [];
+      let ancestor: MempoolTransactionExtended;
+      for (const cluster of clusters) {
+        for (const memberTxid of cluster) {
+          const mempoolTx = mempool[memberTxid];
+          if (mempoolTx) {
+            // ugly micro-optimization to avoid allocating new arrays
+            ancestors.length = 0;
+            descendants.length = 0;
+            let matched = false;
+            cluster.forEach(txid => {
+              ancestor = mempool[txid];
+              if (txid === memberTxid) {
+                matched = true;
               } else {
-                ancestors.push(relative);
+                if (!ancestor) {
+                  console.log('txid missing from mempool! ', txid, candidates?.txs[txid]);
+                  return;
+                }
+                const relative = {
+                  txid: txid,
+                  fee: ancestor.fee,
+                  weight: (ancestor.adjustedVsize * 4),
+                };
+                if (matched) {
+                  descendants.push(relative);
+                  if (!mempoolTx.lastBoosted || (ancestor.firstSeen && ancestor.firstSeen > mempoolTx.lastBoosted)) {
+                    mempoolTx.lastBoosted = ancestor.firstSeen;
+                  }
+                } else {
+                  ancestors.push(relative);
+                }
               }
+            });
+            if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) {
+              mempoolTx.cpfpDirty = true;
             }
-          });
-          if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) {
-            mempoolTx.cpfpDirty = true;
+            // ugly micro-optimization to avoid allocating new arrays or objects
+            if (mempoolTx.ancestors) {
+              mempoolTx.ancestors.length = 0;
+            } else {
+              mempoolTx.ancestors = [];
+            }
+            if (mempoolTx.descendants) {
+              mempoolTx.descendants.length = 0;
+            } else {
+              mempoolTx.descendants = [];
+            }
+            mempoolTx.ancestors.push(...ancestors);
+            mempoolTx.descendants.push(...descendants);
+            mempoolTx.cpfpChecked = true;
           }
-          // ugly micro-optimization to avoid allocating new arrays or objects
-          if (mempoolTx.ancestors) {
-            mempoolTx.ancestors.length = 0;
-          } else {
-            mempoolTx.ancestors = [];
-          }
-          if (mempoolTx.descendants) {
-            mempoolTx.descendants.length = 0;
-          } else {
-            mempoolTx.descendants = [];
-          }
-          mempoolTx.ancestors.push(...ancestors);
-          mempoolTx.descendants.push(...descendants);
-          mempoolTx.cpfpChecked = true;
         }
       }
     }
@@ -471,34 +475,42 @@ class MempoolBlocks {
         const txid = block[i];
         if (txid in mempool) {
           mempoolTx = mempool[txid];
-          // save position in projected blocks
-          mempoolTx.position = {
-            block: blockIndex,
-            vsize: totalVsize + (mempoolTx.vsize / 2),
-          };
 
-          if (txid in accelerations) {
-            acceleration = accelerations[txid];
-            if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
-              if (!mempoolTx.acceleration) {
-                mempoolTx.cpfpDirty = true;
-              }
-              mempoolTx.acceleration = true;
-              mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools;
-              mempoolTx.acceleratedAt = acceleration?.added;
-              mempoolTx.feeDelta = acceleration?.feeDelta;
-              for (const ancestor of mempoolTx.ancestors || []) {
-                if (!(ancestor.txid in mempool)) {
-                  continue;
+          if (!dryRun) {
+            // save position in projected blocks
+            mempoolTx.position = {
+              block: blockIndex,
+              vsize: totalVsize + (mempoolTx.vsize / 2),
+            };
+
+            if (txid in accelerations) {
+              acceleration = accelerations[txid];
+              if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
+                if (!mempoolTx.acceleration) {
+                  mempoolTx.cpfpDirty = true;
                 }
-                if (!mempool[ancestor.txid].acceleration) {
-                  mempool[ancestor.txid].cpfpDirty = true;
+                mempoolTx.acceleration = true;
+                mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools;
+                mempoolTx.acceleratedAt = acceleration?.added;
+                mempoolTx.feeDelta = acceleration?.feeDelta;
+                for (const ancestor of mempoolTx.ancestors || []) {
+                  if (!(ancestor.txid in mempool)) {
+                    continue;
+                  }
+                  if (!mempool[ancestor.txid].acceleration) {
+                    mempool[ancestor.txid].cpfpDirty = true;
+                  }
+                  mempool[ancestor.txid].acceleration = true;
+                  mempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy;
+                  mempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt;
+                  mempool[ancestor.txid].feeDelta = mempoolTx.feeDelta;
+                  isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy;
+                }
+              } else {
+                if (mempoolTx.acceleration) {
+                  mempoolTx.cpfpDirty = true;
+                  delete mempoolTx.acceleration;
                 }
-                mempool[ancestor.txid].acceleration = true;
-                mempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy;
-                mempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt;
-                mempool[ancestor.txid].feeDelta = mempoolTx.feeDelta;
-                isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy;
               }
             } else {
               if (mempoolTx.acceleration) {
@@ -506,11 +518,6 @@ class MempoolBlocks {
                 delete mempoolTx.acceleration;
               }
             }
-          } else {
-            if (mempoolTx.acceleration) {
-              mempoolTx.cpfpDirty = true;
-              delete mempoolTx.acceleration;
-            }
           }
 
           // online calculation of stack-of-blocks fee stats
@@ -548,6 +555,115 @@ class MempoolBlocks {
     return mempoolBlocks;
   }
 
+  public processClusterMempoolBlocks(projectedBlocks: ProjectedBlock[], newMempool: { [txid: string]: MempoolTransactionExtended }, accelerations: { [txid: string]: Acceleration }, saveResults = true, accelerationPool?: number): MempoolBlockWithTransactions[] {
+    const lastBlockIndex = projectedBlocks.length - 1;
+    let hasBlockStack = projectedBlocks.length >= 8;
+    let stackWeight = 0;
+    let feeStatsCalculator: OnlineFeeStatsCalculator | null = null;
+    if (hasBlockStack) {
+      stackWeight = projectedBlocks[lastBlockIndex].weight;
+      hasBlockStack = stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS;
+      feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
+    }
+
+    const isAcceleratedBy: { [txid: string]: number[] | false } = {};
+
+    const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
+    let mempoolTx: MempoolTransactionExtended;
+    let acceleration: Acceleration;
+    const mempoolBlocks: MempoolBlockWithTransactions[] = [];
+
+    for (let blockIndex = 0; blockIndex < projectedBlocks.length; blockIndex++) {
+      const projected = projectedBlocks[blockIndex];
+      let totalSize = 0;
+      let totalVsize = 0;
+      let totalWeight = 0;
+      let totalFees = 0;
+      const transactions: MempoolTransactionExtended[] = [];
+      const validTxids: string[] = [];
+
+      for (const txid of projected.txids) {
+        if (txid in newMempool) {
+          mempoolTx = newMempool[txid];
+          validTxids.push(txid);
+
+          // save position in projected blocks
+          mempoolTx.position = {
+            block: blockIndex,
+            vsize: totalVsize + (mempoolTx.vsize / 2),
+          };
+
+          if (txid in accelerations) {
+            acceleration = accelerations[txid];
+            if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
+              if (!mempoolTx.acceleration) {
+                mempoolTx.cpfpDirty = true;
+              }
+              mempoolTx.acceleration = true;
+              mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools;
+              mempoolTx.acceleratedAt = acceleration?.added;
+              mempoolTx.feeDelta = acceleration?.feeDelta;
+              for (const ancestor of mempoolTx.ancestors || []) {
+                if (!(ancestor.txid in newMempool)) {
+                  continue;
+                }
+                if (!newMempool[ancestor.txid].acceleration) {
+                  newMempool[ancestor.txid].cpfpDirty = true;
+                }
+                newMempool[ancestor.txid].acceleration = true;
+                newMempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy;
+                newMempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt;
+                newMempool[ancestor.txid].feeDelta = mempoolTx.feeDelta;
+                isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy;
+              }
+            } else {
+              if (mempoolTx.acceleration) {
+                mempoolTx.cpfpDirty = true;
+                delete mempoolTx.acceleration;
+              }
+            }
+          } else {
+            if (mempoolTx.acceleration) {
+              mempoolTx.cpfpDirty = true;
+              delete mempoolTx.acceleration;
+            }
+          }
+
+          if (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) {
+            feeStatsCalculator.processNext(mempoolTx);
+          }
+
+          totalSize += mempoolTx.size;
+          totalVsize += mempoolTx.vsize;
+          totalWeight += mempoolTx.weight;
+          totalFees += mempoolTx.fee;
+
+          if (totalVsize <= sizeLimit) {
+            transactions.push(mempoolTx);
+          }
+        }
+      }
+
+      mempoolBlocks[blockIndex] = this.dataToMempoolBlocks(
+        validTxids,
+        transactions,
+        totalSize,
+        totalWeight,
+        totalFees,
+        (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined,
+      );
+    }
+
+    if (saveResults) {
+      const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
+      this.mempoolBlocks = mempoolBlocks;
+      this.mempoolBlockDeltas = deltas;
+      this.updateAccelerationPositions(newMempool, accelerations, mempoolBlocks);
+    }
+
+    return mempoolBlocks;
+  }
+
   private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions {
     if (!feeStats) {
       feeStats = Common.calcEffectiveFeeStatistics(transactions);

backend/src/api/mempool.ts

@@ -13,6 +13,7 @@ import { Acceleration } from './services/acceleration';
 import accelerationApi from './services/acceleration';
 import redisCache from './redis-cache';
 import blocks from './blocks';
+import { ClusterMempool } from '../cluster-mempool/cluster-mempool';
 
 class Mempool {
   private inSync: boolean = false;
@@ -22,6 +23,7 @@ class Mempool {
   private spendMap = new Map<string, MempoolTransactionExtended>();
   private recentlyDeleted: MempoolTransactionExtended[][] = []; // buffer of transactions deleted in recent mempool updates
   private mempoolInfo: IBitcoinApi.MempoolInfo;
+  public clusterMempool: ClusterMempool | null = null;
   private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[],
     deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[]) => void) | undefined;
   private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[],
@@ -62,6 +64,9 @@ class Mempool {
       minrelaytxfee: isLiquid ? 0.00000100 : 0.00001000
     };
     this.txPerSecondInterval = setInterval(this.updateTxPerSecond.bind(this), 1000);
+    if (config.MEMPOOL.CLUSTER_MEMPOOL) {
+      this.clusterMempool = new ClusterMempool(this.mempoolCache, this.accelerations);
+    }
   }
 
   /**
@@ -153,6 +158,9 @@ class Mempool {
       await redisCache.$flushTransactions();
       logger.debug(`Finished migrating cache transactions in ${((Date.now() - redisTimer) / 1000).toFixed(2)} seconds`);
     }
+    if (config.MEMPOOL.CLUSTER_MEMPOOL) {
+      this.clusterMempool = new ClusterMempool(this.mempoolCache, this.accelerations);
+    }
     if (this.mempoolChangedCallback) {
       this.mempoolChangedCallback(this.mempoolCache, [], [], []);
     }
@@ -387,6 +395,14 @@ class Mempool {
       hasChange = true;
     }
 
+    if (config.MEMPOOL.CLUSTER_MEMPOOL && (newTransactions.length || deletedTransactions.length || accelerationDelta.length)) {
+      this.clusterMempool?.applyMempoolChange({
+        added: newTransactions,
+        removed: deletedTransactions.map(tx => tx.txid),
+        accelerations: this.getAccelerations(),
+      });
+    }
+
     this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize);
 
     const candidatesChanged = candidates?.added?.length || candidates?.removed?.length;

backend/src/api/transaction-utils.ts

@@ -256,13 +256,14 @@ class TransactionUtils {
 
   // returns the most significant 4 bytes of the txid as an integer
   public txidToOrdering(txid: string): number {
-    return parseInt(
-      txid.substr(62, 2) +
-        txid.substr(60, 2) +
-        txid.substr(58, 2) +
-        txid.substr(56, 2),
-      16
-    );
+    // Parse last 4 bytes of txid as little-endian uint32, without string allocation
+    let result = 0;
+    for (let i = 62; i >= 56; i -= 2) {
+      const hi = txid.charCodeAt(i);
+      const lo = txid.charCodeAt(i + 1);
+      result = result * 256 + (hi < 58 ? hi - 48 : hi - 87) * 16 + (lo < 58 ? lo - 48 : lo - 87);
+    }
+    return result;
   }
 
   public addInnerScriptsToVin(vin: IEsploraApi.Vin): void {

backend/src/api/websocket-handler.ts

@@ -3,7 +3,7 @@ import * as WebSocket from 'ws';
 import {
   BlockExtended, TransactionExtended, MempoolTransactionExtended, WebsocketResponse,
   OptimizedStatistic, ILoadingIndicators, GbtCandidates, TxTrackingInfo,
-  MempoolDelta, MempoolDeltaTxids
+  MempoolDelta, MempoolDeltaTxids, CpfpInfo
 } from '../mempool.interfaces';
 import blocks from './blocks';
 import memPool from './mempool';
@@ -16,16 +16,12 @@ import transactionUtils from './transaction-utils';
 import rbfCache, { ReplacementInfo } from './rbf-cache';
 import difficultyAdjustment from './difficulty-adjustment';
 import feeApi from './fee-api';
-import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
-import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
-import Audit from './audit';
 import priceUpdater from '../tasks/price-updater';
 import { ApiPrice } from '../repositories/PricesRepository';
 import { Acceleration } from './services/acceleration';
 import accelerationApi from './services/acceleration';
 import mempool from './mempool';
 import statistics from './statistics/statistics';
-import accelerationRepository from '../repositories/AccelerationRepository';
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import walletApi from './services/wallets';
 
@@ -297,11 +293,14 @@ class WebsocketHandler {
 
             if (txids.length) {
               client['track-txs'] = txids;
+              client['track-txs-updates'] = 0;
             } else {
               client['track-txs'] = null;
+              client['track-txs-updates'] = 0;
             }
 
             if (Object.keys(txs).length) {
+              client['track-txs-updates'] = (client['track-txs-updates'] || 0) + Object.keys(txs).length;
               response['tracked-txs'] = JSON.stringify(txs);
             }
           }
@@ -326,10 +325,13 @@ class WebsocketHandler {
             if (Object.keys(addressMap).length > config.MEMPOOL.MAX_TRACKED_ADDRESSES) {
               response['track-addresses-error'] = `"too many addresses requested, this connection supports tracking a maximum of ${config.MEMPOOL.MAX_TRACKED_ADDRESSES} addresses"`;
               client['track-addresses'] = null;
+              client['track-addresses-updates'] = 0;
             } else if (Object.keys(addressMap).length > 0) {
               client['track-addresses'] = addressMap;
+              client['track-addresses-updates'] = 0;
             } else {
               client['track-addresses'] = null;
+              client['track-addresses-updates'] = 0;
             }
           }
 
@@ -649,7 +651,10 @@ class WebsocketHandler {
       removed = candidates?.removed || [];
     }
 
-    if (config.MEMPOOL.RUST_GBT) {
+    if (config.MEMPOOL.CLUSTER_MEMPOOL) {
+      const cmBlocks = mempool.clusterMempool?.getBlocks(config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT) ?? [];
+      mempoolBlocks.processClusterMempoolBlocks(cmBlocks, newMempool, mempool.getAccelerations());
+    } else if (config.MEMPOOL.RUST_GBT) {
       await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, newMempool, added, removed, candidates, true);
     } else {
       await mempoolBlocks.$updateBlockTemplates(transactionIds, newMempool, added, removed, candidates, accelerationDelta, true, true);
@@ -778,6 +783,8 @@ class WebsocketHandler {
       removed: websocketAccelerationDelta.filter(txid => !accelerations[txid]),
     };
 
+    const cpfpUpdatesSent = new Set<string>();
+
     // TODO - Fix indentation after PR is merged
     for (const server of this.webSocketServers) {
     server.clients.forEach(async (client) => {
@@ -855,6 +862,8 @@ class WebsocketHandler {
         }
 
         if (Object.keys(addressMap).length > 0) {
+          client['track-addresses-updates'] =
+            (client['track-addresses-updates'] || 0) + this.countAddressTransactions(addressMap);
           response['multi-address-transactions'] = JSON.stringify(addressMap);
         }
       }
@@ -946,15 +955,23 @@ class WebsocketHandler {
             calculateMempoolTxCpfp(mempoolTx, newMempool);
           }
           if (mempoolTx.cpfpDirty) {
-            positionData['cpfp'] = {
-              ancestors: mempoolTx.ancestors,
+            const cpfp: CpfpInfo = {
+              ancestors: mempoolTx.ancestors || [],
               bestDescendant: mempoolTx.bestDescendant || null,
-              descendants: mempoolTx.descendants || null,
-              effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize || null,
+              descendants: mempoolTx.descendants,
+              effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize,
               sigops: mempoolTx.sigops,
               adjustedVsize: mempoolTx.adjustedVsize,
               acceleration: mempoolTx.acceleration,
             };
+            if (config.MEMPOOL.CLUSTER_MEMPOOL && mempoolTx.clusterId != null) {
+              const cluster = mempool.clusterMempool?.getClusterForApi(mempoolTx.txid);
+              if (cluster) {
+                cpfp.cluster = cluster;
+              }
+            }
+            positionData['cpfp'] = cpfp;
+            cpfpUpdatesSent.add(trackTxid);
           }
           response['txPosition'] = JSON.stringify(positionData);
         }
@@ -992,11 +1009,18 @@ class WebsocketHandler {
               txInfo.cpfp = {
                 ancestors: mempoolTx.ancestors,
                 bestDescendant: mempoolTx.bestDescendant || null,
-                descendants: mempoolTx.descendants || null,
-                effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize || null,
+                descendants: mempoolTx.descendants,
+                effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize,
                 sigops: mempoolTx.sigops,
                 adjustedVsize: mempoolTx.adjustedVsize,
               };
+              if (config.MEMPOOL.CLUSTER_MEMPOOL && mempoolTx.clusterId != null) {
+                const cluster = mempool.clusterMempool?.getClusterForApi(mempoolTx.txid);
+                if (cluster) {
+                  (txInfo.cpfp as CpfpInfo).cluster = cluster;
+                }
+              }
+              cpfpUpdatesSent.add(txid);
             }
             txHasInfo = true;
           }
@@ -1005,6 +1029,7 @@ class WebsocketHandler {
           }
         }
         if (Object.keys(txs).length) {
+          client['track-txs-updates'] = (client['track-txs-updates'] || 0) + Object.keys(txs).length;
           response['tracked-txs'] = JSON.stringify(txs);
         }
       }
@@ -1047,129 +1072,33 @@ class WebsocketHandler {
       }
     });
     }
+
+    for (const txid of cpfpUpdatesSent) {
+      if (newMempool[txid]) {
+        newMempool[txid].cpfpDirty = false;
+      }
+    }
   }
 
-  /** @asyncUnsafe */
-  async handleNewBlock(block: BlockExtended, txIds: string[], transactions: MempoolTransactionExtended[]): Promise<void> {
+  /** @asyncSafe */
+  async handleNewBlock(
+    block: BlockExtended,
+    txIds: string[],
+    transactions: MempoolTransactionExtended[],
+    rbfTransactions: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }}
+  ): Promise<void> {
     if (!this.webSocketServers.length) {
       throw new Error('No WebSocket.Server have been set');
     }
 
-    const blockTransactions = structuredClone(transactions);
-
     this.printLogs();
-    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
-      await statistics.runStatistics();
-    }
 
     const _memPool = memPool.getMempool();
-    const candidateTxs = memPool.getMempoolCandidates();
-    let candidates: GbtCandidates | undefined = (memPool.limitGBT && candidateTxs) ? { txs: candidateTxs, added: [], removed: [] } : undefined;
-    let transactionIds: string[] = (memPool.limitGBT) ? Object.keys(candidates?.txs || {}) : Object.keys(_memPool);
-
-    if (config.DATABASE.ENABLED) {
-      const accelerations = Object.values(mempool.getAccelerations());
-      await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, structuredClone(transactions));
-    }
-
-    const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
-    memPool.handleRbfTransactions(rbfTransactions);
-    memPool.removeFromSpendMap(transactions);
-
-    if (config.MEMPOOL.AUDIT && memPool.isInSync()) {
-      let projectedBlocks;
-      const auditMempool = _memPool;
-      const isAccelerated = accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations()));
-
-      if (config.MEMPOOL.RUST_GBT) {
-        const added = memPool.limitGBT ? (candidates?.added || []) : [];
-        const removed = memPool.limitGBT ? (candidates?.removed || []) : [];
-        projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, auditMempool, added, removed, candidates, isAccelerated, block.extras.pool.id);
-      } else {
-        projectedBlocks = await mempoolBlocks.$makeBlockTemplates(transactionIds, auditMempool, candidates, false, isAccelerated, block.extras.pool.id);
-      }
-
-      if (Common.indexingEnabled()) {
-        const { unseen, censored, added, prioritized, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(block.height, blockTransactions, projectedBlocks, auditMempool);
-        const matchRate = Math.round(score * 100 * 100) / 100;
-
-        const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
-
-        let totalFees = 0;
-        let totalWeight = 0;
-        for (const tx of stripped) {
-          totalFees += tx.fee;
-          totalWeight += (tx.vsize * 4);
-        }
-
-        void BlocksSummariesRepository.$saveTemplate({
-          height: block.height,
-          template: {
-            id: block.id,
-            transactions: stripped,
-          },
-          version: 1,
-        });
-
-        void BlocksAuditsRepository.$saveAudit({
-          version: 1,
-          time: block.timestamp,
-          height: block.height,
-          hash: block.id,
-          unseenTxs: unseen,
-          addedTxs: added,
-          prioritizedTxs: prioritized,
-          missingTxs: censored,
-          freshTxs: fresh,
-          sigopTxs: sigop,
-          fullrbfTxs: fullrbf,
-          acceleratedTxs: accelerated,
-          matchRate: matchRate,
-          expectedFees: totalFees,
-          expectedWeight: totalWeight,
-        });
-
-        if (block.extras) {
-          block.extras.matchRate = matchRate;
-          block.extras.expectedFees = totalFees;
-          block.extras.expectedWeight = totalWeight;
-          block.extras.similarity = similarity;
-        }
-      }
-    } else if (block.extras) {
-      const mBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
-      if (mBlocks?.length && mBlocks[0].transactions) {
-        block.extras.similarity = Common.getSimilarity(mBlocks[0], transactions);
-      }
-    }
-
     const confirmedTxids: { [txid: string]: boolean } = {};
-
-    // Update mempool to remove transactions included in the new block
     for (const txId of txIds) {
-      delete _memPool[txId];
-      rbfCache.mined(txId);
       confirmedTxids[txId] = true;
     }
 
-    if (memPool.limitGBT) {
-      const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null;
-      const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1;
-      candidates = memPool.getNextCandidates(minFeeMempool, minFeeTip, transactions);
-      transactionIds = Object.keys(candidates?.txs || {});
-    } else {
-      candidates = undefined;
-      transactionIds = Object.keys(memPool.getMempool());
-    }
-
-
-    if (config.MEMPOOL.RUST_GBT) {
-      const added = memPool.limitGBT ? (candidates?.added || []) : [];
-      const removed = memPool.limitGBT ? (candidates?.removed || []) : transactions;
-      await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, _memPool, added, removed, candidates, true);
-    } else {
-      await mempoolBlocks.$makeBlockTemplates(transactionIds, _memPool, candidates, true, true);
-    }
     const mBlocks = mempoolBlocks.getMempoolBlocks();
     const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
 
@@ -1302,6 +1231,7 @@ class WebsocketHandler {
           }
         }
         if (Object.keys(txs).length) {
+          client['track-txs-updates'] = (client['track-txs-updates'] || 0) + Object.keys(txs).length;
           response['tracked-txs'] = JSON.stringify(txs);
         }
       }
@@ -1337,6 +1267,8 @@ class WebsocketHandler {
         }
 
         if (Object.keys(addressMap).length > 0) {
+          client['track-addresses-updates'] =
+            (client['track-addresses-updates'] || 0) + this.countAddressTransactions(addressMap);
           response['multi-address-transactions'] = JSON.stringify(addressMap);
         }
       }
@@ -1434,10 +1366,6 @@ class WebsocketHandler {
       }
     });
     }
-
-    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
-      await statistics.runStatistics();
-    }
   }
 
   public handleNewStratumJob(job: StratumJob): void {
@@ -1546,25 +1474,72 @@ class WebsocketHandler {
     if (this.webSocketServers.length) {
       let numTxSubs = 0;
       let numTxsSubs = 0;
+      let numAddressSubs = 0;
+      let numAddressesSubs = 0;
       let numProjectedSubs = 0;
       let numRbfSubs = 0;
+      let trackedTxsTotal = 0;
+      let trackedAddressesTotal = 0;
+      let trackedTxsMax = 0;
+      let trackedAddressesMax = 0;
+      let trackTxsTrackedTotal = 0;
+      let trackTxsTrackedMax = 0;
+      let trackAddressesTrackedTotal = 0;
+      let trackAddressesTrackedMax = 0;
+      let trackTxsUpdatesTotal = 0;
+      let trackTxsUpdatesMax = 0;
+      let trackAddressesUpdatesTotal = 0;
+      let trackAddressesUpdatesMax = 0;
 
-      // TODO - Fix indentation after PR is merged
       for (const server of this.webSocketServers) {
-      server.clients.forEach((client) => {
-        if (client['track-tx']) {
-          numTxSubs++;
-        }
-        if (client['track-txs']) {
-          numTxsSubs++;
-        }
-        if (client['track-mempool-block'] != null && client['track-mempool-block'] >= 0) {
-          numProjectedSubs++;
-        }
-        if (client['track-rbf']) {
-          numRbfSubs++;
-        }
-      });
+        server.clients.forEach((client) => {
+          let trackedTxCount = 0;
+          let trackedAddressCount = 0;
+
+          if (client['track-tx']) {
+            numTxSubs++;
+            trackedTxCount += 1;
+          }
+          if (client['track-txs']) {
+            numTxsSubs++;
+            trackedTxCount += client['track-txs'].length;
+          }
+          if (client['track-address']) {
+            numAddressSubs++;
+            trackedAddressCount += 1;
+          }
+          if (client['track-addresses']) {
+            numAddressesSubs++;
+            const addressCount = Object.keys(client['track-addresses']).length;
+            trackedAddressCount += addressCount;
+            trackAddressesTrackedTotal += addressCount;
+            trackAddressesTrackedMax = Math.max(trackAddressesTrackedMax, addressCount);
+            const updates = client['track-addresses-updates'] || 0;
+            trackAddressesUpdatesTotal += updates;
+            trackAddressesUpdatesMax = Math.max(trackAddressesUpdatesMax, updates);
+            client['track-addresses-updates'] = 0;
+          }
+          if (client['track-mempool-block'] != null && client['track-mempool-block'] >= 0) {
+            numProjectedSubs++;
+          }
+          if (client['track-rbf']) {
+            numRbfSubs++;
+          }
+          if (client['track-txs']) {
+            const txCount = client['track-txs'].length;
+            trackTxsTrackedTotal += txCount;
+            trackTxsTrackedMax = Math.max(trackTxsTrackedMax, txCount);
+            const updates = client['track-txs-updates'] || 0;
+            trackTxsUpdatesTotal += updates;
+            trackTxsUpdatesMax = Math.max(trackTxsUpdatesMax, updates);
+            client['track-txs-updates'] = 0;
+          }
+
+          trackedTxsTotal += trackedTxCount;
+          trackedAddressesTotal += trackedAddressCount;
+          trackedTxsMax = Math.max(trackedTxsMax, trackedTxCount);
+          trackedAddressesMax = Math.max(trackedAddressesMax, trackedAddressCount);
+        });
       }
 
       let count = 0;
@@ -1573,12 +1548,33 @@ class WebsocketHandler {
       }
       const diff = count - this.numClients;
       this.numClients = count;
-      logger.debug(`${count} websocket clients | ${this.numConnected} connected | ${this.numDisconnected} disconnected | (${diff >= 0 ? '+' : ''}${diff})`);
-      logger.debug(`websocket subscriptions: track-tx: ${numTxSubs}, track-txs: ${numTxsSubs}, track-mempool-block: ${numProjectedSubs} track-rbf: ${numRbfSubs}`);
+      const trackedTxsAvg = count > 0 ? trackedTxsTotal / count : 0;
+      const trackedAddressesAvg = count > 0 ? trackedAddressesTotal / count : 0;
+      const trackTxsTrackedAvg = numTxsSubs > 0 ? trackTxsTrackedTotal / numTxsSubs : 0;
+      const trackAddressesTrackedAvg =
+        numAddressesSubs > 0 ? trackAddressesTrackedTotal / numAddressesSubs : 0;
+      const trackTxsUpdatesAvg = numTxsSubs > 0 ? trackTxsUpdatesTotal / numTxsSubs : 0;
+      const trackAddressesUpdatesAvg =
+        numAddressesSubs > 0 ? trackAddressesUpdatesTotal / numAddressesSubs : 0;
+      logger.debug(
+        `${count} websocket clients | ${this.numConnected} connected | ${this.numDisconnected} disconnected | (${diff >= 0 ? '+' : ''}${diff}) | tracked txs: total=${trackedTxsTotal}, avg=${trackedTxsAvg.toFixed(2)}, max=${trackedTxsMax} | tracked addresses: total=${trackedAddressesTotal}, avg=${trackedAddressesAvg.toFixed(2)}, max=${trackedAddressesMax} | ws-subscriptions: tx=${numTxSubs},txs=${numTxsSubs},address=${numAddressSubs},addresses=${numAddressesSubs},txs-tracked-avg=${trackTxsTrackedAvg.toFixed(2)},txs-tracked-max=${trackTxsTrackedMax},addresses-tracked-avg=${trackAddressesTrackedAvg.toFixed(2)},addresses-tracked-max=${trackAddressesTrackedMax},txs-updates-avg=${trackTxsUpdatesAvg.toFixed(2)},txs-updates-max=${trackTxsUpdatesMax},addresses-updates-avg=${trackAddressesUpdatesAvg.toFixed(2)},addresses-updates-max=${trackAddressesUpdatesMax}`
+      );
+      logger.debug(`websocket subscriptions: track-tx: ${numTxSubs}, track-txs: ${numTxsSubs}, track-address: ${numAddressSubs}, track-addresses: ${numAddressesSubs}, track-mempool-block: ${numProjectedSubs} track-rbf: ${numRbfSubs}`);
       this.numConnected = 0;
       this.numDisconnected = 0;
     }
   }
+
+  private countAddressTransactions(addressMap: { [address: string]: AddressTransactions }): number {
+    return Object.values(addressMap).reduce(
+      (total, transactions) =>
+        total
+        + transactions.mempool.length
+        + transactions.confirmed.length
+        + transactions.removed.length,
+      0,
+    );
+  }
 }
 
 export default new WebsocketHandler();

backend/src/cluster-mempool/block-builder.ts

@@ -0,0 +1,165 @@
+import { PairingHeap } from '../utils/pairing-heap';
+import { LinearizationChunk } from './linearize';
+import { MempoolTransactionExtended } from '../mempool.interfaces';
+import { Cluster } from './cluster-mempool';
+
+export interface ProjectedBlock {
+  txids: string[];
+  weight: number;
+  sigops: number;
+}
+
+interface ChunkHeapEntry {
+  fee: number;
+  weight: number;
+  sigops: number;
+  equalFeeratePrefixWeight: number;
+  maxOrder: number;
+  clusterId: number;
+  chunkIndex: number;
+}
+
+const BLOCK_WEIGHT_UNITS = 4_000_000;
+const MAX_BLOCK_SIGOPS_COST = 80_000;
+const COINBASE_RESERVED_WEIGHT = 8000;
+const MAX_CONSECUTIVE_FAILURES = 1000;
+const MAX_WASTED_WEIGHT = 4000;
+
+function chunkHeapHigherPriority(a: ChunkHeapEntry, b: ChunkHeapEntry): boolean {
+  const feerateDiff = a.fee * b.weight - b.fee * a.weight;
+  if (feerateDiff !== 0) {
+    return feerateDiff > 0;
+  }
+  if (a.equalFeeratePrefixWeight !== b.equalFeeratePrefixWeight) {
+    return a.equalFeeratePrefixWeight < b.equalFeeratePrefixWeight;
+  }
+  return a.maxOrder < b.maxOrder;
+}
+
+function equalFeerate(a: LinearizationChunk, b: LinearizationChunk): boolean {
+  return a.fee * b.weight === b.fee * a.weight;
+}
+
+function makeChunkHeapEntry(cluster: Cluster, chunkIndex: number, mempool: { [txid: string]: MempoolTransactionExtended }): ChunkHeapEntry {
+  const chunk = cluster.chunks[chunkIndex];
+  let maxOrder = 0;
+  let sigops = 0;
+  for (const tx of chunk.txs) {
+    if (tx.order > maxOrder) {
+      maxOrder = tx.order;
+    }
+    const mempoolTx = mempool[tx.txid];
+    if (mempoolTx) {
+      sigops += mempoolTx.sigops || 0;
+    }
+  }
+  let prefixWeight = chunk.weight;
+  for (let i = chunkIndex - 1; i >= 0; i--) {
+    if (equalFeerate(cluster.chunks[i], chunk)) {
+      prefixWeight += cluster.chunks[i].weight;
+    } else {
+      break;
+    }
+  }
+  return {
+    fee: chunk.fee,
+    weight: chunk.weight,
+    sigops,
+    equalFeeratePrefixWeight: prefixWeight,
+    maxOrder,
+    clusterId: cluster.id,
+    chunkIndex,
+  };
+}
+
+function buildChunkHeap(clusters: Map<number, Cluster>, mempool: { [txid: string]: MempoolTransactionExtended }): PairingHeap<ChunkHeapEntry> {
+  const heap = new PairingHeap<ChunkHeapEntry>(chunkHeapHigherPriority);
+  for (const cluster of clusters.values()) {
+    if (cluster.chunks.length > 0) {
+      heap.add(makeChunkHeapEntry(cluster, 0, mempool));
+    }
+  }
+  return heap;
+}
+
+export function assembleBlocks(
+  n: number,
+  clusters: Map<number, Cluster>,
+  mempool: { [txid: string]: MempoolTransactionExtended },
+  enforceLimit: boolean,
+): ProjectedBlock[] {
+  const heap = buildChunkHeap(clusters, mempool);
+  const blocks: ProjectedBlock[] = [];
+  for (let blockIdx = 0; blockIdx < n; blockIdx++) {
+    const limited = enforceLimit || blockIdx < n - 1;
+    const maxWeight = limited ? BLOCK_WEIGHT_UNITS : Infinity;
+    const maxSigops = limited ? MAX_BLOCK_SIGOPS_COST : Infinity;
+    const block = fillBlock(heap, clusters, mempool, maxWeight, maxSigops);
+    if (block.txids.length === 0) {
+      break;
+    }
+    blocks.push(block);
+  }
+  return blocks;
+}
+
+function fillBlock(
+  heap: PairingHeap<ChunkHeapEntry>,
+  clusters: Map<number, Cluster>,
+  mempool: { [txid: string]: MempoolTransactionExtended },
+  maxWeight: number,
+  maxSigops: number,
+): ProjectedBlock {
+  const block: ProjectedBlock = { txids: [], weight: COINBASE_RESERVED_WEIGHT, sigops: 0 };
+  const deferred: ChunkHeapEntry[] = [];
+  let consecutiveFailed = 0;
+  let full = false;
+
+  while (!heap.isEmpty() && !full) {
+    const entry = heap.pop() as ChunkHeapEntry;
+    const cluster = clusters.get(entry.clusterId);
+    const chunk = cluster?.chunks[entry.chunkIndex];
+
+    if (!cluster || !chunk) {
+      // stale entry
+    } else if (block.weight + entry.weight < maxWeight
+      && block.sigops + entry.sigops < maxSigops) {
+      consecutiveFailed = 0;
+      block.weight += chunkWeight(chunk, mempool);
+      block.sigops += entry.sigops;
+      for (const tx of chunk.txs) {
+        block.txids.push(tx.txid);
+      }
+      if (entry.chunkIndex + 1 < cluster.chunks.length) {
+        heap.add(makeChunkHeapEntry(cluster, entry.chunkIndex + 1, mempool));
+      }
+    } else {
+      deferred.push(entry);
+      consecutiveFailed++;
+      if (consecutiveFailed > MAX_CONSECUTIVE_FAILURES
+        && block.weight + MAX_WASTED_WEIGHT > maxWeight) {
+        full = true;
+      }
+    }
+  }
+
+  for (const entry of deferred) {
+    heap.add(entry);
+  }
+
+  return block;
+}
+
+function chunkWeight(
+  chunk: LinearizationChunk,
+  mempool: { [txid: string]: MempoolTransactionExtended },
+): number {
+  let weight = 0;
+  for (const clusterTx of chunk.txs) {
+    const mempoolTx = mempool[clusterTx.txid];
+    if (mempoolTx) {
+      weight += mempoolTx.weight;
+    }
+  }
+  return weight;
+}

backend/src/cluster-mempool/cluster-mempool.ts

@@ -0,0 +1,703 @@
+import { ClusterTx, DepGraph, sortTopological, subgraph } from './depgraph';
+import { linearizeCluster, LinearizationChunk } from './linearize';
+import { ProjectedBlock, assembleBlocks } from './block-builder';
+import { Ancestor, CpfpClusterData, CpfpClusterTx, MempoolTransactionExtended } from '../mempool.interfaces';
+import logger from '../logger';
+
+export interface MempoolDiff {
+  added: MempoolTransactionExtended[];
+  removed: string[];
+  accelerations: { [txid: string]: { feeDelta: number } };
+}
+
+export interface ClusterInfo {
+  clusterId: number;
+  chunkIndex: number;
+  chunkFeerate: number;
+}
+
+export { ProjectedBlock };
+
+export interface Cluster {
+  id: number;
+  depgraph: DepGraph;
+  txs: Map<string, ClusterTx>;
+  linearization: ClusterTx[];
+  chunks: LinearizationChunk[];
+  dirty: boolean;
+}
+
+export interface TxCpfpData {
+  effectiveFeePerVsize: number;
+  clusterId: number;
+  chunkIndex: number;
+  cpfpDirty: boolean;
+  cpfpChecked: boolean;
+  ancestors: Ancestor[];
+  descendants: Ancestor[];
+}
+
+const DEFAULT_COST_BUDGET = 75000;
+
+export class ClusterMempool {
+  private clusters = new Map<number, Cluster>();
+  private txToCluster = new Map<string, number>();
+  private parentMap = new Map<string, Set<string>>();
+  private spentBy = new Map<string, string>();
+  private mempool: Readonly<{ [txid: string]: MempoolTransactionExtended }>;
+  private accelerations: { [txid: string]: { feeDelta: number } } = {};
+  private nextClusterId = 0;
+  private modifyTxs: boolean;
+  private costBudget: number = DEFAULT_COST_BUDGET;
+
+  constructor(mempool: { [txid: string]: MempoolTransactionExtended }, accelerations?: { [txid: string]: { feeDelta: number } }, modifyTxs: boolean = true, costBudget: number = DEFAULT_COST_BUDGET) {
+    this.mempool = mempool;
+    if (accelerations) {
+      this.accelerations = accelerations;
+    }
+    this.modifyTxs = modifyTxs;
+    this.costBudget = costBudget;
+    this.buildFromMempool();
+  }
+
+  applyMempoolChange(diff: MempoolDiff): void {
+    this.processRemovals(diff.removed);
+    this.splitDisconnectedClusters();
+    this.processAccelerationChanges(diff.accelerations);
+    this.processAdditions(diff.added);
+    this.relinearizeDirtyClusters();
+  }
+
+  getBlocks(n: number, enforceLimit = false): ProjectedBlock[] {
+    return assembleBlocks(n, this.clusters, this.mempool, enforceLimit);
+  }
+
+  getCluster(clusterId: number): CpfpClusterData | null {
+    const cluster = this.clusters.get(clusterId);
+    if (!cluster) {
+      return null;
+    }
+    return this.buildClusterData(cluster);
+  }
+
+  getClusterInfo(txid: string): ClusterInfo | null {
+    const match = this.getClusterForTx(txid);
+    if (!match) {
+      return null;
+    }
+    return this.findChunkInfo(match.cluster, match.clusterTx);
+  }
+
+  getClusterForApi(txid: string): (CpfpClusterData & { chunkIndex: number }) | null {
+    const info = this.getClusterInfo(txid);
+    if (!info) {
+      return null;
+    }
+    const cluster = this.getCluster(info.clusterId);
+    if (!cluster || cluster.txs.length <= 1) {
+      return null;
+    }
+    return { ...cluster, chunkIndex: info.chunkIndex };
+  }
+
+  getCpfpDataForTx(txid: string): TxCpfpData | null {
+    const clusterInfo = this.getClusterForTx(txid);
+    if (!clusterInfo) {
+      return null;
+    }
+    const chunkInfo = this.findChunkInfo(clusterInfo.cluster, clusterInfo.clusterTx);
+    if (!chunkInfo) {
+      return null;
+    }
+    const chunk = clusterInfo.cluster.chunks[chunkInfo?.chunkIndex];
+    const chunkSet = chunk.txs.length > 1 ? new Set(chunk.txs) : null;
+    return {
+      effectiveFeePerVsize: chunkInfo.chunkFeerate,
+      clusterId: clusterInfo.cluster.id,
+      chunkIndex: chunkInfo.chunkIndex,
+      cpfpDirty: true,
+      cpfpChecked: true,
+      ancestors: chunkSet ? this.getChunkRelatives(clusterInfo.clusterTx, chunkSet, 'ancestors') : [],
+      descendants: chunkSet ? this.getChunkRelatives(clusterInfo.clusterTx, chunkSet, 'descendants') : [],
+    };
+  }
+
+  getClusterCount(): number {
+    return this.clusters.size;
+  }
+
+  getTxCount(): number {
+    return this.txToCluster.size;
+  }
+
+  private getClusterForTx(txid: string): { cluster: Cluster; clusterTx: ClusterTx } | null {
+    const clusterId = this.txToCluster.get(txid);
+    if (clusterId === undefined) {
+      return null;
+    }
+    const cluster = this.clusters.get(clusterId);
+    if (!cluster) {
+      return null;
+    }
+    const clusterTx = cluster.txs.get(txid);
+    if (!clusterTx) {
+      return null;
+    }
+    return { cluster, clusterTx };
+  }
+
+  private buildFromMempool(): void {
+    this.buildRelativeMaps();
+    const components = this.findMempoolComponents();
+    for (const component of components) {
+      this.createClusterFromTxids(component);
+    }
+  }
+
+  private buildRelativeMaps(): void {
+    this.parentMap.clear();
+    this.spentBy.clear();
+    for (const txid in this.mempool) {
+      const tx = this.mempool[txid];
+      const txParents = new Set<string>();
+      for (const vin of tx.vin) {
+        if (!vin.is_coinbase && this.mempool[vin.txid]) {
+          txParents.add(vin.txid);
+          this.spentBy.set(`${vin.txid}:${vin.vout}`, txid);
+        }
+      }
+      if (txParents.size > 0) {
+        this.parentMap.set(txid, txParents);
+      }
+    }
+  }
+
+  private findMempoolComponents(): Set<string>[] {
+    const visited = new Set<string>();
+    const components: Set<string>[] = [];
+
+    for (const txid in this.mempool) {
+      if (!visited.has(txid)) {
+        const component = this.dfsComponent(txid, visited);
+        components.push(component);
+      }
+    }
+    return components;
+  }
+
+  private dfsComponent(
+    startTxid: string,
+    visited: Set<string>
+  ): Set<string> {
+    const component = new Set<string>();
+    const stack = [startTxid];
+    while (stack.length > 0) {
+      const current = stack.pop();
+      if (current !== undefined && !visited.has(current)) {
+        visited.add(current);
+        component.add(current);
+
+        const txParents = this.parentMap.get(current);
+        if (txParents) {
+          for (const p of txParents) {
+            if (!visited.has(p)) {
+              stack.push(p);
+            }
+          }
+        }
+
+        const tx = this.mempool[current];
+        if (tx) {
+          for (let vout = 0; vout < tx.vout.length; vout++) {
+            const child = this.spentBy.get(`${current}:${vout}`);
+            if (child && !visited.has(child)) {
+              stack.push(child);
+            }
+          }
+        }
+      }
+    }
+    return component;
+  }
+
+  private effectiveFee(txid: string, tx: MempoolTransactionExtended): number {
+    return tx.fee + (this.accelerations[txid]?.feeDelta || 0);
+  }
+
+  private adjustedWeight(tx: MempoolTransactionExtended): number {
+    return Math.max(tx.weight, (tx.sigops || 0) * 20);
+  }
+
+  private createClusterFromTxids(
+    txids: Set<string>
+  ): Cluster | null {
+    const clusterId = this.nextClusterId++;
+    const depgraph = new DepGraph();
+    const txMap = new Map<string, ClusterTx>();
+
+    for (const txid of txids) {
+      const tx = this.mempool[txid];
+      if (!tx) {
+        logger.warn(`Warning: missing mempool tx ${txid} during cluster creation, skipping`);
+        return null;
+      }
+      const clusterTx = depgraph.addTransaction(txid, this.effectiveFee(txid, tx), this.adjustedWeight(tx), tx.order ?? 0);
+      txMap.set(txid, clusterTx);
+    }
+
+    for (const txid of txids) {
+      const txParents = this.parentMap.get(txid);
+      if (txParents) {
+        for (const parentTxid of txParents) {
+          if (txids.has(parentTxid)) {
+            const parentTx = txMap.get(parentTxid);
+            const childTx = txMap.get(txid);
+            if (parentTx && childTx) {
+              depgraph.addDependency(parentTx, childTx);
+            }
+          }
+        }
+      }
+    }
+
+    const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), this.costBudget);
+
+    const cluster: Cluster = {
+      id: clusterId,
+      depgraph,
+      txs: txMap,
+      linearization,
+      chunks,
+      dirty: false,
+    };
+
+    this.clusters.set(clusterId, cluster);
+    for (const txid of txids) {
+      this.txToCluster.set(txid, clusterId);
+    }
+
+    if (this.modifyTxs) {
+      this.writeBackCluster(cluster);
+    }
+    return cluster;
+  }
+
+  public writeBackClusters(): void {
+    for (const cluster of this.clusters.values()) {
+      this.writeBackCluster(cluster);
+    }
+  }
+
+  private writeBackCluster(cluster: Cluster): void {
+    for (let chunkIdx = 0; chunkIdx < cluster.chunks.length; chunkIdx++) {
+      const chunk = cluster.chunks[chunkIdx];
+      const chunkFeerate = chunk.weight > 0 ? (chunk.fee * 4) / chunk.weight : 0;
+      const chunkSet = chunk.txs.length > 1 ? new Set(chunk.txs) : null;
+      for (const clusterTx of chunk.txs) {
+        this.writeBackTx(cluster, clusterTx, chunkIdx, chunkFeerate, chunkSet);
+      }
+    }
+  }
+
+  private writeBackTx(
+    cluster: Cluster,
+    clusterTx: ClusterTx,
+    chunkIdx: number,
+    chunkFeerate: number,
+    chunkSet: Set<ClusterTx> | null
+  ): void {
+    const txid = clusterTx.txid;
+    if (!this.mempool[txid]) {
+      logger.warn(`ClusterMempool.writeBackTx: ${txid} missing from mempool (cluster ${cluster.id})`);
+      return;
+    }
+    const tx = this.mempool[txid];
+    if (tx.effectiveFeePerVsize !== chunkFeerate || tx.clusterId !== cluster.id) {
+      tx.cpfpDirty = true;
+    }
+    tx.effectiveFeePerVsize = chunkFeerate;
+    tx.clusterId = cluster.id;
+    tx.chunkIndex = chunkIdx;
+    tx.cpfpChecked = true;
+
+    if (chunkSet) {
+      tx.ancestors = this.getChunkRelatives(clusterTx, chunkSet, 'ancestors');
+      tx.descendants = this.getChunkRelatives(clusterTx, chunkSet, 'descendants');
+    } else {
+      tx.ancestors = [];
+      tx.descendants = [];
+    }
+  }
+
+  private getChunkRelatives(
+    clusterTx: ClusterTx,
+    chunkSet: Set<ClusterTx>,
+    direction: 'ancestors' | 'descendants'
+  ): { txid: string; fee: number; weight: number }[] {
+    const relatives: { txid: string; fee: number; weight: number }[] = [];
+    const related = direction === 'ancestors' ? clusterTx.ancestors : clusterTx.descendants;
+    for (const rel of related) {
+      if (rel !== clusterTx && chunkSet.has(rel)) {
+        const mempoolTx = this.mempool[rel.txid];
+        if (mempoolTx) {
+          relatives.push({ txid: rel.txid, fee: mempoolTx.fee, weight: mempoolTx.weight });
+        } else {
+          logger.warn(`ClusterMempool.getChunkRelatives: ${rel.txid} missing from mempool`);
+        }
+      }
+    }
+    return relatives;
+  }
+
+  private processRemovals(removed: string[]): void {
+    for (const txid of removed) {
+      const tx = this.mempool[txid];
+      if (tx) {
+        for (const vin of tx.vin) {
+          if (!vin.is_coinbase) {
+            this.spentBy.delete(`${vin.txid}:${vin.vout}`);
+          }
+        }
+      } else if (this.txToCluster.has(txid)) {
+        logger.warn(`ClusterMempool.processRemovals: ${txid} missing from mempool, spentBy cleanup skipped`);
+      }
+    }
+
+    for (const txid of removed) {
+      const match = this.getClusterForTx(txid);
+      if (match) {
+        match.cluster.depgraph.removeTransactions(new Set([match.clusterTx]));
+        match.cluster.txs.delete(txid);
+        match.cluster.linearization = match.cluster.linearization.filter(t => t !== match.clusterTx);
+        this.txToCluster.delete(txid);
+        match.cluster.dirty = true;
+      }
+    }
+  }
+
+  private splitDisconnectedClusters(): void {
+    for (const [clusterId, cluster] of this.clusters.entries()) {
+      if (cluster.dirty) {
+        if (cluster.depgraph.size === 0) {
+          this.clusters.delete(clusterId);
+        } else {
+          const components = cluster.depgraph.findConnectedComponents();
+          if (components.length > 1) {
+            this.clusters.delete(clusterId);
+            for (const component of components) {
+              this.splitComponentToCluster(cluster, component);
+            }
+          }
+        }
+      }
+    }
+  }
+
+  private splitComponentToCluster(sourceCluster: Cluster, component: Set<ClusterTx>): void {
+    const newClusterId = this.nextClusterId++;
+    const { depgraph: newDepgraph, txMap } = subgraph(component);
+
+    const newTxs = new Map<string, ClusterTx>();
+    for (const oldTx of component) {
+      const newTx = txMap.get(oldTx);
+      if (newTx) {
+        newTxs.set(oldTx.txid, newTx);
+        this.txToCluster.set(oldTx.txid, newClusterId);
+      }
+    }
+
+    const newLinearization: ClusterTx[] = [];
+    for (const oldTx of sourceCluster.linearization) {
+      if (component.has(oldTx)) {
+        const newTx = txMap.get(oldTx);
+        if (newTx) {
+          newLinearization.push(newTx);
+        }
+      }
+    }
+
+    const newCluster: Cluster = {
+      id: newClusterId,
+      dirty: true,
+      depgraph: newDepgraph,
+      txs: newTxs,
+      linearization: newLinearization,
+      chunks: [],
+    };
+    this.clusters.set(newClusterId, newCluster);
+  }
+
+  private processAdditions(added: MempoolTransactionExtended[]): void {
+    for (const tx of added) {
+      const txid = tx.txid;
+
+      for (const vin of tx.vin) {
+        if (!vin.is_coinbase) {
+          this.spentBy.set(`${vin.txid}:${vin.vout}`, txid);
+        }
+      }
+
+      const { relatedClusterIds, parentTxids, childTxids } = this.findRelatedClusters(tx);
+
+      if (relatedClusterIds.size === 0) {
+        this.addSingletonCluster(tx);
+      } else if (relatedClusterIds.size === 1) {
+        this.addToExistingCluster(tx, relatedClusterIds, parentTxids, childTxids);
+      } else {
+        this.mergeAndAddToCluster(tx, relatedClusterIds, parentTxids, childTxids);
+      }
+    }
+  }
+
+  private findRelatedClusters(tx: MempoolTransactionExtended): {
+    relatedClusterIds: Set<number>;
+    parentTxids: string[];
+    childTxids: string[];
+  } {
+    const relatedClusterIds = new Set<number>();
+    const parentTxids: string[] = [];
+    const childTxids: string[] = [];
+
+    for (const vin of tx.vin) {
+      if (!vin.is_coinbase && this.mempool[vin.txid]) {
+        const parentCluster = this.txToCluster.get(vin.txid);
+        if (parentCluster !== undefined) {
+          relatedClusterIds.add(parentCluster);
+          parentTxids.push(vin.txid);
+        }
+      }
+    }
+
+    for (let vout = 0; vout < tx.vout.length; vout++) {
+      const childTxid = this.spentBy.get(`${tx.txid}:${vout}`);
+      if (childTxid && this.mempool[childTxid]) {
+        const childCluster = this.txToCluster.get(childTxid);
+        if (childCluster !== undefined) {
+          relatedClusterIds.add(childCluster);
+          childTxids.push(childTxid);
+        }
+      }
+    }
+
+    return { relatedClusterIds, parentTxids, childTxids };
+  }
+
+  private addSingletonCluster(tx: MempoolTransactionExtended): void {
+    const txid = tx.txid;
+    const clusterId = this.nextClusterId++;
+    const depgraph = new DepGraph();
+    const clusterTx = depgraph.addTransaction(txid, this.effectiveFee(txid, tx), this.adjustedWeight(tx), tx.order ?? 0);
+
+    const cluster: Cluster = {
+      id: clusterId,
+      dirty: true,
+      depgraph,
+      txs: new Map([[txid, clusterTx]]),
+      linearization: [clusterTx],
+      chunks: [],
+    };
+    this.clusters.set(clusterId, cluster);
+    this.txToCluster.set(txid, clusterId);
+  }
+
+  private addToExistingCluster(
+    tx: MempoolTransactionExtended,
+    relatedClusterIds: Set<number>,
+    parentTxids: string[],
+    childTxids: string[],
+  ): void {
+    const txid = tx.txid;
+    const clusterId = relatedClusterIds.values().next().value;
+    const cluster = this.clusters.get(clusterId);
+    if (!cluster) {
+      return;
+    }
+    const clusterTx = cluster.depgraph.addTransaction(txid, this.effectiveFee(txid, tx), this.adjustedWeight(tx), tx.order ?? 0);
+    cluster.txs.set(txid, clusterTx);
+    cluster.linearization.push(clusterTx);
+    this.txToCluster.set(txid, clusterId);
+
+    this.addParentDeps(cluster, clusterTx, parentTxids);
+    this.addChildDeps(cluster, clusterTx, childTxids);
+    cluster.dirty = true;
+  }
+
+  private mergeAndAddToCluster(
+    tx: MempoolTransactionExtended,
+    relatedClusterIds: Set<number>,
+    parentTxids: string[],
+    childTxids: string[],
+  ): void {
+    const clusterIterator = relatedClusterIds.values();
+    const primaryId: number = clusterIterator.next().value;
+    const primary = this.clusters.get(primaryId);
+    if (!primary) {
+      return;
+    }
+
+    for (const clusterId of clusterIterator) {
+      const other = this.clusters.get(clusterId);
+      if (other) {
+        this.mergeClusterInto(primary, other);
+        this.clusters.delete(clusterId);
+      }
+    }
+
+    const clusterTx = primary.depgraph.addTransaction(tx.txid, this.effectiveFee(tx.txid, tx), this.adjustedWeight(tx), tx.order ?? 0);
+    primary.txs.set(tx.txid, clusterTx);
+    primary.linearization.push(clusterTx);
+    this.txToCluster.set(tx.txid, primaryId);
+
+    this.addParentDeps(primary, clusterTx, parentTxids);
+    this.addChildDeps(primary, clusterTx, childTxids);
+    primary.dirty = true;
+  }
+
+  private addParentDeps(cluster: Cluster, childTx: ClusterTx, parentTxids: string[]): void {
+    for (const parentTxid of parentTxids) {
+      const parentTx = cluster.txs.get(parentTxid);
+      if (parentTx) {
+        cluster.depgraph.addDependency(parentTx, childTx);
+      }
+    }
+  }
+
+  private addChildDeps(cluster: Cluster, parentTx: ClusterTx, childTxids: string[]): void {
+    for (const childTxid of childTxids) {
+      const childTx = cluster.txs.get(childTxid);
+      if (childTx) {
+        cluster.depgraph.addDependency(parentTx, childTx);
+      }
+    }
+  }
+
+  private mergeClusterInto(primary: Cluster, other: Cluster): void {
+    for (const [txid, otherTx] of other.txs) {
+      const newTx = primary.depgraph.addTransaction(txid, otherTx.effectiveFee, otherTx.weight, otherTx.order);
+      primary.txs.set(txid, newTx);
+      this.txToCluster.set(txid, primary.id);
+    }
+
+    for (const otherTx of other.depgraph.getTxs()) {
+      for (const parent of otherTx.parents) {
+        const newChild = primary.txs.get(otherTx.txid);
+        const newParent = primary.txs.get(parent.txid);
+        if (newChild && newParent) {
+          primary.depgraph.addDependency(newParent, newChild);
+        }
+      }
+    }
+
+    for (const otherTx of other.linearization) {
+      const newTx = primary.txs.get(otherTx.txid);
+      if (newTx) {
+        primary.linearization.push(newTx);
+      }
+    }
+  }
+
+  private processAccelerationChanges(newAccelerations: { [txid: string]: { feeDelta: number } }): void {
+    const changed = new Set<string>();
+    for (const txid in newAccelerations) {
+      if ((newAccelerations[txid]?.feeDelta || 0) !== (this.accelerations[txid]?.feeDelta || 0)) {
+        changed.add(txid);
+      }
+    }
+    for (const txid in this.accelerations) {
+      if (!newAccelerations[txid]) {
+        changed.add(txid);
+      }
+    }
+    this.accelerations = newAccelerations;
+    for (const txid of changed) {
+      const tx = this.mempool[txid];
+      if (!tx) {
+        continue;
+      }
+      const match = this.getClusterForTx(txid);
+      if (match) {
+        match.clusterTx.effectiveFee = this.effectiveFee(txid, tx);
+        match.cluster.dirty = true;
+      }
+    }
+  }
+
+  private relinearizeDirtyClusters(): void {
+    for (const [clusterId, cluster] of this.clusters.entries()) {
+      if (cluster.dirty) {
+        cluster.dirty = false;
+        const newId = this.nextClusterId++;
+        this.clusters.delete(clusterId);
+        cluster.id = newId;
+        this.clusters.set(newId, cluster);
+        for (const txid of cluster.txs.keys()) {
+          this.txToCluster.set(txid, newId);
+        }
+
+        const { linearization, chunks } = linearizeCluster(
+          cluster.depgraph.getTxs(),
+          this.costBudget,
+          cluster.linearization,
+        );
+        cluster.linearization = linearization;
+        cluster.chunks = chunks;
+
+        if (this.modifyTxs) {
+          this.writeBackCluster(cluster);
+        }
+      }
+    }
+  }
+
+  private buildClusterData(cluster: Cluster): CpfpClusterData {
+    const txs: CpfpClusterTx[] = [];
+    const txToFlatIdx = new Map<ClusterTx, number>();
+
+    for (const chunk of cluster.chunks) {
+      const ordered = sortTopological(new Set(chunk.txs));
+      for (const clusterTx of ordered) {
+        if (this.mempool[clusterTx.txid]) {
+          txToFlatIdx.set(clusterTx, txs.length);
+          const parents: number[] = [];
+          for (const parentTx of clusterTx.parents) {
+            const flatIdx = txToFlatIdx.get(parentTx);
+            if (flatIdx !== undefined) {
+              parents.push(flatIdx);
+            }
+          }
+          const mempoolTx = this.mempool[clusterTx.txid];
+          txs.push({ txid: clusterTx.txid, fee: mempoolTx.fee, weight: mempoolTx.weight, parents });
+        } else {
+          logger.warn(`ClusterMempool.buildClusterData: ${clusterTx.txid} missing from mempool (cluster ${cluster.id})`);
+        }
+      }
+    }
+
+    let offset = 0;
+    const chunks = cluster.chunks.map(chunk => {
+      const count = chunk.txs.length;
+      const chunkEntry = {
+        txs: Array.from({ length: count }, (_, i) => offset + i),
+        feerate: chunk.weight > 0 ? (chunk.fee * 4) / chunk.weight : 0,
+      };
+      offset += count;
+      return chunkEntry;
+    });
+
+    return { txs, chunks };
+  }
+
+  private findChunkInfo(cluster: Cluster, tx: ClusterTx): ClusterInfo | null {
+    for (let chunkIdx = 0; chunkIdx < cluster.chunks.length; chunkIdx++) {
+      const chunk = cluster.chunks[chunkIdx];
+      if (chunk.txs.includes(tx)) {
+        return {
+          clusterId: cluster.id,
+          chunkIndex: chunkIdx,
+          chunkFeerate: chunk.weight > 0 ? (chunk.fee * 4) / chunk.weight : 0,
+        };
+      }
+    }
+    return null;
+  }
+}

backend/src/cluster-mempool/depgraph.ts

@@ -0,0 +1,165 @@
+import logger from '../logger';
+
+export class ClusterTx {
+  txid: string;
+  effectiveFee: number;
+  weight: number;
+  order: number;
+  ancestors: Set<ClusterTx>;
+  descendants: Set<ClusterTx>;
+  parents: Set<ClusterTx>;
+  children: Set<ClusterTx>;
+
+  constructor(txid: string, effectiveFee: number, weight: number, order: number) {
+    this.txid = txid;
+    this.effectiveFee = effectiveFee;
+    this.weight = weight;
+    this.order = order;
+    this.ancestors = new Set([this]);
+    this.descendants = new Set([this]);
+    this.parents = new Set();
+    this.children = new Set();
+  }
+}
+
+export class DepGraph {
+  private txs: Set<ClusterTx> = new Set();
+
+  get size(): number {
+    return this.txs.size;
+  }
+
+  addTransaction(txid: string, fee: number, weight: number, order: number = 0): ClusterTx {
+    const tx = new ClusterTx(txid, fee, weight, order);
+    this.txs.add(tx);
+    return tx;
+  }
+
+  addDependency(parent: ClusterTx, child: ClusterTx): void {
+    if (!this.txs.has(parent) || !this.txs.has(child)) {
+      logger.warn(`Warning: invalid dependency, skipping`);
+      return;
+    }
+
+    parent.children.add(child);
+    child.parents.add(parent);
+
+    if (child.ancestors.has(parent)) {
+      return;
+    }
+
+    for (const descendant of child.descendants) {
+      for (const ancestor of parent.ancestors) {
+        descendant.ancestors.add(ancestor);
+        ancestor.descendants.add(descendant);
+      }
+    }
+  }
+
+  removeTransactions(toRemove: Set<ClusterTx>): void {
+    for (const tx of toRemove) {
+      for (const parent of tx.parents) {
+        parent.children.delete(tx);
+      }
+      for (const child of tx.children) {
+        child.parents.delete(tx);
+      }
+      this.txs.delete(tx);
+    }
+
+    for (const tx of this.txs) {
+      for (const removed of toRemove) {
+        tx.ancestors.delete(removed);
+        tx.descendants.delete(removed);
+      }
+    }
+
+    this.rederiveAncestorsDescendants();
+  }
+
+  private rederiveAncestorsDescendants(): void {
+    const ordered = [...this.txs].sort((a, b) => a.ancestors.size - b.ancestors.size);
+    for (const tx of this.txs) {
+      tx.ancestors = new Set([tx]);
+      tx.descendants = new Set([tx]);
+    }
+    for (const tx of ordered) {
+      for (const parent of tx.parents) {
+        for (const ancestor of parent.ancestors) {
+          tx.ancestors.add(ancestor);
+        }
+      }
+      for (const ancestor of tx.ancestors) {
+        ancestor.descendants.add(tx);
+      }
+    }
+  }
+
+  hasTx(tx: ClusterTx): boolean {
+    return this.txs.has(tx);
+  }
+
+  getTxs(): Set<ClusterTx> {
+    return this.txs;
+  }
+
+  findConnectedComponents(): Set<ClusterTx>[] {
+    const visited = new Set<ClusterTx>();
+    const components: Set<ClusterTx>[] = [];
+
+    for (const tx of this.txs) {
+      if (!visited.has(tx)) {
+        const component = new Set<ClusterTx>();
+        const stack: ClusterTx[] = [tx];
+        while (stack.length > 0) {
+          const node = stack.pop();
+          if (node && !visited.has(node)) {
+            visited.add(node);
+            component.add(node);
+            for (const a of node.ancestors) {
+              if (!visited.has(a) && this.txs.has(a)) {
+                stack.push(a);
+              }
+            }
+            for (const d of node.descendants) {
+              if (!visited.has(d) && this.txs.has(d)) {
+                stack.push(d);
+              }
+            }
+          }
+        }
+        components.push(component);
+      }
+    }
+    return components;
+  }
+
+}
+
+export function sortTopological(subset: Set<ClusterTx>): ClusterTx[] {
+  return [...subset].sort((a, b) => a.ancestors.size - b.ancestors.size);
+}
+
+export function subgraph(txSubset: Set<ClusterTx>): { depgraph: DepGraph; txMap: Map<ClusterTx, ClusterTx> } {
+  const newGraph = new DepGraph();
+  const txMap = new Map<ClusterTx, ClusterTx>();
+
+  for (const oldTx of txSubset) {
+    const newTx = newGraph.addTransaction(oldTx.txid, oldTx.effectiveFee, oldTx.weight, oldTx.order);
+    txMap.set(oldTx, newTx);
+  }
+
+  for (const oldTx of txSubset) {
+    for (const parent of oldTx.parents) {
+      if (txSubset.has(parent)) {
+        const newChild = txMap.get(oldTx);
+        const newParent = txMap.get(parent);
+        if (newChild && newParent) {
+          newGraph.addDependency(newParent, newChild);
+        }
+      }
+    }
+  }
+
+  return { depgraph: newGraph, txMap };
+}

backend/src/config.ts

@@ -34,6 +34,8 @@ interface IConfig {
     POOLS_JSON_TREE_URL: string,
     POOLS_UPDATE_DELAY: number,
     AUDIT: boolean;
+    CLUSTER_MEMPOOL: boolean;
+    CLUSTER_MEMPOOL_INDEXING: boolean;
     RUST_GBT: boolean;
     LIMIT_GBT: boolean;
     CPFP_INDEXING: boolean;
@@ -205,6 +207,8 @@ const defaults: IConfig = {
     'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
     'POOLS_UPDATE_DELAY': 604800, // in seconds, default is one week
     'AUDIT': false,
+    'CLUSTER_MEMPOOL': false,
+    'CLUSTER_MEMPOOL_INDEXING': false,
     'RUST_GBT': true,
     'LIMIT_GBT': false,
     'CPFP_INDEXING': false,

backend/src/index.ts

@@ -334,7 +334,6 @@ class Server {
     if (config.MEMPOOL.ENABLED) {
       statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler));
       memPool.setAsyncMempoolChangedCallback(websocketHandler.$handleMempoolChange.bind(websocketHandler));
-      blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
     }
     if (config.FIAT_PRICE.ENABLED) {
       priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));

backend/src/indexer.ts

@@ -139,12 +139,12 @@ class Indexer {
     switch (task) {
       case 'blocksPrices': {
         if (!['testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) && config.FIAT_PRICE.ENABLED) {
-          let lastestPriceId;
+          let latestPriceId;
           try {
-            lastestPriceId = await PricesRepository.$getLatestPriceId();
+            latestPriceId = await PricesRepository.$getLatestPriceId();
           } catch (e) {
             logger.debug('failed to fetch latest price id from db: ' + (e instanceof Error ? e.message : e));
-          }          if (priceUpdater.historyInserted === false || lastestPriceId === null) {
+          }          if (priceUpdater.historyInserted === false || latestPriceId === null) {
             logger.debug(`Blocks prices indexer is waiting for the price updater to complete`, logger.tags.mining);
             this.scheduleSingleTask(task, 10000);
           } else {

backend/src/mempool.interfaces.ts

@@ -28,6 +28,11 @@ export interface PoolStats extends PoolInfo {
   emptyBlocks: number;
 }
 
+export enum TemplateAlgorithm {
+  legacy = 0,
+  clusterMempool = 1,
+}
+
 export interface BlockAudit {
   version: number,
   time: number,
@@ -45,6 +50,7 @@ export interface BlockAudit {
   expectedFees?: number,
   expectedWeight?: number,
   template?: any[];
+  templateAlgorithm?: TemplateAlgorithm,
 }
 
 export interface TransactionAudit {
@@ -132,6 +138,8 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
   replacement?: boolean;
   uid?: number;
   flags?: number;
+  clusterId?: number;
+  chunkIndex?: number;
 }
 
 export interface MempoolTransactionExtended extends TransactionExtended {
@@ -227,6 +235,7 @@ export interface CpfpInfo {
   adjustedVsize?: number,
   acceleration?: boolean,
   fee?: number;
+  cluster?: CpfpClusterData & { chunkIndex: number };
 }
 
 export interface TransactionStripped {
@@ -388,6 +397,25 @@ export interface CpfpCluster {
   height: number,
   txs: Ancestor[],
   effectiveFeePerVsize: number,
+  templateAlgorithm?: TemplateAlgorithm,
+  clusterData?: CpfpClusterData,
+}
+
+export interface CpfpClusterTx {
+  txid: string;
+  fee: number;
+  weight: number;
+  parents: number[];
+}
+
+export interface CpfpClusterChunk {
+  txs: number[];
+  feerate: number;
+}
+
+export interface CpfpClusterData {
+  txs: CpfpClusterTx[];
+  chunks: CpfpClusterChunk[];
 }
 
 export interface CpfpSummary {

backend/src/replication/AuditReplication.ts

@@ -114,6 +114,7 @@ class AuditReplication {
     });
     await blocksAuditsRepository.$saveAudit({
       version: auditSummary.version || 0,
+      templateAlgorithm: auditSummary.templateAlgorithm ?? 0,
       hash: blockHash,
       height: auditSummary.height,
       time: auditSummary.timestamp || auditSummary.time,

backend/src/repositories/BlocksAuditsRepository.ts

@@ -1,7 +1,7 @@
 import DB from '../database';
 import logger from '../logger';
 import bitcoinApi from '../api/bitcoin/bitcoin-api-factory';
-import { BlockAudit, AuditScore, TransactionAudit, TransactionStripped } from '../mempool.interfaces';
+import { BlockAudit, AuditScore, TransactionAudit, TransactionStripped, TemplateAlgorithm } from '../mempool.interfaces';
 
 interface MigrationAudit {
   version: number,
@@ -18,8 +18,8 @@ class BlocksAuditRepositories {
   /** @asyncSafe */
   public async $saveAudit(audit: BlockAudit): Promise<void> {
     try {
-      await DB.query(`INSERT INTO blocks_audits(version, time, height, hash, unseen_txs, missing_txs, added_txs, prioritized_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight)
-        VALUE (?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.version, audit.time, audit.height, audit.hash, JSON.stringify(audit.unseenTxs), JSON.stringify(audit.missingTxs),
+      await DB.query(`INSERT INTO blocks_audits(version, template_algo, time, height, hash, unseen_txs, missing_txs, added_txs, prioritized_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight)
+        VALUE (?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.version, audit.templateAlgorithm ?? 0, audit.time, audit.height, audit.hash, JSON.stringify(audit.unseenTxs), JSON.stringify(audit.missingTxs),
           JSON.stringify(audit.addedTxs), JSON.stringify(audit.prioritizedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), JSON.stringify(audit.acceleratedTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]);
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
@@ -80,6 +80,7 @@ class BlocksAuditRepositories {
       const [rows]: any[] = await DB.query(
         `SELECT
           blocks_audits.version,
+          blocks_audits.template_algo as templateAlgorithm,
           blocks_audits.height,
           blocks_audits.hash as id,
           UNIX_TIMESTAMP(blocks_audits.time) as timestamp,
@@ -120,6 +121,23 @@ class BlocksAuditRepositories {
     }
   }
 
+  /** @asyncSafe */
+  public async $getBlockTemplateAlgo(hash: string): Promise<TemplateAlgorithm | null> {
+    try {
+      const [rows]: any[] = await DB.query(
+        `SELECT template_algo FROM blocks_audits WHERE hash = ?`,
+        [hash]
+      );
+      if (rows.length) {
+        return rows[0].template_algo as TemplateAlgorithm;
+      }
+      return null;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block template algo from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return null;
+    }
+  }
+
   /** @asyncSafe */
   public async $getBlockTxAudit(hash: string, txid: string): Promise<TransactionAudit | null> {
     try {

backend/src/repositories/CpfpRepository.ts

@@ -1,31 +1,47 @@
 import { RowDataPacket } from 'mysql2';
 import DB from '../database';
 import logger from '../logger';
-import { Ancestor, CpfpCluster } from '../mempool.interfaces';
+import { Ancestor, CpfpCluster, CpfpClusterData, CpfpClusterTx, TemplateAlgorithm } from '../mempool.interfaces';
 import transactionRepository from '../repositories/TransactionRepository';
 
 class CpfpRepository {
-  public async $batchSaveClusters(clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[]): Promise<boolean> {
+  public async $batchSaveClusters(clusters: CpfpCluster[]): Promise<boolean> {
     try {
-      const clusterValues: [string, number, Buffer, number][] = [];
+      const clusterValues: [string, number, Buffer, number, number][] = [];
       const txs: { txid: string, cluster: string }[] = [];
 
       for (const cluster of clusters) {
         if (cluster.txs?.length) {
-          const roundedEffectiveFee = Math.round(cluster.effectiveFeePerVsize * 100) / 100;
-          const equalFee = cluster.txs.length > 1 && cluster.txs.reduce((acc, tx) => {
-            return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee);
-          }, true);
-          if (!equalFee) {
+          const isCM = cluster.templateAlgorithm === TemplateAlgorithm.clusterMempool;
+
+          if (isCM && cluster.clusterData) {
             clusterValues.push([
               cluster.root,
               cluster.height,
-              Buffer.from(this.pack(cluster.txs)),
-              cluster.effectiveFeePerVsize
+              Buffer.from(this.packCM(cluster.clusterData)),
+              0,
+              TemplateAlgorithm.clusterMempool,
             ]);
-            for (const tx of cluster.txs) {
+            for (const tx of cluster.clusterData.txs) {
               txs.push({ txid: tx.txid, cluster: cluster.root });
             }
+          } else {
+            const roundedEffectiveFee = Math.round(cluster.effectiveFeePerVsize * 100) / 100;
+            const equalFee = cluster.txs.length > 1 && cluster.txs.reduce((acc, tx) => {
+              return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee);
+            }, true);
+            if (!equalFee) {
+              clusterValues.push([
+                cluster.root,
+                cluster.height,
+                Buffer.from(this.pack(cluster.txs)),
+                cluster.effectiveFeePerVsize,
+                TemplateAlgorithm.legacy,
+              ]);
+              for (const tx of cluster.txs) {
+                txs.push({ txid: tx.txid, cluster: cluster.root });
+              }
+            }
           }
         }
       }
@@ -42,11 +58,11 @@ class CpfpRepository {
       while (chunkIndex < clusterValues.length) {
         const chunk = clusterValues.slice(chunkIndex, chunkIndex + maxChunk);
         let query = `
-            INSERT IGNORE INTO compact_cpfp_clusters(root, height, txs, fee_rate)
+            INSERT IGNORE INTO compact_cpfp_clusters(root, height, txs, fee_rate, template_algo)
             VALUES
         `;
         query += chunk.map(chunk => {
-          return (' (UNHEX(?), ?, ?, ?)');
+          return (' (UNHEX(?), ?, ?, ?, ?)');
         }) + ';';
         const values = chunk.flat();
         queries.push({
@@ -86,8 +102,16 @@ class CpfpRepository {
     );
     const cluster = clusterRows[0];
     if (cluster?.txs) {
-      cluster.effectiveFeePerVsize = cluster.fee_rate;
-      cluster.txs = this.unpack(cluster.txs);
+      if (cluster.template_algo === TemplateAlgorithm.clusterMempool) {
+        cluster.templateAlgorithm = TemplateAlgorithm.clusterMempool;
+        cluster.clusterData = this.unpackCM(cluster.txs);
+        cluster.txs = cluster.clusterData.txs.map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee }));
+        cluster.effectiveFeePerVsize = 0;
+      } else {
+        cluster.templateAlgorithm = TemplateAlgorithm.legacy;
+        cluster.effectiveFeePerVsize = cluster.fee_rate;
+        cluster.txs = this.unpack(cluster.txs);
+      }
       return cluster;
     }
     return;
@@ -105,8 +129,16 @@ class CpfpRepository {
     );
     return clusterRows.map(cluster => {
       if (cluster?.txs) {
-        cluster.effectiveFeePerVsize = cluster.fee_rate;
-        cluster.txs = this.unpack(cluster.txs);
+        if (cluster.template_algo === TemplateAlgorithm.clusterMempool) {
+          cluster.templateAlgorithm = TemplateAlgorithm.clusterMempool;
+          cluster.clusterData = this.unpackCM(cluster.txs);
+          cluster.txs = cluster.clusterData.txs.map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee }));
+          cluster.effectiveFeePerVsize = 0;
+        } else {
+          cluster.templateAlgorithm = TemplateAlgorithm.legacy;
+          cluster.effectiveFeePerVsize = cluster.fee_rate;
+          cluster.txs = this.unpack(cluster.txs);
+        }
         return cluster;
       } else {
         return null;
@@ -119,16 +151,16 @@ class CpfpRepository {
     try {
       const [rows] = await DB.query(
         `
-          SELECT txs, height, root from compact_cpfp_clusters
+          SELECT txs, height, root, template_algo from compact_cpfp_clusters
           WHERE height >= ?
         `,
         [height]
       ) as RowDataPacket[][];
       if (rows?.length) {
         for (const clusterToDelete of rows) {
-          const txs = this.unpack(clusterToDelete?.txs);
-          for (const tx of txs) {
-            await transactionRepository.$removeTransaction(tx.txid);
+          const txids = this.extractTxids(clusterToDelete);
+          for (const txid of txids) {
+            await transactionRepository.$removeTransaction(txid);
           }
         }
       }
@@ -150,16 +182,16 @@ class CpfpRepository {
     try {
       const [rows] = await DB.query(
         `
-          SELECT txs, height, root from compact_cpfp_clusters
+          SELECT txs, height, root, template_algo from compact_cpfp_clusters
           WHERE height = ?
         `,
         [height]
       ) as RowDataPacket[][];
       if (rows?.length) {
         for (const clusterToDelete of rows) {
-          const txs = this.unpack(clusterToDelete?.txs);
-          for (const tx of txs) {
-            await transactionRepository.$removeTransaction(tx.txid);
+          const txids = this.extractTxids(clusterToDelete);
+          for (const txid of txids) {
+            await transactionRepository.$removeTransaction(txid);
           }
         }
       }
@@ -176,6 +208,13 @@ class CpfpRepository {
     }
   }
 
+  private extractTxids(row: any): string[] {
+    if (row.template_algo === TemplateAlgorithm.clusterMempool) {
+      return this.unpackCM(row.txs).txs.map(tx => tx.txid);
+    }
+    return this.unpack(row.txs).map(tx => tx.txid);
+  }
+
   // insert a dummy row to mark that we've indexed as far as this block
   public async $insertProgressMarker(height: number): Promise<void> {
     try {
@@ -245,6 +284,117 @@ class CpfpRepository {
     }
   }
 
+  /**
+   * Pack cluster mempool data into binary format:
+   * [num_chunks: uint16]
+   * Per chunk: [num_txs: uint16]
+   * Per tx (in linearization order, grouped by chunk):
+   *   [txid: 32 bytes LE] [weight: uint32] [fee: uint64] [num_parents: uint8] [parent_indices: uint8 each]
+   */
+  public packCM(clusterData: CpfpClusterData): ArrayBuffer {
+    const headerSize = 2;
+    const chunkHeadersSize = clusterData.chunks.length * 2;
+    let txDataSize = 0;
+    for (const tx of clusterData.txs) {
+      txDataSize += 32 + 4 + 8 + 1 + tx.parents.length;
+    }
+    const totalSize = headerSize + chunkHeadersSize + txDataSize;
+
+    const buf = new ArrayBuffer(totalSize);
+    const view = new DataView(buf);
+    let offset = 0;
+
+    view.setUint16(offset, clusterData.chunks.length);
+    offset += 2;
+
+    for (const chunk of clusterData.chunks) {
+      view.setUint16(offset, chunk.txs.length);
+      offset += 2;
+    }
+
+    for (const tx of clusterData.txs) {
+      for (let x = 0; x < 32; x++) {
+        view.setUint8(offset + (31 - x), parseInt(tx.txid.slice(x * 2, (x * 2) + 2), 16));
+      }
+      offset += 32;
+      view.setUint32(offset, tx.weight);
+      offset += 4;
+      view.setBigUint64(offset, BigInt(Math.round(tx.fee)));
+      offset += 8;
+      view.setUint8(offset, tx.parents.length);
+      offset += 1;
+      for (const parentIdx of tx.parents) {
+        view.setUint8(offset, parentIdx);
+        offset += 1;
+      }
+    }
+
+    return buf;
+  }
+
+  public unpackCM(buf: Buffer): CpfpClusterData {
+    if (!buf) {
+      return { txs: [], chunks: [] };
+    }
+
+    try {
+      const arrayBuffer = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
+      const view = new DataView(arrayBuffer);
+      let offset = 0;
+
+      const numChunks = view.getUint16(offset);
+      offset += 2;
+
+      const chunkSizes: number[] = [];
+      for (let i = 0; i < numChunks; i++) {
+        chunkSizes.push(view.getUint16(offset));
+        offset += 2;
+      }
+
+      const txs: CpfpClusterTx[] = [];
+      const chunks: { txs: number[], feerate: number }[] = [];
+      let txIndex = 0;
+
+      for (let chunkIdx = 0; chunkIdx < numChunks; chunkIdx++) {
+        const chunkTxIndices: number[] = [];
+        let chunkFee = 0;
+        let chunkWeight = 0;
+
+        for (let t = 0; t < chunkSizes[chunkIdx]; t++) {
+          const txid = Array.from(new Uint8Array(arrayBuffer, offset, 32)).reverse().map(b => b.toString(16).padStart(2, '0')).join('');
+          offset += 32;
+          const weight = view.getUint32(offset);
+          offset += 4;
+          const fee = Number(view.getBigUint64(offset));
+          offset += 8;
+          const numParents = view.getUint8(offset);
+          offset += 1;
+          const parents: number[] = [];
+          for (let p = 0; p < numParents; p++) {
+            parents.push(view.getUint8(offset));
+            offset += 1;
+          }
+
+          txs.push({ txid, fee, weight, parents });
+          chunkTxIndices.push(txIndex);
+          chunkFee += fee;
+          chunkWeight += weight;
+          txIndex++;
+        }
+
+        chunks.push({
+          txs: chunkTxIndices,
+          feerate: chunkWeight > 0 ? (chunkFee * 4) / chunkWeight : 0,
+        });
+      }
+
+      return { txs, chunks };
+    } catch (e) {
+      logger.warn(`Failed to unpack CM CPFP cluster. Reason: ` + (e instanceof Error ? e.message : e));
+      return { txs: [], chunks: [] };
+    }
+  }
+
   // returns `true` if two sets of CPFP clusters are deeply identical
   public compareClusters(clustersA: CpfpCluster[], clustersB: CpfpCluster[]): boolean {
     if (clustersA.length !== clustersB.length) {

backend/src/repositories/TransactionRepository.ts

@@ -1,6 +1,6 @@
 import DB from '../database';
 import logger from '../logger';
-import { Ancestor, CpfpInfo } from '../mempool.interfaces';
+import { Ancestor, CpfpCluster, CpfpInfo, TemplateAlgorithm } from '../mempool.interfaces';
 import cpfpRepository from './CpfpRepository';
 
 class TransactionRepository {
@@ -72,6 +72,9 @@ class TransactionRepository {
         const clusterId = txRows[0].root.toLowerCase();
         const cluster = await cpfpRepository.$getCluster(clusterId);
         if (cluster) {
+          if (cluster.templateAlgorithm === TemplateAlgorithm.clusterMempool && cluster.clusterData) {
+            return this.convertCpfpCM(txid, cluster);
+          }
           return this.convertCpfp(txid, cluster);
         }
       }
@@ -116,6 +119,95 @@ class TransactionRepository {
       effectiveFeePerVsize: cluster.effectiveFeePerVsize,
     };
   }
+
+  private convertCpfpCM(txid: string, cluster: CpfpCluster): CpfpInfo {
+    const clusterData = cluster.clusterData;
+    if (!clusterData) {
+      return { ancestors: [], descendants: [], effectiveFeePerVsize: 0 };
+    }
+
+    // Find which chunk this tx belongs to
+    let txFlatIdx = -1;
+    let txChunkIndex = -1;
+    for (let i = 0; i < clusterData.txs.length; i++) {
+      if (clusterData.txs[i].txid === txid) {
+        txFlatIdx = i;
+        break;
+      }
+    }
+
+    // Find the chunk containing this tx
+    for (let chunkIdx = 0; chunkIdx < clusterData.chunks.length; chunkIdx++) {
+      if (clusterData.chunks[chunkIdx].txs.includes(txFlatIdx)) {
+        txChunkIndex = chunkIdx;
+        break;
+      }
+    }
+
+    // Derive ancestors/descendants from in-chunk depgraph parents
+    // For CM, ancestors are the tx's depgraph parents within the cluster,
+    // descendants are txs that depend on this tx
+    const ancestors: Ancestor[] = [];
+    const descendants: Ancestor[] = [];
+
+    if (txFlatIdx >= 0) {
+      // Build child map
+      const childMap = new Map<number, number[]>();
+      for (let i = 0; i < clusterData.txs.length; i++) {
+        for (const parentIdx of clusterData.txs[i].parents) {
+          let children = childMap.get(parentIdx);
+          if (!children) {
+            children = [];
+            childMap.set(parentIdx, children);
+          }
+          children.push(i);
+        }
+      }
+
+      const ancestorSet = new Set<number>();
+      const stack = [...clusterData.txs[txFlatIdx].parents];
+      while (stack.length) {
+        const idx = stack.pop();
+        if (idx === undefined || ancestorSet.has(idx)) {
+          continue;
+        }
+        ancestorSet.add(idx);
+        stack.push(...clusterData.txs[idx].parents);
+      }
+
+      const descendantSet = new Set<number>();
+      const dStack = [...(childMap.get(txFlatIdx) || [])];
+      while (dStack.length) {
+        const idx = dStack.pop();
+        if (idx === undefined || descendantSet.has(idx)) {
+          continue;
+        }
+        descendantSet.add(idx);
+        dStack.push(...(childMap.get(idx) || []));
+      }
+
+      for (const idx of ancestorSet) {
+        const tx = clusterData.txs[idx];
+        ancestors.push({ txid: tx.txid, weight: tx.weight, fee: tx.fee });
+      }
+      for (const idx of descendantSet) {
+        const tx = clusterData.txs[idx];
+        descendants.push({ txid: tx.txid, weight: tx.weight, fee: tx.fee });
+      }
+    }
+
+    const effectiveFeePerVsize = txChunkIndex >= 0 ? clusterData.chunks[txChunkIndex].feerate : 0;
+
+    return {
+      ancestors,
+      descendants,
+      effectiveFeePerVsize,
+      cluster: {
+        ...clusterData,
+        chunkIndex: txChunkIndex,
+      },
+    };
+  }
 }
 
 export default new TransactionRepository();

backend/tsconfig.build.json

@@ -1,6 +1,6 @@
 {
   "extends": "./tsconfig",
-  "exclude": ["**/*.test.*", "**/__mocks__/*", "**/__tests__/*"],
+  "exclude": ["**/*.test.*", "**/__mocks__/*", "**/__tests__/*", "**/__e2e__/*"],
   "compilerOptions": {
     "types": ["node"]
   },