From edaf5a64542a12a6776c0d3d23fbe4c4fbac1927 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 21 Mar 2026 11:14:04 +0000 Subject: [PATCH] optimize cluster mempool --- backend/jest.config.ts | 1 + .../cluster-mempool/cluster-mempool.test.ts | 4 + .../cluster-mempool/linearize.test.ts | 50 +++---- backend/src/api/block-processor.ts | 73 ++++++---- backend/src/api/blocks.ts | 15 +- backend/src/api/cpfp.ts | 82 +++++++---- .../src/cluster-mempool/cluster-mempool.ts | 135 +++++++++++------- backend/src/cluster-mempool/linearize.ts | 8 +- backend/src/indexer.ts | 6 +- backend/tsconfig.build.json | 2 +- 10 files changed, 226 insertions(+), 150 deletions(-) diff --git a/backend/jest.config.ts b/backend/jest.config.ts index fd6e5abf3..78c4ca054 100644 --- a/backend/jest.config.ts +++ b/backend/jest.config.ts @@ -17,6 +17,7 @@ const config: Config.InitialOptions = { './testSetup.ts', ], testPathIgnorePatterns: [ + '/dist/', '/node_modules/', '/__integration_tests__/', 'test-utils\\.ts$', diff --git a/backend/src/__tests__/cluster-mempool/cluster-mempool.test.ts b/backend/src/__tests__/cluster-mempool/cluster-mempool.test.ts index d4fbb7e9a..11d99b6eb 100644 --- a/backend/src/__tests__/cluster-mempool/cluster-mempool.test.ts +++ b/backend/src/__tests__/cluster-mempool/cluster-mempool.test.ts @@ -198,6 +198,7 @@ describe('ClusterMempool', () => { expect(cm.getClusterCount()).toBe(3); const bridge = makeTx(txid('d1'), 400, 100, [a, b, c]); + mempool[bridge.txid] = bridge; cm.applyMempoolChange({ added: [bridge], removed: [], accelerations: {} }); expect(cm.getClusterCount()).toBe(1); @@ -215,6 +216,7 @@ describe('ClusterMempool', () => { expect(cm.getClusterCount()).toBe(1); const c = makeTx(txid('c1'), 300, 100, [a]); + mempool[c.txid] = c; cm.applyMempoolChange({ added: [c], removed: [], accelerations: {} }); expect(cm.getClusterCount()).toBe(1); @@ -227,9 +229,11 @@ describe('ClusterMempool', () => { const cm = new ClusterMempool(mempool); const b = makeTx(txid('b1'), 200, 100, [a]); + mempool[b.txid] = b; cm.applyMempoolChange({ added: [b], removed: [], accelerations: {} }); const c = makeTx(txid('c1'), 300, 100, [txid('b1')]); + mempool[c.txid] = c; cm.applyMempoolChange({ added: [c], removed: [], accelerations: {} }); expect(cm.getClusterCount()).toBe(1); diff --git a/backend/src/__tests__/cluster-mempool/linearize.test.ts b/backend/src/__tests__/cluster-mempool/linearize.test.ts index da5d93a73..1371c59c2 100644 --- a/backend/src/__tests__/cluster-mempool/linearize.test.ts +++ b/backend/src/__tests__/cluster-mempool/linearize.test.ts @@ -107,7 +107,7 @@ describe('spanningForestLinearize', () => { const b = dg.addTransaction('b', 500, 100); const c = dg.addTransaction('c', 300, 100); - const result = spanningForestLinearize(dg.getTxs()); + const result = spanningForestLinearize(dg.getTxs(), 75000); expect(result[0]).toBe(b); expect(result[1]).toBe(c); expect(result[2]).toBe(a); @@ -119,7 +119,7 @@ describe('spanningForestLinearize', () => { const child = dg.addTransaction('child', 500, 100); dg.addDependency(parent, child); - const result = spanningForestLinearize(dg.getTxs()); + const result = spanningForestLinearize(dg.getTxs(), 75000); expect(result.indexOf(parent)).toBeLessThan(result.indexOf(child)); }); @@ -129,7 +129,7 @@ describe('spanningForestLinearize', () => { const b = dg.addTransaction('b', 10000, 100); dg.addDependency(a, b); - const { chunks } = linearizeCluster(dg.getTxs()); + const { chunks } = linearizeCluster(dg.getTxs(), 75000); expect(chunks.length).toBe(1); expect(chunks[0].txs.length).toBe(2); }); @@ -139,7 +139,7 @@ describe('spanningForestLinearize', () => { const a = dg.addTransaction('a', 1000, 100); const b = dg.addTransaction('b', 100, 100); - const { chunks } = linearizeCluster(dg.getTxs()); + const { chunks } = linearizeCluster(dg.getTxs(), 75000); expect(chunks.length).toBe(2); expect(chunks[0].txs).toContain(a); expect(chunks[1].txs).toContain(b); @@ -149,13 +149,13 @@ describe('spanningForestLinearize', () => { const dg = new DepGraph(); const a = dg.addTransaction('a', 1000, 100); - const result = spanningForestLinearize(dg.getTxs()); + const result = spanningForestLinearize(dg.getTxs(), 75000); expect(result).toEqual([a]); }); it('should handle empty graph', () => { const dg = new DepGraph(); - const result = spanningForestLinearize(dg.getTxs()); + const result = spanningForestLinearize(dg.getTxs(), 75000); expect(result).toEqual([]); }); }); @@ -171,7 +171,7 @@ describe('minimize', () => { dg.addDependency(txs[i - 1], txs[i]); } - const { chunks } = linearizeCluster(dg.getTxs()); + const { chunks } = linearizeCluster(dg.getTxs(), 75000); expect(chunks.length).toBe(5); for (const chunk of chunks) { expect(chunk.txs.length).toBe(1); @@ -184,7 +184,7 @@ describe('minimize', () => { const child = dg.addTransaction('child', 900, 100); dg.addDependency(parent, child); - const { chunks } = linearizeCluster(dg.getTxs()); + const { chunks } = linearizeCluster(dg.getTxs(), 75000); expect(chunks.length).toBe(1); expect(chunks[0].txs.length).toBe(2); }); @@ -195,7 +195,7 @@ describe('minimize', () => { const child = dg.addTransaction('child', 1240, 248); dg.addDependency(parent, child); - const { chunks } = linearizeCluster(dg.getTxs()); + const { chunks } = linearizeCluster(dg.getTxs(), 75000); expect(chunks.length).toBe(2); }); @@ -204,7 +204,7 @@ describe('minimize', () => { dg.addTransaction('a', 100, 100); dg.addTransaction('b', 100, 100); - const { chunks } = linearizeCluster(dg.getTxs()); + const { chunks } = linearizeCluster(dg.getTxs(), 75000); expect(chunks.length).toBe(2); expect(chunks[0].txs.length).toBe(1); expect(chunks[1].txs.length).toBe(1); @@ -319,7 +319,7 @@ describe('SFL adversarial topologies', () => { dg.addDependency(root, child); } - const { linearization, chunks } = linearizeCluster(dg.getTxs()); + const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000); verifyLinearization(dg.getTxs(), linearization, chunks); }); @@ -336,7 +336,7 @@ describe('SFL adversarial topologies', () => { dg.addDependency(i < 2 ? mid1 : mid2, leaf); } - const { linearization, chunks } = linearizeCluster(dg.getTxs()); + const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000); verifyLinearization(dg.getTxs(), linearization, chunks); }); @@ -358,7 +358,7 @@ describe('SFL adversarial topologies', () => { prev2 = tx; } - const { linearization, chunks } = linearizeCluster(dg.getTxs()); + const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000); verifyLinearization(dg.getTxs(), linearization, chunks); }); @@ -366,7 +366,7 @@ describe('SFL adversarial topologies', () => { const { depgraph, txs } = buildChain(6, 10, 100); txs[5].effectiveFee = 50000; - const { linearization, chunks } = linearizeCluster(depgraph.getTxs()); + const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), 75000); verifyLinearization(depgraph.getTxs(), linearization, chunks); expect(chunks[0].txs.length).toBeGreaterThan(1); }); @@ -379,7 +379,7 @@ describe('SFL adversarial topologies', () => { dg.addDependency(a, c); dg.addDependency(b, c); - const { linearization, chunks } = linearizeCluster(dg.getTxs()); + const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000 ); verifyLinearization(dg.getTxs(), linearization, chunks); const firstChunkFee = chunks[0].fee; const firstChunkSize = chunks[0].weight; @@ -396,7 +396,7 @@ describe('linearizeCluster', () => { dg.addDependency(a, b); dg.addDependency(b, c); - const { linearization } = linearizeCluster(dg.getTxs()); + const { linearization } = linearizeCluster(dg.getTxs(), 75000); expect(linearization.indexOf(a)).toBeLessThan(linearization.indexOf(b)); expect(linearization.indexOf(b)).toBeLessThan(linearization.indexOf(c)); }); @@ -407,7 +407,7 @@ describe('linearizeCluster', () => { dg.addTransaction(`tx${i}`, Math.floor(Math.random() * 10000) + 100, Math.floor(Math.random() * 500) + 50); } - const { chunks } = linearizeCluster(dg.getTxs()); + const { chunks } = linearizeCluster(dg.getTxs(), 75000); for (let i = 1; i < chunks.length; i++) { const prevRate = chunks[i - 1].fee * chunks[i].weight; const curRate = chunks[i].fee * chunks[i - 1].weight; @@ -427,7 +427,7 @@ describe('linearizeCluster', () => { dg.addDependency(b, d); dg.addDependency(c, d); - const { linearization, chunks } = linearizeCluster(dg.getTxs()); + const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000); verifyLinearization(dg.getTxs(), linearization, chunks); }); @@ -438,8 +438,8 @@ describe('linearizeCluster', () => { const c = dg.addTransaction('c', 1000, 100); const suboptimal = [b, a, c]; - const { chunks: hintChunks } = linearizeCluster(dg.getTxs(), suboptimal); - const { chunks: freshChunks } = linearizeCluster(dg.getTxs()); + const { chunks: hintChunks } = linearizeCluster(dg.getTxs(), 75000, suboptimal); + const { chunks: freshChunks } = linearizeCluster(dg.getTxs(), 75000); const hintFirstFeerate = hintChunks[0].fee * freshChunks[0].weight; const freshFirstFeerate = freshChunks[0].fee * hintChunks[0].weight; @@ -453,7 +453,7 @@ describe('linearizeCluster', () => { const c = dg.addTransaction('c', 100, 100); const optimal = [a, b, c]; - const { linearization } = linearizeCluster(dg.getTxs(), optimal); + const { linearization } = linearizeCluster(dg.getTxs(), 75000, optimal); expect(linearization).toEqual(optimal); }); @@ -465,21 +465,21 @@ describe('linearizeCluster', () => { dg.addDependency(a, c); dg.addDependency(b, c); - const result1 = linearizeCluster(dg.getTxs()); - const result2 = linearizeCluster(dg.getTxs()); + const result1 = linearizeCluster(dg.getTxs(), 75000); + const result2 = linearizeCluster(dg.getTxs(), 75000); verifyLinearization(dg.getTxs(), result1.linearization, result1.chunks); verifyLinearization(dg.getTxs(), result2.linearization, result2.chunks); }); it('should pass invariant checks for fan-out topology', () => { const { depgraph } = buildFanOut(6, 100, 100, 500, 100); - const { linearization, chunks } = linearizeCluster(depgraph.getTxs()); + const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), 75000); verifyLinearization(depgraph.getTxs(), linearization, chunks); }); it('should pass invariant checks for star topology', () => { const { depgraph } = buildStar(5, 100, 100, 300, 100); - const { linearization, chunks } = linearizeCluster(depgraph.getTxs()); + const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), 75000); verifyLinearization(depgraph.getTxs(), linearization, chunks); }); }); diff --git a/backend/src/api/block-processor.ts b/backend/src/api/block-processor.ts index 7d8ce4f68..eb0eaaeaf 100644 --- a/backend/src/api/block-processor.ts +++ b/backend/src/api/block-processor.ts @@ -11,7 +11,7 @@ import { } from '../mempool.interfaces'; import { IEsploraApi } from './bitcoin/esplora-api.interface'; import { Acceleration } from './services/acceleration'; -import { calculateGoodBlockCpfp, calculateClusterMempoolBlockCpfp, calculateFastBlockCpfp } from './cpfp'; +import { calculateGoodBlockCpfp, calculateClusterMempoolBlockCpfp, calculateFastBlockCpfp, BlockCpfpData } from './cpfp'; import mempoolBlocks from './mempool-blocks'; import memPool from './mempool'; import Audit, { AuditResult } from './audit'; @@ -62,7 +62,6 @@ class BlockProcessor { poolAccelerations ); - logger.debug(`Block #${block.height} detected template algorithm: ${templateAlgorithm === TemplateAlgorithm.clusterMempool ? 'cluster mempool' : 'legacy GBT'}`); const blockExtended = await blocks.$getBlockExtended(block, cpfpSummary.transactions, pool); const blockSummary = blocks.summarizeBlockTransactions(block.id, block.height, cpfpSummary.transactions); @@ -120,7 +119,7 @@ class BlockProcessor { let projectedBlocks: MempoolBlockWithTransactions[]; if (templateAlgorithm === TemplateAlgorithm.clusterMempool) { - const clusterMempool = memPool.clusterMempool ?? new ClusterMempool(auditMempool, accelerations); + const clusterMempool = memPool.clusterMempool ?? new ClusterMempool(auditMempool, accelerations, true, 75000); const cmBlocks = clusterMempool.getBlocks(config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT) ?? []; projectedBlocks = mempoolBlocks.processClusterMempoolBlocks( cmBlocks, @@ -153,7 +152,7 @@ class BlockProcessor { ); } - const auditResult = Audit.auditBlock(block.height, structuredClone(transactions), projectedBlocks, auditMempool); + const auditResult = Audit.auditBlock(block.height, transactions, projectedBlocks, auditMempool); const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : []; @@ -173,6 +172,32 @@ class BlockProcessor { } } +function saveCpfpDataToTransactions(transactions: MempoolTransactionExtended[], cpfpData: BlockCpfpData): void { + for (const tx of transactions) { + if (cpfpData.txs[tx.txid]) { + Object.assign(tx, cpfpData.txs[tx.txid]); + } + } +} + +export function saveCpfpDataToCpfpSummary(transactions: MempoolTransactionExtended[], cpfpData: BlockCpfpData): CpfpSummary { + saveCpfpDataToTransactions(transactions, cpfpData); + return { + transactions, + clusters: cpfpData.clusters, + version: cpfpData.version, + }; +} + +/** + * + * @param height + * @param blockTransactions + * @param poolAccelerations + * @param fast + * + * saves effective fee rates from detected algorithm to blockTransactions + */ export function detectTemplateAlgorithm( height: number, blockTransactions: MempoolTransactionExtended[], @@ -183,58 +208,46 @@ export function detectTemplateAlgorithm( const network = config.MEMPOOL.NETWORK || 'mainnet'; const activationHeight = CM_ACTIVATION_HEIGHT[network] ?? Infinity; - // always need the legacy CPFP summary - const legacyCpfpSummary = fast ? calculateFastBlockCpfp( + const legacyCpfpData = fast ? calculateFastBlockCpfp( height, - structuredClone(blockTransactions), + blockTransactions, ) : calculateGoodBlockCpfp( height, - structuredClone(blockTransactions), + blockTransactions, poolAccelerations ); - // assume legacy below the activation height if (height < activationHeight) { return { templateAlgorithm: TemplateAlgorithm.legacy, - cpfpSummary: legacyCpfpSummary, + cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, legacyCpfpData), }; } - // calculate single-block CPFP rates for each algorithm - const clusterCpfpSummary = calculateClusterMempoolBlockCpfp( + const clusterCpfpData = calculateClusterMempoolBlockCpfp( height, - structuredClone(blockTransactions), + blockTransactions, poolAccelerations ); - const clusterRates = new Map(); - const legacyRates = new Map(); - for (const tx of clusterCpfpSummary.transactions) { - clusterRates.set(tx.txid, tx.effectiveFeePerVsize); - } - for (const tx of legacyCpfpSummary.transactions) { - legacyRates.set(tx.txid, tx.effectiveFeePerVsize); - } - const clusterTxs = blockTransactions.map(tx => ({...tx, effectiveFeePerVsize: clusterRates.get(tx.txid) || tx.effectiveFeePerVsize})); - const legacyTxs = blockTransactions.map(tx => ({...tx, effectiveFeePerVsize: legacyRates.get(tx.txid) || tx.effectiveFeePerVsize})); - // identify apparent prioritizations using each algorithm's rates - const clusterPrioritization = transactionUtils.identifyPrioritizedTransactions(clusterTxs, 'effectiveFeePerVsize'); - const legacyPrioritization = transactionUtils.identifyPrioritizedTransactions(legacyTxs, 'effectiveFeePerVsize'); + const clusterTxs = blockTransactions.map(tx => ({ txid: tx.txid, rate: clusterCpfpData.txs[tx.txid].effectiveFeePerVsize ?? tx.effectiveFeePerVsize })); + const legacyTxs = blockTransactions.map(tx => ({ txid: tx.txid, rate: legacyCpfpData.txs[tx.txid].effectiveFeePerVsize ?? tx.effectiveFeePerVsize })); + const clusterPrioritization = transactionUtils.identifyPrioritizedTransactions(clusterTxs, 'rate'); + const legacyPrioritization = transactionUtils.identifyPrioritizedTransactions(legacyTxs, 'rate'); - // choose the best fitting algorithm (or legacy if tied) const clusterCount = clusterPrioritization.prioritized.length + clusterPrioritization.deprioritized.length; const legacyCount = legacyPrioritization.prioritized.length + legacyPrioritization.deprioritized.length; - logger.debug(`Prioritization counts - cluster: ${clusterCount}, legacy: ${legacyCount}`); + if (clusterCount < legacyCount) { + saveCpfpDataToTransactions(blockTransactions, clusterCpfpData); return { templateAlgorithm: TemplateAlgorithm.clusterMempool, - cpfpSummary: clusterCpfpSummary, + cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, clusterCpfpData), }; } else { return { templateAlgorithm: TemplateAlgorithm.legacy, - cpfpSummary: legacyCpfpSummary, + cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, legacyCpfpData), }; } } diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index faded1e08..613b77932 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -31,11 +31,10 @@ import rbfCache from './rbf-cache'; import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; import mempoolBlocks from './mempool-blocks'; import statistics from './statistics/statistics'; -import { Acceleration } from './services/acceleration'; import { calcBitsDifference } from './difficulty-adjustment'; import AccelerationRepository from '../repositories/AccelerationRepository'; -import { calculateFastBlockCpfp, calculateGoodBlockCpfp, calculateClusterMempoolBlockCpfp } from './cpfp'; -import blockProcessor, { BlockProcessingResult, detectTemplateAlgorithm } from './block-processor'; +import { calculateGoodBlockCpfp } from './cpfp'; +import blockProcessor, { BlockProcessingResult, detectTemplateAlgorithm, saveCpfpDataToCpfpSummary } from './block-processor'; import mempool from './mempool'; import CpfpRepository from '../repositories/CpfpRepository'; import { parseDATUMTemplateCreator } from '../utils/bitcoin-script'; @@ -852,7 +851,8 @@ class Blocks { // fetch transactions txs = (await bitcoinApi.$getTxsForBlock(blockHash, true)).map(tx => transactionUtils.extendMempoolTransaction(tx)) || []; // add CPFP - const cpfpSummary = calculateGoodBlockCpfp(height, txs, []); + const blockCpfpData = calculateGoodBlockCpfp(height, txs, []); + const cpfpSummary = saveCpfpDataToCpfpSummary(txs, blockCpfpData); // classify const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, height, cpfpSummary.transactions); await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 2); @@ -889,7 +889,8 @@ class Blocks { } templateTxs.push(tx || templateTx); } - const cpfpSummary = calculateGoodBlockCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as MempoolTransactionExtended[], []); + const blockCpfpData = calculateGoodBlockCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as MempoolTransactionExtended[], []); + const cpfpSummary = saveCpfpDataToCpfpSummary(templateTxs as MempoolTransactionExtended[], blockCpfpData); // classify const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, height, cpfpSummary.transactions); const classifiedTxMap: { [txid: string]: TransactionClassified } = {}; @@ -1218,7 +1219,7 @@ class Blocks { await AccelerationRepository.$indexAccelerationsForBlock( blockExtended, Object.values(accelerations), - structuredClone(cpfpSummary.transactions) + cpfpSummary.transactions ); this.updateTimerProgress(timer, `indexed accelerations for ${this.currentBlockHeight}`); @@ -1763,7 +1764,7 @@ class Blocks { } if (transactions?.length != null) { - const { cpfpSummary } = await detectTemplateAlgorithm(height, transactions, []); + const { cpfpSummary } = await detectTemplateAlgorithm(height, transactions, [], true); if (!stale) { await this.$saveCpfp(hash, height, cpfpSummary); diff --git a/backend/src/api/cpfp.ts b/backend/src/api/cpfp.ts index 53385f925..30232c90b 100644 --- a/backend/src/api/cpfp.ts +++ b/backend/src/api/cpfp.ts @@ -1,4 +1,4 @@ -import { Ancestor, CpfpCluster, CpfpInfo, CpfpSummary, MempoolTransactionExtended, TemplateAlgorithm, TransactionExtended } from '../mempool.interfaces'; +import { Ancestor, CpfpCluster, CpfpInfo, MempoolTransactionExtended, TemplateAlgorithm, TransactionExtended } from '../mempool.interfaces'; import { GraphTx, convertToGraphTx, expandRelativesGraph, initializeRelatives, makeBlockTemplate, mempoolComparator, removeAncestors, setAncestorScores } from './mini-miner'; import memPool from './mempool'; import { Acceleration } from './acceleration/acceleration'; @@ -7,15 +7,24 @@ import { ClusterMempool } from '../cluster-mempool/cluster-mempool'; const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction const MAX_CLUSTER_ITERATIONS = 100; -export function calculateFastBlockCpfp(height: number, transactions: MempoolTransactionExtended[], saveRelatives: boolean = false): CpfpSummary { +type TransactionCpfpData = Partial; +export interface BlockCpfpData { + txs: Record, + clusters: CpfpCluster[]; + version: number; +} + +export function calculateFastBlockCpfp(height: number, transactions: MempoolTransactionExtended[], saveRelatives: boolean = false): BlockCpfpData { const clusters: CpfpCluster[] = []; // list of all cpfp clusters in this block const clusterMap: { [txid: string]: CpfpCluster } = {}; // map transactions to their cpfp cluster let clusterTxs: TransactionExtended[] = []; // working list of elements of the current cluster let ancestors: { [txid: string]: boolean } = {}; // working set of ancestors of the current cluster root const txMap: { [txid: string]: TransactionExtended } = {}; + const cpfpData: Record = {}; // initialize the txMap for (const tx of transactions) { txMap[tx.txid] = tx; + cpfpData[tx.txid] = {}; } // reverse pass to identify CPFP clusters for (let i = transactions.length - 1; i >= 0; i--) { @@ -39,7 +48,9 @@ export function calculateFastBlockCpfp(height: number, transactions: MempoolTran clusters.push(cluster); } clusterTxs.forEach(tx => { - txMap[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize; + cpfpData[tx.txid] = { + effectiveFeePerVsize + }; if (cluster) { clusterMap[tx.txid] = cluster; } @@ -55,17 +66,19 @@ export function calculateFastBlockCpfp(height: number, transactions: MempoolTran } // forward pass to enforce ancestor rate caps for (const tx of transactions) { - let minAncestorRate = tx.effectiveFeePerVsize; + const txRate = cpfpData[tx.txid]?.effectiveFeePerVsize ?? tx.effectiveFeePerVsize; + let minAncestorRate = txRate; for (const vin of tx.vin) { - if (txMap[vin.txid]?.effectiveFeePerVsize) { - minAncestorRate = Math.min(minAncestorRate, txMap[vin.txid].effectiveFeePerVsize); + const vinRate = cpfpData[vin.txid]?.effectiveFeePerVsize ?? txMap[vin.txid]?.effectiveFeePerVsize; + if (vinRate) { + minAncestorRate = Math.min(minAncestorRate, vinRate); } } // check rounded values to skip cases with almost identical fees const roundedMinAncestorRate = Math.ceil(minAncestorRate); - const roundedEffectiveFeeRate = Math.floor(tx.effectiveFeePerVsize); + const roundedEffectiveFeeRate = Math.floor(txRate); if (roundedMinAncestorRate < roundedEffectiveFeeRate) { - tx.effectiveFeePerVsize = minAncestorRate; + cpfpData[tx.txid].effectiveFeePerVsize = minAncestorRate; if (!clusterMap[tx.txid]) { // add a single-tx cluster to record the dependent rate const cluster = { @@ -85,23 +98,25 @@ export function calculateFastBlockCpfp(height: number, transactions: MempoolTran if (saveRelatives) { for (const cluster of clusters) { cluster.txs.forEach((member, index) => { - txMap[member.txid].descendants = cluster.txs.slice(0, index).reverse(); - txMap[member.txid].ancestors = cluster.txs.slice(index + 1).reverse(); - txMap[member.txid].effectiveFeePerVsize = cluster.effectiveFeePerVsize; + cpfpData[member.txid].descendants = cluster.txs.slice(0, index).reverse(); + cpfpData[member.txid].ancestors = cluster.txs.slice(index + 1).reverse(); + cpfpData[member.txid].effectiveFeePerVsize = cluster.effectiveFeePerVsize; }); } } return { - transactions, + txs: cpfpData, clusters, version: 1, }; } -export function calculateGoodBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): CpfpSummary { +export function calculateGoodBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): BlockCpfpData { const txMap: { [txid: string]: MempoolTransactionExtended } = {}; + const cpfpData: Record = {}; for (const tx of transactions) { txMap[tx.txid] = tx; + cpfpData[tx.txid] = {}; } const template = makeBlockTemplate(transactions, accelerations, 1, Infinity, Infinity); const clusters = new Map(); @@ -111,15 +126,15 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran if (cluster.length > 1 && root && !clusters.has(root)) { clusters.set(root, cluster); } - txMap[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize; + cpfpData[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize; } const clusterArray: CpfpCluster[] = []; for (const cluster of clusters.values()) { for (const txid of cluster) { - const mempoolTx = txMap[txid]; - if (mempoolTx) { + const mempoolTxCpfpData = cpfpData[txid]; + if (mempoolTxCpfpData) { const ancestors: Ancestor[] = []; const descendants: Ancestor[] = []; let matched = false; @@ -139,10 +154,10 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran } } }); - if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) { - mempoolTx.cpfpDirty = true; + if (mempoolTxCpfpData.ancestors?.length !== ancestors.length || mempoolTxCpfpData.descendants?.length !== descendants.length) { + mempoolTxCpfpData.cpfpDirty = true; } - Object.assign(mempoolTx, { ancestors, descendants, bestDescendant: null, cpfpChecked: true }); + Object.assign(mempoolTxCpfpData, { ancestors, descendants, bestDescendant: null, cpfpChecked: true }); } } const root = cluster[cluster.length - 1]; @@ -154,21 +169,23 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran fee: txMap[txid].fee, weight: (txMap[txid].adjustedVsize * 4) || txMap[txid].weight, })), - effectiveFeePerVsize: txMap[root].effectiveFeePerVsize, + effectiveFeePerVsize: cpfpData[root].effectiveFeePerVsize ?? txMap[root].effectiveFeePerVsize, }); } return { - transactions: transactions.map(tx => txMap[tx.txid]), + txs: cpfpData, clusters: clusterArray, version: 2, }; } -export function calculateClusterMempoolBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): CpfpSummary { +export function calculateClusterMempoolBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): BlockCpfpData { const txMap: { [txid: string]: MempoolTransactionExtended } = {}; + const cpfpData: Record = {}; for (const tx of transactions) { txMap[tx.txid] = tx; + cpfpData[tx.txid] = {}; } const accelMap: { [txid: string]: { feeDelta: number } } = {}; @@ -176,16 +193,25 @@ export function calculateClusterMempoolBlockCpfp(height: number, transactions: M accelMap[acc.txid] = { feeDelta: acc.max_bid }; } - const cm = new ClusterMempool(txMap, accelMap); + const cm = new ClusterMempool(txMap, accelMap, false, 25000); const seenClusters = new Set(); const clusters: CpfpCluster[] = []; - for (const tx of transactions) { - if (tx.clusterId !== undefined && !seenClusters.has(tx.clusterId)) { - seenClusters.add(tx.clusterId); + for (const txid in cpfpData) { + const txCpfpData = cm.getCpfpDataForTx(txid); + if (!txCpfpData) { + continue; + } + cpfpData[txid].effectiveFeePerVsize = txCpfpData.effectiveFeePerVsize; + cpfpData[txid].clusterId = txCpfpData.clusterId; + cpfpData[txid].chunkIndex = txCpfpData.chunkIndex; + cpfpData[txid].ancestors = txCpfpData.ancestors; + cpfpData[txid].descendants = txCpfpData.descendants; + if (txCpfpData.clusterId !== undefined && !seenClusters.has(txCpfpData.clusterId)) { + seenClusters.add(txCpfpData.clusterId); - const clusterData = cm.getCluster(tx.clusterId); + const clusterData = cm.getCluster(txCpfpData.clusterId); if (clusterData && clusterData.txs.length > 1) { let totalFee = 0; let totalWeight = 0; @@ -206,7 +232,7 @@ export function calculateClusterMempoolBlockCpfp(height: number, transactions: M } return { - transactions: transactions.map(tx => txMap[tx.txid]), + txs: cpfpData, clusters, version: 3, }; diff --git a/backend/src/cluster-mempool/cluster-mempool.ts b/backend/src/cluster-mempool/cluster-mempool.ts index f01abce45..98d94553c 100644 --- a/backend/src/cluster-mempool/cluster-mempool.ts +++ b/backend/src/cluster-mempool/cluster-mempool.ts @@ -1,7 +1,7 @@ import { ClusterTx, DepGraph, sortTopological, subgraph } from './depgraph'; import { linearizeCluster, LinearizationChunk } from './linearize'; import { ProjectedBlock, assembleBlocks } from './block-builder'; -import { CpfpClusterData, CpfpClusterTx, MempoolTransactionExtended } from '../mempool.interfaces'; +import { Ancestor, CpfpClusterData, CpfpClusterTx, MempoolTransactionExtended } from '../mempool.interfaces'; import logger from '../logger'; export interface MempoolDiff { @@ -27,19 +27,36 @@ export interface Cluster { dirty: boolean; } +export interface TxCpfpData { + effectiveFeePerVsize: number; + clusterId: number; + chunkIndex: number; + cpfpDirty: boolean; + cpfpChecked: boolean; + ancestors: Ancestor[]; + descendants: Ancestor[]; +} + +const DEFAULT_COST_BUDGET = 75000; + export class ClusterMempool { private clusters = new Map(); private txToCluster = new Map(); + private parentMap = new Map>(); private spentBy = new Map(); private mempool: Readonly<{ [txid: string]: MempoolTransactionExtended }>; private accelerations: { [txid: string]: { feeDelta: number } } = {}; private nextClusterId = 0; + private modifyTxs: boolean; + private costBudget: number = DEFAULT_COST_BUDGET; - constructor(mempool: { [txid: string]: MempoolTransactionExtended }, accelerations?: { [txid: string]: { feeDelta: number } }) { + constructor(mempool: { [txid: string]: MempoolTransactionExtended }, accelerations?: { [txid: string]: { feeDelta: number } }, modifyTxs: boolean = true, costBudget: number = DEFAULT_COST_BUDGET) { this.mempool = mempool; if (accelerations) { this.accelerations = accelerations; } + this.modifyTxs = modifyTxs; + this.costBudget = costBudget; this.buildFromMempool(); } @@ -83,6 +100,28 @@ export class ClusterMempool { return { ...cluster, chunkIndex: info.chunkIndex }; } + getCpfpDataForTx(txid: string): TxCpfpData | null { + const clusterInfo = this.getClusterForTx(txid); + if (!clusterInfo) { + return null; + } + const chunkInfo = this.findChunkInfo(clusterInfo.cluster, clusterInfo.clusterTx); + if (!chunkInfo) { + return null; + } + const chunk = clusterInfo.cluster.chunks[chunkInfo?.chunkIndex]; + const chunkSet = chunk.txs.length > 1 ? new Set(chunk.txs) : null; + return { + effectiveFeePerVsize: chunkInfo.chunkFeerate, + clusterId: clusterInfo.cluster.id, + chunkIndex: chunkInfo.chunkIndex, + cpfpDirty: true, + cpfpChecked: true, + ancestors: chunkSet ? this.getChunkRelatives(clusterInfo.clusterTx, chunkSet, 'ancestors') : [], + descendants: chunkSet ? this.getChunkRelatives(clusterInfo.clusterTx, chunkSet, 'descendants') : [], + }; + } + getClusterCount(): number { return this.clusters.size; } @@ -108,51 +147,38 @@ export class ClusterMempool { } private buildFromMempool(): void { - const parentMap = this.buildMempoolParentMap(); - this.spentBy = this.buildSpentByMap(); - const components = this.findMempoolComponents(parentMap); + this.buildRelativeMaps(); + const components = this.findMempoolComponents(); for (const component of components) { - this.createClusterFromTxids(component, parentMap); + this.createClusterFromTxids(component); } } - private buildMempoolParentMap(): Map> { - const parents = new Map>(); - for (const [txid, tx] of Object.entries(this.mempool)) { + private buildRelativeMaps(): void { + this.parentMap.clear(); + this.spentBy.clear(); + for (const txid in this.mempool) { + const tx = this.mempool[txid]; const txParents = new Set(); for (const vin of tx.vin) { if (!vin.is_coinbase && this.mempool[vin.txid]) { txParents.add(vin.txid); + this.spentBy.set(`${vin.txid}:${vin.vout}`, txid); } } if (txParents.size > 0) { - parents.set(txid, txParents); + this.parentMap.set(txid, txParents); } } - return parents; } - private buildSpentByMap(): Map { - const spentBy = new Map(); - for (const [txid, tx] of Object.entries(this.mempool)) { - for (const vin of tx.vin) { - if (!vin.is_coinbase) { - spentBy.set(`${vin.txid}:${vin.vout}`, txid); - } - } - } - return spentBy; - } - - private findMempoolComponents( - parentMap: Map> - ): Set[] { + private findMempoolComponents(): Set[] { const visited = new Set(); const components: Set[] = []; - for (const txid of Object.keys(this.mempool)) { + for (const txid in this.mempool) { if (!visited.has(txid)) { - const component = this.dfsComponent(txid, parentMap, visited); + const component = this.dfsComponent(txid, visited); components.push(component); } } @@ -161,7 +187,6 @@ export class ClusterMempool { private dfsComponent( startTxid: string, - parentMap: Map>, visited: Set ): Set { const component = new Set(); @@ -172,7 +197,7 @@ export class ClusterMempool { visited.add(current); component.add(current); - const txParents = parentMap.get(current); + const txParents = this.parentMap.get(current); if (txParents) { for (const p of txParents) { if (!visited.has(p)) { @@ -204,8 +229,7 @@ export class ClusterMempool { } private createClusterFromTxids( - txids: Set, - parentMap: Map> + txids: Set ): Cluster | null { const clusterId = this.nextClusterId++; const depgraph = new DepGraph(); @@ -222,7 +246,7 @@ export class ClusterMempool { } for (const txid of txids) { - const txParents = parentMap.get(txid); + const txParents = this.parentMap.get(txid); if (txParents) { for (const parentTxid of txParents) { if (txids.has(parentTxid)) { @@ -236,7 +260,7 @@ export class ClusterMempool { } } - const { linearization, chunks } = linearizeCluster(depgraph.getTxs()); + const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), this.costBudget); const cluster: Cluster = { id: clusterId, @@ -252,10 +276,18 @@ export class ClusterMempool { this.txToCluster.set(txid, clusterId); } - this.writeBackCluster(cluster); + if (this.modifyTxs) { + this.writeBackCluster(cluster); + } return cluster; } + public writeBackClusters(): void { + for (const cluster of this.clusters.values()) { + this.writeBackCluster(cluster); + } + } + private writeBackCluster(cluster: Cluster): void { for (let chunkIdx = 0; chunkIdx < cluster.chunks.length; chunkIdx++) { const chunk = cluster.chunks[chunkIdx]; @@ -344,7 +376,7 @@ export class ClusterMempool { } private splitDisconnectedClusters(): void { - for (const [clusterId, cluster] of [...this.clusters]) { + for (const [clusterId, cluster] of this.clusters.entries()) { if (cluster.dirty) { if (cluster.depgraph.size === 0) { this.clusters.delete(clusterId); @@ -475,7 +507,7 @@ export class ClusterMempool { childTxids: string[], ): void { const txid = tx.txid; - const clusterId = [...relatedClusterIds][0]; + const clusterId = relatedClusterIds.values().next().value; const cluster = this.clusters.get(clusterId); if (!cluster) { return; @@ -496,27 +528,25 @@ export class ClusterMempool { parentTxids: string[], childTxids: string[], ): void { - const txid = tx.txid; - const clusterIds = [...relatedClusterIds]; - const primaryId = clusterIds[0]; + const clusterIterator = relatedClusterIds.values(); + const primaryId: number = clusterIterator.next().value; const primary = this.clusters.get(primaryId); if (!primary) { return; } - for (let i = 1; i < clusterIds.length; i++) { - const otherId = clusterIds[i]; - const other = this.clusters.get(otherId); + for (const clusterId of clusterIterator) { + const other = this.clusters.get(clusterId); if (other) { this.mergeClusterInto(primary, other); - this.clusters.delete(otherId); + this.clusters.delete(clusterId); } } - const clusterTx = primary.depgraph.addTransaction(txid, this.effectiveFee(txid, tx), this.adjustedWeight(tx), tx.order ?? 0); - primary.txs.set(txid, clusterTx); + const clusterTx = primary.depgraph.addTransaction(tx.txid, this.effectiveFee(tx.txid, tx), this.adjustedWeight(tx), tx.order ?? 0); + primary.txs.set(tx.txid, clusterTx); primary.linearization.push(clusterTx); - this.txToCluster.set(txid, primaryId); + this.txToCluster.set(tx.txid, primaryId); this.addParentDeps(primary, clusterTx, parentTxids); this.addChildDeps(primary, clusterTx, childTxids); @@ -568,12 +598,12 @@ export class ClusterMempool { private processAccelerationChanges(newAccelerations: { [txid: string]: { feeDelta: number } }): void { const changed = new Set(); - for (const txid of Object.keys(newAccelerations)) { + for (const txid in newAccelerations) { if ((newAccelerations[txid]?.feeDelta || 0) !== (this.accelerations[txid]?.feeDelta || 0)) { changed.add(txid); } } - for (const txid of Object.keys(this.accelerations)) { + for (const txid in this.accelerations) { if (!newAccelerations[txid]) { changed.add(txid); } @@ -593,7 +623,7 @@ export class ClusterMempool { } private relinearizeDirtyClusters(): void { - for (const [clusterId, cluster] of [...this.clusters]) { + for (const [clusterId, cluster] of this.clusters.entries()) { if (cluster.dirty) { cluster.dirty = false; const newId = this.nextClusterId++; @@ -606,12 +636,15 @@ export class ClusterMempool { const { linearization, chunks } = linearizeCluster( cluster.depgraph.getTxs(), + this.costBudget, cluster.linearization, ); cluster.linearization = linearization; cluster.chunks = chunks; - this.writeBackCluster(cluster); + if (this.modifyTxs) { + this.writeBackCluster(cluster); + } } } } diff --git a/backend/src/cluster-mempool/linearize.ts b/backend/src/cluster-mempool/linearize.ts index cc6ca8394..8b1c0465d 100644 --- a/backend/src/cluster-mempool/linearize.ts +++ b/backend/src/cluster-mempool/linearize.ts @@ -161,12 +161,10 @@ const enum MergeDir { Up, Down, Both } interface SFLCost { cost: number; } -const DEFAULT_COST_BUDGET = 75_000; - export function spanningForestLinearize( txs: Set, + costBudget: number, existingLinearization?: ClusterTx[], - costBudget: number = DEFAULT_COST_BUDGET, ): ClusterTx[] { const allTxs = [...txs]; if (allTxs.length === 0) { @@ -1199,10 +1197,10 @@ function bfsComponentWithinChunk( export function linearizeCluster( txs: Set, + costBudget: number, existingLinearization?: ClusterTx[], - costBudget?: number, ): { linearization: ClusterTx[]; chunks: LinearizationChunk[] } { - let linearization = spanningForestLinearize(txs, existingLinearization, costBudget); + let linearization = spanningForestLinearize(txs, costBudget, existingLinearization); linearization = postLinearize(linearization); let chunks = chunkify(linearization); chunks = minimizeChunks(chunks); diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 327bbbebf..46256cc29 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -139,12 +139,12 @@ class Indexer { switch (task) { case 'blocksPrices': { if (!['testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) && config.FIAT_PRICE.ENABLED) { - let lastestPriceId; + let latestPriceId; try { - lastestPriceId = await PricesRepository.$getLatestPriceId(); + latestPriceId = await PricesRepository.$getLatestPriceId(); } catch (e) { logger.debug('failed to fetch latest price id from db: ' + (e instanceof Error ? e.message : e)); - } if (priceUpdater.historyInserted === false || lastestPriceId === null) { + } if (priceUpdater.historyInserted === false || latestPriceId === null) { logger.debug(`Blocks prices indexer is waiting for the price updater to complete`, logger.tags.mining); this.scheduleSingleTask(task, 10000); } else { diff --git a/backend/tsconfig.build.json b/backend/tsconfig.build.json index e3d61b71c..225f0944a 100644 --- a/backend/tsconfig.build.json +++ b/backend/tsconfig.build.json @@ -1,6 +1,6 @@ { "extends": "./tsconfig", - "exclude": ["**/*.test.*", "**/__mocks__/*", "**/__tests__/*"], + "exclude": ["**/*.test.*", "**/__mocks__/*", "**/__tests__/*", "**/__e2e__/*"], "compilerOptions": { "types": ["node"] },