diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index c2715153b..f0d447582 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -31,6 +31,7 @@ "AUDIT": false, "RUST_GBT": true, "LIMIT_GBT": false, + "CLUSTER_MEMPOOL": false, "CPFP_INDEXING": false, "DISK_CACHE_BLOCK_INTERVAL": 6, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, diff --git a/backend/mempool-config.test.json b/backend/mempool-config.test.json index 2522ca90f..b0ca4cd04 100644 --- a/backend/mempool-config.test.json +++ b/backend/mempool-config.test.json @@ -31,6 +31,7 @@ "AUDIT": false, "RUST_GBT": true, "LIMIT_GBT": false, + "CLUSTER_MEMPOOL": false, "CPFP_INDEXING": false, "DISK_CACHE_BLOCK_INTERVAL": 6, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 0ca5654a5..56feefa67 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -32,6 +32,7 @@ "AUDIT": true, "RUST_GBT": false, "LIMIT_GBT": false, + "CLUSTER_MEMPOOL": false, "CPFP_INDEXING": true, "MAX_BLOCKS_BULK_QUERY": 999, "DISK_CACHE_BLOCK_INTERVAL": 999, diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index cf81a5f7f..ec44eeeff 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -45,6 +45,7 @@ describe('Mempool Backend Config', () => { AUDIT: false, RUST_GBT: true, LIMIT_GBT: false, + CLUSTER_MEMPOOL: false, CPFP_INDEXING: false, MAX_BLOCKS_BULK_QUERY: 0, DISK_CACHE_BLOCK_INTERVAL: 6, diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index 237a7caec..9d9b5f262 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -12,7 +12,7 @@ import backendInfo from '../backend-info'; import transactionUtils from '../transaction-utils'; import { IEsploraApi } from './esplora-api.interface'; import loadingIndicators from '../loading-indicators'; -import { TransactionExtended } from '../../mempool.interfaces'; +import { CpfpInfo, TransactionExtended } from '../../mempool.interfaces'; import logger from '../../logger'; import blocks from '../blocks'; import bitcoinClient from './bitcoin-client'; @@ -191,11 +191,11 @@ class BitcoinRoutes { const tx = mempool.getMempool()[req.params.txId]; if (tx) { if (tx?.cpfpChecked) { - res.json({ - ancestors: tx.ancestors, + const response: CpfpInfo & { acceleratedBy?: number[], acceleratedAt?: number, feeDelta?: number } = { + ancestors: tx.ancestors || [], bestDescendant: tx.bestDescendant || null, - descendants: tx.descendants || null, - effectiveFeePerVsize: tx.effectiveFeePerVsize || null, + descendants: tx.descendants, + effectiveFeePerVsize: tx.effectiveFeePerVsize, sigops: tx.sigops, fee: tx.fee, adjustedVsize: tx.adjustedVsize, @@ -203,7 +203,14 @@ class BitcoinRoutes { acceleratedBy: tx.acceleratedBy || undefined, acceleratedAt: tx.acceleratedAt || undefined, feeDelta: tx.feeDelta || undefined, - }); + }; + if (config.MEMPOOL.CLUSTER_MEMPOOL && tx.clusterId != null) { + const cluster = mempool.clusterMempool?.getClusterForApi(req.params.txId); + if (cluster) { + response.cluster = cluster; + } + } + res.json(response); return; } diff --git a/backend/src/api/cpfp.ts b/backend/src/api/cpfp.ts index ad601361c..53385f925 100644 --- a/backend/src/api/cpfp.ts +++ b/backend/src/api/cpfp.ts @@ -1,7 +1,8 @@ -import { Ancestor, CpfpCluster, CpfpInfo, CpfpSummary, MempoolTransactionExtended, TransactionExtended } from '../mempool.interfaces'; +import { Ancestor, CpfpCluster, CpfpInfo, CpfpSummary, MempoolTransactionExtended, TemplateAlgorithm, TransactionExtended } from '../mempool.interfaces'; import { GraphTx, convertToGraphTx, expandRelativesGraph, initializeRelatives, makeBlockTemplate, mempoolComparator, removeAncestors, setAncestorScores } from './mini-miner'; import memPool from './mempool'; import { Acceleration } from './acceleration/acceleration'; +import { ClusterMempool } from '../cluster-mempool/cluster-mempool'; const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction const MAX_CLUSTER_ITERATIONS = 100; @@ -164,6 +165,53 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran }; } +export function calculateClusterMempoolBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): CpfpSummary { + const txMap: { [txid: string]: MempoolTransactionExtended } = {}; + for (const tx of transactions) { + txMap[tx.txid] = tx; + } + + const accelMap: { [txid: string]: { feeDelta: number } } = {}; + for (const acc of accelerations) { + accelMap[acc.txid] = { feeDelta: acc.max_bid }; + } + + const cm = new ClusterMempool(txMap, accelMap); + + const seenClusters = new Set(); + const clusters: CpfpCluster[] = []; + + for (const tx of transactions) { + if (tx.clusterId !== undefined && !seenClusters.has(tx.clusterId)) { + seenClusters.add(tx.clusterId); + + const clusterData = cm.getCluster(tx.clusterId); + if (clusterData && clusterData.txs.length > 1) { + let totalFee = 0; + let totalWeight = 0; + for (const t of clusterData.txs) { + totalFee += t.fee; + totalWeight += t.weight; + } + clusters.push({ + root: clusterData.txs[0].txid, + height, + txs: clusterData.txs.map(t => ({ txid: t.txid, weight: t.weight, fee: t.fee })), + effectiveFeePerVsize: totalFee / (totalWeight / 4), + templateAlgorithm: TemplateAlgorithm.clusterMempool, + clusterData, + }); + } + } + } + + return { + transactions: transactions.map(tx => txMap[tx.txid]), + clusters, + version: 3, + }; +} + /** * Takes a mempool transaction and a copy of the current mempool, and calculates the CPFP data for * that transaction (and all others in the same cluster) diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 43bf05eec..17ccf84e6 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -8,6 +8,7 @@ import path from 'path'; import mempool from './mempool'; import { Acceleration } from './services/acceleration'; import PoolsRepository from '../repositories/PoolsRepository'; +import { ProjectedBlock } from '../cluster-mempool/cluster-mempool'; const MAX_UINT32 = Math.pow(2, 32) - 1; @@ -238,7 +239,7 @@ class MempoolBlocks { } /** @asyncSafe */ - public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { + public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number, dryRun = false): Promise { const start = Date.now(); // reset mempool short ids @@ -278,7 +279,7 @@ class MempoolBlocks { const expectedSize = transactions.length; const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length; logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${expectedSize} in the mempool, ${overflow.length} were unmineable`); - const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults, dryRun); logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; } catch (e) { @@ -296,16 +297,15 @@ class MempoolBlocks { } /** @asyncSafe */ - public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise { + public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number, dryRun = false): Promise { // GBT optimization requires that uids never get too sparse // as a sanity check, we should also explicitly prevent uint32 uid overflow if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * transactions.length), MAX_UINT32)) { this.resetRustGbt(); } - if (!this.rustInitialized) { - // need to reset the worker - return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, true, useAccelerations, accelerationPool); + if (!this.rustInitialized || dryRun) { + return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, !dryRun, useAccelerations, accelerationPool, dryRun); } const start = Date.now(); @@ -344,7 +344,7 @@ class MempoolBlocks { if (transactions.length !== resultMempoolSize) { throw new Error(`GBT returned wrong number of transactions ${transactions.length} vs ${resultMempoolSize}, cache is probably out of sync`); } else { - const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, true); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, !dryRun, dryRun); this.removeUids(removedTxs); logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; @@ -356,20 +356,22 @@ class MempoolBlocks { } } - private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations: { [txid: string]: Acceleration }, accelerationPool, saveResults): MempoolBlockWithTransactions[] { - for (const txid of Object.keys(candidates?.txs ?? mempool)) { - if (txid in mempool) { - mempool[txid].cpfpDirty = false; - mempool[txid].ancestors = []; - mempool[txid].descendants = []; - mempool[txid].bestDescendant = null; + private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations: { [txid: string]: Acceleration }, accelerationPool, saveResults, dryRun = false): MempoolBlockWithTransactions[] { + if (!dryRun) { + for (const txid of Object.keys(candidates?.txs ?? mempool)) { + if (txid in mempool) { + mempool[txid].cpfpDirty = false; + mempool[txid].ancestors = []; + mempool[txid].descendants = []; + mempool[txid].bestDescendant = null; + } } - } - for (const [txid, rate] of rates) { - if (txid in mempool) { - mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize); - mempool[txid].effectiveFeePerVsize = rate; - mempool[txid].cpfpChecked = true; + for (const [txid, rate] of rates) { + if (txid in mempool) { + mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize); + mempool[txid].effectiveFeePerVsize = rate; + mempool[txid].cpfpChecked = true; + } } } @@ -387,58 +389,60 @@ class MempoolBlocks { feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]); } - const ancestors: Ancestor[] = []; - const descendants: Ancestor[] = []; - let ancestor: MempoolTransactionExtended; - for (const cluster of clusters) { - for (const memberTxid of cluster) { - const mempoolTx = mempool[memberTxid]; - if (mempoolTx) { - // ugly micro-optimization to avoid allocating new arrays - ancestors.length = 0; - descendants.length = 0; - let matched = false; - cluster.forEach(txid => { - ancestor = mempool[txid]; - if (txid === memberTxid) { - matched = true; - } else { - if (!ancestor) { - console.log('txid missing from mempool! ', txid, candidates?.txs[txid]); - return; - } - const relative = { - txid: txid, - fee: ancestor.fee, - weight: (ancestor.adjustedVsize * 4), - }; - if (matched) { - descendants.push(relative); - if (!mempoolTx.lastBoosted || (ancestor.firstSeen && ancestor.firstSeen > mempoolTx.lastBoosted)) { - mempoolTx.lastBoosted = ancestor.firstSeen; - } + if (!dryRun) { + const ancestors: Ancestor[] = []; + const descendants: Ancestor[] = []; + let ancestor: MempoolTransactionExtended; + for (const cluster of clusters) { + for (const memberTxid of cluster) { + const mempoolTx = mempool[memberTxid]; + if (mempoolTx) { + // ugly micro-optimization to avoid allocating new arrays + ancestors.length = 0; + descendants.length = 0; + let matched = false; + cluster.forEach(txid => { + ancestor = mempool[txid]; + if (txid === memberTxid) { + matched = true; } else { - ancestors.push(relative); + if (!ancestor) { + console.log('txid missing from mempool! ', txid, candidates?.txs[txid]); + return; + } + const relative = { + txid: txid, + fee: ancestor.fee, + weight: (ancestor.adjustedVsize * 4), + }; + if (matched) { + descendants.push(relative); + if (!mempoolTx.lastBoosted || (ancestor.firstSeen && ancestor.firstSeen > mempoolTx.lastBoosted)) { + mempoolTx.lastBoosted = ancestor.firstSeen; + } + } else { + ancestors.push(relative); + } } + }); + if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) { + mempoolTx.cpfpDirty = true; } - }); - if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) { - mempoolTx.cpfpDirty = true; + // ugly micro-optimization to avoid allocating new arrays or objects + if (mempoolTx.ancestors) { + mempoolTx.ancestors.length = 0; + } else { + mempoolTx.ancestors = []; + } + if (mempoolTx.descendants) { + mempoolTx.descendants.length = 0; + } else { + mempoolTx.descendants = []; + } + mempoolTx.ancestors.push(...ancestors); + mempoolTx.descendants.push(...descendants); + mempoolTx.cpfpChecked = true; } - // ugly micro-optimization to avoid allocating new arrays or objects - if (mempoolTx.ancestors) { - mempoolTx.ancestors.length = 0; - } else { - mempoolTx.ancestors = []; - } - if (mempoolTx.descendants) { - mempoolTx.descendants.length = 0; - } else { - mempoolTx.descendants = []; - } - mempoolTx.ancestors.push(...ancestors); - mempoolTx.descendants.push(...descendants); - mempoolTx.cpfpChecked = true; } } } @@ -471,34 +475,42 @@ class MempoolBlocks { const txid = block[i]; if (txid in mempool) { mempoolTx = mempool[txid]; - // save position in projected blocks - mempoolTx.position = { - block: blockIndex, - vsize: totalVsize + (mempoolTx.vsize / 2), - }; - if (txid in accelerations) { - acceleration = accelerations[txid]; - if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) { - if (!mempoolTx.acceleration) { - mempoolTx.cpfpDirty = true; - } - mempoolTx.acceleration = true; - mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools; - mempoolTx.acceleratedAt = acceleration?.added; - mempoolTx.feeDelta = acceleration?.feeDelta; - for (const ancestor of mempoolTx.ancestors || []) { - if (!(ancestor.txid in mempool)) { - continue; + if (!dryRun) { + // save position in projected blocks + mempoolTx.position = { + block: blockIndex, + vsize: totalVsize + (mempoolTx.vsize / 2), + }; + + if (txid in accelerations) { + acceleration = accelerations[txid]; + if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) { + if (!mempoolTx.acceleration) { + mempoolTx.cpfpDirty = true; } - if (!mempool[ancestor.txid].acceleration) { - mempool[ancestor.txid].cpfpDirty = true; + mempoolTx.acceleration = true; + mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools; + mempoolTx.acceleratedAt = acceleration?.added; + mempoolTx.feeDelta = acceleration?.feeDelta; + for (const ancestor of mempoolTx.ancestors || []) { + if (!(ancestor.txid in mempool)) { + continue; + } + if (!mempool[ancestor.txid].acceleration) { + mempool[ancestor.txid].cpfpDirty = true; + } + mempool[ancestor.txid].acceleration = true; + mempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy; + mempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt; + mempool[ancestor.txid].feeDelta = mempoolTx.feeDelta; + isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy; + } + } else { + if (mempoolTx.acceleration) { + mempoolTx.cpfpDirty = true; + delete mempoolTx.acceleration; } - mempool[ancestor.txid].acceleration = true; - mempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy; - mempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt; - mempool[ancestor.txid].feeDelta = mempoolTx.feeDelta; - isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy; } } else { if (mempoolTx.acceleration) { @@ -506,11 +518,6 @@ class MempoolBlocks { delete mempoolTx.acceleration; } } - } else { - if (mempoolTx.acceleration) { - mempoolTx.cpfpDirty = true; - delete mempoolTx.acceleration; - } } // online calculation of stack-of-blocks fee stats @@ -548,6 +555,115 @@ class MempoolBlocks { return mempoolBlocks; } + public processClusterMempoolBlocks(projectedBlocks: ProjectedBlock[], newMempool: { [txid: string]: MempoolTransactionExtended }, accelerations: { [txid: string]: Acceleration }, saveResults = true, accelerationPool?: number): MempoolBlockWithTransactions[] { + const lastBlockIndex = projectedBlocks.length - 1; + let hasBlockStack = projectedBlocks.length >= 8; + let stackWeight = 0; + let feeStatsCalculator: OnlineFeeStatsCalculator | null = null; + if (hasBlockStack) { + stackWeight = projectedBlocks[lastBlockIndex].weight; + hasBlockStack = stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS; + feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]); + } + + const isAcceleratedBy: { [txid: string]: number[] | false } = {}; + + const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2; + let mempoolTx: MempoolTransactionExtended; + let acceleration: Acceleration; + const mempoolBlocks: MempoolBlockWithTransactions[] = []; + + for (let blockIndex = 0; blockIndex < projectedBlocks.length; blockIndex++) { + const projected = projectedBlocks[blockIndex]; + let totalSize = 0; + let totalVsize = 0; + let totalWeight = 0; + let totalFees = 0; + const transactions: MempoolTransactionExtended[] = []; + const validTxids: string[] = []; + + for (const txid of projected.txids) { + if (txid in newMempool) { + mempoolTx = newMempool[txid]; + validTxids.push(txid); + + // save position in projected blocks + mempoolTx.position = { + block: blockIndex, + vsize: totalVsize + (mempoolTx.vsize / 2), + }; + + if (txid in accelerations) { + acceleration = accelerations[txid]; + if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) { + if (!mempoolTx.acceleration) { + mempoolTx.cpfpDirty = true; + } + mempoolTx.acceleration = true; + mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools; + mempoolTx.acceleratedAt = acceleration?.added; + mempoolTx.feeDelta = acceleration?.feeDelta; + for (const ancestor of mempoolTx.ancestors || []) { + if (!(ancestor.txid in newMempool)) { + continue; + } + if (!newMempool[ancestor.txid].acceleration) { + newMempool[ancestor.txid].cpfpDirty = true; + } + newMempool[ancestor.txid].acceleration = true; + newMempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy; + newMempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt; + newMempool[ancestor.txid].feeDelta = mempoolTx.feeDelta; + isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy; + } + } else { + if (mempoolTx.acceleration) { + mempoolTx.cpfpDirty = true; + delete mempoolTx.acceleration; + } + } + } else { + if (mempoolTx.acceleration) { + mempoolTx.cpfpDirty = true; + delete mempoolTx.acceleration; + } + } + + if (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) { + feeStatsCalculator.processNext(mempoolTx); + } + + totalSize += mempoolTx.size; + totalVsize += mempoolTx.vsize; + totalWeight += mempoolTx.weight; + totalFees += mempoolTx.fee; + + if (totalVsize <= sizeLimit) { + transactions.push(mempoolTx); + } + } + } + + mempoolBlocks[blockIndex] = this.dataToMempoolBlocks( + validTxids, + transactions, + totalSize, + totalWeight, + totalFees, + (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined, + ); + } + + if (saveResults) { + const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks); + this.mempoolBlocks = mempoolBlocks; + this.mempoolBlockDeltas = deltas; + this.updateAccelerationPositions(newMempool, accelerations, mempoolBlocks); + } + + return mempoolBlocks; + } + private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions { if (!feeStats) { feeStats = Common.calcEffectiveFeeStatistics(transactions); diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index d83347fd6..1bd919c6c 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -13,6 +13,7 @@ import { Acceleration } from './services/acceleration'; import accelerationApi from './services/acceleration'; import redisCache from './redis-cache'; import blocks from './blocks'; +import { ClusterMempool } from '../cluster-mempool/cluster-mempool'; class Mempool { private inSync: boolean = false; @@ -22,6 +23,7 @@ class Mempool { private spendMap = new Map(); private recentlyDeleted: MempoolTransactionExtended[][] = []; // buffer of transactions deleted in recent mempool updates private mempoolInfo: IBitcoinApi.MempoolInfo; + public clusterMempool: ClusterMempool | null = null; private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[]) => void) | undefined; private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[], @@ -62,6 +64,9 @@ class Mempool { minrelaytxfee: isLiquid ? 0.00000100 : 0.00001000 }; this.txPerSecondInterval = setInterval(this.updateTxPerSecond.bind(this), 1000); + if (config.MEMPOOL.CLUSTER_MEMPOOL) { + this.clusterMempool = new ClusterMempool(this.mempoolCache, this.accelerations); + } } /** @@ -153,6 +158,9 @@ class Mempool { await redisCache.$flushTransactions(); logger.debug(`Finished migrating cache transactions in ${((Date.now() - redisTimer) / 1000).toFixed(2)} seconds`); } + if (config.MEMPOOL.CLUSTER_MEMPOOL) { + this.clusterMempool = new ClusterMempool(this.mempoolCache, this.accelerations); + } if (this.mempoolChangedCallback) { this.mempoolChangedCallback(this.mempoolCache, [], [], []); } @@ -387,6 +395,14 @@ class Mempool { hasChange = true; } + if (config.MEMPOOL.CLUSTER_MEMPOOL && (newTransactions.length || deletedTransactions.length || accelerationDelta.length)) { + this.clusterMempool?.applyMempoolChange({ + added: newTransactions, + removed: deletedTransactions.map(tx => tx.txid), + accelerations: this.getAccelerations(), + }); + } + this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize); const candidatesChanged = candidates?.added?.length || candidates?.removed?.length; diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index 0345282b0..caf589708 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -256,13 +256,14 @@ class TransactionUtils { // returns the most significant 4 bytes of the txid as an integer public txidToOrdering(txid: string): number { - return parseInt( - txid.substr(62, 2) + - txid.substr(60, 2) + - txid.substr(58, 2) + - txid.substr(56, 2), - 16 - ); + // Parse last 4 bytes of txid as little-endian uint32, without string allocation + let result = 0; + for (let i = 62; i >= 56; i -= 2) { + const hi = txid.charCodeAt(i); + const lo = txid.charCodeAt(i + 1); + result = result * 256 + (hi < 58 ? hi - 48 : hi - 87) * 16 + (lo < 58 ? lo - 48 : lo - 87); + } + return result; } public addInnerScriptsToVin(vin: IEsploraApi.Vin): void { diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index e11982e9d..bcdd71da5 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -3,7 +3,7 @@ import * as WebSocket from 'ws'; import { BlockExtended, TransactionExtended, MempoolTransactionExtended, WebsocketResponse, OptimizedStatistic, ILoadingIndicators, GbtCandidates, TxTrackingInfo, - MempoolDelta, MempoolDeltaTxids + MempoolDelta, MempoolDeltaTxids, TemplateAlgorithm, CpfpInfo } from '../mempool.interfaces'; import blocks from './blocks'; import memPool from './mempool'; @@ -35,7 +35,7 @@ interface AddressTransactions { removed: MempoolTransactionExtended[], } import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; -import { calculateMempoolTxCpfp } from './cpfp'; +import { calculateMempoolTxCpfp, calculateGoodBlockCpfp, calculateClusterMempoolBlockCpfp } from './cpfp'; import stratumApi, { StratumJob } from './services/stratum'; // valid 'want' subscriptions @@ -649,7 +649,10 @@ class WebsocketHandler { removed = candidates?.removed || []; } - if (config.MEMPOOL.RUST_GBT) { + if (config.MEMPOOL.CLUSTER_MEMPOOL) { + const cmBlocks = mempool.clusterMempool?.getBlocks(config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT) ?? []; + mempoolBlocks.processClusterMempoolBlocks(cmBlocks, newMempool, mempool.getAccelerations()); + } else if (config.MEMPOOL.RUST_GBT) { await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, newMempool, added, removed, candidates, true); } else { await mempoolBlocks.$updateBlockTemplates(transactionIds, newMempool, added, removed, candidates, accelerationDelta, true, true); @@ -946,15 +949,22 @@ class WebsocketHandler { calculateMempoolTxCpfp(mempoolTx, newMempool); } if (mempoolTx.cpfpDirty) { - positionData['cpfp'] = { - ancestors: mempoolTx.ancestors, + const cpfp: CpfpInfo = { + ancestors: mempoolTx.ancestors || [], bestDescendant: mempoolTx.bestDescendant || null, - descendants: mempoolTx.descendants || null, - effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize || null, + descendants: mempoolTx.descendants, + effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize, sigops: mempoolTx.sigops, adjustedVsize: mempoolTx.adjustedVsize, acceleration: mempoolTx.acceleration, }; + if (config.MEMPOOL.CLUSTER_MEMPOOL && mempoolTx.clusterId != null) { + const cluster = mempool.clusterMempool?.getClusterForApi(mempoolTx.txid); + if (cluster) { + cpfp.cluster = cluster; + } + } + positionData['cpfp'] = cpfp; } response['txPosition'] = JSON.stringify(positionData); } @@ -992,11 +1002,17 @@ class WebsocketHandler { txInfo.cpfp = { ancestors: mempoolTx.ancestors, bestDescendant: mempoolTx.bestDescendant || null, - descendants: mempoolTx.descendants || null, - effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize || null, + descendants: mempoolTx.descendants, + effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize, sigops: mempoolTx.sigops, adjustedVsize: mempoolTx.adjustedVsize, }; + if (config.MEMPOOL.CLUSTER_MEMPOOL && mempoolTx.clusterId != null) { + const cluster = mempool.clusterMempool?.getClusterForApi(mempoolTx.txid); + if (cluster) { + (txInfo.cpfp as CpfpInfo).cluster = cluster; + } + } } txHasInfo = true; } diff --git a/backend/src/config.ts b/backend/src/config.ts index 9c1762378..6850d8de6 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -34,6 +34,7 @@ interface IConfig { POOLS_JSON_TREE_URL: string, POOLS_UPDATE_DELAY: number, AUDIT: boolean; + CLUSTER_MEMPOOL: boolean; RUST_GBT: boolean; LIMIT_GBT: boolean; CPFP_INDEXING: boolean; @@ -205,6 +206,7 @@ const defaults: IConfig = { 'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', 'POOLS_UPDATE_DELAY': 604800, // in seconds, default is one week 'AUDIT': false, + 'CLUSTER_MEMPOOL': false, 'RUST_GBT': true, 'LIMIT_GBT': false, 'CPFP_INDEXING': false, diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index a0888bb50..23b1af694 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -28,6 +28,11 @@ export interface PoolStats extends PoolInfo { emptyBlocks: number; } +export enum TemplateAlgorithm { + legacy = 0, + clusterMempool = 1, +} + export interface BlockAudit { version: number, time: number, @@ -132,6 +137,8 @@ export interface TransactionExtended extends IEsploraApi.Transaction { replacement?: boolean; uid?: number; flags?: number; + clusterId?: number; + chunkIndex?: number; } export interface MempoolTransactionExtended extends TransactionExtended { @@ -227,6 +234,7 @@ export interface CpfpInfo { adjustedVsize?: number, acceleration?: boolean, fee?: number; + cluster?: CpfpClusterData & { chunkIndex: number }; } export interface TransactionStripped { diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index ee8e329a6..4dcb45cd2 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -29,6 +29,7 @@ "AUDIT": __MEMPOOL_AUDIT__, "RUST_GBT": __MEMPOOL_RUST_GBT__, "LIMIT_GBT": __MEMPOOL_LIMIT_GBT__, + "CLUSTER_MEMPOOL": __MEMPOOL_CLUSTER_MEMPOOL__, "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__, "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__, "DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__, diff --git a/docker/backend/start.sh b/docker/backend/start.sh index ae0bc616f..d5ea316bb 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -33,6 +33,7 @@ __MEMPOOL_POOLS_UPDATE_DELAY__=${MEMPOOL_POOLS_UPDATE_DELAY:=604800} __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false} __MEMPOOL_RUST_GBT__=${MEMPOOL_RUST_GBT:=true} __MEMPOOL_LIMIT_GBT__=${MEMPOOL_LIMIT_GBT:=false} +__MEMPOOL_CLUSTER_MEMPOOL__=${MEMPOOL_CLUSTER_MEMPOOL:=false} __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false} __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0} __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6} @@ -197,6 +198,7 @@ sed -i "s!__MEMPOOL_POOLS_UPDATE_DELAY__!${__MEMPOOL_POOLS_UPDATE_DELAY__}!g" me sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json sed -i "s!__MEMPOOL_LIMIT_GBT__!${__MEMPOOL_LIMIT_GBT__}!g" mempool-config.json +sed -i "s!__MEMPOOL_CLUSTER_MEMPOOL__!${__MEMPOOL_CLUSTER_MEMPOOL__}!g" mempool-config.json sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json