GlcoinCore/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2026-08-13 12:33:11 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Integrate cluster mempool into block projections

Browse source
This commit is contained in:
Mononaut 2026-03-07 11:01:12 +00:00 • committed by mononaut
parent 4811f9c5ce
commit 1f2e09c569
No known key found for this signature in database
GPG key ID: BFD16BE592A9CD8D
14 changed files with 342 additions and 121 deletions
Download patch file Download diff file Expand all files Collapse all files

1
backend/mempool-config.sample.json
View file

@ -31,6 +31,7 @@
"AUDIT": false,
"RUST_GBT": true,
"LIMIT_GBT": false,
"CLUSTER_MEMPOOL": false,
"CPFP_INDEXING": false,
"DISK_CACHE_BLOCK_INTERVAL": 6,
"MAX_PUSH_TX_SIZE_WEIGHT": 4000000,

1
backend/mempool-config.test.json
View file

@ -31,6 +31,7 @@
"AUDIT": false,
"RUST_GBT": true,
"LIMIT_GBT": false,
"CLUSTER_MEMPOOL": false,
"CPFP_INDEXING": false,
"DISK_CACHE_BLOCK_INTERVAL": 6,
"MAX_PUSH_TX_SIZE_WEIGHT": 4000000,

1
backend/src/__fixtures__/mempool-config.template.json
View file

@ -32,6 +32,7 @@
"AUDIT": true,
"RUST_GBT": false,
"LIMIT_GBT": false,
"CLUSTER_MEMPOOL": false,
"CPFP_INDEXING": true,
"MAX_BLOCKS_BULK_QUERY": 999,
"DISK_CACHE_BLOCK_INTERVAL": 999,

1
backend/src/__tests__/config.test.ts
View file

@ -45,6 +45,7 @@ describe('Mempool Backend Config', () => {
AUDIT: false,
RUST_GBT: true,
LIMIT_GBT: false,
CLUSTER_MEMPOOL: false,
CPFP_INDEXING: false,
MAX_BLOCKS_BULK_QUERY: 0,
DISK_CACHE_BLOCK_INTERVAL: 6,

19
backend/src/api/bitcoin/bitcoin.routes.ts
View file

@ -12,7 +12,7 @@ import backendInfo from '../backend-info';
import transactionUtils from '../transaction-utils';
import { IEsploraApi } from './esplora-api.interface';
import loadingIndicators from '../loading-indicators';
import { TransactionExtended } from '../../mempool.interfaces';
import { CpfpInfo, TransactionExtended } from '../../mempool.interfaces';
import logger from '../../logger';
import blocks from '../blocks';
import bitcoinClient from './bitcoin-client';
@ -191,11 +191,11 @@ class BitcoinRoutes {
const tx = mempool.getMempool()[req.params.txId];
if (tx) {
if (tx?.cpfpChecked) {
res.json({
ancestors: tx.ancestors,
const response: CpfpInfo & { acceleratedBy?: number[], acceleratedAt?: number, feeDelta?: number } = {
ancestors: tx.ancestors || [],
bestDescendant: tx.bestDescendant || null,
descendants: tx.descendants || null,
effectiveFeePerVsize: tx.effectiveFeePerVsize || null,
descendants: tx.descendants,
effectiveFeePerVsize: tx.effectiveFeePerVsize,
sigops: tx.sigops,
fee: tx.fee,
adjustedVsize: tx.adjustedVsize,
@ -203,7 +203,14 @@ class BitcoinRoutes {
acceleratedBy: tx.acceleratedBy || undefined,
acceleratedAt: tx.acceleratedAt || undefined,
feeDelta: tx.feeDelta || undefined,
});
};
if (config.MEMPOOL.CLUSTER_MEMPOOL && tx.clusterId != null) {
const cluster = mempool.clusterMempool?.getClusterForApi(req.params.txId);
if (cluster) {
response.cluster = cluster;
}
}
res.json(response);
return;
}

50
backend/src/api/cpfp.ts
View file

@ -1,7 +1,8 @@
import { Ancestor, CpfpCluster, CpfpInfo, CpfpSummary, MempoolTransactionExtended, TransactionExtended } from '../mempool.interfaces';
import { Ancestor, CpfpCluster, CpfpInfo, CpfpSummary, MempoolTransactionExtended, TemplateAlgorithm, TransactionExtended } from '../mempool.interfaces';
import { GraphTx, convertToGraphTx, expandRelativesGraph, initializeRelatives, makeBlockTemplate, mempoolComparator, removeAncestors, setAncestorScores } from './mini-miner';
import memPool from './mempool';
import { Acceleration } from './acceleration/acceleration';
import { ClusterMempool } from '../cluster-mempool/cluster-mempool';
const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction
const MAX_CLUSTER_ITERATIONS = 100;
@ -164,6 +165,53 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran
};
}
export function calculateClusterMempoolBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): CpfpSummary {
const txMap: { [txid: string]: MempoolTransactionExtended } = {};
for (const tx of transactions) {
txMap[tx.txid] = tx;
}
const accelMap: { [txid: string]: { feeDelta: number } } = {};
for (const acc of accelerations) {
accelMap[acc.txid] = { feeDelta: acc.max_bid };
}
const cm = new ClusterMempool(txMap, accelMap);
const seenClusters = new Set<number>();
const clusters: CpfpCluster[] = [];
for (const tx of transactions) {
if (tx.clusterId !== undefined && !seenClusters.has(tx.clusterId)) {
seenClusters.add(tx.clusterId);
const clusterData = cm.getCluster(tx.clusterId);
if (clusterData && clusterData.txs.length > 1) {
let totalFee = 0;
let totalWeight = 0;
for (const t of clusterData.txs) {
totalFee += t.fee;
totalWeight += t.weight;
}
clusters.push({
root: clusterData.txs[0].txid,
height,
txs: clusterData.txs.map(t => ({ txid: t.txid, weight: t.weight, fee: t.fee })),
effectiveFeePerVsize: totalFee / (totalWeight / 4),
templateAlgorithm: TemplateAlgorithm.clusterMempool,
clusterData,
});
}
}
}
return {
transactions: transactions.map(tx => txMap[tx.txid]),
clusters,
version: 3,
};
}
/**
* Takes a mempool transaction and a copy of the current mempool, and calculates the CPFP data for
* that transaction (and all others in the same cluster)

312
backend/src/api/mempool-blocks.ts
View file

@ -8,6 +8,7 @@ import path from 'path';
import mempool from './mempool';
import { Acceleration } from './services/acceleration';
import PoolsRepository from '../repositories/PoolsRepository';
import { ProjectedBlock } from '../cluster-mempool/cluster-mempool';
const MAX_UINT32 = Math.pow(2, 32) - 1;
@ -238,7 +239,7 @@ class MempoolBlocks {
}
/** @asyncSafe */
public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number, dryRun = false): Promise<MempoolBlockWithTransactions[]> {
const start = Date.now();
// reset mempool short ids
@ -278,7 +279,7 @@ class MempoolBlocks {
const expectedSize = transactions.length;
const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length;
logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${expectedSize} in the mempool, ${overflow.length} were unmineable`);
const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults);
const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults, dryRun);
logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
return processed;
} catch (e) {
@ -296,16 +297,15 @@ class MempoolBlocks {
}
/** @asyncSafe */
public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number, dryRun = false): Promise<MempoolBlockWithTransactions[]> {
// GBT optimization requires that uids never get too sparse
// as a sanity check, we should also explicitly prevent uint32 uid overflow
if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * transactions.length), MAX_UINT32)) {
this.resetRustGbt();
}
if (!this.rustInitialized) {
// need to reset the worker
return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, true, useAccelerations, accelerationPool);
if (!this.rustInitialized || dryRun) {
return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, !dryRun, useAccelerations, accelerationPool, dryRun);
}
const start = Date.now();
@ -344,7 +344,7 @@ class MempoolBlocks {
if (transactions.length !== resultMempoolSize) {
throw new Error(`GBT returned wrong number of transactions ${transactions.length} vs ${resultMempoolSize}, cache is probably out of sync`);
} else {
const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, true);
const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, !dryRun, dryRun);
this.removeUids(removedTxs);
logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
return processed;
@ -356,20 +356,22 @@ class MempoolBlocks {
}
}
private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations: { [txid: string]: Acceleration }, accelerationPool, saveResults): MempoolBlockWithTransactions[] {
for (const txid of Object.keys(candidates?.txs ?? mempool)) {
if (txid in mempool) {
mempool[txid].cpfpDirty = false;
mempool[txid].ancestors = [];
mempool[txid].descendants = [];
mempool[txid].bestDescendant = null;
private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations: { [txid: string]: Acceleration }, accelerationPool, saveResults, dryRun = false): MempoolBlockWithTransactions[] {
if (!dryRun) {
for (const txid of Object.keys(candidates?.txs ?? mempool)) {
if (txid in mempool) {
mempool[txid].cpfpDirty = false;
mempool[txid].ancestors = [];
mempool[txid].descendants = [];
mempool[txid].bestDescendant = null;
}
}
}
for (const [txid, rate] of rates) {
if (txid in mempool) {
mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize);
mempool[txid].effectiveFeePerVsize = rate;
mempool[txid].cpfpChecked = true;
for (const [txid, rate] of rates) {
if (txid in mempool) {
mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize);
mempool[txid].effectiveFeePerVsize = rate;
mempool[txid].cpfpChecked = true;
}
}
}
@ -387,58 +389,60 @@ class MempoolBlocks {
feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
}
const ancestors: Ancestor[] = [];
const descendants: Ancestor[] = [];
let ancestor: MempoolTransactionExtended;
for (const cluster of clusters) {
for (const memberTxid of cluster) {
const mempoolTx = mempool[memberTxid];
if (mempoolTx) {
// ugly micro-optimization to avoid allocating new arrays
ancestors.length = 0;
descendants.length = 0;
let matched = false;
cluster.forEach(txid => {
ancestor = mempool[txid];
if (txid === memberTxid) {
matched = true;
} else {
if (!ancestor) {
console.log('txid missing from mempool! ', txid, candidates?.txs[txid]);
return;
}
const relative = {
txid: txid,
fee: ancestor.fee,
weight: (ancestor.adjustedVsize * 4),
};
if (matched) {
descendants.push(relative);
if (!mempoolTx.lastBoosted || (ancestor.firstSeen && ancestor.firstSeen > mempoolTx.lastBoosted)) {
mempoolTx.lastBoosted = ancestor.firstSeen;
}
if (!dryRun) {
const ancestors: Ancestor[] = [];
const descendants: Ancestor[] = [];
let ancestor: MempoolTransactionExtended;
for (const cluster of clusters) {
for (const memberTxid of cluster) {
const mempoolTx = mempool[memberTxid];
if (mempoolTx) {
// ugly micro-optimization to avoid allocating new arrays
ancestors.length = 0;
descendants.length = 0;
let matched = false;
cluster.forEach(txid => {
ancestor = mempool[txid];
if (txid === memberTxid) {
matched = true;
} else {
ancestors.push(relative);
if (!ancestor) {
console.log('txid missing from mempool! ', txid, candidates?.txs[txid]);
return;
}
const relative = {
txid: txid,
fee: ancestor.fee,
weight: (ancestor.adjustedVsize * 4),
};
if (matched) {
descendants.push(relative);
if (!mempoolTx.lastBoosted || (ancestor.firstSeen && ancestor.firstSeen > mempoolTx.lastBoosted)) {
mempoolTx.lastBoosted = ancestor.firstSeen;
}
} else {
ancestors.push(relative);
}
}
});
if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) {
mempoolTx.cpfpDirty = true;
}
});
if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) {
mempoolTx.cpfpDirty = true;
// ugly micro-optimization to avoid allocating new arrays or objects
if (mempoolTx.ancestors) {
mempoolTx.ancestors.length = 0;
} else {
mempoolTx.ancestors = [];
}
if (mempoolTx.descendants) {
mempoolTx.descendants.length = 0;
} else {
mempoolTx.descendants = [];
}
mempoolTx.ancestors.push(...ancestors);
mempoolTx.descendants.push(...descendants);
mempoolTx.cpfpChecked = true;
}
// ugly micro-optimization to avoid allocating new arrays or objects
if (mempoolTx.ancestors) {
mempoolTx.ancestors.length = 0;
} else {
mempoolTx.ancestors = [];
}
if (mempoolTx.descendants) {
mempoolTx.descendants.length = 0;
} else {
mempoolTx.descendants = [];
}
mempoolTx.ancestors.push(...ancestors);
mempoolTx.descendants.push(...descendants);
mempoolTx.cpfpChecked = true;
}
}
}
@ -471,34 +475,42 @@ class MempoolBlocks {
const txid = block[i];
if (txid in mempool) {
mempoolTx = mempool[txid];
// save position in projected blocks
mempoolTx.position = {
block: blockIndex,
vsize: totalVsize + (mempoolTx.vsize / 2),
};
if (txid in accelerations) {
acceleration = accelerations[txid];
if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
if (!mempoolTx.acceleration) {
mempoolTx.cpfpDirty = true;
}
mempoolTx.acceleration = true;
mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools;
mempoolTx.acceleratedAt = acceleration?.added;
mempoolTx.feeDelta = acceleration?.feeDelta;
for (const ancestor of mempoolTx.ancestors || []) {
if (!(ancestor.txid in mempool)) {
continue;
if (!dryRun) {
// save position in projected blocks
mempoolTx.position = {
block: blockIndex,
vsize: totalVsize + (mempoolTx.vsize / 2),
};
if (txid in accelerations) {
acceleration = accelerations[txid];
if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
if (!mempoolTx.acceleration) {
mempoolTx.cpfpDirty = true;
}
if (!mempool[ancestor.txid].acceleration) {
mempool[ancestor.txid].cpfpDirty = true;
mempoolTx.acceleration = true;
mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools;
mempoolTx.acceleratedAt = acceleration?.added;
mempoolTx.feeDelta = acceleration?.feeDelta;
for (const ancestor of mempoolTx.ancestors || []) {
if (!(ancestor.txid in mempool)) {
continue;
}
if (!mempool[ancestor.txid].acceleration) {
mempool[ancestor.txid].cpfpDirty = true;
}
mempool[ancestor.txid].acceleration = true;
mempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy;
mempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt;
mempool[ancestor.txid].feeDelta = mempoolTx.feeDelta;
isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy;
}
} else {
if (mempoolTx.acceleration) {
mempoolTx.cpfpDirty = true;
delete mempoolTx.acceleration;
}
mempool[ancestor.txid].acceleration = true;
mempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy;
mempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt;
mempool[ancestor.txid].feeDelta = mempoolTx.feeDelta;
isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy;
}
} else {
if (mempoolTx.acceleration) {
@ -506,11 +518,6 @@ class MempoolBlocks {
delete mempoolTx.acceleration;
}
}
} else {
if (mempoolTx.acceleration) {
mempoolTx.cpfpDirty = true;
delete mempoolTx.acceleration;
}
}
// online calculation of stack-of-blocks fee stats
@ -548,6 +555,115 @@ class MempoolBlocks {
return mempoolBlocks;
}
public processClusterMempoolBlocks(projectedBlocks: ProjectedBlock[], newMempool: { [txid: string]: MempoolTransactionExtended }, accelerations: { [txid: string]: Acceleration }, saveResults = true, accelerationPool?: number): MempoolBlockWithTransactions[] {
const lastBlockIndex = projectedBlocks.length - 1;
let hasBlockStack = projectedBlocks.length >= 8;
let stackWeight = 0;
let feeStatsCalculator: OnlineFeeStatsCalculator | null = null;
if (hasBlockStack) {
stackWeight = projectedBlocks[lastBlockIndex].weight;
hasBlockStack = stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS;
feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
}
const isAcceleratedBy: { [txid: string]: number[] | false } = {};
const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
let mempoolTx: MempoolTransactionExtended;
let acceleration: Acceleration;
const mempoolBlocks: MempoolBlockWithTransactions[] = [];
for (let blockIndex = 0; blockIndex < projectedBlocks.length; blockIndex++) {
const projected = projectedBlocks[blockIndex];
let totalSize = 0;
let totalVsize = 0;
let totalWeight = 0;
let totalFees = 0;
const transactions: MempoolTransactionExtended[] = [];
const validTxids: string[] = [];
for (const txid of projected.txids) {
if (txid in newMempool) {
mempoolTx = newMempool[txid];
validTxids.push(txid);
// save position in projected blocks
mempoolTx.position = {
block: blockIndex,
vsize: totalVsize + (mempoolTx.vsize / 2),
};
if (txid in accelerations) {
acceleration = accelerations[txid];
if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
if (!mempoolTx.acceleration) {
mempoolTx.cpfpDirty = true;
}
mempoolTx.acceleration = true;
mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools;
mempoolTx.acceleratedAt = acceleration?.added;
mempoolTx.feeDelta = acceleration?.feeDelta;
for (const ancestor of mempoolTx.ancestors || []) {
if (!(ancestor.txid in newMempool)) {
continue;
}
if (!newMempool[ancestor.txid].acceleration) {
newMempool[ancestor.txid].cpfpDirty = true;
}
newMempool[ancestor.txid].acceleration = true;
newMempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy;
newMempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt;
newMempool[ancestor.txid].feeDelta = mempoolTx.feeDelta;
isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy;
}
} else {
if (mempoolTx.acceleration) {
mempoolTx.cpfpDirty = true;
delete mempoolTx.acceleration;
}
}
} else {
if (mempoolTx.acceleration) {
mempoolTx.cpfpDirty = true;
delete mempoolTx.acceleration;
}
}
if (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) {
feeStatsCalculator.processNext(mempoolTx);
}
totalSize += mempoolTx.size;
totalVsize += mempoolTx.vsize;
totalWeight += mempoolTx.weight;
totalFees += mempoolTx.fee;
if (totalVsize <= sizeLimit) {
transactions.push(mempoolTx);
}
}
}
mempoolBlocks[blockIndex] = this.dataToMempoolBlocks(
validTxids,
transactions,
totalSize,
totalWeight,
totalFees,
(hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined,
);
}
if (saveResults) {
const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
this.mempoolBlocks = mempoolBlocks;
this.mempoolBlockDeltas = deltas;
this.updateAccelerationPositions(newMempool, accelerations, mempoolBlocks);
}
return mempoolBlocks;
}
private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions {
if (!feeStats) {
feeStats = Common.calcEffectiveFeeStatistics(transactions);

16
backend/src/api/mempool.ts
View file

@ -13,6 +13,7 @@ import { Acceleration } from './services/acceleration';
import accelerationApi from './services/acceleration';
import redisCache from './redis-cache';
import blocks from './blocks';
import { ClusterMempool } from '../cluster-mempool/cluster-mempool';
class Mempool {
private inSync: boolean = false;
@ -22,6 +23,7 @@ class Mempool {
private spendMap = new Map<string, MempoolTransactionExtended>();
private recentlyDeleted: MempoolTransactionExtended[][] = []; // buffer of transactions deleted in recent mempool updates
private mempoolInfo: IBitcoinApi.MempoolInfo;
public clusterMempool: ClusterMempool | null = null;
private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[],
deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[]) => void) | undefined;
private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[],
@ -62,6 +64,9 @@ class Mempool {
minrelaytxfee: isLiquid ? 0.00000100 : 0.00001000
};
this.txPerSecondInterval = setInterval(this.updateTxPerSecond.bind(this), 1000);
if (config.MEMPOOL.CLUSTER_MEMPOOL) {
this.clusterMempool = new ClusterMempool(this.mempoolCache, this.accelerations);
}
}
/**
@ -153,6 +158,9 @@ class Mempool {
await redisCache.$flushTransactions();
logger.debug(`Finished migrating cache transactions in ${((Date.now() - redisTimer) / 1000).toFixed(2)} seconds`);
}
if (config.MEMPOOL.CLUSTER_MEMPOOL) {
this.clusterMempool = new ClusterMempool(this.mempoolCache, this.accelerations);
}
if (this.mempoolChangedCallback) {
this.mempoolChangedCallback(this.mempoolCache, [], [], []);
}
@ -387,6 +395,14 @@ class Mempool {
hasChange = true;
}
if (config.MEMPOOL.CLUSTER_MEMPOOL && (newTransactions.length || deletedTransactions.length || accelerationDelta.length)) {
this.clusterMempool?.applyMempoolChange({
added: newTransactions,
removed: deletedTransactions.map(tx => tx.txid),
accelerations: this.getAccelerations(),
});
}
this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize);
const candidatesChanged = candidates?.added?.length || candidates?.removed?.length;

15
backend/src/api/transaction-utils.ts
View file

@ -256,13 +256,14 @@ class TransactionUtils {
// returns the most significant 4 bytes of the txid as an integer
public txidToOrdering(txid: string): number {
return parseInt(
txid.substr(62, 2) +
txid.substr(60, 2) +
txid.substr(58, 2) +
txid.substr(56, 2),
16
);
// Parse last 4 bytes of txid as little-endian uint32, without string allocation
let result = 0;
for (let i = 62; i >= 56; i -= 2) {
const hi = txid.charCodeAt(i);
const lo = txid.charCodeAt(i + 1);
result = result * 256 + (hi < 58 ? hi - 48 : hi - 87) * 16 + (lo < 58 ? lo - 48 : lo - 87);
}
return result;
}
public addInnerScriptsToVin(vin: IEsploraApi.Vin): void {

34
backend/src/api/websocket-handler.ts
View file

@ -3,7 +3,7 @@ import * as WebSocket from 'ws';
import {
BlockExtended, TransactionExtended, MempoolTransactionExtended, WebsocketResponse,
OptimizedStatistic, ILoadingIndicators, GbtCandidates, TxTrackingInfo,
MempoolDelta, MempoolDeltaTxids
MempoolDelta, MempoolDeltaTxids, TemplateAlgorithm, CpfpInfo
} from '../mempool.interfaces';
import blocks from './blocks';
import memPool from './mempool';
@ -35,7 +35,7 @@ interface AddressTransactions {
removed: MempoolTransactionExtended[],
}
import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
import { calculateMempoolTxCpfp } from './cpfp';
import { calculateMempoolTxCpfp, calculateGoodBlockCpfp, calculateClusterMempoolBlockCpfp } from './cpfp';
import stratumApi, { StratumJob } from './services/stratum';
// valid 'want' subscriptions
@ -649,7 +649,10 @@ class WebsocketHandler {
removed = candidates?.removed || [];
}
if (config.MEMPOOL.RUST_GBT) {
if (config.MEMPOOL.CLUSTER_MEMPOOL) {
const cmBlocks = mempool.clusterMempool?.getBlocks(config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT) ?? [];
mempoolBlocks.processClusterMempoolBlocks(cmBlocks, newMempool, mempool.getAccelerations());
} else if (config.MEMPOOL.RUST_GBT) {
await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, newMempool, added, removed, candidates, true);
} else {
await mempoolBlocks.$updateBlockTemplates(transactionIds, newMempool, added, removed, candidates, accelerationDelta, true, true);
@ -946,15 +949,22 @@ class WebsocketHandler {
calculateMempoolTxCpfp(mempoolTx, newMempool);
}
if (mempoolTx.cpfpDirty) {
positionData['cpfp'] = {
ancestors: mempoolTx.ancestors,
const cpfp: CpfpInfo = {
ancestors: mempoolTx.ancestors || [],
bestDescendant: mempoolTx.bestDescendant || null,
descendants: mempoolTx.descendants || null,
effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize || null,
descendants: mempoolTx.descendants,
effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize,
sigops: mempoolTx.sigops,
adjustedVsize: mempoolTx.adjustedVsize,
acceleration: mempoolTx.acceleration,
};
if (config.MEMPOOL.CLUSTER_MEMPOOL && mempoolTx.clusterId != null) {
const cluster = mempool.clusterMempool?.getClusterForApi(mempoolTx.txid);
if (cluster) {
cpfp.cluster = cluster;
}
}
positionData['cpfp'] = cpfp;
}
response['txPosition'] = JSON.stringify(positionData);
}
@ -992,11 +1002,17 @@ class WebsocketHandler {
txInfo.cpfp = {
ancestors: mempoolTx.ancestors,
bestDescendant: mempoolTx.bestDescendant || null,
descendants: mempoolTx.descendants || null,
effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize || null,
descendants: mempoolTx.descendants,
effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize,
sigops: mempoolTx.sigops,
adjustedVsize: mempoolTx.adjustedVsize,
};
if (config.MEMPOOL.CLUSTER_MEMPOOL && mempoolTx.clusterId != null) {
const cluster = mempool.clusterMempool?.getClusterForApi(mempoolTx.txid);
if (cluster) {
(txInfo.cpfp as CpfpInfo).cluster = cluster;
}
}
}
txHasInfo = true;
}

2
backend/src/config.ts
View file

@ -34,6 +34,7 @@ interface IConfig {
POOLS_JSON_TREE_URL: string,
POOLS_UPDATE_DELAY: number,
AUDIT: boolean;
CLUSTER_MEMPOOL: boolean;
RUST_GBT: boolean;
LIMIT_GBT: boolean;
CPFP_INDEXING: boolean;
@ -205,6 +206,7 @@ const defaults: IConfig = {
'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
'POOLS_UPDATE_DELAY': 604800, // in seconds, default is one week
'AUDIT': false,
'CLUSTER_MEMPOOL': false,
'RUST_GBT': true,
'LIMIT_GBT': false,
'CPFP_INDEXING': false,

8
backend/src/mempool.interfaces.ts
View file

@ -28,6 +28,11 @@ export interface PoolStats extends PoolInfo {
emptyBlocks: number;
}
export enum TemplateAlgorithm {
legacy = 0,
clusterMempool = 1,
}
export interface BlockAudit {
version: number,
time: number,
@ -132,6 +137,8 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
replacement?: boolean;
uid?: number;
flags?: number;
clusterId?: number;
chunkIndex?: number;
}
export interface MempoolTransactionExtended extends TransactionExtended {
@ -227,6 +234,7 @@ export interface CpfpInfo {
adjustedVsize?: number,
acceleration?: boolean,
fee?: number;
cluster?: CpfpClusterData & { chunkIndex: number };
}
export interface TransactionStripped {

1
docker/backend/mempool-config.json
View file

@ -29,6 +29,7 @@
"AUDIT": __MEMPOOL_AUDIT__,
"RUST_GBT": __MEMPOOL_RUST_GBT__,
"LIMIT_GBT": __MEMPOOL_LIMIT_GBT__,
"CLUSTER_MEMPOOL": __MEMPOOL_CLUSTER_MEMPOOL__,
"CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
"MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__,
"DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__,

2
docker/backend/start.sh
View file

@ -33,6 +33,7 @@ __MEMPOOL_POOLS_UPDATE_DELAY__=${MEMPOOL_POOLS_UPDATE_DELAY:=604800}
__MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false}
__MEMPOOL_RUST_GBT__=${MEMPOOL_RUST_GBT:=true}
__MEMPOOL_LIMIT_GBT__=${MEMPOOL_LIMIT_GBT:=false}
__MEMPOOL_CLUSTER_MEMPOOL__=${MEMPOOL_CLUSTER_MEMPOOL:=false}
__MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
__MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6}
@ -197,6 +198,7 @@ sed -i "s!__MEMPOOL_POOLS_UPDATE_DELAY__!${__MEMPOOL_POOLS_UPDATE_DELAY__}!g" me
sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json
sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json
sed -i "s!__MEMPOOL_LIMIT_GBT__!${__MEMPOOL_LIMIT_GBT__}!g" mempool-config.json
sed -i "s!__MEMPOOL_CLUSTER_MEMPOOL__!${__MEMPOOL_CLUSTER_MEMPOOL__}!g" mempool-config.json
sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json