diff --git a/backend/.eslintrc.js b/backend/.eslintrc.js index 15bc17cd9..b53232ee0 100644 --- a/backend/.eslintrc.js +++ b/backend/.eslintrc.js @@ -15,7 +15,7 @@ module.exports = { "plugin:@typescript-eslint/recommended", "prettier" ], - "ignorePatterns": ["dist", "eslint-local-rules", ".eslintrc.js", "testSetup*.ts", "jest.integration.*.ts", "__tests__", "*.config.ts"], + "ignorePatterns": ["dist", "eslint-local-rules", ".eslintrc.js", "testSetup*.ts", "jest.integration.*.ts", "__tests__", "__e2e__", "*.config.ts"], "overrides": [ { "files": ["src/__integration_tests__/**/*"], diff --git a/backend/jest.config.ts b/backend/jest.config.ts index 7989fca81..78c4ca054 100644 --- a/backend/jest.config.ts +++ b/backend/jest.config.ts @@ -17,8 +17,10 @@ const config: Config.InitialOptions = { './testSetup.ts', ], testPathIgnorePatterns: [ + '/dist/', '/node_modules/', '/__integration_tests__/', + 'test-utils\\.ts$', ], }; export default config; diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index c2715153b..b11616b2a 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -31,6 +31,8 @@ "AUDIT": false, "RUST_GBT": true, "LIMIT_GBT": false, + "CLUSTER_MEMPOOL": false, + "CLUSTER_MEMPOOL_INDEXING": false, "CPFP_INDEXING": false, "DISK_CACHE_BLOCK_INTERVAL": 6, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, diff --git a/backend/mempool-config.test.json b/backend/mempool-config.test.json index 2522ca90f..c309099b9 100644 --- a/backend/mempool-config.test.json +++ b/backend/mempool-config.test.json @@ -31,6 +31,8 @@ "AUDIT": false, "RUST_GBT": true, "LIMIT_GBT": false, + "CLUSTER_MEMPOOL": false, + "CLUSTER_MEMPOOL_INDEXING": false, "CPFP_INDEXING": false, "DISK_CACHE_BLOCK_INTERVAL": 6, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, diff --git a/backend/src/__e2e__/cluster-mempool/harness/rpc-client.ts b/backend/src/__e2e__/cluster-mempool/harness/rpc-client.ts new file mode 100644 index 000000000..6ef621028 --- /dev/null +++ b/backend/src/__e2e__/cluster-mempool/harness/rpc-client.ts @@ -0,0 +1,137 @@ +import http from 'http'; + +export interface RpcConfig { + host: string; + port: number; + user: string; + pass: string; +} + +interface RpcResponse { + result: any; + error: { code: number; message: string } | null; + id: string; +} + +export class RpcClient { + private config: RpcConfig; + private idCounter = 0; + + constructor(config: RpcConfig) { + this.config = config; + } + + private async call(method: string, params: any[] = []): Promise { + const id = String(++this.idCounter); + const body = JSON.stringify({ jsonrpc: '2.0', id, method, params }); + const auth = Buffer.from(`${this.config.user}:${this.config.pass}`).toString('base64'); + + return new Promise((resolve, reject) => { + const req = http.request( + { + hostname: this.config.host, + port: this.config.port, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Basic ${auth}`, + 'Content-Length': Buffer.byteLength(body), + }, + }, + (res) => { + let data = ''; + res.on('data', (chunk) => (data += chunk)); + res.on('end', () => { + try { + const parsed: RpcResponse = JSON.parse(data); + if (parsed.error) { + reject(new Error(`RPC error ${parsed.error.code}: ${parsed.error.message}`)); + } else { + resolve(parsed.result); + } + } catch (e) { + reject(new Error(`Failed to parse RPC response: ${data.slice(0, 500)}`)); + } + }); + }, + ); + req.on('error', reject); + req.write(body); + req.end(); + }); + } + + async getRawMempool(): Promise { + return this.call('getrawmempool'); + } + + async getRawTransaction(txid: string, verbose = true): Promise { + return this.call('getrawtransaction', [txid, verbose]); + } + + async getBlockTemplate(rules: string[] = ['segwit']): Promise { + return this.call('getblocktemplate', [{ rules }]); + } + + async getMempoolEntry(txid: string): Promise { + return this.call('getmempoolentry', [txid]); + } + + async getMempoolCluster(txid: string): Promise { + return this.call('getmempoolcluster', [txid]); + } + + async getBlockCount(): Promise { + return this.call('getblockcount'); + } + + async batch(calls: { method: string; params: any[] }[]): Promise { + if (calls.length === 0) { + return []; + } + const bodies = calls.map((c) => ({ + jsonrpc: '2.0', + id: String(++this.idCounter), + method: c.method, + params: c.params, + })); + const body = JSON.stringify(bodies); + const auth = Buffer.from(`${this.config.user}:${this.config.pass}`).toString('base64'); + + return new Promise((resolve, reject) => { + const req = http.request( + { + hostname: this.config.host, + port: this.config.port, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Basic ${auth}`, + 'Content-Length': Buffer.byteLength(body), + }, + }, + (res) => { + let data = ''; + res.on('data', (chunk) => (data += chunk)); + res.on('end', () => { + try { + const parsed: RpcResponse[] = JSON.parse(data); + const results = parsed.map((r) => { + if (r.error) { + return { error: r.error }; + } + return r.result; + }); + resolve(results); + } catch (e) { + reject(new Error(`Failed to parse batch RPC response: ${data.slice(0, 500)}`)); + } + }); + }, + ); + req.on('error', reject); + req.write(body); + req.end(); + }); + } +} diff --git a/backend/src/__e2e__/cluster-mempool/harness/run-harness.ts b/backend/src/__e2e__/cluster-mempool/harness/run-harness.ts new file mode 100644 index 000000000..3e252122b --- /dev/null +++ b/backend/src/__e2e__/cluster-mempool/harness/run-harness.ts @@ -0,0 +1,461 @@ +/** + * Cluster Mempool Test Harness + * + * Standalone script that compares our ClusterMempool block template ordering + * against Bitcoin Core's getblocktemplate output. + * + * Uses the backend's own transactionUtils to fetch and convert transactions, + * ensuring fields (sigops, adjustedVsize, etc.) match exactly. + * + * Usage: + * npx ts-node src/__tests__/cluster-mempool/harness/run-harness.ts [options] + * + * Options: + * --host Override CORE_RPC host + * --port Override CORE_RPC port + * --user Override CORE_RPC username + * --pass Override CORE_RPC password + * --interval Comparison interval in ms (default: 30000) + * --poll Mempool poll interval in ms (default: 1000) + * --max Max comparisons before exit (0 = unlimited, default: 0) + */ + +import { RpcClient, RpcConfig } from './rpc-client'; + +// ─── CLI Parsing (must happen before backend imports) ─────────────────────── + +interface CliOptions { + rpcOverrides: Partial; + comparisonInterval: number; + pollInterval: number; + maxComparisons: number; +} + +function parseArgs(): CliOptions { + const args = process.argv.slice(2); + const overrides: Partial = {}; + let comparisonInterval = 30_000; + let pollInterval = 1000; + let maxComparisons = 0; + + for (let i = 0; i < args.length; i++) { + switch (args[i]) { + case '--host': + overrides.host = args[++i]; + break; + case '--port': + overrides.port = parseInt(args[++i], 10); + break; + case '--user': + overrides.user = args[++i]; + break; + case '--pass': + overrides.pass = args[++i]; + break; + case '--interval': + comparisonInterval = parseInt(args[++i], 10); + break; + case '--poll': + pollInterval = parseInt(args[++i], 10); + break; + case '--max': + maxComparisons = parseInt(args[++i], 10); + break; + } + } + + return { rpcOverrides: overrides, comparisonInterval, pollInterval, maxComparisons }; +} + +const cliOptions = parseArgs(); + +// ─── Backend Module Loading ───────────────────────────────────────────────── +// Import config first, apply CLI overrides, then import modules that depend on it. +// This works because Node's require() caches modules — when transactionUtils +// (via bitcoinClient) reads config, it sees our modified values. + +const config = require('../../../config').default; + +if (cliOptions.rpcOverrides.host) { config.CORE_RPC.HOST = cliOptions.rpcOverrides.host; } +if (cliOptions.rpcOverrides.port) { config.CORE_RPC.PORT = cliOptions.rpcOverrides.port; } +if (cliOptions.rpcOverrides.user) { config.CORE_RPC.USERNAME = cliOptions.rpcOverrides.user; } +if (cliOptions.rpcOverrides.pass) { config.CORE_RPC.PASSWORD = cliOptions.rpcOverrides.pass; } + +// Now load modules that depend on config +const bitcoinApi = require('../../../api/bitcoin/bitcoin-api-factory').default; +const transactionUtils = require('../../../api/transaction-utils').default; +const { ClusterMempool } = require('../../../cluster-mempool/cluster-mempool'); + +import { MempoolTransactionExtended } from '../../../mempool.interfaces'; +import { MempoolDiff } from '../../../cluster-mempool/cluster-mempool'; + +// ─── Main Harness ─────────────────────────────────────────────────────────── + +const TX_FETCH_BATCH_SIZE = 1000; + +class Harness { + private rpc: RpcClient; + private clusterMempool: InstanceType | null = null; + private mempool: { [txid: string]: MempoolTransactionExtended } = {}; + private knownTxids = new Set(); + private lastBlockHeight = -1; + private comparisonInterval: number; + private pollInterval: number; + private maxComparisons: number; + private pollTimer: ReturnType | null = null; + private polling = false; + private nextComparisonTime = 0; + private stats = { + templateMatches: 0, + templateMismatches: 0, + comparisons: 0, + }; + + constructor(rpcConfig: RpcConfig, opts: CliOptions) { + this.rpc = new RpcClient(rpcConfig); + this.comparisonInterval = opts.comparisonInterval; + this.pollInterval = opts.pollInterval; + this.maxComparisons = opts.maxComparisons; + } + + async run(): Promise { + console.log('=== Cluster Mempool Test Harness ==='); + console.log(`RPC: ${config.CORE_RPC.HOST}:${config.CORE_RPC.PORT}`); + console.log(`Backend: ${config.MEMPOOL.BACKEND}`); + console.log(`Poll interval: ${this.pollInterval}ms`); + console.log(`Comparison interval: ${this.comparisonInterval}ms`); + console.log(); + + // Step 1: Fetch full mempool + console.log('Fetching full mempool...'); + const txids: string[] = await bitcoinApi.$getRawMempool(); + console.log(`Got ${txids.length} txids from getrawmempool`); + + const fetchStart = Date.now(); + await this.fetchTransactions(txids); + const fetchTime = Date.now() - fetchStart; + console.log(`Fetched ${Object.keys(this.mempool).length} transactions in ${fetchTime}ms`); + + // Step 2: Initialize ClusterMempool + console.log('Building cluster mempool...'); + const buildStart = Date.now(); + this.clusterMempool = new ClusterMempool(this.mempool); + const buildTime = Date.now() - buildStart; + console.log(`Cluster mempool built in ${buildTime}ms`); + console.log(` Clusters: ${this.clusterMempool.getClusterCount()}`); + console.log(` Transactions: ${this.clusterMempool.getTxCount()}`); + console.log(); + + // Record initial block height + this.lastBlockHeight = await this.rpc.getBlockCount(); + + // Run initial comparison, then start polling + await this.runComparison(); + this.nextComparisonTime = Date.now() + this.comparisonInterval; + this.pollTimer = setInterval(() => this.pollMempool(), this.pollInterval); + + console.log('\nHarness running. Press Ctrl+C to stop.\n'); + + process.on('SIGINT', () => this.shutdown()); + process.on('SIGTERM', () => this.shutdown()); + } + + private async fetchTransactions(txids: string[]): Promise { + const added: MempoolTransactionExtended[] = []; + + for (let offset = 0; offset < txids.length; offset += TX_FETCH_BATCH_SIZE) { + let batch = txids.slice(offset, offset + TX_FETCH_BATCH_SIZE); + let txs: MempoolTransactionExtended[] = []; + let tries = 0; + while (batch.length && tries < 20) { + try { + tries++; + txs = txs.concat(await transactionUtils.$getMempoolTransactionsExtended(batch, false, false, false)); + let missing: string[] = []; + console.log(`txs: ${txs.length} of ${batch.length} fetched`); + for (const txid of batch) { + if (!txs.some(tx => tx.txid === txid)) { + console.log(`missing ${txid} at offset ${offset}, retrying`); + missing.push(txid); + } + } + batch = missing; + if (batch.length) { + await new Promise(resolve => setTimeout(resolve, 500)); + } + } catch (err: any) { + console.log(` Fetch batch failed at offset ${offset}: ${err.message}, retrying`); + } + } + + for (const tx of txs) { + this.mempool[tx.txid] = tx; + this.knownTxids.add(tx.txid); + added.push(tx); + } + + if (txids.length > TX_FETCH_BATCH_SIZE && (offset + batch.length) % 5000 < TX_FETCH_BATCH_SIZE) { + console.log(` ${offset + batch.length} / ${txids.length} fetched...`); + } + } + + return added; + } + + private async pollMempool(): Promise { + if (this.polling || !this.clusterMempool) { + return; + } + this.polling = true; + + try { + // Check for new block + const height = await this.rpc.getBlockCount(); + const newBlock = height > this.lastBlockHeight; + if (newBlock) { + console.log(`\n--- New block detected (height ${height}) ---`); + this.lastBlockHeight = height; + } + + // Get current mempool + const currentTxids = await bitcoinApi.$getRawMempool(); + const currentSet = new Set(currentTxids); + + // Find added and removed + const addedTxids: string[] = []; + for (const txid of currentTxids) { + if (!this.knownTxids.has(txid)) { + addedTxids.push(txid); + } + } + const removedTxids: string[] = []; + for (const txid of this.knownTxids) { + if (!currentSet.has(txid)) { + removedTxids.push(txid); + } + } + + if (addedTxids.length === 0 && removedTxids.length === 0) { + if (newBlock) { + await this.runComparison(); + } + return; + } + + const added = await this.fetchTransactions(addedTxids); + + // Apply diff before deleting from cache — processRemovals needs the tx data + const diff: MempoolDiff = { added, removed: removedTxids, accelerations: {} }; + const t0 = Date.now(); + this.clusterMempool.applyMempoolChange(diff); + const dt = Date.now() - t0; + for (const txid of removedTxids) { + delete this.mempool[txid]; + this.knownTxids.delete(txid); + } + console.log( + `Applied${newBlock ? ' block' : ''} diff: +${added.length} -${removedTxids.length} txs in ${dt}ms` + + ` (${this.clusterMempool.getTxCount()} txs, ${this.clusterMempool.getClusterCount()} clusters)` + ); + + if (newBlock) { + await this.runComparison(); + } else if (Date.now() >= this.nextComparisonTime) { + await this.runComparison(); + this.nextComparisonTime = Date.now() + this.comparisonInterval; + } + } catch (err: any) { + console.error(`Poll error: ${err.message}`); + } finally { + this.polling = false; + } + } + + private async runComparison(): Promise { + if (!this.clusterMempool) { + return; + } + this.stats.comparisons++; + console.log(`\n=== Comparison #${this.stats.comparisons} ===`); + + // Step 1: Get Core's block template + let template: any; + try { + template = await this.rpc.getBlockTemplate(['segwit']); + } catch (err: any) { + console.log(`getblocktemplate failed: ${err.message}`); + return; + } + const coreTxids: string[] = template.transactions.map((t: any) => t.txid); + const coreSet = new Set(coreTxids); + + const missingTxids: string[] = []; + for (const txid of coreSet) { + if (!this.mempool[txid]) { + missingTxids.push(txid); + } + } + if (missingTxids.length) { + const added = await this.fetchTransactions(missingTxids); + if (added.length) { + console.log(`Reconciled: +${added.length} missing txs`); + this.clusterMempool.applyMempoolChange({ added, removed: [], accelerations: {} }); + } + } + + const ourBlocks = this.clusterMempool.getBlocks(1, true); + const ourTxids: string[] = ourBlocks[0]?.txids || []; + + await this.compareTemplateOrdering(coreTxids, ourTxids); + + this.printStats(); + + if (this.maxComparisons > 0 && this.stats.comparisons >= this.maxComparisons) { + console.log(`\nReached ${this.maxComparisons} comparisons, stopping.`); + this.shutdown(); + } + } + + private async compareTemplateOrdering(coreTxids: string[], ourTxids: string[]): Promise { + const coreSet = new Set(coreTxids); + const ourSet = new Set(ourTxids); + + let inBoth = 0; + let onlyCore = 0; + let onlyOurs = 0; + for (const txid of coreSet) { + if (ourSet.has(txid)) { + inBoth++; + } else { + onlyCore++; + } + } + for (const txid of ourSet) { + if (!coreSet.has(txid)) { + onlyOurs++; + } + } + + console.log(`Template sets: Core ${coreTxids.length} txs, Ours ${ourTxids.length} txs`); + console.log(` In both: ${inBoth}, Only Core: ${onlyCore}, Only ours: ${onlyOurs}`); + + // Build position maps for the intersection + const corePos = new Map(); + for (let i = 0; i < coreTxids.length; i++) { + if (ourSet.has(coreTxids[i])) { + corePos.set(coreTxids[i], i); + } + } + + // Filter to shared txids in each order + const sharedInCoreOrder = coreTxids.filter(t => ourSet.has(t)); + const sharedInOurOrder = ourTxids.filter(t => coreSet.has(t)); + + let exactMatches = 0; + let firstDivergence = -1; + for (let i = 0; i < sharedInCoreOrder.length; i++) { + if (sharedInCoreOrder[i] === sharedInOurOrder[i]) { + exactMatches++; + } else if (firstDivergence === -1) { + firstDivergence = i; + } + } + + const orderMatchRate = sharedInCoreOrder.length > 0 + ? (exactMatches / sharedInCoreOrder.length * 100).toFixed(1) + : 'N/A'; + console.log(` Ordering: ${orderMatchRate}% exact position match (${exactMatches}/${sharedInCoreOrder.length} shared txs)`); + + if (firstDivergence >= 0) { + const coreTx = sharedInCoreOrder[firstDivergence]; + const ourTx = sharedInOurOrder[firstDivergence]; + console.log(`\n First divergence at position ${firstDivergence}:`); + console.log(` Core wants: ${coreTx}`); + console.log(` We placed: ${ourTx} (Core has this at position ${corePos.get(ourTx)})`); + + for (const [label, txid] of [['Core tx', coreTx], ['Our tx', ourTx]] as const) { + const inOurMempool = !!this.mempool[txid]; + const info = this.clusterMempool?.getClusterInfo(txid); + console.log(`\n --- ${label}: ${txid} ---`); + console.log(` In our mempool: ${inOurMempool}`); + if (inOurMempool) { + const tx = this.mempool[txid]; + console.log(` fee=${tx.fee} weight=${tx.weight} vsize=${tx.vsize} sigops=${tx.sigops} adjustedVsize=${tx.adjustedVsize}`); + } + if (info) { + console.log(` clusterId=${info.clusterId} chunkIndex=${info.chunkIndex} chunkFeerate=${info.chunkFeerate.toFixed(6)}`); + const cluster = this.clusterMempool?.getCluster(info.clusterId); + if (cluster) { + console.log(` Cluster has ${cluster.chunks.length} chunk(s):`); + for (let ci = 0; ci < cluster.chunks.length; ci++) { + const c = cluster.chunks[ci]; + console.log(` chunk[${ci}]: ${c.txs.length} txs, feerate=${c.feerate.toFixed(6)}`); + if (ci === info.chunkIndex) { + const chunkTxids = c.txs.map((idx: number) => cluster.txs[idx]?.txid).filter(Boolean); + for (const tid of chunkTxids) { + const t = this.mempool[tid]; + const parents = t?.vin + ?.filter(v => !v.is_coinbase && this.mempool[v.txid] && chunkTxids.includes(v.txid)) + .map(v => v.txid.substring(0, 12)) || []; + console.log(` ${tid.substring(0, 12)} fee=${t?.fee} size=${(t?.weight || 0) / 4} sigops=${t?.sigops} parents=[${parents.join(', ')}]`); + } + } + } + } + } else { + console.log(` NOT in our cluster mempool`); + } + } + + for (const [label, txid] of [['Core', coreTx], ['Our', ourTx]] as const) { + try { + const coreCluster = await this.rpc.getMempoolCluster(txid); + console.log(`\n --- Core cluster for ${label} tx ${txid.substring(0, 12)} ---`); + console.log(` Raw response: ${JSON.stringify(coreCluster).substring(0, 2000)}`); + } catch (e: any) { + console.log(` Failed to get Core cluster for ${label} tx: ${e.message}`); + } + } + + this.stats.templateMismatches++; + } else if (sharedInCoreOrder.length === coreTxids.length && sharedInCoreOrder.length === ourTxids.length) { + console.log(` PERFECT MATCH`); + this.stats.templateMatches++; + } else { + console.log(` Ordering matches for shared txs, but sets differ`); + this.stats.templateMatches++; + } + } + + private printStats(): void { + console.log('\n--- Cumulative Stats ---'); + const total = this.stats.templateMatches + this.stats.templateMismatches; + console.log(`Template comparisons: ${total} (${this.stats.templateMatches} perfect, ${this.stats.templateMismatches} divergent)`); + } + + private shutdown(): void { + console.log('\nShutting down...'); + if (this.pollTimer) { + clearInterval(this.pollTimer); + } + this.printStats(); + process.exit(0); + } +} + +// ─── Entry Point ──────────────────────────────────────────────────────────── + +const rpcConfig: RpcConfig = { + host: config.CORE_RPC.HOST, + port: config.CORE_RPC.PORT, + user: config.CORE_RPC.USERNAME, + pass: config.CORE_RPC.PASSWORD, +}; + +console.log(`Connecting to Bitcoin Core at ${rpcConfig.host}:${rpcConfig.port}`); + +const harness = new Harness(rpcConfig, cliOptions); +harness.run().catch((err) => { + console.error('Harness failed:', err); + process.exit(1); +}); diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 0ca5654a5..688a9ba37 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -32,6 +32,8 @@ "AUDIT": true, "RUST_GBT": false, "LIMIT_GBT": false, + "CLUSTER_MEMPOOL": false, + "CLUSTER_MEMPOOL_INDEXING": false, "CPFP_INDEXING": true, "MAX_BLOCKS_BULK_QUERY": 999, "DISK_CACHE_BLOCK_INTERVAL": 999, diff --git a/backend/src/__tests__/cluster-mempool/cluster-mempool.test.ts b/backend/src/__tests__/cluster-mempool/cluster-mempool.test.ts new file mode 100644 index 000000000..11d99b6eb --- /dev/null +++ b/backend/src/__tests__/cluster-mempool/cluster-mempool.test.ts @@ -0,0 +1,477 @@ +import { ClusterMempool } from '../../cluster-mempool/cluster-mempool'; +import { makeTx, txid } from './test-utils'; +import { MempoolTransactionExtended } from '../../mempool.interfaces'; + +function buildMempool(txs: MempoolTransactionExtended[]): { [txid: string]: MempoolTransactionExtended } { + const mempool: { [txid: string]: MempoolTransactionExtended } = {}; + for (const tx of txs) { + mempool[tx.txid] = tx; + } + return mempool; +} + +describe('ClusterMempool', () => { + describe('constructor', () => { + it('should build clusters from mempool', () => { + const parentId = txid('a1'); + const childId = txid('a2'); + const mempool = buildMempool([ + makeTx(parentId, 100, 100), + makeTx(childId, 5000, 100, [parentId]), + ]); + const cm = new ClusterMempool(mempool); + expect(cm.getClusterCount()).toBe(1); + expect(cm.getTxCount()).toBe(2); + }); + + it('should create separate clusters for unrelated txs', () => { + const mempool = buildMempool([ + makeTx(txid('a1'), 100, 100), + makeTx(txid('b1'), 200, 100), + ]); + const cm = new ClusterMempool(mempool); + expect(cm.getClusterCount()).toBe(2); + }); + }); + + describe('getClusterInfo', () => { + it('should return cluster info for a known tx', () => { + const parentId = txid('a1'); + const childId = txid('a2'); + const mempool = buildMempool([ + makeTx(parentId, 100, 100), + makeTx(childId, 5000, 100, [parentId]), + ]); + const cm = new ClusterMempool(mempool); + const info = cm.getClusterInfo(parentId); + expect(info).not.toBeNull(); + expect(info?.chunkFeerate).toBeGreaterThan(0); + }); + + it('should return null for unknown tx', () => { + const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]); + const cm = new ClusterMempool(mempool); + expect(cm.getClusterInfo(txid('zz'))).toBeNull(); + }); + }); + + describe('getCluster', () => { + it('should return cluster data with correct topology', () => { + const parentId = txid('a1'); + const childId = txid('a2'); + const mempool = buildMempool([ + makeTx(parentId, 100, 100), + makeTx(childId, 5000, 100, [parentId]), + ]); + const cm = new ClusterMempool(mempool); + const info = cm.getClusterInfo(parentId); + expect(info).not.toBeNull(); + const data = cm.getCluster(info!.clusterId); + expect(data).not.toBeNull(); + expect(data!.txs.length).toBe(2); + expect(data!.chunks.length).toBeGreaterThan(0); + }); + + it('should return null for unknown cluster id', () => { + const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]); + const cm = new ClusterMempool(mempool); + expect(cm.getCluster(9999)).toBeNull(); + }); + }); + + describe('applyMempoolChange', () => { + it('should handle adding a new singleton tx', () => { + const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]); + const cm = new ClusterMempool(mempool); + const initialCount = cm.getClusterCount(); + + cm.applyMempoolChange({ + added: [makeTx(txid('b1'), 200, 100)], + removed: [], + accelerations: {}, + }); + + expect(cm.getClusterCount()).toBe(initialCount + 1); + expect(cm.getTxCount()).toBe(2); + }); + + it('should handle removing a tx', () => { + const parentId = txid('a1'); + const childId = txid('a2'); + const mempool = buildMempool([ + makeTx(parentId, 100, 100), + makeTx(childId, 5000, 100, [parentId]), + ]); + const cm = new ClusterMempool(mempool); + + cm.applyMempoolChange({ + added: [], + removed: [childId], + accelerations: {}, + }); + + expect(cm.getTxCount()).toBe(1); + expect(cm.getClusterInfo(childId)).toBeNull(); + expect(cm.getClusterInfo(parentId)).not.toBeNull(); + }); + + it('should split cluster when middle tx is removed', () => { + const a = txid('a1'); + const b = txid('b1'); + const c = txid('c1'); + const mempool = buildMempool([ + makeTx(a, 100, 100), + makeTx(b, 200, 100, [a]), + makeTx(c, 300, 100, [b]), + ]); + const cm = new ClusterMempool(mempool); + expect(cm.getClusterCount()).toBe(1); + + cm.applyMempoolChange({ + added: [], + removed: [b], + accelerations: {}, + }); + + expect(cm.getTxCount()).toBe(2); + const infoA = cm.getClusterInfo(a); + const infoC = cm.getClusterInfo(c); + expect(infoA).not.toBeNull(); + expect(infoC).not.toBeNull(); + expect(infoA!.clusterId).not.toBe(infoC!.clusterId); + }); + + it('should handle fee changes via acceleration', () => { + const parentId = txid('a1'); + const childId = txid('a2'); + const mempool = buildMempool([ + makeTx(parentId, 100, 100), + makeTx(childId, 100, 100, [parentId]), + ]); + const cm = new ClusterMempool(mempool); + const infoBefore = cm.getClusterInfo(childId); + + cm.applyMempoolChange({ + added: [], + removed: [], + accelerations: { [childId]: { feeDelta: 49900 } }, + }); + + const infoAfter = cm.getClusterInfo(childId); + expect(infoAfter).not.toBeNull(); + expect(infoAfter!.clusterId).not.toBe(infoBefore!.clusterId); + }); + + it('should merge clusters when new tx connects them', () => { + const a = txid('a1'); + const b = txid('b1'); + const mempool = buildMempool([ + makeTx(a, 100, 100), + makeTx(b, 200, 100), + ]); + const cm = new ClusterMempool(mempool); + expect(cm.getClusterCount()).toBe(2); + + const bridgeTx = makeTx(txid('c1'), 300, 100, [a, b]); + cm.applyMempoolChange({ + added: [bridgeTx], + removed: [], + accelerations: {}, + }); + + expect(cm.getClusterCount()).toBe(1); + expect(cm.getTxCount()).toBe(3); + }); + }); + + describe('cluster merging', () => { + it('should merge 3 separate clusters when new tx bridges them', () => { + const a = txid('a1'); + const b = txid('b1'); + const c = txid('c1'); + const mempool = buildMempool([ + makeTx(a, 100, 100), + makeTx(b, 200, 100), + makeTx(c, 300, 100), + ]); + const cm = new ClusterMempool(mempool); + expect(cm.getClusterCount()).toBe(3); + + const bridge = makeTx(txid('d1'), 400, 100, [a, b, c]); + mempool[bridge.txid] = bridge; + cm.applyMempoolChange({ added: [bridge], removed: [], accelerations: {} }); + + expect(cm.getClusterCount()).toBe(1); + expect(cm.getTxCount()).toBe(4); + }); + + it('should grow cluster by 1 when new tx has parents in same cluster', () => { + const a = txid('a1'); + const b = txid('b1'); + const mempool = buildMempool([ + makeTx(a, 100, 100), + makeTx(b, 200, 100, [a]), + ]); + const cm = new ClusterMempool(mempool); + expect(cm.getClusterCount()).toBe(1); + + const c = makeTx(txid('c1'), 300, 100, [a]); + mempool[c.txid] = c; + cm.applyMempoolChange({ added: [c], removed: [], accelerations: {} }); + + expect(cm.getClusterCount()).toBe(1); + expect(cm.getTxCount()).toBe(3); + }); + + it('should grow chain incrementally', () => { + const a = txid('a1'); + const mempool = buildMempool([makeTx(a, 100, 100)]); + const cm = new ClusterMempool(mempool); + + const b = makeTx(txid('b1'), 200, 100, [a]); + mempool[b.txid] = b; + cm.applyMempoolChange({ added: [b], removed: [], accelerations: {} }); + + const c = makeTx(txid('c1'), 300, 100, [txid('b1')]); + mempool[c.txid] = c; + cm.applyMempoolChange({ added: [c], removed: [], accelerations: {} }); + + expect(cm.getClusterCount()).toBe(1); + expect(cm.getTxCount()).toBe(3); + }); + }); + + describe('cluster splitting', () => { + it('should split star into singletons when center is removed', () => { + const center = txid('center'); + const leaves = Array.from({ length: 5 }, (_, i) => txid(`leaf${i}`)); + const centerTx = makeTx(center, 100, 100); + for (let i = 1; i < 5; i++) { + centerTx.vout.push({ + scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000, + }); + } + const leafTxs = leaves.map((l, i) => { + const tx = makeTx(l, 200, 100, [center]); + tx.vin[0].vout = i; + return tx; + }); + const mempool = buildMempool([centerTx, ...leafTxs]); + const cm = new ClusterMempool(mempool); + expect(cm.getClusterCount()).toBe(1); + + cm.applyMempoolChange({ added: [], removed: [center], accelerations: {} }); + + expect(cm.getTxCount()).toBe(5); + expect(cm.getClusterCount()).toBe(5); + }); + + it('should shrink cluster without splitting when leaf is removed', () => { + const a = txid('a1'); + const b = txid('b1'); + const c = txid('c1'); + const aTx = makeTx(a, 100, 100); + aTx.vout.push({ scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000 }); + const bTx = makeTx(b, 200, 100, [a]); + bTx.vin[0].vout = 0; + const cTx = makeTx(c, 300, 100, [a]); + cTx.vin[0].vout = 1; + const mempool = buildMempool([aTx, bTx, cTx]); + const cm = new ClusterMempool(mempool); + expect(cm.getClusterCount()).toBe(1); + + cm.applyMempoolChange({ added: [], removed: [c], accelerations: {} }); + + expect(cm.getClusterCount()).toBe(1); + expect(cm.getTxCount()).toBe(2); + }); + + it('should produce singleton when tx is removed from 2-tx cluster', () => { + const a = txid('a1'); + const b = txid('b1'); + const mempool = buildMempool([ + makeTx(a, 100, 100), + makeTx(b, 200, 100, [a]), + ]); + const cm = new ClusterMempool(mempool); + + cm.applyMempoolChange({ added: [], removed: [b], accelerations: {} }); + + expect(cm.getClusterCount()).toBe(1); + expect(cm.getTxCount()).toBe(1); + expect(cm.getClusterInfo(a)).not.toBeNull(); + }); + + it('should create 3+ components when removing a tx that bridges multiple subgraphs', () => { + const hub = txid('hub'); + const a = txid('a1'); + const b = txid('b1'); + const c = txid('c1'); + const hubTx = makeTx(hub, 100, 100); + hubTx.vout.push( + { scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000 }, + { scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000 }, + ); + const txA = makeTx(a, 200, 100, [hub]); + txA.vin[0].vout = 0; + const txB = makeTx(b, 300, 100, [hub]); + txB.vin[0].vout = 1; + const txC = makeTx(c, 400, 100, [hub]); + txC.vin[0].vout = 2; + const mempool = buildMempool([hubTx, txA, txB, txC]); + const cm = new ClusterMempool(mempool); + expect(cm.getClusterCount()).toBe(1); + + cm.applyMempoolChange({ added: [], removed: [hub], accelerations: {} }); + + expect(cm.getClusterCount()).toBe(3); + expect(cm.getTxCount()).toBe(3); + }); + }); + + describe('fee changes via accelerations', () => { + it('should increase chunk feerate when acceleration added', () => { + const a = txid('a1'); + const mempool = buildMempool([makeTx(a, 100, 100)]); + const cm = new ClusterMempool(mempool); + const infoBefore = cm.getClusterInfo(a); + expect(infoBefore).not.toBeNull(); + + cm.applyMempoolChange({ + added: [], + removed: [], + accelerations: { [a]: { feeDelta: 9900 } }, + }); + + const infoAfter = cm.getClusterInfo(a); + expect(infoAfter).not.toBeNull(); + expect(infoAfter!.chunkFeerate).toBeGreaterThan(infoBefore!.chunkFeerate); + }); + + it('should decrease chunk feerate when acceleration removed', () => { + const a = txid('a1'); + const mempool = buildMempool([makeTx(a, 100, 100)]); + const cm = new ClusterMempool(mempool, { [a]: { feeDelta: 9900 } }); + const infoBefore = cm.getClusterInfo(a); + expect(infoBefore).not.toBeNull(); + + cm.applyMempoolChange({ + added: [], + removed: [], + accelerations: {}, + }); + + const infoAfter = cm.getClusterInfo(a); + expect(infoAfter).not.toBeNull(); + expect(infoAfter!.chunkFeerate).toBeLessThan(infoBefore!.chunkFeerate); + }); + + it('should reorder chunks when acceleration shifts priorities', () => { + const a = txid('a1'); + const b = txid('b1'); + const mempool = buildMempool([ + makeTx(a, 100, 100), + makeTx(b, 200, 100, [a]), + ]); + const cm = new ClusterMempool(mempool); + const infoBBefore = cm.getClusterInfo(b); + expect(infoBBefore).not.toBeNull(); + + cm.applyMempoolChange({ + added: [], + removed: [], + accelerations: { [b]: { feeDelta: 49800 } }, + }); + + const infoBAfter = cm.getClusterInfo(b); + expect(infoBAfter).not.toBeNull(); + expect(infoBAfter!.chunkFeerate).toBeGreaterThan(infoBBefore!.chunkFeerate); + }); + }); + + describe('getBlocks', () => { + it('should return projected blocks', () => { + const txs: MempoolTransactionExtended[] = []; + for (let i = 0; i < 10; i++) { + txs.push(makeTx(txid(`t${i}`), 1000 * (i + 1), 100)); + } + const mempool = buildMempool(txs); + const cm = new ClusterMempool(mempool); + const blocks = cm.getBlocks(3); + expect(blocks.length).toBeGreaterThan(0); + expect(blocks[0].txids.length).toBeGreaterThan(0); + }); + + it('should return empty array for empty mempool', () => { + const cm = new ClusterMempool({}); + const blocks = cm.getBlocks(3); + expect(blocks.length).toBe(0); + }); + + it('should respect chunk ordering for single-cluster mempool', () => { + const a = txid('a1'); + const b = txid('b1'); + const c = txid('c1'); + const mempool = buildMempool([ + makeTx(a, 3000, 100), + makeTx(b, 200, 100, [a]), + makeTx(c, 100, 100, [b]), + ]); + const cm = new ClusterMempool(mempool); + const blocks = cm.getBlocks(1); + expect(blocks.length).toBe(1); + const txids = blocks[0].txids; + expect(txids.indexOf(a)).toBeLessThan(txids.indexOf(b)); + expect(txids.indexOf(b)).toBeLessThan(txids.indexOf(c)); + }); + + it('should maintain topological validity within blocks', () => { + const a = txid('a1'); + const b = txid('b1'); + const c = txid('c1'); + const d = txid('d1'); + const aTx = makeTx(a, 1000, 100); + aTx.vout.push({ scriptpubkey: '', scriptpubkey_asm: '', scriptpubkey_type: 'v0_p2wpkh', value: 50000 }); + const bTx = makeTx(b, 500, 100, [a]); + bTx.vin[0].vout = 0; + const cTx = makeTx(c, 500, 100, [a]); + cTx.vin[0].vout = 1; + const mempool = buildMempool([aTx, bTx, cTx, makeTx(d, 200, 100, [b, c])]); + const cm = new ClusterMempool(mempool); + const blocks = cm.getBlocks(1); + const txids = blocks[0].txids; + + expect(txids.indexOf(a)).toBeLessThan(txids.indexOf(b)); + expect(txids.indexOf(a)).toBeLessThan(txids.indexOf(c)); + expect(txids.indexOf(b)).toBeLessThan(txids.indexOf(d)); + expect(txids.indexOf(c)).toBeLessThan(txids.indexOf(d)); + }); + }); + + describe('empty and degenerate cases', () => { + it('should handle empty diff with no changes', () => { + const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]); + const cm = new ClusterMempool(mempool); + const countBefore = cm.getClusterCount(); + const txCountBefore = cm.getTxCount(); + + cm.applyMempoolChange({ added: [], removed: [], accelerations: {} }); + + expect(cm.getClusterCount()).toBe(countBefore); + expect(cm.getTxCount()).toBe(txCountBefore); + }); + + it('should not crash when removing nonexistent tx', () => { + const mempool = buildMempool([makeTx(txid('a1'), 100, 100)]); + const cm = new ClusterMempool(mempool); + + cm.applyMempoolChange({ + added: [], + removed: [txid('nonexistent')], + accelerations: {}, + }); + + expect(cm.getTxCount()).toBe(1); + }); + }); +}); diff --git a/backend/src/__tests__/cluster-mempool/depgraph.test.ts b/backend/src/__tests__/cluster-mempool/depgraph.test.ts new file mode 100644 index 000000000..8fd9c48f1 --- /dev/null +++ b/backend/src/__tests__/cluster-mempool/depgraph.test.ts @@ -0,0 +1,494 @@ +import { DepGraph, sortTopological, subgraph } from '../../cluster-mempool/depgraph'; +import { buildChain, buildFanOut, buildDiamond, buildStar } from './test-utils'; + +describe('DepGraph', () => { + describe('addTransaction', () => { + it('should add a transaction and return a ClusterTx', () => { + const dg = new DepGraph(); + const tx = dg.addTransaction('tx0', 1000, 100); + expect(dg.size).toBe(1); + expect(tx.effectiveFee).toBe(1000); + expect(tx.weight).toBe(100); + expect(tx.txid).toBe('tx0'); + }); + + it('should assign distinct ClusterTx objects', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + const c = dg.addTransaction('c', 300, 30); + expect(a).not.toBe(b); + expect(b).not.toBe(c); + expect(dg.size).toBe(3); + }); + + it('should include self in ancestors and descendants', () => { + const dg = new DepGraph(); + const tx = dg.addTransaction('tx0', 100, 10); + expect(tx.ancestors.has(tx)).toBe(true); + expect(tx.descendants.has(tx)).toBe(true); + }); + + it('should handle large clusters', () => { + const dg = new DepGraph(); + for (let i = 0; i < 100; i++) { + dg.addTransaction(`tx${i}`, 100, 10); + } + expect(dg.size).toBe(100); + }); + }); + + describe('addDependency', () => { + it('should establish parent-child relationship', () => { + const dg = new DepGraph(); + const parent = dg.addTransaction('parent', 100, 10); + const child = dg.addTransaction('child', 200, 20); + dg.addDependency(parent, child); + + expect(child.ancestors.has(parent)).toBe(true); + expect(parent.descendants.has(child)).toBe(true); + expect(child.parents.has(parent)).toBe(true); + expect(parent.children.has(child)).toBe(true); + }); + + it('should propagate ancestors transitively', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + const c = dg.addTransaction('c', 300, 30); + dg.addDependency(a, b); + dg.addDependency(b, c); + + expect(c.ancestors.has(a)).toBe(true); + expect(c.ancestors.has(b)).toBe(true); + expect(c.ancestors.has(c)).toBe(true); + + expect(a.descendants.has(a)).toBe(true); + expect(a.descendants.has(b)).toBe(true); + expect(a.descendants.has(c)).toBe(true); + }); + + it('should handle diamond dependencies', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + const c = dg.addTransaction('c', 300, 30); + const d = dg.addTransaction('d', 400, 40); + dg.addDependency(a, b); + dg.addDependency(a, c); + dg.addDependency(b, d); + dg.addDependency(c, d); + + expect(d.ancestors.size).toBe(4); + expect(a.descendants.size).toBe(4); + }); + }); + + describe('removeTransactions', () => { + it('should remove transactions and update sets', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + const c = dg.addTransaction('c', 300, 30); + dg.addDependency(a, b); + dg.addDependency(b, c); + + dg.removeTransactions(new Set([b])); + expect(dg.size).toBe(2); + expect(dg.hasTx(b)).toBe(false); + expect(c.ancestors.has(b)).toBe(false); + expect(a.descendants.has(b)).toBe(false); + }); + + it('should handle slot reuse after removal', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + dg.addTransaction('b', 200, 20); + dg.removeTransactions(new Set([a])); + const c = dg.addTransaction('c', 300, 30); + expect(dg.size).toBe(2); + expect(c.txid).toBe('c'); + }); + }); + + describe('dependsOn (via ancestors)', () => { + it('should correctly identify dependencies', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + const c = dg.addTransaction('c', 300, 30); + dg.addDependency(a, b); + + expect(b.ancestors.has(a)).toBe(true); + expect(a.ancestors.has(b)).toBe(false); + expect(c.ancestors.has(a)).toBe(false); + }); + }); + + describe('findConnectedComponents', () => { + it('should find a single component', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + dg.addDependency(a, b); + + const components = dg.findConnectedComponents(); + expect(components.length).toBe(1); + expect(components[0].size).toBe(2); + }); + + it('should find multiple disconnected components', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + const c = dg.addTransaction('c', 300, 30); + const d = dg.addTransaction('d', 400, 40); + dg.addDependency(a, b); + dg.addDependency(c, d); + + const components = dg.findConnectedComponents(); + expect(components.length).toBe(2); + }); + + it('should handle isolated transactions', () => { + const dg = new DepGraph(); + dg.addTransaction('a', 100, 10); + dg.addTransaction('b', 200, 20); + dg.addTransaction('c', 300, 30); + + const components = dg.findConnectedComponents(); + expect(components.length).toBe(3); + }); + }); + + describe('parents / children (direct)', () => { + it('should return only direct parents, not transitive', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + const c = dg.addTransaction('c', 300, 30); + dg.addDependency(a, b); + dg.addDependency(b, c); + + expect(c.parents.has(b)).toBe(true); + expect(c.parents.has(a)).toBe(false); + }); + + it('should return only direct children', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + const c = dg.addTransaction('c', 300, 30); + dg.addDependency(a, b); + dg.addDependency(b, c); + + expect(a.children.has(b)).toBe(true); + expect(a.children.has(c)).toBe(false); + }); + }); + + describe('appendTopo', () => { + it('should output in topological order', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + const c = dg.addTransaction('c', 300, 30); + dg.addDependency(a, b); + dg.addDependency(b, c); + + const output = sortTopological(new Set([c, a, b])); + expect(output.indexOf(a)).toBeLessThan(output.indexOf(b)); + expect(output.indexOf(b)).toBeLessThan(output.indexOf(c)); + }); + }); + + describe('restrict', () => { + it('should create a subgraph with correct deps', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + dg.addTransaction('c', 300, 30); + dg.addDependency(a, b); + + const { depgraph: sub, txMap } = subgraph(new Set([a, b])); + expect(sub.size).toBe(2); + const newA = txMap.get(a)!; + const newB = txMap.get(b)!; + expect(newB.ancestors.has(newA)).toBe(true); + }); + }); + + describe('graceful error handling', () => { + it('should no-op addDependency with non-member txs', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const dg2 = new DepGraph(); + const foreign = dg2.addTransaction('foreign', 200, 20); + dg.addDependency(a, foreign); + dg.addDependency(foreign, a); + expect(a.ancestors.size).toBe(1); + }); + }); + + describe('deep chain topology', () => { + it('should track ancestors and descendants at each depth', () => { + const { depgraph, txs } = buildChain(20, 100, 10); + expect(depgraph.size).toBe(20); + + expect(txs[19].ancestors.size).toBe(20); + expect(txs[0].descendants.size).toBe(20); + + expect(txs[10].ancestors.size).toBe(11); + expect(txs[10].descendants.size).toBe(10); + + expect(txs[10].parents.size).toBe(1); + expect(txs[10].parents.has(txs[9])).toBe(true); + expect(txs[10].children.size).toBe(1); + expect(txs[10].children.has(txs[11])).toBe(true); + }); + }); + + describe('wide fan-out topology', () => { + it('should track parent-children relationships for 1 parent with 10 children', () => { + const { depgraph, parent, children } = buildFanOut(10, 100, 10, 50, 10); + expect(depgraph.size).toBe(11); + expect(parent.children.size).toBe(10); + expect(parent.descendants.size).toBe(11); + + for (const child of children) { + expect(child.ancestors.size).toBe(2); + expect(child.ancestors.has(parent)).toBe(true); + expect(child.parents.size).toBe(1); + expect(child.parents.has(parent)).toBe(true); + } + }); + }); + + describe('wide fan-in topology', () => { + it('should track many parents converging to one child', () => { + const dg = new DepGraph(); + const parents: any[] = []; + for (let i = 0; i < 10; i++) { + parents.push(dg.addTransaction(`p${i}`, 100, 10)); + } + const child = dg.addTransaction('child', 500, 50); + for (const p of parents) { + dg.addDependency(p, child); + } + + expect(child.parents.size).toBe(10); + expect(child.ancestors.size).toBe(11); + for (const p of parents) { + expect(p.descendants.has(child)).toBe(true); + } + }); + }); + + describe('multiple diamonds in sequence', () => { + it('should handle A→B,C→D→E,F→G topology', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 100, 10); + const c = dg.addTransaction('c', 100, 10); + const d = dg.addTransaction('d', 100, 10); + const e = dg.addTransaction('e', 100, 10); + const f = dg.addTransaction('f', 100, 10); + const g = dg.addTransaction('g', 100, 10); + dg.addDependency(a, b); + dg.addDependency(a, c); + dg.addDependency(b, d); + dg.addDependency(c, d); + dg.addDependency(d, e); + dg.addDependency(d, f); + dg.addDependency(e, g); + dg.addDependency(f, g); + + expect(g.ancestors.size).toBe(7); + expect(a.descendants.size).toBe(7); + expect(d.parents.size).toBe(2); + expect(d.children.size).toBe(2); + }); + }); + + describe('disconnected subgraphs', () => { + it('should coexist in one DepGraph', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + dg.addDependency(a, b); + + const c = dg.addTransaction('c', 300, 30); + const d = dg.addTransaction('d', 400, 40); + dg.addDependency(c, d); + + const e = dg.addTransaction('e', 500, 50); + + expect(dg.size).toBe(5); + expect(b.ancestors.has(c)).toBe(false); + expect(a.descendants.has(d)).toBe(false); + expect(e.ancestors.size).toBe(1); + expect(dg.findConnectedComponents().length).toBe(3); + }); + }); + + describe('removeTransactions edge cases', () => { + it('should remove a leaf without affecting siblings', () => { + const { depgraph, parent, children } = buildFanOut(3, 100, 10, 50, 10); + depgraph.removeTransactions(new Set([children[2]])); + expect(depgraph.size).toBe(3); + expect(parent.children.size).toBe(2); + expect(depgraph.hasTx(children[0])).toBe(true); + expect(depgraph.hasTx(children[1])).toBe(true); + }); + + it('should remove a root without affecting unrelated txs', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + dg.addDependency(a, b); + dg.removeTransactions(new Set([a])); + expect(dg.size).toBe(1); + expect(dg.hasTx(a)).toBe(false); + expect(b.ancestors.size).toBe(1); + expect(b.ancestors.has(b)).toBe(true); + }); + + it('should break transitive edges when middle of chain is removed', () => { + const { depgraph, txs } = buildChain(5, 100, 10); + depgraph.removeTransactions(new Set([txs[2]])); + expect(depgraph.size).toBe(4); + expect(txs[0].descendants.has(txs[3])).toBe(false); + expect(txs[0].descendants.has(txs[1])).toBe(true); + expect(txs[3].ancestors.has(txs[0])).toBe(false); + expect(txs[3].descendants.has(txs[4])).toBe(true); + }); + + it('should handle batch removal of multiple txs', () => { + const { depgraph, txs } = buildChain(5, 100, 10); + depgraph.removeTransactions(new Set([txs[1], txs[3]])); + expect(depgraph.size).toBe(3); + expect(depgraph.hasTx(txs[0])).toBe(true); + expect(depgraph.hasTx(txs[2])).toBe(true); + expect(depgraph.hasTx(txs[4])).toBe(true); + }); + + it('should result in empty graph when all txs removed', () => { + const { depgraph, txs } = buildChain(3, 100, 10); + depgraph.removeTransactions(new Set(txs)); + expect(depgraph.size).toBe(0); + expect(depgraph.getTxs().size).toBe(0); + }); + + it('should produce clean state when new tx is added after removal', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + dg.addDependency(a, b); + dg.removeTransactions(new Set([a])); + + const c = dg.addTransaction('c', 300, 30); + expect(c.ancestors.size).toBe(1); + expect(c.descendants.size).toBe(1); + expect(c.ancestors.has(c)).toBe(true); + expect(b.ancestors.has(c)).toBe(false); + }); + }); + + describe('findConnectedComponents after removal', () => { + it('should split into 2 components when bridge tx is removed', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + const c = dg.addTransaction('c', 300, 30); + const d = dg.addTransaction('d', 400, 40); + dg.addDependency(a, b); + dg.addDependency(b, c); + dg.addDependency(d, c); + + dg.removeTransactions(new Set([b])); + const components = dg.findConnectedComponents(); + expect(components.length).toBe(2); + const sizes = components.map(comp => comp.size).sort((x, y) => x - y); + expect(sizes).toEqual([1, 2]); + }); + + it('should produce N singletons when center of star is removed', () => { + const { depgraph, center, leaves } = buildStar(5, 100, 10, 50, 10); + depgraph.removeTransactions(new Set([center])); + const components = depgraph.findConnectedComponents(); + expect(components.length).toBe(5); + for (const comp of components) { + expect(comp.size).toBe(1); + } + }); + + it('should remain 1 component when non-bridge tx is removed', () => { + const { depgraph, txs } = buildDiamond([100, 200, 300, 400], [10, 20, 30, 40]); + depgraph.removeTransactions(new Set([txs[1]])); + const components = depgraph.findConnectedComponents(); + expect(components.length).toBe(1); + }); + }); + + describe('restrict edge cases', () => { + it('should restrict to a single tx', () => { + const { depgraph, txs } = buildChain(3, 100, 10); + const { depgraph: sub } = subgraph(new Set([txs[1]])); + expect(sub.size).toBe(1); + }); + + it('should preserve edges within partial chain subset', () => { + const { depgraph, txs } = buildChain(5, 100, 10); + const subset = new Set([txs[0], txs[1], txs[2]]); + const { depgraph: sub, txMap } = subgraph(subset); + expect(sub.size).toBe(3); + + const newA = txMap.get(txs[0]); + const newB = txMap.get(txs[1]); + const newC = txMap.get(txs[2]); + if (!newA || !newB || !newC) { + throw new Error('txMap missing entries'); + } + expect(newB.ancestors.has(newA)).toBe(true); + expect(newC.ancestors.has(newB)).toBe(true); + }); + }); + + describe('sortTopological edge cases', () => { + it('should handle subset with no internal dependencies', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + const c = dg.addTransaction('c', 300, 30); + + const output = sortTopological(new Set([a, b, c])); + expect(output.length).toBe(3); + expect(new Set(output).size).toBe(3); + }); + + it('should handle single-tx subset', () => { + const { txs } = buildChain(3, 100, 10); + const output = sortTopological(new Set([txs[1]])); + expect(output).toEqual([txs[1]]); + }); + }); + + describe('addDependency edge cases', () => { + it('should be idempotent when adding the same dependency twice', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + dg.addDependency(a, b); + dg.addDependency(a, b); + expect(b.ancestors.size).toBe(2); + expect(a.descendants.size).toBe(2); + }); + + it('should handle redundant edge when parent is already transitive ancestor', () => { + const { depgraph, txs } = buildChain(3, 100, 10); + depgraph.addDependency(txs[0], txs[2]); + expect(txs[2].ancestors.size).toBe(3); + expect(txs[2].parents.size).toBe(2); + }); + }); +}); diff --git a/backend/src/__tests__/cluster-mempool/linearize.test.ts b/backend/src/__tests__/cluster-mempool/linearize.test.ts new file mode 100644 index 000000000..1371c59c2 --- /dev/null +++ b/backend/src/__tests__/cluster-mempool/linearize.test.ts @@ -0,0 +1,485 @@ +import { DepGraph } from '../../cluster-mempool/depgraph'; +import { chunkify, postLinearize, spanningForestLinearize, linearizeCluster } from '../../cluster-mempool/linearize'; +import { buildChain, buildFanOut, buildStar, verifyLinearization, verifyTopologicalOrder } from './test-utils'; + +describe('chunkify', () => { + it('should create one chunk per tx when feerates are decreasing', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 300, 10); + const b = dg.addTransaction('b', 200, 10); + const c = dg.addTransaction('c', 100, 10); + + const chunks = chunkify([a, b, c]); + expect(chunks.length).toBe(3); + expect(chunks[0].txs).toEqual([a]); + expect(chunks[1].txs).toEqual([b]); + expect(chunks[2].txs).toEqual([c]); + }); + + it('should merge all into one chunk when feerates are increasing', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 10); + const c = dg.addTransaction('c', 300, 10); + + const chunks = chunkify([a, b, c]); + expect(chunks.length).toBe(1); + expect(chunks[0].txs).toEqual([a, b, c]); + expect(chunks[0].fee).toBe(600); + expect(chunks[0].weight).toBe(30); + }); + + it('should NOT merge equal feerates (matching Core behavior)', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 100, 10); + + const chunks = chunkify([a, b]); + expect(chunks.length).toBe(2); + }); + + it('should handle single transaction', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 500, 50); + + const chunks = chunkify([a]); + expect(chunks.length).toBe(1); + expect(chunks[0].fee).toBe(500); + expect(chunks[0].weight).toBe(50); + }); + + it('should handle empty linearization', () => { + const chunks = chunkify([]); + expect(chunks.length).toBe(0); + }); + + it('should produce decreasing chunk feerates', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 50, 10); + const c = dg.addTransaction('c', 200, 10); + const d = dg.addTransaction('d', 30, 10); + + const chunks = chunkify([a, c, b, d]); + for (let i = 1; i < chunks.length; i++) { + const prevRate = chunks[i - 1].fee / chunks[i - 1].weight; + const curRate = chunks[i].fee / chunks[i].weight; + expect(prevRate).toBeGreaterThanOrEqual(curRate); + } + }); +}); + +describe('postLinearize', () => { + it('should improve a bad linearization', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 100); + const b = dg.addTransaction('b', 500, 100); + + const result = postLinearize([a, b]); + expect(result[0]).toBe(b); + expect(result[1]).toBe(a); + }); + + it('should not violate dependencies', () => { + const dg = new DepGraph(); + const parent = dg.addTransaction('parent', 100, 100); + const child = dg.addTransaction('child', 500, 100); + dg.addDependency(parent, child); + + const result = postLinearize([parent, child]); + expect(result.indexOf(parent)).toBeLessThan(result.indexOf(child)); + }); + + it('should handle already-optimal ordering', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 500, 100); + const b = dg.addTransaction('b', 100, 100); + + const result = postLinearize([a, b]); + expect(result).toEqual([a, b]); + }); +}); + +describe('spanningForestLinearize', () => { + it('should sort independent transactions by feerate', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 100); + const b = dg.addTransaction('b', 500, 100); + const c = dg.addTransaction('c', 300, 100); + + const result = spanningForestLinearize(dg.getTxs(), 75000); + expect(result[0]).toBe(b); + expect(result[1]).toBe(c); + expect(result[2]).toBe(a); + }); + + it('should respect dependencies', () => { + const dg = new DepGraph(); + const parent = dg.addTransaction('parent', 100, 100); + const child = dg.addTransaction('child', 500, 100); + dg.addDependency(parent, child); + + const result = spanningForestLinearize(dg.getTxs(), 75000); + expect(result.indexOf(parent)).toBeLessThan(result.indexOf(child)); + }); + + it('should handle CPFP pattern', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 100); + const b = dg.addTransaction('b', 10000, 100); + dg.addDependency(a, b); + + const { chunks } = linearizeCluster(dg.getTxs(), 75000); + expect(chunks.length).toBe(1); + expect(chunks[0].txs.length).toBe(2); + }); + + it('should separate high and low feerate independent txs', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 1000, 100); + const b = dg.addTransaction('b', 100, 100); + + const { chunks } = linearizeCluster(dg.getTxs(), 75000); + expect(chunks.length).toBe(2); + expect(chunks[0].txs).toContain(a); + expect(chunks[1].txs).toContain(b); + }); + + it('should handle single transaction', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 1000, 100); + + const result = spanningForestLinearize(dg.getTxs(), 75000); + expect(result).toEqual([a]); + }); + + it('should handle empty graph', () => { + const dg = new DepGraph(); + const result = spanningForestLinearize(dg.getTxs(), 75000); + expect(result).toEqual([]); + }); +}); + +describe('minimize', () => { + it('should keep equal-feerate chain as individual chunks', () => { + const dg = new DepGraph(); + const txs: any[] = []; + for (let i = 0; i < 5; i++) { + txs.push(dg.addTransaction(`tx${i}`, 19, 140)); + } + for (let i = 1; i < 5; i++) { + dg.addDependency(txs[i - 1], txs[i]); + } + + const { chunks } = linearizeCluster(dg.getTxs(), 75000); + expect(chunks.length).toBe(5); + for (const chunk of chunks) { + expect(chunk.txs.length).toBe(1); + } + }); + + it('should merge chain where child has strictly higher feerate', () => { + const dg = new DepGraph(); + const parent = dg.addTransaction('parent', 100, 200); + const child = dg.addTransaction('child', 900, 100); + dg.addDependency(parent, child); + + const { chunks } = linearizeCluster(dg.getTxs(), 75000); + expect(chunks.length).toBe(1); + expect(chunks[0].txs.length).toBe(2); + }); + + it('should split parent-child with equal feerate', () => { + const dg = new DepGraph(); + const parent = dg.addTransaction('parent', 1720, 344); + const child = dg.addTransaction('child', 1240, 248); + dg.addDependency(parent, child); + + const { chunks } = linearizeCluster(dg.getTxs(), 75000); + expect(chunks.length).toBe(2); + }); + + it('should split disconnected equal-feerate components', () => { + const dg = new DepGraph(); + dg.addTransaction('a', 100, 100); + dg.addTransaction('b', 100, 100); + + const { chunks } = linearizeCluster(dg.getTxs(), 75000); + expect(chunks.length).toBe(2); + expect(chunks[0].txs.length).toBe(1); + expect(chunks[1].txs.length).toBe(1); + }); +}); + +describe('chunkify edge cases', () => { + it('should produce N separate chunks when all feerates are equal', () => { + const dg = new DepGraph(); + const txs: any[] = []; + for (let i = 0; i < 5; i++) { + txs.push(dg.addTransaction(`tx${i}`, 100, 10)); + } + const chunks = chunkify(txs); + expect(chunks.length).toBe(5); + }); + + it('should handle alternating high/low feerates', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 1000, 10); + const b = dg.addTransaction('b', 100, 10); + const c = dg.addTransaction('c', 1000, 10); + const d = dg.addTransaction('d', 100, 10); + + const chunks = chunkify([a, b, c, d]); + for (let i = 1; i < chunks.length; i++) { + const prevRate = chunks[i - 1].fee * chunks[i].weight; + const curRate = chunks[i].fee * chunks[i - 1].weight; + expect(prevRate).toBeGreaterThanOrEqual(curRate); + } + }); + + it('should merge all when single very high feerate tx is at the end', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 10, 10); + const b = dg.addTransaction('b', 10, 10); + const c = dg.addTransaction('c', 10, 10); + const d = dg.addTransaction('d', 10000, 10); + + const chunks = chunkify([a, b, c, d]); + expect(chunks.length).toBe(1); + expect(chunks[0].txs.length).toBe(4); + }); + + it('should maintain non-increasing feerates for 50+ tx linearization', () => { + const dg = new DepGraph(); + const txs: any[] = []; + for (let i = 0; i < 50; i++) { + txs.push(dg.addTransaction(`tx${i}`, 5000 - i * 100, 100)); + } + const chunks = chunkify(txs); + for (let i = 1; i < chunks.length; i++) { + const prevRate = chunks[i - 1].fee * chunks[i].weight; + const curRate = chunks[i].fee * chunks[i - 1].weight; + expect(prevRate).toBeGreaterThanOrEqual(curRate); + } + }); + + it('should handle zero-fee transaction', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 1000, 100); + const b = dg.addTransaction('b', 0, 100); + + const chunks = chunkify([a, b]); + expect(chunks.length).toBe(2); + expect(chunks[1].fee).toBe(0); + }); +}); + +describe('postLinearize edge cases', () => { + it('should sort three independent txs by feerate', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 100); + const b = dg.addTransaction('b', 500, 100); + const c = dg.addTransaction('c', 300, 100); + + const result = postLinearize([a, c, b]); + expect(result[0]).toBe(b); + expect(result[2]).toBe(a); + }); + + it('should respect parent-child dependency even when child has higher feerate', () => { + const dg = new DepGraph(); + const parent = dg.addTransaction('parent', 100, 100); + const child = dg.addTransaction('child', 1000, 100); + dg.addDependency(parent, child); + + const result = postLinearize([parent, child]); + expect(result[0]).toBe(parent); + expect(result[1]).toBe(child); + }); + + it('should handle chain A→B→C with CPFP-like feerates', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 100); + const b = dg.addTransaction('b', 200, 100); + const c = dg.addTransaction('c', 10000, 100); + dg.addDependency(a, b); + dg.addDependency(b, c); + + const result = postLinearize([a, b, c]); + verifyTopologicalOrder(result); + }); +}); + +describe('SFL adversarial topologies', () => { + it('should handle comb pattern: one root with many children at different feerates', () => { + const dg = new DepGraph(); + const root = dg.addTransaction('root', 100, 100); + for (let i = 0; i < 8; i++) { + const child = dg.addTransaction(`child${i}`, (i + 1) * 500, 100); + dg.addDependency(root, child); + } + + const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000); + verifyLinearization(dg.getTxs(), linearization, chunks); + }); + + it('should handle inverted tree: many leaves → intermediates → root', () => { + const dg = new DepGraph(); + const root = dg.addTransaction('root', 100, 100); + const mid1 = dg.addTransaction('mid1', 200, 100); + const mid2 = dg.addTransaction('mid2', 300, 100); + dg.addDependency(root, mid1); + dg.addDependency(root, mid2); + + for (let i = 0; i < 4; i++) { + const leaf = dg.addTransaction(`leaf${i}`, 5000, 100); + dg.addDependency(i < 2 ? mid1 : mid2, leaf); + } + + const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000); + verifyLinearization(dg.getTxs(), linearization, chunks); + }); + + it('should handle two parallel chains with shared root', () => { + const dg = new DepGraph(); + const root = dg.addTransaction('root', 100, 100); + + let prev1: any = root; + for (let i = 0; i < 5; i++) { + const tx = dg.addTransaction(`chain1_${i}`, 200, 100); + dg.addDependency(prev1, tx); + prev1 = tx; + } + + let prev2: any = root; + for (let i = 0; i < 3; i++) { + const tx = dg.addTransaction(`chain2_${i}`, 300, 100); + dg.addDependency(prev2, tx); + prev2 = tx; + } + + const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000); + verifyLinearization(dg.getTxs(), linearization, chunks); + }); + + it('should handle deep CPFP: low-fee chain with high-fee tip', () => { + const { depgraph, txs } = buildChain(6, 10, 100); + txs[5].effectiveFee = 50000; + + const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), 75000); + verifyLinearization(depgraph.getTxs(), linearization, chunks); + expect(chunks[0].txs.length).toBeGreaterThan(1); + }); + + it('should find better result than ancestor-feerate for overlapping high-feerate subsets', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 100); + const b = dg.addTransaction('b', 100, 100); + const c = dg.addTransaction('c', 10000, 100); + dg.addDependency(a, c); + dg.addDependency(b, c); + + const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000 ); + verifyLinearization(dg.getTxs(), linearization, chunks); + const firstChunkFee = chunks[0].fee; + const firstChunkSize = chunks[0].weight; + expect(firstChunkFee / firstChunkSize).toBeGreaterThan(100 / 100); + }); +}); + +describe('linearizeCluster', () => { + it('should produce valid topological linearization', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 10); + const b = dg.addTransaction('b', 200, 20); + const c = dg.addTransaction('c', 300, 30); + dg.addDependency(a, b); + dg.addDependency(b, c); + + const { linearization } = linearizeCluster(dg.getTxs(), 75000); + expect(linearization.indexOf(a)).toBeLessThan(linearization.indexOf(b)); + expect(linearization.indexOf(b)).toBeLessThan(linearization.indexOf(c)); + }); + + it('should produce monotonically decreasing chunk feerates', () => { + const dg = new DepGraph(); + for (let i = 0; i < 10; i++) { + dg.addTransaction(`tx${i}`, Math.floor(Math.random() * 10000) + 100, Math.floor(Math.random() * 500) + 50); + } + + const { chunks } = linearizeCluster(dg.getTxs(), 75000); + for (let i = 1; i < chunks.length; i++) { + const prevRate = chunks[i - 1].fee * chunks[i].weight; + const curRate = chunks[i].fee * chunks[i - 1].weight; + expect(prevRate).toBeGreaterThanOrEqual(curRate); + } + }); + + it('should handle complex diamond dependency graph', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 1000, 100); + const b = dg.addTransaction('b', 500, 100); + const c = dg.addTransaction('c', 100, 100); + const d = dg.addTransaction('d', 300, 100); + + dg.addDependency(a, b); + dg.addDependency(a, c); + dg.addDependency(b, d); + dg.addDependency(c, d); + + const { linearization, chunks } = linearizeCluster(dg.getTxs(), 75000); + verifyLinearization(dg.getTxs(), linearization, chunks); + }); + + it('should produce at-least-as-good result when given suboptimal hint', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 500, 100); + const b = dg.addTransaction('b', 100, 100); + const c = dg.addTransaction('c', 1000, 100); + + const suboptimal = [b, a, c]; + const { chunks: hintChunks } = linearizeCluster(dg.getTxs(), 75000, suboptimal); + const { chunks: freshChunks } = linearizeCluster(dg.getTxs(), 75000); + + const hintFirstFeerate = hintChunks[0].fee * freshChunks[0].weight; + const freshFirstFeerate = freshChunks[0].fee * hintChunks[0].weight; + expect(hintFirstFeerate).toBeGreaterThanOrEqual(freshFirstFeerate - 1); + }); + + it('should preserve an already-optimal linearization', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 1000, 100); + const b = dg.addTransaction('b', 500, 100); + const c = dg.addTransaction('c', 100, 100); + + const optimal = [a, b, c]; + const { linearization } = linearizeCluster(dg.getTxs(), 75000, optimal); + expect(linearization).toEqual(optimal); + }); + + it('should produce valid linearizations on repeated calls', () => { + const dg = new DepGraph(); + const a = dg.addTransaction('a', 100, 100); + const b = dg.addTransaction('b', 100, 100); + const c = dg.addTransaction('c', 100, 100); + dg.addDependency(a, c); + dg.addDependency(b, c); + + const result1 = linearizeCluster(dg.getTxs(), 75000); + const result2 = linearizeCluster(dg.getTxs(), 75000); + verifyLinearization(dg.getTxs(), result1.linearization, result1.chunks); + verifyLinearization(dg.getTxs(), result2.linearization, result2.chunks); + }); + + it('should pass invariant checks for fan-out topology', () => { + const { depgraph } = buildFanOut(6, 100, 100, 500, 100); + const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), 75000); + verifyLinearization(depgraph.getTxs(), linearization, chunks); + }); + + it('should pass invariant checks for star topology', () => { + const { depgraph } = buildStar(5, 100, 100, 300, 100); + const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), 75000); + verifyLinearization(depgraph.getTxs(), linearization, chunks); + }); +}); diff --git a/backend/src/__tests__/cluster-mempool/test-utils.ts b/backend/src/__tests__/cluster-mempool/test-utils.ts new file mode 100644 index 000000000..c2cf18ba8 --- /dev/null +++ b/backend/src/__tests__/cluster-mempool/test-utils.ts @@ -0,0 +1,175 @@ +import { MempoolTransactionExtended } from '../../mempool.interfaces'; +import { ClusterTx, DepGraph } from '../../cluster-mempool/depgraph'; +import { LinearizationChunk } from '../../cluster-mempool/linearize'; + +export function makeTx( + txid: string, + fee: number, + vsize: number, + parentTxids: string[] = [], +): MempoolTransactionExtended { + const vin = parentTxids.length > 0 + ? parentTxids.map(ptxid => ({ + txid: ptxid, + vout: 0, + is_coinbase: false, + scriptsig: '', + scriptsig_asm: '', + inner_redeemscript_asm: '', + inner_witnessscript_asm: '', + sequence: 0, + witness: [] as string[], + prevout: null, + })) + : [{ + txid: '0000000000000000000000000000000000000000000000000000000000000000', + vout: 0, + is_coinbase: false, + scriptsig: '', + scriptsig_asm: '', + inner_redeemscript_asm: '', + inner_witnessscript_asm: '', + sequence: 0, + witness: [] as string[], + prevout: null, + }]; + + return { + txid, + version: 2, + locktime: 0, + size: vsize, + weight: vsize * 4, + fee, + vin, + vout: [{ + scriptpubkey: '', + scriptpubkey_asm: '', + scriptpubkey_type: 'v0_p2wpkh', + value: 50000, + }], + status: { confirmed: false }, + vsize, + feePerVsize: fee / vsize, + effectiveFeePerVsize: fee / vsize, + order: 0, + sigops: 0, + adjustedVsize: vsize, + adjustedFeePerVsize: fee / vsize, + } as MempoolTransactionExtended; +} + +export function txid(short: string): string { + return short.padStart(64, '0'); +} + +export function buildChain( + n: number, + baseFee: number, + baseSize: number, +): { depgraph: DepGraph; txs: ClusterTx[] } { + const depgraph = new DepGraph(); + const txs: ClusterTx[] = []; + for (let i = 0; i < n; i++) { + txs.push(depgraph.addTransaction(`chain_${i}`, baseFee, baseSize)); + } + for (let i = 1; i < n; i++) { + depgraph.addDependency(txs[i - 1], txs[i]); + } + return { depgraph, txs }; +} + +export function buildFanOut( + nChildren: number, + parentFee: number, + parentSize: number, + childFee: number, + childSize: number, +): { depgraph: DepGraph; parent: ClusterTx; children: ClusterTx[] } { + const depgraph = new DepGraph(); + const parent = depgraph.addTransaction('fanout_parent', parentFee, parentSize); + const children: ClusterTx[] = []; + for (let i = 0; i < nChildren; i++) { + const child = depgraph.addTransaction(`fanout_child_${i}`, childFee, childSize); + depgraph.addDependency(parent, child); + children.push(child); + } + return { depgraph, parent, children }; +} + +export function buildDiamond( + fees: [number, number, number, number], + sizes: [number, number, number, number], +): { depgraph: DepGraph; txs: [ClusterTx, ClusterTx, ClusterTx, ClusterTx] } { + const depgraph = new DepGraph(); + const a = depgraph.addTransaction('diamond_a', fees[0], sizes[0]); + const b = depgraph.addTransaction('diamond_b', fees[1], sizes[1]); + const c = depgraph.addTransaction('diamond_c', fees[2], sizes[2]); + const d = depgraph.addTransaction('diamond_d', fees[3], sizes[3]); + depgraph.addDependency(a, b); + depgraph.addDependency(a, c); + depgraph.addDependency(b, d); + depgraph.addDependency(c, d); + return { depgraph, txs: [a, b, c, d] }; +} + +export function buildStar( + nLeaves: number, + centerFee: number, + centerSize: number, + leafFee: number, + leafSize: number, +): { depgraph: DepGraph; center: ClusterTx; leaves: ClusterTx[] } { + const depgraph = new DepGraph(); + const center = depgraph.addTransaction('star_center', centerFee, centerSize); + const leaves: ClusterTx[] = []; + for (let i = 0; i < nLeaves; i++) { + const leaf = depgraph.addTransaction(`star_leaf_${i}`, leafFee, leafSize); + depgraph.addDependency(center, leaf); + leaves.push(leaf); + } + return { depgraph, center, leaves }; +} + +export function verifyTopologicalOrder(ordering: ClusterTx[]): void { + const positionMap = new Map(); + for (let i = 0; i < ordering.length; i++) { + positionMap.set(ordering[i], i); + } + for (const tx of ordering) { + for (const parent of tx.parents) { + const parentPos = positionMap.get(parent); + const childPos = positionMap.get(tx); + if (parentPos !== undefined && childPos !== undefined) { + expect(parentPos).toBeLessThan(childPos); + } + } + } +} + +export function verifyLinearization( + txs: Set, + linearization: ClusterTx[], + chunks: LinearizationChunk[], +): void { + expect(linearization.length).toBe(txs.size); + const linSet = new Set(linearization); + expect(linSet.size).toBe(linearization.length); + for (const tx of txs) { + expect(linSet.has(tx)).toBe(true); + } + + verifyTopologicalOrder(linearization); + + for (let i = 1; i < chunks.length; i++) { + const prevRate = chunks[i - 1].fee * chunks[i].weight; + const curRate = chunks[i].fee * chunks[i - 1].weight; + expect(prevRate).toBeGreaterThanOrEqual(curRate); + } + + const chunkTxs = chunks.flatMap(c => c.txs); + expect(chunkTxs.length).toBe(linearization.length); + for (let i = 0; i < chunkTxs.length; i++) { + expect(chunkTxs[i]).toBe(linearization[i]); + } +} diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index cf81a5f7f..42a0b3fc8 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -45,6 +45,8 @@ describe('Mempool Backend Config', () => { AUDIT: false, RUST_GBT: true, LIMIT_GBT: false, + CLUSTER_MEMPOOL: false, + CLUSTER_MEMPOOL_INDEXING: false, CPFP_INDEXING: false, MAX_BLOCKS_BULK_QUERY: 0, DISK_CACHE_BLOCK_INTERVAL: 6, diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index 7b90c516e..cc1201919 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -6,11 +6,28 @@ import transactionUtils from './transaction-utils'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners +export interface AuditResult { + unseen: string[]; + censored: string[]; + added: string[]; + prioritized: string[]; + fresh: string[]; + sigop: string[]; + fullrbf: string[]; + accelerated: string[]; + matchRate: number; + similarity: number; +} + class Audit { - auditBlock(height: number, transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }) - : { unseen: string[], censored: string[], added: string[], prioritized: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } { + auditBlock( + height: number, + transactions: MempoolTransactionExtended[], + projectedBlocks: MempoolBlockWithTransactions[], + mempool: { [txId: string]: MempoolTransactionExtended } + ): AuditResult { if (!projectedBlocks?.[0]?.transactionIds || !mempool) { - return { unseen: [], censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 1, similarity: 1 }; + return { unseen: [], censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], matchRate: 100, similarity: 1 }; } const matches: string[] = []; // present in both mined block and template @@ -176,6 +193,8 @@ class Audit { } const similarity = projectedWeight ? matchedWeight / projectedWeight : 1; + const matchRate = Math.round(score * 100 * 100) / 100; + return { unseen, censored: Object.keys(isCensored), @@ -185,7 +204,7 @@ class Audit { sigop: [], fullrbf: rbf, accelerated, - score, + matchRate, similarity, }; } diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index 237a7caec..9d9b5f262 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -12,7 +12,7 @@ import backendInfo from '../backend-info'; import transactionUtils from '../transaction-utils'; import { IEsploraApi } from './esplora-api.interface'; import loadingIndicators from '../loading-indicators'; -import { TransactionExtended } from '../../mempool.interfaces'; +import { CpfpInfo, TransactionExtended } from '../../mempool.interfaces'; import logger from '../../logger'; import blocks from '../blocks'; import bitcoinClient from './bitcoin-client'; @@ -191,11 +191,11 @@ class BitcoinRoutes { const tx = mempool.getMempool()[req.params.txId]; if (tx) { if (tx?.cpfpChecked) { - res.json({ - ancestors: tx.ancestors, + const response: CpfpInfo & { acceleratedBy?: number[], acceleratedAt?: number, feeDelta?: number } = { + ancestors: tx.ancestors || [], bestDescendant: tx.bestDescendant || null, - descendants: tx.descendants || null, - effectiveFeePerVsize: tx.effectiveFeePerVsize || null, + descendants: tx.descendants, + effectiveFeePerVsize: tx.effectiveFeePerVsize, sigops: tx.sigops, fee: tx.fee, adjustedVsize: tx.adjustedVsize, @@ -203,7 +203,14 @@ class BitcoinRoutes { acceleratedBy: tx.acceleratedBy || undefined, acceleratedAt: tx.acceleratedAt || undefined, feeDelta: tx.feeDelta || undefined, - }); + }; + if (config.MEMPOOL.CLUSTER_MEMPOOL && tx.clusterId != null) { + const cluster = mempool.clusterMempool?.getClusterForApi(req.params.txId); + if (cluster) { + response.cluster = cluster; + } + } + res.json(response); return; } diff --git a/backend/src/api/block-processor.ts b/backend/src/api/block-processor.ts new file mode 100644 index 000000000..6556e907e --- /dev/null +++ b/backend/src/api/block-processor.ts @@ -0,0 +1,262 @@ +import config from '../config'; +import logger from '../logger'; +import { + BlockExtended, + BlockSummary, + PoolTag, + MempoolTransactionExtended, + CpfpSummary, + TemplateAlgorithm, + MempoolBlockWithTransactions, +} from '../mempool.interfaces'; +import { IEsploraApi } from './bitcoin/esplora-api.interface'; +import { Acceleration } from './services/acceleration'; +import { calculateGoodBlockCpfp, calculateClusterMempoolBlockCpfp, calculateFastBlockCpfp, BlockCpfpData } from './cpfp'; +import mempoolBlocks from './mempool-blocks'; +import memPool from './mempool'; +import Audit, { AuditResult } from './audit'; +import blocks from './blocks'; +import transactionUtils from './transaction-utils'; +import { ClusterMempool } from '../cluster-mempool/cluster-mempool'; +import { Common } from './common'; +import accelerationApi from './services/acceleration'; + +interface ProcessedAudit extends AuditResult { + expectedFees: number; + expectedWeight: number; + projectedBlocks: MempoolBlockWithTransactions[]; +} + +export interface BlockProcessingResult { + templateAlgorithm: TemplateAlgorithm; + cpfpSummary: CpfpSummary; + blockExtended: BlockExtended; + blockSummary: BlockSummary; + auditResult?: ProcessedAudit; +} + +const CM_ACTIVATION_HEIGHT: { [network: string]: number } = { + 'mainnet': 940000, + 'testnet': 4860000, + 'testnet4': 125000, + 'signet': 294000, + 'regtest': 0, +}; + +class BlockProcessor { + + /** @asyncUnsafe */ + public async $processNewBlock( + block: IEsploraApi.Block, + transactions: MempoolTransactionExtended[], + pool: PoolTag, + accelerations: Record + ): Promise { + const poolAccelerations = Object.values(accelerations) + .filter(a => a.pools.includes(pool.uniqueId)) + .map(a => ({ txid: a.txid, max_bid: a.feeDelta })); + + const { templateAlgorithm, cpfpSummary } = detectTemplateAlgorithm( + block.height, + transactions, + poolAccelerations + ); + + + const blockExtended = await blocks.$getBlockExtended(block, cpfpSummary.transactions, pool); + const blockSummary = blocks.summarizeBlockTransactions(block.id, block.height, cpfpSummary.transactions); + + let auditResult: ProcessedAudit | undefined; + if (config.MEMPOOL.AUDIT && memPool.isInSync()) { + auditResult = await this.$runAudit( + blockExtended, + transactions, + templateAlgorithm, + pool, + accelerations + ); + + if (blockExtended.extras) { + blockExtended.extras.matchRate = auditResult.matchRate; + blockExtended.extras.expectedFees = auditResult.expectedFees; + blockExtended.extras.expectedWeight = auditResult.expectedWeight; + blockExtended.extras.similarity = auditResult.similarity; + } + } else if (blockExtended.extras) { + const mBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); + if (mBlocks?.length && mBlocks[0].transactions) { + blockExtended.extras.similarity = Common.getSimilarity(mBlocks[0], transactions); + } + } + + return { + templateAlgorithm, + cpfpSummary, + blockExtended, + blockSummary, + auditResult, + }; + } + + private async $runAudit( + block: BlockExtended, + transactions: MempoolTransactionExtended[], + templateAlgorithm: TemplateAlgorithm, + pool: PoolTag, + accelerations: Record + ): Promise { + const auditMempool = memPool.getMempool(); + const isAccelerated = accelerationApi.isAcceleratedBlock(block, Object.values(accelerations)); + + const candidateTxs = memPool.getMempoolCandidates(); + const candidates = (memPool.limitGBT && candidateTxs) + ? { txs: candidateTxs, added: [], removed: [] } + : undefined; + const transactionIds = (memPool.limitGBT) + ? Object.keys(candidates?.txs || {}) + : Object.keys(auditMempool); + + let projectedBlocks: MempoolBlockWithTransactions[]; + + if (templateAlgorithm === TemplateAlgorithm.clusterMempool) { + const clusterMempool = memPool.clusterMempool ?? new ClusterMempool(auditMempool, accelerations, true, 75000); + const cmBlocks = clusterMempool.getBlocks(config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT) ?? []; + projectedBlocks = mempoolBlocks.processClusterMempoolBlocks( + cmBlocks, + auditMempool, + accelerations, + false, + pool.uniqueId + ); + } else if (config.MEMPOOL.RUST_GBT) { + const added = memPool.limitGBT ? (candidates?.added || []) : []; + const removed = memPool.limitGBT ? (candidates?.removed || []) : []; + projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates( + transactionIds, + auditMempool, + added, + removed, + candidates, + isAccelerated, + pool.uniqueId, + true + ); + } else { + projectedBlocks = await mempoolBlocks.$makeBlockTemplates( + transactionIds, + auditMempool, + candidates, + false, + isAccelerated, + pool.uniqueId + ); + } + + const auditResult = Audit.auditBlock(block.height, transactions, projectedBlocks, auditMempool); + + const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : []; + + let totalFees = 0; + let totalWeight = 0; + for (const tx of stripped) { + totalFees += tx.fee; + totalWeight += (tx.vsize * 4); + } + + return { + ...auditResult, + expectedFees: totalFees, + expectedWeight: totalWeight, + projectedBlocks, + }; + } +} + +function saveCpfpDataToTransactions(transactions: MempoolTransactionExtended[], cpfpData: BlockCpfpData): void { + for (const tx of transactions) { + if (cpfpData.txs[tx.txid]) { + Object.assign(tx, cpfpData.txs[tx.txid]); + } + } +} + +export function saveCpfpDataToCpfpSummary(transactions: MempoolTransactionExtended[], cpfpData: BlockCpfpData): CpfpSummary { + saveCpfpDataToTransactions(transactions, cpfpData); + return { + transactions, + clusters: cpfpData.clusters, + version: cpfpData.version, + }; +} + +/** + * + * @param height + * @param blockTransactions + * @param poolAccelerations + * @param fast + * + * saves effective fee rates from detected algorithm to blockTransactions + */ +export function detectTemplateAlgorithm( + height: number, + blockTransactions: MempoolTransactionExtended[], + poolAccelerations: { txid: string; max_bid: number }[], + fast: boolean = false +): { templateAlgorithm: TemplateAlgorithm; cpfpSummary: CpfpSummary } { + + const legacyCpfpData = fast ? calculateFastBlockCpfp( + height, + blockTransactions, + ) : calculateGoodBlockCpfp( + height, + blockTransactions, + poolAccelerations + ); + + if (!config.MEMPOOL.CLUSTER_MEMPOOL_INDEXING) { + return { + templateAlgorithm: TemplateAlgorithm.legacy, + cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, legacyCpfpData), + }; + } + + const network = config.MEMPOOL.NETWORK || 'mainnet'; + const activationHeight = CM_ACTIVATION_HEIGHT[network] ?? Infinity; + + if (height < activationHeight) { + return { + templateAlgorithm: TemplateAlgorithm.legacy, + cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, legacyCpfpData), + }; + } + + const clusterCpfpData = calculateClusterMempoolBlockCpfp( + height, + blockTransactions, + poolAccelerations + ); + + const clusterTxs = blockTransactions.map(tx => ({ txid: tx.txid, rate: clusterCpfpData.txs[tx.txid].effectiveFeePerVsize ?? tx.effectiveFeePerVsize })); + const legacyTxs = blockTransactions.map(tx => ({ txid: tx.txid, rate: legacyCpfpData.txs[tx.txid].effectiveFeePerVsize ?? tx.effectiveFeePerVsize })); + const clusterPrioritization = transactionUtils.identifyPrioritizedTransactions(clusterTxs, 'rate'); + const legacyPrioritization = transactionUtils.identifyPrioritizedTransactions(legacyTxs, 'rate'); + + const clusterCount = clusterPrioritization.prioritized.length + clusterPrioritization.deprioritized.length; + const legacyCount = legacyPrioritization.prioritized.length + legacyPrioritization.deprioritized.length; + + if (clusterCount < legacyCount) { + saveCpfpDataToTransactions(blockTransactions, clusterCpfpData); + return { + templateAlgorithm: TemplateAlgorithm.clusterMempool, + cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, clusterCpfpData), + }; + } else { + return { + templateAlgorithm: TemplateAlgorithm.legacy, + cpfpSummary: saveCpfpDataToCpfpSummary(blockTransactions, legacyCpfpData), + }; + } +} + +export default new BlockProcessor(); diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index deb42bff2..613b77932 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -2,7 +2,7 @@ import config from '../config'; import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; import logger from '../logger'; import memPool from './mempool'; -import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified, BlockAudit, TransactionAudit } from '../mempool.interfaces'; +import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified, BlockAudit, TransactionAudit, TemplateAlgorithm } from '../mempool.interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; import transactionUtils from './transaction-utils'; @@ -28,9 +28,13 @@ import chainTips from './chain-tips'; import websocketHandler from './websocket-handler'; import redisCache from './redis-cache'; import rbfCache from './rbf-cache'; +import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; +import mempoolBlocks from './mempool-blocks'; +import statistics from './statistics/statistics'; import { calcBitsDifference } from './difficulty-adjustment'; import AccelerationRepository from '../repositories/AccelerationRepository'; -import { calculateFastBlockCpfp, calculateGoodBlockCpfp } from './cpfp'; +import { calculateGoodBlockCpfp } from './cpfp'; +import blockProcessor, { BlockProcessingResult, detectTemplateAlgorithm, saveCpfpDataToCpfpSummary } from './block-processor'; import mempool from './mempool'; import CpfpRepository from '../repositories/CpfpRepository'; import { parseDATUMTemplateCreator } from '../utils/bitcoin-script'; @@ -46,7 +50,6 @@ class Blocks { private previousDifficultyRetarget = 0; private quarterEpochBlockTime: number | null = null; private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = []; - private newAsyncBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: MempoolTransactionExtended[]) => Promise)[] = []; private classifyingBlocks: boolean = false; private oldestCoreLogTimestamp: number | undefined | null = undefined; @@ -74,10 +77,6 @@ class Blocks { this.newBlockCallbacks.push(fn); } - public setNewAsyncBlockCallback(fn: (block: BlockExtended, txIds: string[], transactions: MempoolTransactionExtended[]) => Promise) { - this.newAsyncBlockCallbacks.push(fn); - } - /** * Return the list of transaction for a block * @param blockHash @@ -252,7 +251,7 @@ class Blocks { * * @asyncUnsafe */ - private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise { + public async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[], providedPool?: PoolTag): Promise { const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); const blk: Partial = Object.assign({}, block); @@ -335,7 +334,9 @@ class Blocks { if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) { let pool: PoolTag; - if (coinbaseTx !== undefined) { + if (providedPool) { + pool = providedPool; + } else if (coinbaseTx !== undefined) { pool = await this.$findBlockMiner(coinbaseTx); } else { if (config.DATABASE.ENABLED === true) { @@ -386,7 +387,7 @@ class Blocks { return blk; } - private async $getBlockStats(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise { + public async $getBlockStats(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise { if (!block.stale) { return bitcoinClient.getBlockStats(block.id); } @@ -496,6 +497,128 @@ class Blocks { } } + /** @asyncUnsafe */ + private async $applyBlockTransactionsToMempool( + txIds: string[], + transactions: MempoolTransactionExtended[] + ): Promise<{ rbfTransactions: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }}}> { + const _memPool = memPool.getMempool(); + + const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap()); + memPool.handleRbfTransactions(rbfTransactions); + memPool.removeFromSpendMap(transactions); + + if (config.MEMPOOL.CLUSTER_MEMPOOL) { + memPool.clusterMempool?.applyMempoolChange({ + added: [], + removed: txIds, + accelerations: mempool.getAccelerations(), + }); + } + + for (const txId of txIds) { + delete _memPool[txId]; + rbfCache.mined(txId); + } + + let candidates; + let transactionIds: string[]; + + if (memPool.limitGBT) { + const minFeeMempool = await bitcoinSecondClient.getRawMemPool(); + const minFeeTip = await bitcoinSecondClient.getBlockCount(); + candidates = memPool.getNextCandidates(minFeeMempool, minFeeTip, transactions); + transactionIds = Object.keys(candidates?.txs || {}); + } else { + candidates = undefined; + transactionIds = Object.keys(memPool.getMempool()); + } + + if (config.MEMPOOL.CLUSTER_MEMPOOL) { + const cmBlocks = mempool.clusterMempool?.getBlocks(config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT) ?? []; + mempoolBlocks.processClusterMempoolBlocks(cmBlocks, _memPool, mempool.getAccelerations()); + } else if (config.MEMPOOL.RUST_GBT) { + const added = memPool.limitGBT ? (candidates?.added || []) : []; + const removed = memPool.limitGBT ? (candidates?.removed || []) : transactions; + await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, _memPool, added, removed, candidates, true); + } else { + await mempoolBlocks.$makeBlockTemplates(transactionIds, _memPool, candidates, true, true); + } + + return { rbfTransactions }; + } + + /** @asyncUnsafe */ + private async $saveBlockData( + processingResult: BlockProcessingResult, + timer: number + ): Promise { + const blockExtended = processingResult.blockExtended; + const cpfpSummary = processingResult.cpfpSummary; + + let latestPriceId; + try { + latestPriceId = await PricesRepository.$getLatestPriceId(); + this.updateTimerProgress(timer, `got latest price id ${this.currentBlockHeight}`); + } catch (e) { + logger.debug('failed to fetch latest price id from db: ' + (e instanceof Error ? e.message : e)); + } + if (priceUpdater.historyInserted === true && latestPriceId !== null) { + await blocksRepository.$saveBlockPrices([{ + height: blockExtended.height, + priceId: latestPriceId, + }]); + this.updateTimerProgress(timer, `saved prices for ${this.currentBlockHeight}`); + } else { + logger.debug(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining); + indexer.scheduleSingleTask('blocksPrices', 10000); + } + + if (Common.blocksSummariesIndexingEnabled() === true) { + // indexes the summary as a side effect + await this.$getStrippedBlockTransactions(blockExtended.id, true, false, cpfpSummary, blockExtended.height); + this.updateTimerProgress(timer, `saved block summary for ${this.currentBlockHeight}`); + } + + if (config.MEMPOOL.CPFP_INDEXING) { + // can be slow, and isn't critical, so don't await + void this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary); + this.updateTimerProgress(timer, `saved cpfp for ${this.currentBlockHeight}`); + } + + if (processingResult.auditResult) { + void BlocksSummariesRepository.$saveTemplate({ + height: blockExtended.height, + template: { + id: blockExtended.id, + transactions: processingResult.auditResult.projectedBlocks[0].transactions, + }, + version: 1, + }); + this.updateTimerProgress(timer, `saved audit template for ${this.currentBlockHeight}`); + + void BlocksAuditsRepository.$saveAudit({ + version: 1, + templateAlgorithm: processingResult.templateAlgorithm, + time: blockExtended.timestamp, + height: blockExtended.height, + hash: blockExtended.id, + unseenTxs: processingResult.auditResult.unseen, + addedTxs: processingResult.auditResult.added, + prioritizedTxs: processingResult.auditResult.prioritized, + missingTxs: processingResult.auditResult.censored, + freshTxs: processingResult.auditResult.fresh, + sigopTxs: processingResult.auditResult.sigop, + fullrbfTxs: processingResult.auditResult.fullrbf, + acceleratedTxs: processingResult.auditResult.accelerated, + matchRate: processingResult.auditResult.matchRate, + expectedFees: processingResult.auditResult.expectedFees, + expectedWeight: processingResult.auditResult.expectedWeight, + }); + this.updateTimerProgress(timer, `saved audit results for ${this.currentBlockHeight}`); + } + } + /** * [INDEXING] Index all blocks summaries for the block txs visualization */ @@ -728,7 +851,8 @@ class Blocks { // fetch transactions txs = (await bitcoinApi.$getTxsForBlock(blockHash, true)).map(tx => transactionUtils.extendMempoolTransaction(tx)) || []; // add CPFP - const cpfpSummary = calculateGoodBlockCpfp(height, txs, []); + const blockCpfpData = calculateGoodBlockCpfp(height, txs, []); + const cpfpSummary = saveCpfpDataToCpfpSummary(txs, blockCpfpData); // classify const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, height, cpfpSummary.transactions); await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 2); @@ -765,7 +889,8 @@ class Blocks { } templateTxs.push(tx || templateTx); } - const cpfpSummary = calculateGoodBlockCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as MempoolTransactionExtended[], []); + const blockCpfpData = calculateGoodBlockCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as MempoolTransactionExtended[], []); + const cpfpSummary = saveCpfpDataToCpfpSummary(templateTxs as MempoolTransactionExtended[], blockCpfpData); // classify const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, height, cpfpSummary.transactions); const classifiedTxMap: { [txid: string]: TransactionClassified } = {}; @@ -1054,59 +1179,55 @@ class Blocks { } } - let accelerations = Object.values(mempool.getAccelerations()); - if (accelerations?.length > 0) { - const pool = await this.$findBlockMiner(transactionUtils.stripCoinbaseTransaction(transactions[0])); - accelerations = accelerations.filter(a => a.pools.includes(pool.uniqueId)); - } - const cpfpSummary: CpfpSummary = calculateGoodBlockCpfp(block.height, transactions, accelerations.map(a => ({ txid: a.txid, max_bid: a.feeDelta }))); - const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions); - const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, block.height, cpfpSummary.transactions); + const pool = await this.$findBlockMiner(transactionUtils.stripCoinbaseTransaction(transactions[0])); + const accelerations = mempool.getAccelerations(); + + const processingResult = await blockProcessor.$processNewBlock( + block, + transactions, + pool, + accelerations + ); + + const blockExtended = processingResult.blockExtended; + const blockSummary = processingResult.blockSummary; + const cpfpSummary = processingResult.cpfpSummary; this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`); - if (Common.indexingEnabled()) { - if (!fastForwarded) { - await this.$handleReorgs(blockExtended, timer); - } + if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) { + await statistics.runStatistics(); + } + const { rbfTransactions } = await this.$applyBlockTransactionsToMempool(txIds, cpfpSummary.transactions); + this.updateTimerProgress(timer, `applied mempool changes for ${this.currentBlockHeight}`); + + if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) { + await statistics.runStatistics(); + } + + if (Common.indexingEnabled() && !fastForwarded) { + await this.$handleReorgs(blockExtended, timer); + } + + await websocketHandler.handleNewBlock(blockExtended, txIds, cpfpSummary.transactions, rbfTransactions); + this.updateTimerProgress(timer, `sent websocket updates for ${this.currentBlockHeight}`); + + if (Common.indexingEnabled()) { await blocksRepository.$saveBlockInDatabase(blockExtended); this.updateTimerProgress(timer, `saved ${this.currentBlockHeight} to database`); - if (!fastForwarded) { - let lastestPriceId; - try { - lastestPriceId = await PricesRepository.$getLatestPriceId(); - this.updateTimerProgress(timer, `got latest price id ${this.currentBlockHeight}`); - } catch (e) { - logger.debug('failed to fetch latest price id from db: ' + (e instanceof Error ? e.message : e)); - } - if (priceUpdater.historyInserted === true && lastestPriceId !== null) { - await blocksRepository.$saveBlockPrices([{ - height: blockExtended.height, - priceId: lastestPriceId, - }]); - this.updateTimerProgress(timer, `saved prices for ${this.currentBlockHeight}`); - } else { - logger.debug(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining); - indexer.scheduleSingleTask('blocksPrices', 10000); - } + await AccelerationRepository.$indexAccelerationsForBlock( + blockExtended, + Object.values(accelerations), + cpfpSummary.transactions + ); + this.updateTimerProgress(timer, `indexed accelerations for ${this.currentBlockHeight}`); - // Save blocks summary for visualization if it's enabled - if (Common.blocksSummariesIndexingEnabled() === true) { - await this.$getStrippedBlockTransactions(blockExtended.id, true, false, cpfpSummary, blockExtended.height); - this.updateTimerProgress(timer, `saved block summary for ${this.currentBlockHeight}`); - } - if (config.MEMPOOL.CPFP_INDEXING) { - void this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary); - this.updateTimerProgress(timer, `saved cpfp for ${this.currentBlockHeight}`); - } + if (!fastForwarded) { + await this.$saveBlockData(processingResult, timer); } } - // start async callbacks - this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`); - const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, cpfpSummary.transactions)); - if (block.height % 2016 === 0) { if (Common.indexingEnabled()) { let adjustment; @@ -1144,11 +1265,6 @@ class Blocks { await chainTips.updateOrphanedBlocks(); } - // wait for pending async callbacks to finish - this.updateTimerProgress(timer, `waiting for async callbacks to complete for ${this.currentBlockHeight}`); - await Promise.all(callbackPromises); - this.updateTimerProgress(timer, `async callbacks completed for ${this.currentBlockHeight}`); - this.blocks.push(blockExtended); if (this.blocks.length > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4) { this.blocks = this.blocks.slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4); @@ -1648,16 +1764,16 @@ class Blocks { } if (transactions?.length != null) { - const summary = calculateFastBlockCpfp(height, transactions); + const { cpfpSummary } = await detectTemplateAlgorithm(height, transactions, [], true); if (!stale) { - await this.$saveCpfp(hash, height, summary); + await this.$saveCpfp(hash, height, cpfpSummary); } - const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions); + const effectiveFeeStats = Common.calcEffectiveFeeStatistics(cpfpSummary.transactions); await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats); - return summary; + return cpfpSummary; } else { logger.err(`Cannot index CPFP for block ${height} - missing transaction data`); return null; diff --git a/backend/src/api/cpfp.ts b/backend/src/api/cpfp.ts index ad601361c..30232c90b 100644 --- a/backend/src/api/cpfp.ts +++ b/backend/src/api/cpfp.ts @@ -1,20 +1,30 @@ -import { Ancestor, CpfpCluster, CpfpInfo, CpfpSummary, MempoolTransactionExtended, TransactionExtended } from '../mempool.interfaces'; +import { Ancestor, CpfpCluster, CpfpInfo, MempoolTransactionExtended, TemplateAlgorithm, TransactionExtended } from '../mempool.interfaces'; import { GraphTx, convertToGraphTx, expandRelativesGraph, initializeRelatives, makeBlockTemplate, mempoolComparator, removeAncestors, setAncestorScores } from './mini-miner'; import memPool from './mempool'; import { Acceleration } from './acceleration/acceleration'; +import { ClusterMempool } from '../cluster-mempool/cluster-mempool'; const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction const MAX_CLUSTER_ITERATIONS = 100; -export function calculateFastBlockCpfp(height: number, transactions: MempoolTransactionExtended[], saveRelatives: boolean = false): CpfpSummary { +type TransactionCpfpData = Partial; +export interface BlockCpfpData { + txs: Record, + clusters: CpfpCluster[]; + version: number; +} + +export function calculateFastBlockCpfp(height: number, transactions: MempoolTransactionExtended[], saveRelatives: boolean = false): BlockCpfpData { const clusters: CpfpCluster[] = []; // list of all cpfp clusters in this block const clusterMap: { [txid: string]: CpfpCluster } = {}; // map transactions to their cpfp cluster let clusterTxs: TransactionExtended[] = []; // working list of elements of the current cluster let ancestors: { [txid: string]: boolean } = {}; // working set of ancestors of the current cluster root const txMap: { [txid: string]: TransactionExtended } = {}; + const cpfpData: Record = {}; // initialize the txMap for (const tx of transactions) { txMap[tx.txid] = tx; + cpfpData[tx.txid] = {}; } // reverse pass to identify CPFP clusters for (let i = transactions.length - 1; i >= 0; i--) { @@ -38,7 +48,9 @@ export function calculateFastBlockCpfp(height: number, transactions: MempoolTran clusters.push(cluster); } clusterTxs.forEach(tx => { - txMap[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize; + cpfpData[tx.txid] = { + effectiveFeePerVsize + }; if (cluster) { clusterMap[tx.txid] = cluster; } @@ -54,17 +66,19 @@ export function calculateFastBlockCpfp(height: number, transactions: MempoolTran } // forward pass to enforce ancestor rate caps for (const tx of transactions) { - let minAncestorRate = tx.effectiveFeePerVsize; + const txRate = cpfpData[tx.txid]?.effectiveFeePerVsize ?? tx.effectiveFeePerVsize; + let minAncestorRate = txRate; for (const vin of tx.vin) { - if (txMap[vin.txid]?.effectiveFeePerVsize) { - minAncestorRate = Math.min(minAncestorRate, txMap[vin.txid].effectiveFeePerVsize); + const vinRate = cpfpData[vin.txid]?.effectiveFeePerVsize ?? txMap[vin.txid]?.effectiveFeePerVsize; + if (vinRate) { + minAncestorRate = Math.min(minAncestorRate, vinRate); } } // check rounded values to skip cases with almost identical fees const roundedMinAncestorRate = Math.ceil(minAncestorRate); - const roundedEffectiveFeeRate = Math.floor(tx.effectiveFeePerVsize); + const roundedEffectiveFeeRate = Math.floor(txRate); if (roundedMinAncestorRate < roundedEffectiveFeeRate) { - tx.effectiveFeePerVsize = minAncestorRate; + cpfpData[tx.txid].effectiveFeePerVsize = minAncestorRate; if (!clusterMap[tx.txid]) { // add a single-tx cluster to record the dependent rate const cluster = { @@ -84,23 +98,25 @@ export function calculateFastBlockCpfp(height: number, transactions: MempoolTran if (saveRelatives) { for (const cluster of clusters) { cluster.txs.forEach((member, index) => { - txMap[member.txid].descendants = cluster.txs.slice(0, index).reverse(); - txMap[member.txid].ancestors = cluster.txs.slice(index + 1).reverse(); - txMap[member.txid].effectiveFeePerVsize = cluster.effectiveFeePerVsize; + cpfpData[member.txid].descendants = cluster.txs.slice(0, index).reverse(); + cpfpData[member.txid].ancestors = cluster.txs.slice(index + 1).reverse(); + cpfpData[member.txid].effectiveFeePerVsize = cluster.effectiveFeePerVsize; }); } } return { - transactions, + txs: cpfpData, clusters, version: 1, }; } -export function calculateGoodBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): CpfpSummary { +export function calculateGoodBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): BlockCpfpData { const txMap: { [txid: string]: MempoolTransactionExtended } = {}; + const cpfpData: Record = {}; for (const tx of transactions) { txMap[tx.txid] = tx; + cpfpData[tx.txid] = {}; } const template = makeBlockTemplate(transactions, accelerations, 1, Infinity, Infinity); const clusters = new Map(); @@ -110,15 +126,15 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran if (cluster.length > 1 && root && !clusters.has(root)) { clusters.set(root, cluster); } - txMap[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize; + cpfpData[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize; } const clusterArray: CpfpCluster[] = []; for (const cluster of clusters.values()) { for (const txid of cluster) { - const mempoolTx = txMap[txid]; - if (mempoolTx) { + const mempoolTxCpfpData = cpfpData[txid]; + if (mempoolTxCpfpData) { const ancestors: Ancestor[] = []; const descendants: Ancestor[] = []; let matched = false; @@ -138,10 +154,10 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran } } }); - if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) { - mempoolTx.cpfpDirty = true; + if (mempoolTxCpfpData.ancestors?.length !== ancestors.length || mempoolTxCpfpData.descendants?.length !== descendants.length) { + mempoolTxCpfpData.cpfpDirty = true; } - Object.assign(mempoolTx, { ancestors, descendants, bestDescendant: null, cpfpChecked: true }); + Object.assign(mempoolTxCpfpData, { ancestors, descendants, bestDescendant: null, cpfpChecked: true }); } } const root = cluster[cluster.length - 1]; @@ -153,17 +169,75 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran fee: txMap[txid].fee, weight: (txMap[txid].adjustedVsize * 4) || txMap[txid].weight, })), - effectiveFeePerVsize: txMap[root].effectiveFeePerVsize, + effectiveFeePerVsize: cpfpData[root].effectiveFeePerVsize ?? txMap[root].effectiveFeePerVsize, }); } return { - transactions: transactions.map(tx => txMap[tx.txid]), + txs: cpfpData, clusters: clusterArray, version: 2, }; } +export function calculateClusterMempoolBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): BlockCpfpData { + const txMap: { [txid: string]: MempoolTransactionExtended } = {}; + const cpfpData: Record = {}; + for (const tx of transactions) { + txMap[tx.txid] = tx; + cpfpData[tx.txid] = {}; + } + + const accelMap: { [txid: string]: { feeDelta: number } } = {}; + for (const acc of accelerations) { + accelMap[acc.txid] = { feeDelta: acc.max_bid }; + } + + const cm = new ClusterMempool(txMap, accelMap, false, 25000); + + const seenClusters = new Set(); + const clusters: CpfpCluster[] = []; + + for (const txid in cpfpData) { + const txCpfpData = cm.getCpfpDataForTx(txid); + if (!txCpfpData) { + continue; + } + cpfpData[txid].effectiveFeePerVsize = txCpfpData.effectiveFeePerVsize; + cpfpData[txid].clusterId = txCpfpData.clusterId; + cpfpData[txid].chunkIndex = txCpfpData.chunkIndex; + cpfpData[txid].ancestors = txCpfpData.ancestors; + cpfpData[txid].descendants = txCpfpData.descendants; + if (txCpfpData.clusterId !== undefined && !seenClusters.has(txCpfpData.clusterId)) { + seenClusters.add(txCpfpData.clusterId); + + const clusterData = cm.getCluster(txCpfpData.clusterId); + if (clusterData && clusterData.txs.length > 1) { + let totalFee = 0; + let totalWeight = 0; + for (const t of clusterData.txs) { + totalFee += t.fee; + totalWeight += t.weight; + } + clusters.push({ + root: clusterData.txs[0].txid, + height, + txs: clusterData.txs.map(t => ({ txid: t.txid, weight: t.weight, fee: t.fee })), + effectiveFeePerVsize: totalFee / (totalWeight / 4), + templateAlgorithm: TemplateAlgorithm.clusterMempool, + clusterData, + }); + } + } + } + + return { + txs: cpfpData, + clusters, + version: 3, + }; +} + /** * Takes a mempool transaction and a copy of the current mempool, and calculates the CPFP data for * that transaction (and all others in the same cluster) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 5a5314a05..3932179ed 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 109; + private static currentVersion = 111; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -1246,6 +1246,15 @@ class DatabaseMigration { `); await this.updateToSchemaVersion(109); } + if (databaseSchemaVersion < 110 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `blocks_audits` ADD template_algo TINYINT UNSIGNED NOT NULL DEFAULT 0'); + await this.updateToSchemaVersion(110); + } + + if (databaseSchemaVersion < 111 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `compact_cpfp_clusters` ADD template_algo TINYINT UNSIGNED NOT NULL DEFAULT 0'); + await this.updateToSchemaVersion(111); + } } /** diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 43bf05eec..17ccf84e6 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -8,6 +8,7 @@ import path from 'path'; import mempool from './mempool'; import { Acceleration } from './services/acceleration'; import PoolsRepository from '../repositories/PoolsRepository'; +import { ProjectedBlock } from '../cluster-mempool/cluster-mempool'; const MAX_UINT32 = Math.pow(2, 32) - 1; @@ -238,7 +239,7 @@ class MempoolBlocks { } /** @asyncSafe */ - public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { + public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number, dryRun = false): Promise { const start = Date.now(); // reset mempool short ids @@ -278,7 +279,7 @@ class MempoolBlocks { const expectedSize = transactions.length; const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length; logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${expectedSize} in the mempool, ${overflow.length} were unmineable`); - const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults, dryRun); logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; } catch (e) { @@ -296,16 +297,15 @@ class MempoolBlocks { } /** @asyncSafe */ - public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise { + public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number, dryRun = false): Promise { // GBT optimization requires that uids never get too sparse // as a sanity check, we should also explicitly prevent uint32 uid overflow if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * transactions.length), MAX_UINT32)) { this.resetRustGbt(); } - if (!this.rustInitialized) { - // need to reset the worker - return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, true, useAccelerations, accelerationPool); + if (!this.rustInitialized || dryRun) { + return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, !dryRun, useAccelerations, accelerationPool, dryRun); } const start = Date.now(); @@ -344,7 +344,7 @@ class MempoolBlocks { if (transactions.length !== resultMempoolSize) { throw new Error(`GBT returned wrong number of transactions ${transactions.length} vs ${resultMempoolSize}, cache is probably out of sync`); } else { - const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, true); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, !dryRun, dryRun); this.removeUids(removedTxs); logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; @@ -356,20 +356,22 @@ class MempoolBlocks { } } - private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations: { [txid: string]: Acceleration }, accelerationPool, saveResults): MempoolBlockWithTransactions[] { - for (const txid of Object.keys(candidates?.txs ?? mempool)) { - if (txid in mempool) { - mempool[txid].cpfpDirty = false; - mempool[txid].ancestors = []; - mempool[txid].descendants = []; - mempool[txid].bestDescendant = null; + private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations: { [txid: string]: Acceleration }, accelerationPool, saveResults, dryRun = false): MempoolBlockWithTransactions[] { + if (!dryRun) { + for (const txid of Object.keys(candidates?.txs ?? mempool)) { + if (txid in mempool) { + mempool[txid].cpfpDirty = false; + mempool[txid].ancestors = []; + mempool[txid].descendants = []; + mempool[txid].bestDescendant = null; + } } - } - for (const [txid, rate] of rates) { - if (txid in mempool) { - mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize); - mempool[txid].effectiveFeePerVsize = rate; - mempool[txid].cpfpChecked = true; + for (const [txid, rate] of rates) { + if (txid in mempool) { + mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize); + mempool[txid].effectiveFeePerVsize = rate; + mempool[txid].cpfpChecked = true; + } } } @@ -387,58 +389,60 @@ class MempoolBlocks { feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]); } - const ancestors: Ancestor[] = []; - const descendants: Ancestor[] = []; - let ancestor: MempoolTransactionExtended; - for (const cluster of clusters) { - for (const memberTxid of cluster) { - const mempoolTx = mempool[memberTxid]; - if (mempoolTx) { - // ugly micro-optimization to avoid allocating new arrays - ancestors.length = 0; - descendants.length = 0; - let matched = false; - cluster.forEach(txid => { - ancestor = mempool[txid]; - if (txid === memberTxid) { - matched = true; - } else { - if (!ancestor) { - console.log('txid missing from mempool! ', txid, candidates?.txs[txid]); - return; - } - const relative = { - txid: txid, - fee: ancestor.fee, - weight: (ancestor.adjustedVsize * 4), - }; - if (matched) { - descendants.push(relative); - if (!mempoolTx.lastBoosted || (ancestor.firstSeen && ancestor.firstSeen > mempoolTx.lastBoosted)) { - mempoolTx.lastBoosted = ancestor.firstSeen; - } + if (!dryRun) { + const ancestors: Ancestor[] = []; + const descendants: Ancestor[] = []; + let ancestor: MempoolTransactionExtended; + for (const cluster of clusters) { + for (const memberTxid of cluster) { + const mempoolTx = mempool[memberTxid]; + if (mempoolTx) { + // ugly micro-optimization to avoid allocating new arrays + ancestors.length = 0; + descendants.length = 0; + let matched = false; + cluster.forEach(txid => { + ancestor = mempool[txid]; + if (txid === memberTxid) { + matched = true; } else { - ancestors.push(relative); + if (!ancestor) { + console.log('txid missing from mempool! ', txid, candidates?.txs[txid]); + return; + } + const relative = { + txid: txid, + fee: ancestor.fee, + weight: (ancestor.adjustedVsize * 4), + }; + if (matched) { + descendants.push(relative); + if (!mempoolTx.lastBoosted || (ancestor.firstSeen && ancestor.firstSeen > mempoolTx.lastBoosted)) { + mempoolTx.lastBoosted = ancestor.firstSeen; + } + } else { + ancestors.push(relative); + } } + }); + if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) { + mempoolTx.cpfpDirty = true; } - }); - if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) { - mempoolTx.cpfpDirty = true; + // ugly micro-optimization to avoid allocating new arrays or objects + if (mempoolTx.ancestors) { + mempoolTx.ancestors.length = 0; + } else { + mempoolTx.ancestors = []; + } + if (mempoolTx.descendants) { + mempoolTx.descendants.length = 0; + } else { + mempoolTx.descendants = []; + } + mempoolTx.ancestors.push(...ancestors); + mempoolTx.descendants.push(...descendants); + mempoolTx.cpfpChecked = true; } - // ugly micro-optimization to avoid allocating new arrays or objects - if (mempoolTx.ancestors) { - mempoolTx.ancestors.length = 0; - } else { - mempoolTx.ancestors = []; - } - if (mempoolTx.descendants) { - mempoolTx.descendants.length = 0; - } else { - mempoolTx.descendants = []; - } - mempoolTx.ancestors.push(...ancestors); - mempoolTx.descendants.push(...descendants); - mempoolTx.cpfpChecked = true; } } } @@ -471,34 +475,42 @@ class MempoolBlocks { const txid = block[i]; if (txid in mempool) { mempoolTx = mempool[txid]; - // save position in projected blocks - mempoolTx.position = { - block: blockIndex, - vsize: totalVsize + (mempoolTx.vsize / 2), - }; - if (txid in accelerations) { - acceleration = accelerations[txid]; - if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) { - if (!mempoolTx.acceleration) { - mempoolTx.cpfpDirty = true; - } - mempoolTx.acceleration = true; - mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools; - mempoolTx.acceleratedAt = acceleration?.added; - mempoolTx.feeDelta = acceleration?.feeDelta; - for (const ancestor of mempoolTx.ancestors || []) { - if (!(ancestor.txid in mempool)) { - continue; + if (!dryRun) { + // save position in projected blocks + mempoolTx.position = { + block: blockIndex, + vsize: totalVsize + (mempoolTx.vsize / 2), + }; + + if (txid in accelerations) { + acceleration = accelerations[txid]; + if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) { + if (!mempoolTx.acceleration) { + mempoolTx.cpfpDirty = true; } - if (!mempool[ancestor.txid].acceleration) { - mempool[ancestor.txid].cpfpDirty = true; + mempoolTx.acceleration = true; + mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools; + mempoolTx.acceleratedAt = acceleration?.added; + mempoolTx.feeDelta = acceleration?.feeDelta; + for (const ancestor of mempoolTx.ancestors || []) { + if (!(ancestor.txid in mempool)) { + continue; + } + if (!mempool[ancestor.txid].acceleration) { + mempool[ancestor.txid].cpfpDirty = true; + } + mempool[ancestor.txid].acceleration = true; + mempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy; + mempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt; + mempool[ancestor.txid].feeDelta = mempoolTx.feeDelta; + isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy; + } + } else { + if (mempoolTx.acceleration) { + mempoolTx.cpfpDirty = true; + delete mempoolTx.acceleration; } - mempool[ancestor.txid].acceleration = true; - mempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy; - mempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt; - mempool[ancestor.txid].feeDelta = mempoolTx.feeDelta; - isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy; } } else { if (mempoolTx.acceleration) { @@ -506,11 +518,6 @@ class MempoolBlocks { delete mempoolTx.acceleration; } } - } else { - if (mempoolTx.acceleration) { - mempoolTx.cpfpDirty = true; - delete mempoolTx.acceleration; - } } // online calculation of stack-of-blocks fee stats @@ -548,6 +555,115 @@ class MempoolBlocks { return mempoolBlocks; } + public processClusterMempoolBlocks(projectedBlocks: ProjectedBlock[], newMempool: { [txid: string]: MempoolTransactionExtended }, accelerations: { [txid: string]: Acceleration }, saveResults = true, accelerationPool?: number): MempoolBlockWithTransactions[] { + const lastBlockIndex = projectedBlocks.length - 1; + let hasBlockStack = projectedBlocks.length >= 8; + let stackWeight = 0; + let feeStatsCalculator: OnlineFeeStatsCalculator | null = null; + if (hasBlockStack) { + stackWeight = projectedBlocks[lastBlockIndex].weight; + hasBlockStack = stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS; + feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]); + } + + const isAcceleratedBy: { [txid: string]: number[] | false } = {}; + + const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2; + let mempoolTx: MempoolTransactionExtended; + let acceleration: Acceleration; + const mempoolBlocks: MempoolBlockWithTransactions[] = []; + + for (let blockIndex = 0; blockIndex < projectedBlocks.length; blockIndex++) { + const projected = projectedBlocks[blockIndex]; + let totalSize = 0; + let totalVsize = 0; + let totalWeight = 0; + let totalFees = 0; + const transactions: MempoolTransactionExtended[] = []; + const validTxids: string[] = []; + + for (const txid of projected.txids) { + if (txid in newMempool) { + mempoolTx = newMempool[txid]; + validTxids.push(txid); + + // save position in projected blocks + mempoolTx.position = { + block: blockIndex, + vsize: totalVsize + (mempoolTx.vsize / 2), + }; + + if (txid in accelerations) { + acceleration = accelerations[txid]; + if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) { + if (!mempoolTx.acceleration) { + mempoolTx.cpfpDirty = true; + } + mempoolTx.acceleration = true; + mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools; + mempoolTx.acceleratedAt = acceleration?.added; + mempoolTx.feeDelta = acceleration?.feeDelta; + for (const ancestor of mempoolTx.ancestors || []) { + if (!(ancestor.txid in newMempool)) { + continue; + } + if (!newMempool[ancestor.txid].acceleration) { + newMempool[ancestor.txid].cpfpDirty = true; + } + newMempool[ancestor.txid].acceleration = true; + newMempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy; + newMempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt; + newMempool[ancestor.txid].feeDelta = mempoolTx.feeDelta; + isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy; + } + } else { + if (mempoolTx.acceleration) { + mempoolTx.cpfpDirty = true; + delete mempoolTx.acceleration; + } + } + } else { + if (mempoolTx.acceleration) { + mempoolTx.cpfpDirty = true; + delete mempoolTx.acceleration; + } + } + + if (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) { + feeStatsCalculator.processNext(mempoolTx); + } + + totalSize += mempoolTx.size; + totalVsize += mempoolTx.vsize; + totalWeight += mempoolTx.weight; + totalFees += mempoolTx.fee; + + if (totalVsize <= sizeLimit) { + transactions.push(mempoolTx); + } + } + } + + mempoolBlocks[blockIndex] = this.dataToMempoolBlocks( + validTxids, + transactions, + totalSize, + totalWeight, + totalFees, + (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined, + ); + } + + if (saveResults) { + const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks); + this.mempoolBlocks = mempoolBlocks; + this.mempoolBlockDeltas = deltas; + this.updateAccelerationPositions(newMempool, accelerations, mempoolBlocks); + } + + return mempoolBlocks; + } + private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions { if (!feeStats) { feeStats = Common.calcEffectiveFeeStatistics(transactions); diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index d83347fd6..1bd919c6c 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -13,6 +13,7 @@ import { Acceleration } from './services/acceleration'; import accelerationApi from './services/acceleration'; import redisCache from './redis-cache'; import blocks from './blocks'; +import { ClusterMempool } from '../cluster-mempool/cluster-mempool'; class Mempool { private inSync: boolean = false; @@ -22,6 +23,7 @@ class Mempool { private spendMap = new Map(); private recentlyDeleted: MempoolTransactionExtended[][] = []; // buffer of transactions deleted in recent mempool updates private mempoolInfo: IBitcoinApi.MempoolInfo; + public clusterMempool: ClusterMempool | null = null; private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[]) => void) | undefined; private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[], @@ -62,6 +64,9 @@ class Mempool { minrelaytxfee: isLiquid ? 0.00000100 : 0.00001000 }; this.txPerSecondInterval = setInterval(this.updateTxPerSecond.bind(this), 1000); + if (config.MEMPOOL.CLUSTER_MEMPOOL) { + this.clusterMempool = new ClusterMempool(this.mempoolCache, this.accelerations); + } } /** @@ -153,6 +158,9 @@ class Mempool { await redisCache.$flushTransactions(); logger.debug(`Finished migrating cache transactions in ${((Date.now() - redisTimer) / 1000).toFixed(2)} seconds`); } + if (config.MEMPOOL.CLUSTER_MEMPOOL) { + this.clusterMempool = new ClusterMempool(this.mempoolCache, this.accelerations); + } if (this.mempoolChangedCallback) { this.mempoolChangedCallback(this.mempoolCache, [], [], []); } @@ -387,6 +395,14 @@ class Mempool { hasChange = true; } + if (config.MEMPOOL.CLUSTER_MEMPOOL && (newTransactions.length || deletedTransactions.length || accelerationDelta.length)) { + this.clusterMempool?.applyMempoolChange({ + added: newTransactions, + removed: deletedTransactions.map(tx => tx.txid), + accelerations: this.getAccelerations(), + }); + } + this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize); const candidatesChanged = candidates?.added?.length || candidates?.removed?.length; diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index 0345282b0..caf589708 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -256,13 +256,14 @@ class TransactionUtils { // returns the most significant 4 bytes of the txid as an integer public txidToOrdering(txid: string): number { - return parseInt( - txid.substr(62, 2) + - txid.substr(60, 2) + - txid.substr(58, 2) + - txid.substr(56, 2), - 16 - ); + // Parse last 4 bytes of txid as little-endian uint32, without string allocation + let result = 0; + for (let i = 62; i >= 56; i -= 2) { + const hi = txid.charCodeAt(i); + const lo = txid.charCodeAt(i + 1); + result = result * 256 + (hi < 58 ? hi - 48 : hi - 87) * 16 + (lo < 58 ? lo - 48 : lo - 87); + } + return result; } public addInnerScriptsToVin(vin: IEsploraApi.Vin): void { diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index e11982e9d..92b3d909e 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -3,7 +3,7 @@ import * as WebSocket from 'ws'; import { BlockExtended, TransactionExtended, MempoolTransactionExtended, WebsocketResponse, OptimizedStatistic, ILoadingIndicators, GbtCandidates, TxTrackingInfo, - MempoolDelta, MempoolDeltaTxids + MempoolDelta, MempoolDeltaTxids, CpfpInfo } from '../mempool.interfaces'; import blocks from './blocks'; import memPool from './mempool'; @@ -16,16 +16,12 @@ import transactionUtils from './transaction-utils'; import rbfCache, { ReplacementInfo } from './rbf-cache'; import difficultyAdjustment from './difficulty-adjustment'; import feeApi from './fee-api'; -import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository'; -import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; -import Audit from './audit'; import priceUpdater from '../tasks/price-updater'; import { ApiPrice } from '../repositories/PricesRepository'; import { Acceleration } from './services/acceleration'; import accelerationApi from './services/acceleration'; import mempool from './mempool'; import statistics from './statistics/statistics'; -import accelerationRepository from '../repositories/AccelerationRepository'; import bitcoinApi from './bitcoin/bitcoin-api-factory'; import walletApi from './services/wallets'; @@ -649,7 +645,10 @@ class WebsocketHandler { removed = candidates?.removed || []; } - if (config.MEMPOOL.RUST_GBT) { + if (config.MEMPOOL.CLUSTER_MEMPOOL) { + const cmBlocks = mempool.clusterMempool?.getBlocks(config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT) ?? []; + mempoolBlocks.processClusterMempoolBlocks(cmBlocks, newMempool, mempool.getAccelerations()); + } else if (config.MEMPOOL.RUST_GBT) { await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, newMempool, added, removed, candidates, true); } else { await mempoolBlocks.$updateBlockTemplates(transactionIds, newMempool, added, removed, candidates, accelerationDelta, true, true); @@ -778,6 +777,8 @@ class WebsocketHandler { removed: websocketAccelerationDelta.filter(txid => !accelerations[txid]), }; + const cpfpUpdatesSent = new Set(); + // TODO - Fix indentation after PR is merged for (const server of this.webSocketServers) { server.clients.forEach(async (client) => { @@ -946,15 +947,23 @@ class WebsocketHandler { calculateMempoolTxCpfp(mempoolTx, newMempool); } if (mempoolTx.cpfpDirty) { - positionData['cpfp'] = { - ancestors: mempoolTx.ancestors, + const cpfp: CpfpInfo = { + ancestors: mempoolTx.ancestors || [], bestDescendant: mempoolTx.bestDescendant || null, - descendants: mempoolTx.descendants || null, - effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize || null, + descendants: mempoolTx.descendants, + effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize, sigops: mempoolTx.sigops, adjustedVsize: mempoolTx.adjustedVsize, acceleration: mempoolTx.acceleration, }; + if (config.MEMPOOL.CLUSTER_MEMPOOL && mempoolTx.clusterId != null) { + const cluster = mempool.clusterMempool?.getClusterForApi(mempoolTx.txid); + if (cluster) { + cpfp.cluster = cluster; + } + } + positionData['cpfp'] = cpfp; + cpfpUpdatesSent.add(trackTxid); } response['txPosition'] = JSON.stringify(positionData); } @@ -992,11 +1001,18 @@ class WebsocketHandler { txInfo.cpfp = { ancestors: mempoolTx.ancestors, bestDescendant: mempoolTx.bestDescendant || null, - descendants: mempoolTx.descendants || null, - effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize || null, + descendants: mempoolTx.descendants, + effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize, sigops: mempoolTx.sigops, adjustedVsize: mempoolTx.adjustedVsize, }; + if (config.MEMPOOL.CLUSTER_MEMPOOL && mempoolTx.clusterId != null) { + const cluster = mempool.clusterMempool?.getClusterForApi(mempoolTx.txid); + if (cluster) { + (txInfo.cpfp as CpfpInfo).cluster = cluster; + } + } + cpfpUpdatesSent.add(txid); } txHasInfo = true; } @@ -1047,129 +1063,33 @@ class WebsocketHandler { } }); } + + for (const txid of cpfpUpdatesSent) { + if (newMempool[txid]) { + newMempool[txid].cpfpDirty = false; + } + } } - /** @asyncUnsafe */ - async handleNewBlock(block: BlockExtended, txIds: string[], transactions: MempoolTransactionExtended[]): Promise { + /** @asyncSafe */ + async handleNewBlock( + block: BlockExtended, + txIds: string[], + transactions: MempoolTransactionExtended[], + rbfTransactions: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }} + ): Promise { if (!this.webSocketServers.length) { throw new Error('No WebSocket.Server have been set'); } - const blockTransactions = structuredClone(transactions); - this.printLogs(); - if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) { - await statistics.runStatistics(); - } const _memPool = memPool.getMempool(); - const candidateTxs = memPool.getMempoolCandidates(); - let candidates: GbtCandidates | undefined = (memPool.limitGBT && candidateTxs) ? { txs: candidateTxs, added: [], removed: [] } : undefined; - let transactionIds: string[] = (memPool.limitGBT) ? Object.keys(candidates?.txs || {}) : Object.keys(_memPool); - - if (config.DATABASE.ENABLED) { - const accelerations = Object.values(mempool.getAccelerations()); - await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, structuredClone(transactions)); - } - - const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap()); - memPool.handleRbfTransactions(rbfTransactions); - memPool.removeFromSpendMap(transactions); - - if (config.MEMPOOL.AUDIT && memPool.isInSync()) { - let projectedBlocks; - const auditMempool = _memPool; - const isAccelerated = accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); - - if (config.MEMPOOL.RUST_GBT) { - const added = memPool.limitGBT ? (candidates?.added || []) : []; - const removed = memPool.limitGBT ? (candidates?.removed || []) : []; - projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, auditMempool, added, removed, candidates, isAccelerated, block.extras.pool.id); - } else { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(transactionIds, auditMempool, candidates, false, isAccelerated, block.extras.pool.id); - } - - if (Common.indexingEnabled()) { - const { unseen, censored, added, prioritized, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(block.height, blockTransactions, projectedBlocks, auditMempool); - const matchRate = Math.round(score * 100 * 100) / 100; - - const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : []; - - let totalFees = 0; - let totalWeight = 0; - for (const tx of stripped) { - totalFees += tx.fee; - totalWeight += (tx.vsize * 4); - } - - void BlocksSummariesRepository.$saveTemplate({ - height: block.height, - template: { - id: block.id, - transactions: stripped, - }, - version: 1, - }); - - void BlocksAuditsRepository.$saveAudit({ - version: 1, - time: block.timestamp, - height: block.height, - hash: block.id, - unseenTxs: unseen, - addedTxs: added, - prioritizedTxs: prioritized, - missingTxs: censored, - freshTxs: fresh, - sigopTxs: sigop, - fullrbfTxs: fullrbf, - acceleratedTxs: accelerated, - matchRate: matchRate, - expectedFees: totalFees, - expectedWeight: totalWeight, - }); - - if (block.extras) { - block.extras.matchRate = matchRate; - block.extras.expectedFees = totalFees; - block.extras.expectedWeight = totalWeight; - block.extras.similarity = similarity; - } - } - } else if (block.extras) { - const mBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); - if (mBlocks?.length && mBlocks[0].transactions) { - block.extras.similarity = Common.getSimilarity(mBlocks[0], transactions); - } - } - const confirmedTxids: { [txid: string]: boolean } = {}; - - // Update mempool to remove transactions included in the new block for (const txId of txIds) { - delete _memPool[txId]; - rbfCache.mined(txId); confirmedTxids[txId] = true; } - if (memPool.limitGBT) { - const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null; - const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1; - candidates = memPool.getNextCandidates(minFeeMempool, minFeeTip, transactions); - transactionIds = Object.keys(candidates?.txs || {}); - } else { - candidates = undefined; - transactionIds = Object.keys(memPool.getMempool()); - } - - - if (config.MEMPOOL.RUST_GBT) { - const added = memPool.limitGBT ? (candidates?.added || []) : []; - const removed = memPool.limitGBT ? (candidates?.removed || []) : transactions; - await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, _memPool, added, removed, candidates, true); - } else { - await mempoolBlocks.$makeBlockTemplates(transactionIds, _memPool, candidates, true, true); - } const mBlocks = mempoolBlocks.getMempoolBlocks(); const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); @@ -1434,10 +1354,6 @@ class WebsocketHandler { } }); } - - if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) { - await statistics.runStatistics(); - } } public handleNewStratumJob(job: StratumJob): void { diff --git a/backend/src/cluster-mempool/block-builder.ts b/backend/src/cluster-mempool/block-builder.ts new file mode 100644 index 000000000..3f6a4b687 --- /dev/null +++ b/backend/src/cluster-mempool/block-builder.ts @@ -0,0 +1,165 @@ +import { PairingHeap } from '../utils/pairing-heap'; +import { LinearizationChunk } from './linearize'; +import { MempoolTransactionExtended } from '../mempool.interfaces'; +import { Cluster } from './cluster-mempool'; + +export interface ProjectedBlock { + txids: string[]; + weight: number; + sigops: number; +} + +interface ChunkHeapEntry { + fee: number; + weight: number; + sigops: number; + equalFeeratePrefixWeight: number; + maxOrder: number; + clusterId: number; + chunkIndex: number; +} + +const BLOCK_WEIGHT_UNITS = 4_000_000; +const MAX_BLOCK_SIGOPS_COST = 80_000; +const COINBASE_RESERVED_WEIGHT = 8000; +const MAX_CONSECUTIVE_FAILURES = 1000; +const MAX_WASTED_WEIGHT = 4000; + +function chunkHeapHigherPriority(a: ChunkHeapEntry, b: ChunkHeapEntry): boolean { + const feerateDiff = a.fee * b.weight - b.fee * a.weight; + if (feerateDiff !== 0) { + return feerateDiff > 0; + } + if (a.equalFeeratePrefixWeight !== b.equalFeeratePrefixWeight) { + return a.equalFeeratePrefixWeight < b.equalFeeratePrefixWeight; + } + return a.maxOrder < b.maxOrder; +} + +function equalFeerate(a: LinearizationChunk, b: LinearizationChunk): boolean { + return a.fee * b.weight === b.fee * a.weight; +} + +function makeChunkHeapEntry(cluster: Cluster, chunkIndex: number, mempool: { [txid: string]: MempoolTransactionExtended }): ChunkHeapEntry { + const chunk = cluster.chunks[chunkIndex]; + let maxOrder = 0; + let sigops = 0; + for (const tx of chunk.txs) { + if (tx.order > maxOrder) { + maxOrder = tx.order; + } + const mempoolTx = mempool[tx.txid]; + if (mempoolTx) { + sigops += mempoolTx.sigops || 0; + } + } + let prefixWeight = chunk.weight; + for (let i = chunkIndex - 1; i >= 0; i--) { + if (equalFeerate(cluster.chunks[i], chunk)) { + prefixWeight += cluster.chunks[i].weight; + } else { + break; + } + } + return { + fee: chunk.fee, + weight: chunk.weight, + sigops, + equalFeeratePrefixWeight: prefixWeight, + maxOrder, + clusterId: cluster.id, + chunkIndex, + }; +} + +function buildChunkHeap(clusters: Map, mempool: { [txid: string]: MempoolTransactionExtended }): PairingHeap { + const heap = new PairingHeap(chunkHeapHigherPriority); + for (const cluster of clusters.values()) { + if (cluster.chunks.length > 0) { + heap.add(makeChunkHeapEntry(cluster, 0, mempool)); + } + } + return heap; +} + +export function assembleBlocks( + n: number, + clusters: Map, + mempool: { [txid: string]: MempoolTransactionExtended }, + enforceLimit: boolean, +): ProjectedBlock[] { + const heap = buildChunkHeap(clusters, mempool); + const blocks: ProjectedBlock[] = []; + for (let blockIdx = 0; blockIdx < n; blockIdx++) { + const limited = enforceLimit || blockIdx < n - 1; + const maxWeight = limited ? BLOCK_WEIGHT_UNITS : Infinity; + const maxSigops = limited ? MAX_BLOCK_SIGOPS_COST : Infinity; + const block = fillBlock(heap, clusters, mempool, maxWeight, maxSigops); + if (block.txids.length === 0) { + break; + } + blocks.push(block); + } + return blocks; +} + +function fillBlock( + heap: PairingHeap, + clusters: Map, + mempool: { [txid: string]: MempoolTransactionExtended }, + maxWeight: number, + maxSigops: number, +): ProjectedBlock { + const block: ProjectedBlock = { txids: [], weight: COINBASE_RESERVED_WEIGHT, sigops: 0 }; + const deferred: ChunkHeapEntry[] = []; + let consecutiveFailed = 0; + let full = false; + + while (!heap.isEmpty() && !full) { + const entry = heap.pop() as ChunkHeapEntry; + const cluster = clusters.get(entry.clusterId); + const chunk = cluster?.chunks[entry.chunkIndex]; + + if (!cluster || !chunk) { + // stale entry + } else if (block.weight + entry.weight < maxWeight + && block.sigops + entry.sigops < maxSigops) { + consecutiveFailed = 0; + block.weight += chunkWeight(chunk, mempool); + block.sigops += entry.sigops; + for (const tx of chunk.txs) { + block.txids.push(tx.txid); + } + if (entry.chunkIndex + 1 < cluster.chunks.length) { + heap.add(makeChunkHeapEntry(cluster, entry.chunkIndex + 1, mempool)); + } + } else { + deferred.push(entry); + consecutiveFailed++; + if (consecutiveFailed > MAX_CONSECUTIVE_FAILURES + && block.weight + MAX_WASTED_WEIGHT > maxWeight) { + full = true; + } + } + } + + for (const entry of deferred) { + heap.add(entry); + } + + return block; +} + +function chunkWeight( + chunk: LinearizationChunk, + mempool: { [txid: string]: MempoolTransactionExtended }, +): number { + let weight = 0; + for (const clusterTx of chunk.txs) { + const mempoolTx = mempool[clusterTx.txid]; + if (mempoolTx) { + weight += mempoolTx.weight; + } + } + return weight; +} diff --git a/backend/src/cluster-mempool/cluster-mempool.ts b/backend/src/cluster-mempool/cluster-mempool.ts new file mode 100644 index 000000000..98d94553c --- /dev/null +++ b/backend/src/cluster-mempool/cluster-mempool.ts @@ -0,0 +1,703 @@ +import { ClusterTx, DepGraph, sortTopological, subgraph } from './depgraph'; +import { linearizeCluster, LinearizationChunk } from './linearize'; +import { ProjectedBlock, assembleBlocks } from './block-builder'; +import { Ancestor, CpfpClusterData, CpfpClusterTx, MempoolTransactionExtended } from '../mempool.interfaces'; +import logger from '../logger'; + +export interface MempoolDiff { + added: MempoolTransactionExtended[]; + removed: string[]; + accelerations: { [txid: string]: { feeDelta: number } }; +} + +export interface ClusterInfo { + clusterId: number; + chunkIndex: number; + chunkFeerate: number; +} + +export { ProjectedBlock }; + +export interface Cluster { + id: number; + depgraph: DepGraph; + txs: Map; + linearization: ClusterTx[]; + chunks: LinearizationChunk[]; + dirty: boolean; +} + +export interface TxCpfpData { + effectiveFeePerVsize: number; + clusterId: number; + chunkIndex: number; + cpfpDirty: boolean; + cpfpChecked: boolean; + ancestors: Ancestor[]; + descendants: Ancestor[]; +} + +const DEFAULT_COST_BUDGET = 75000; + +export class ClusterMempool { + private clusters = new Map(); + private txToCluster = new Map(); + private parentMap = new Map>(); + private spentBy = new Map(); + private mempool: Readonly<{ [txid: string]: MempoolTransactionExtended }>; + private accelerations: { [txid: string]: { feeDelta: number } } = {}; + private nextClusterId = 0; + private modifyTxs: boolean; + private costBudget: number = DEFAULT_COST_BUDGET; + + constructor(mempool: { [txid: string]: MempoolTransactionExtended }, accelerations?: { [txid: string]: { feeDelta: number } }, modifyTxs: boolean = true, costBudget: number = DEFAULT_COST_BUDGET) { + this.mempool = mempool; + if (accelerations) { + this.accelerations = accelerations; + } + this.modifyTxs = modifyTxs; + this.costBudget = costBudget; + this.buildFromMempool(); + } + + applyMempoolChange(diff: MempoolDiff): void { + this.processRemovals(diff.removed); + this.splitDisconnectedClusters(); + this.processAccelerationChanges(diff.accelerations); + this.processAdditions(diff.added); + this.relinearizeDirtyClusters(); + } + + getBlocks(n: number, enforceLimit = false): ProjectedBlock[] { + return assembleBlocks(n, this.clusters, this.mempool, enforceLimit); + } + + getCluster(clusterId: number): CpfpClusterData | null { + const cluster = this.clusters.get(clusterId); + if (!cluster) { + return null; + } + return this.buildClusterData(cluster); + } + + getClusterInfo(txid: string): ClusterInfo | null { + const match = this.getClusterForTx(txid); + if (!match) { + return null; + } + return this.findChunkInfo(match.cluster, match.clusterTx); + } + + getClusterForApi(txid: string): (CpfpClusterData & { chunkIndex: number }) | null { + const info = this.getClusterInfo(txid); + if (!info) { + return null; + } + const cluster = this.getCluster(info.clusterId); + if (!cluster || cluster.txs.length <= 1) { + return null; + } + return { ...cluster, chunkIndex: info.chunkIndex }; + } + + getCpfpDataForTx(txid: string): TxCpfpData | null { + const clusterInfo = this.getClusterForTx(txid); + if (!clusterInfo) { + return null; + } + const chunkInfo = this.findChunkInfo(clusterInfo.cluster, clusterInfo.clusterTx); + if (!chunkInfo) { + return null; + } + const chunk = clusterInfo.cluster.chunks[chunkInfo?.chunkIndex]; + const chunkSet = chunk.txs.length > 1 ? new Set(chunk.txs) : null; + return { + effectiveFeePerVsize: chunkInfo.chunkFeerate, + clusterId: clusterInfo.cluster.id, + chunkIndex: chunkInfo.chunkIndex, + cpfpDirty: true, + cpfpChecked: true, + ancestors: chunkSet ? this.getChunkRelatives(clusterInfo.clusterTx, chunkSet, 'ancestors') : [], + descendants: chunkSet ? this.getChunkRelatives(clusterInfo.clusterTx, chunkSet, 'descendants') : [], + }; + } + + getClusterCount(): number { + return this.clusters.size; + } + + getTxCount(): number { + return this.txToCluster.size; + } + + private getClusterForTx(txid: string): { cluster: Cluster; clusterTx: ClusterTx } | null { + const clusterId = this.txToCluster.get(txid); + if (clusterId === undefined) { + return null; + } + const cluster = this.clusters.get(clusterId); + if (!cluster) { + return null; + } + const clusterTx = cluster.txs.get(txid); + if (!clusterTx) { + return null; + } + return { cluster, clusterTx }; + } + + private buildFromMempool(): void { + this.buildRelativeMaps(); + const components = this.findMempoolComponents(); + for (const component of components) { + this.createClusterFromTxids(component); + } + } + + private buildRelativeMaps(): void { + this.parentMap.clear(); + this.spentBy.clear(); + for (const txid in this.mempool) { + const tx = this.mempool[txid]; + const txParents = new Set(); + for (const vin of tx.vin) { + if (!vin.is_coinbase && this.mempool[vin.txid]) { + txParents.add(vin.txid); + this.spentBy.set(`${vin.txid}:${vin.vout}`, txid); + } + } + if (txParents.size > 0) { + this.parentMap.set(txid, txParents); + } + } + } + + private findMempoolComponents(): Set[] { + const visited = new Set(); + const components: Set[] = []; + + for (const txid in this.mempool) { + if (!visited.has(txid)) { + const component = this.dfsComponent(txid, visited); + components.push(component); + } + } + return components; + } + + private dfsComponent( + startTxid: string, + visited: Set + ): Set { + const component = new Set(); + const stack = [startTxid]; + while (stack.length > 0) { + const current = stack.pop(); + if (current !== undefined && !visited.has(current)) { + visited.add(current); + component.add(current); + + const txParents = this.parentMap.get(current); + if (txParents) { + for (const p of txParents) { + if (!visited.has(p)) { + stack.push(p); + } + } + } + + const tx = this.mempool[current]; + if (tx) { + for (let vout = 0; vout < tx.vout.length; vout++) { + const child = this.spentBy.get(`${current}:${vout}`); + if (child && !visited.has(child)) { + stack.push(child); + } + } + } + } + } + return component; + } + + private effectiveFee(txid: string, tx: MempoolTransactionExtended): number { + return tx.fee + (this.accelerations[txid]?.feeDelta || 0); + } + + private adjustedWeight(tx: MempoolTransactionExtended): number { + return Math.max(tx.weight, (tx.sigops || 0) * 20); + } + + private createClusterFromTxids( + txids: Set + ): Cluster | null { + const clusterId = this.nextClusterId++; + const depgraph = new DepGraph(); + const txMap = new Map(); + + for (const txid of txids) { + const tx = this.mempool[txid]; + if (!tx) { + logger.warn(`Warning: missing mempool tx ${txid} during cluster creation, skipping`); + return null; + } + const clusterTx = depgraph.addTransaction(txid, this.effectiveFee(txid, tx), this.adjustedWeight(tx), tx.order ?? 0); + txMap.set(txid, clusterTx); + } + + for (const txid of txids) { + const txParents = this.parentMap.get(txid); + if (txParents) { + for (const parentTxid of txParents) { + if (txids.has(parentTxid)) { + const parentTx = txMap.get(parentTxid); + const childTx = txMap.get(txid); + if (parentTx && childTx) { + depgraph.addDependency(parentTx, childTx); + } + } + } + } + } + + const { linearization, chunks } = linearizeCluster(depgraph.getTxs(), this.costBudget); + + const cluster: Cluster = { + id: clusterId, + depgraph, + txs: txMap, + linearization, + chunks, + dirty: false, + }; + + this.clusters.set(clusterId, cluster); + for (const txid of txids) { + this.txToCluster.set(txid, clusterId); + } + + if (this.modifyTxs) { + this.writeBackCluster(cluster); + } + return cluster; + } + + public writeBackClusters(): void { + for (const cluster of this.clusters.values()) { + this.writeBackCluster(cluster); + } + } + + private writeBackCluster(cluster: Cluster): void { + for (let chunkIdx = 0; chunkIdx < cluster.chunks.length; chunkIdx++) { + const chunk = cluster.chunks[chunkIdx]; + const chunkFeerate = chunk.weight > 0 ? (chunk.fee * 4) / chunk.weight : 0; + const chunkSet = chunk.txs.length > 1 ? new Set(chunk.txs) : null; + for (const clusterTx of chunk.txs) { + this.writeBackTx(cluster, clusterTx, chunkIdx, chunkFeerate, chunkSet); + } + } + } + + private writeBackTx( + cluster: Cluster, + clusterTx: ClusterTx, + chunkIdx: number, + chunkFeerate: number, + chunkSet: Set | null + ): void { + const txid = clusterTx.txid; + if (!this.mempool[txid]) { + logger.warn(`ClusterMempool.writeBackTx: ${txid} missing from mempool (cluster ${cluster.id})`); + return; + } + const tx = this.mempool[txid]; + if (tx.effectiveFeePerVsize !== chunkFeerate || tx.clusterId !== cluster.id) { + tx.cpfpDirty = true; + } + tx.effectiveFeePerVsize = chunkFeerate; + tx.clusterId = cluster.id; + tx.chunkIndex = chunkIdx; + tx.cpfpChecked = true; + + if (chunkSet) { + tx.ancestors = this.getChunkRelatives(clusterTx, chunkSet, 'ancestors'); + tx.descendants = this.getChunkRelatives(clusterTx, chunkSet, 'descendants'); + } else { + tx.ancestors = []; + tx.descendants = []; + } + } + + private getChunkRelatives( + clusterTx: ClusterTx, + chunkSet: Set, + direction: 'ancestors' | 'descendants' + ): { txid: string; fee: number; weight: number }[] { + const relatives: { txid: string; fee: number; weight: number }[] = []; + const related = direction === 'ancestors' ? clusterTx.ancestors : clusterTx.descendants; + for (const rel of related) { + if (rel !== clusterTx && chunkSet.has(rel)) { + const mempoolTx = this.mempool[rel.txid]; + if (mempoolTx) { + relatives.push({ txid: rel.txid, fee: mempoolTx.fee, weight: mempoolTx.weight }); + } else { + logger.warn(`ClusterMempool.getChunkRelatives: ${rel.txid} missing from mempool`); + } + } + } + return relatives; + } + + private processRemovals(removed: string[]): void { + for (const txid of removed) { + const tx = this.mempool[txid]; + if (tx) { + for (const vin of tx.vin) { + if (!vin.is_coinbase) { + this.spentBy.delete(`${vin.txid}:${vin.vout}`); + } + } + } else if (this.txToCluster.has(txid)) { + logger.warn(`ClusterMempool.processRemovals: ${txid} missing from mempool, spentBy cleanup skipped`); + } + } + + for (const txid of removed) { + const match = this.getClusterForTx(txid); + if (match) { + match.cluster.depgraph.removeTransactions(new Set([match.clusterTx])); + match.cluster.txs.delete(txid); + match.cluster.linearization = match.cluster.linearization.filter(t => t !== match.clusterTx); + this.txToCluster.delete(txid); + match.cluster.dirty = true; + } + } + } + + private splitDisconnectedClusters(): void { + for (const [clusterId, cluster] of this.clusters.entries()) { + if (cluster.dirty) { + if (cluster.depgraph.size === 0) { + this.clusters.delete(clusterId); + } else { + const components = cluster.depgraph.findConnectedComponents(); + if (components.length > 1) { + this.clusters.delete(clusterId); + for (const component of components) { + this.splitComponentToCluster(cluster, component); + } + } + } + } + } + } + + private splitComponentToCluster(sourceCluster: Cluster, component: Set): void { + const newClusterId = this.nextClusterId++; + const { depgraph: newDepgraph, txMap } = subgraph(component); + + const newTxs = new Map(); + for (const oldTx of component) { + const newTx = txMap.get(oldTx); + if (newTx) { + newTxs.set(oldTx.txid, newTx); + this.txToCluster.set(oldTx.txid, newClusterId); + } + } + + const newLinearization: ClusterTx[] = []; + for (const oldTx of sourceCluster.linearization) { + if (component.has(oldTx)) { + const newTx = txMap.get(oldTx); + if (newTx) { + newLinearization.push(newTx); + } + } + } + + const newCluster: Cluster = { + id: newClusterId, + dirty: true, + depgraph: newDepgraph, + txs: newTxs, + linearization: newLinearization, + chunks: [], + }; + this.clusters.set(newClusterId, newCluster); + } + + private processAdditions(added: MempoolTransactionExtended[]): void { + for (const tx of added) { + const txid = tx.txid; + + for (const vin of tx.vin) { + if (!vin.is_coinbase) { + this.spentBy.set(`${vin.txid}:${vin.vout}`, txid); + } + } + + const { relatedClusterIds, parentTxids, childTxids } = this.findRelatedClusters(tx); + + if (relatedClusterIds.size === 0) { + this.addSingletonCluster(tx); + } else if (relatedClusterIds.size === 1) { + this.addToExistingCluster(tx, relatedClusterIds, parentTxids, childTxids); + } else { + this.mergeAndAddToCluster(tx, relatedClusterIds, parentTxids, childTxids); + } + } + } + + private findRelatedClusters(tx: MempoolTransactionExtended): { + relatedClusterIds: Set; + parentTxids: string[]; + childTxids: string[]; + } { + const relatedClusterIds = new Set(); + const parentTxids: string[] = []; + const childTxids: string[] = []; + + for (const vin of tx.vin) { + if (!vin.is_coinbase && this.mempool[vin.txid]) { + const parentCluster = this.txToCluster.get(vin.txid); + if (parentCluster !== undefined) { + relatedClusterIds.add(parentCluster); + parentTxids.push(vin.txid); + } + } + } + + for (let vout = 0; vout < tx.vout.length; vout++) { + const childTxid = this.spentBy.get(`${tx.txid}:${vout}`); + if (childTxid && this.mempool[childTxid]) { + const childCluster = this.txToCluster.get(childTxid); + if (childCluster !== undefined) { + relatedClusterIds.add(childCluster); + childTxids.push(childTxid); + } + } + } + + return { relatedClusterIds, parentTxids, childTxids }; + } + + private addSingletonCluster(tx: MempoolTransactionExtended): void { + const txid = tx.txid; + const clusterId = this.nextClusterId++; + const depgraph = new DepGraph(); + const clusterTx = depgraph.addTransaction(txid, this.effectiveFee(txid, tx), this.adjustedWeight(tx), tx.order ?? 0); + + const cluster: Cluster = { + id: clusterId, + dirty: true, + depgraph, + txs: new Map([[txid, clusterTx]]), + linearization: [clusterTx], + chunks: [], + }; + this.clusters.set(clusterId, cluster); + this.txToCluster.set(txid, clusterId); + } + + private addToExistingCluster( + tx: MempoolTransactionExtended, + relatedClusterIds: Set, + parentTxids: string[], + childTxids: string[], + ): void { + const txid = tx.txid; + const clusterId = relatedClusterIds.values().next().value; + const cluster = this.clusters.get(clusterId); + if (!cluster) { + return; + } + const clusterTx = cluster.depgraph.addTransaction(txid, this.effectiveFee(txid, tx), this.adjustedWeight(tx), tx.order ?? 0); + cluster.txs.set(txid, clusterTx); + cluster.linearization.push(clusterTx); + this.txToCluster.set(txid, clusterId); + + this.addParentDeps(cluster, clusterTx, parentTxids); + this.addChildDeps(cluster, clusterTx, childTxids); + cluster.dirty = true; + } + + private mergeAndAddToCluster( + tx: MempoolTransactionExtended, + relatedClusterIds: Set, + parentTxids: string[], + childTxids: string[], + ): void { + const clusterIterator = relatedClusterIds.values(); + const primaryId: number = clusterIterator.next().value; + const primary = this.clusters.get(primaryId); + if (!primary) { + return; + } + + for (const clusterId of clusterIterator) { + const other = this.clusters.get(clusterId); + if (other) { + this.mergeClusterInto(primary, other); + this.clusters.delete(clusterId); + } + } + + const clusterTx = primary.depgraph.addTransaction(tx.txid, this.effectiveFee(tx.txid, tx), this.adjustedWeight(tx), tx.order ?? 0); + primary.txs.set(tx.txid, clusterTx); + primary.linearization.push(clusterTx); + this.txToCluster.set(tx.txid, primaryId); + + this.addParentDeps(primary, clusterTx, parentTxids); + this.addChildDeps(primary, clusterTx, childTxids); + primary.dirty = true; + } + + private addParentDeps(cluster: Cluster, childTx: ClusterTx, parentTxids: string[]): void { + for (const parentTxid of parentTxids) { + const parentTx = cluster.txs.get(parentTxid); + if (parentTx) { + cluster.depgraph.addDependency(parentTx, childTx); + } + } + } + + private addChildDeps(cluster: Cluster, parentTx: ClusterTx, childTxids: string[]): void { + for (const childTxid of childTxids) { + const childTx = cluster.txs.get(childTxid); + if (childTx) { + cluster.depgraph.addDependency(parentTx, childTx); + } + } + } + + private mergeClusterInto(primary: Cluster, other: Cluster): void { + for (const [txid, otherTx] of other.txs) { + const newTx = primary.depgraph.addTransaction(txid, otherTx.effectiveFee, otherTx.weight, otherTx.order); + primary.txs.set(txid, newTx); + this.txToCluster.set(txid, primary.id); + } + + for (const otherTx of other.depgraph.getTxs()) { + for (const parent of otherTx.parents) { + const newChild = primary.txs.get(otherTx.txid); + const newParent = primary.txs.get(parent.txid); + if (newChild && newParent) { + primary.depgraph.addDependency(newParent, newChild); + } + } + } + + for (const otherTx of other.linearization) { + const newTx = primary.txs.get(otherTx.txid); + if (newTx) { + primary.linearization.push(newTx); + } + } + } + + private processAccelerationChanges(newAccelerations: { [txid: string]: { feeDelta: number } }): void { + const changed = new Set(); + for (const txid in newAccelerations) { + if ((newAccelerations[txid]?.feeDelta || 0) !== (this.accelerations[txid]?.feeDelta || 0)) { + changed.add(txid); + } + } + for (const txid in this.accelerations) { + if (!newAccelerations[txid]) { + changed.add(txid); + } + } + this.accelerations = newAccelerations; + for (const txid of changed) { + const tx = this.mempool[txid]; + if (!tx) { + continue; + } + const match = this.getClusterForTx(txid); + if (match) { + match.clusterTx.effectiveFee = this.effectiveFee(txid, tx); + match.cluster.dirty = true; + } + } + } + + private relinearizeDirtyClusters(): void { + for (const [clusterId, cluster] of this.clusters.entries()) { + if (cluster.dirty) { + cluster.dirty = false; + const newId = this.nextClusterId++; + this.clusters.delete(clusterId); + cluster.id = newId; + this.clusters.set(newId, cluster); + for (const txid of cluster.txs.keys()) { + this.txToCluster.set(txid, newId); + } + + const { linearization, chunks } = linearizeCluster( + cluster.depgraph.getTxs(), + this.costBudget, + cluster.linearization, + ); + cluster.linearization = linearization; + cluster.chunks = chunks; + + if (this.modifyTxs) { + this.writeBackCluster(cluster); + } + } + } + } + + private buildClusterData(cluster: Cluster): CpfpClusterData { + const txs: CpfpClusterTx[] = []; + const txToFlatIdx = new Map(); + + for (const chunk of cluster.chunks) { + const ordered = sortTopological(new Set(chunk.txs)); + for (const clusterTx of ordered) { + if (this.mempool[clusterTx.txid]) { + txToFlatIdx.set(clusterTx, txs.length); + const parents: number[] = []; + for (const parentTx of clusterTx.parents) { + const flatIdx = txToFlatIdx.get(parentTx); + if (flatIdx !== undefined) { + parents.push(flatIdx); + } + } + const mempoolTx = this.mempool[clusterTx.txid]; + txs.push({ txid: clusterTx.txid, fee: mempoolTx.fee, weight: mempoolTx.weight, parents }); + } else { + logger.warn(`ClusterMempool.buildClusterData: ${clusterTx.txid} missing from mempool (cluster ${cluster.id})`); + } + } + } + + let offset = 0; + const chunks = cluster.chunks.map(chunk => { + const count = chunk.txs.length; + const chunkEntry = { + txs: Array.from({ length: count }, (_, i) => offset + i), + feerate: chunk.weight > 0 ? (chunk.fee * 4) / chunk.weight : 0, + }; + offset += count; + return chunkEntry; + }); + + return { txs, chunks }; + } + + private findChunkInfo(cluster: Cluster, tx: ClusterTx): ClusterInfo | null { + for (let chunkIdx = 0; chunkIdx < cluster.chunks.length; chunkIdx++) { + const chunk = cluster.chunks[chunkIdx]; + if (chunk.txs.includes(tx)) { + return { + clusterId: cluster.id, + chunkIndex: chunkIdx, + chunkFeerate: chunk.weight > 0 ? (chunk.fee * 4) / chunk.weight : 0, + }; + } + } + return null; + } +} diff --git a/backend/src/cluster-mempool/depgraph.ts b/backend/src/cluster-mempool/depgraph.ts new file mode 100644 index 000000000..76e98e3f2 --- /dev/null +++ b/backend/src/cluster-mempool/depgraph.ts @@ -0,0 +1,165 @@ +import logger from '../logger'; + +export class ClusterTx { + txid: string; + effectiveFee: number; + weight: number; + order: number; + ancestors: Set; + descendants: Set; + parents: Set; + children: Set; + + constructor(txid: string, effectiveFee: number, weight: number, order: number) { + this.txid = txid; + this.effectiveFee = effectiveFee; + this.weight = weight; + this.order = order; + this.ancestors = new Set([this]); + this.descendants = new Set([this]); + this.parents = new Set(); + this.children = new Set(); + } +} + +export class DepGraph { + private txs: Set = new Set(); + + get size(): number { + return this.txs.size; + } + + addTransaction(txid: string, fee: number, weight: number, order: number = 0): ClusterTx { + const tx = new ClusterTx(txid, fee, weight, order); + this.txs.add(tx); + return tx; + } + + addDependency(parent: ClusterTx, child: ClusterTx): void { + if (!this.txs.has(parent) || !this.txs.has(child)) { + logger.warn(`Warning: invalid dependency, skipping`); + return; + } + + parent.children.add(child); + child.parents.add(parent); + + if (child.ancestors.has(parent)) { + return; + } + + for (const descendant of child.descendants) { + for (const ancestor of parent.ancestors) { + descendant.ancestors.add(ancestor); + ancestor.descendants.add(descendant); + } + } + } + + removeTransactions(toRemove: Set): void { + for (const tx of toRemove) { + for (const parent of tx.parents) { + parent.children.delete(tx); + } + for (const child of tx.children) { + child.parents.delete(tx); + } + this.txs.delete(tx); + } + + for (const tx of this.txs) { + for (const removed of toRemove) { + tx.ancestors.delete(removed); + tx.descendants.delete(removed); + } + } + + this.rederiveAncestorsDescendants(); + } + + private rederiveAncestorsDescendants(): void { + const ordered = [...this.txs].sort((a, b) => a.ancestors.size - b.ancestors.size); + for (const tx of this.txs) { + tx.ancestors = new Set([tx]); + tx.descendants = new Set([tx]); + } + for (const tx of ordered) { + for (const parent of tx.parents) { + for (const ancestor of parent.ancestors) { + tx.ancestors.add(ancestor); + } + } + for (const ancestor of tx.ancestors) { + ancestor.descendants.add(tx); + } + } + } + + hasTx(tx: ClusterTx): boolean { + return this.txs.has(tx); + } + + getTxs(): Set { + return this.txs; + } + + findConnectedComponents(): Set[] { + const visited = new Set(); + const components: Set[] = []; + + for (const tx of this.txs) { + if (!visited.has(tx)) { + const component = new Set(); + const stack: ClusterTx[] = [tx]; + while (stack.length > 0) { + const node = stack.pop(); + if (node && !visited.has(node)) { + visited.add(node); + component.add(node); + for (const a of node.ancestors) { + if (!visited.has(a) && this.txs.has(a)) { + stack.push(a); + } + } + for (const d of node.descendants) { + if (!visited.has(d) && this.txs.has(d)) { + stack.push(d); + } + } + } + } + components.push(component); + } + } + return components; + } + +} + +export function sortTopological(subset: Set): ClusterTx[] { + return [...subset].sort((a, b) => a.ancestors.size - b.ancestors.size); +} + +export function subgraph(txSubset: Set): { depgraph: DepGraph; txMap: Map } { + const newGraph = new DepGraph(); + const txMap = new Map(); + + for (const oldTx of txSubset) { + const newTx = newGraph.addTransaction(oldTx.txid, oldTx.effectiveFee, oldTx.weight, oldTx.order); + txMap.set(oldTx, newTx); + } + + for (const oldTx of txSubset) { + for (const parent of oldTx.parents) { + if (txSubset.has(parent)) { + const newChild = txMap.get(oldTx); + const newParent = txMap.get(parent); + if (newChild && newParent) { + newGraph.addDependency(newParent, newChild); + } + } + } + } + + return { depgraph: newGraph, txMap }; +} diff --git a/backend/src/cluster-mempool/linearize.ts b/backend/src/cluster-mempool/linearize.ts new file mode 100644 index 000000000..8b1c0465d --- /dev/null +++ b/backend/src/cluster-mempool/linearize.ts @@ -0,0 +1,1314 @@ +import { ClusterTx } from './depgraph'; + +function higherFeerate(aFee: number, aWeight: number, bFee: number, bWeight: number): boolean { + return aFee * bWeight > bFee * aWeight; +} + +export interface LinearizationChunk { + txs: ClusterTx[]; + fee: number; + weight: number; +} + +export function chunkify(linearization: ClusterTx[]): LinearizationChunk[] { + const chunks: LinearizationChunk[] = []; + + for (const tx of linearization) { + chunks.push({ txs: [tx], fee: tx.effectiveFee, weight: tx.weight }); + + while (chunks.length >= 2) { + const last = chunks[chunks.length - 1]; + const prev = chunks[chunks.length - 2]; + if (higherFeerate(last.fee, last.weight, prev.fee, prev.weight)) { + prev.txs.push(...last.txs); + prev.fee += last.fee; + prev.weight += last.weight; + chunks.pop(); + } else { + break; + } + } + } + + return chunks; +} + +export function postLinearize(linearization: ClusterTx[]): ClusterTx[] { + if (linearization.length <= 1) { + return [...linearization]; + } + let result = [...linearization]; + result = postLinearizePass(result, true); + result = postLinearizePass(result, false); + return result; +} + +interface PostLinGroup { + txs: ClusterTx[]; + deps: Set; + fee: number; + weight: number; +} + +function postLinearizePass(lin: ClusterTx[], forward: boolean): ClusterTx[] { + const n = lin.length; + if (n <= 1) { + return [...lin]; + } + + const input = forward ? lin : [...lin].reverse(); + const feeMul = forward ? 1 : -1; + + const groups: PostLinGroup[] = []; + const seen = new Set(); + + for (const tx of input) { + const deps = new Set(); + const related = forward ? tx.parents : tx.children; + for (const r of related) { + if (seen.has(r)) { + deps.add(r); + } + } + seen.add(tx); + + groups.push({ + txs: [tx], + deps, + fee: tx.effectiveFee * feeMul, + weight: tx.weight, + }); + + let pos = groups.length - 1; + while (pos > 0) { + const cur = groups[pos]; + const prev = groups[pos - 1]; + if (groupsDepsOverlap(cur, prev)) { + mergeGroupIntoCurrent(groups, pos); + pos--; + } else if (higherFeerate(cur.fee, cur.weight, prev.fee, prev.weight)) { + swapAdjacentGroups(groups, pos); + pos--; + } else { + break; + } + } + } + + const result: ClusterTx[] = []; + for (const g of groups) { + for (const tx of g.txs) { + result.push(tx); + } + } + + if (!forward) { + result.reverse(); + } + + return result; +} + +function groupsDepsOverlap(cur: PostLinGroup, prev: PostLinGroup): boolean { + for (const tx of prev.txs) { + if (cur.deps.has(tx)) { + return true; + } + } + return false; +} + +function mergeGroupIntoCurrent(groups: PostLinGroup[], pos: number): void { + const cur = groups[pos]; + const prev = groups[pos - 1]; + prev.txs.push(...cur.txs); + for (const d of cur.deps) { + prev.deps.add(d); + } + prev.fee += cur.fee; + prev.weight += cur.weight; + groups.splice(pos, 1); +} + +function swapAdjacentGroups(groups: PostLinGroup[], pos: number): void { + const tmp = groups[pos]; + groups[pos] = groups[pos - 1]; + groups[pos - 1] = tmp; +} + +function pickRandomTx(txs: Set): ClusterTx | null { + const arr = [...txs]; + if (arr.length === 0) { + return null; + } + return arr[Math.floor(Math.random() * arr.length)]; +} + +interface SFLChunk { + id: number; + txs: Set; + fee: number; + weight: number; +} + +interface SFLDependency { + parent: ClusterTx; + child: ClusterTx; + active: boolean; +} + +const enum MergeDir { Up, Down, Both } + +interface SFLCost { cost: number; } + +export function spanningForestLinearize( + txs: Set, + costBudget: number, + existingLinearization?: ClusterTx[], +): ClusterTx[] { + const allTxs = [...txs]; + if (allTxs.length === 0) { + return []; + } + if (allTxs.length === 1) { + return [...allTxs]; + } + + const deps = collectDirectDeps(allTxs); + + if (deps.length === 0) { + return sortByFeerateDesc(allTxs); + } + + const { chunks, txToChunk, nextChunkId: startNextId } = initSFLChunks(allTxs); + + const cost: SFLCost = { cost: 87 * allTxs.length + 4 * deps.length }; + + if (existingLinearization && existingLinearization.length > 0) { + for (const tx of existingLinearization) { + const chunkId = txs.has(tx) ? txToChunk.get(tx) : undefined; + if (chunkId !== undefined) { + mergeUpwards(chunkId, deps, chunks, txToChunk, cost); + } + } + } + + let nextId = makeTopological(deps, chunks, txToChunk, startNextId, cost); + if (cost.cost < costBudget) { + nextId = optimizeSFL(deps, chunks, txToChunk, nextId, costBudget, cost); + } + if (cost.cost < costBudget) { + minimizeSFL(deps, chunks, txToChunk, nextId, costBudget, cost); + } + + return extractLinearization(chunks, txToChunk); +} + +function collectDirectDeps(txs: ClusterTx[]): SFLDependency[] { + const deps: SFLDependency[] = []; + for (const tx of txs) { + for (const parent of tx.parents) { + deps.push({ parent, child: tx, active: false }); + } + } + return deps; +} + +function sortByFeerateDesc(txs: ClusterTx[]): ClusterTx[] { + return [...txs].sort((a, b) => { + if (higherFeerate(a.effectiveFee, a.weight, b.effectiveFee, b.weight)) { + return -1; + } + if (higherFeerate(b.effectiveFee, b.weight, a.effectiveFee, a.weight)) { + return 1; + } + return a.order - b.order; + }); +} + +function initSFLChunks( + txs: ClusterTx[] +): { chunks: Map; txToChunk: Map; nextChunkId: number } { + let nextChunkId = 0; + const txToChunk = new Map(); + const chunks = new Map(); + + for (const tx of txs) { + const chunkId = nextChunkId++; + chunks.set(chunkId, { + id: chunkId, + txs: new Set([tx]), + fee: tx.effectiveFee, + weight: tx.weight, + }); + txToChunk.set(tx, chunkId); + } + + return { chunks, txToChunk, nextChunkId }; +} + +function mergeChunks( + dstId: number, + srcId: number, + chunks: Map, + txToChunk: Map +): void { + const dst = chunks.get(dstId); + const src = chunks.get(srcId); + if (!dst || !src) { + return; + } + for (const tx of src.txs) { + dst.txs.add(tx); + txToChunk.set(tx, dstId); + } + dst.fee += src.fee; + dst.weight += src.weight; + chunks.delete(srcId); +} + +function activateInternalDeps( + chunkId: number, + deps: SFLDependency[], + txToChunk: Map, + cost: SFLCost, + chunks?: Map, +): void { + for (const d of deps) { + if (!d.active) { + const c1 = txToChunk.get(d.parent); + const c2 = txToChunk.get(d.child); + if (c1 === chunkId && c2 === chunkId) { + d.active = true; + } + } + } + const mergedChunkSize = chunks?.get(chunkId)?.txs.size ?? 0; + cost.cost += 10 * mergedChunkSize + 1; +} + +function pickMergeCandidateUp( + chunkId: number, + chunk: SFLChunk, + deps: SFLDependency[], + chunks: Map, + txToChunk: Map, + cost: SFLCost, +): number | null { + let bestChunkId: number | null = null; + let bestFee = 0; + let bestWeight = 0; + let bestTiebreak = 0; + const visited = new Set(); + for (const dep of deps) { + if (!dep.active) { + const childChunk = txToChunk.get(dep.child); + const parentChunkId = txToChunk.get(dep.parent); + if (childChunk === chunkId && parentChunkId !== chunkId && parentChunkId !== undefined) { + visited.add(parentChunkId); + const pChunk = chunks.get(parentChunkId); + if (pChunk && !higherFeerate(pChunk.fee, pChunk.weight, chunk.fee, chunk.weight)) { + const tiebreak = Math.random(); + if (bestChunkId === null + || higherFeerate(bestFee, bestWeight, pChunk.fee, pChunk.weight) + || (!higherFeerate(pChunk.fee, pChunk.weight, bestFee, bestWeight) && tiebreak > bestTiebreak)) { + bestChunkId = parentChunkId; + bestFee = pChunk.fee; + bestWeight = pChunk.weight; + bestTiebreak = tiebreak; + } + } + } + } + } + cost.cost += 8 * visited.size; + return bestChunkId; +} + +function mergeStep( + dir: MergeDir.Up | MergeDir.Down, + chunkId: number, + deps: SFLDependency[], + chunks: Map, + txToChunk: Map, + cost: SFLCost, +): number | null { + if (dir === MergeDir.Up) { + return mergeStepUp(chunkId, deps, chunks, txToChunk, cost); + } + return mergeStepDown(chunkId, deps, chunks, txToChunk, cost); +} + +function mergeStepUp( + chunkId: number, + deps: SFLDependency[], + chunks: Map, + txToChunk: Map, + cost: SFLCost, +): number | null { + const chunk = chunks.get(chunkId); + if (!chunk) { + return null; + } + const parentChunkId = pickMergeCandidateUp(chunkId, chunk, deps, chunks, txToChunk, cost); + if (parentChunkId === null) { + return null; + } + const dep = pickRandomCrossChunkDep(parentChunkId, chunkId, deps, txToChunk, chunks, cost); + if (!dep) { + return null; + } + dep.active = true; + mergeChunks(chunkId, parentChunkId, chunks, txToChunk); + activateInternalDeps(chunkId, deps, txToChunk, cost, chunks); + return chunkId; +} + +function mergeStepDown( + chunkId: number, + deps: SFLDependency[], + chunks: Map, + txToChunk: Map, + cost: SFLCost, +): number | null { + const chunk = chunks.get(chunkId); + if (!chunk) { + return null; + } + const childChunkId = pickMergeCandidateDown(chunkId, chunk, deps, chunks, txToChunk, cost); + if (childChunkId === null) { + return null; + } + const dep = pickRandomCrossChunkDep(chunkId, childChunkId, deps, txToChunk, chunks, cost); + if (!dep) { + return null; + } + dep.active = true; + mergeChunks(chunkId, childChunkId, chunks, txToChunk); + activateInternalDeps(chunkId, deps, txToChunk, cost, chunks); + return chunkId; +} + +function makeTopological( + deps: SFLDependency[], + chunks: Map, + txToChunk: Map, + nextChunkId: number, + cost: SFLCost, +): number { + const queue: number[] = []; + const onQueue = new Set(); + for (const [chunkId] of chunks) { + queue.push(chunkId); + const j = Math.floor(Math.random() * queue.length); + if (j !== queue.length - 1) { + [queue[queue.length - 1], queue[j]] = [queue[j], queue[queue.length - 1]]; + } + onQueue.add(chunkId); + } + + const mergedChunks = new Set(); + const initDir: MergeDir = Math.random() < 0.5 ? MergeDir.Up : MergeDir.Down; + let numSteps = 0; + + while (queue.length > 0) { + const chunkId = queue.shift(); + if (chunkId === undefined) { + break; + } + onQueue.delete(chunkId); + if (!chunks.has(chunkId)) { + continue; + } + numSteps++; + + const dir = mergedChunks.has(chunkId) ? MergeDir.Both : initDir; + const first = Math.random() < 0.5 ? MergeDir.Up : MergeDir.Down; + const second = first === MergeDir.Up ? MergeDir.Down : MergeDir.Up; + + let result: number | null = null; + if (dir === MergeDir.Both || dir === first) { + result = mergeStep(first, chunkId, deps, chunks, txToChunk, cost); + } + if (result === null && (dir === MergeDir.Both || dir === second)) { + result = mergeStep(second, chunkId, deps, chunks, txToChunk, cost); + } + + if (result !== null) { + if (!onQueue.has(result)) { + onQueue.add(result); + queue.push(result); + } + mergedChunks.add(result); + } + } + + cost.cost += 20 * chunks.size + 28 * numSteps; + + return nextChunkId; +} + +function mergeUpwards( + chunkId: number, + deps: SFLDependency[], + chunks: Map, + txToChunk: Map, + cost: SFLCost, +): void { + let done = false; + while (!done) { + done = mergeStepUp(chunkId, deps, chunks, txToChunk, cost) === null; + } +} + +function pickMergeCandidateDown( + chunkId: number, + chunk: SFLChunk, + deps: SFLDependency[], + chunks: Map, + txToChunk: Map, + cost: SFLCost, +): number | null { + let bestChunkId: number | null = null; + let bestFee = 0; + let bestWeight = 0; + let bestTiebreak = 0; + const visited = new Set(); + for (const dep of deps) { + if (!dep.active) { + const parentChunk = txToChunk.get(dep.parent); + const childChunkId = txToChunk.get(dep.child); + if (parentChunk === chunkId && childChunkId !== chunkId && childChunkId !== undefined) { + visited.add(childChunkId); + const cChunk = chunks.get(childChunkId); + if (cChunk && !higherFeerate(chunk.fee, chunk.weight, cChunk.fee, cChunk.weight)) { + const tiebreak = Math.random(); + if (bestChunkId === null + || higherFeerate(cChunk.fee, cChunk.weight, bestFee, bestWeight) + || (!higherFeerate(bestFee, bestWeight, cChunk.fee, cChunk.weight) && tiebreak > bestTiebreak)) { + bestChunkId = childChunkId; + bestFee = cChunk.fee; + bestWeight = cChunk.weight; + bestTiebreak = tiebreak; + } + } + } + } + } + cost.cost += 8 * visited.size; + return bestChunkId; +} + +function pickRandomCrossChunkDep( + topChunkId: number, + bottomChunkId: number, + deps: SFLDependency[], + txToChunk: Map, + chunks: Map, + cost: SFLCost, +): SFLDependency | null { + const topChunk = chunks.get(topChunkId); + const candidates: SFLDependency[] = []; + let scanSteps = 0; + for (const d of deps) { + if (!d.active) { + const pChunk = txToChunk.get(d.parent); + const cChunk = txToChunk.get(d.child); + if (pChunk === topChunkId && cChunk === bottomChunkId) { + candidates.push(d); + } + scanSteps++; + } + } + cost.cost += 2 * (topChunk?.txs.size ?? 0); + cost.cost += 3 * scanSteps + 5; + if (candidates.length === 0) { + return null; + } + return candidates[Math.floor(Math.random() * candidates.length)]; +} + +function mergeDownwards( + chunkId: number, + deps: SFLDependency[], + chunks: Map, + txToChunk: Map, + cost: SFLCost, +): void { + let done = false; + while (!done) { + done = mergeStepDown(chunkId, deps, chunks, txToChunk, cost) === null; + } +} + +function buildChunkAdjacency( + deps: SFLDependency[], + chunkTxs: Set, + excludeDep?: SFLDependency +): Map> { + const adj = new Map>(); + for (const tx of chunkTxs) { + adj.set(tx, new Set()); + } + for (const d of deps) { + if (d !== excludeDep && d.active && chunkTxs.has(d.parent) && chunkTxs.has(d.child)) { + const parentAdj = adj.get(d.parent); + const childAdj = adj.get(d.child); + if (parentAdj) { + parentAdj.add(d.child); + } + if (childAdj) { + childAdj.add(d.parent); + } + } + } + return adj; +} + +function bfsReachable(adj: Map>, start: ClusterTx): Set { + const visited = new Set(); + const queue: ClusterTx[] = [start]; + visited.add(start); + while (queue.length > 0) { + const node = queue.shift(); + if (node) { + const neighbors = adj.get(node); + if (neighbors) { + for (const neighbor of neighbors) { + if (!visited.has(neighbor)) { + visited.add(neighbor); + queue.push(neighbor); + } + } + } + } + } + return visited; +} + +function optimizeSFL( + deps: SFLDependency[], + chunks: Map, + txToChunk: Map, + nextChunkId: number, + maxCost: number, + cost: SFLCost, +): number { + const queue: number[] = []; + const onQueue = new Set(); + for (const [chunkId] of chunks) { + queue.push(chunkId); + const j = Math.floor(Math.random() * queue.length); + if (j !== queue.length - 1) { + [queue[queue.length - 1], queue[j]] = [queue[j], queue[queue.length - 1]]; + } + onQueue.add(chunkId); + } + + cost.cost += 13 * chunks.size; + + while (cost.cost < maxCost) { + let chunkId: number | undefined; + let numPopped = 0; + while (queue.length > 0) { + const candidate = queue.shift(); + if (candidate === undefined) { + break; + } + numPopped++; + onQueue.delete(candidate); + if (chunks.has(candidate)) { + chunkId = candidate; + break; + } + } + cost.cost += 1 * numPopped + 4; + if (chunkId === undefined) { + break; + } + + const chunk = chunks.get(chunkId); + if (!chunk) { + break; + } + + const split = pickDependencyToSplit(deps, chunk, chunkId, txToChunk, cost); + if (split) { + const result = splitAndMerge( + split.dep, split.parentSide, chunkId, chunk, + deps, chunks, txToChunk, nextChunkId, cost, + ); + nextChunkId = result.nextChunkId; + + if (!onQueue.has(chunkId) && chunks.has(chunkId)) { + onQueue.add(chunkId); + queue.push(chunkId); + } + if (!onQueue.has(result.childChunkId) && chunks.has(result.childChunkId)) { + onQueue.add(result.childChunkId); + queue.push(result.childChunkId); + } + } + } + + return nextChunkId; +} + +function computeDepTopSet( + dep: SFLDependency, + chunk: SFLChunk, + chunkId: number, + deps: SFLDependency[], + txToChunk: Map, +): { parentSide: Set; topFee: number; topWeight: number } | null { + if (!dep.active + || txToChunk.get(dep.parent) !== chunkId + || txToChunk.get(dep.child) !== chunkId) { + return null; + } + const adj = buildChunkAdjacency(deps, chunk.txs, dep); + const parentSide = bfsReachable(adj, dep.parent); + if (parentSide.has(dep.child)) { + return null; + } + let topFee = 0; + let topWeight = 0; + for (const tx of parentSide) { + topFee += tx.effectiveFee; + topWeight += tx.weight; + } + return { parentSide, topFee, topWeight }; +} + +function pickDependencyToSplit( + deps: SFLDependency[], + chunk: SFLChunk, + chunkId: number, + txToChunk: Map, + cost: SFLCost, +): { dep: SFLDependency; parentSide: Set } | null { + let best: { dep: SFLDependency; parentSide: Set } | null = null; + let bestTiebreak = 0; + + for (const dep of deps) { + const split = computeDepTopSet(dep, chunk, chunkId, deps, txToChunk); + if (split && split.topFee * chunk.weight > chunk.fee * split.topWeight) { + const tiebreak = Math.random(); + if (tiebreak >= bestTiebreak) { + bestTiebreak = tiebreak; + best = { dep, parentSide: split.parentSide }; + } + } + } + + cost.cost += 8 * chunk.txs.size + 9; + return best; +} + +function splitAndMerge( + dep: SFLDependency, + parentSide: Set, + parentChunkId: number, + parentChunk: SFLChunk, + deps: SFLDependency[], + chunks: Map, + txToChunk: Map, + nextChunkId: number, + cost: SFLCost, +): { nextChunkId: number; childChunkId: number } { + dep.active = false; + + const origChunkSize = parentChunk.txs.size; + + const childSide = new Set(); + for (const tx of parentChunk.txs) { + if (!parentSide.has(tx)) { + childSide.add(tx); + } + } + + let childFee = 0; + let childWeight = 0; + for (const tx of childSide) { + childFee += tx.effectiveFee; + childWeight += tx.weight; + } + + const childChunkId = nextChunkId++; + chunks.set(childChunkId, { + id: childChunkId, + txs: childSide, + fee: childFee, + weight: childWeight, + }); + + let parentFee = 0; + let parentWeight = 0; + for (const tx of parentSide) { + parentFee += tx.effectiveFee; + parentWeight += tx.weight; + } + parentChunk.txs = parentSide; + parentChunk.fee = parentFee; + parentChunk.weight = parentWeight; + + for (const tx of childSide) { + txToChunk.set(tx, childChunkId); + } + + for (const d of deps) { + if (d.active && txToChunk.get(d.parent) !== txToChunk.get(d.child)) { + d.active = false; + } + } + + cost.cost += 11 * (origChunkSize - 1) + 8; + + let needsSelfMerge = false; + for (const d of deps) { + if (!d.active && txToChunk.get(d.parent) === parentChunkId && txToChunk.get(d.child) === childChunkId) { + needsSelfMerge = true; + break; + } + } + + if (needsSelfMerge) { + const selfDep = pickRandomCrossChunkDep(childChunkId, parentChunkId, deps, txToChunk, chunks, cost); + if (selfDep) { + selfDep.active = true; + mergeChunks(childChunkId, parentChunkId, chunks, txToChunk); + activateInternalDeps(childChunkId, deps, txToChunk, cost, chunks); + } + } else { + mergeUpwards(parentChunkId, deps, chunks, txToChunk, cost); + mergeDownwards(childChunkId, deps, chunks, txToChunk, cost); + } + + return { nextChunkId, childChunkId }; +} + +interface MinimizeQueueEntry { + chunkId: number; + pivot: ClusterTx; + movePivotDown: boolean; + secondStage: boolean; +} + +function minimizeSFL( + deps: SFLDependency[], + chunks: Map, + txToChunk: Map, + nextChunkId: number, + maxCost: number, + cost: SFLCost, +): number { + const queue: MinimizeQueueEntry[] = []; + + for (const [chunkId, chunk] of chunks) { + const pivot = pickRandomTx(chunk.txs); + if (pivot) { + queue.push({ chunkId, pivot, movePivotDown: Math.random() < 0.5, secondStage: false }); + const j = Math.floor(Math.random() * queue.length); + if (j !== queue.length - 1) { + [queue[queue.length - 1], queue[j]] = [queue[j], queue[queue.length - 1]]; + } + } + } + + cost.cost += 18 * chunks.size; + + while (queue.length > 0 && cost.cost < maxCost) { + const entry = queue.shift(); + if (!entry) { + break; + } + + const chunk = chunks.get(entry.chunkId); + if (chunk) { + nextChunkId = minimizeChunkStep( + chunk, entry, + deps, chunks, txToChunk, nextChunkId, queue, cost, + ); + } + } + + return nextChunkId; +} + +function minimizeChunkStep( + chunk: SFLChunk, + entry: MinimizeQueueEntry, + deps: SFLDependency[], + chunks: Map, + txToChunk: Map, + nextChunkId: number, + queue: MinimizeQueueEntry[], + cost: SFLCost, +): number { + const { chunkId, pivot, movePivotDown, secondStage } = entry; + + let haveAny = false; + let bestDep: SFLDependency | null = null; + let bestParentSide: Set | null = null; + let bestTiebreak = 0; + + for (const dep of deps) { + const split = computeDepTopSet(dep, chunk, chunkId, deps, txToChunk); + if (split && split.topFee * chunk.weight === chunk.fee * split.topWeight) { + haveAny = true; + if (movePivotDown !== split.parentSide.has(pivot)) { + const tiebreak = Math.random(); + if (tiebreak > bestTiebreak) { + bestTiebreak = tiebreak; + bestDep = dep; + bestParentSide = split.parentSide; + } + } + } + } + + cost.cost += 11 * chunk.txs.size + 11; + + if (!haveAny) { + cost.cost += 7; + return nextChunkId; + } + + if (!bestDep || !bestParentSide) { + if (!secondStage) { + queue.push({ chunkId, pivot, movePivotDown: !movePivotDown, secondStage: true }); + } + cost.cost += 7; + return nextChunkId; + } + + const result = splitAndMerge( + bestDep, bestParentSide, chunkId, chunk, + deps, chunks, txToChunk, nextChunkId, cost, + ); + nextChunkId = result.nextChunkId; + const childChunkId = result.childChunkId; + + cost.cost += 17 + 7; + + if (movePivotDown) { + const parentPivot = pickRandomTx(chunks.get(chunkId)?.txs ?? new Set()); + if (parentPivot) { + queue.push({ chunkId, pivot: parentPivot, movePivotDown: Math.random() < 0.5, secondStage: false }); + } + queue.push({ chunkId: childChunkId, pivot, movePivotDown, secondStage }); + } else { + queue.push({ chunkId, pivot, movePivotDown, secondStage }); + const childPivot = pickRandomTx(chunks.get(childChunkId)?.txs ?? new Set()); + if (childPivot) { + queue.push({ chunkId: childChunkId, pivot: childPivot, movePivotDown: Math.random() < 0.5, secondStage: false }); + } + } + + if (queue.length >= 2 && Math.random() < 0.5) { + const last = queue.length - 1; + [queue[last], queue[last - 1]] = [queue[last - 1], queue[last]]; + } + + return nextChunkId; +} + +function chunkCmp(a: SFLChunk, b: SFLChunk, chunkMaxOrder: Map): number { + if (higherFeerate(a.fee, a.weight, b.fee, b.weight)) { + return -1; + } + if (higherFeerate(b.fee, b.weight, a.fee, a.weight)) { + return 1; + } + if (a.weight !== b.weight) { + return b.weight - a.weight; + } + return (chunkMaxOrder.get(a.id) ?? 0) - (chunkMaxOrder.get(b.id) ?? 0); +} + +function txCmp(a: ClusterTx, b: ClusterTx): number { + if (higherFeerate(a.effectiveFee, a.weight, b.effectiveFee, b.weight)) { + return -1; + } + if (higherFeerate(b.effectiveFee, b.weight, a.effectiveFee, a.weight)) { + return 1; + } + if (a.weight !== b.weight) { + return b.weight - a.weight; + } + return a.order - b.order; +} + +function extractLinearization( + chunks: Map, + txToChunk: Map, +): ClusterTx[] { + const chunkList = [...chunks.values()]; + + const chunkMaxOrder = new Map(); + for (const c of chunkList) { + let max = 0; + for (const tx of c.txs) { + if (tx.order > max) { + max = tx.order; + } + } + chunkMaxOrder.set(c.id, max); + } + + const { chunkDeps, chunkChildren } = buildChunkDependencies(chunkList, txToChunk); + + return emitLinearization(chunkList, chunkDeps, chunkChildren, chunkMaxOrder, chunks); +} + +function buildChunkDependencies( + chunkList: SFLChunk[], + txToChunk: Map +): { chunkDeps: Map; chunkChildren: Map } { + const chunkDeps = new Map(); + const chunkChildren = new Map(); + + for (const c of chunkList) { + chunkChildren.set(c.id, []); + } + + for (const c of chunkList) { + const depChunks = new Set(); + for (const tx of c.txs) { + for (const parent of tx.parents) { + const parentChunk = txToChunk.get(parent); + if (parentChunk !== undefined && parentChunk !== c.id) { + depChunks.add(parentChunk); + } + } + } + chunkDeps.set(c.id, depChunks.size); + for (const d of depChunks) { + const children = chunkChildren.get(d); + if (children) { + children.push(c.id); + } + } + } + + return { chunkDeps, chunkChildren }; +} + +function emitLinearization( + chunkList: SFLChunk[], + chunkDeps: Map, + chunkChildren: Map, + chunkMaxOrder: Map, + chunkMap: Map +): ClusterTx[] { + const result: ClusterTx[] = []; + + const readyChunks: SFLChunk[] = []; + for (const c of chunkList) { + if (chunkDeps.get(c.id) === 0) { + readyChunks.push(c); + } + } + readyChunks.sort((a, b) => chunkCmp(a, b, chunkMaxOrder)); + + while (readyChunks.length > 0) { + const chunk = readyChunks.shift(); + if (!chunk) { + break; + } + + emitChunkTxs(chunk, result); + + const children = chunkChildren.get(chunk.id); + if (children) { + for (const childChunkId of children) { + const prevCount = chunkDeps.get(childChunkId) ?? 0; + const newCount = prevCount - 1; + chunkDeps.set(childChunkId, newCount); + if (newCount === 0) { + const childChunk = chunkMap.get(childChunkId); + if (childChunk) { + insertSortedChunk(readyChunks, childChunk, chunkMaxOrder); + } + } + } + } + } + + return result; +} + +function emitChunkTxs( + chunk: SFLChunk, + result: ClusterTx[] +): void { + const txSet = new Set(chunk.txs); + const txDepCount = new Map(); + for (const tx of txSet) { + let count = 0; + for (const parent of tx.parents) { + if (txSet.has(parent)) { + count++; + } + } + txDepCount.set(tx, count); + } + + const readyTxs: ClusterTx[] = []; + for (const tx of txSet) { + if (txDepCount.get(tx) === 0) { + readyTxs.push(tx); + } + } + readyTxs.sort((a, b) => txCmp(a, b)); + + const emitted = new Set(); + while (readyTxs.length > 0) { + const best = readyTxs.shift(); + if (!best) { + break; + } + result.push(best); + emitted.add(best); + + for (const child of best.children) { + if (txSet.has(child) && !emitted.has(child)) { + const prevCount = txDepCount.get(child) ?? 0; + const newCount = prevCount - 1; + txDepCount.set(child, newCount); + if (newCount === 0) { + insertSortedTx(readyTxs, child); + } + } + } + } +} + +function insertSortedChunk(arr: SFLChunk[], item: SFLChunk, chunkMaxOrder: Map): void { + const idx = arr.findIndex(e => chunkCmp(item, e, chunkMaxOrder) < 0); + if (idx === -1) { + arr.push(item); + } else { + arr.splice(idx, 0, item); + } +} + +function insertSortedTx(arr: ClusterTx[], item: ClusterTx): void { + const idx = arr.findIndex(e => txCmp(item, e) < 0); + if (idx === -1) { + arr.push(item); + } else { + arr.splice(idx, 0, item); + } +} + +function minimizeChunks(chunks: LinearizationChunk[]): LinearizationChunk[] { + const result: LinearizationChunk[] = []; + + for (const chunk of chunks) { + if (chunk.txs.length <= 1) { + result.push(chunk); + } else { + const subChunks = splitChunkByComponents(chunk); + result.push(...subChunks); + } + } + + return result; +} + +function splitChunkByComponents(chunk: LinearizationChunk): LinearizationChunk[] { + const txSet = new Set(chunk.txs); + const visited = new Set(); + const components: ClusterTx[][] = []; + + for (const tx of chunk.txs) { + if (!visited.has(tx)) { + const component = bfsComponentWithinChunk(tx, txSet, visited); + components.push(component); + } + } + + if (components.length <= 1) { + return [chunk]; + } + + const posInLin = new Map(); + for (let i = 0; i < chunk.txs.length; i++) { + posInLin.set(chunk.txs[i], i); + } + + components.sort((a, b) => { + let aFee = 0, aWeight = 0; + for (const t of a) { aFee += t.effectiveFee; aWeight += t.weight; } + let bFee = 0, bWeight = 0; + for (const t of b) { bFee += t.effectiveFee; bWeight += t.weight; } + if (higherFeerate(aFee, aWeight, bFee, bWeight)) { return -1; } + if (higherFeerate(bFee, bWeight, aFee, aWeight)) { return 1; } + const aMin = Math.min(...a.map(t => posInLin.get(t) ?? 0)); + const bMin = Math.min(...b.map(t => posInLin.get(t) ?? 0)); + return aMin - bMin; + }); + + return components.map(comp => { + comp.sort((a, b) => (posInLin.get(a) ?? 0) - (posInLin.get(b) ?? 0)); + + let fee = 0; + let weight = 0; + for (const tx of comp) { + fee += tx.effectiveFee; + weight += tx.weight; + } + return { txs: comp, fee, weight }; + }); +} + +function bfsComponentWithinChunk( + start: ClusterTx, + txSet: Set, + visited: Set +): ClusterTx[] { + const component: ClusterTx[] = []; + const queue: ClusterTx[] = [start]; + visited.add(start); + + while (queue.length > 0) { + const node = queue.shift(); + if (!node) { + break; + } + component.push(node); + for (const parent of node.parents) { + if (txSet.has(parent) && !visited.has(parent)) { + visited.add(parent); + queue.push(parent); + } + } + for (const child of node.children) { + if (txSet.has(child) && !visited.has(child)) { + visited.add(child); + queue.push(child); + } + } + } + + return component; +} + +export function linearizeCluster( + txs: Set, + costBudget: number, + existingLinearization?: ClusterTx[], +): { linearization: ClusterTx[]; chunks: LinearizationChunk[] } { + let linearization = spanningForestLinearize(txs, costBudget, existingLinearization); + linearization = postLinearize(linearization); + let chunks = chunkify(linearization); + chunks = minimizeChunks(chunks); + linearization = chunks.flatMap(c => c.txs); + chunks = chunkify(linearization); + chunks = canonicalizeChunkOrder(chunks); + linearization = chunks.flatMap(c => c.txs); + return { linearization, chunks }; +} + +function canonicalizeChunkOrder(chunks: LinearizationChunk[]): LinearizationChunk[] { + if (chunks.length <= 1) { + return chunks; + } + + const txToChunkIdx = new Map(); + for (let i = 0; i < chunks.length; i++) { + for (const tx of chunks[i].txs) { + txToChunkIdx.set(tx, i); + } + } + + const { depCount, chunkChildren } = buildCanonicalChunkDeps(chunks, txToChunkIdx); + const maxOrder = computeChunkMaxOrder(chunks); + + const ready: number[] = []; + for (let i = 0; i < chunks.length; i++) { + if (depCount[i] === 0) { + ready.push(i); + } + } + ready.sort((a, b) => canonicalChunkCmp(chunks, maxOrder, a, b)); + + const result: LinearizationChunk[] = []; + while (ready.length > 0) { + const idx = ready.shift(); + if (idx === undefined) { + break; + } + result.push(chunks[idx]); + + for (const childIdx of chunkChildren[idx]) { + depCount[childIdx]--; + if (depCount[childIdx] === 0) { + insertSortedCanonicalChunk(ready, childIdx, chunks, maxOrder); + } + } + } + + return result; +} + +function buildCanonicalChunkDeps( + chunks: LinearizationChunk[], + txToChunkIdx: Map, +): { depCount: number[]; chunkChildren: number[][] } { + const depCount = new Array(chunks.length).fill(0); + const chunkChildren: number[][] = chunks.map(() => []); + const seen: Set[] = chunks.map(() => new Set()); + + for (let i = 0; i < chunks.length; i++) { + for (const tx of chunks[i].txs) { + for (const parent of tx.parents) { + const parentIdx = txToChunkIdx.get(parent); + if (parentIdx !== undefined && parentIdx !== i && !seen[i].has(parentIdx)) { + seen[i].add(parentIdx); + depCount[i]++; + chunkChildren[parentIdx].push(i); + } + } + } + } + + return { depCount, chunkChildren }; +} + +function computeChunkMaxOrder(chunks: LinearizationChunk[]): number[] { + return chunks.map(chunk => { + let max = 0; + for (const tx of chunk.txs) { + if (tx.order > max) { + max = tx.order; + } + } + return max; + }); +} + +function canonicalChunkCmp(chunks: LinearizationChunk[], maxOrder: number[], a: number, b: number): number { + const ac = chunks[a]; + const bc = chunks[b]; + if (higherFeerate(ac.fee, ac.weight, bc.fee, bc.weight)) { + return -1; + } + if (higherFeerate(bc.fee, bc.weight, ac.fee, ac.weight)) { + return 1; + } + if (ac.weight !== bc.weight) { + return ac.weight - bc.weight; + } + return maxOrder[a] - maxOrder[b]; +} + +function insertSortedCanonicalChunk(ready: number[], idx: number, chunks: LinearizationChunk[], maxOrder: number[]): void { + const insertPos = ready.findIndex(r => canonicalChunkCmp(chunks, maxOrder, idx, r) < 0); + if (insertPos === -1) { + ready.push(idx); + } else { + ready.splice(insertPos, 0, idx); + } +} diff --git a/backend/src/config.ts b/backend/src/config.ts index 9c1762378..6ac08a2f1 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -34,6 +34,8 @@ interface IConfig { POOLS_JSON_TREE_URL: string, POOLS_UPDATE_DELAY: number, AUDIT: boolean; + CLUSTER_MEMPOOL: boolean; + CLUSTER_MEMPOOL_INDEXING: boolean; RUST_GBT: boolean; LIMIT_GBT: boolean; CPFP_INDEXING: boolean; @@ -205,6 +207,8 @@ const defaults: IConfig = { 'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', 'POOLS_UPDATE_DELAY': 604800, // in seconds, default is one week 'AUDIT': false, + 'CLUSTER_MEMPOOL': false, + 'CLUSTER_MEMPOOL_INDEXING': false, 'RUST_GBT': true, 'LIMIT_GBT': false, 'CPFP_INDEXING': false, diff --git a/backend/src/index.ts b/backend/src/index.ts index c2a3d998c..2e7e9a13d 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -334,7 +334,6 @@ class Server { if (config.MEMPOOL.ENABLED) { statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler)); memPool.setAsyncMempoolChangedCallback(websocketHandler.$handleMempoolChange.bind(websocketHandler)); - blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler)); } if (config.FIAT_PRICE.ENABLED) { priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler)); diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 327bbbebf..46256cc29 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -139,12 +139,12 @@ class Indexer { switch (task) { case 'blocksPrices': { if (!['testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) && config.FIAT_PRICE.ENABLED) { - let lastestPriceId; + let latestPriceId; try { - lastestPriceId = await PricesRepository.$getLatestPriceId(); + latestPriceId = await PricesRepository.$getLatestPriceId(); } catch (e) { logger.debug('failed to fetch latest price id from db: ' + (e instanceof Error ? e.message : e)); - } if (priceUpdater.historyInserted === false || lastestPriceId === null) { + } if (priceUpdater.historyInserted === false || latestPriceId === null) { logger.debug(`Blocks prices indexer is waiting for the price updater to complete`, logger.tags.mining); this.scheduleSingleTask(task, 10000); } else { diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index a0888bb50..24e01d4dd 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -28,6 +28,11 @@ export interface PoolStats extends PoolInfo { emptyBlocks: number; } +export enum TemplateAlgorithm { + legacy = 0, + clusterMempool = 1, +} + export interface BlockAudit { version: number, time: number, @@ -45,6 +50,7 @@ export interface BlockAudit { expectedFees?: number, expectedWeight?: number, template?: any[]; + templateAlgorithm?: TemplateAlgorithm, } export interface TransactionAudit { @@ -132,6 +138,8 @@ export interface TransactionExtended extends IEsploraApi.Transaction { replacement?: boolean; uid?: number; flags?: number; + clusterId?: number; + chunkIndex?: number; } export interface MempoolTransactionExtended extends TransactionExtended { @@ -227,6 +235,7 @@ export interface CpfpInfo { adjustedVsize?: number, acceleration?: boolean, fee?: number; + cluster?: CpfpClusterData & { chunkIndex: number }; } export interface TransactionStripped { @@ -388,6 +397,25 @@ export interface CpfpCluster { height: number, txs: Ancestor[], effectiveFeePerVsize: number, + templateAlgorithm?: TemplateAlgorithm, + clusterData?: CpfpClusterData, +} + +export interface CpfpClusterTx { + txid: string; + fee: number; + weight: number; + parents: number[]; +} + +export interface CpfpClusterChunk { + txs: number[]; + feerate: number; +} + +export interface CpfpClusterData { + txs: CpfpClusterTx[]; + chunks: CpfpClusterChunk[]; } export interface CpfpSummary { diff --git a/backend/src/replication/AuditReplication.ts b/backend/src/replication/AuditReplication.ts index d92614746..562de5e2f 100644 --- a/backend/src/replication/AuditReplication.ts +++ b/backend/src/replication/AuditReplication.ts @@ -114,6 +114,7 @@ class AuditReplication { }); await blocksAuditsRepository.$saveAudit({ version: auditSummary.version || 0, + templateAlgorithm: auditSummary.templateAlgorithm ?? 0, hash: blockHash, height: auditSummary.height, time: auditSummary.timestamp || auditSummary.time, diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index e682cab59..9d1be0eb6 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -1,7 +1,7 @@ import DB from '../database'; import logger from '../logger'; import bitcoinApi from '../api/bitcoin/bitcoin-api-factory'; -import { BlockAudit, AuditScore, TransactionAudit, TransactionStripped } from '../mempool.interfaces'; +import { BlockAudit, AuditScore, TransactionAudit, TransactionStripped, TemplateAlgorithm } from '../mempool.interfaces'; interface MigrationAudit { version: number, @@ -18,8 +18,8 @@ class BlocksAuditRepositories { /** @asyncSafe */ public async $saveAudit(audit: BlockAudit): Promise { try { - await DB.query(`INSERT INTO blocks_audits(version, time, height, hash, unseen_txs, missing_txs, added_txs, prioritized_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight) - VALUE (?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.version, audit.time, audit.height, audit.hash, JSON.stringify(audit.unseenTxs), JSON.stringify(audit.missingTxs), + await DB.query(`INSERT INTO blocks_audits(version, template_algo, time, height, hash, unseen_txs, missing_txs, added_txs, prioritized_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight) + VALUE (?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.version, audit.templateAlgorithm ?? 0, audit.time, audit.height, audit.hash, JSON.stringify(audit.unseenTxs), JSON.stringify(audit.missingTxs), JSON.stringify(audit.addedTxs), JSON.stringify(audit.prioritizedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), JSON.stringify(audit.acceleratedTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart @@ -80,6 +80,7 @@ class BlocksAuditRepositories { const [rows]: any[] = await DB.query( `SELECT blocks_audits.version, + blocks_audits.template_algo as templateAlgorithm, blocks_audits.height, blocks_audits.hash as id, UNIX_TIMESTAMP(blocks_audits.time) as timestamp, @@ -120,6 +121,23 @@ class BlocksAuditRepositories { } } + /** @asyncSafe */ + public async $getBlockTemplateAlgo(hash: string): Promise { + try { + const [rows]: any[] = await DB.query( + `SELECT template_algo FROM blocks_audits WHERE hash = ?`, + [hash] + ); + if (rows.length) { + return rows[0].template_algo as TemplateAlgorithm; + } + return null; + } catch (e: any) { + logger.err(`Cannot fetch block template algo from db. Reason: ` + (e instanceof Error ? e.message : e)); + return null; + } + } + /** @asyncSafe */ public async $getBlockTxAudit(hash: string, txid: string): Promise { try { diff --git a/backend/src/repositories/CpfpRepository.ts b/backend/src/repositories/CpfpRepository.ts index 3be43eb64..f3f9dad5c 100644 --- a/backend/src/repositories/CpfpRepository.ts +++ b/backend/src/repositories/CpfpRepository.ts @@ -1,31 +1,47 @@ import { RowDataPacket } from 'mysql2'; import DB from '../database'; import logger from '../logger'; -import { Ancestor, CpfpCluster } from '../mempool.interfaces'; +import { Ancestor, CpfpCluster, CpfpClusterData, CpfpClusterTx, TemplateAlgorithm } from '../mempool.interfaces'; import transactionRepository from '../repositories/TransactionRepository'; class CpfpRepository { - public async $batchSaveClusters(clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[]): Promise { + public async $batchSaveClusters(clusters: CpfpCluster[]): Promise { try { - const clusterValues: [string, number, Buffer, number][] = []; + const clusterValues: [string, number, Buffer, number, number][] = []; const txs: { txid: string, cluster: string }[] = []; for (const cluster of clusters) { if (cluster.txs?.length) { - const roundedEffectiveFee = Math.round(cluster.effectiveFeePerVsize * 100) / 100; - const equalFee = cluster.txs.length > 1 && cluster.txs.reduce((acc, tx) => { - return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee); - }, true); - if (!equalFee) { + const isCM = cluster.templateAlgorithm === TemplateAlgorithm.clusterMempool; + + if (isCM && cluster.clusterData) { clusterValues.push([ cluster.root, cluster.height, - Buffer.from(this.pack(cluster.txs)), - cluster.effectiveFeePerVsize + Buffer.from(this.packCM(cluster.clusterData)), + 0, + TemplateAlgorithm.clusterMempool, ]); - for (const tx of cluster.txs) { + for (const tx of cluster.clusterData.txs) { txs.push({ txid: tx.txid, cluster: cluster.root }); } + } else { + const roundedEffectiveFee = Math.round(cluster.effectiveFeePerVsize * 100) / 100; + const equalFee = cluster.txs.length > 1 && cluster.txs.reduce((acc, tx) => { + return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee); + }, true); + if (!equalFee) { + clusterValues.push([ + cluster.root, + cluster.height, + Buffer.from(this.pack(cluster.txs)), + cluster.effectiveFeePerVsize, + TemplateAlgorithm.legacy, + ]); + for (const tx of cluster.txs) { + txs.push({ txid: tx.txid, cluster: cluster.root }); + } + } } } } @@ -42,11 +58,11 @@ class CpfpRepository { while (chunkIndex < clusterValues.length) { const chunk = clusterValues.slice(chunkIndex, chunkIndex + maxChunk); let query = ` - INSERT IGNORE INTO compact_cpfp_clusters(root, height, txs, fee_rate) + INSERT IGNORE INTO compact_cpfp_clusters(root, height, txs, fee_rate, template_algo) VALUES `; query += chunk.map(chunk => { - return (' (UNHEX(?), ?, ?, ?)'); + return (' (UNHEX(?), ?, ?, ?, ?)'); }) + ';'; const values = chunk.flat(); queries.push({ @@ -86,8 +102,16 @@ class CpfpRepository { ); const cluster = clusterRows[0]; if (cluster?.txs) { - cluster.effectiveFeePerVsize = cluster.fee_rate; - cluster.txs = this.unpack(cluster.txs); + if (cluster.template_algo === TemplateAlgorithm.clusterMempool) { + cluster.templateAlgorithm = TemplateAlgorithm.clusterMempool; + cluster.clusterData = this.unpackCM(cluster.txs); + cluster.txs = cluster.clusterData.txs.map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee })); + cluster.effectiveFeePerVsize = 0; + } else { + cluster.templateAlgorithm = TemplateAlgorithm.legacy; + cluster.effectiveFeePerVsize = cluster.fee_rate; + cluster.txs = this.unpack(cluster.txs); + } return cluster; } return; @@ -105,8 +129,16 @@ class CpfpRepository { ); return clusterRows.map(cluster => { if (cluster?.txs) { - cluster.effectiveFeePerVsize = cluster.fee_rate; - cluster.txs = this.unpack(cluster.txs); + if (cluster.template_algo === TemplateAlgorithm.clusterMempool) { + cluster.templateAlgorithm = TemplateAlgorithm.clusterMempool; + cluster.clusterData = this.unpackCM(cluster.txs); + cluster.txs = cluster.clusterData.txs.map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee })); + cluster.effectiveFeePerVsize = 0; + } else { + cluster.templateAlgorithm = TemplateAlgorithm.legacy; + cluster.effectiveFeePerVsize = cluster.fee_rate; + cluster.txs = this.unpack(cluster.txs); + } return cluster; } else { return null; @@ -119,16 +151,16 @@ class CpfpRepository { try { const [rows] = await DB.query( ` - SELECT txs, height, root from compact_cpfp_clusters + SELECT txs, height, root, template_algo from compact_cpfp_clusters WHERE height >= ? `, [height] ) as RowDataPacket[][]; if (rows?.length) { for (const clusterToDelete of rows) { - const txs = this.unpack(clusterToDelete?.txs); - for (const tx of txs) { - await transactionRepository.$removeTransaction(tx.txid); + const txids = this.extractTxids(clusterToDelete); + for (const txid of txids) { + await transactionRepository.$removeTransaction(txid); } } } @@ -150,16 +182,16 @@ class CpfpRepository { try { const [rows] = await DB.query( ` - SELECT txs, height, root from compact_cpfp_clusters + SELECT txs, height, root, template_algo from compact_cpfp_clusters WHERE height = ? `, [height] ) as RowDataPacket[][]; if (rows?.length) { for (const clusterToDelete of rows) { - const txs = this.unpack(clusterToDelete?.txs); - for (const tx of txs) { - await transactionRepository.$removeTransaction(tx.txid); + const txids = this.extractTxids(clusterToDelete); + for (const txid of txids) { + await transactionRepository.$removeTransaction(txid); } } } @@ -176,6 +208,13 @@ class CpfpRepository { } } + private extractTxids(row: any): string[] { + if (row.template_algo === TemplateAlgorithm.clusterMempool) { + return this.unpackCM(row.txs).txs.map(tx => tx.txid); + } + return this.unpack(row.txs).map(tx => tx.txid); + } + // insert a dummy row to mark that we've indexed as far as this block public async $insertProgressMarker(height: number): Promise { try { @@ -245,6 +284,117 @@ class CpfpRepository { } } + /** + * Pack cluster mempool data into binary format: + * [num_chunks: uint16] + * Per chunk: [num_txs: uint16] + * Per tx (in linearization order, grouped by chunk): + * [txid: 32 bytes LE] [weight: uint32] [fee: uint64] [num_parents: uint8] [parent_indices: uint8 each] + */ + public packCM(clusterData: CpfpClusterData): ArrayBuffer { + const headerSize = 2; + const chunkHeadersSize = clusterData.chunks.length * 2; + let txDataSize = 0; + for (const tx of clusterData.txs) { + txDataSize += 32 + 4 + 8 + 1 + tx.parents.length; + } + const totalSize = headerSize + chunkHeadersSize + txDataSize; + + const buf = new ArrayBuffer(totalSize); + const view = new DataView(buf); + let offset = 0; + + view.setUint16(offset, clusterData.chunks.length); + offset += 2; + + for (const chunk of clusterData.chunks) { + view.setUint16(offset, chunk.txs.length); + offset += 2; + } + + for (const tx of clusterData.txs) { + for (let x = 0; x < 32; x++) { + view.setUint8(offset + (31 - x), parseInt(tx.txid.slice(x * 2, (x * 2) + 2), 16)); + } + offset += 32; + view.setUint32(offset, tx.weight); + offset += 4; + view.setBigUint64(offset, BigInt(Math.round(tx.fee))); + offset += 8; + view.setUint8(offset, tx.parents.length); + offset += 1; + for (const parentIdx of tx.parents) { + view.setUint8(offset, parentIdx); + offset += 1; + } + } + + return buf; + } + + public unpackCM(buf: Buffer): CpfpClusterData { + if (!buf) { + return { txs: [], chunks: [] }; + } + + try { + const arrayBuffer = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength); + const view = new DataView(arrayBuffer); + let offset = 0; + + const numChunks = view.getUint16(offset); + offset += 2; + + const chunkSizes: number[] = []; + for (let i = 0; i < numChunks; i++) { + chunkSizes.push(view.getUint16(offset)); + offset += 2; + } + + const txs: CpfpClusterTx[] = []; + const chunks: { txs: number[], feerate: number }[] = []; + let txIndex = 0; + + for (let chunkIdx = 0; chunkIdx < numChunks; chunkIdx++) { + const chunkTxIndices: number[] = []; + let chunkFee = 0; + let chunkWeight = 0; + + for (let t = 0; t < chunkSizes[chunkIdx]; t++) { + const txid = Array.from(new Uint8Array(arrayBuffer, offset, 32)).reverse().map(b => b.toString(16).padStart(2, '0')).join(''); + offset += 32; + const weight = view.getUint32(offset); + offset += 4; + const fee = Number(view.getBigUint64(offset)); + offset += 8; + const numParents = view.getUint8(offset); + offset += 1; + const parents: number[] = []; + for (let p = 0; p < numParents; p++) { + parents.push(view.getUint8(offset)); + offset += 1; + } + + txs.push({ txid, fee, weight, parents }); + chunkTxIndices.push(txIndex); + chunkFee += fee; + chunkWeight += weight; + txIndex++; + } + + chunks.push({ + txs: chunkTxIndices, + feerate: chunkWeight > 0 ? (chunkFee * 4) / chunkWeight : 0, + }); + } + + return { txs, chunks }; + } catch (e) { + logger.warn(`Failed to unpack CM CPFP cluster. Reason: ` + (e instanceof Error ? e.message : e)); + return { txs: [], chunks: [] }; + } + } + // returns `true` if two sets of CPFP clusters are deeply identical public compareClusters(clustersA: CpfpCluster[], clustersB: CpfpCluster[]): boolean { if (clustersA.length !== clustersB.length) { diff --git a/backend/src/repositories/TransactionRepository.ts b/backend/src/repositories/TransactionRepository.ts index b5067f790..772bfd9ae 100644 --- a/backend/src/repositories/TransactionRepository.ts +++ b/backend/src/repositories/TransactionRepository.ts @@ -1,6 +1,6 @@ import DB from '../database'; import logger from '../logger'; -import { Ancestor, CpfpInfo } from '../mempool.interfaces'; +import { Ancestor, CpfpCluster, CpfpInfo, TemplateAlgorithm } from '../mempool.interfaces'; import cpfpRepository from './CpfpRepository'; class TransactionRepository { @@ -72,6 +72,9 @@ class TransactionRepository { const clusterId = txRows[0].root.toLowerCase(); const cluster = await cpfpRepository.$getCluster(clusterId); if (cluster) { + if (cluster.templateAlgorithm === TemplateAlgorithm.clusterMempool && cluster.clusterData) { + return this.convertCpfpCM(txid, cluster); + } return this.convertCpfp(txid, cluster); } } @@ -116,6 +119,95 @@ class TransactionRepository { effectiveFeePerVsize: cluster.effectiveFeePerVsize, }; } + + private convertCpfpCM(txid: string, cluster: CpfpCluster): CpfpInfo { + const clusterData = cluster.clusterData; + if (!clusterData) { + return { ancestors: [], descendants: [], effectiveFeePerVsize: 0 }; + } + + // Find which chunk this tx belongs to + let txFlatIdx = -1; + let txChunkIndex = -1; + for (let i = 0; i < clusterData.txs.length; i++) { + if (clusterData.txs[i].txid === txid) { + txFlatIdx = i; + break; + } + } + + // Find the chunk containing this tx + for (let chunkIdx = 0; chunkIdx < clusterData.chunks.length; chunkIdx++) { + if (clusterData.chunks[chunkIdx].txs.includes(txFlatIdx)) { + txChunkIndex = chunkIdx; + break; + } + } + + // Derive ancestors/descendants from in-chunk depgraph parents + // For CM, ancestors are the tx's depgraph parents within the cluster, + // descendants are txs that depend on this tx + const ancestors: Ancestor[] = []; + const descendants: Ancestor[] = []; + + if (txFlatIdx >= 0) { + // Build child map + const childMap = new Map(); + for (let i = 0; i < clusterData.txs.length; i++) { + for (const parentIdx of clusterData.txs[i].parents) { + let children = childMap.get(parentIdx); + if (!children) { + children = []; + childMap.set(parentIdx, children); + } + children.push(i); + } + } + + const ancestorSet = new Set(); + const stack = [...clusterData.txs[txFlatIdx].parents]; + while (stack.length) { + const idx = stack.pop(); + if (idx === undefined || ancestorSet.has(idx)) { + continue; + } + ancestorSet.add(idx); + stack.push(...clusterData.txs[idx].parents); + } + + const descendantSet = new Set(); + const dStack = [...(childMap.get(txFlatIdx) || [])]; + while (dStack.length) { + const idx = dStack.pop(); + if (idx === undefined || descendantSet.has(idx)) { + continue; + } + descendantSet.add(idx); + dStack.push(...(childMap.get(idx) || [])); + } + + for (const idx of ancestorSet) { + const tx = clusterData.txs[idx]; + ancestors.push({ txid: tx.txid, weight: tx.weight, fee: tx.fee }); + } + for (const idx of descendantSet) { + const tx = clusterData.txs[idx]; + descendants.push({ txid: tx.txid, weight: tx.weight, fee: tx.fee }); + } + } + + const effectiveFeePerVsize = txChunkIndex >= 0 ? clusterData.chunks[txChunkIndex].feerate : 0; + + return { + ancestors, + descendants, + effectiveFeePerVsize, + cluster: { + ...clusterData, + chunkIndex: txChunkIndex, + }, + }; + } } export default new TransactionRepository(); diff --git a/backend/tsconfig.build.json b/backend/tsconfig.build.json index e3d61b71c..225f0944a 100644 --- a/backend/tsconfig.build.json +++ b/backend/tsconfig.build.json @@ -1,6 +1,6 @@ { "extends": "./tsconfig", - "exclude": ["**/*.test.*", "**/__mocks__/*", "**/__tests__/*"], + "exclude": ["**/*.test.*", "**/__mocks__/*", "**/__tests__/*", "**/__e2e__/*"], "compilerOptions": { "types": ["node"] }, diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index ee8e329a6..4c3e075ce 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -29,6 +29,8 @@ "AUDIT": __MEMPOOL_AUDIT__, "RUST_GBT": __MEMPOOL_RUST_GBT__, "LIMIT_GBT": __MEMPOOL_LIMIT_GBT__, + "CLUSTER_MEMPOOL": __MEMPOOL_CLUSTER_MEMPOOL__, + "CLUSTER_MEMPOOL_INDEXING": __MEMPOOL_CLUSTER_MEMPOOL_INDEXING__, "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__, "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__, "DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__, diff --git a/docker/backend/start.sh b/docker/backend/start.sh index ae0bc616f..bb1351b05 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -33,6 +33,8 @@ __MEMPOOL_POOLS_UPDATE_DELAY__=${MEMPOOL_POOLS_UPDATE_DELAY:=604800} __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false} __MEMPOOL_RUST_GBT__=${MEMPOOL_RUST_GBT:=true} __MEMPOOL_LIMIT_GBT__=${MEMPOOL_LIMIT_GBT:=false} +__MEMPOOL_CLUSTER_MEMPOOL__=${MEMPOOL_CLUSTER_MEMPOOL:=false} +__MEMPOOL_CLUSTER_MEMPOOL_INDEXING__=${MEMPOOL_CLUSTER_MEMPOOL_INDEXING:=false} __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false} __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0} __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6} @@ -197,6 +199,8 @@ sed -i "s!__MEMPOOL_POOLS_UPDATE_DELAY__!${__MEMPOOL_POOLS_UPDATE_DELAY__}!g" me sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json sed -i "s!__MEMPOOL_LIMIT_GBT__!${__MEMPOOL_LIMIT_GBT__}!g" mempool-config.json +sed -i "s!__MEMPOOL_CLUSTER_MEMPOOL__!${__MEMPOOL_CLUSTER_MEMPOOL__}!g" mempool-config.json +sed -i "s!__MEMPOOL_CLUSTER_MEMPOOL_INDEXING__!${__MEMPOOL_CLUSTER_MEMPOOL_INDEXING__}!g" mempool-config.json sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json diff --git a/frontend/src/app/components/cluster-diagram/cluster-diagram.component.html b/frontend/src/app/components/cluster-diagram/cluster-diagram.component.html new file mode 100644 index 000000000..53cd9861b --- /dev/null +++ b/frontend/src/app/components/cluster-diagram/cluster-diagram.component.html @@ -0,0 +1,93 @@ +
+ + + + + + + + + + + + + + + + + + {{ outline.feerate | feeRounding }} sat/vB + + + + + + + + + + + {{ node.feerate | feeRounding }} + + + + +
+

{{ hoverNode.tx.txid | shortenString }}

+

{{ hoverNode.tx.fee | number }} sat

+

+

{{ hoverNode.feerate | feeRounding }} sat/vB

+
+
diff --git a/frontend/src/app/components/cluster-diagram/cluster-diagram.component.scss b/frontend/src/app/components/cluster-diagram/cluster-diagram.component.scss new file mode 100644 index 000000000..eb55be138 --- /dev/null +++ b/frontend/src/app/components/cluster-diagram/cluster-diagram.component.scss @@ -0,0 +1,93 @@ +.graph-container { + position: relative; + width: 100%; + background: var(--stat-box-bg); + padding: 10px 0; + overflow-x: auto; + text-align: center; + + svg { + display: inline-block; + } +} + +.chunk-border { + stroke-dasharray: 6 4; + stroke: var(--fg); + stroke-width: 1.5; + stroke-linejoin: round; + fill: none; + + &.inactive { + opacity: 0.35; + } +} + +.chunk-label { + font-size: 11px; + fill: var(--fg); + + &.inactive { + opacity: 0.4; + } +} + +.edge-hitarea { + cursor: pointer; +} + +.edge-line { + stroke-width: 2; + transition: stroke 150ms; + pointer-events: none; +} + +.node-group { + cursor: pointer; + + &.inactive-fill { + opacity: 0.45; + } +} + +.node-rect { + stroke: transparent; + stroke-width: 4; + transition: stroke 150ms; + + &.current { + stroke: var(--mainnet-alt); + } + + &.related { + stroke: rgba(255, 255, 255, 0.6); + } + + &.hovered { + stroke: white; + } +} + +.node-label { + font-size: 12px; + fill: white; + pointer-events: none; + font-weight: 700; +} + +.cluster-tooltip { + position: absolute; + background: color-mix(in srgb, var(--active-bg) 95%, transparent); + border-radius: 4px; + box-shadow: 1px 1px 10px rgba(0, 0, 0, 0.5); + color: var(--tooltip-grey); + padding: 10px 15px; + text-align: left; + pointer-events: none; + max-width: 350px; + + p { + margin: 0; + white-space: nowrap; + } +} diff --git a/frontend/src/app/components/cluster-diagram/cluster-diagram.component.ts b/frontend/src/app/components/cluster-diagram/cluster-diagram.component.ts new file mode 100644 index 000000000..59e327285 --- /dev/null +++ b/frontend/src/app/components/cluster-diagram/cluster-diagram.component.ts @@ -0,0 +1,212 @@ +import { Component, Input, OnChanges, SimpleChanges, ChangeDetectionStrategy, ElementRef, ViewChild, ChangeDetectorRef, HostListener } from '@angular/core'; +import { Router } from '@angular/router'; +import { CpfpClusterTx, CpfpClusterChunk } from '@app/interfaces/node-api.interface'; +import { ThemeService } from '@app/services/theme.service'; +import { StateService } from '@app/services/state.service'; +import { feeLevels } from '@app/app.constants'; +import { computeGridLayout, GridLayout } from './cluster-layout'; +import { renderLayout, RenderedNode, RenderedEdge, RenderedChunkOutline, NODE_W, NODE_H } from './cluster-renderer'; + +const NODE_RX = 6; +const RESIZE_DEBOUNCE_MS = 100; + +@Component({ + selector: 'app-cluster-diagram', + templateUrl: './cluster-diagram.component.html', + styleUrls: ['./cluster-diagram.component.scss'], + standalone: false, + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class ClusterDiagramComponent implements OnChanges { + @Input() cluster: { txs: CpfpClusterTx[]; chunks: CpfpClusterChunk[]; chunkIndex: number }; + @Input() txid: string; + + @ViewChild('graphContainer', { static: true }) graphContainer: ElementRef; + @ViewChild('tooltip') tooltipElement: ElementRef; + + txs: CpfpClusterTx[] = []; + chunks: CpfpClusterChunk[] = []; + gridLayout: GridLayout | null = null; + nodes: RenderedNode[] = []; + edges: RenderedEdge[] = []; + chunkOutlines: RenderedChunkOutline[] = []; + svgWidth = 0; + svgHeight = 0; + activeChunkIndex = 0; + + hoverNode: RenderedNode | null = null; + tooltipPosition = { x: 0, y: 0 }; + + readonly nodeW = NODE_W; + readonly nodeH = NODE_H; + readonly nodeRx = NODE_RX; + + private resizeTimer: ReturnType | null = null; + + constructor( + private router: Router, + private themeService: ThemeService, + private stateService: StateService, + private cd: ChangeDetectorRef, + ) {} + + ngOnChanges(changes: SimpleChanges): void { + if (!this.cluster?.txs?.length) { return; } + this.activeChunkIndex = this.cluster.chunkIndex; + const clusterChanged = changes['cluster']; + if (clusterChanged) { + this.computeLayout(); + } + this.renderLayout(); + } + + private computeLayout(): void { + this.txs = this.cluster.txs; + this.chunks = this.cluster.chunks; + this.gridLayout = computeGridLayout(this.txs, this.chunks); + } + + private renderLayout(): void { + if (!this.gridLayout) { + return; + } + const containerWidth = this.graphContainer?.nativeElement?.clientWidth || 600; + const colors = this.themeService.mempoolFeeColors; + const getColor = (feerate: number): string => { + let index = feeLevels.findIndex((level: number) => feerate < level); + if (index < 0) { index = feeLevels.length; } + index = Math.min(index, colors.length - 1); + return '#' + colors[index]; + }; + + const result = renderLayout(this.gridLayout, { + containerWidth, + activeChunkIndex: this.activeChunkIndex, + currentTxid: this.txid, + txFees: this.txs.map(tx => tx.fee), + txWeights: this.txs.map(tx => tx.weight), + txids: this.txs.map(tx => tx.txid), + chunkFeerates: this.chunks.map(c => c.feerate), + getColor, + }); + + this.nodes = result.nodes; + this.edges = result.edges; + this.chunkOutlines = result.chunkOutlines; + this.svgWidth = result.svgWidth; + this.svgHeight = result.svgHeight; + } + + onNodeEnter(node: RenderedNode, event: MouseEvent): void { + this.hoverNode = node; + this.clearHighlights(); + node.hovered = true; + for (const edge of this.edges) { + if (edge.parentIndex === node.index) { + this.nodes[edge.childIndex].related = true; + edge.highlighted = true; + } else if (edge.childIndex === node.index) { + this.nodes[edge.parentIndex].related = true; + edge.highlighted = true; + } + } + this.updateTooltipPosition(event); + this.cd.markForCheck(); + } + + onNodeMove(event: MouseEvent): void { + this.updateTooltipPosition(event); + this.cd.markForCheck(); + } + + onNodeLeave(): void { + this.hoverNode = null; + this.clearHighlights(); + this.cd.markForCheck(); + } + + onEdgeEnter(edgeIndex: number): void { + this.clearHighlights(); + const edge = this.edges[edgeIndex]; + edge.highlighted = true; + this.nodes[edge.parentIndex].related = true; + this.nodes[edge.childIndex].related = true; + this.cd.markForCheck(); + } + + onEdgeLeave(): void { + this.clearHighlights(); + this.cd.markForCheck(); + } + + onNodeClick(node: RenderedNode): void { + const network = this.stateService.network; + const prefix = network && network !== 'mainnet' ? `/${network}` : ''; + this.router.navigate([prefix + '/tx/', node.tx.txid]); + } + + private clearHighlights(): void { + for (const node of this.nodes) { + node.hovered = false; + node.related = false; + } + for (const edge of this.edges) { + edge.highlighted = false; + } + } + + private updateTooltipPosition(event: MouseEvent): void { + if (!this.graphContainer) { return; } + const container = this.graphContainer.nativeElement; + const rect = container.getBoundingClientRect(); + let x = event.clientX - rect.left + container.scrollLeft + 15; + let y = event.clientY - rect.top + container.scrollTop + 15; + + if (this.tooltipElement) { + const tipRect = this.tooltipElement.nativeElement.getBoundingClientRect(); + const visibleLeft = container.scrollLeft; + const visibleRight = visibleLeft + rect.width; + const visibleTop = container.scrollTop; + const visibleBottom = visibleTop + rect.height; + + if (x + tipRect.width > visibleRight) { + x = Math.max(visibleLeft, visibleRight - tipRect.width - 10); + } + if (x < visibleLeft) { + x = visibleLeft; + } + if (y + tipRect.height > visibleBottom) { + y = y - tipRect.height - 30; + } + if (y < visibleTop) { + y = visibleTop; + } + } + + this.tooltipPosition = { x, y }; + } + + trackByNodeIndex(_index: number, node: RenderedNode): number { + return node.index; + } + + trackByEdgeId(_index: number, edge: RenderedEdge): string { + return edge.gradientId; + } + + trackByChunkIndex(_index: number, outline: RenderedChunkOutline): number { + return outline.chunkIndex; + } + + @HostListener('window:resize') + onResize(): void { + if (this.resizeTimer !== null) { + clearTimeout(this.resizeTimer); + } + this.resizeTimer = setTimeout(() => { + this.resizeTimer = null; + this.renderLayout(); + this.cd.markForCheck(); + }, RESIZE_DEBOUNCE_MS); + } +} diff --git a/frontend/src/app/components/cluster-diagram/cluster-layout.ts b/frontend/src/app/components/cluster-diagram/cluster-layout.ts new file mode 100644 index 000000000..c9d0e2c38 --- /dev/null +++ b/frontend/src/app/components/cluster-diagram/cluster-layout.ts @@ -0,0 +1,1245 @@ +export function cellKey(col: number, row: number): string { + return `${col}:${row}`; +} +export function cellCol(key: string): number { + return parseInt(key, 10); +} +export function cellRow(key: string): number { + return parseInt(key.substring(key.indexOf(':') + 1), 10); +} + +export interface GridNode { + index: number; + col: number; + row: number; + chunkIndex: number; +} + +export interface SlotInfo { + slot: number; + count: number; +} + +export interface GridEdge { + parent: number; + child: number; + waypoints: { col: number; row: number }[]; + exitSlot: number; + exitCount: number; + entrySlot: number; + entryCount: number; + verticalSlots: SlotInfo[]; + horizontalSlots: SlotInfo[]; +} + +export interface ChunkRegion { + chunkIndex: number; + cells: Set; + bridges: Set; +} + +export interface GridLayout { + nodes: GridNode[]; + edges: GridEdge[]; + chunks: ChunkRegion[]; + cols: number; + rows: number; + rowLaneCounts: number[]; + colLaneCounts: number[]; +} + +interface TxInput { + parents: number[]; +} + +interface ChunkInput { + txs: number[]; +} + +export function computeGridLayout(txs: TxInput[], chunks: ChunkInput[]): GridLayout { + const n = txs.length; + if (n === 0) { + return { nodes: [], edges: [], chunks: [], cols: 0, rows: 0, rowLaneCounts: [], colLaneCounts: [] }; + } + + const children: number[][] = txs.map(() => []); + for (let i = 0; i < n; i++) { + for (const p of txs[i].parents) { + children[p].push(i); + } + } + + const txChunk = new Int32Array(n); + for (let ci = 0; ci < chunks.length; ci++) { + for (const ti of chunks[ci].txs) { + txChunk[ti] = ci; + } + } + + const cols = assignColumns(txs, children, n); + const maxCol = Math.max(...cols); + + const colNodes: number[][] = Array.from({ length: maxCol + 1 }, () => []); + for (let i = 0; i < n; i++) { + colNodes[cols[i]].push(i); + } + + const totalRows = Math.max(...colNodes.map(cn => cn.length)); + const rows = assignRows(txs, children, txChunk, colNodes, n, maxCol, totalRows); + const maxRow = Math.max(...rows); + + const nodes: GridNode[] = []; + for (let i = 0; i < n; i++) { + nodes.push({ index: i, col: cols[i], row: rows[i], chunkIndex: txChunk[i] }); + } + + const edges = routeEdges(txs, nodes, maxRow); + optimizeCrossings(edges, nodes, maxRow); + const rowLaneCounts = new Array(maxRow + 1).fill(0); + assignSlots(edges, rowLaneCounts); + + const colLaneCounts = new Array(maxCol + 1).fill(0); + for (const e of edges) { + for (let si = 0; si < e.verticalSlots.length; si++) { + const col = e.waypoints[si].col; + colLaneCounts[col] = Math.max(colLaneCounts[col], e.verticalSlots[si].count); + } + } + + const chunkRegions = computeChunkRegions(nodes, chunks); + + return { nodes, edges, chunks: chunkRegions, cols: maxCol + 1, rows: maxRow + 1, rowLaneCounts, colLaneCounts }; +} + +function assignColumns(txs: TxInput[], children: number[][], n: number): Int32Array { + const cols = new Int32Array(n); + for (let i = 0; i < n; i++) { + if (txs[i].parents.length === 0) { + cols[i] = 0; + } else { + let maxParentCol = 0; + for (const p of txs[i].parents) { + maxParentCol = Math.max(maxParentCol, cols[p]); + } + cols[i] = maxParentCol + 1; + } + } + + const maxCol = Math.max(...cols); + for (let i = 0; i < n; i++) { + if (children[i].length === 0) { + cols[i] = maxCol; + } + } + return cols; +} + +function assignRows( + txs: TxInput[], children: number[][], txChunk: Int32Array, + colNodes: number[][], n: number, maxCol: number, totalRows: number, +): Int32Array { + const rows = new Int32Array(n); + const placed = new Uint8Array(n); + + for (let c = maxCol; c >= 0; c--) { + const layer = colNodes[c]; + if (layer.length === 0) { continue; } + + const targets = layer.map(i => { + if (children[i].length > 0) { + let sum = 0; + let count = 0; + for (const ch of children[i]) { + if (placed[ch]) { + sum += rows[ch]; + count++; + } + } + if (count > 0) { return sum / count; } + } + return (totalRows - 1) / 2; + }); + + const sorted = layer.map((nodeIdx, arrIdx) => ({ nodeIdx, arrIdx, target: targets[arrIdx], chunk: txChunk[nodeIdx] })); + sorted.sort((a, b) => { + const dt = a.target - b.target; + if (Math.abs(dt) > 0.001) { return dt; } + return a.chunk - b.chunk; + }); + + const slots = pickSlots(sorted.map(s => s.target), totalRows); + for (let k = 0; k < sorted.length; k++) { + rows[sorted[k].nodeIdx] = slots[k]; + placed[sorted[k].nodeIdx] = 1; + } + } + + for (let pass = 0; pass < 4; pass++) { + const forward = pass % 2 === 0; + for (let c = forward ? 0 : maxCol; forward ? c <= maxCol : c >= 0; forward ? c++ : c--) { + const layer = colNodes[c]; + if (layer.length <= 1) { continue; } + + const targets = layer.map(i => { + const neighbors = forward ? txs[i].parents : children[i]; + if (neighbors.length === 0) { return rows[i]; } + let sum = 0; + for (const nb of neighbors) { sum += rows[nb]; } + return sum / neighbors.length; + }); + + const indexed = layer.map((_, idx) => idx); + indexed.sort((a, b) => targets[a] - targets[b]); + + const currentSlots = layer.map(i => rows[i]).sort((a, b) => a - b); + for (let k = 0; k < indexed.length; k++) { + rows[layer[indexed[k]]] = currentSlots[k]; + } + } + } + + compactRows(rows, n); + for (let i = 0; i < n; i++) { rows[i] *= 2; } + return rows; +} + +function pickSlots(targets: number[], totalRows: number): number[] { + const count = targets.length; + if (count >= totalRows) { + return targets.map((_, i) => i); + } + + const used = new Set(); + const result = new Array(count); + const order = targets.map((t, i) => ({ i, t })).sort((a, b) => a.t - b.t); + + for (const { i, t } of order) { + let best = Math.round(t); + best = Math.max(0, Math.min(totalRows - 1, best)); + if (used.has(best)) { + let found = false; + for (let d = 1; d < totalRows; d++) { + if (best + d < totalRows && !used.has(best + d)) { best = best + d; found = true; break; } + if (best - d >= 0 && !used.has(best - d)) { best = best - d; found = true; break; } + } + if (!found) { best = 0; } + } + used.add(best); + result[i] = best; + } + + return result; +} + +function compactRows(rows: Int32Array, n: number): void { + const usedRows = new Set(); + for (let i = 0; i < n; i++) { usedRows.add(rows[i]); } + const sorted = [...usedRows].sort((a, b) => a - b); + const remap = new Map(); + sorted.forEach((r, idx) => remap.set(r, idx)); + for (let i = 0; i < n; i++) { rows[i] = remap.get(rows[i]) ?? 0; } +} + +function routeEdges(txs: TxInput[], nodes: GridNode[], maxRow: number): GridEdge[] { + const nodeAt = new Map(); + for (const nd of nodes) { + nodeAt.set(cellKey(nd.col, nd.row), nd.index); + } + + const edges: GridEdge[] = []; + for (let i = 0; i < txs.length; i++) { + for (const p of txs[i].parents) { + const pNode = nodes[p]; + const cNode = nodes[i]; + const waypoints = computeWaypoints(pNode, cNode, nodeAt, maxRow); + edges.push({ + parent: p, child: i, waypoints, + exitSlot: 0, exitCount: 1, + entrySlot: 0, entryCount: 1, + verticalSlots: waypoints.map(() => ({ slot: 0, count: 1 })).slice(0, -1), + horizontalSlots: waypoints.length > 2 + ? waypoints.slice(1, -1).map(() => ({ slot: 0, count: 1 })) + : [], + }); + } + } + + return edges; +} + +function computeWaypoints( + pNode: GridNode, cNode: GridNode, + nodeAt: Map, maxRow: number +): { col: number; row: number }[] { + const waypoints: { col: number; row: number }[] = [{ col: pNode.col, row: pNode.row }]; + + if (cNode.col - pNode.col <= 1) { + waypoints.push({ col: cNode.col, row: cNode.row }); + return waypoints; + } + + for (let c = pNode.col + 1; c < cNode.col; c++) { + const frac = (c - pNode.col) / (cNode.col - pNode.col); + const idealRow = pNode.row + frac * (cNode.row - pNode.row); + let bestRow = Math.max(0, Math.min(maxRow, Math.round(idealRow))); + + if (nodeAt.has(cellKey(c, bestRow))) { + for (let dr = 1; dr <= maxRow; dr++) { + if (bestRow + dr <= maxRow && !nodeAt.has(cellKey(c, bestRow + dr))) { bestRow = bestRow + dr; break; } + if (bestRow - dr >= 0 && !nodeAt.has(cellKey(c, bestRow - dr))) { bestRow = bestRow - dr; break; } + } + } + + waypoints.push({ col: c, row: bestRow }); + } + + waypoints.push({ col: cNode.col, row: cNode.row }); + return waypoints; +} + +interface VSeg { + edgeIdx: number; + segIdx: number; + fromRow: number; + toRow: number; + minRow: number; + maxRow: number; +} + +interface Seg { + edgeIdx: number; + type: 'exit' | 'h' | 'entry'; + hIdx: number; + row: number; + col: number; + nextRow: number | null; + prevRow: number | null; +} + +type CellOrder = (Seg | Seg[])[]; + +interface LaneSeg { + edgeIdx: number; + col: number; + type: 'exit' | 'entry' | 'h' | 'approach'; + hIdx: number; + order: number; +} + +interface SlotIndex { + verticalGroups: Map; + cellIndex: Map; +} + +function cellOrderFlatLen(ord: CellOrder): number { + let n = 0; + for (const item of ord) { n += Array.isArray(item) ? item.length : 1; } + return n; +} + +function cellOrderFindPos(ord: CellOrder, edgeIdx: number): { pos: number; tied: boolean } | null { + let pos = 0; + for (const item of ord) { + if (Array.isArray(item)) { + if (item.some(s => s.edgeIdx === edgeIdx)) { return { pos, tied: true }; } + pos += item.length; + } else { + if (item.edgeIdx === edgeIdx) { return { pos, tied: false }; } + pos++; + } + } + return null; +} + +function writeOrderToSlots(ord: CellOrder, edges: GridEdge[]): void { + const count = cellOrderFlatLen(ord); + let pos = 0; + for (const item of ord) { + const segs = Array.isArray(item) ? item : [item]; + for (const seg of segs) { + if (seg.type === 'exit') { + edges[seg.edgeIdx].exitSlot = pos; + edges[seg.edgeIdx].exitCount = count; + } else if (seg.type === 'entry') { + edges[seg.edgeIdx].entrySlot = pos; + edges[seg.edgeIdx].entryCount = count; + } else { + edges[seg.edgeIdx].horizontalSlots[seg.hIdx] = { slot: pos, count }; + } + pos++; + } + } +} + +function buildSlotIndex(edges: GridEdge[]): SlotIndex { + const verticalGroups = new Map(); + const cellIndex = new Map(); + + for (let ei = 0; ei < edges.length; ei++) { + const wp = edges[ei].waypoints; + + for (let i = 0; i < wp.length - 1; i++) { + let vg = verticalGroups.get(wp[i].col); + if (!vg) { vg = []; verticalGroups.set(wp[i].col, vg); } + vg.push({ + edgeIdx: ei, segIdx: i, fromRow: wp[i].row, toRow: wp[i + 1].row, + minRow: Math.min(wp[i].row, wp[i + 1].row), + maxRow: Math.max(wp[i].row, wp[i + 1].row), + }); + } + + const hPositions: { type: 'exit' | 'h' | 'entry'; hIdx: number; row: number; col: number }[] = []; + hPositions.push({ type: 'exit', hIdx: -1, row: wp[0].row, col: wp[0].col }); + for (let j = 1; j < wp.length - 1; j++) { + hPositions.push({ type: 'h', hIdx: j - 1, row: wp[j].row, col: wp[j].col }); + } + hPositions.push({ type: 'entry', hIdx: -1, row: wp[wp.length - 1].row, col: wp[wp.length - 1].col }); + + for (let k = 0; k < hPositions.length; k++) { + const hp = hPositions[k]; + const nextRow = k < hPositions.length - 1 ? hPositions[k + 1].row : null; + const prevRow = k > 0 ? hPositions[k - 1].row : null; + const seg: Seg = { edgeIdx: ei, type: hp.type, hIdx: hp.hIdx, row: hp.row, col: hp.col, nextRow, prevRow }; + const key = cellKey(seg.col, seg.row); + let arr = cellIndex.get(key); + if (!arr) { arr = []; cellIndex.set(key, arr); } + arr.push(seg); + } + } + + return { verticalGroups, cellIndex }; +} + +function initialHorizontalOrdering(cellIndex: Map, edges: GridEdge[]): Map { + const cellOrderings = new Map(); + + for (const [key, segs] of cellIndex) { + segs.sort((a, b) => { + if (a.nextRow === null && b.nextRow === null) { return 0; } + if (a.nextRow === null) { return 1; } + if (b.nextRow === null) { return -1; } + return a.nextRow - b.nextRow; + }); + + const ordering: CellOrder = []; + let i = 0; + while (i < segs.length) { + let j = i + 1; + while (j < segs.length) { + const sameGroup = (segs[i].nextRow === null && segs[j].nextRow === null) + || (segs[i].nextRow !== null && segs[j].nextRow !== null && segs[i].nextRow === segs[j].nextRow); + if (!sameGroup) { break; } + j++; + } + if (j - i === 1) { + ordering.push(segs[i]); + } else { + ordering.push(segs.slice(i, j)); + } + i = j; + } + + cellOrderings.set(key, ordering); + writeOrderToSlots(ordering, edges); + } + + return cellOrderings; +} + +function compareTiedSegs( + a: Seg, b: Seg, row: number, + hPos: Map, vSlot: Map, + forward: boolean, +): number { + const aH = hPos.get(a.edgeIdx); + const bH = hPos.get(b.edgeIdx); + if (aH !== undefined && bH !== undefined && aH !== bH) { return aH - bH; } + + const aGoesV = aH === undefined; + const bGoesV = bH === undefined; + const aNb = aGoesV ? (forward ? a.prevRow : a.nextRow) : null; + const bNb = bGoesV ? (forward ? b.prevRow : b.nextRow) : null; + const aDir = aNb !== null ? (aNb < row ? -1 : aNb > row ? 1 : 0) : 0; + const bDir = bNb !== null ? (bNb < row ? -1 : bNb > row ? 1 : 0) : 0; + if (aDir !== bDir) { return aDir - bDir; } + + if (aGoesV && bGoesV) { + const aV = vSlot.get(a.edgeIdx); + const bV = vSlot.get(b.edgeIdx); + if (aV !== undefined && bV !== undefined && aV !== bV) { + const travelDown = forward ? aDir < 0 : aDir > 0; + if (travelDown) { return bV - aV; } + const travelUp = forward ? aDir > 0 : aDir < 0; + if (travelUp) { return aV - bV; } + } + } + + if (!aGoesV && bGoesV) { return -1; } + if (aGoesV && !bGoesV) { return 1; } + return 0; +} + +function compareJoiners( + a: VSeg, b: VSeg, + cellOrderings: Map, edges: GridEdge[], +): number { + const aGroup = a.toRow > a.fromRow ? 0 : a.toRow === a.fromRow ? 1 : 2; + const bGroup = b.toRow > b.fromRow ? 0 : b.toRow === b.fromRow ? 1 : 2; + if (aGroup !== bGroup) { return aGroup - bGroup; } + + const aWp = edges[a.edgeIdx].waypoints; + const aSrcOrd = cellOrderings.get(cellKey(aWp[a.segIdx].col, aWp[a.segIdx].row)); + const aPos = (aSrcOrd ? cellOrderFindPos(aSrcOrd, a.edgeIdx) : null)?.pos ?? 0; + + const bWp = edges[b.edgeIdx].waypoints; + const bSrcOrd = cellOrderings.get(cellKey(bWp[b.segIdx].col, bWp[b.segIdx].row)); + const bPos = (bSrcOrd ? cellOrderFindPos(bSrcOrd, b.edgeIdx) : null)?.pos ?? 0; + + if (aGroup === 0) { + if (a.toRow !== b.toRow) { return b.toRow - a.toRow; } + const aDist = a.toRow - a.fromRow, bDist = b.toRow - b.fromRow; + if (aDist !== bDist) { return aDist - bDist; } + return bPos - aPos; + } else if (aGroup === 2) { + if (a.toRow !== b.toRow) { return a.toRow - b.toRow; } + const aDist = a.fromRow - a.toRow, bDist = b.fromRow - b.toRow; + if (aDist !== bDist) { return aDist - bDist; } + return aPos - bPos; + } else { + return aPos - bPos; + } +} + +function compareContinuers(a: VSeg, b: VSeg, prevVSlot: Map): number { + const aV = prevVSlot.get(a.edgeIdx) ?? 0; + const bV = prevVSlot.get(b.edgeIdx) ?? 0; + return aV - bV; +} + +function recordHPositions( + ord: CellOrder, row: number, + posMap: Map>, +): void { + let pm = posMap.get(row); + if (!pm) { pm = new Map(); posMap.set(row, pm); } + let pos = 0; + for (const item of ord) { + if (Array.isArray(item)) { + for (const seg of item) { pm.set(seg.edgeIdx, pos); } + pos += item.length; + } else { + pm.set(item.edgeIdx, pos); + pos++; + } + } +} + +function sortAndRegroupTied( + items: Seg[], row: number, + hPos: Map, vSlot: Map, + forward: boolean, +): CellOrder { + const sorted = [...items]; + sorted.sort((a, b) => compareTiedSegs(a, b, row, hPos, vSlot, forward)); + const result: CellOrder = []; + let k = 0; + while (k < sorted.length) { + let l = k + 1; + while (l < sorted.length && compareTiedSegs(sorted[l - 1], sorted[l], row, hPos, vSlot, forward) === 0) { l++; } + if (l - k === 1) { result.push(sorted[k]); } + else { result.push(sorted.slice(k, l)); } + k = l; + } + return result; +} + +function getRowsAtCol(cellIndex: Map, col: number): number[] { + const rowSet = new Set(); + for (const key of cellIndex.keys()) { + if (cellCol(key) === col) { rowSet.add(cellRow(key)); } + } + return [...rowSet].sort((a, b) => a - b); +} + +function resolveOrderingTies( + cellIndex: Map, cellOrderings: Map, + edges: GridEdge[], col: number, + hPosMap: Map>, vSlot: Map, + forward: boolean, +): void { + for (const row of getRowsAtCol(cellIndex, col)) { + const key = cellKey(col, row); + let ordering = cellOrderings.get(key); + if (!ordering) { continue; } + + if (ordering.some(item => Array.isArray(item))) { + const hPos = hPosMap.get(row) || new Map(); + const newOrdering: CellOrder = []; + for (const item of ordering) { + if (!Array.isArray(item)) { + newOrdering.push(item); + } else { + newOrdering.push(...sortAndRegroupTied(item, row, hPos, vSlot, forward)); + } + } + cellOrderings.set(key, newOrdering); + writeOrderToSlots(newOrdering, edges); + ordering = newOrdering; + } + + recordHPositions(ordering, row, hPosMap); + } +} + +function propagateOrderingsLTR( + cellIndex: Map, cellOrderings: Map, + verticalGroups: Map, edges: GridEdge[], +): void { + const allCols = new Set(); + for (const [key] of cellIndex) { allCols.add(cellCol(key)); } + for (const col of verticalGroups.keys()) { allCols.add(col); } + const sortedCols = [...allCols].sort((a, b) => a - b); + + const prevHPos = new Map>(); + const prevVSlot = new Map(); + + for (const col of sortedCols) { + resolveOrderingTies(cellIndex, cellOrderings, edges, col, prevHPos, prevVSlot, true); + + const vGroup = verticalGroups.get(col); + if (vGroup && vGroup.length > 0) { + const clusters = clusterOverlapping(vGroup); + for (const cluster of clusters) { + if (cluster.length <= 1) { + edges[cluster[0].edgeIdx].verticalSlots[cluster[0].segIdx] = { slot: 0, count: 1 }; + prevVSlot.set(cluster[0].edgeIdx, 0); + continue; + } + + const joiners: VSeg[] = []; + const continuers: VSeg[] = []; + for (const vseg of cluster) { + const wp = edges[vseg.edgeIdx].waypoints; + if (vseg.segIdx > 0 && wp[vseg.segIdx - 1].col === col) { continuers.push(vseg); } + else { joiners.push(vseg); } + } + + joiners.sort((a, b) => compareJoiners(a, b, cellOrderings, edges)); + continuers.sort((a, b) => compareContinuers(a, b, prevVSlot)); + + const ordered = [...joiners, ...continuers]; + const joinerCount = joiners.length; + const totalCount = ordered.length; + let slot = 0; + for (let i = 0; i < ordered.length; i++) { + if (i > 0) { + const bothJoiners = i < joinerCount && i - 1 < joinerCount; + const bothContinuers = i >= joinerCount && i - 1 >= joinerCount; + const cmp = bothJoiners ? compareJoiners(ordered[i - 1], ordered[i], cellOrderings, edges) + : bothContinuers ? compareContinuers(ordered[i - 1], ordered[i], prevVSlot) + : 1; + if (cmp !== 0) { slot = i; } + } + edges[ordered[i].edgeIdx].verticalSlots[ordered[i].segIdx] = { slot, count: totalCount }; + prevVSlot.set(ordered[i].edgeIdx, slot); + } + } + } + } +} + +function getDestHPos(v: VSeg, edges: GridEdge[], cellOrderings: Map): number { + const wp = edges[v.edgeIdx].waypoints; + const destOrd = cellOrderings.get(cellKey(wp[v.segIdx + 1].col, wp[v.segIdx + 1].row)); + if (!destOrd) { return 0; } + return (cellOrderFindPos(destOrd, v.edgeIdx))?.pos ?? 0; +} + +function propagateOrderingsRTL( + cellIndex: Map, cellOrderings: Map, + verticalGroups: Map, edges: GridEdge[], +): void { + const allCols = new Set(); + for (const [key] of cellIndex) { allCols.add(cellCol(key)); } + for (const col of verticalGroups.keys()) { allCols.add(col); } + const sortedCols = [...allCols].sort((a, b) => b - a); + + const nextHPos = new Map>(); + const nextVSlot = new Map(); + + for (const col of sortedCols) { + const vGroup = verticalGroups.get(col); + if (vGroup && vGroup.length > 0) { + const clusters = clusterOverlapping(vGroup); + for (const cluster of clusters) { + if (cluster.length <= 1) { + const v = cluster[0]; + nextVSlot.set(v.edgeIdx, edges[v.edgeIdx].verticalSlots[v.segIdx].slot); + continue; + } + + cluster.sort((a, b) => + edges[a.edgeIdx].verticalSlots[a.segIdx].slot - edges[b.edgeIdx].verticalSlots[b.segIdx].slot + ); + + const sorted: VSeg[] = []; + let i = 0; + while (i < cluster.length) { + const s = edges[cluster[i].edgeIdx].verticalSlots[cluster[i].segIdx].slot; + let j = i + 1; + while (j < cluster.length && edges[cluster[j].edgeIdx].verticalSlots[cluster[j].segIdx].slot === s) { j++; } + + if (j - i === 1) { + sorted.push(cluster[i]); + } else { + const tied = cluster.slice(i, j); + tied.sort((a, b) => { + const aG = a.toRow > a.fromRow ? 0 : a.toRow === a.fromRow ? 1 : 2; + const bG = b.toRow > b.fromRow ? 0 : b.toRow === b.fromRow ? 1 : 2; + if (aG !== bG) { return aG - bG; } + const aH = getDestHPos(a, edges, cellOrderings); + const bH = getDestHPos(b, edges, cellOrderings); + if (aH === bH) { return 0; } + if (aG === 0) { return bH - aH; } + return aH - bH; + }); + sorted.push(...tied); + } + i = j; + } + + let vSlot = 0; + for (let k = 0; k < sorted.length; k++) { + if (k > 0) { + const prev = sorted[k - 1], curr = sorted[k]; + const prevS = edges[prev.edgeIdx].verticalSlots[prev.segIdx].slot; + const currS = edges[curr.edgeIdx].verticalSlots[curr.segIdx].slot; + if (prevS !== currS) { + vSlot = k; + } else { + const pG = prev.toRow > prev.fromRow ? 0 : prev.toRow === prev.fromRow ? 1 : 2; + const cG = curr.toRow > curr.fromRow ? 0 : curr.toRow === curr.fromRow ? 1 : 2; + if (pG !== cG || getDestHPos(prev, edges, cellOrderings) !== getDestHPos(curr, edges, cellOrderings)) { vSlot = k; } + } + } + edges[sorted[k].edgeIdx].verticalSlots[sorted[k].segIdx] = { slot: vSlot, count: sorted.length }; + nextVSlot.set(sorted[k].edgeIdx, vSlot); + } + } + } + + resolveOrderingTies(cellIndex, cellOrderings, edges, col, nextHPos, nextVSlot, false); + } +} + +function addLaneSeg(map: Map, row: number, seg: LaneSeg): void { + let arr = map.get(row); + if (!arr) { arr = []; map.set(row, arr); } + arr.push(seg); +} + +function assignHorizontalLanes(edges: GridEdge[], rowLaneCounts: number[]): void { + const rowSegs = new Map(); + for (let ei = 0; ei < edges.length; ei++) { + const e = edges[ei]; + const wp = e.waypoints; + addLaneSeg(rowSegs, wp[0].row, { edgeIdx: ei, col: wp[0].col, type: 'exit', hIdx: -1, order: e.exitSlot }); + if (wp.length > 1 && wp[0].row !== wp[1].row && wp[0].col !== wp[1].col) { + addLaneSeg(rowSegs, wp[0].row, { edgeIdx: ei, col: wp[1].col, type: 'approach', hIdx: -1, order: e.exitSlot }); + } + addLaneSeg(rowSegs, wp[wp.length - 1].row, { edgeIdx: ei, col: wp[wp.length - 1].col, type: 'entry', hIdx: -1, order: e.entrySlot }); + if (wp.length > 1 && wp[wp.length - 1].row !== wp[wp.length - 2].row && wp[wp.length - 1].col !== wp[wp.length - 2].col) { + addLaneSeg(rowSegs, wp[wp.length - 1].row, { edgeIdx: ei, col: wp[wp.length - 2].col, type: 'approach', hIdx: -1, order: e.entrySlot }); + } + for (let j = 1; j < wp.length - 1; j++) { + addLaneSeg(rowSegs, wp[j].row, { edgeIdx: ei, col: wp[j].col, type: 'h', hIdx: j - 1, order: e.horizontalSlots[j - 1].slot }); + if (wp[j].row !== wp[j - 1].row && wp[j].col !== wp[j - 1].col) { + addLaneSeg(rowSegs, wp[j].row, { edgeIdx: ei, col: wp[j - 1].col, type: 'approach', hIdx: -1, order: e.horizontalSlots[j - 1].slot }); + } + if (wp[j].row !== wp[j + 1].row && wp[j].col !== wp[j + 1].col) { + addLaneSeg(rowSegs, wp[j].row, { edgeIdx: ei, col: wp[j + 1].col, type: 'approach', hIdx: -1, order: e.horizontalSlots[j - 1].slot }); + } + } + } + + for (const [row, segs] of rowSegs) { + const edgeSegMap = new Map(); + for (const seg of segs) { + let arr = edgeSegMap.get(seg.edgeIdx); + if (!arr) { arr = []; edgeSegMap.set(seg.edgeIdx, arr); } + arr.push(seg); + } + + const spans: { edgeIdx: number; minCol: number; maxCol: number; colOrders: Map; segments: LaneSeg[] }[] = []; + for (const [edgeIdx, edgeSegments] of edgeSegMap) { + edgeSegments.sort((a, b) => a.col - b.col); + let start = 0; + for (let i = 1; i <= edgeSegments.length; i++) { + if (i === edgeSegments.length || edgeSegments[i].col !== edgeSegments[i - 1].col + 1) { + const spanSegs = edgeSegments.slice(start, i); + const colOrders = new Map(); + for (const s of spanSegs) { if (s.type !== 'approach') { colOrders.set(s.col, s.order); } } + spans.push({ edgeIdx, minCol: spanSegs[0].col, maxCol: spanSegs[spanSegs.length - 1].col, colOrders, segments: spanSegs }); + start = i; + } + } + } + + if (spans.length === 0) { continue; } + + const lanes = assignLanes( + spans.map(s => ({ minPos: s.minCol, maxPos: s.maxCol })), + (i, j) => { + const a = spans[i], b = spans[j]; + for (let c = Math.max(a.minCol, b.minCol); c <= Math.min(a.maxCol, b.maxCol); c++) { + const oa = a.colOrders.get(c), ob = b.colOrders.get(c); + if (oa !== undefined && ob !== undefined) { return oa - ob; } + } + return 0; + } + ); + + let minLane = Infinity, maxLane = -Infinity; + for (const l of lanes) { + if (l < minLane) { minLane = l; } + if (l > maxLane) { maxLane = l; } + } + const count = maxLane - minLane + 1; + if (row < rowLaneCounts.length) { rowLaneCounts[row] = count; } + + for (let si = 0; si < spans.length; si++) { + const normLane = lanes[si]; + for (const seg of spans[si].segments) { + if (seg.type === 'exit') { + edges[seg.edgeIdx].exitSlot = normLane; + edges[seg.edgeIdx].exitCount = count; + } else if (seg.type === 'entry') { + edges[seg.edgeIdx].entrySlot = normLane; + edges[seg.edgeIdx].entryCount = count; + } else if (seg.type === 'h') { + edges[seg.edgeIdx].horizontalSlots[seg.hIdx] = { slot: normLane, count }; + } + } + } + } +} + +function assignVerticalLanes(edges: GridEdge[], verticalGroups: Map): void { + for (const [, vSegs] of verticalGroups) { + if (vSegs.length === 0) { continue; } + + const vLanes = assignLanes( + vSegs.map(v => ({ minPos: v.minRow, maxPos: v.maxRow })), + (i, j) => edges[vSegs[i].edgeIdx].verticalSlots[vSegs[i].segIdx].slot + - edges[vSegs[j].edgeIdx].verticalSlots[vSegs[j].segIdx].slot + ); + + let vMinLane = Infinity, vMaxLane = -Infinity; + for (const l of vLanes) { + if (l < vMinLane) { vMinLane = l; } + if (l > vMaxLane) { vMaxLane = l; } + } + const vCount = vMaxLane - vMinLane + 1; + + for (let i = 0; i < vSegs.length; i++) { + edges[vSegs[i].edgeIdx].verticalSlots[vSegs[i].segIdx] = { slot: vLanes[i], count: vCount }; + } + } +} + +function assignSlots(edges: GridEdge[], rowLaneCounts: number[]): void { + const { verticalGroups, cellIndex } = buildSlotIndex(edges); + const cellOrderings = initialHorizontalOrdering(cellIndex, edges); + propagateOrderingsLTR(cellIndex, cellOrderings, verticalGroups, edges); + propagateOrderingsRTL(cellIndex, cellOrderings, verticalGroups, edges); + assignHorizontalLanes(edges, rowLaneCounts); + assignVerticalLanes(edges, verticalGroups); +} + +function assignLanes( + lines: { minPos: number; maxPos: number }[], + compare: (i: number, j: number) => number, +): number[] { + const n = lines.length; + if (n === 0) { return []; } + + const adj: number[][] = Array.from({ length: n }, () => []); + for (let i = 0; i < n; i++) { + for (let j = i + 1; j < n; j++) { + const lo = Math.max(lines[i].minPos, lines[j].minPos); + const hi = Math.min(lines[i].maxPos, lines[j].maxPos); + if (lo >= hi) { continue; } + const cmp = compare(i, j); + if (cmp < 0 || (cmp === 0 && i < j)) { + adj[i].push(j); + } else { + adj[j].push(i); + } + } + } + + const inDeg = new Array(n).fill(0); + for (let i = 0; i < n; i++) { for (const j of adj[i]) { inDeg[j]++; } } + const queue: number[] = []; + for (let i = 0; i < n; i++) { if (inDeg[i] === 0) { queue.push(i); } } + const topo: number[] = []; + const tmpDeg = [...inDeg]; + let qi = 0; + while (qi < queue.length) { + const u = queue[qi++]; + topo.push(u); + for (const v of adj[u]) { if (--tmpDeg[v] === 0) { queue.push(v); } } + } + + const minLane = new Array(n).fill(0); + for (const u of topo) { + for (const v of adj[u]) { minLane[v] = Math.max(minLane[v], minLane[u] + 1); } + } + const totalLanes = Math.max(...minLane) + 1; + + const distToSink = new Array(n).fill(0); + for (let i = topo.length - 1; i >= 0; i--) { + const u = topo[i]; + for (const v of adj[u]) { distToSink[u] = Math.max(distToSink[u], distToSink[v] + 1); } + } + + const lane = new Array(n); + for (let i = 0; i < n; i++) { + lane[i] = Math.floor((minLane[i] + totalLanes - 1 - distToSink[i]) / 2); + } + return lane; +} + + +function clusterOverlapping(items: T[]): T[][] { + const clusters: T[][] = []; + const assigned = new Uint8Array(items.length); + + for (let i = 0; i < items.length; i++) { + if (assigned[i]) { continue; } + const cluster = [items[i]]; + assigned[i] = 1; + + let changed = true; + while (changed) { + changed = false; + for (let j = 0; j < items.length; j++) { + if (assigned[j]) { continue; } + let overlaps = false; + for (const member of cluster) { + if (items[j].minRow <= member.maxRow && items[j].maxRow >= member.minRow) { + overlaps = true; + break; + } + } + if (overlaps) { + cluster.push(items[j]); + assigned[j] = 1; + changed = true; + } + } + } + + clusters.push(cluster); + } + + return clusters; +} + +function countEdgeCrossings(edges: GridEdge[]): Map { + const counts = new Map(); + for (const e of edges) { counts.set(e, 0); } + + for (let i = 0; i < edges.length; i++) { + for (let j = i + 1; j < edges.length; j++) { + if (edgesCross(edges[i], edges[j])) { + counts.set(edges[i], (counts.get(edges[i]) ?? 0) + 1); + counts.set(edges[j], (counts.get(edges[j]) ?? 0) + 1); + } + } + } + return counts; +} + +function edgesCross(a: GridEdge, b: GridEdge): boolean { + for (let i = 0; i < a.waypoints.length - 1; i++) { + for (let j = 0; j < b.waypoints.length - 1; j++) { + if (segmentsCross(a.waypoints[i], a.waypoints[i + 1], b.waypoints[j], b.waypoints[j + 1])) { + return true; + } + } + } + return false; +} + +function segmentsCross( + a1: { col: number; row: number }, a2: { col: number; row: number }, + b1: { col: number; row: number }, b2: { col: number; row: number } +): boolean { + if (a1.col === a2.col && b1.col === b2.col) { + if (a1.col !== b1.col) { return false; } + const aMin = Math.min(a1.row, a2.row), aMax = Math.max(a1.row, a2.row); + const bMin = Math.min(b1.row, b2.row), bMax = Math.max(b1.row, b2.row); + return aMin < bMax && bMin < aMax; + } + + if (a1.col === a2.col || b1.col === b2.col) { return false; } + + const aMinCol = Math.min(a1.col, a2.col), aMaxCol = Math.max(a1.col, a2.col); + const bMinCol = Math.min(b1.col, b2.col), bMaxCol = Math.max(b1.col, b2.col); + const overlapStart = Math.max(aMinCol, bMinCol); + const overlapEnd = Math.min(aMaxCol, bMaxCol); + if (overlapStart >= overlapEnd) { return false; } + + const aRowAtStart = a1.row + (a2.row - a1.row) * (overlapStart - a1.col) / (a2.col - a1.col); + const aRowAtEnd = a1.row + (a2.row - a1.row) * (overlapEnd - a1.col) / (a2.col - a1.col); + const bRowAtStart = b1.row + (b2.row - b1.row) * (overlapStart - b1.col) / (b2.col - b1.col); + const bRowAtEnd = b1.row + (b2.row - b1.row) * (overlapEnd - b1.col) / (b2.col - b1.col); + + return (aRowAtStart - bRowAtStart) * (aRowAtEnd - bRowAtEnd) < 0; +} + +function optimizeCrossings(edges: GridEdge[], nodes: GridNode[], maxRow: number): void { + const nodeAt = new Map(); + for (const nd of nodes) { nodeAt.set(cellKey(nd.col, nd.row), nd.index); } + + for (let iteration = 0; iteration < 5; iteration++) { + const counts = countEdgeCrossings(edges); + const crossingEdges = edges.filter(e => (counts.get(e) ?? 0) > 0); + if (crossingEdges.length === 0) { break; } + + crossingEdges.sort((a, b) => (counts.get(b) ?? 0) - (counts.get(a) ?? 0)); + let improved = false; + + for (const edge of crossingEdges) { + const pNode = nodes[edge.parent]; + const cNode = nodes[edge.child]; + if (cNode.col - pNode.col <= 1) { continue; } + + const currentCrossings = countSingleEdgeCrossings(edge, edges); + if (currentCrossings === 0) { continue; } + + const originalWaypoints = edge.waypoints.map(w => ({ ...w })); + let bestWaypoints = originalWaypoints; + let bestCrossings = currentCrossings; + + const offsets = [1, -1, 2, -2]; + for (const offset of offsets) { + const alt = computeAlternativeWaypoints(pNode, cNode, nodeAt, maxRow, offset); + edge.waypoints = alt; + const altCrossings = countSingleEdgeCrossings(edge, edges); + if (altCrossings < bestCrossings) { + bestWaypoints = alt; + bestCrossings = altCrossings; + } + } + + edge.waypoints = bestCrossings < currentCrossings ? bestWaypoints : originalWaypoints; + if (bestCrossings < currentCrossings) { improved = true; } + } + + if (!improved) { break; } + } +} + +function computeAlternativeWaypoints( + pNode: GridNode, cNode: GridNode, + nodeAt: Map, + maxRow: number, offset: number +): { col: number; row: number }[] { + const waypoints: { col: number; row: number }[] = [{ col: pNode.col, row: pNode.row }]; + + for (let c = pNode.col + 1; c < cNode.col; c++) { + const frac = (c - pNode.col) / (cNode.col - pNode.col); + const idealRow = pNode.row + frac * (cNode.row - pNode.row); + let targetRow = Math.max(0, Math.min(maxRow, Math.round(idealRow) + offset)); + + if (nodeAt.has(cellKey(c, targetRow))) { + for (let dr = 1; dr <= maxRow; dr++) { + if (targetRow + dr <= maxRow && !nodeAt.has(cellKey(c, targetRow + dr))) { targetRow = targetRow + dr; break; } + if (targetRow - dr >= 0 && !nodeAt.has(cellKey(c, targetRow - dr))) { targetRow = targetRow - dr; break; } + } + } + + waypoints.push({ col: c, row: targetRow }); + } + + waypoints.push({ col: cNode.col, row: cNode.row }); + return waypoints; +} + +function countSingleEdgeCrossings(edge: GridEdge, allEdges: GridEdge[]): number { + let count = 0; + for (const other of allEdges) { + if (other !== edge && edgesCross(edge, other)) { count++; } + } + return count; +} + +function computeChunkRegions(nodes: GridNode[], chunks: ChunkInput[]): ChunkRegion[] { + const regions: ChunkRegion[] = []; + + for (let ci = 0; ci < chunks.length; ci++) { + if (chunks[ci].txs.length <= 1) { + const cells = new Set(); + if (chunks[ci].txs.length === 1) { + const nd = nodes[chunks[ci].txs[0]]; + cells.add(cellKey(nd.col, nd.row)); + } + regions.push({ chunkIndex: ci, cells, bridges: new Set() }); + continue; + } + + const memberSet = new Set(chunks[ci].txs); + const memberNodes = chunks[ci].txs.map(ti => nodes[ti]); + + let minCol = Infinity, maxC = -Infinity, minRow = Infinity, maxR = -Infinity; + for (const nd of memberNodes) { + minCol = Math.min(minCol, nd.col); + maxC = Math.max(maxC, nd.col); + minRow = Math.min(minRow, nd.row); + maxR = Math.max(maxR, nd.row); + } + + const nodeAtCell = new Map(); + for (const nd of nodes) { + if (nd.col >= minCol && nd.col <= maxC && nd.row >= minRow && nd.row <= maxR) { + nodeAtCell.set(cellKey(nd.col, nd.row), nd.index); + } + } + + const cells = new Set(); + for (let c = minCol; c <= maxC; c++) { + for (let r = minRow; r <= maxR; r++) { + const key = cellKey(c, r); + const occupant = nodeAtCell.get(key); + if (occupant === undefined || memberSet.has(occupant)) { + cells.add(key); + } + } + } + + const blocked = new Set(); + for (const [key, idx] of nodeAtCell) { + if (!memberSet.has(idx)) { blocked.add(key); } + } + + const bridges = new Set(); + connectDisconnectedRegions(cells, bridges, blocked); + trimTabs(cells, memberNodes); + + regions.push({ chunkIndex: ci, cells, bridges }); + } + + return regions; +} + +function connectDisconnectedRegions(cells: Set, bridges: Set, blocked: Set): void { + const components = findConnectedComponents(cells); + if (components.length <= 1) { return; } + + const mainComponent = components.reduce((a, b) => a.size > b.size ? a : b); + for (const component of components) { + if (component === mainComponent) { continue; } + + let bestDist = Infinity, bestFrom = '', bestTo = ''; + for (const from of component) { + for (const to of mainComponent) { + const fc = cellCol(from), fr = cellRow(from); + const tc = cellCol(to), tr = cellRow(to); + const dist = Math.abs(fc - tc) + Math.abs(fr - tr); + if (dist < bestDist) { bestDist = dist; bestFrom = from; bestTo = to; } + } + } + + const prev = new Map(); + const queue = [bestFrom]; + prev.set(bestFrom, ''); + let found = false; + + for (let qi = 0; qi < queue.length && !found; qi++) { + const key = queue[qi]; + const c = cellCol(key), r = cellRow(key); + for (const [nc, nr] of [[c - 1, r], [c + 1, r], [c, r - 1], [c, r + 1]]) { + const nk = cellKey(nc, nr); + if (prev.has(nk)) { continue; } + if (blocked.has(nk)) { continue; } + prev.set(nk, key); + queue.push(nk); + if (nk === bestTo) { found = true; break; } + } + } + + if (found) { + let cur = bestTo; + while (cur !== bestFrom) { + const p = prev.get(cur); + if (p === undefined) { break; } + if (!cells.has(cur)) { cells.add(cur); } + const cc = cellCol(cur), cr = cellRow(cur); + const pc = cellCol(p), pr = cellRow(p); + if (pc < cc) { bridges.add(`${pc}:${pr}:right`); } + else if (pc > cc) { bridges.add(`${pc}:${pr}:left`); } + else if (pr < cr) { bridges.add(`${pc}:${pr}:bottom`); } + else { bridges.add(`${pc}:${pr}:top`); } + cur = p; + } + } + + for (const cell of component) { mainComponent.add(cell); } + } +} + +function findConnectedComponents(cells: Set): Set[] { + const visited = new Set(); + const components: Set[] = []; + + for (const cell of cells) { + if (visited.has(cell)) { continue; } + const component = new Set(); + const queue = [cell]; + visited.add(cell); + + let qi = 0; + while (qi < queue.length) { + const current = queue[qi++]; + component.add(current); + const c = cellCol(current), r = cellRow(current); + + for (const [nc, nr] of [[c - 1, r], [c + 1, r], [c, r - 1], [c, r + 1]]) { + const key = cellKey(nc, nr); + if (cells.has(key) && !visited.has(key)) { + visited.add(key); + queue.push(key); + } + } + } + + components.push(component); + } + + return components; +} + +function trimTabs(cells: Set, memberNodes: GridNode[]): void { + const memberCells = new Set(memberNodes.map(nd => cellKey(nd.col, nd.row))); + + for (let pass = 0; pass < 10; pass++) { + let changed = false; + + for (const cell of [...cells]) { + if (memberCells.has(cell)) { continue; } + const c = cellCol(cell), r = cellRow(cell); + let connectedSides = 0; + for (const [nc, nr] of [[c - 1, r], [c + 1, r], [c, r - 1], [c, r + 1]]) { + if (cells.has(cellKey(nc, nr))) { connectedSides++; } + } + if (connectedSides <= 1) { + cells.delete(cell); + changed = true; + } + } + + if (!changed) { break; } + } +} diff --git a/frontend/src/app/components/cluster-diagram/cluster-renderer.ts b/frontend/src/app/components/cluster-diagram/cluster-renderer.ts new file mode 100644 index 000000000..fdd509043 --- /dev/null +++ b/frontend/src/app/components/cluster-diagram/cluster-renderer.ts @@ -0,0 +1,640 @@ +import { GridLayout, GridNode, GridEdge, ChunkRegion, cellKey, cellCol, cellRow } from './cluster-layout'; + +export interface RenderedNode { + index: number; + tx: { txid: string; fee: number; weight: number }; + x: number; + y: number; + rectX: number; + rectY: number; + color: string; + chunkIndex: number; + feerate: number; + inactive: boolean; + isCurrent: boolean; + hovered: boolean; + related: boolean; +} + +export interface RenderedEdge { + parentIndex: number; + childIndex: number; + path: string; + gradientId: string; + markerId: string; + parentColor: string; + childColor: string; + parentInactive: boolean; + childInactive: boolean; + x1: number; + y1: number; + x2: number; + y2: number; + highlighted: boolean; +} + +export interface RenderedChunkOutline { + chunkIndex: number; + path: string; + feerate: number; + labelX: number; + labelY: number; +} + +export interface RenderParams { + containerWidth: number; + activeChunkIndex: number; + currentTxid: string; + txFees: number[]; + txWeights: number[]; + txids: string[]; + chunkFeerates: number[]; + getColor: (feerate: number) => string; +} + +interface RenderResult { + nodes: RenderedNode[]; + edges: RenderedEdge[]; + chunkOutlines: RenderedChunkOutline[]; + svgWidth: number; + svgHeight: number; +} + +interface GridGeometry { + colLeftX: number[]; + rowTopY: number[]; + rowHeights: number[]; + gutterWidths: number[]; + cellW: number; + totalCols: number; + totalRows: number; +} + +export const NODE_W = 64; +export const NODE_H = 30; +const MIN_CELL_W = 80; +const MAX_CELL_W = 120; +const CELL_PAD_Y = 16; +const MARGIN_X = 20; +const MARGIN_Y = 30; +const LANE_SPACING = 6; +const MIN_GUTTER_W = 3 * LANE_SPACING; +const MIN_GUTTER_H = 3 * LANE_SPACING; +const OUTLINE_PAD = 6; + +export function renderLayout(layout: GridLayout, params: RenderParams): RenderResult { + if (layout.nodes.length === 0) { + return { nodes: [], edges: [], chunkOutlines: [], svgWidth: 0, svgHeight: 0 }; + } + + const cellH = NODE_H + CELL_PAD_Y; + + const rowHeights: number[] = []; + for (let r = 0; r < layout.rows; r++) { + const laneCount = (layout.rowLaneCounts && layout.rowLaneCounts[r]) || 0; + if (r % 2 === 0) { + rowHeights.push(Math.max(cellH, (laneCount + 1) * LANE_SPACING)); + } else { + rowHeights.push(Math.max(MIN_GUTTER_H, (laneCount + 1) * LANE_SPACING)); + } + } + + const rowTopY: number[] = [MARGIN_Y]; + for (let r = 1; r < layout.rows; r++) { + rowTopY.push(rowTopY[r - 1] + rowHeights[r - 1]); + } + + const gutterWidths: number[] = []; + for (let c = 0; c < layout.cols - 1; c++) { + const laneCount = (layout.colLaneCounts && layout.colLaneCounts[c]) || 0; + gutterWidths.push(Math.max(MIN_GUTTER_W, (laneCount + 1) * LANE_SPACING)); + } + const totalGutterW = gutterWidths.reduce((a, b) => a + b, 0); + + const fixedWidth = MARGIN_X * 2 + totalGutterW; + const cellW = layout.cols > 0 + ? Math.max(MIN_CELL_W, Math.min(MAX_CELL_W, (params.containerWidth - fixedWidth) / layout.cols)) + : MIN_CELL_W; + + const colLeftX: number[] = [MARGIN_X]; + for (let c = 1; c < layout.cols; c++) { + colLeftX.push(colLeftX[c - 1] + cellW + gutterWidths[c - 1]); + } + + const geo: GridGeometry = { + colLeftX, rowTopY, rowHeights, gutterWidths, cellW, + totalCols: layout.cols, totalRows: layout.rows, + }; + + const nodes = renderNodes(layout.nodes, params, geo); + const edges = renderEdges(layout.edges, nodes, geo); + const chunkOutlines = renderChunkOutlines(layout.chunks, params, geo); + + const svgWidth = MARGIN_X * 2 + layout.cols * cellW + totalGutterW; + const svgHeight = rowTopY[layout.rows - 1] + rowHeights[layout.rows - 1] + MARGIN_Y; + + return { nodes, edges, chunkOutlines, svgWidth, svgHeight }; +} + +function geoCellX(geo: GridGeometry, col: number): number { + return geo.colLeftX[col] + geo.cellW / 2; +} + +function geoCellY(geo: GridGeometry, row: number): number { + if (row < 0 || row >= geo.totalRows) { return MARGIN_Y; } + return geo.rowTopY[row] + geo.rowHeights[row] / 2; +} + +function geoGutterCenterX(geo: GridGeometry, col: number): number { + return geo.colLeftX[col] + geo.cellW + geo.gutterWidths[col] / 2; +} + +function geoCellLeft(geo: GridGeometry, col: number): number { + return geo.colLeftX[col]; +} + +function geoCellTop(geo: GridGeometry, row: number): number { + if (row < 0 || row >= geo.totalRows) { return MARGIN_Y; } + return geo.rowTopY[row]; +} + +function geoCellRight(geo: GridGeometry, col: number): number { + return geo.colLeftX[col] + geo.cellW; +} + +function geoCellBottom(geo: GridGeometry, row: number): number { + if (row < 0 || row >= geo.totalRows) { return MARGIN_Y; } + return geo.rowTopY[row] + geo.rowHeights[row]; +} + +function borderX(geo: GridGeometry, b: number): number { + if (b <= 0) { return geoCellLeft(geo, 0) - OUTLINE_PAD; } + if (b >= geo.totalCols) { return geoCellRight(geo, geo.totalCols - 1) + OUTLINE_PAD; } + return (geoCellRight(geo, b - 1) + geoCellLeft(geo, b)) / 2; +} + +function borderY(geo: GridGeometry, b: number): number { + if (b <= 0) { return geoCellTop(geo, 0) - OUTLINE_PAD; } + if (b >= geo.totalRows) { return geoCellBottom(geo, geo.totalRows - 1) + OUTLINE_PAD; } + return (geoCellBottom(geo, b - 1) + geoCellTop(geo, b)) / 2; +} + +function renderNodes(gridNodes: GridNode[], params: RenderParams, geo: GridGeometry): RenderedNode[] { + return gridNodes.map(gn => { + const x = geoCellX(geo, gn.col); + const y = geoCellY(geo, gn.row); + const feerate = params.txFees[gn.index] / (params.txWeights[gn.index] / 4); + const color = params.getColor(feerate); + const inactive = gn.chunkIndex !== params.activeChunkIndex; + + return { + index: gn.index, + tx: { + txid: params.txids[gn.index], + fee: params.txFees[gn.index], + weight: params.txWeights[gn.index], + }, + x, y, + rectX: x - NODE_W / 2, + rectY: y - NODE_H / 2, + color, + chunkIndex: gn.chunkIndex, + feerate, + inactive, + isCurrent: params.txids[gn.index] === params.currentTxid, + hovered: false, + related: false, + }; + }); +} + +interface FanInfo { + nodeEdgeOffset: number; + fanDist: number; +} + +function computeFanInfos( + gridEdges: GridEdge[], + side: 'exit' | 'entry', +): (FanInfo | undefined)[] { + const nodeHalfH = NODE_H / 2; + const nodeKey = side === 'exit' ? 'parent' : 'child'; + + const groups = new Map(); + for (let i = 0; i < gridEdges.length; i++) { + const nodeIdx = gridEdges[i][nodeKey]; + let group = groups.get(nodeIdx); + if (!group) { group = []; groups.set(nodeIdx, group); } + group.push(i); + } + + const fanInfos: (FanInfo | undefined)[] = new Array(gridEdges.length); + + for (const [, edgeIndices] of groups) { + edgeIndices.sort((a, b) => { + const sa = side === 'exit' ? gridEdges[a].exitSlot : gridEdges[a].entrySlot; + const sb = side === 'exit' ? gridEdges[b].exitSlot : gridEdges[b].entrySlot; + return sa - sb; + }); + + let anyNeedsFan = false; + for (const ei of edgeIndices) { + const slot = side === 'exit' ? gridEdges[ei].exitSlot : gridEdges[ei].entrySlot; + const count = side === 'exit' ? gridEdges[ei].exitCount : gridEdges[ei].entryCount; + if (Math.abs(slotToOffset(slot, count)) > nodeHalfH) { + anyNeedsFan = true; + break; + } + } + + if (!anyNeedsFan) { continue; } + + const nodeCount = edgeIndices.length; + const edgeSpacing = (NODE_H - 2) / Math.max(1, nodeCount - 1); + + let maxDy = 0; + const nodeEdgeOffsets: number[] = []; + for (let k = 0; k < edgeIndices.length; k++) { + const slot = side === 'exit' ? gridEdges[edgeIndices[k]].exitSlot : gridEdges[edgeIndices[k]].entrySlot; + const count = side === 'exit' ? gridEdges[edgeIndices[k]].exitCount : gridEdges[edgeIndices[k]].entryCount; + const laneOffset = slotToOffset(slot, count); + const nodeEdgeOffset = (k - (nodeCount - 1) / 2) * edgeSpacing; + nodeEdgeOffsets.push(nodeEdgeOffset); + maxDy = Math.max(maxDy, Math.abs(laneOffset - nodeEdgeOffset)); + } + + const fanDist = Math.min(20, Math.max(10, maxDy * 1.5)); + + for (let k = 0; k < edgeIndices.length; k++) { + fanInfos[edgeIndices[k]] = { nodeEdgeOffset: nodeEdgeOffsets[k], fanDist }; + } + } + + return fanInfos; +} + +function renderEdges( + gridEdges: GridEdge[], renderedNodes: RenderedNode[], geo: GridGeometry, +): RenderedEdge[] { + const exitFanInfos = computeFanInfos(gridEdges, 'exit'); + const entryFanInfos = computeFanInfos(gridEdges, 'entry'); + + return gridEdges.map((ge, idx) => { + const pNode = renderedNodes[ge.parent]; + const cNode = renderedNodes[ge.child]; + + const points = waypointsToPixels(ge, geo, exitFanInfos[idx], entryFanInfos[idx]); + const path = buildEdgePath(points); + + const first = points[0]; + const last = points[points.length - 1]; + + return { + parentIndex: ge.parent, + childIndex: ge.child, + path, + gradientId: `edge-grad-${idx}`, + markerId: `edge-arrow-${idx}`, + parentColor: pNode.color, + childColor: cNode.color, + parentInactive: pNode.inactive, + childInactive: cNode.inactive, + x1: first.x, y1: first.y, + x2: last.x, y2: last.y, + highlighted: false, + }; + }); +} + +interface PathPoint { + x: number; + y: number; + sigmoid?: boolean; +} + +function waypointsToPixels( + edge: GridEdge, geo: GridGeometry, + exitFan?: FanInfo, entryFan?: FanInfo, +): PathPoint[] { + const wp = edge.waypoints; + if (wp.length < 2) { return []; } + + const exitOffset = slotToOffset(edge.exitSlot, edge.exitCount); + const entryOffset = slotToOffset(edge.entrySlot, edge.entryCount); + const points: PathPoint[] = []; + + const startNodeX = geoCellX(geo, wp[0].col) + NODE_W / 2; + const cy0 = geoCellY(geo, wp[0].row); + + if (exitFan) { + points.push({ x: startNodeX, y: cy0 + exitFan.nodeEdgeOffset }); + points.push({ x: startNodeX + exitFan.fanDist, y: cy0 + exitOffset, sigmoid: true }); + } else { + points.push({ x: startNodeX, y: cy0 + exitOffset }); + } + + let curY = cy0 + exitOffset; + for (let i = 0; i < wp.length - 1; i++) { + const vInfo = edge.verticalSlots[i]; + const gx = geoGutterCenterX(geo, wp[i].col) + slotToOffset(vInfo.slot, vInfo.count); + const isLast = i === wp.length - 2; + + let nextY: number; + if (isLast) { + nextY = geoCellY(geo, wp[i + 1].row) + entryOffset; + } else { + const hInfo = edge.horizontalSlots[i]; + nextY = geoCellY(geo, wp[i + 1].row) + slotToOffset(hInfo.slot, hInfo.count); + } + + if (points[points.length - 1].x !== gx) { + points.push({ x: gx, y: curY }); + } + + if (curY !== nextY) { + points.push({ x: gx, y: nextY }); + curY = nextY; + } + + if (!isLast) { + const nextVInfo = edge.verticalSlots[i + 1]; + const nextGx = geoGutterCenterX(geo, wp[i + 1].col) + slotToOffset(nextVInfo.slot, nextVInfo.count); + points.push({ x: nextGx, y: curY }); + } + } + + const endNodeX = geoCellX(geo, wp[wp.length - 1].col) - NODE_W / 2; + const cyLast = geoCellY(geo, wp[wp.length - 1].row); + + if (entryFan) { + points.push({ x: endNodeX - entryFan.fanDist, y: curY }); + points.push({ x: endNodeX, y: cyLast + entryFan.nodeEdgeOffset, sigmoid: true }); + } else { + points.push({ x: endNodeX, y: curY }); + } + + return points; +} + +function slotToOffset(slot: number, count: number): number { + if (count <= 1) { return 0; } + return (slot - (count - 1) / 2) * LANE_SPACING; +} + +function buildEdgePath(points: PathPoint[]): string { + if (points.length === 0) { return ''; } + const parts = [`M ${points[0].x} ${points[0].y}`]; + + for (let i = 1; i < points.length; i++) { + const prev = points[i - 1]; + const curr = points[i]; + const next = points[i + 1]; + + if (curr.sigmoid) { + const mx = (prev.x + curr.x) / 2; + parts.push(`C ${mx} ${prev.y} ${mx} ${curr.y} ${curr.x} ${curr.y}`); + continue; + } + + if (next && next.sigmoid) { + parts.push(`L ${curr.x} ${curr.y}`); + } else if (next && isCorner(prev, curr, next)) { + const segLen1 = Math.abs(curr.x - prev.x) + Math.abs(curr.y - prev.y); + const segLen2 = Math.abs(next.x - curr.x) + Math.abs(next.y - curr.y); + const r = Math.min(5, segLen1 / 2, segLen2 / 2); + + const dx1 = Math.sign(curr.x - prev.x); + const dy1 = Math.sign(curr.y - prev.y); + const dx2 = Math.sign(next.x - curr.x); + const dy2 = Math.sign(next.y - curr.y); + + parts.push(`L ${curr.x - dx1 * r} ${curr.y - dy1 * r}`); + parts.push(`Q ${curr.x} ${curr.y} ${curr.x + dx2 * r} ${curr.y + dy2 * r}`); + } else { + parts.push(`L ${curr.x} ${curr.y}`); + } + } + + return parts.join(' '); +} + +function isCorner( + a: { x: number; y: number }, + b: { x: number; y: number }, + c: { x: number; y: number }, +): boolean { + const dx1 = Math.sign(b.x - a.x); + const dy1 = Math.sign(b.y - a.y); + const dx2 = Math.sign(c.x - b.x); + const dy2 = Math.sign(c.y - b.y); + return dx1 !== dx2 || dy1 !== dy2; +} + +function renderChunkOutlines( + chunks: ChunkRegion[], params: RenderParams, geo: GridGeometry, +): RenderedChunkOutline[] { + return chunks.filter(ch => ch.cells.size > 1).map(ch => { + const segments = collectBoundarySegments(ch, geo); + const loops = connectSegments(segments); + const path = loops.map(loop => loopToPath(loop, 4)).join(' '); + + let labelX = Infinity, labelY = Infinity; + for (const cell of ch.cells) { + const c = cellCol(cell), r = cellRow(cell); + const cx = (geoCellLeft(geo, c) + geoCellRight(geo, c)) / 2; + const ty = borderY(geo, r); + if (ty < labelY || (ty === labelY && cx < labelX)) { + labelX = cx; + labelY = ty - 4; + } + } + + return { + chunkIndex: ch.chunkIndex, + path, + feerate: params.chunkFeerates[ch.chunkIndex], + labelX, + labelY, + }; + }); +} + +interface Segment { + x1: number; y1: number; + x2: number; y2: number; +} + +function collectBoundarySegments(chunk: ChunkRegion, geo: GridGeometry): Segment[] { + const segments: Segment[] = []; + + for (const cell of chunk.cells) { + const c = cellCol(cell), r = cellRow(cell); + const left = borderX(geo, c); + const right = borderX(geo, c + 1); + const top = borderY(geo, r); + const bottom = borderY(geo, r + 1); + + const hasAbove = chunk.cells.has(cellKey(c, r - 1)); + const hasBelow = chunk.cells.has(cellKey(c, r + 1)); + const hasLeft = chunk.cells.has(cellKey(c - 1, r)); + const hasRight = chunk.cells.has(cellKey(c + 1, r)); + + if (!hasAbove) { segments.push({ x1: left, y1: top, x2: right, y2: top }); } + if (!hasBelow) { segments.push({ x1: right, y1: bottom, x2: left, y2: bottom }); } + if (!hasLeft) { segments.push({ x1: left, y1: bottom, x2: left, y2: top }); } + if (!hasRight) { segments.push({ x1: right, y1: top, x2: right, y2: bottom }); } + } + + return segments; +} + +function endpointKey(x: number, y: number): string { + return `${Math.round(x)}:${Math.round(y)}`; +} + +function connectSegments(segments: Segment[]): { x: number; y: number }[][] { + if (segments.length === 0) { return []; } + + const endpointMap = new Map(); + for (let i = 0; i < segments.length; i++) { + const s = segments[i]; + const k1 = endpointKey(s.x1, s.y1); + const k2 = endpointKey(s.x2, s.y2); + let arr1 = endpointMap.get(k1); + if (!arr1) { arr1 = []; endpointMap.set(k1, arr1); } + arr1.push(i); + let arr2 = endpointMap.get(k2); + if (!arr2) { arr2 = []; endpointMap.set(k2, arr2); } + arr2.push(i); + } + + const loops: { x: number; y: number }[][] = []; + const used = new Uint8Array(segments.length); + + for (let start = 0; start < segments.length; start++) { + if (used[start]) { continue; } + used[start] = 1; + + const loop: { x: number; y: number }[] = [ + { x: segments[start].x1, y: segments[start].y1 }, + { x: segments[start].x2, y: segments[start].y2 }, + ]; + + for (let safety = 0; safety < segments.length * 2; safety++) { + const tail = loop[loop.length - 1]; + const head = loop[0]; + const eps = 0.5; + + if (loop.length > 2 && Math.abs(tail.x - head.x) < eps && Math.abs(tail.y - head.y) < eps) { + loop.pop(); + break; + } + + const candidates = endpointMap.get(endpointKey(tail.x, tail.y)); + let found = false; + if (candidates) { + for (const i of candidates) { + if (used[i]) { continue; } + const s = segments[i]; + if (Math.abs(s.x1 - tail.x) < eps && Math.abs(s.y1 - tail.y) < eps) { + used[i] = 1; + loop.push({ x: s.x2, y: s.y2 }); + found = true; + break; + } + if (Math.abs(s.x2 - tail.x) < eps && Math.abs(s.y2 - tail.y) < eps) { + used[i] = 1; + loop.push({ x: s.x1, y: s.y1 }); + found = true; + break; + } + } + } + + if (!found) { break; } + } + + if (loop.length >= 3) { + loops.push(filterCollinear(loop)); + } + } + + return loops; +} + +function filterCollinear(points: { x: number; y: number }[]): { x: number; y: number }[] { + const result: { x: number; y: number }[] = []; + const n = points.length; + + for (let i = 0; i < n; i++) { + const prev = points[(i - 1 + n) % n]; + const curr = points[i]; + const next = points[(i + 1) % n]; + + const dx1 = Math.sign(curr.x - prev.x); + const dy1 = Math.sign(curr.y - prev.y); + const dx2 = Math.sign(next.x - curr.x); + const dy2 = Math.sign(next.y - curr.y); + + if (dx1 !== dx2 || dy1 !== dy2) { + result.push(curr); + } + } + + return result.length >= 3 ? result : points; +} + +function loopToPath(points: { x: number; y: number }[], radius: number): string { + const n = points.length; + if (n < 3) { return ''; } + + const parts: string[] = []; + const firstCorner = computeCornerEntry(points, 0, radius); + parts.push(`M ${firstCorner.x} ${firstCorner.y}`); + + for (let i = 0; i < n; i++) { + const next = (i + 1) % n; + const exit = computeCornerExit(points, i, radius); + const entry = computeCornerEntry(points, next, radius); + + if (i > 0 || !pointsEqual(firstCorner, exit)) { + parts.push(`L ${exit.x} ${exit.y}`); + } + + const curr = points[next]; + parts.push(`Q ${curr.x} ${curr.y} ${entry.x} ${entry.y}`); + } + + parts.push('Z'); + return parts.join(' '); +} + +function computeCornerEntry( + points: { x: number; y: number }[], idx: number, radius: number, +): { x: number; y: number } { + const n = points.length; + const prev = points[(idx - 1 + n) % n]; + const curr = points[idx]; + const dist = Math.abs(curr.x - prev.x) + Math.abs(curr.y - prev.y); + const r = Math.min(radius, dist / 2); + const dx = Math.sign(curr.x - prev.x); + const dy = Math.sign(curr.y - prev.y); + return { x: curr.x - dx * r, y: curr.y - dy * r }; +} + +function computeCornerExit( + points: { x: number; y: number }[], idx: number, radius: number, +): { x: number; y: number } { + const n = points.length; + const curr = points[idx]; + const next = points[(idx + 1) % n]; + const dist = Math.abs(next.x - curr.x) + Math.abs(next.y - curr.y); + const r = Math.min(radius, dist / 2); + const dx = Math.sign(next.x - curr.x); + const dy = Math.sign(next.y - curr.y); + return { x: curr.x + dx * r, y: curr.y + dy * r }; +} + +function pointsEqual(a: { x: number; y: number }, b: { x: number; y: number }): boolean { + return Math.abs(a.x - b.x) < 0.01 && Math.abs(a.y - b.y) < 0.01; +} diff --git a/frontend/src/app/components/transaction/cpfp-info.component.html b/frontend/src/app/components/transaction/cpfp-info.component.html index e407bf576..41079efaf 100644 --- a/frontend/src/app/components/transaction/cpfp-info.component.html +++ b/frontend/src/app/components/transaction/cpfp-info.component.html @@ -1,7 +1,3 @@ -
-
-

Related Transactions

-
diff --git a/frontend/src/app/components/transaction/transaction-raw.component.html b/frontend/src/app/components/transaction/transaction-raw.component.html index 85646eaf8..ff9a2abf3 100644 --- a/frontend/src/app/components/transaction/transaction-raw.component.html +++ b/frontend/src/app/components/transaction/transaction-raw.component.html @@ -90,10 +90,34 @@ [hasEffectiveFeeRate]="hasEffectiveFeeRate" [cpfpInfo]="cpfpInfo" [hasCpfp]="hasCpfp" - (toggleCpfp$)="this.showCpfpDetails = !this.showCpfpDetails" + (toggleCpfp$)="toggleCpfp()" > - + +
+
+
+

Cluster Mempool Chunkingbeta

+

Related Transactions

+
+
+ +
+
+ +
+ + +
+ + + + +
diff --git a/frontend/src/app/components/transaction/transaction-raw.component.scss b/frontend/src/app/components/transaction/transaction-raw.component.scss index 936e86ada..cf5d08818 100644 --- a/frontend/src/app/components/transaction/transaction-raw.component.scss +++ b/frontend/src/app/components/transaction/transaction-raw.component.scss @@ -150,6 +150,16 @@ } } +.cpfp-beta { + font-size: 10px; + padding: 3px 5px; +} + +h2 .cpfp-beta { + margin-left: 5px; + vertical-align: super; +} + .subtitle-block { display: flex; flex-direction: row; diff --git a/frontend/src/app/components/transaction/transaction-raw.component.ts b/frontend/src/app/components/transaction/transaction-raw.component.ts index e12e97f47..cfcf77e26 100644 --- a/frontend/src/app/components/transaction/transaction-raw.component.ts +++ b/frontend/src/app/components/transaction/transaction-raw.component.ts @@ -68,7 +68,7 @@ export class TransactionRawComponent implements OnInit, OnDestroy { fetchCpfp: boolean; cpfpInfo: CpfpInfo | null; hasCpfp: boolean = false; - showCpfpDetails = false; + cpfpMode: 'advanced' | 'simple' | null = null; mempoolBlocksSubscription: Subscription; constructor( @@ -90,6 +90,10 @@ export class TransactionRawComponent implements OnInit, OnDestroy { this.seoService.setTitle($localize`:@@d7f92e6fe26fba6fff568cbdae5db4a5c8c6a55c:Preview Transaction`); this.seoService.setDescription($localize`:@@meta.description.preview-tx:Preview a transaction to the Bitcoin${seoDescriptionNetwork(this.stateService.network)} network using the transaction's raw hex data.`); this.websocketService.want(['blocks', 'mempool-blocks']); + const cpfpParam = this.route.snapshot.queryParams['cpfp']; + if (cpfpParam === 'advanced' || cpfpParam === 'simple') { + this.cpfpMode = cpfpParam; + } this.pushTxForm = this.formBuilder.group({ txRaw: ['', Validators.required], }); @@ -334,7 +338,7 @@ export class TransactionRawComponent implements OnInit, OnDestroy { this.isLoadingCpfpInfo = false; this.isLoadingBroadcast = false; this.adjustedVsize = null; - this.showCpfpDetails = false; + this.cpfpMode = null; this.hasCpfp = false; this.fetchCpfp = false; this.cpfpInfo = null; @@ -389,6 +393,27 @@ export class TransactionRawComponent implements OnInit, OnDestroy { this.stateService.hideFlow.next(!showFlow); } + toggleCpfp() { + const newMode = this.cpfpMode ? null : (this.cpfpInfo?.cluster ? 'advanced' : 'simple'); + this.updateCpfpMode(newMode); + } + + toggleCpfpView() { + const newMode = this.cpfpMode === 'advanced' ? 'simple' : 'advanced'; + this.updateCpfpMode(newMode); + } + + private updateCpfpMode(mode: 'advanced' | 'simple' | null) { + this.cpfpMode = mode; + this.router.navigate([], { + relativeTo: this.route, + queryParams: { cpfp: mode }, + queryParamsHandling: 'merge', + preserveFragment: true, + replaceUrl: true, + }); + } + setFlowEnabled() { this.flowEnabled = !this.hideFlow; } diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index e61556023..bb6b33e3e 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -73,14 +73,38 @@ [isCached]="isCached" [ETA$]="ETA$" (accelerateClicked)="onAccelerateClicked()" - (toggleCpfp$)="this.showCpfpDetails = !this.showCpfpDetails" + (toggleCpfp$)="toggleCpfp()" > } - + +
+
+
+

Cluster Mempool Chunkingbeta

+

Related Transactions

+
+
+ +
+
+ +
+ + +
+ + + + + diff --git a/frontend/src/app/components/transaction/transaction.component.scss b/frontend/src/app/components/transaction/transaction.component.scss index 942a68076..180eea926 100644 --- a/frontend/src/app/components/transaction/transaction.component.scss +++ b/frontend/src/app/components/transaction/transaction.component.scss @@ -206,6 +206,16 @@ bottom: -13px; } +.cpfp-beta { + font-size: 10px; + padding: 3px 5px; +} + +h2 .cpfp-beta { + margin-left: 5px; + vertical-align: super; +} + .subtitle-block { display: flex; flex-direction: row; diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index 4c15628ed..0ecbecf90 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -119,7 +119,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { isAcceleration: boolean = false; accelerationCanceled: boolean = false; filters: Filter[] = []; - showCpfpDetails = false; + cpfpMode: 'advanced' | 'simple' | null = null; miningStats: MiningStats; fetchCpfp$ = new Subject(); transactionTimes$ = new Subject(); @@ -221,6 +221,10 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { ngOnInit() { this.enterpriseService.page(); this.isDetailsOpen = this.route.snapshot.queryParams['showDetails'] === 'true'; + const cpfpParam = this.route.snapshot.queryParams['cpfp']; + if (cpfpParam === 'advanced' || cpfpParam === 'simple') { + this.cpfpMode = cpfpParam; + } const urlParams = new URLSearchParams(window.location.search); this.forceAccelerationSummary = !!urlParams.get('cash_request_id'); @@ -1090,7 +1094,6 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.rbfInfo = null; this.rbfReplaces = []; this.filters = []; - this.showCpfpDetails = false; this.showAccelerationDetails = false; this.accelerationFlowCompleted = false; this.accelerationInfo = null; @@ -1101,6 +1104,8 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.auditStatus = null; this.accelerationPositions = null; this.isDetailsOpen = this.route.snapshot.queryParams['showDetails'] === 'true'; + const cpfpParam = this.route.snapshot.queryParams['cpfp']; + this.cpfpMode = (cpfpParam === 'advanced' || cpfpParam === 'simple') ? cpfpParam : null; document.body.scrollTo(0, 0); this.isAcceleration = false; this.isAccelerated$.next(this.isAcceleration); @@ -1118,6 +1123,28 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.graphHeight = this.graphExpanded ? this.maxInOut * 15 : Math.min(360, this.maxInOut * 80); } + toggleCpfp() { + // cycle: null (closed) -> 'advanced' (open) -> null (closed) + const newMode = this.cpfpMode ? null : (this.cpfpInfo?.cluster ? 'advanced' : 'simple'); + this.updateCpfpMode(newMode); + } + + toggleCpfpView() { + const newMode = this.cpfpMode === 'advanced' ? 'simple' : 'advanced'; + this.updateCpfpMode(newMode); + } + + private updateCpfpMode(mode: 'advanced' | 'simple' | null) { + this.cpfpMode = mode; + this.router.navigate([], { + relativeTo: this.route, + queryParams: { cpfp: mode }, + queryParamsHandling: 'merge', + preserveFragment: true, + replaceUrl: true, + }); + } + toggleGraph() { const showFlow = !this.flowEnabled; this.stateService.hideFlow.next(!showFlow); diff --git a/frontend/src/app/components/transaction/transaction.module.ts b/frontend/src/app/components/transaction/transaction.module.ts index a05191346..609c9a961 100644 --- a/frontend/src/app/components/transaction/transaction.module.ts +++ b/frontend/src/app/components/transaction/transaction.module.ts @@ -11,6 +11,7 @@ import { AccelerateCheckout } from '@components/accelerate-checkout/accelerate-c import { AccelerateFeeGraphComponent } from '@components/accelerate-checkout/accelerate-fee-graph.component'; import { TransactionRawComponent } from '@components/transaction/transaction-raw.component'; import { CpfpInfoComponent } from '@components/transaction/cpfp-info.component'; +import { ClusterDiagramComponent } from '@components/cluster-diagram/cluster-diagram.component'; const routes: Routes = [ { @@ -57,6 +58,7 @@ export class TransactionRoutingModule { } AccelerateFeeGraphComponent, TransactionRawComponent, CpfpInfoComponent, + ClusterDiagramComponent, ], exports: [ TransactionComponent, @@ -64,6 +66,7 @@ export class TransactionRoutingModule { } AccelerateCheckout, AccelerateFeeGraphComponent, CpfpInfoComponent, + ClusterDiagramComponent, ] }) export class TransactionModule { } diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index ac449fbf3..a48ff3f05 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -32,6 +32,23 @@ export interface CpfpInfo { acceleratedBy?: number[]; acceleratedAt?: number; feeDelta?: number; + cluster?: { + txs: CpfpClusterTx[]; + chunks: CpfpClusterChunk[]; + chunkIndex: number; + }; +} + +export interface CpfpClusterTx { + txid: string; + fee: number; + weight: number; + parents: number[]; +} + +export interface CpfpClusterChunk { + txs: number[]; + feerate: number; } export interface RbfInfo { diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index b69c0f2c9..3a135f701 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -18,6 +18,8 @@ "AUDIT": true, "CPFP_INDEXING": true, "RUST_GBT": true, + "CLUSTER_MEMPOOL": true, + "CLUSTER_MEMPOOL_INDEXING": true, "USE_SECOND_NODE_FOR_MINFEE": true, "DISK_CACHE_BLOCK_INTERVAL": 1, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, diff --git a/production/mempool-config.regtest.json b/production/mempool-config.regtest.json index 151b139fc..0c3b92e89 100644 --- a/production/mempool-config.regtest.json +++ b/production/mempool-config.regtest.json @@ -12,6 +12,8 @@ "POOLS_UPDATE_DELAY": 3600, "AUDIT": true, "RUST_GBT": true, + "CLUSTER_MEMPOOL": true, + "CLUSTER_MEMPOOL_INDEXING": true, "POLL_RATE_MS": 500, "DISK_CACHE_BLOCK_INTERVAL": 1, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, diff --git a/production/mempool-config.signet.json b/production/mempool-config.signet.json index 87ecd9a0b..8155ced03 100644 --- a/production/mempool-config.signet.json +++ b/production/mempool-config.signet.json @@ -12,6 +12,8 @@ "POOLS_UPDATE_DELAY": 3600, "AUDIT": true, "RUST_GBT": true, + "CLUSTER_MEMPOOL": true, + "CLUSTER_MEMPOOL_INDEXING": true, "POLL_RATE_MS": 1000, "DISK_CACHE_BLOCK_INTERVAL": 1, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, diff --git a/production/mempool-config.testnet.json b/production/mempool-config.testnet.json index 67a570e9e..fdd5d78f7 100644 --- a/production/mempool-config.testnet.json +++ b/production/mempool-config.testnet.json @@ -12,6 +12,8 @@ "POOLS_UPDATE_DELAY": 3600, "AUDIT": true, "RUST_GBT": true, + "CLUSTER_MEMPOOL": true, + "CLUSTER_MEMPOOL_INDEXING": true, "POLL_RATE_MS": 1000, "DISK_CACHE_BLOCK_INTERVAL": 1, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, diff --git a/production/mempool-config.testnet4.json b/production/mempool-config.testnet4.json index 29c1591fc..19489194c 100644 --- a/production/mempool-config.testnet4.json +++ b/production/mempool-config.testnet4.json @@ -12,6 +12,8 @@ "POOLS_UPDATE_DELAY": 3600, "AUDIT": true, "RUST_GBT": true, + "CLUSTER_MEMPOOL": true, + "CLUSTER_MEMPOOL_INDEXING": true, "POLL_RATE_MS": 1000, "DISK_CACHE_BLOCK_INTERVAL": 1, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,