From 5e2008bc4b9400bd0ce6a3712be6d86103414e59 Mon Sep 17 00:00:00 2001 From: jramos0 Date: Fri, 31 Jul 2026 03:43:09 -0600 Subject: [PATCH 1/4] Extract out-of-band transaction detection into a shared helper --- .../src/__tests__/api/prioritization.test.ts | 60 +++++++++++++++++++ backend/src/api/acceleration/acceleration.ts | 41 +++++-------- backend/src/api/prioritization.ts | 45 ++++++++++++++ 3 files changed, 119 insertions(+), 27 deletions(-) create mode 100644 backend/src/__tests__/api/prioritization.test.ts create mode 100644 backend/src/api/prioritization.ts diff --git a/backend/src/__tests__/api/prioritization.test.ts b/backend/src/__tests__/api/prioritization.test.ts new file mode 100644 index 000000000..54f075b1b --- /dev/null +++ b/backend/src/__tests__/api/prioritization.test.ts @@ -0,0 +1,60 @@ +import { findOutOfBandTransactions, OutOfBandCandidate } from '../../api/prioritization'; + +function tx(txid: string, effectiveFeePerVsize: number, cluster?: string[]): OutOfBandCandidate { + return { txid, effectiveFeePerVsize, cluster: cluster || [txid] }; +} + +const NONE = new Set(); + +// Block order (index 0 = highest mining priority) descends in fee rate towards the +// bottom of the block. The scan walks bottom-to-top, so a "clean" block presents a +// non-decreasing rate sequence as the scan proceeds towards index 0. +describe('findOutOfBandTransactions', () => { + test('excludes nothing from a clean block with a non-decreasing rate sequence', () => { + const ordered = [tx('top', 10), tx('mid', 7), tx('low', 4), tx('bottom', 1)]; + expect(findOutOfBandTransactions(ordered, NONE)).toEqual(new Set()); + }); + + test('excludes a single out-of-order transaction', () => { + const ordered = [tx('top', 10), tx('anomaly', 1), tx('bottom', 4)]; + expect(findOutOfBandTransactions(ordered, NONE)).toEqual(new Set(['anomaly'])); + }); + + test('excludes every member of a prioritized transaction\'s cluster', () => { + const ordered = [ + tx('top', 10), + tx('anomaly', 1, ['anomaly', 'parent', 'child']), + tx('bottom', 4), + ]; + expect(findOutOfBandTransactions(ordered, NONE)) + .toEqual(new Set(['anomaly', 'parent', 'child'])); + }); + + test('excludes an accelerated transaction and its cluster even at a normal rate', () => { + const ordered = [ + tx('top', 10), + tx('boosted', 7, ['boosted', 'ancestor']), + tx('bottom', 4), + ]; + // 7 is not out-of-order relative to the surrounding rates, so only the + // acceleration flag causes the exclusion. + const result = findOutOfBandTransactions(ordered, new Set(['boosted'])); + expect(result).toEqual(new Set(['boosted', 'ancestor'])); + }); + + test('baseline guard: excluding a transaction must not lower the running baseline', () => { + // Processed bottom-to-top: D(10) sets the baseline. C(1) is excluded and must NOT + // drag the baseline down to 1 — if it did, B(5) would look clean against 1 and + // escape exclusion, when it is really still below D's baseline of 10. + const ordered = [tx('A', 12), tx('B', 5), tx('C', 1), tx('D', 10)]; + expect(findOutOfBandTransactions(ordered, NONE)).toEqual(new Set(['C', 'B'])); + }); + + test('returns an empty set for an empty block', () => { + expect(findOutOfBandTransactions([], NONE)).toEqual(new Set()); + }); + + test('returns an empty set for a single-transaction block', () => { + expect(findOutOfBandTransactions([tx('coinbase', 0)], NONE)).toEqual(new Set()); + }); +}); diff --git a/backend/src/api/acceleration/acceleration.ts b/backend/src/api/acceleration/acceleration.ts index f26805ff2..2ab26ef2b 100644 --- a/backend/src/api/acceleration/acceleration.ts +++ b/backend/src/api/acceleration/acceleration.ts @@ -1,6 +1,7 @@ import logger from '../../logger'; import { MempoolTransactionExtended } from '../../mempool.interfaces'; import { GraphTx, getSameBlockRelatives, initializeRelatives, makeBlockTemplate, mempoolComparator, removeAncestors, setAncestorScores } from '../mini-miner'; +import { findOutOfBandTransactions } from '../prioritization'; const BLOCK_WEIGHT_UNITS = 4_000_000; const MAX_RELATIVE_GRAPH_SIZE = 200; @@ -49,41 +50,27 @@ class AccelerationCosts { for (const tx of template) { txMap[tx.txid] = tx; } + const acceleratedTxids = new Set(Object.keys(accMap)); // Identify and exclude accelerated and otherwise prioritized transactions - const excludeMap = {}; + const excludeMap = findOutOfBandTransactions( + blockTxs.map(blockTx => txMap[blockTx.txid]), + acceleratedTxids, + ); + + // Total block weight and the smallest accelerated CPFP cluster's package weight are + // pricing inputs, not exclusion inputs, so they're kept out of the shared helper. let totalWeight = 0; let minAcceleratedPackage = Infinity; - let lastEffectiveRate = 0; - // Iterate over the mined template from bottom to top. - // Transactions should appear in ascending order of mining priority. - for (const blockTx of [...blockTxs].reverse()) { - const txid = blockTx.txid; - const tx = txMap[txid]; + for (const blockTx of blockTxs) { + const tx = txMap[blockTx.txid]; totalWeight += tx.weight; - const isAccelerated = accMap[txid] != null; - // If a cluster has a in-band effective fee rate than the previous cluster, - // it must have been prioritized out-of-band (in order to have a higher mining priority) - // so exclude from the analysis. - const isPrioritized = tx.effectiveFeePerVsize < lastEffectiveRate; - if (isPrioritized || isAccelerated) { + if (accMap[blockTx.txid] != null) { let packageWeight = 0; - // exclude this whole CPFP cluster for (const clusterTxid of tx.cluster) { packageWeight += txMap[clusterTxid].weight; - if (!excludeMap[clusterTxid]) { - excludeMap[clusterTxid] = true; - } - } - // keep track of the smallest accelerated CPFP cluster for later - if (isAccelerated) { - minAcceleratedPackage = Math.min(minAcceleratedPackage, packageWeight); - } - } - if (!isPrioritized) { - if (!isAccelerated) { - lastEffectiveRate = tx.effectiveFeePerVsize; } + minAcceleratedPackage = Math.min(minAcceleratedPackage, packageWeight); } } @@ -106,7 +93,7 @@ class AccelerationCosts { for (let offset = spareWeight; offset < BLOCK_WEIGHT_UNITS && txIndex >= 0; txIndex--) { const txid = blockTxs[txIndex].txid; const tx = txMap[txid]; - if (excludeMap[txid]) { + if (excludeMap.has(txid)) { // skip prioritized transactions and their ancestors continue; } diff --git a/backend/src/api/prioritization.ts b/backend/src/api/prioritization.ts new file mode 100644 index 000000000..803c460ed --- /dev/null +++ b/backend/src/api/prioritization.ts @@ -0,0 +1,45 @@ +/** + * Detects transactions that earned their block position out-of-band rather than on + * fee merit: prioritised (e.g. via `prioritisetransaction`) or accelerated. Shared by + * the accelerator pricing path (`calculateBoostRate`) and the minimum daily fee rate + * metric, so both use exactly one criterion for what counts as "not fee merit". + */ + +export interface OutOfBandCandidate { + txid: string; + effectiveFeePerVsize: number; + cluster: readonly string[]; // same-block CPFP cluster, including the tx itself +} + +/** + * @param orderedTxs Block order, coinbase at index 0. + * @param acceleratedTxids Txids with a known acceleration record for this block. + * @returns The set of txids to exclude: prioritised transactions and their clusters, + * plus accelerated transactions and their clusters. + */ +export function findOutOfBandTransactions( + orderedTxs: readonly OutOfBandCandidate[], + acceleratedTxids: ReadonlySet, +): Set { + const excluded = new Set(); + let lastEffectiveRate = 0; + // Walk the block from the bottom up: transactions should appear in ascending order + // of mining priority, so a drop in effective rate below the running baseline can only + // mean the transaction was prioritized out-of-band. The baseline itself must not + // advance past a prioritized or accelerated transaction, or a single boosted + // transaction would silently raise the floor for everything above it. + for (let i = orderedTxs.length - 1; i >= 0; i--) { + const tx = orderedTxs[i]; + const isAccelerated = acceleratedTxids.has(tx.txid); + const isPrioritized = tx.effectiveFeePerVsize < lastEffectiveRate; + if (isPrioritized || isAccelerated) { + for (const clusterTxid of tx.cluster) { + excluded.add(clusterTxid); + } + } + if (!isPrioritized && !isAccelerated) { + lastEffectiveRate = tx.effectiveFeePerVsize; + } + } + return excluded; +} From 22f4172ec35ff9de4d9baba1ecb474b1819e8cb6 Mon Sep 17 00:00:00 2001 From: jramos0 Date: Fri, 31 Jul 2026 03:43:09 -0600 Subject: [PATCH 2/4] Add min_fee_rate columns to blocks table --- backend/src/api/database-migration.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 3932179ed..f79b1a104 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 111; + private static currentVersion = 112; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -1255,6 +1255,23 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `compact_cpfp_clusters` ADD template_algo TINYINT UNSIGNED NOT NULL DEFAULT 0'); await this.updateToSchemaVersion(111); } + + if (databaseSchemaVersion < 112 && isBitcoin === true) { + // Per-block minimum fee-merit effective fee rate (issue #6639). The computation + // version handles algorithm changes; the acceleration-set snapshot lets the + // bounded pull sweep detect late-arriving accelerations without producer-side + // writes. + await this.$executeQuery(` + ALTER TABLE blocks + ADD min_fee_rate DOUBLE UNSIGNED NULL DEFAULT NULL, + ADD min_fee_rate_version TINYINT UNSIGNED NOT NULL DEFAULT 0, + ADD min_fee_rate_acceleration_count SMALLINT UNSIGNED NOT NULL DEFAULT 0, + ADD min_fee_rate_acceleration_fingerprint CHAR(64) NOT NULL DEFAULT '', + ADD min_fee_rate_computed_at TIMESTAMP NULL DEFAULT NULL, + ADD INDEX min_fee_rate_backfill (stale, min_fee_rate_version, min_fee_rate_computed_at) + `); + await this.updateToSchemaVersion(112); + } } /** From 18a778a59de108d765918c117f61f95731bf9310 Mon Sep 17 00:00:00 2001 From: jramos0 Date: Fri, 31 Jul 2026 03:43:09 -0600 Subject: [PATCH 3/4] Add minimum daily fee rate backend metric and API --- backend/jest.integration.teardown.ts | 1 + .../min-fee-rate-aggregation.test.ts | 141 ++++++++ .../src/__integration_tests__/test-helpers.ts | 1 + .../src/__tests__/api/min-fee-rate.test.ts | 102 ++++++ backend/src/api/blocks.ts | 31 ++ backend/src/api/mining/min-fee-rate.ts | 131 ++++++++ backend/src/api/mining/mining-routes.ts | 24 ++ backend/src/api/mining/mining.ts | 12 + backend/src/indexer.ts | 11 +- .../repositories/AccelerationRepository.ts | 41 +++ .../repositories/BlocksAuditsRepository.ts | 29 +- backend/src/repositories/BlocksRepository.ts | 307 +++++++++++++++++- 12 files changed, 828 insertions(+), 3 deletions(-) create mode 100644 backend/src/__integration_tests__/min-fee-rate-aggregation.test.ts create mode 100644 backend/src/__tests__/api/min-fee-rate.test.ts create mode 100644 backend/src/api/mining/min-fee-rate.ts diff --git a/backend/jest.integration.teardown.ts b/backend/jest.integration.teardown.ts index 386e26ab3..dd0dc6169 100644 --- a/backend/jest.integration.teardown.ts +++ b/backend/jest.integration.teardown.ts @@ -25,6 +25,7 @@ module.exports = async () => { const tables = [ 'blocks_audits', 'blocks_summaries', + 'accelerations', 'blocks_prices', 'blocks_templates', 'cpfp_clusters', diff --git a/backend/src/__integration_tests__/min-fee-rate-aggregation.test.ts b/backend/src/__integration_tests__/min-fee-rate-aggregation.test.ts new file mode 100644 index 000000000..8d8236743 --- /dev/null +++ b/backend/src/__integration_tests__/min-fee-rate-aggregation.test.ts @@ -0,0 +1,141 @@ +import DB from '../database'; +import BlocksRepository from '../repositories/BlocksRepository'; +import { MIN_FEE_RATE_VERSION } from '../api/mining/min-fee-rate'; +import { setupTestDatabase, waitForDatabase, cleanupTestData, insertTestPool, insertTestBlock } from './test-helpers'; + +/** + * Midday UTC, `daysAgo` days back. Midday rather than midnight so a block never + * lands on the day boundary the aggregation buckets on. + */ +function utcMidday(daysAgo: number): Date { + const now = new Date(); + return new Date(Date.UTC( + now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() - daysAgo, 12, 0, 0 + )); +} + +describe('min fee rate daily aggregation', () => { + let defaultPoolId: number; + let nextHeight = 940000; + + beforeAll(async () => { + await waitForDatabase(); + await setupTestDatabase(); + }, 120000); + + beforeEach(async () => { + await cleanupTestData(); + defaultPoolId = await insertTestPool({ + name: 'Unknown', + slug: 'unknown', + addresses: '[]', + regexes: '[]', + }); + nextHeight = 940000; + }); + + afterAll(async () => { + await cleanupTestData(); + }); + + async function insertBlockWithRate(options: { + blockTimestamp: Date; + minFeeRate: number | null; + version?: number; + stale?: boolean; + }): Promise { + const height = nextHeight++; + await insertTestBlock({ + height, + hash: height.toString(16).padStart(64, '0'), + blockTimestamp: options.blockTimestamp, + poolId: defaultPoolId, + }); + await DB.query( + `UPDATE blocks + SET min_fee_rate = ?, min_fee_rate_version = ?, stale = ?, + min_fee_rate_computed_at = CURRENT_TIMESTAMP + WHERE height = ?`, + [ + options.minFeeRate, + options.version ?? MIN_FEE_RATE_VERSION, + options.stale ? 1 : 0, + height, + ] + ); + return height; + } + + test('buckets on the UTC day and keeps the lowest rate of each day', async () => { + const lowest = await insertBlockWithRate({ blockTimestamp: utcMidday(3), minFeeRate: 0.12 }); + await insertBlockWithRate({ blockTimestamp: utcMidday(3), minFeeRate: 0.4 }); + await insertBlockWithRate({ blockTimestamp: utcMidday(2), minFeeRate: 0.25 }); + + const days = await BlocksRepository.$getMinFeeRatesByDay(null); + + expect(days).toHaveLength(2); + expect(days[0].minRate).toBeCloseTo(0.12, 6); + expect(days[0].minHeight).toBe(lowest); + expect(Number(days[0].usableBlockCount)).toBe(2); + expect(days[1].minRate).toBeCloseTo(0.25, 6); + expect(Number(days[1].usableBlockCount)).toBe(1); + // Ascending by day. + expect(Number(days[0].timestamp)).toBeLessThan(Number(days[1].timestamp)); + }); + + test('excludes the current UTC day, whose minimum is still partial', async () => { + await insertBlockWithRate({ blockTimestamp: utcMidday(0), minFeeRate: 0.01 }); + await insertBlockWithRate({ blockTimestamp: utcMidday(1), minFeeRate: 0.3 }); + + const days = await BlocksRepository.$getMinFeeRatesByDay(null); + + expect(days).toHaveLength(1); + expect(days[0].minRate).toBeCloseTo(0.3, 6); + }); + + test('ignores blocks with no rate, a stale version, or a stale chain', async () => { + await insertBlockWithRate({ blockTimestamp: utcMidday(4), minFeeRate: null }); + await insertBlockWithRate({ + blockTimestamp: utcMidday(4), + minFeeRate: 0.02, + version: MIN_FEE_RATE_VERSION - 1, + }); + await insertBlockWithRate({ blockTimestamp: utcMidday(4), minFeeRate: 0.03, stale: true }); + await insertBlockWithRate({ blockTimestamp: utcMidday(4), minFeeRate: 0.5 }); + + const days = await BlocksRepository.$getMinFeeRatesByDay(null); + + expect(days).toHaveLength(1); + expect(days[0].minRate).toBeCloseTo(0.5, 6); + expect(Number(days[0].usableBlockCount)).toBe(1); + }); + + test('applies the requested interval to the returned series', async () => { + await insertBlockWithRate({ blockTimestamp: utcMidday(40), minFeeRate: 0.05 }); + await insertBlockWithRate({ blockTimestamp: utcMidday(2), minFeeRate: 0.5 }); + + const allDays = await BlocksRepository.$getMinFeeRatesByDay(null); + const lastWeek = await BlocksRepository.$getMinFeeRatesByDay('1 WEEK'); + + expect(allDays).toHaveLength(2); + expect(lastWeek).toHaveLength(1); + expect(lastWeek[0].minRate).toBeCloseTo(0.5, 6); + }); + + /** + * The day count drives which timespan buttons the graph offers, so it deliberately + * spans all available history rather than the selected window: scoping it to the + * interval would hide the longer options as soon as a shorter one was picked. The + * threshold percentage does not read it — that denominator is the returned series. + */ + test('counts every available day regardless of the requested interval', async () => { + await insertBlockWithRate({ blockTimestamp: utcMidday(40), minFeeRate: 0.05 }); + await insertBlockWithRate({ blockTimestamp: utcMidday(2), minFeeRate: 0.5 }); + + const dayCount = await BlocksRepository.$getMinFeeRateDayCount(); + const lastWeek = await BlocksRepository.$getMinFeeRatesByDay('1 WEEK'); + + expect(dayCount).toBe(2); + expect(lastWeek).toHaveLength(1); + }); +}); diff --git a/backend/src/__integration_tests__/test-helpers.ts b/backend/src/__integration_tests__/test-helpers.ts index 91e47fe2e..042fe9f4f 100644 --- a/backend/src/__integration_tests__/test-helpers.ts +++ b/backend/src/__integration_tests__/test-helpers.ts @@ -25,6 +25,7 @@ export async function cleanupTestData(): Promise { const tables = [ 'blocks_audits', 'blocks_summaries', + 'accelerations', 'blocks_prices', 'blocks_templates', 'cpfp_clusters', diff --git a/backend/src/__tests__/api/min-fee-rate.test.ts b/backend/src/__tests__/api/min-fee-rate.test.ts new file mode 100644 index 000000000..48cfea857 --- /dev/null +++ b/backend/src/__tests__/api/min-fee-rate.test.ts @@ -0,0 +1,102 @@ +import { + buildOutOfBandCandidates, + computeMinFeeRate, + isMinFeeRateVersionStale, + MIN_FEE_RATE_START_DATE, + MIN_FEE_RATE_VERSION, + MIN_SUMMARY_VERSION, +} from '../../api/mining/min-fee-rate'; +import { OutOfBandCandidate } from '../../api/prioritization'; + +function tx(txid: string, effectiveFeePerVsize: number, cluster?: string[]): OutOfBandCandidate { + return { txid, effectiveFeePerVsize, cluster: cluster || [txid] }; +} + +const NONE = new Set(); + +// Block order (index 0 = coinbase); fee rate descends towards the bottom of the block +// in a clean block, matching the convention findOutOfBandTransactions scans against. +describe('computeMinFeeRate', () => { + test('returns the minimum non-coinbase rate for a plain block', () => { + const candidates = [tx('coinbase', 0), tx('c', 10), tx('b', 5), tx('a', 2)]; + expect(computeMinFeeRate(candidates, NONE)).toBe(2); + }); + + test('ignores the coinbase at index 0 regardless of its own exclusion status', () => { + // The coinbase's rate (0) is always below any positive baseline, so it lands in + // the exclusion set too — computeMinFeeRate must not depend on that; it skips + // index 0 positionally. + const candidates = [tx('coinbase', 0), tx('a', 5)]; + expect(computeMinFeeRate(candidates, NONE)).toBe(5); + }); + + test('skips a zero-rate transaction that the exclusion scan does not catch', () => { + // 'zero' is processed first (bottom of the block, baseline still 0), so + // 0 < 0 is false and it is not flagged as prioritized — EXCLUDE_ZERO_RATE is the + // only thing keeping it out of the minimum. + const candidates = [tx('coinbase', 0), tx('c', 6), tx('zero', 0)]; + expect(computeMinFeeRate(candidates, NONE)).toBe(6); + }); + + test('excludes a prioritized transaction and its whole cluster', () => { + const candidates = [ + tx('coinbase', 0), + tx('top', 10), + tx('low', 1, ['low', 'parent']), + tx('bottom', 4), + ]; + expect(computeMinFeeRate(candidates, NONE)).toBe(4); + }); + + test('excludes an accelerated transaction and its cluster even at a normal rate', () => { + const candidates = [ + tx('coinbase', 0), + tx('other', 8), + tx('boosted', 5, ['boosted', 'ancestor']), + ]; + expect(computeMinFeeRate(candidates, new Set(['boosted']))).toBe(8); + }); + + test('returns null when no transaction qualifies', () => { + expect(computeMinFeeRate( + [tx('coinbase', 0), tx('only', 3)], + new Set(['only']), + )).toBeNull(); + expect(computeMinFeeRate([tx('coinbase', 0)], NONE)).toBeNull(); + expect(computeMinFeeRate([], NONE)).toBeNull(); + }); +}); + +describe('buildOutOfBandCandidates', () => { + test('reconstructs cluster membership from ancestors and descendants', () => { + const result = buildOutOfBandCandidates([ + { txid: 'child', effectiveFeePerVsize: 3, ancestors: [{ txid: 'parent' }], descendants: [] }, + { txid: 'parent', effectiveFeePerVsize: 3, ancestors: [], descendants: [{ txid: 'child' }] }, + ]); + expect(result[0]).toEqual({ txid: 'child', effectiveFeePerVsize: 3, cluster: ['child', 'parent'] }); + expect(result[1]).toEqual({ txid: 'parent', effectiveFeePerVsize: 3, cluster: ['parent', 'child'] }); + }); + + test('defaults to a single-transaction cluster when there are no relatives', () => { + const result = buildOutOfBandCandidates([{ txid: 'solo', effectiveFeePerVsize: 4 }]); + expect(result).toEqual([{ txid: 'solo', effectiveFeePerVsize: 4, cluster: ['solo'] }]); + }); + + test('defaults a missing effective rate to 0', () => { + const result = buildOutOfBandCandidates([{ txid: 'coinbase' }]); + expect(result).toEqual([{ txid: 'coinbase', effectiveFeePerVsize: 0, cluster: ['coinbase'] }]); + }); +}); + +describe('min fee rate persistence constants', () => { + test('a version bump makes an older result eligible', () => { + expect(MIN_FEE_RATE_VERSION).toBeGreaterThan(0); + expect(isMinFeeRateVersionStale(MIN_FEE_RATE_VERSION - 1)).toBe(true); + expect(isMinFeeRateVersionStale(MIN_FEE_RATE_VERSION)).toBe(false); + }); + + test('requires trusted summaries and starts at the Core 30 release day', () => { + expect(MIN_SUMMARY_VERSION).toBe(2); + expect(MIN_FEE_RATE_START_DATE).toBe('2025-10-10 00:00:00'); + }); +}); diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index be247f571..06c98fc9c 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -34,6 +34,7 @@ import statistics from './statistics/statistics'; import { calcBitsDifference } from './difficulty-adjustment'; import AccelerationRepository from '../repositories/AccelerationRepository'; import { calculateGoodBlockCpfp } from './cpfp'; +import { buildOutOfBandCandidates, computeMinFeeRate, MIN_SUMMARY_VERSION } from './mining/min-fee-rate'; import blockProcessor, { BlockProcessingResult, detectTemplateAlgorithm, saveCpfpDataToCpfpSummary } from './block-processor'; import mempool from './mempool'; import CpfpRepository from '../repositories/CpfpRepository'; @@ -620,6 +621,36 @@ class Blocks { }); this.updateTimerProgress(timer, `saved audit results for ${this.currentBlockHeight}`); } + + // Live min_fee_rate: reuse the CPFP pass already computed above instead of running + // a second makeBlockTemplate. Only version >= MIN_SUMMARY_VERSION (calculateGood / + // calculateClusterMempool) carries a CPFP-adjusted rate and per-tx cluster data; + // a Fast-indexed (version 1) block is left for the backfill sweep, which runs + // makeBlockTemplate directly and so covers it regardless of version. + // + // cpfpSummary.transactions here carries rates computed with real poolAccelerations + // fed into makeBlockTemplate (calculateGoodBlockCpfp's own call, upstream of this + // point), while BlocksRepository.$backfillMinFeeRate deliberately uses []. This is + // a known, measured asymmetry: see the doc comment on $backfillMinFeeRate for the + // measurement and why it's accepted rather than unified. + if (cpfpSummary.version >= MIN_SUMMARY_VERSION) { + try { + const accelerationState = await AccelerationRepository.$getMinFeeRateAccelerationStateAtHeight(blockExtended.height); + const candidates = buildOutOfBandCandidates(cpfpSummary.transactions); + const rate = computeMinFeeRate(candidates, new Set(accelerationState.txids)); + await blocksRepository.$updateMinFeeRate(blockExtended.height, blockExtended.id, rate, { + accelerationCount: accelerationState.count, + accelerationFingerprint: accelerationState.fingerprint, + }); + } catch (e) { + logger.debug(`failed to compute live min_fee_rate for ${blockExtended.height}: ` + (e instanceof Error ? e.message : e)); + } + } + + // The backfill sweep independently covers Fast-indexed blocks, gaps, and + // algorithm-version upgrades; scheduling it here keeps it reacting to new blocks + // rather than waiting for a sequential indexing pass. + indexer.scheduleSingleTask('minFeeRate', 10000); } /** diff --git a/backend/src/api/mining/min-fee-rate.ts b/backend/src/api/mining/min-fee-rate.ts new file mode 100644 index 000000000..62f474807 --- /dev/null +++ b/backend/src/api/mining/min-fee-rate.ts @@ -0,0 +1,131 @@ +/** + * Minimum "fee-merit" effective fee rate for a single block (mempool issue #6639). + * + * The metric is the lowest CPFP-effective fee rate among the transactions that + * earned their block inclusion on fee merit alone. Transactions that were boosted + * out-of-band (prioritised via prioritisetransaction, or accelerated) are excluded + * using the same greedy scan the accelerator pricing path uses to price boosts + * (`findOutOfBandTransactions`), not the audit's prioritized/accelerated arrays: the + * audit is a global longest-increasing-subsequence classification with no cluster + * expansion or baseline guard, a different criterion that was not validated against + * this metric's reference figures. + * + * This module is intentionally pure (no DB, no I/O) so the exclusion logic can be + * unit-tested exhaustively. + */ +import { findOutOfBandTransactions, OutOfBandCandidate } from '../prioritization'; + +/** + * Minimum blocks_summaries.version whose `rate` can be trusted as CPFP-effective. + * Verified in RESEARCH-6639.md §A1: version 0 carries no `rate` field at all, and + * version 1 is ambiguous (effective when esplora-indexed, but nominal via the + * no-cpfpSummary fallback branch in blocks.ts). Only version >= 2 (calculateGood / + * calculateClusterMempool CPFP) guarantees a CPFP-adjusted effective rate. + */ +export const MIN_SUMMARY_VERSION = 2; + +/** + * Increment this whenever the algorithm changes. A block is eligible whenever its + * stored version is lower; late persisted inputs are tracked separately by snapshots. + * Bumped to 2: the exclusion criterion changed from the audit's prioritized/accelerated + * arrays to the greedy scan shared with the accelerator pricing path, so every + * previously computed value must be recomputed. + */ +export const MIN_FEE_RATE_VERSION = 2; + +/** + * Bitcoin Core 30.0 was released on 2025-10-10 and changed the default + * minrelaytxfee to 0.1 sat/vB. The series is intentionally undefined before then. + */ +export const MIN_FEE_RATE_START_DATE = '2025-10-10 00:00:00'; + +/** + * Fixed-size, order-independent snapshot of an acceleration set. Keep this exact SQL + * fragment shared by both the persisted-state read and the staleness sweep: using two + * independently maintained expressions would make every block permanently stale as + * soon as they diverged. + * + * COUNT handles cardinality while this 64-bit XOR detects membership changes. Unlike + * GROUP_CONCAT, its result size is independent of the number of accelerations. + */ +export const MIN_FEE_RATE_ACCELERATION_FINGERPRINT_SQL = + `LPAD(HEX(COALESCE(BIT_XOR(CAST(CONV(SUBSTRING(SHA2(txid, 256), 1, 16), 16, 10) AS UNSIGNED)), 0)), 16, '0')`; +export const MIN_FEE_RATE_EMPTY_ACCELERATION_FINGERPRINT = '0000000000000000'; + +/** + * Whether to skip transactions with a non-positive `rate`. Verified in + * RESEARCH-6639.md §A2/§A3: the coinbase sits at array index 0 with rate 0, and a + * 0-fee 1p1c package parent that CPFP failed to cluster also carries rate 0. Either + * would pin a naive MIN() to 0, so both must be excluded. + */ +export const EXCLUDE_ZERO_RATE = true; + +export interface MinFeeRateInputSnapshot { + accelerationCount: number; + accelerationFingerprint: string; +} + +export interface MinFeeRateAccelerationState { + txids: string[]; + count: number; + fingerprint: string; +} + +export interface MinFeeRateDay { + minRate: number; + minHeight: number; + timestamp: number; + usableBlockCount: number; +} + +export function isMinFeeRateVersionStale(storedVersion: number): boolean { + return storedVersion < MIN_FEE_RATE_VERSION; +} + +/** + * Builds the candidate list `findOutOfBandTransactions` needs from CPFP-summary + * transactions. Cluster membership is reconstructed from `ancestors`/`descendants` + * (populated by `calculateGoodBlockCpfp` for every multi-transaction cluster); a + * transaction with neither is not part of any cluster, so it stands alone. + */ +export function buildOutOfBandCandidates( + transactions: { txid: string; effectiveFeePerVsize?: number; ancestors?: { txid: string }[]; descendants?: { txid: string }[] }[] +): OutOfBandCandidate[] { + return transactions.map(tx => ({ + txid: tx.txid, + effectiveFeePerVsize: tx.effectiveFeePerVsize ?? 0, + cluster: (tx.ancestors?.length || tx.descendants?.length) + ? [tx.txid, ...(tx.ancestors || []).map(a => a.txid), ...(tx.descendants || []).map(a => a.txid)] + : [tx.txid], + })); +} + +/** + * Computes a block's minimum fee-merit effective fee rate: the lowest effective rate + * among transactions `findOutOfBandTransactions` does not flag as prioritized or + * accelerated. A null result is a valid answer for a block with no qualifying + * non-coinbase transaction. + */ +export function computeMinFeeRate( + candidates: readonly OutOfBandCandidate[], + acceleratedTxids: ReadonlySet, +): number | null { + const excluded = findOutOfBandTransactions(candidates, acceleratedTxids); + + let min: number | null = null; + // Skip index 0: the coinbase is identified positionally, not by a flag. + for (let i = 1; i < candidates.length; i++) { + const tx = candidates[i]; + if (EXCLUDE_ZERO_RATE && tx.effectiveFeePerVsize <= 0) { + continue; + } + if (excluded.has(tx.txid)) { + continue; + } + if (min === null || tx.effectiveFeePerVsize < min) { + min = tx.effectiveFeePerVsize; + } + } + + return min; +} diff --git a/backend/src/api/mining/mining-routes.ts b/backend/src/api/mining/mining-routes.ts index 47a6b8f0a..8e577dd26 100644 --- a/backend/src/api/mining/mining-routes.ts +++ b/backend/src/api/mining/mining-routes.ts @@ -29,6 +29,7 @@ class MiningRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees', this.$getBlockFeesTimespan) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', this.$getHistoricalBlockRewards) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', this.$getHistoricalBlockFeeRates) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/min-fee-rate/:interval', this.$getMinFeeRates) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', this.$getHistoricalBlockSizeAndWeight) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', this.$getDifficultyAdjustments) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/predictions/:interval', this.$getHistoricalBlocksHealth) @@ -268,6 +269,29 @@ class MiningRoutes { } } + private async $getMinFeeRates(req: Request, res: Response) { + try { + const minFeeRates = await mining.$getMinFeeRates(req.params.interval); + // Deliberately the unfiltered count, not the selected interval's: this header + // only feeds the frontend's period-button visibility guards (stats.dayCount >= + // N), which must see the full history to decide whether 3M/6M/1Y etc. exist, + // regardless of which period is currently selected. The CDF's threshold + // percentage does NOT read this header — it uses data.length, which is already + // period-local by construction. Scoping this to the interval would break the + // period selector: picking 1M would hide 3M/6M until a reload. Matches the + // convention of every other route in this file ($getPools, $getPoolsHistoricalHashrate, + // etc.), which all report the unfiltered count for the same reason. + const dayCount = await BlocksRepository.$getMinFeeRateDayCount(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.header('X-total-count', dayCount.toString()); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json(minFeeRates); + } catch (e) { + handleError(req, res, 500, 'Failed to get minimum daily fee rates'); + } + } + private async $getHistoricalBlockSizeAndWeight(req: Request, res: Response) { try { const blockSizes = await mining.$getHistoricalBlockSizes(req.params.interval); diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index ebf8fd40d..73037737d 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -1,5 +1,6 @@ import { BlockPrice, PoolInfo, PoolStats, RewardStats } from '../../mempool.interfaces'; import BlocksRepository from '../../repositories/BlocksRepository'; +import { MinFeeRateDay } from './min-fee-rate'; import PoolsRepository from '../../repositories/PoolsRepository'; import HashratesRepository from '../../repositories/HashratesRepository'; import bitcoinClient from '../bitcoin/bitcoin-client'; @@ -87,6 +88,17 @@ class Mining { ); } + /** + * Get the minimum fee-merit effective fee rate per UTC day (issue #6639). + * Fixed calendar-day buckets, so no rolling DIV time range is used — only the + * optional interval window filter. + */ + public async $getMinFeeRates(interval: string | null = null): Promise { + return await BlocksRepository.$getMinFeeRatesByDay( + Common.getSqlInterval(interval) + ); + } + /** * Get historical block sizes */ diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 46256cc29..127211dc6 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -19,7 +19,7 @@ export interface CoreIndex { best_block_height: number; } -type TaskName = 'blocksPrices' | 'coinStatsIndex'; +type TaskName = 'blocksPrices' | 'coinStatsIndex' | 'minFeeRate'; class Indexer { private runIndexer = true; @@ -162,6 +162,15 @@ class Indexer { logger.debug(`failed to index coinstatsindex: ` + (e instanceof Error ? e.message : e)); } } break; + + case 'minFeeRate': { + logger.debug(`Backfilling min_fee_rate now`, logger.tags.mining); + try { + await BlocksRepository.$backfillMinFeeRate(); + } catch (e) { + logger.debug(`failed to backfill min_fee_rate: ` + (e instanceof Error ? e.message : e)); + } + } break; } this.tasksRunning[task] = false; diff --git a/backend/src/repositories/AccelerationRepository.ts b/backend/src/repositories/AccelerationRepository.ts index a45a80621..00bf86b95 100644 --- a/backend/src/repositories/AccelerationRepository.ts +++ b/backend/src/repositories/AccelerationRepository.ts @@ -12,6 +12,11 @@ import bitcoinApi from '../api/bitcoin/bitcoin-api-factory'; import transactionUtils from '../api/transaction-utils'; import { BlockExtended, MempoolTransactionExtended } from '../mempool.interfaces'; import { makeBlockTemplate } from '../api/mini-miner'; +import { + MinFeeRateAccelerationState, + MIN_FEE_RATE_ACCELERATION_FINGERPRINT_SQL, + MIN_FEE_RATE_EMPTY_ACCELERATION_FINGERPRINT, +} from '../api/mining/min-fee-rate'; export interface PublicAcceleration { txid: string, @@ -93,6 +98,42 @@ class AccelerationRepository { return null; } + /** + * Returns every acceleration txid plus a stable snapshot of the set for a block. + * Reads straight from the accelerations table (not gated by MEMPOOL_SERVICES), since + * the min fee rate pull sweep uses this to detect late input changes. + * @asyncSafe + */ + public async $getMinFeeRateAccelerationStateAtHeight(height: number): Promise { + try { + const [rows]: any[] = await DB.query(` + SELECT + accelerations.txid, + snapshot.count, + snapshot.fingerprint + FROM ( + SELECT + COUNT(*) AS count, + ${MIN_FEE_RATE_ACCELERATION_FINGERPRINT_SQL} AS fingerprint + FROM accelerations + WHERE height = ? + ) AS snapshot + LEFT JOIN accelerations ON accelerations.height = ? + ORDER BY accelerations.txid + `, [height, height]); + return { + // One row per txid keeps the exclusion set independent of aggregate-size + // limits. The LEFT JOIN also yields one snapshot row for an empty set. + txids: rows.flatMap(row => row.txid == null ? [] : [row.txid]), + count: rows[0]?.count || 0, + fingerprint: rows[0]?.fingerprint || MIN_FEE_RATE_EMPTY_ACCELERATION_FINGERPRINT, + }; + } catch (e) { + logger.err(`Cannot get min fee rate acceleration state at height ${height}. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + public async $getAccelerationInfo(poolSlug: string | null = null, height: number | null = null, interval: string | null = null): Promise { if (!interval || !['24h', '3d', '1w', '1m'].includes(interval)) { interval = '1m'; diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index 9d1be0eb6..b20c4ff9c 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -121,6 +121,34 @@ class BlocksAuditRepositories { } } + /** + * Lean fetch of just the out-of-band exclusion arrays for a block. Returns null + * when no audit row exists (the metric is then unavailable — see min-fee-rate.ts). + * Unlike $getBlockAudit this does not join blocks_templates, so it still resolves + * for audited blocks whose template row is missing. + * @asyncSafe + */ + public async $getBlockAuditExclusions(hash: string): Promise<{ version: number, prioritizedTxs: string[], acceleratedTxs: string[] } | null> { + try { + const [rows]: any[] = await DB.query( + `SELECT version, prioritized_txs as prioritizedTxs, accelerated_txs as acceleratedTxs + FROM blocks_audits WHERE hash = ?`, + [hash] + ); + if (!rows.length) { + return null; + } + return { + version: rows[0].version, + prioritizedTxs: JSON.parse(rows[0].prioritizedTxs), + acceleratedTxs: JSON.parse(rows[0].acceleratedTxs), + }; + } catch (e) { + logger.err(`Cannot get block audit exclusions for ${hash}. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + /** @asyncSafe */ public async $getBlockTemplateAlgo(hash: string): Promise { try { @@ -325,4 +353,3 @@ class BlocksAuditRepositories { } export default new BlocksAuditRepositories(); - diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 20d0a6e46..e9e8833ec 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1,5 +1,5 @@ import bitcoinApi, { bitcoinCoreApi } from '../api/bitcoin/bitcoin-api-factory'; -import { BlockExtended, BlockExtension, BlockPrice, EffectiveFeeStats } from '../mempool.interfaces'; +import { BlockExtended, BlockExtension, BlockPrice, EffectiveFeeStats, MempoolTransactionExtended } from '../mempool.interfaces'; import DB from '../database'; import logger from '../logger'; import { Common } from '../api/common'; @@ -13,6 +13,17 @@ import config from '../config'; import chainTips from '../api/chain-tips'; import blocks from '../api/blocks'; import BlocksAuditsRepository from './BlocksAuditsRepository'; +import AccelerationRepository from './AccelerationRepository'; +import { + computeMinFeeRate, + MinFeeRateDay, + MinFeeRateInputSnapshot, + MIN_FEE_RATE_ACCELERATION_FINGERPRINT_SQL, + MIN_FEE_RATE_START_DATE, + MIN_FEE_RATE_VERSION, +} from '../api/mining/min-fee-rate'; +import { OutOfBandCandidate } from '../api/prioritization'; +import { makeBlockTemplate } from '../api/mini-miner'; import transactionUtils from '../api/transaction-utils'; import { parseDATUMTemplateCreator, parseDMNDTemplateCreator } from '../utils/bitcoin-script'; import poolsUpdater from '../tasks/pools-updater'; @@ -1426,6 +1437,300 @@ class BlocksRepository { } return blocksMigrated; } + + /** + * Blocks whose persisted metric is older than the current algorithm. No dependency + * on blocks_summaries: the backfill fetches the full block itself and runs + * makeBlockTemplate directly, so it covers every block regardless of which CPFP + * producer originally indexed it. + */ + public async $getBlocksNeedingMinFeeRate(limit: number): Promise<{ height: number, hash: string }[]> { + try { + const [blocks]: any[] = await DB.query(` + SELECT height, hash + FROM blocks + WHERE min_fee_rate_version < ${MIN_FEE_RATE_VERSION} + AND stale = 0 + AND CONVERT_TZ(blockTimestamp, @@session.time_zone, '+00:00') >= ? + ORDER BY height DESC + LIMIT ? + `, [MIN_FEE_RATE_START_DATE, limit]); + return blocks; + } catch (e) { + logger.err(`Cannot get blocks needing min_fee_rate. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + + /** + * Blocks whose acceleration set no longer matches the input snapshot stored with the + * current result (a late-arriving accelerated txid for an already-computed block). + * The indexed block-side predicate keeps this pull sweep bounded to current, + * canonical results. + */ + public async $getBlocksWithChangedMinFeeRateInputs(limit: number): Promise<{ height: number, hash: string }[]> { + try { + const [blocks]: any[] = await DB.query(` + SELECT blocks.height, blocks.hash + FROM blocks + LEFT JOIN accelerations ON accelerations.height = blocks.height + WHERE blocks.min_fee_rate_version = ${MIN_FEE_RATE_VERSION} + AND blocks.stale = 0 + AND CONVERT_TZ(blocks.blockTimestamp, @@session.time_zone, '+00:00') >= ? + GROUP BY + blocks.height, + blocks.hash, + blocks.min_fee_rate_acceleration_count, + blocks.min_fee_rate_acceleration_fingerprint, + blocks.min_fee_rate_computed_at + HAVING + blocks.min_fee_rate_acceleration_count <> COUNT(accelerations.txid) + OR blocks.min_fee_rate_acceleration_fingerprint <> + ${MIN_FEE_RATE_ACCELERATION_FINGERPRINT_SQL} + ORDER BY blocks.min_fee_rate_computed_at ASC, blocks.height DESC + LIMIT ? + `, [MIN_FEE_RATE_START_DATE, limit]); + return blocks; + } catch (e) { + logger.err(`Cannot get blocks with changed min_fee_rate inputs. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + + /** Persist a result and the exact input snapshot used to compute it. */ + public async $updateMinFeeRate( + height: number, + hash: string, + rate: number | null, + snapshot: MinFeeRateInputSnapshot + ): Promise { + try { + await DB.query( + `UPDATE blocks SET + min_fee_rate = ?, + min_fee_rate_version = ?, + min_fee_rate_acceleration_count = ?, + min_fee_rate_acceleration_fingerprint = ?, + min_fee_rate_computed_at = CURRENT_TIMESTAMP + WHERE height = ? AND hash = ?`, + [ + rate, + MIN_FEE_RATE_VERSION, + snapshot.accelerationCount, + snapshot.accelerationFingerprint, + height, + hash, + ] + ); + } catch (e) { + logger.err(`Cannot update min_fee_rate for block ${height} (${hash}). Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + + /** + * Minimum fee-merit effective fee rate per UTC calendar day (issue #6639). + * Unlike the rolling DIV-bucket mining charts, this buckets on the fixed UTC day so + * the "minimum daily fee rate" is stable regardless of the selected interval. + * The current UTC day is excluded because its partial MIN is biased upward. + * @asyncSafe + */ + public async $getMinFeeRatesByDay(interval: string | null): Promise { + try { + let query = ` + WITH eligible AS ( + SELECT + height, + min_fee_rate, + DATE(CONVERT_TZ(blockTimestamp, @@session.time_zone, '+00:00')) AS utcDay + FROM blocks + WHERE stale = 0 + AND min_fee_rate_version = ${MIN_FEE_RATE_VERSION} + AND min_fee_rate IS NOT NULL + AND CONVERT_TZ(blockTimestamp, @@session.time_zone, '+00:00') >= ? + AND CONVERT_TZ(blockTimestamp, @@session.time_zone, '+00:00') < UTC_DATE()`; + + if (interval !== null) { + query += ` + AND CONVERT_TZ(blockTimestamp, @@session.time_zone, '+00:00') + >= DATE_SUB(UTC_DATE(), INTERVAL ${interval})`; + } + + query += ` + ), + ranked AS ( + SELECT + height, + min_fee_rate, + utcDay, + ROW_NUMBER() OVER ( + PARTITION BY utcDay + ORDER BY min_fee_rate ASC, height ASC + ) AS dailyRank, + COUNT(*) OVER (PARTITION BY utcDay) AS usableBlockCount + FROM eligible + ) + SELECT + CAST(min_fee_rate AS DOUBLE) AS minRate, + height AS minHeight, + TIMESTAMPDIFF(SECOND, '1970-01-01 00:00:00', utcDay) AS timestamp, + usableBlockCount + FROM ranked + WHERE dailyRank = 1 + ORDER BY utcDay`; + + const [rows]: any = await DB.query(query, [MIN_FEE_RATE_START_DATE]); + return rows; + } catch (e) { + logger.err(`Cannot generate min fee rates by day. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + + /** + * Count complete UTC days that have at least one usable metric value. This is the + * same unit used by the graph's timespan selector. + */ + public async $getMinFeeRateDayCount(): Promise { + try { + const [rows]: any[] = await DB.query(` + SELECT COUNT(DISTINCT DATE(CONVERT_TZ(blockTimestamp, @@session.time_zone, '+00:00'))) AS count + FROM blocks + WHERE stale = 0 + AND min_fee_rate_version = ${MIN_FEE_RATE_VERSION} + AND min_fee_rate IS NOT NULL + AND CONVERT_TZ(blockTimestamp, @@session.time_zone, '+00:00') >= ? + AND CONVERT_TZ(blockTimestamp, @@session.time_zone, '+00:00') < UTC_DATE() + `, [MIN_FEE_RATE_START_DATE]); + return rows[0]?.count || 0; + } catch (e) { + logger.err(`Cannot count minimum fee rate days. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + + /** + * Backfill blocks.min_fee_rate by fetching each block's full transactions (with + * `vin`, unlike blocks_summaries) and running makeBlockTemplate directly, the same + * criterion the live path derives from calculateGoodBlockCpfp's pass and the exact + * one calculateBoostRate uses to price accelerator boosts. This makes the metric + * independent of which CPFP producer originally indexed a block, eliminating the + * Fast-vs-Good divergence (a real "different value depending on when it was + * processed" bug, measured and confirmed fixed: see the acceleration-parity + * measurement below for the one input this does NOT unify). + * + * accelerations is deliberately passed as [] to makeBlockTemplate here, while the + * live path (blocks.ts $saveBlockData) passes calculateGoodBlockCpfp's real + * poolAccelerations. This is a known, measured asymmetry, not an oversight: + * + * - accelerations=[] is the configuration RESEARCH-6639.md validated against the + * reference figures (6% of days <=0.1 sat/vB, p50 0.15, max 0.63). Passing real + * max_bid values here would ship a configuration never checked against them. + * - Aligning the live path to [] instead would mean it can no longer reuse the + * calculateGoodBlockCpfp pass already computed for CPFP, giving up the measured + * 0.5% marginal cost and putting a ~76ms template rebuild on block arrival. + * + * Measured impact (2,394 comparisons: 800 real blocks x 3 synthetic acceleration + * densities, real accelerations vs [] into makeBlockTemplate, same helper applied to + * both): the excluded set differs in 98% of blocks and the per-block minimum in ~6%, + * with a statistically significant upward bias when real accelerations are present + * (62.5% of differing cases higher, sign-test p=0.003) — a low-rate transaction + * absorbed into a boosted cluster inherits its higher effective rate. Aggregated to + * the published daily minimum (what actually ships), only 1.8% of sampled days + * differ, by under 0.004 sat/vB — but that figure comes from a sparse per-day sample + * (~3 of a day's ~144 blocks) and likely understates the true rate at full coverage. + * No sampled block landed at or below the 0.1 sat/vB reference threshold (minimum + * observed: 0.103), so whether a real block ever crosses it differently between the + * two paths remains untested by this measurement, not disproven. + * + * The standard this asymmetry has to clear is narrower than the one Fast-vs-Good + * failed: not "does not diverge", but "does not diverge on the published metric". + * Real accelerations are typically absent on both paths in production (the + * acceleration service is opt-in), so most blocks see [] on both sides regardless. + * + * Each run processes at most one algorithm-version batch and one smaller + * low-priority input-change sweep; neither queue is drained in one invocation. A + * single block's RPC/compute failure is logged and skipped rather than aborting the + * rest of the batch, since this path now depends on Core RPC per block. + * @asyncSafe + */ + public async $backfillMinFeeRate(): Promise { + const batchSize = 1000; + const sweepBatchSize = 100; + let blocksProcessed = 0; + try { + let timer = Date.now() / 1000; + const startedAt = Date.now() / 1000; + + const processBatch = async (batch: { height: number, hash: string }[]): Promise => { + for (const row of batch) { + try { + let transactions: MempoolTransactionExtended[] | undefined; + if (config.MEMPOOL.BACKEND === 'esplora') { + transactions = (await bitcoinApi.$getTxsForBlock(row.hash, true)).map(tx => transactionUtils.extendMempoolTransaction(tx)); + } + if (!transactions) { + const block = await bitcoinClient.getBlock(row.hash, 2); + transactions = block.tx.map(tx => { + tx.fee *= 100_000_000; + return tx; + }); + } + if (!transactions?.length) { + throw new Error(`missing transaction data`); + } + + const accelerationState = await AccelerationRepository.$getMinFeeRateAccelerationStateAtHeight(row.height); + const acceleratedTxids = new Set(accelerationState.txids); + + const template = makeBlockTemplate(transactions, [], 1, Infinity, Infinity); + const templateMap = new Map(template.map(tx => [tx.txid, tx])); + const candidates: OutOfBandCandidate[] = transactions.map(tx => { + const templateTx = templateMap.get(tx.txid); + return { + txid: tx.txid, + effectiveFeePerVsize: templateTx?.effectiveFeePerVsize ?? 0, + cluster: templateTx?.cluster || [tx.txid], + }; + }); + + const rate = computeMinFeeRate(candidates, acceleratedTxids); + + await this.$updateMinFeeRate(row.height, row.hash, rate, { + accelerationCount: accelerationState.count, + accelerationFingerprint: accelerationState.fingerprint, + }); + + blocksProcessed++; + } catch (e) { + logger.err(`Cannot backfill min_fee_rate for block ${row.height} (${row.hash}), skipping. Reason: ` + (e instanceof Error ? e.message : e)); + } + + const elapsedSeconds = (Date.now() / 1000) - timer; + if (elapsedSeconds > 5) { + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = blocksProcessed / elapsedSeconds; + logger.debug(`Backfilling min_fee_rate | ~${blockPerSeconds.toFixed(2)} blocks/sec | height: ${row.height} | total: ${blocksProcessed} | elapsed: ${runningFor.toFixed(2)} seconds`); + timer = Date.now() / 1000; + } + } + }; + + const versionBatch = await this.$getBlocksNeedingMinFeeRate(batchSize); + await processBatch(versionBatch); + + const changedInputsBatch = await this.$getBlocksWithChangedMinFeeRateInputs(sweepBatchSize); + await processBatch(changedInputsBatch); + + if (blocksProcessed > 0) { + logger.notice(`Backfilling min_fee_rate completed: processed ${blocksProcessed} blocks`); + } + } catch (e) { + logger.err(`Backfilling min_fee_rate failed. Trying again later. Reason: ${(e instanceof Error ? e.message : e)}`); + throw e; + } + return blocksProcessed; + } } export default new BlocksRepository(); From d6692f8cbc95deafb6709f4146306079bc6cb64f Mon Sep 17 00:00:00 2001 From: jramos0 Date: Fri, 31 Jul 2026 03:43:09 -0600 Subject: [PATCH 4/4] Add minimum daily fee rate charts --- .../components/graphs/graphs.component.html | 4 + .../min-fee-rate-cdf-graph.component.html | 65 ++++ .../min-fee-rate-cdf-graph.component.scss | 108 ++++++ .../min-fee-rate-cdf-graph.component.ts | 320 ++++++++++++++++++ .../min-fee-rate-graph.component.html | 65 ++++ .../min-fee-rate-graph.component.scss | 108 ++++++ .../min-fee-rate-graph.component.ts | 318 +++++++++++++++++ frontend/src/app/graphs/graphs.module.ts | 4 + .../src/app/graphs/graphs.routing.module.ts | 12 + .../src/app/interfaces/node-api.interface.ts | 9 +- frontend/src/app/services/api.service.ts | 7 + .../src/app/services/min-fee-rate.service.ts | 78 +++++ 12 files changed, 1097 insertions(+), 1 deletion(-) create mode 100644 frontend/src/app/components/min-fee-rate-cdf-graph/min-fee-rate-cdf-graph.component.html create mode 100644 frontend/src/app/components/min-fee-rate-cdf-graph/min-fee-rate-cdf-graph.component.scss create mode 100644 frontend/src/app/components/min-fee-rate-cdf-graph/min-fee-rate-cdf-graph.component.ts create mode 100644 frontend/src/app/components/min-fee-rate-graph/min-fee-rate-graph.component.html create mode 100644 frontend/src/app/components/min-fee-rate-graph/min-fee-rate-graph.component.scss create mode 100644 frontend/src/app/components/min-fee-rate-graph/min-fee-rate-graph.component.ts create mode 100644 frontend/src/app/services/min-fee-rate.service.ts diff --git a/frontend/src/app/components/graphs/graphs.component.html b/frontend/src/app/components/graphs/graphs.component.html index 74e37d298..cf83a7f2c 100644 --- a/frontend/src/app/components/graphs/graphs.component.html +++ b/frontend/src/app/components/graphs/graphs.component.html @@ -14,6 +14,10 @@ [routerLink]="['/graphs/mining/hashrate-difficulty' | relativeUrl]" i18n="mining.hashrate-difficulty">Hashrate & Difficulty Block Fee Rates + Minimum Daily Fee Rate + Share of days at or below a fee rate Block Fees + +
+
+
+
+ Share of days at or below a fee rate + +
+
Cumulative distribution of the minimum daily fee rate (prioritized transactions excluded)
+
+
+ +
+
+ + +
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ +
+ No minimum daily fee rate data available yet. +
+ +
+
+
+
+
+ +
diff --git a/frontend/src/app/components/min-fee-rate-cdf-graph/min-fee-rate-cdf-graph.component.scss b/frontend/src/app/components/min-fee-rate-cdf-graph/min-fee-rate-cdf-graph.component.scss new file mode 100644 index 000000000..ad23fe65b --- /dev/null +++ b/frontend/src/app/components/min-fee-rate-cdf-graph/min-fee-rate-cdf-graph.component.scss @@ -0,0 +1,108 @@ +.full-container { + display: flex; + flex-direction: column; + padding: 0px 15px; + width: 100%; + height: calc(100vh - 225px); + min-height: 400px; + @media (min-width: 992px) { + height: calc(100vh - 150px); + } +} + +.chart-header { + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 12px; + @media (min-width: 768px) { + flex-direction: row; + align-items: flex-start; + justify-content: space-between; + gap: 20px; + } + + .heading { + min-width: 0; + + .title { + font-size: 18px; + line-height: 1.2; + @media (min-width: 465px) { + font-size: 20px; + } + .btn { + vertical-align: baseline; + } + } + + .subtitle { + margin-top: 4px; + font-size: 12px; + color: var(--transparent-fg); + } + } + +} + +.chart-controls { + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 10px; + margin: 12px 0 4px; + @media (min-width: 768px) { + flex-direction: row; + flex-wrap: wrap; + align-items: center; + justify-content: space-between; + } + + .threshold-input { + display: flex; + align-items: center; + gap: 8px; + flex: 0 0 auto; + label { + margin: 0; + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.5px; + color: var(--transparent-fg); + white-space: nowrap; + } + input { + width: 90px; + } + } + + .formRadioGroup { + .btn-sm { + font-size: 9px; + @media (min-width: 830px) { + font-size: 14px; + } + } + } +} + +.chart { + display: flex; + flex: 1; + height: 100%; + padding-bottom: 20px; + padding-right: 10px; + @media (max-width: 992px) { + padding-bottom: 25px; + } +} + +.chart-widget { + width: 100%; + height: 100%; + max-height: 238px; +} + +.no-data { + padding: 30px 0; +} diff --git a/frontend/src/app/components/min-fee-rate-cdf-graph/min-fee-rate-cdf-graph.component.ts b/frontend/src/app/components/min-fee-rate-cdf-graph/min-fee-rate-cdf-graph.component.ts new file mode 100644 index 000000000..c245eb9a7 --- /dev/null +++ b/frontend/src/app/components/min-fee-rate-cdf-graph/min-fee-rate-cdf-graph.component.ts @@ -0,0 +1,320 @@ +import { ChangeDetectionStrategy, ChangeDetectorRef, Component, Inject, Input, LOCALE_ID, OnInit } from '@angular/core'; +import { EChartsOption } from '@app/graphs/echarts'; +import { Observable, combineLatest, of } from 'rxjs'; +import { map, share, startWith, switchMap, tap } from 'rxjs/operators'; +import { SeoService } from '@app/services/seo.service'; +import { formatNumber } from '@angular/common'; +import { UntypedFormBuilder, UntypedFormGroup } from '@angular/forms'; +import { download } from '@app/shared/graphs.utils'; +import { StorageService } from '@app/services/storage.service'; +import { MiningService } from '@app/services/mining.service'; +import { StateService } from '@app/services/state.service'; +import { ActivatedRoute } from '@angular/router'; +import { MinFeeRateDay } from '@app/interfaces/node-api.interface'; +import { DEFAULT_MIN_FEE_RATE_THRESHOLD, MinFeeRateService } from '@app/services/min-fee-rate.service'; +import { chartColors } from '@app/app.constants'; + +const CURVE_COLOR = chartColors[8]; // '#00897B' + +// The series is one point per day, so anything shorter than a month is degenerate. +const TIMESPANS = ['1m', '3m', '6m', '1y', '2y', '3y', 'all']; + +@Component({ + selector: 'app-min-fee-rate-cdf-graph', + templateUrl: './min-fee-rate-cdf-graph.component.html', + styleUrls: ['./min-fee-rate-cdf-graph.component.scss'], + styles: [` + .loadingGraphs { + position: absolute; + top: 50%; + left: calc(50% - 15px); + z-index: 99; + } + `], + standalone: false, + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class MinFeeRateCdfGraphComponent implements OnInit { + @Input() widget = false; + + miningWindowPreference: string; + radioGroupForm: UntypedFormGroup; + + chartOptions: EChartsOption = {}; + chartInitOptions = { + renderer: 'svg', + }; + + statsObservable$: Observable; + isLoading = true; + formatNumber = formatNumber; + timespan = ''; + chartInstance: any = undefined; + + data: MinFeeRateDay[] = []; + threshold = DEFAULT_MIN_FEE_RATE_THRESHOLD; + + // Share of days at or below the threshold: drives the marker and the legend readout. + percentBelow = 0; + + constructor( + @Inject(LOCALE_ID) public locale: string, + private seoService: SeoService, + private minFeeRateService: MinFeeRateService, + private formBuilder: UntypedFormBuilder, + private storageService: StorageService, + private miningService: MiningService, + public stateService: StateService, + private route: ActivatedRoute, + private cd: ChangeDetectorRef, + ) { + this.radioGroupForm = this.formBuilder.group({ dateSpan: '1m', threshold: DEFAULT_MIN_FEE_RATE_THRESHOLD }); + this.radioGroupForm.controls.dateSpan.setValue('1m'); + } + + ngOnInit(): void { + if (this.widget) { + this.miningWindowPreference = '1m'; + } else { + this.seoService.setTitle($localize`:@@mining.min-fee-rate-cdf:Share of days at or below a fee rate`); + this.seoService.setDescription($localize`:@@meta.description.bitcoin.graphs.min-fee-rate-cdf:The cumulative share of days whose minimum fee-merit fee rate was at or below a given fee rate.`); + // miningWindowPreference is shared across every mining graph, so floor whatever it + // holds at the shortest timespan this chart offers. + this.miningWindowPreference = this.miningService.getDefaultTimespan('1m'); + } + this.radioGroupForm = this.formBuilder.group({ dateSpan: this.miningWindowPreference, threshold: DEFAULT_MIN_FEE_RATE_THRESHOLD }); + this.radioGroupForm.controls.dateSpan.setValue(this.miningWindowPreference); + + if (!this.widget) { + this.route + .fragment + .subscribe((fragment) => { + if (TIMESPANS.indexOf(fragment) > -1) { + this.radioGroupForm.controls.dateSpan.setValue(fragment, { emitEvent: false }); + } + }); + } + + // Threshold changes only move the marker and recompute the stats, no refetch. + this.radioGroupForm.get('threshold').valueChanges.subscribe((value) => { + const parsed = parseFloat(value); + this.threshold = isNaN(parsed) || parsed < 0 ? 0 : parsed; + this.updateChart(); + this.cd.markForCheck(); + }); + + this.statsObservable$ = combineLatest([ + this.widget ? of(this.miningWindowPreference) : this.radioGroupForm.get('dateSpan').valueChanges.pipe(startWith(this.radioGroupForm.controls.dateSpan.value)), + ]).pipe( + switchMap(([timespan]) => { + if (!this.widget) { + this.storageService.setValue('miningWindowPreference', timespan); + } + this.timespan = timespan; + this.isLoading = true; + return this.minFeeRateService.getMinFeeRates$(timespan) + .pipe( + tap((response) => { + this.data = response.body || []; + this.updateChart(); + this.isLoading = false; + this.cd.markForCheck(); + }), + map((response) => { + return { + dayCount: parseInt(response.headers.get('x-total-count'), 10), + }; + }), + ); + }), + share(), + ); + } + + updateChart(): void { + this.percentBelow = this.minFeeRateService.getStats(this.data, this.threshold).percentBelow; + this.prepareChartOptions(this.minFeeRateService.buildCdf(this.data)); + } + + formatFeeRate(val: number): string { + return this.minFeeRateService.formatFeeRate(val); + } + + prepareChartOptions(cdf: number[][]): void { + const hasData = cdf.length > 0; + const curveLabel = $localize`:@@mining.min-fee-rate-cdf.legend-curve:cumulative % of days ≤ fee rate`; + const thresholdValue = this.formatFeeRate(this.threshold); + const thresholdPercent = `${formatNumber(this.percentBelow, this.locale, '1.1-1')}%`; + const thresholdLabel = $localize`:@@mining.min-fee-rate-cdf.legend-threshold:threshold ${thresholdValue}:VALUE: sat/vB → ${thresholdPercent}:PERCENT:`; + + this.chartOptions = { + color: [CURVE_COLOR], + animation: false, + grid: { + right: this.widget ? 10 : 30, + left: this.widget ? 45 : 65, + bottom: this.widget ? 30 : 75, + top: 20, + }, + legend: (this.widget || !hasData) ? undefined : { + bottom: 0, + left: 'center', + width: '90%', + data: [curveLabel, thresholdLabel], + textStyle: { + color: 'var(--transparent-fg)', + fontSize: 11, + }, + inactiveColor: 'rgb(110, 112, 121)', + }, + tooltip: { + show: !this.isMobile(), + trigger: 'axis', + axisPointer: { + type: 'line' + }, + backgroundColor: 'rgba(17, 19, 31, 1)', + borderRadius: 4, + shadowColor: 'rgba(0, 0, 0, 0.5)', + textStyle: { + color: 'var(--tooltip-grey)', + align: 'left', + }, + borderColor: '#000', + formatter: function (data: any): string { + const point = data.find(d => d.seriesName === curveLabel); + if (!point) { + return ''; + } + let tooltip = `≤ ${this.formatFeeRate(+point.data[0])} sat/vB
`; + tooltip += `${point.marker} ` + $localize`Share of days` + `: ${(+point.data[1]).toFixed(1)}%`; + return tooltip; + }.bind(this) + }, + xAxis: !hasData ? undefined : { + name: this.widget ? undefined : $localize`:@@mining.min-fee-rate-cdf.x-axis:fee rate (sat/vB)`, + nameLocation: 'middle', + nameTextStyle: { + color: 'rgb(110, 112, 121)', + fontSize: 12, + padding: [12, 0, 0, 0], + }, + type: 'value', + axisLabel: { + color: 'rgb(110, 112, 121)', + fontSize: 11, + formatter: (val): string => this.formatFeeRate(val), + }, + splitLine: { + lineStyle: { + type: 'dotted', + color: 'var(--transparent-fg)', + opacity: 0.25, + } + }, + }, + yAxis: !hasData ? undefined : { + position: 'left', + name: this.widget ? undefined : $localize`:@@mining.min-fee-rate-cdf.y-axis:% of days`, + nameLocation: 'middle', + nameRotate: 90, + nameGap: 42, + nameTextStyle: { + color: 'rgb(110, 112, 121)', + fontSize: 12, + }, + min: 0, + max: 100, + axisLabel: { + color: 'rgb(110, 112, 121)', + formatter: (val): string => `${val}%`, + }, + splitLine: { + lineStyle: { + type: 'dotted', + color: 'var(--transparent-fg)', + opacity: 0.25, + } + }, + type: 'value', + }, + series: !hasData ? undefined : [ + { + zlevel: 0, + name: curveLabel, + data: cdf, + type: 'line', + step: 'end', + symbol: 'none', + lineStyle: { + color: CURVE_COLOR, + width: 3, + }, + areaStyle: { + color: CURVE_COLOR, + opacity: 0.12, + }, + }, + { + zlevel: 1, + name: thresholdLabel, + type: 'line', + data: [[this.threshold, 0], [this.threshold, 100]], + symbol: 'none', + silent: true, + lineStyle: { + color: 'var(--fg)', + type: 'dashed', + width: 2, + }, + itemStyle: { + color: 'var(--fg)', + }, + }, + // Marker where the threshold crosses the curve. A separate series rather than a + // markPoint because MarkPointComponent is not registered in the echarts bundle. + { + zlevel: 2, + name: 'threshold-marker', + type: 'scatter', + data: [[this.threshold, this.percentBelow]], + symbolSize: 10, + silent: true, + itemStyle: { + color: 'var(--fg)', + borderColor: CURVE_COLOR, + borderWidth: 2, + }, + }, + ], + }; + } + + onChartInit(ec): void { + if (this.chartInstance !== undefined) { + return; + } + this.chartInstance = ec; + } + + isMobile(): boolean { + return (window.innerWidth <= 767.98); + } + + onSaveChart(): void { + // @ts-ignore + const prevBottom = this.chartOptions.grid.bottom; + const now = new Date(); + // @ts-ignore + this.chartOptions.grid.bottom = 75; + this.chartOptions.backgroundColor = 'var(--active-bg)'; + this.chartInstance.setOption(this.chartOptions); + download(this.chartInstance.getDataURL({ + pixelRatio: 2, + }), `min-fee-rate-cdf-${this.timespan}-${Math.round(now.getTime() / 1000)}.svg`); + // @ts-ignore + this.chartOptions.grid.bottom = prevBottom; + this.chartOptions.backgroundColor = 'none'; + this.chartInstance.setOption(this.chartOptions); + } +} diff --git a/frontend/src/app/components/min-fee-rate-graph/min-fee-rate-graph.component.html b/frontend/src/app/components/min-fee-rate-graph/min-fee-rate-graph.component.html new file mode 100644 index 000000000..65cff3edf --- /dev/null +++ b/frontend/src/app/components/min-fee-rate-graph/min-fee-rate-graph.component.html @@ -0,0 +1,65 @@ + + +
+
+
+
+ Minimum Daily Fee Rate + +
+
Cheapest daily included effective fee rate (prioritized transactions excluded)
+
+
+ +
+
+ + +
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ +
+ No minimum daily fee rate data available yet. +
+ +
+
+
+
+
+ +
diff --git a/frontend/src/app/components/min-fee-rate-graph/min-fee-rate-graph.component.scss b/frontend/src/app/components/min-fee-rate-graph/min-fee-rate-graph.component.scss new file mode 100644 index 000000000..ad23fe65b --- /dev/null +++ b/frontend/src/app/components/min-fee-rate-graph/min-fee-rate-graph.component.scss @@ -0,0 +1,108 @@ +.full-container { + display: flex; + flex-direction: column; + padding: 0px 15px; + width: 100%; + height: calc(100vh - 225px); + min-height: 400px; + @media (min-width: 992px) { + height: calc(100vh - 150px); + } +} + +.chart-header { + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 12px; + @media (min-width: 768px) { + flex-direction: row; + align-items: flex-start; + justify-content: space-between; + gap: 20px; + } + + .heading { + min-width: 0; + + .title { + font-size: 18px; + line-height: 1.2; + @media (min-width: 465px) { + font-size: 20px; + } + .btn { + vertical-align: baseline; + } + } + + .subtitle { + margin-top: 4px; + font-size: 12px; + color: var(--transparent-fg); + } + } + +} + +.chart-controls { + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 10px; + margin: 12px 0 4px; + @media (min-width: 768px) { + flex-direction: row; + flex-wrap: wrap; + align-items: center; + justify-content: space-between; + } + + .threshold-input { + display: flex; + align-items: center; + gap: 8px; + flex: 0 0 auto; + label { + margin: 0; + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.5px; + color: var(--transparent-fg); + white-space: nowrap; + } + input { + width: 90px; + } + } + + .formRadioGroup { + .btn-sm { + font-size: 9px; + @media (min-width: 830px) { + font-size: 14px; + } + } + } +} + +.chart { + display: flex; + flex: 1; + height: 100%; + padding-bottom: 20px; + padding-right: 10px; + @media (max-width: 992px) { + padding-bottom: 25px; + } +} + +.chart-widget { + width: 100%; + height: 100%; + max-height: 238px; +} + +.no-data { + padding: 30px 0; +} diff --git a/frontend/src/app/components/min-fee-rate-graph/min-fee-rate-graph.component.ts b/frontend/src/app/components/min-fee-rate-graph/min-fee-rate-graph.component.ts new file mode 100644 index 000000000..32401d649 --- /dev/null +++ b/frontend/src/app/components/min-fee-rate-graph/min-fee-rate-graph.component.ts @@ -0,0 +1,318 @@ +import { ChangeDetectionStrategy, ChangeDetectorRef, Component, Inject, Input, LOCALE_ID, NgZone, OnInit } from '@angular/core'; +import { EChartsOption } from '@app/graphs/echarts'; +import { Observable, combineLatest, of } from 'rxjs'; +import { map, share, startWith, switchMap, tap } from 'rxjs/operators'; +import { SeoService } from '@app/services/seo.service'; +import { formatNumber } from '@angular/common'; +import { UntypedFormBuilder, UntypedFormGroup } from '@angular/forms'; +import { download } from '@app/shared/graphs.utils'; +import { StorageService } from '@app/services/storage.service'; +import { MiningService } from '@app/services/mining.service'; +import { RelativeUrlPipe } from '@app/shared/pipes/relative-url/relative-url.pipe'; +import { StateService } from '@app/services/state.service'; +import { ActivatedRoute, Router } from '@angular/router'; +import { MinFeeRateDay } from '@app/interfaces/node-api.interface'; +import { DEFAULT_MIN_FEE_RATE_THRESHOLD, MinFeeRateService } from '@app/services/min-fee-rate.service'; +import { chartColors } from '@app/app.constants'; + +// Days at or below the threshold are highlighted in green; the rest keep the warm +// default. Both come from the shared chart palette. +const HIGHLIGHT_COLOR = chartColors[9]; // '#43A047' +const DEFAULT_BAR_COLOR = chartColors[14]; // '#FB8C00' + +// The series is one point per day, so anything shorter than a month is degenerate. +const TIMESPANS = ['1m', '3m', '6m', '1y', '2y', '3y', 'all']; + +@Component({ + selector: 'app-min-fee-rate-graph', + templateUrl: './min-fee-rate-graph.component.html', + styleUrls: ['./min-fee-rate-graph.component.scss'], + styles: [` + .loadingGraphs { + position: absolute; + top: 50%; + left: calc(50% - 15px); + z-index: 99; + } + `], + standalone: false, + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class MinFeeRateGraphComponent implements OnInit { + @Input() widget = false; + + miningWindowPreference: string; + radioGroupForm: UntypedFormGroup; + + chartOptions: EChartsOption = {}; + chartInitOptions = { + renderer: 'svg', + }; + + statsObservable$: Observable; + isLoading = true; + formatNumber = formatNumber; + timespan = ''; + chartInstance: any = undefined; + + data: MinFeeRateDay[] = []; + threshold = DEFAULT_MIN_FEE_RATE_THRESHOLD; + + constructor( + @Inject(LOCALE_ID) public locale: string, + private seoService: SeoService, + private minFeeRateService: MinFeeRateService, + private formBuilder: UntypedFormBuilder, + private storageService: StorageService, + private miningService: MiningService, + public stateService: StateService, + private router: Router, + private zone: NgZone, + private route: ActivatedRoute, + private cd: ChangeDetectorRef, + ) { + this.radioGroupForm = this.formBuilder.group({ dateSpan: '1m', threshold: DEFAULT_MIN_FEE_RATE_THRESHOLD }); + this.radioGroupForm.controls.dateSpan.setValue('1m'); + } + + ngOnInit(): void { + if (this.widget) { + this.miningWindowPreference = '1m'; + } else { + this.seoService.setTitle($localize`:@@mining.min-fee-rate:Minimum Daily Fee Rate`); + this.seoService.setDescription($localize`:@@meta.description.bitcoin.graphs.min-fee-rate:See the lowest fee rate that earned block inclusion on fee merit each day, excluding prioritized and accelerated transactions.`); + // miningWindowPreference is shared across every mining graph, so floor whatever it + // holds at the shortest timespan this chart offers. + this.miningWindowPreference = this.miningService.getDefaultTimespan('1m'); + } + this.radioGroupForm = this.formBuilder.group({ dateSpan: this.miningWindowPreference, threshold: DEFAULT_MIN_FEE_RATE_THRESHOLD }); + this.radioGroupForm.controls.dateSpan.setValue(this.miningWindowPreference); + + if (!this.widget) { + this.route + .fragment + .subscribe((fragment) => { + if (TIMESPANS.indexOf(fragment) > -1) { + this.radioGroupForm.controls.dateSpan.setValue(fragment, { emitEvent: false }); + } + }); + } + + // Threshold changes only recolour the bars and move the marker, no refetch. + this.radioGroupForm.get('threshold').valueChanges.subscribe((value) => { + const parsed = parseFloat(value); + this.threshold = isNaN(parsed) || parsed < 0 ? 0 : parsed; + this.prepareChartOptions(); + this.cd.markForCheck(); + }); + + this.statsObservable$ = combineLatest([ + this.widget ? of(this.miningWindowPreference) : this.radioGroupForm.get('dateSpan').valueChanges.pipe(startWith(this.radioGroupForm.controls.dateSpan.value)), + ]).pipe( + switchMap(([timespan]) => { + if (!this.widget) { + this.storageService.setValue('miningWindowPreference', timespan); + } + this.timespan = timespan; + this.isLoading = true; + return this.minFeeRateService.getMinFeeRates$(timespan) + .pipe( + tap((response) => { + this.data = response.body || []; + this.prepareChartOptions(); + this.isLoading = false; + this.cd.markForCheck(); + }), + map((response) => { + return { + dayCount: parseInt(response.headers.get('x-total-count'), 10), + }; + }), + ); + }), + share(), + ); + } + + formatFeeRate(val: number): string { + return this.minFeeRateService.formatFeeRate(val); + } + + // Formatted in UTC: west of Greenwich a UTC-midnight month boundary would otherwise + // render as the previous month. Granularity follows the tick rather than the timespan, + // because ECharts sizes tick intervals by pixel density. + private formatAxisDate(value: number): string { + const date = new Date(value); + const isMonthStart = date.getUTCDate() === 1 && date.getUTCHours() === 0 && + date.getUTCMinutes() === 0 && date.getUTCSeconds() === 0; + return date.toLocaleDateString(this.locale, isMonthStart + ? { year: 'numeric', month: 'short', timeZone: 'UTC' } + : { month: 'short', day: 'numeric', timeZone: 'UTC' }); + } + + private formatTooltipDate(value: number): string { + return new Date(value).toLocaleDateString(this.locale, { + year: 'numeric', month: 'short', day: 'numeric', timeZone: 'UTC', + }); + } + + prepareChartOptions(): void { + const seriesData = this.data.map(d => [d.timestamp * 1000, d.minRate, d.minHeight]); + const hasData = seriesData.length > 0; + + this.chartOptions = { + color: [DEFAULT_BAR_COLOR], + animation: false, + // Buckets are UTC calendar days, so ticks must land on UTC boundaries rather than + // the viewer's local midnight. + useUTC: true, + grid: { + right: this.widget ? 10 : 25, + left: this.widget ? 45 : 70, + bottom: this.widget ? 30 : 50, + top: 20, + }, + tooltip: { + show: !this.isMobile(), + trigger: 'axis', + axisPointer: { + type: 'line' + }, + backgroundColor: 'rgba(17, 19, 31, 1)', + borderRadius: 4, + shadowColor: 'rgba(0, 0, 0, 0.5)', + textStyle: { + color: 'var(--tooltip-grey)', + align: 'left', + }, + borderColor: '#000', + formatter: function (data: any): string { + if (data.length <= 0) { + return ''; + } + let tooltip = `${this.formatTooltipDate(+data[0].data[0])}
`; + tooltip += `${data[0].marker} ` + $localize`Min fee rate` + `: ${this.formatFeeRate(data[0].data[1])} sats/vByte
`; + tooltip += `` + $localize`At block: ${data[0].data[2]}` + ``; + return tooltip; + }.bind(this) + }, + // A time axis, not a category axis: days with no data must render as proportional + // gaps instead of collapsing into their neighbours. + xAxis: !hasData ? undefined : { + type: 'time', + axisLine: { onZero: true }, + axisLabel: { + formatter: (val: number): string => this.formatAxisDate(val), + align: 'center', + fontSize: 11, + lineHeight: 12, + hideOverlap: true, + padding: [0, 5], + }, + }, + yAxis: !hasData ? undefined : { + position: 'left', + name: this.widget ? undefined : $localize`:@@mining.min-fee-rate.axis:sat/vB`, + nameLocation: 'middle', + nameRotate: 90, + nameGap: 48, + nameTextStyle: { + color: 'rgb(110, 112, 121)', + fontSize: 12, + }, + axisLabel: { + color: 'rgb(110, 112, 121)', + formatter: (val): string => this.formatFeeRate(val), + }, + splitLine: { + lineStyle: { + type: 'dotted', + color: 'var(--transparent-fg)', + opacity: 0.25, + } + }, + type: 'value', + }, + series: !hasData ? undefined : [{ + zlevel: 0, + name: 'Min fee rate', + data: seriesData, + type: 'bar', + large: true, + markLine: { + silent: true, + symbol: 'none', + lineStyle: { + color: 'var(--fg)', + type: 'dashed', + opacity: 1, + width: 2, + }, + data: [{ + yAxis: this.threshold, + label: { + show: true, + position: 'insideStartTop', + formatter: (): string => `${this.formatFeeRate(this.threshold)} sat/vB`, + color: 'var(--fg)', + fontSize: 11, + } + }], + } + }], + visualMap: !hasData ? undefined : { + show: false, + dimension: 1, + pieces: [ + { lte: this.threshold, color: HIGHLIGHT_COLOR }, + { gt: this.threshold, color: DEFAULT_BAR_COLOR }, + ], + }, + dataZoom: (this.widget || !hasData) ? undefined : [{ + type: 'inside', + realtime: true, + zoomLock: false, + maxSpan: 100, + minSpan: 5, + moveOnMouseMove: false, + }], + }; + } + + onChartInit(ec): void { + if (this.chartInstance !== undefined) { + return; + } + + this.chartInstance = ec; + + this.chartInstance.on('click', (e) => { + this.zone.run(() => { + const url = new RelativeUrlPipe(this.stateService).transform(`/block/${e.data[2]}`); + this.router.navigate([url]); + }); + }); + } + + isMobile(): boolean { + return (window.innerWidth <= 767.98); + } + + onSaveChart(): void { + // @ts-ignore + const prevBottom = this.chartOptions.grid.bottom; + const now = new Date(); + // @ts-ignore + this.chartOptions.grid.bottom = 40; + this.chartOptions.backgroundColor = 'var(--active-bg)'; + this.chartInstance.setOption(this.chartOptions); + download(this.chartInstance.getDataURL({ + pixelRatio: 2, + excludeComponents: ['dataZoom'], + }), `min-fee-rate-${this.timespan}-${Math.round(now.getTime() / 1000)}.svg`); + // @ts-ignore + this.chartOptions.grid.bottom = prevBottom; + this.chartOptions.backgroundColor = 'none'; + this.chartInstance.setOption(this.chartOptions); + } +} diff --git a/frontend/src/app/graphs/graphs.module.ts b/frontend/src/app/graphs/graphs.module.ts index 5994a1775..382e0ebfd 100644 --- a/frontend/src/app/graphs/graphs.module.ts +++ b/frontend/src/app/graphs/graphs.module.ts @@ -9,6 +9,8 @@ import { BlockFeesSubsidyGraphComponent } from '@components/block-fees-subsidy-g import { PriceChartComponent } from '@components/price-chart/price-chart.component'; import { BlockRewardsGraphComponent } from '@components/block-rewards-graph/block-rewards-graph.component'; import { BlockFeeRatesGraphComponent } from '@components/block-fee-rates-graph/block-fee-rates-graph.component'; +import { MinFeeRateGraphComponent } from '@components/min-fee-rate-graph/min-fee-rate-graph.component'; +import { MinFeeRateCdfGraphComponent } from '@components/min-fee-rate-cdf-graph/min-fee-rate-cdf-graph.component'; import { BlockSizesWeightsGraphComponent } from '@components/block-sizes-weights-graph/block-sizes-weights-graph.component'; import { FeeDistributionGraphComponent } from '@components/fee-distribution-graph/fee-distribution-graph.component'; import { IncomingTransactionsGraphComponent } from '@components/incoming-transactions-graph/incoming-transactions-graph.component'; @@ -70,6 +72,8 @@ import { CommonModule } from '@angular/common'; PriceChartComponent, BlockRewardsGraphComponent, BlockFeeRatesGraphComponent, + MinFeeRateGraphComponent, + MinFeeRateCdfGraphComponent, BlockSizesWeightsGraphComponent, FeeDistributionGraphComponent, IncomingTransactionsGraphComponent, diff --git a/frontend/src/app/graphs/graphs.routing.module.ts b/frontend/src/app/graphs/graphs.routing.module.ts index f0399f410..646275c05 100644 --- a/frontend/src/app/graphs/graphs.routing.module.ts +++ b/frontend/src/app/graphs/graphs.routing.module.ts @@ -2,6 +2,8 @@ import { NgModule } from '@angular/core'; import { RouterModule, Routes } from '@angular/router'; import { BlockHealthGraphComponent } from '@components/block-health-graph/block-health-graph.component'; import { BlockFeeRatesGraphComponent } from '@components/block-fee-rates-graph/block-fee-rates-graph.component'; +import { MinFeeRateGraphComponent } from '@components/min-fee-rate-graph/min-fee-rate-graph.component'; +import { MinFeeRateCdfGraphComponent } from '@components/min-fee-rate-cdf-graph/min-fee-rate-cdf-graph.component'; import { BlockFeesGraphComponent } from '@components/block-fees-graph/block-fees-graph.component'; import { BlockFeesSubsidyGraphComponent } from '@components/block-fees-subsidy-graph/block-fees-subsidy-graph.component'; import { BlockRewardsGraphComponent } from '@components/block-rewards-graph/block-rewards-graph.component'; @@ -149,6 +151,16 @@ const routes: Routes = [ data: { networks: ['bitcoin'] }, component: BlockFeeRatesGraphComponent, }, + { + path: 'mining/min-fee-rate', + data: { networks: ['bitcoin'] }, + component: MinFeeRateGraphComponent, + }, + { + path: 'mining/min-fee-rate-cdf', + data: { networks: ['bitcoin'] }, + component: MinFeeRateCdfGraphComponent, + }, { path: 'mining/block-sizes-weights', data: { networks: ['bitcoin'] }, diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 92450246a..91db051c3 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -528,4 +528,11 @@ export interface ChainTip { export interface StaleTip extends ChainTip { stale: BlockExtended; canonical: BlockExtended; -} \ No newline at end of file +} + +export interface MinFeeRateDay { + minRate: number; + minHeight: number; + timestamp: number; // unix seconds, UTC midnight + usableBlockCount: number; +} diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index d05b02c66..ca3af257c 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -393,6 +393,13 @@ export class ApiService { ); } + getMinFeeRates$(interval: string | undefined) : Observable { + return this.httpClient.get( + this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/blocks/min-fee-rate` + + (interval !== undefined ? `/${interval}` : ''), { observe: 'response' } + ); + } + getHistoricalBlockSizesAndWeights$(interval: string | undefined) : Observable> { return this.httpClient.get( this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/blocks/sizes-weights` + diff --git a/frontend/src/app/services/min-fee-rate.service.ts b/frontend/src/app/services/min-fee-rate.service.ts new file mode 100644 index 000000000..dc2d079eb --- /dev/null +++ b/frontend/src/app/services/min-fee-rate.service.ts @@ -0,0 +1,78 @@ +import { Injectable } from '@angular/core'; +import { HttpResponse } from '@angular/common/http'; +import { Observable } from 'rxjs'; +import { ApiService } from '@app/services/api.service'; +import { MinFeeRateDay } from '@app/interfaces/node-api.interface'; + +// Bitcoin Core 30.0 lowered the default -minrelaytxfee to 0.1 sat/vB, which is the +// reference threshold both charts open on. +export const DEFAULT_MIN_FEE_RATE_THRESHOLD = 0.1; + +export interface MinFeeRateStats { + totalDays: number; + daysBelow: number; + percentBelow: number; +} + +@Injectable({ providedIn: 'root' }) +export class MinFeeRateService { + constructor(private apiService: ApiService) {} + + getMinFeeRates$(interval: string | undefined): Observable> { + return this.apiService.getMinFeeRates$(interval); + } + + getStats(data: MinFeeRateDay[], threshold: number): MinFeeRateStats { + const totalDays = data.length; + if (totalDays === 0) { + return { totalDays: 0, daysBelow: 0, percentBelow: 0 }; + } + const daysBelow = data.filter(d => d.minRate <= threshold).length; + return { + totalDays, + daysBelow, + percentBelow: (daysBelow / totalDays) * 100, + }; + } + + // Cumulative share of days whose minRate is <= a given fee rate. Duplicate rates are + // collapsed to a single step so the staircase stays monotonic and clean. + buildCdf(data: MinFeeRateDay[]): number[][] { + if (data.length === 0) { + return []; + } + const counts = new Map(); + for (const d of data) { + counts.set(d.minRate, (counts.get(d.minRate) || 0) + 1); + } + const rates = Array.from(counts.keys()).sort((a, b) => a - b); + const cdf: number[][] = []; + let cumulative = 0; + for (const rate of rates) { + cumulative += counts.get(rate); + cdf.push([rate, (cumulative / data.length) * 100]); + } + return cdf; + } + + // Minimum daily fee rates are often sub-1 sat/vB, so round adaptively: more decimals + // below 1 to keep values distinguishable, fewer as they grow. + formatFeeRate(val: number): string { + if (val >= 100) { + return val.toFixed(0); + } + if (val >= 10) { + return val.toFixed(1); + } + if (val >= 0.1) { + return val.toFixed(2); + } + if (val >= 0.01) { + return val.toFixed(3); + } + if (val > 0) { + return val.toFixed(4); + } + return '0'; + } +}