replace staleTip cache with stale block cache

backend/src/api/chain-tips.ts

@@ -31,20 +31,22 @@ export interface OrphanedBlock {
 
 class ChainTips {
   private chainTips: ChainTip[] = [];
-  private staleTips: Record<number, StaleTip> = {};
+  private validChainTips: ChainTip[] = []; // 'valid-fork' and 'valid-headers' only, in descending height order
+  private staleBlocks: Record<string, BlockExtended> = {};
   private orphanedBlocks: { [hash: string]: OrphanedBlock } = {};
   private blockCache: { [hash: string]: OrphanedBlock } = {};
   private orphansByHeight: { [height: number]: OrphanedBlock[] } = {};
   private indexingOrphanedBlocks = false;
   private indexingQueue: { blockhash?: string, block?: IEsploraApi.Block, tip: OrphanedBlock }[] = [];
 
-  private staleTipsCacheSize = 50;
+  private staleBlocksCacheSize = 50;
   private maxIndexingQueueSize = 100;
 
   /** @asyncSafe */
   public async updateOrphanedBlocks(): Promise<void> {
     try {
       this.chainTips = await bitcoinClient.getChainTips();
+      this.validChainTips = this.chainTips.filter(tip => tip.status === 'valid-fork' || tip.status === 'valid-headers').sort((a, b) => b.height - a.height);
 
       const activeTipHeight = this.chainTips.find(tip => tip.status === 'active')?.height || (await bitcoinApi.$getBlockHeightTip());
       let minIndexHeight = 0;
@@ -122,13 +124,7 @@ class ChainTips {
         this.orphansByHeight[orphan.height].push(orphan);
       }
 
-      const heightsToKeep = new Set(this.chainTips.filter(tip => tip.status !== 'active').map(tip => tip.height));
-      const heightsToRemove: number[] = Object.keys(this.staleTips).map(Number).filter(height => !heightsToKeep.has(height));
-      for (const height of heightsToRemove) {
-        delete this.staleTips[height];
-      }
-
-      this.trimStaleTipsCache();
+      this.trimStaleBlocksCache();
 
       // index new orphaned blocks in the background
       void this.$indexOrphanedBlocks();
@@ -160,7 +156,7 @@ class ChainTips {
         }
         let staleBlock: BlockExtended | undefined;
         const alreadyIndexed = await BlocksSummariesRepository.$isSummaryIndexed(block.id);
-        const needToCache = Object.keys(this.staleTips).length < this.staleTipsCacheSize || block.height > Object.keys(this.staleTips).map(Number).sort((a, b) => b - a)[this.staleTipsCacheSize - 1];
+        const needToCache = this.shouldCacheStaleBlock(block.id, block.height);
         if (!alreadyIndexed) {
           staleBlock = await blocks.$indexBlock(block.id, block, true);
           await blocks.$indexBlockSummary(block.id, block.height, true);
@@ -171,16 +167,9 @@ class ChainTips {
         }
 
         if (staleBlock && needToCache) {
-          const canonicalBlock = await blocks.$indexBlockByHeight(staleBlock.height);
-          this.staleTips[staleBlock.height] = {
-            height: staleBlock.height,
-            hash: staleBlock.id,
-            branchlen: tip.branchlen,
-            status: tip.status,
-            stale: staleBlock,
-            canonical: canonicalBlock,
-          };
-          this.trimStaleTipsCache();
+          // ensure the canonical block is correctly indexed
+          await blocks.$indexBlockByHeight(staleBlock.height);
+          this.cacheStaleBlock(staleBlock);
         }
       } catch (e) {
         logger.err(`Failed to index orphaned block ${block?.id} at height ${block?.height}. Reason: ${e instanceof Error ? e.message : e}`);
@@ -189,12 +178,42 @@ class ChainTips {
     this.indexingOrphanedBlocks = false;
   }
 
-  private trimStaleTipsCache(): void {
-    const staleTipHeights = Object.keys(this.staleTips).map(Number).sort((a, b) => b - a);
-    if (staleTipHeights.length > this.staleTipsCacheSize) {
-      const heightsToDiscard = staleTipHeights.slice(this.staleTipsCacheSize);
-      for (const height of heightsToDiscard) {
-        delete this.staleTips[height];
+  private shouldCacheStaleBlock(hash: string, height: number): boolean {
+    // already cached
+    if (this.staleBlocks[hash]) {
+      return false;
+    }
+
+    // cache is not full
+    const cachedBlocks = Object.values(this.staleBlocks);
+    if (cachedBlocks.length < this.staleBlocksCacheSize) {
+      return true;
+    }
+
+    // otherwise cache if this block is newer than the oldest in the cache
+    const oldestCachedHeight = cachedBlocks.reduce((min, block) => Math.min(min, block.height), Infinity);
+    return height >= oldestCachedHeight;
+  }
+
+  private cacheStaleBlock(block: BlockExtended): void {
+    this.staleBlocks[block.id] = block;
+    this.trimStaleBlocksCache();
+  }
+
+  // evict the oldest stale blocks until the cache is within the size limit
+  private trimStaleBlocksCache(): void {
+    // sort by height
+    const cachedBlocks = Object.values(this.staleBlocks).sort((a, b) => {
+      if (b.height !== a.height) {
+        return b.height - a.height;
+      }
+      // tie-break by hash
+      return a.id.localeCompare(b.id);
+    });
+    // delete everything beyond the size limit
+    if (cachedBlocks.length > this.staleBlocksCacheSize) {
+      for (const block of cachedBlocks.slice(this.staleBlocksCacheSize)) {
+        delete this.staleBlocks[block.id];
       }
     }
   }
@@ -211,41 +230,27 @@ class ChainTips {
     return this.chainTips;
   }
 
-  public getStaleTips(): StaleTip[] {
-    return Object.values(this.staleTips).sort((a, b) => b.height - a.height);
-  }
-
-  /** @asyncUnsafe */
+  /** @asyncSafe */
   public async $getStaleTipsPage(fromHeight: number | undefined, limit: number): Promise<StaleTip[]> {
-    const cacheTips = this.getStaleTips();
-
-    const allStaleTips = this.chainTips.filter(chainTip => chainTip.status === 'valid-fork' || chainTip.status === 'valid-headers').sort((a, b) => b.height - a.height);
-    const staleTipsPage = fromHeight === undefined ? allStaleTips.slice(0, limit) : allStaleTips.filter(tip => tip.height < fromHeight).slice(0, limit);
-
-    const cacheTipsMap = new Map<string, StaleTip>();
-    for (const cacheTip of cacheTips) {
-      cacheTipsMap.set(cacheTip.hash, cacheTip);
-    }
+    const start = fromHeight === undefined ? 0 : this.validChainTips.findIndex(tip => tip.height < fromHeight);
+    const staleTipsPage = start === -1 ? [] : this.validChainTips.slice(start, start + limit);
 
+    // hydrate tips with block data
     const tips: StaleTip[] = [];
     for (const staleTip of staleTipsPage) {
-      const cachedTip = cacheTipsMap.get(staleTip.hash);
-      if (cachedTip) {
-        tips.push(cachedTip);
-        continue;
+      // fetch blocks from caches if available, or DB otherwise
+      const canonical = blocks.getBlocks().find(block => block.height === staleTip.height) || await BlocksRepository.$getBlockByHeight(staleTip.height);
+      let stale: BlockExtended | null | undefined = this.staleBlocks[staleTip.hash];
+      if (!stale) {
+        stale = await BlocksRepository.$getBlockByHash(staleTip.hash);
       }
-
-      const canonical = await BlocksRepository.$getBlockByHeight(staleTip.height);
-      const stale = await BlocksRepository.$getBlockByHash(staleTip.hash);
+      // skip tips with missing block data
       if (!canonical || !stale) {
         continue;
       }
 
       tips.push({
-        height: staleTip.height,
-        hash: staleTip.hash,
-        branchlen: staleTip.branchlen,
-        status: staleTip.status,
+        ...staleTip,
         stale,
         canonical,
       });
@@ -276,4 +281,4 @@ class ChainTips {
   }
 }
 
-export default new ChainTips();
+export default new ChainTips();

backend/src/repositories/BlocksRepository.ts

@@ -577,6 +577,7 @@ class BlocksRepository {
 
   /**
    * Get one block by hash
+   * @asyncSafe
    */
   public async $getBlockByHash(hash: string): Promise<BlockExtended | null> {
     try {