More misc improvements, prepping for 2.0:

- major memory usage improvements for loading txs with inputs: appropriate input-vouts are now extracted from their source txs and summarized (asm/hex are dropped since they're never used), rather than keeping the whole of input-txs in memory; this dramatically improves memory usage (and avoids crashing due to OOMEs), particularly when loading txs with "heavy" inputs (i.e. many, large input txs); this is relevant for /tx/TXID pages, and particularly important for the new /block-analysis pages which necessarily load many, many txs
- cache-key prefixes for coreApi's caching functionality: to avoid loading old-format data from a persistent cache and barfing on it
- several important code-reuse fixes: there were almost-duplicate chunks of code that were essentially calls to getRawTransactionWithInputs - now just use that function
- new minor /admin page for showing memory usage
- on homepage Network Summary: show full tx count (non abbreviated)
- use new domain everywhere

CHANGELOG.md

@@ -39,6 +39,8 @@
 	* **IMPORTANT**: Use of `/block-analysis` can put heavy memory pressure on this app, depending on the details of the block being analyzed. If your app is crashing, consider setting a higher memory ceiling: `node --max_old_space_size=XXX bin/www` (where `XXX` is measured in MB).
 * Change `/mempool-summary` to load data via ajax (UX improvement to give feedback while loading large data sets)
 * Zero-indexing for tx index-in-block values
+* Reduced memory usage
+* Versioning for cache keys if using persistent cache (redis)
 * Configurable UI "sub-header" links
 * Start of RPC API versioning support
 * Tweaked styling across site
@@ -46,6 +48,7 @@
 	* Remove "Bitcoin Explorer" H1 (it's redundant)
 	* Hide the "Date" (timestamp) column for recent blocks (the Age+TTM is more valuable)
 * Updated miner configs
+* Lots of minor bug fixes
 
 #### v1.1.9
 ##### 2020-02-23

README.md

@@ -9,13 +9,13 @@ This tool is intended to be a simple, self-hosted explorer for the Bitcoin block
 
 Whatever reasons one might have for running a full node (trustlessness, technical curiosity, supporting the network, etc) it's helpful to appreciate the "fullness" of your node. With this explorer, you can not only explore the blockchain (in the traditional sense of the term "explorer"), but also explore the functional capabilities of your own node.
 
-Live demo available at: [https://btc-explorer.com](https://btc-explorer.com)
+Live demo available at: [https://explorer.btc21.org](https://explorer.btc21.org)
 
 # Features
 
-* Browse blocks
-* View block details
-* View transaction details, with navigation "backward" via spent transaction outputs
+* Network Summary "dashboard"
+* View details of blocks, transactions, and addresses
+* Analysis tools for viewing stats on blocks, transactions, and miner activity
 * View JSON content used to generate most pages
 * Search by transaction ID, block hash/height, and address
 * Optional transaction history for addresses by querying from ElectrumX, blockchain.com, blockchair.com, or blockcypher.com
@@ -71,7 +71,7 @@ See `btc-rpc-explorer --help` for the full list of CLI options.
 
 # Support
 
-* [https://donation.btc21.org](https://donate.btc21.org/apps/2TBP2GuQnYXGBiHQkmf4jNuMh6eN/pos)
+* [https://donate.btc21.org](https://donate.btc21.org/apps/2TBP2GuQnYXGBiHQkmf4jNuMh6eN/pos)
 
 
 [npm-ver-img]: https://img.shields.io/npm/v/btc-rpc-explorer.svg?style=flat

app/api/coreApi.js

@@ -16,6 +16,10 @@ var rpcApi = require("./rpcApi.js");
 //var rpcApi = require("./mockApi.js");
 
 
+// this value should be incremented whenever data format changes, to avoid
+// pulling old-format data from a persistent cache
+var cacheKeyVersion = "v0";
+
 function onCacheEvent(cacheType, hitOrMiss, cacheKey) {
 	//debugLog(`cache.${cacheType}.${hitOrMiss}: ${cacheKey}`);
 }
@@ -84,7 +88,10 @@ function getGenesisCoinbaseTransactionId() {
 
 
 function tryCacheThenRpcApi(cache, cacheKey, cacheMaxAge, rpcApiFunction, cacheConditionFunction) {
-	//debugLog("tryCache: " + cacheKey + ", " + cacheMaxAge);
+	var versionedCacheKey = `${cacheKeyVersion}-${cacheKey}`;
+
+	//debugLog("tryCache: " + versionedCacheKey + ", " + cacheMaxAge);
+	
 	if (cacheConditionFunction == null) {
 		cacheConditionFunction = function(obj) {
 			return true;
@@ -101,7 +108,7 @@ function tryCacheThenRpcApi(cache, cacheKey, cacheMaxAge, rpcApiFunction, cacheC
 			} else {
 				rpcApiFunction().then(function(rpcResult) {
 					if (rpcResult != null && cacheConditionFunction(rpcResult)) {
-						cache.set(cacheKey, rpcResult, cacheMaxAge);
+						cache.set(versionedCacheKey, rpcResult, cacheMaxAge);
 					}
 
 					resolve(rpcResult);
@@ -112,13 +119,13 @@ function tryCacheThenRpcApi(cache, cacheKey, cacheMaxAge, rpcApiFunction, cacheC
 			}
 		};
 
-		cache.get(cacheKey).then(function(result) {
+		cache.get(versionedCacheKey).then(function(result) {
 			cacheResult = result;
 
 			finallyFunc();
 			
 		}).catch(function(err) {
-			utils.logError("nds9fc2eg621tf3", err, {cacheKey:cacheKey});
+			utils.logError("nds9fc2eg621tf3", err, {cacheKey:versionedCacheKey});
 
 			finallyFunc();
 		});
@@ -387,49 +394,8 @@ function getMempoolDetails(start, count) {
 				txids.push(resultTxids[i]);
 			}
 
-			getRawTransactions(txids).then(function(transactions) {
-				var maxInputsTracked = config.site.txMaxInput;
-				var vinTxids = [];
-				for (var i = 0; i < transactions.length; i++) {
-					var transaction = transactions[i];
-
-					if (transaction && transaction.vin) {
-						for (var j = 0; j < Math.min(maxInputsTracked, transaction.vin.length); j++) {
-							if (transaction.vin[j].txid) {
-								vinTxids.push(transaction.vin[j].txid);
-							}
-						}
-					}
-				}
-
-				var txInputsByTransaction = {};
-				getRawTransactions(vinTxids).then(function(vinTransactions) {
-					var vinTxById = {};
-
-					vinTransactions.forEach(function(tx) {
-						vinTxById[tx.txid] = tx;
-					});
-
-					transactions.forEach(function(tx) {
-						txInputsByTransaction[tx.txid] = {};
-
-						if (tx && tx.vin) {
-							for (var i = 0; i < Math.min(maxInputsTracked, tx.vin.length); i++) {
-								if (vinTxById[tx.vin[i].txid]) {
-									txInputsByTransaction[tx.txid][i] = vinTxById[tx.vin[i].txid];
-								}
-							}
-						}
-					});
-
-					resolve({ txCount:resultTxids.length, transactions:transactions, txInputsByTransaction:txInputsByTransaction });
-
-				}).catch(function(err) {
-					reject(err);
-				});
-
-			}).catch(function(err) {
-				reject(err);
+			getRawTransactionsWithInputs(txids, config.site.txMaxInput).then(function(result) {
+				resolve({ txCount:resultTxids.length, transactions:result.transactions, txInputsByTransaction:result.txInputsByTransaction });
 			});
 
 		}).catch(function(err) {
@@ -696,6 +662,37 @@ function getRawTransaction(txid) {
 	return tryCacheThenRpcApi(txCache, "getRawTransaction-" + txid, 3600000, rpcApiFunction, shouldCacheTransaction);
 }
 
+/*
+	This function pulls raw tx data and then summarizes the outputs. It's used in memory-constrained situations.
+*/
+function getSummarizedTransactionOutput(txid, voutIndex) {
+	var rpcApiFunction = function() {
+		return new Promise(function(resolve, reject) {
+			rpcApi.getRawTransaction(txid).then(function(rawTx) {
+				var vout = rawTx.vout[voutIndex];
+				if (vout.scriptPubKey) {
+					if (vout.scriptPubKey.asm) {
+						delete vout.scriptPubKey.asm;
+					}
+
+					if (vout.scriptPubKey.hex) {
+						delete vout.scriptPubKey.hex;
+					}
+				}
+
+				vout.txid = txid;
+
+				resolve(vout);
+
+			}).catch(function(err) {
+				reject(err);
+			});
+		});
+	};
+
+	return tryCacheThenRpcApi(txCache, `txoSummary-${txid}-${voutIndex}`, 3600000, rpcApiFunction, shouldCacheTransaction);
+}
+
 function getTxUtxos(tx) {
 	return new Promise(function(resolve, reject) {
 		var promises = [];
@@ -813,7 +810,7 @@ function summarizeBlockAnalysisData(blockHeight, tx, inputs) {
 	} else {
 		for (var i = 0; i < tx.vin.length; i++) {
 			var vin = tx.vin[i];
-			var inputVout = inputs[i].vout[vin.vout];
+			var inputVout = inputs[i];
 
 			txSummary.totalInput = txSummary.totalInput.plus(new Decimal(inputVout.value));
 
@@ -866,34 +863,45 @@ function getRawTransactionsWithInputs(txids, maxInputs=-1) {
 				maxInputsTracked = maxInputs;
 			}
 
-			var vinTxids = [];
+			var vinIds = [];
 			for (var i = 0; i < transactions.length; i++) {
 				var transaction = transactions[i];
 
 				if (transaction && transaction.vin) {
 					for (var j = 0; j < Math.min(maxInputsTracked, transaction.vin.length); j++) {
 						if (transaction.vin[j].txid) {
-							vinTxids.push(transaction.vin[j].txid);
+							vinIds.push({txid:transaction.vin[j].txid, voutIndex:transaction.vin[j].vout});
 						}
 					}
 				}
 			}
 
-			var txInputsByTransaction = {};
-			getRawTransactions(vinTxids).then(function(vinTransactions) {
-				var vinTxById = {};
+			var promises = [];
 
-				vinTransactions.forEach(function(tx) {
-					vinTxById[tx.txid] = tx;
-				});
+			for (var i = 0; i < vinIds.length; i++) {
+				var vinId = vinIds[i];
+
+				promises.push(getSummarizedTransactionOutput(vinId.txid, vinId.voutIndex));
+			}
+
+			Promise.all(promises).then(function(promiseResults) {
+				var summarizedTxOutputs = {};
+				for (var i = 0; i < promiseResults.length; i++) {
+					var summarizedTxOutput = promiseResults[i];
+
+					summarizedTxOutputs[`${summarizedTxOutput.txid}:${summarizedTxOutput.n}`] = summarizedTxOutput;
+				}
+
+				var txInputsByTransaction = {};
 
 				transactions.forEach(function(tx) {
 					txInputsByTransaction[tx.txid] = {};
 
 					if (tx && tx.vin) {
 						for (var i = 0; i < Math.min(maxInputsTracked, tx.vin.length); i++) {
-							if (vinTxById[tx.vin[i].txid]) {
-								txInputsByTransaction[tx.txid][i] = vinTxById[tx.vin[i].txid];
+							var summarizedTxOutput = summarizedTxOutputs[`${tx.vin[i].txid}:${tx.vin[i].vout}`];
+							if (summarizedTxOutput) {
+								txInputsByTransaction[tx.txid][i] = summarizedTxOutput;
 							}
 						}
 					}
@@ -918,54 +926,19 @@ function getBlockByHashWithTransactions(blockHash, txLimit, txOffset) {
 				txids.push(block.tx[i]);
 			}
 
-			getRawTransactions(txids).then(function(transactions) {
-				if (transactions.length == txids.length) {
-					block.coinbaseTx = transactions[0];
+			getRawTransactionsWithInputs(txids, config.site.txMaxInput).then(function(txsResult) {
+				if (txsResult.transactions && txsResult.transactions.length > 0) {
+					block.coinbaseTx = txsResult.transactions[0];
 					block.totalFees = utils.getBlockTotalFeesFromCoinbaseTxAndBlockHeight(block.coinbaseTx, block.height);
 					block.miner = utils.getMinerFromCoinbaseTx(block.coinbaseTx);
 				}
 
-				// if we're on page 2, we don't really want it anymore...
+				// if we're on page 2, we don't really want the coinbase tx in the tx list anymore
 				if (txOffset > 0) {
 					transactions.shift();
 				}
 
-				var maxInputsTracked = config.site.txMaxInput;
-				var vinTxids = [];
-				for (var i = 0; i < transactions.length; i++) {
-					var transaction = transactions[i];
-
-					if (transaction && transaction.vin) {
-						for (var j = 0; j < Math.min(maxInputsTracked, transaction.vin.length); j++) {
-							if (transaction.vin[j].txid) {
-								vinTxids.push(transaction.vin[j].txid);
-							}
-						}
-					}
-				}
-
-				var txInputsByTransaction = {};
-				getRawTransactions(vinTxids).then(function(vinTransactions) {
-					var vinTxById = {};
-
-					vinTransactions.forEach(function(tx) {
-						vinTxById[tx.txid] = tx;
-					});
-
-					transactions.forEach(function(tx) {
-						txInputsByTransaction[tx.txid] = {};
-
-						if (tx && tx.vin) {
-							for (var i = 0; i < Math.min(maxInputsTracked, tx.vin.length); i++) {
-								if (vinTxById[tx.vin[i].txid]) {
-									txInputsByTransaction[tx.txid][i] = vinTxById[tx.vin[i].txid];
-								}
-							}
-						}
-
-						resolve({ getblock:block, transactions:transactions, txInputsByTransaction:txInputsByTransaction });
-					});
-				});
+				resolve({ getblock:block, transactions:txsResult.transactions, txInputsByTransaction:txsResult.txInputsByTransaction });
 			});
 		});
 	});

app/utils.js

@@ -411,7 +411,7 @@ function getTxTotalInputOutputValues(tx, txInputs, blockHeight) {
 
 				if (txInput) {
 					try {
-						var vout = txInput.vout[tx.vin[i].vout];
+						var vout = txInput;
 						if (vout.value) {
 							totalInputValue = totalInputValue.plus(new Decimal(vout.value));
 						}

docs/Server-Setup.md

@@ -1,4 +1,4 @@
-### Setup of https://btc-explorer.com on Ubuntu 16.04
+### Setup of https://explorer.btc21.org on Ubuntu 16.04
 
     apt update
     apt upgrade
@@ -11,9 +11,9 @@
     apt upgrade
     apt install python-certbot-nginx
     
-Copy content from [./btc-explorer.com.conf](./btc-explorer.com.conf) into `/etc/nginx/sites-available/btc-explorer.com.conf`
+Copy content from [./explorer.btc21.org.conf](./explorer.btc21.org.conf) into `/etc/nginx/sites-available/explorer.btc21.org.conf`
 
-    certbot --nginx -d btc-explorer.com
+    certbot --nginx -d explorer.btc21.org
     cd /etc/ssl/certs
     openssl dhparam -out dhparam.pem 4096
     cd /home/bitcoin

docs/btc-explorer.com.conf

@@ -1,17 +1,17 @@
 ## http://domain.com redirects to https://domain.com
 server {
-	server_name btc-explorer.com;
+	server_name explorer.btc21.org;
 	listen 80;
 	#listen [::]:80 ipv6only=on;
 
         location / {
-                return 301 https://btc-explorer.com$request_uri;
+                return 301 https://explorer.btc21.org$request_uri;
         }
 }
 
 ## Serves httpS://domain.com
 server {
-	server_name btc-explorer.com;
+	server_name explorer.btc21.org;
 	listen 443 ssl http2;
 	#listen [::]:443 ssl http2 ipv6only=on;
 	

routes/baseActionsRouter.js

@@ -22,6 +22,8 @@ var coreApi = require("./../app/api/coreApi.js");
 var addressApi = require("./../app/api/addressApi.js");
 var rpcApi = require("./../app/api/rpcApi.js");
 
+const v8 = require('v8');
+
 const forceCsrf = csurf({ ignoreMethods: [] });
 
 router.get("/", function(req, res, next) {
@@ -792,13 +794,16 @@ router.get("/tx/:transactionId", function(req, res, next) {
 
 	res.locals.result = {};
 
-	coreApi.getRawTransaction(txid).then(function(rawTxResult) {
-		res.locals.result.getrawtransaction = rawTxResult;
+	coreApi.getRawTransactionsWithInputs([txid]).then(function(rawTxResult) {
+		var tx = rawTxResult.transactions[0];
+
+		res.locals.result.getrawtransaction = tx;
+		res.locals.result.txInputs = rawTxResult.txInputsByTransaction[txid];
 
 		var promises = [];
 
 		promises.push(new Promise(function(resolve, reject) {
-			coreApi.getTxUtxos(rawTxResult).then(function(utxos) {
+			coreApi.getTxUtxos(tx).then(function(utxos) {
 				res.locals.utxos = utxos;
 				
 				resolve();
@@ -810,7 +815,7 @@ router.get("/tx/:transactionId", function(req, res, next) {
 			});
 		}));
 
-		if (rawTxResult.confirmations == null) {
+		if (tx.confirmations == null) {
 			promises.push(new Promise(function(resolve, reject) {
 				coreApi.getMempoolTxDetails(txid, true).then(function(mempoolDetails) {
 					res.locals.mempoolDetails = mempoolDetails;
@@ -826,40 +831,23 @@ router.get("/tx/:transactionId", function(req, res, next) {
 		}
 
 		promises.push(new Promise(function(resolve, reject) {
-			global.rpcClient.command('getblock', rawTxResult.blockhash, function(err3, result3, resHeaders3) {
+			global.rpcClient.command('getblock', tx.blockhash, function(err3, result3, resHeaders3) {
 				res.locals.result.getblock = result3;
 
-				var txids = [];
-				for (var i = 0; i < rawTxResult.vin.length; i++) {
-					if (!rawTxResult.vin[i].coinbase) {
-						txids.push(rawTxResult.vin[i].txid);
-					}
-				}
-
-				coreApi.getRawTransactions(txids).then(function(txInputs) {
-					res.locals.result.txInputs = txInputs;
-
-					resolve();
-				});
+				resolve();
 			});
 		}));
 
 		Promise.all(promises).then(function() {
 			res.render("transaction");
 
-			next();
-
-		}).catch(function(err) {
-			res.locals.pageErrors.push(utils.logError("1237y4ewssgt", err));
-
-			res.render("transaction");
-
 			next();
 		});
 
-		
 	}).catch(function(err) {
 		res.locals.userMessage = "Failed to load transaction with txid=" + txid + ": " + err;
+		
+		res.locals.pageErrors.push(utils.logError("1237y4ewssgt", err));
 
 		res.render("transaction");
 
@@ -1053,12 +1041,12 @@ router.get("/address/:address", function(req, res, next) {
 											var vinJ = tx.vin[j];
 
 											if (txInput != null) {
-												if (txInput.vout[vinJ.vout] && txInput.vout[vinJ.vout].scriptPubKey && txInput.vout[vinJ.vout].scriptPubKey.addresses && txInput.vout[vinJ.vout].scriptPubKey.addresses.includes(address)) {
+												if (txInput && txInput.scriptPubKey && txInput.scriptPubKey.addresses && txInput.scriptPubKey.addresses.includes(address)) {
 													if (addrLossesByTx[tx.txid] == null) {
 														addrLossesByTx[tx.txid] = new Decimal(0);
 													}
 
-													addrLossesByTx[tx.txid] = addrLossesByTx[tx.txid].plus(new Decimal(txInput.vout[vinJ.vout].value));
+													addrLossesByTx[tx.txid] = addrLossesByTx[tx.txid].plus(new Decimal(txInput.value));
 												}
 											}
 										}
@@ -1448,6 +1436,14 @@ router.get("/tools", function(req, res, next) {
 	next();
 });
 
+router.get("/admin", function(req, res, next) {
+	res.locals.memstats = v8.getHeapStatistics();
+
+	res.render("admin");
+
+	next();
+});
+
 router.get("/changelog", function(req, res, next) {
 	res.locals.changelogHtml = marked(global.changelogMarkdown);
 

views/admin.pug

@@ -0,0 +1,11 @@
+extends layout
+
+block headContent
+	title Admin
+
+block content
+	h1.h3 Admin
+	hr
+
+	pre
+		code.json #{JSON.stringify(memstats, null, 4)}

views/includes/index-network-summary.pug

@@ -89,8 +89,7 @@ div.row.index-summary
 						span Total Txs
 					td.text-right.text-monospace
 						if (txStats && txStats.totalTxCount)
-							- var totalTxData = utils.formatLargeNumber(txStats.totalTxCount, 2);
-							span.border-dotted(title=`${txStats.totalTxCount.toLocaleString()}`, data-toggle="tooltip") #{totalTxData[0]} #{totalTxData[1].abbreviation}
+							span #{txStats.totalTxCount.toLocaleString()}
 						else
 							span ???
 

views/includes/transaction-io-details.pug

@@ -22,8 +22,7 @@ div.row.text-monospace
 					- var vout = null;
 					if (txInputs && txInputs[txVinIndex])
 						- var txInput = txInputs[txVinIndex];
-						if (txInput.vout && txInput.vout[txVin.vout])
-							- var vout = txInput.vout[txVin.vout];
+						- var vout = txInput;
 
 				if (txVin.coinbase || vout)
 					div.clearfix

views/transaction.pug

@@ -30,7 +30,7 @@ block content
 			- totalInputValue = totalInputValue.plus(new Decimal(coinConfig.blockRewardFunction(result.getblock.height, global.activeBlockchain)));
 		each txInput, txInputIndex in result.txInputs
 			if (txInput)
-				- var vout = txInput.vout[result.getrawtransaction.vin[txInputIndex].vout];
+				- var vout = txInput;
 				if (vout.value)
 					- totalInputValue = totalInputValue.plus(new Decimal(vout.value));