mirror of
https://github.com/mempool/mempool.git
synced 2026-08-13 12:33:11 +02:00
377 lines
12 KiB
JavaScript
377 lines
12 KiB
JavaScript
const https = require('https');
|
|
const fsSync = require('fs');
|
|
const crypto = require('crypto');
|
|
const path = require('node:path');
|
|
|
|
// Configuration
|
|
const LOG_TAG = '[sync-assets]';
|
|
const CONFIG_FILE_NAME = 'mempool-frontend-config.json';
|
|
|
|
const config = {
|
|
verbose: parseInt(process.env.VERBOSE) === 1,
|
|
mempoolCDN: parseInt(process.env.MEMPOOL_CDN) === 1,
|
|
dryRun: parseInt(process.env.DRY_RUN) === 1,
|
|
githubToken: process.env.GITHUB_TOKEN,
|
|
};
|
|
|
|
// Early exit if SKIP_SYNC is set
|
|
if (parseInt(process.env.SKIP_SYNC) === 1) {
|
|
console.log(`${LOG_TAG} SKIP_SYNC is set, not checking any assets`);
|
|
process.exit(0);
|
|
}
|
|
|
|
// Log configuration
|
|
if (config.verbose) console.log(`${LOG_TAG} VERBOSE is set, logs will be more verbose`);
|
|
if (config.mempoolCDN) console.log(`${LOG_TAG} MEMPOOL_CDN is set, assets will be downloaded from mempool.space`);
|
|
if (config.dryRun) console.log(`${LOG_TAG} DRY_RUN is set, not downloading any assets`);
|
|
|
|
// Setup assets path
|
|
const ASSETS_PATH = (() => {
|
|
if (!process.argv[2]) {
|
|
throw new Error('Resource path argument is not set');
|
|
}
|
|
const rawPath = process.argv[2].endsWith("/") ? process.argv[2] : `${process.argv[2]}/`;
|
|
const normalizedPath = path.resolve(path.normalize(rawPath));
|
|
console.log(`${LOG_TAG} using ASSETS_PATH ${normalizedPath}`);
|
|
|
|
if (!fsSync.existsSync(normalizedPath)) {
|
|
console.log(`${LOG_TAG} ${normalizedPath} does not exist, creating`);
|
|
fsSync.mkdirSync(normalizedPath, { recursive: true });
|
|
}
|
|
|
|
return normalizedPath;
|
|
})();
|
|
|
|
// Load frontend config
|
|
const loadConfig = () => {
|
|
try {
|
|
const rawConfig = fsSync.readFileSync(CONFIG_FILE_NAME, 'utf8');
|
|
console.log(`${LOG_TAG} ${CONFIG_FILE_NAME} file found, using provided config`);
|
|
return JSON.parse(rawConfig);
|
|
} catch (e) {
|
|
if (e.code !== 'ENOENT') throw e;
|
|
console.log(`${LOG_TAG} ${CONFIG_FILE_NAME} file not found, using default config`);
|
|
return {};
|
|
}
|
|
};
|
|
|
|
const configContent = loadConfig();
|
|
|
|
// Utility: Make HTTPS request
|
|
const httpsRequest = (options) => {
|
|
return new Promise((resolve, reject) => {
|
|
https.get(options, (response) => {
|
|
const chunks = [];
|
|
|
|
response.on('data', (chunk) => chunks.push(chunk));
|
|
response.on('end', () => resolve(Buffer.concat(chunks)));
|
|
response.on('error', reject);
|
|
}).on('error', reject);
|
|
});
|
|
};
|
|
|
|
// Utility: Download file
|
|
const downloadFile = (filePath, url) => {
|
|
if (!filePath || !url) {
|
|
if (config.verbose) {
|
|
console.log('skipping malformed download request: ', filePath, url);
|
|
}
|
|
return Promise.resolve();
|
|
}
|
|
|
|
return new Promise((resolve, reject) => {
|
|
https.get(url, { headers: { 'user-agent': 'node.js' } }, (response) => {
|
|
if (response.statusCode < 200 || response.statusCode > 299) {
|
|
reject(new Error(`HTTP Error ${response.statusCode} while fetching '${url}' (destination '${filePath}')`));
|
|
return;
|
|
}
|
|
|
|
const writeStream = fsSync.createWriteStream(filePath);
|
|
response.pipe(writeStream);
|
|
|
|
writeStream.on('finish', () => {
|
|
if (config.verbose) {
|
|
console.log(`${LOG_TAG} \tFinished downloading ${url} to ${filePath}`);
|
|
}
|
|
resolve();
|
|
});
|
|
|
|
writeStream.on('error', reject);
|
|
}).on('error', reject);
|
|
});
|
|
};
|
|
|
|
// Utility: Get local file hash (Git blob format)
|
|
const getLocalHash = (filePath) => {
|
|
const stats = fsSync.statSync(filePath);
|
|
const buffer = fsSync.readFileSync(filePath);
|
|
const bufferWithHeader = Buffer.concat([
|
|
Buffer.from('blob '),
|
|
Buffer.from(`${stats.size}`),
|
|
Buffer.from('\0'),
|
|
buffer
|
|
]);
|
|
const hash = crypto.createHash('sha1').update(bufferWithHeader).digest('hex');
|
|
|
|
if (config.verbose) {
|
|
console.log(`${LOG_TAG} \t\tgetLocalHash ${filePath} ${hash}`);
|
|
}
|
|
|
|
return hash;
|
|
};
|
|
|
|
// Utility: Create GitHub API options
|
|
const createGitHubOptions = (repoPath) => {
|
|
const options = {
|
|
host: 'api.github.com',
|
|
path: repoPath,
|
|
method: 'GET',
|
|
headers: { 'user-agent': 'node.js' }
|
|
};
|
|
|
|
if (config.githubToken) {
|
|
options.headers['authorization'] = `Bearer ${config.githubToken}`;
|
|
options.headers['X-GitHub-Api-Version'] = '2022-11-28';
|
|
}
|
|
|
|
return options;
|
|
};
|
|
|
|
// Utility: Replace URL for CDN
|
|
const getCDNUrl = (url, replacePattern) => {
|
|
return config.mempoolCDN ? url.replace(replacePattern.from, replacePattern.to) : url;
|
|
};
|
|
|
|
// Utility: Ensure directory exists
|
|
const ensureDirectory = (dirPath) => {
|
|
if (!fsSync.existsSync(dirPath)) {
|
|
fsSync.mkdirSync(dirPath, { recursive: true });
|
|
}
|
|
};
|
|
|
|
// Core: Process file item (handles checking and downloading)
|
|
const processFileItem = async (item, options) => {
|
|
const {
|
|
filePath,
|
|
remoteHash,
|
|
downloadUrl,
|
|
cdnPattern,
|
|
itemName,
|
|
downloadDir
|
|
} = options;
|
|
|
|
const fileExists = fsSync.existsSync(filePath);
|
|
|
|
if (fileExists) {
|
|
const localHash = getLocalHash(filePath);
|
|
|
|
if (config.verbose) {
|
|
console.log(`${LOG_TAG} \t\tremote ${itemName} hash ${remoteHash}`);
|
|
}
|
|
|
|
if (localHash !== remoteHash) {
|
|
console.log(`${LOG_TAG} \t\t${itemName} is different on the remote, downloading...`);
|
|
|
|
if (config.dryRun) {
|
|
console.log(`${LOG_TAG} \t\tDRY_RUN is set, not downloading ${itemName} but we should`);
|
|
return false;
|
|
}
|
|
|
|
const url = getCDNUrl(downloadUrl, cdnPattern);
|
|
if (config.verbose) {
|
|
console.log(`${LOG_TAG} \t\tDownloading ${url} to ${filePath}`);
|
|
}
|
|
await downloadFile(filePath, url);
|
|
return true;
|
|
} else {
|
|
console.log(`${LOG_TAG} \t\t${itemName} is already up to date. Skipping.`);
|
|
return false;
|
|
}
|
|
} else {
|
|
console.log(`${LOG_TAG} \t\t${itemName} is missing, downloading...`);
|
|
ensureDirectory(downloadDir);
|
|
|
|
if (config.dryRun) {
|
|
console.log(`${LOG_TAG} \t\tDRY_RUN is set, not downloading ${itemName} but we should`);
|
|
return false;
|
|
}
|
|
|
|
const url = getCDNUrl(downloadUrl, cdnPattern);
|
|
if (config.verbose) {
|
|
console.log(`${LOG_TAG} \t\tDownloading ${url} to ${filePath}`);
|
|
}
|
|
await downloadFile(filePath, url);
|
|
return true;
|
|
}
|
|
};
|
|
|
|
// Core: Fetch GitHub directory contents
|
|
const fetchGitHubContents = async (repoPath, useAuth = false) => {
|
|
if (useAuth && config.githubToken) {
|
|
console.log(`${LOG_TAG} \tDownloading with authentication`);
|
|
}
|
|
|
|
const options = createGitHubOptions(repoPath);
|
|
const responseBody = await httpsRequest(options);
|
|
const contents = JSON.parse(responseBody.toString());
|
|
|
|
if (contents.message) {
|
|
throw new Error(contents.message);
|
|
}
|
|
|
|
return contents;
|
|
};
|
|
|
|
// Main: Download mining pool logos
|
|
const downloadMiningPoolLogos = async () => {
|
|
console.log(`${LOG_TAG} \tChecking if mining pool logos needs downloading or updating...`);
|
|
|
|
try {
|
|
const poolLogos = await fetchGitHubContents('/repos/mempool/mining-pool-logos/contents/', !!config.githubToken);
|
|
|
|
let downloadedCount = 0;
|
|
const validFiles = poolLogos.filter(item => item.type === 'file' && item.download_url && item.name.toLowerCase().endsWith('.svg'));
|
|
|
|
for (const poolLogo of validFiles) {
|
|
if (config.verbose) {
|
|
console.log(`${LOG_TAG} Processing ${poolLogo.name}`);
|
|
}
|
|
|
|
const downloaded = await processFileItem(poolLogo, {
|
|
filePath: `${ASSETS_PATH}/mining-pools/${poolLogo.name}`,
|
|
remoteHash: poolLogo.sha,
|
|
downloadUrl: poolLogo.download_url,
|
|
cdnPattern: {
|
|
from: "raw.githubusercontent.com/mempool/mining-pool-logos/master",
|
|
to: "mempool.space/resources/mining-pools"
|
|
},
|
|
itemName: poolLogo.name,
|
|
downloadDir: `${ASSETS_PATH}/mining-pools/`
|
|
});
|
|
|
|
if (downloaded) downloadedCount++;
|
|
}
|
|
|
|
console.log(`${LOG_TAG} \t\tDownloaded ${downloadedCount} and skipped ${validFiles.length - downloadedCount} existing mining pool logos`);
|
|
} catch (e) {
|
|
throw new Error(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
|
|
}
|
|
};
|
|
|
|
// Main: Download promo video subtitles
|
|
const downloadPromoVideoSubtitles = async () => {
|
|
console.log(`${LOG_TAG} \tChecking if promo video subtitles needs downloading or updating...`);
|
|
|
|
try {
|
|
const subtitles = await fetchGitHubContents('/repos/mempool/mempool-promo/contents/subtitles', !!config.githubToken);
|
|
|
|
let downloadedCount = 0;
|
|
const validFiles = subtitles.filter(item => item.type === 'file' && item.download_url);
|
|
|
|
for (const subtitle of validFiles) {
|
|
if (config.verbose) {
|
|
console.log(`${LOG_TAG} Processing ${subtitle.name}`);
|
|
}
|
|
|
|
const downloaded = await processFileItem(subtitle, {
|
|
filePath: `${ASSETS_PATH}/promo-video/${subtitle.name}`,
|
|
remoteHash: subtitle.sha,
|
|
downloadUrl: subtitle.download_url,
|
|
cdnPattern: {
|
|
from: "raw.githubusercontent.com/mempool/mempool-promo/master/subtitles",
|
|
to: "mempool.space/resources/promo-video"
|
|
},
|
|
itemName: subtitle.name,
|
|
downloadDir: `${ASSETS_PATH}/promo-video/`
|
|
});
|
|
|
|
if (downloaded) downloadedCount++;
|
|
}
|
|
|
|
console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${validFiles.length - downloadedCount} existing video subtitles`);
|
|
} catch (e) {
|
|
throw new Error(`Unable to download video subtitles. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
|
|
}
|
|
};
|
|
|
|
// Main: Download promo video
|
|
const downloadPromoVideo = async () => {
|
|
console.log(`${LOG_TAG} \tChecking if promo video needs downloading or updating...`);
|
|
|
|
try {
|
|
const contents = await fetchGitHubContents('/repos/mempool/mempool-promo/contents', !!config.githubToken);
|
|
|
|
const videoItem = contents.find(item => item.name === 'promo.mp4');
|
|
if (!videoItem) {
|
|
console.log(`${LOG_TAG} \tpromo.mp4 not found in repository`);
|
|
return;
|
|
}
|
|
|
|
await processFileItem(videoItem, {
|
|
filePath: `${ASSETS_PATH}/promo-video/mempool-promo.mp4`,
|
|
remoteHash: videoItem.sha,
|
|
downloadUrl: videoItem.download_url,
|
|
cdnPattern: {
|
|
from: "raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4",
|
|
to: "mempool.space/resources/promo-video/mempool-promo.mp4"
|
|
},
|
|
itemName: 'mempool-promo.mp4',
|
|
downloadDir: `${ASSETS_PATH}/promo-video/`
|
|
});
|
|
} catch (e) {
|
|
throw new Error(`Unable to download video. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
|
|
}
|
|
};
|
|
|
|
// Download Liquid assets if configured
|
|
const downloadLiquidAssets = () => {
|
|
if (configContent.BASE_MODULE !== 'liquid') {
|
|
if (config.verbose) {
|
|
console.log(`${LOG_TAG} BASE_MODULE is not set to Liquid (currently ${configContent.BASE_MODULE}), skipping downloading assets`);
|
|
}
|
|
return;
|
|
}
|
|
|
|
const liquidAssets = [
|
|
{ file: 'assets.json', url: 'https://raw.githubusercontent.com/Blockstream/asset_registry_db/master/index.json' },
|
|
{ file: 'assets.minimal.json', url: 'https://raw.githubusercontent.com/Blockstream/asset_registry_db/master/index.minimal.json' },
|
|
{ file: 'assets-testnet.json', url: 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json' },
|
|
{ file: 'assets-testnet.minimal.json', url: 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json' }
|
|
];
|
|
|
|
console.log(`${LOG_TAG} Downloading assets`);
|
|
liquidAssets.forEach(({ file, url }) => {
|
|
const fileName = file.replace(/^assets/, 'assets');
|
|
console.log(`${LOG_TAG} Downloading ${fileName}`);
|
|
downloadFile(`${ASSETS_PATH}/${fileName}`, url);
|
|
});
|
|
};
|
|
|
|
// Main execution
|
|
(async () => {
|
|
try {
|
|
// Download Liquid assets (non-blocking)
|
|
downloadLiquidAssets();
|
|
|
|
// Download GitHub assets sequentially
|
|
if (config.verbose) {
|
|
console.log(`${LOG_TAG} Downloading mining pool logos`);
|
|
}
|
|
await downloadMiningPoolLogos();
|
|
|
|
if (config.verbose) {
|
|
console.log(`${LOG_TAG} Downloading promo video subtitles`);
|
|
}
|
|
await downloadPromoVideoSubtitles();
|
|
|
|
if (config.verbose) {
|
|
console.log(`${LOG_TAG} Downloading promo video`);
|
|
}
|
|
await downloadPromoVideo();
|
|
|
|
console.log(`${LOG_TAG} Asset synchronization complete`);
|
|
} catch (error) {
|
|
console.error(`${LOG_TAG} Error:`, error.message);
|
|
process.exit(1);
|
|
}
|
|
})();
|