mirror of
https://github.com/mempool/mempool.git
synced 2026-08-13 12:33:11 +02:00
Refactor the sync assets script
This commit is contained in:
Felipe Knorr Kuhn
parent
f2491795af
commit
32af03b44b
1 changed files with 325 additions and 372 deletions
|
|
@ -1,425 +1,378 @@
|
|||
var https = require('https');
|
||||
var fs = require('fs');
|
||||
var crypto = require('crypto');
|
||||
var path = require('node:path');
|
||||
const LOG_TAG = '[sync-assets]';
|
||||
let verbose = false;
|
||||
let MEMPOOL_CDN = false;
|
||||
let DRY_RUN = false;
|
||||
const https = require('https');
|
||||
const fs = require('fs').promises;
|
||||
const fsSync = require('fs');
|
||||
const crypto = require('crypto');
|
||||
const path = require('node:path');
|
||||
|
||||
// Configuration
|
||||
const LOG_TAG = '[sync-assets]';
|
||||
const CONFIG_FILE_NAME = 'mempool-frontend-config.json';
|
||||
|
||||
const config = {
|
||||
verbose: parseInt(process.env.VERBOSE) === 1,
|
||||
mempoolCDN: parseInt(process.env.MEMPOOL_CDN) === 1,
|
||||
dryRun: parseInt(process.env.DRY_RUN) === 1,
|
||||
githubToken: process.env.GITHUB_TOKEN,
|
||||
};
|
||||
|
||||
// Early exit if SKIP_SYNC is set
|
||||
if (parseInt(process.env.SKIP_SYNC) === 1) {
|
||||
console.log(`${LOG_TAG} SKIP_SYNC is set, not checking any assets`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (parseInt(process.env.VERBOSE) === 1) {
|
||||
console.log(`${LOG_TAG} VERBOSE is set, logs will be more verbose`);
|
||||
verbose = true;
|
||||
}
|
||||
// Log configuration
|
||||
if (config.verbose) console.log(`${LOG_TAG} VERBOSE is set, logs will be more verbose`);
|
||||
if (config.mempoolCDN) console.log(`${LOG_TAG} MEMPOOL_CDN is set, assets will be downloaded from mempool.space`);
|
||||
if (config.dryRun) console.log(`${LOG_TAG} DRY_RUN is set, not downloading any assets`);
|
||||
|
||||
if (parseInt(process.env.MEMPOOL_CDN) === 1) {
|
||||
console.log(`${LOG_TAG} MEMPOOL_CDN is set, assets will be downloaded from mempool.space`);
|
||||
MEMPOOL_CDN = true;
|
||||
}
|
||||
|
||||
if (parseInt(process.env.DRY_RUN) === 1) {
|
||||
console.log(`${LOG_TAG} DRY_RUN is set, not downloading any assets`);
|
||||
DRY_RUN = true;
|
||||
}
|
||||
|
||||
const githubSecret = process.env.GITHUB_TOKEN;
|
||||
|
||||
const CONFIG_FILE_NAME = 'mempool-frontend-config.json';
|
||||
let configContent = {};
|
||||
|
||||
var ASSETS_PATH;
|
||||
if (process.argv[2]) {
|
||||
ASSETS_PATH = process.argv[2];
|
||||
ASSETS_PATH += ASSETS_PATH.endsWith("/") ? "" : "/"
|
||||
ASSETS_PATH = path.resolve(path.normalize(ASSETS_PATH));
|
||||
console.log(`[sync-assets] using ASSETS_PATH ${ASSETS_PATH}`);
|
||||
if (!fs.existsSync(ASSETS_PATH)){
|
||||
console.log(`${LOG_TAG} ${ASSETS_PATH} does not exist, creating`);
|
||||
fs.mkdirSync(ASSETS_PATH, { recursive: true });
|
||||
// Setup assets path
|
||||
const ASSETS_PATH = (() => {
|
||||
if (!process.argv[2]) {
|
||||
throw new Error('Resource path argument is not set');
|
||||
}
|
||||
}
|
||||
const rawPath = process.argv[2].endsWith("/") ? process.argv[2] : `${process.argv[2]}/`;
|
||||
const normalizedPath = path.resolve(path.normalize(rawPath));
|
||||
console.log(`${LOG_TAG} using ASSETS_PATH ${normalizedPath}`);
|
||||
|
||||
if (!ASSETS_PATH) {
|
||||
throw new Error('Resource path argument is not set');
|
||||
}
|
||||
if (!fsSync.existsSync(normalizedPath)) {
|
||||
console.log(`${LOG_TAG} ${normalizedPath} does not exist, creating`);
|
||||
fsSync.mkdirSync(normalizedPath, { recursive: true });
|
||||
}
|
||||
|
||||
try {
|
||||
const rawConfig = fs.readFileSync(CONFIG_FILE_NAME);
|
||||
configContent = JSON.parse(rawConfig);
|
||||
console.log(`${LOG_TAG} ${CONFIG_FILE_NAME} file found, using provided config`);
|
||||
} catch (e) {
|
||||
if (e.code !== 'ENOENT') {
|
||||
throw new Error(e);
|
||||
} else {
|
||||
return normalizedPath;
|
||||
})();
|
||||
|
||||
// Load frontend config
|
||||
const loadConfig = () => {
|
||||
try {
|
||||
const rawConfig = fsSync.readFileSync(CONFIG_FILE_NAME, 'utf8');
|
||||
console.log(`${LOG_TAG} ${CONFIG_FILE_NAME} file found, using provided config`);
|
||||
return JSON.parse(rawConfig);
|
||||
} catch (e) {
|
||||
if (e.code !== 'ENOENT') throw e;
|
||||
console.log(`${LOG_TAG} ${CONFIG_FILE_NAME} file not found, using default config`);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
function download(filename, url) {
|
||||
if (!filename || !url) {
|
||||
if (verbose) {
|
||||
console.log('skipping malformed download request: ', filename, url);
|
||||
}
|
||||
return;
|
||||
}
|
||||
https.get(url, (response) => {
|
||||
if (response.statusCode < 200 || response.statusCode > 299) {
|
||||
throw new Error('HTTP Error ' + response.statusCode + ' while fetching \'' + filename + '\'');
|
||||
}
|
||||
response.pipe(fs.createWriteStream(filename));
|
||||
})
|
||||
.on('error', function(e) {
|
||||
throw new Error(e);
|
||||
})
|
||||
.on('finish', () => {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} \tFinished downloading ${url} to ${filename}`);
|
||||
}
|
||||
const configContent = loadConfig();
|
||||
|
||||
// Utility: Make HTTPS request
|
||||
const httpsRequest = (options) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
https.get(options, (response) => {
|
||||
const chunks = [];
|
||||
|
||||
response.on('data', (chunk) => chunks.push(chunk));
|
||||
response.on('end', () => resolve(Buffer.concat(chunks)));
|
||||
response.on('error', reject);
|
||||
}).on('error', reject);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
function getLocalHash(filePath) {
|
||||
const size = fs.statSync(filePath);
|
||||
const buffer = fs.readFileSync(filePath);
|
||||
const bufferWithHeader = Buffer.concat([Buffer.from('blob '), Buffer.from(`${size.size}`), Buffer.from('\0'), buffer]);
|
||||
// Utility: Download file
|
||||
const downloadFile = (filePath, url) => {
|
||||
if (!filePath || !url) {
|
||||
if (config.verbose) {
|
||||
console.log('skipping malformed download request: ', filePath, url);
|
||||
}
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
https.get(url, (response) => {
|
||||
if (response.statusCode < 200 || response.statusCode > 299) {
|
||||
reject(new Error(`HTTP Error ${response.statusCode} while fetching '${filePath}'`));
|
||||
return;
|
||||
}
|
||||
|
||||
const writeStream = fsSync.createWriteStream(filePath);
|
||||
response.pipe(writeStream);
|
||||
|
||||
writeStream.on('finish', () => {
|
||||
if (config.verbose) {
|
||||
console.log(`${LOG_TAG} \tFinished downloading ${url} to ${filePath}`);
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
|
||||
writeStream.on('error', reject);
|
||||
}).on('error', reject);
|
||||
});
|
||||
};
|
||||
|
||||
// Utility: Get local file hash (Git blob format)
|
||||
const getLocalHash = (filePath) => {
|
||||
const stats = fsSync.statSync(filePath);
|
||||
const buffer = fsSync.readFileSync(filePath);
|
||||
const bufferWithHeader = Buffer.concat([
|
||||
Buffer.from('blob '),
|
||||
Buffer.from(`${stats.size}`),
|
||||
Buffer.from('\0'),
|
||||
buffer
|
||||
]);
|
||||
const hash = crypto.createHash('sha1').update(bufferWithHeader).digest('hex');
|
||||
|
||||
if (verbose) {
|
||||
if (config.verbose) {
|
||||
console.log(`${LOG_TAG} \t\tgetLocalHash ${filePath} ${hash}`);
|
||||
}
|
||||
|
||||
return hash;
|
||||
}
|
||||
};
|
||||
|
||||
function downloadMiningPoolLogos$() {
|
||||
return new Promise((resolve, reject) => {
|
||||
console.log(`${LOG_TAG} \tChecking if mining pool logos needs downloading or updating...`);
|
||||
const options = {
|
||||
host: 'api.github.com',
|
||||
path: '/repos/mempool/mining-pool-logos/contents/',
|
||||
method: 'GET',
|
||||
headers: {'user-agent': 'node.js'}
|
||||
};
|
||||
// Utility: Create GitHub API options
|
||||
const createGitHubOptions = (repoPath) => {
|
||||
const options = {
|
||||
host: 'api.github.com',
|
||||
path: repoPath,
|
||||
method: 'GET',
|
||||
headers: { 'user-agent': 'node.js' }
|
||||
};
|
||||
|
||||
if (githubSecret) {
|
||||
console.log(`${LOG_TAG} Downloading the mining pool logos with authentication`);
|
||||
options.headers['authorization'] = `Bearer ${githubSecret}`;
|
||||
options.headers['X-GitHub-Api-Version'] = '2022-11-28';
|
||||
if (config.githubToken) {
|
||||
options.headers['authorization'] = `Bearer ${config.githubToken}`;
|
||||
options.headers['X-GitHub-Api-Version'] = '2022-11-28';
|
||||
}
|
||||
|
||||
return options;
|
||||
};
|
||||
|
||||
// Utility: Replace URL for CDN
|
||||
const getCDNUrl = (url, replacePattern) => {
|
||||
return config.mempoolCDN ? url.replace(replacePattern.from, replacePattern.to) : url;
|
||||
};
|
||||
|
||||
// Utility: Ensure directory exists
|
||||
const ensureDirectory = (dirPath) => {
|
||||
if (!fsSync.existsSync(dirPath)) {
|
||||
fsSync.mkdirSync(dirPath, { recursive: true });
|
||||
}
|
||||
};
|
||||
|
||||
// Core: Process file item (handles checking and downloading)
|
||||
const processFileItem = async (item, options) => {
|
||||
const {
|
||||
filePath,
|
||||
remoteHash,
|
||||
downloadUrl,
|
||||
cdnPattern,
|
||||
itemName,
|
||||
downloadDir
|
||||
} = options;
|
||||
|
||||
const fileExists = fsSync.existsSync(filePath);
|
||||
|
||||
if (fileExists) {
|
||||
const localHash = getLocalHash(filePath);
|
||||
|
||||
if (config.verbose) {
|
||||
console.log(`${LOG_TAG} \t\tremote ${itemName} hash ${remoteHash}`);
|
||||
}
|
||||
|
||||
https.get(options, (response) => {
|
||||
const chunks_of_data = [];
|
||||
if (localHash !== remoteHash) {
|
||||
console.log(`${LOG_TAG} \t\t${itemName} is different on the remote, downloading...`);
|
||||
|
||||
response.on('data', (fragments) => {
|
||||
chunks_of_data.push(fragments);
|
||||
});
|
||||
if (config.dryRun) {
|
||||
console.log(`${LOG_TAG} \t\tDRY_RUN is set, not downloading ${itemName} but we should`);
|
||||
return false;
|
||||
}
|
||||
|
||||
response.on('end', () => {
|
||||
const response_body = Buffer.concat(chunks_of_data);
|
||||
try {
|
||||
const poolLogos = JSON.parse(response_body.toString());
|
||||
if (poolLogos.message) {
|
||||
reject(poolLogos.message);
|
||||
}
|
||||
let downloadedCount = 0;
|
||||
for (const poolLogo of poolLogos) {
|
||||
if (poolLogo.type !== 'file' || poolLogo.download_url == null) {
|
||||
continue;
|
||||
}
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} Processing ${poolLogo.name}`);
|
||||
}
|
||||
console.log(`${ASSETS_PATH}/mining-pools/${poolLogo.name}`);
|
||||
const filePath = `${ASSETS_PATH}/mining-pools/${poolLogo.name}`;
|
||||
if (fs.existsSync(filePath)) {
|
||||
const localHash = getLocalHash(filePath);
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} \t\tremote ${poolLogo.name} logo hash ${poolLogo.sha}`);
|
||||
console.log(`${LOG_TAG} \t\t\tchecking if ${filePath} exists: ${fs.existsSync(filePath)}`);
|
||||
}
|
||||
if (localHash !== poolLogo.sha) {
|
||||
console.log(`${LOG_TAG} \t\t\t\t${poolLogo.name} is different on the remote, downloading...`);
|
||||
let download_url = poolLogo.download_url;
|
||||
if (MEMPOOL_CDN) {
|
||||
download_url = download_url.replace("raw.githubusercontent.com/mempool/mining-pool-logos/master", "mempool.space/resources/mining-pools");
|
||||
}
|
||||
if (DRY_RUN) {
|
||||
console.log(`${LOG_TAG} \t\tDRY_RUN is set, not downloading ${poolLogo.name} but we should`);
|
||||
} else {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} \t\tDownloading ${download_url} to ${filePath}`);
|
||||
}
|
||||
download(filePath, download_url);
|
||||
downloadedCount++;
|
||||
}
|
||||
} else {
|
||||
console.log(`${LOG_TAG} \t\t${poolLogo.name} is already up to date. Skipping.`);
|
||||
}
|
||||
} else {
|
||||
console.log(`${LOG_TAG} \t\t${poolLogo.name} is missing, downloading...`);
|
||||
const miningPoolsDir = `${ASSETS_PATH}/mining-pools/`;
|
||||
if (!fs.existsSync(miningPoolsDir)){
|
||||
fs.mkdirSync(miningPoolsDir, { recursive: true });
|
||||
}
|
||||
let download_url = poolLogo.download_url;
|
||||
if (MEMPOOL_CDN) {
|
||||
download_url = download_url.replace("raw.githubusercontent.com/mempool/mining-pool-logos/master", "mempool.space/resources/mining-pools");
|
||||
}
|
||||
if (DRY_RUN) {
|
||||
console.log(`${LOG_TAG} DRY_RUN is set, not downloading ${poolLogo.name} but it should`);
|
||||
} else {
|
||||
console.log(`${LOG_TAG} \tDownloading ${download_url} to ${filePath}`);
|
||||
download(filePath, download_url);
|
||||
downloadedCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(`${LOG_TAG} \t\tDownloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`);
|
||||
resolve();
|
||||
} catch (e) {
|
||||
reject(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
});
|
||||
const url = getCDNUrl(downloadUrl, cdnPattern);
|
||||
if (config.verbose) {
|
||||
console.log(`${LOG_TAG} \t\tDownloading ${url} to ${filePath}`);
|
||||
}
|
||||
await downloadFile(filePath, url);
|
||||
return true;
|
||||
} else {
|
||||
console.log(`${LOG_TAG} \t\t${itemName} is already up to date. Skipping.`);
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
console.log(`${LOG_TAG} \t\t${itemName} is missing, downloading...`);
|
||||
ensureDirectory(downloadDir);
|
||||
|
||||
response.on('error', (error) => {
|
||||
reject(error);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function downloadPromoVideoSubtiles$() {
|
||||
return new Promise((resolve, reject) => {
|
||||
console.log(`${LOG_TAG} \tChecking if promo video subtitles needs downloading or updating...`);
|
||||
const options = {
|
||||
host: 'api.github.com',
|
||||
path: '/repos/mempool/mempool-promo/contents/subtitles',
|
||||
method: 'GET',
|
||||
headers: {'user-agent': 'node.js'}
|
||||
};
|
||||
|
||||
if (githubSecret) {
|
||||
console.log(`${LOG_TAG} \tDownloading the promo video subtitles with authentication`);
|
||||
options.headers['authorization'] = `Bearer ${githubSecret}`;
|
||||
options.headers['X-GitHub-Api-Version'] = '2022-11-28';
|
||||
if (config.dryRun) {
|
||||
console.log(`${LOG_TAG} \t\tDRY_RUN is set, not downloading ${itemName} but we should`);
|
||||
return false;
|
||||
}
|
||||
|
||||
const url = getCDNUrl(downloadUrl, cdnPattern);
|
||||
if (config.verbose) {
|
||||
console.log(`${LOG_TAG} \t\tDownloading ${url} to ${filePath}`);
|
||||
}
|
||||
await downloadFile(filePath, url);
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
https.get(options, (response) => {
|
||||
const chunks_of_data = [];
|
||||
// Core: Fetch GitHub directory contents
|
||||
const fetchGitHubContents = async (repoPath, useAuth = false) => {
|
||||
if (useAuth && config.githubToken) {
|
||||
console.log(`${LOG_TAG} \tDownloading with authentication`);
|
||||
}
|
||||
|
||||
response.on('data', (fragments) => {
|
||||
chunks_of_data.push(fragments);
|
||||
const options = createGitHubOptions(repoPath);
|
||||
const responseBody = await httpsRequest(options);
|
||||
const contents = JSON.parse(responseBody.toString());
|
||||
|
||||
if (contents.message) {
|
||||
throw new Error(contents.message);
|
||||
}
|
||||
|
||||
return contents;
|
||||
};
|
||||
|
||||
// Main: Download mining pool logos
|
||||
const downloadMiningPoolLogos = async () => {
|
||||
console.log(`${LOG_TAG} \tChecking if mining pool logos needs downloading or updating...`);
|
||||
|
||||
try {
|
||||
const poolLogos = await fetchGitHubContents('/repos/mempool/mining-pool-logos/contents/', !!config.githubToken);
|
||||
|
||||
let downloadedCount = 0;
|
||||
const validFiles = poolLogos.filter(item => item.type === 'file' && item.download_url);
|
||||
|
||||
for (const poolLogo of validFiles) {
|
||||
if (config.verbose) {
|
||||
console.log(`${LOG_TAG} Processing ${poolLogo.name}`);
|
||||
}
|
||||
|
||||
const downloaded = await processFileItem(poolLogo, {
|
||||
filePath: `${ASSETS_PATH}/mining-pools/${poolLogo.name}`,
|
||||
remoteHash: poolLogo.sha,
|
||||
downloadUrl: poolLogo.download_url,
|
||||
cdnPattern: {
|
||||
from: "raw.githubusercontent.com/mempool/mining-pool-logos/master",
|
||||
to: "mempool.space/resources/mining-pools"
|
||||
},
|
||||
itemName: poolLogo.name,
|
||||
downloadDir: `${ASSETS_PATH}/mining-pools/`
|
||||
});
|
||||
|
||||
response.on('end', () => {
|
||||
const response_body = Buffer.concat(chunks_of_data);
|
||||
try {
|
||||
const videoLanguages = JSON.parse(response_body.toString());
|
||||
if (videoLanguages.message) {
|
||||
reject(videoLanguages.message);
|
||||
}
|
||||
let downloadedCount = 0;
|
||||
for (const language of videoLanguages) {
|
||||
if (language.type !== 'file' || language.download_url == null) {
|
||||
continue;
|
||||
}
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} Processing ${language.name}`);
|
||||
}
|
||||
const filePath = `${ASSETS_PATH}/promo-video/${language.name}`;
|
||||
if (fs.existsSync(filePath)) {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} \t${language.name} remote promo video hash ${language.sha}`);
|
||||
}
|
||||
const localHash = getLocalHash(filePath);
|
||||
if (localHash !== language.sha) {
|
||||
console.log(`${LOG_TAG} \t\t${language.name} is different on the remote, updating`);
|
||||
let download_url = language.download_url;
|
||||
if (MEMPOOL_CDN) {
|
||||
download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/subtitles", "mempool.space/resources/promo-video");
|
||||
}
|
||||
if (DRY_RUN) {
|
||||
console.log(`${LOG_TAG} \t\tDRY_RUN is set, not downloading ${language.name} but we should`);
|
||||
} else {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} \t\tdownloading ${download_url} to ${filePath}`);
|
||||
}
|
||||
download(filePath, download_url);
|
||||
downloadedCount++;
|
||||
}
|
||||
} else {
|
||||
console.log(`${LOG_TAG} \t\t${language.name} is already up to date. Skipping.`);
|
||||
}
|
||||
} else {
|
||||
console.log(`${LOG_TAG} \t\t${language.name} is missing, downloading`);
|
||||
const promoVideosDir = `${ASSETS_PATH}/promo-video/`;
|
||||
if (!fs.existsSync(promoVideosDir)){
|
||||
fs.mkdirSync(promoVideosDir, { recursive: true });
|
||||
}
|
||||
|
||||
let download_url = language.download_url;
|
||||
if (MEMPOOL_CDN) {
|
||||
download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/subtitles", "mempool.space/resources/promo-video");
|
||||
}
|
||||
if (DRY_RUN) {
|
||||
console.log(`${LOG_TAG} \tDRY_RUN is set, not downloading ${language.name} but we should`);
|
||||
} else {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`);
|
||||
}
|
||||
download(filePath, download_url);
|
||||
downloadedCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`);
|
||||
resolve();
|
||||
} catch (e) {
|
||||
reject(`Unable to download video subtitles. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
});
|
||||
|
||||
response.on('error', (error) => {
|
||||
reject(error);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function downloadPromoVideo$() {
|
||||
return new Promise((resolve, reject) => {
|
||||
console.log(`${LOG_TAG} \tChecking if promo video needs downloading or updating...`);
|
||||
const options = {
|
||||
host: 'api.github.com',
|
||||
path: '/repos/mempool/mempool-promo/contents',
|
||||
method: 'GET',
|
||||
headers: {'user-agent': 'node.js'}
|
||||
};
|
||||
|
||||
if (githubSecret) {
|
||||
console.log(`${LOG_TAG} \tDownloading the promo video with authentication`);
|
||||
options.headers['authorization'] = `Bearer ${githubSecret}`;
|
||||
options.headers['X-GitHub-Api-Version'] = '2022-11-28';
|
||||
if (downloaded) downloadedCount++;
|
||||
}
|
||||
|
||||
https.get(options, (response) => {
|
||||
const chunks_of_data = [];
|
||||
console.log(`${LOG_TAG} \t\tDownloaded ${downloadedCount} and skipped ${validFiles.length - downloadedCount} existing mining pool logos`);
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
};
|
||||
|
||||
response.on('data', (fragments) => {
|
||||
chunks_of_data.push(fragments);
|
||||
// Main: Download promo video subtitles
|
||||
const downloadPromoVideoSubtitles = async () => {
|
||||
console.log(`${LOG_TAG} \tChecking if promo video subtitles needs downloading or updating...`);
|
||||
|
||||
try {
|
||||
const subtitles = await fetchGitHubContents('/repos/mempool/mempool-promo/contents/subtitles', !!config.githubToken);
|
||||
|
||||
let downloadedCount = 0;
|
||||
const validFiles = subtitles.filter(item => item.type === 'file' && item.download_url);
|
||||
|
||||
for (const subtitle of validFiles) {
|
||||
if (config.verbose) {
|
||||
console.log(`${LOG_TAG} Processing ${subtitle.name}`);
|
||||
}
|
||||
|
||||
const downloaded = await processFileItem(subtitle, {
|
||||
filePath: `${ASSETS_PATH}/promo-video/${subtitle.name}`,
|
||||
remoteHash: subtitle.sha,
|
||||
downloadUrl: subtitle.download_url,
|
||||
cdnPattern: {
|
||||
from: "raw.githubusercontent.com/mempool/mempool-promo/master/subtitles",
|
||||
to: "mempool.space/resources/promo-video"
|
||||
},
|
||||
itemName: subtitle.name,
|
||||
downloadDir: `${ASSETS_PATH}/promo-video/`
|
||||
});
|
||||
|
||||
response.on('end', () => {
|
||||
const response_body = Buffer.concat(chunks_of_data);
|
||||
try {
|
||||
const contents = JSON.parse(response_body.toString());
|
||||
if (contents.message) {
|
||||
reject(contents.message);
|
||||
}
|
||||
for (const item of contents) {
|
||||
if (item.name !== 'promo.mp4') {
|
||||
continue;
|
||||
}
|
||||
const filePath = `${ASSETS_PATH}/promo-video/mempool-promo.mp4`;
|
||||
if (fs.existsSync(filePath)) {
|
||||
const localHash = getLocalHash(filePath);
|
||||
if (downloaded) downloadedCount++;
|
||||
}
|
||||
|
||||
if (localHash !== item.sha) {
|
||||
console.log(`${LOG_TAG} \tmempool-promo.mp4 is different on the remote, updating`);
|
||||
let download_url = item.download_url;
|
||||
if (MEMPOOL_CDN) {
|
||||
download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4", "mempool.space/resources/promo-video/mempool-promo.mp4");
|
||||
}
|
||||
if (DRY_RUN) {
|
||||
console.log(`${LOG_TAG} DRY_RUN is set, not downloading mempool-promo.mp4 but we should`);
|
||||
} else {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`);
|
||||
}
|
||||
download(filePath, download_url);
|
||||
console.log(`${LOG_TAG} \tmempool-promo.mp4 downloaded.`);
|
||||
}
|
||||
} else {
|
||||
console.log(`${LOG_TAG} \t\tmempool-promo.mp4 is already up to date. Skipping.`);
|
||||
}
|
||||
} else {
|
||||
console.log(`${LOG_TAG} \tmempool-promo.mp4 is missing, downloading`);
|
||||
let download_url = item.download_url;
|
||||
if (MEMPOOL_CDN) {
|
||||
download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4", "mempool.space/resources/promo-video/mempool-promo.mp4");
|
||||
}
|
||||
if (DRY_RUN) {
|
||||
console.log(`${LOG_TAG} DRY_RUN is set, not downloading mempool-promo.mp4 but we should`);
|
||||
} else {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`);
|
||||
}
|
||||
download(filePath, download_url);
|
||||
}
|
||||
}
|
||||
}
|
||||
resolve();
|
||||
} catch (e) {
|
||||
reject(`Unable to download video. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
});
|
||||
console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${validFiles.length - downloadedCount} existing video subtitles`);
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to download video subtitles. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
};
|
||||
|
||||
response.on('error', (error) => {
|
||||
reject(error);
|
||||
});
|
||||
// Main: Download promo video
|
||||
const downloadPromoVideo = async () => {
|
||||
console.log(`${LOG_TAG} \tChecking if promo video needs downloading or updating...`);
|
||||
|
||||
try {
|
||||
const contents = await fetchGitHubContents('/repos/mempool/mempool-promo/contents', !!config.githubToken);
|
||||
|
||||
const videoItem = contents.find(item => item.name === 'promo.mp4');
|
||||
if (!videoItem) {
|
||||
console.log(`${LOG_TAG} \tpromo.mp4 not found in repository`);
|
||||
return;
|
||||
}
|
||||
|
||||
await processFileItem(videoItem, {
|
||||
filePath: `${ASSETS_PATH}/promo-video/mempool-promo.mp4`,
|
||||
remoteHash: videoItem.sha,
|
||||
downloadUrl: videoItem.download_url,
|
||||
cdnPattern: {
|
||||
from: "raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4",
|
||||
to: "mempool.space/resources/promo-video/mempool-promo.mp4"
|
||||
},
|
||||
itemName: 'mempool-promo.mp4',
|
||||
downloadDir: `${ASSETS_PATH}/promo-video/`
|
||||
});
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to download video. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
// Download Liquid assets if configured
|
||||
const downloadLiquidAssets = () => {
|
||||
if (configContent.BASE_MODULE !== 'liquid') {
|
||||
if (config.verbose) {
|
||||
console.log(`${LOG_TAG} BASE_MODULE is not set to Liquid (currently ${configContent.BASE_MODULE}), skipping downloading assets`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
if (configContent.BASE_MODULE && configContent.BASE_MODULE === 'liquid') {
|
||||
const assetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_db/master/index.json';
|
||||
const assetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_db/master/index.minimal.json';
|
||||
const testnetAssetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json';
|
||||
const testnetAssetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json';
|
||||
const liquidAssets = [
|
||||
{ file: 'assets.json', url: 'https://raw.githubusercontent.com/Blockstream/asset_registry_db/master/index.json' },
|
||||
{ file: 'assets.minimal.json', url: 'https://raw.githubusercontent.com/Blockstream/asset_registry_db/master/index.minimal.json' },
|
||||
{ file: 'assets-testnet.json', url: 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json' },
|
||||
{ file: 'assets-testnet.minimal.json', url: 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json' }
|
||||
];
|
||||
|
||||
console.log(`${LOG_TAG} Downloading assets`);
|
||||
download(`${ASSETS_PATH}/assets.json`, assetsJsonUrl);
|
||||
liquidAssets.forEach(({ file, url }) => {
|
||||
const fileName = file.replace(/^assets/, 'assets');
|
||||
console.log(`${LOG_TAG} Downloading ${fileName}`);
|
||||
downloadFile(`${ASSETS_PATH}/${fileName}`, url);
|
||||
});
|
||||
};
|
||||
|
||||
console.log(`${LOG_TAG} Downloading assets minimal`);
|
||||
download(`${ASSETS_PATH}/assets.minimal.json`, assetsMinimalJsonUrl);
|
||||
// Main execution
|
||||
(async () => {
|
||||
try {
|
||||
// Download Liquid assets (non-blocking)
|
||||
downloadLiquidAssets();
|
||||
|
||||
console.log(`${LOG_TAG} Downloading testnet assets`);
|
||||
download(`${ASSETS_PATH}/assets-testnet.json`, testnetAssetsJsonUrl);
|
||||
// Download GitHub assets sequentially
|
||||
if (config.verbose) {
|
||||
console.log(`${LOG_TAG} Downloading mining pool logos`);
|
||||
}
|
||||
await downloadMiningPoolLogos();
|
||||
|
||||
console.log(`${LOG_TAG} Downloading testnet assets minimal`);
|
||||
download(`${ASSETS_PATH}/assets-testnet.minimal.json`, testnetAssetsMinimalJsonUrl);
|
||||
} else {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} BASE_MODULE is not set to Liquid (currently ${configContent.BASE_MODULE}), skipping downloading assets`);
|
||||
}
|
||||
}
|
||||
|
||||
(() => {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} Downloading mining pool logos`);
|
||||
}
|
||||
downloadMiningPoolLogos$()
|
||||
.then(() => {
|
||||
if (verbose) {
|
||||
if (config.verbose) {
|
||||
console.log(`${LOG_TAG} Downloading promo video subtitles`);
|
||||
}
|
||||
downloadPromoVideoSubtiles$();
|
||||
})
|
||||
.then(() => {
|
||||
if (verbose) {
|
||||
await downloadPromoVideoSubtitles();
|
||||
|
||||
if (config.verbose) {
|
||||
console.log(`${LOG_TAG} Downloading promo video`);
|
||||
}
|
||||
downloadPromoVideo$();
|
||||
})
|
||||
.catch((error) => {
|
||||
throw new Error(error);
|
||||
});
|
||||
})();
|
||||
await downloadPromoVideo();
|
||||
|
||||
console.log(`${LOG_TAG} Asset synchronization complete`);
|
||||
} catch (error) {
|
||||
console.error(`${LOG_TAG} Error:`, error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue