mirror of
https://github.com/mempool/mempool.git
synced 2026-08-13 12:33:11 +02:00
Merge branch 'master' into natsoni/block-first-seen-improvements
This commit is contained in:
commit
abd01a352b
551 changed files with 31880 additions and 34448 deletions
145
.github/workflows/backend-integration.yml
vendored
Normal file
145
.github/workflows/backend-integration.yml
vendored
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
name: Backend Integration Tests with MariaDB
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, review_requested, synchronize]
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
|
||||
jobs:
|
||||
backend-integration:
|
||||
if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
|
||||
strategy:
|
||||
matrix:
|
||||
node: ["24.13.0"]
|
||||
fail-fast: false
|
||||
runs-on: mempool-ci
|
||||
|
||||
name: Backend Integration Tests - node ${{ matrix.node }}
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
path: ${{ matrix.node }}/integration
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
registry-url: "https://registry.npmjs.org"
|
||||
cache: 'npm'
|
||||
cache-dependency-path: '${{ matrix.node }}/integration/backend/package-lock.json'
|
||||
|
||||
- name: Cache node modules
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ${{ matrix.node }}/integration/backend/node_modules
|
||||
key: ${{ runner.os }}-backend-integration-node-${{ matrix.node }}-${{ hashFiles('${{ matrix.node }}/integration/backend/package-lock.json') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-backend-integration-node-${{ matrix.node }}-
|
||||
${{ runner.os }}-backend-integration-
|
||||
|
||||
- name: Read rust-toolchain file from repository
|
||||
id: gettoolchain
|
||||
run: echo "::set-output name=toolchain::$(cat ./rust/gbt/rust-toolchain)"
|
||||
working-directory: ${{ matrix.node }}/integration
|
||||
|
||||
- name: Cache Rust dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
${{ matrix.node }}/integration/rust/gbt/target/
|
||||
key: ${{ runner.os }}-cargo-integration-${{ hashFiles('${{ matrix.node }}/integration/rust/gbt/**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-integration-
|
||||
${{ runner.os }}-cargo-
|
||||
|
||||
- name: Install ${{ steps.gettoolchain.outputs.toolchain }} Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@f7ccc83f9ed1e5b9c81d8a67d7ad1a747e22a561
|
||||
with:
|
||||
toolchain: ${{ steps.gettoolchain.outputs.toolchain }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ${{ matrix.node }}/integration/backend
|
||||
|
||||
- name: Build backend
|
||||
run: npm run build
|
||||
working-directory: ${{ matrix.node }}/integration/backend
|
||||
|
||||
- name: Verify config file exists
|
||||
run: |
|
||||
ls -la mempool-config.test.json
|
||||
echo "Current directory: ${PWD}"
|
||||
echo "Config file will be: ${{ github.workspace }}/${{ matrix.node }}/integration/backend/mempool-config.test.json"
|
||||
test -f mempool-config.test.json || (echo "ERROR: Config file not found!" && exit 1)
|
||||
working-directory: ${{ matrix.node }}/integration/backend
|
||||
|
||||
- name: Run integration tests (DB auto-starts via Jest)
|
||||
run: |
|
||||
echo "MEMPOOL_CONFIG_FILE=$MEMPOOL_CONFIG_FILE"
|
||||
npm run test:integration
|
||||
working-directory: ${{ matrix.node }}/integration/backend
|
||||
env:
|
||||
MEMPOOL_CONFIG_FILE: ${{ github.workspace }}/${{ matrix.node }}/integration/backend/mempool-config.test.json
|
||||
|
||||
- name: Start MariaDB for server test
|
||||
run: docker compose -f docker-compose.test.yml up -d
|
||||
working-directory: ${{ matrix.node }}/integration/backend
|
||||
|
||||
- name: Wait for MariaDB
|
||||
run: |
|
||||
echo "Waiting for MariaDB to be ready..."
|
||||
for i in {1..30}; do
|
||||
if docker compose -f docker-compose.test.yml exec -T db-test mysqladmin ping -h localhost -u mempool_test -pmempool_test --silent 2>/dev/null; then
|
||||
echo "MariaDB is ready!"
|
||||
break
|
||||
fi
|
||||
echo "Attempt $i/30..."
|
||||
sleep 2
|
||||
done
|
||||
sleep 3
|
||||
working-directory: ${{ matrix.node }}/integration/backend
|
||||
|
||||
- name: Start backend server and verify connectivity
|
||||
run: |
|
||||
# Start server in background
|
||||
node dist/index.js &
|
||||
SERVER_PID=$!
|
||||
|
||||
# Wait for server to start
|
||||
echo "Waiting for server to start..."
|
||||
sleep 10
|
||||
|
||||
# Check if server is still running
|
||||
if ps -p $SERVER_PID > /dev/null 2>&1; then
|
||||
echo "Server started successfully and connected to database!"
|
||||
kill $SERVER_PID 2>/dev/null || true
|
||||
wait $SERVER_PID 2>/dev/null || true
|
||||
exit 0
|
||||
else
|
||||
echo "Server failed to start or exited prematurely"
|
||||
exit 1
|
||||
fi
|
||||
working-directory: ${{ matrix.node }}/integration/backend
|
||||
env:
|
||||
MEMPOOL_CONFIG_FILE: ${{ github.workspace }}/${{ matrix.node }}/integration/backend/mempool-config.test.json
|
||||
|
||||
- name: Cleanup containers
|
||||
if: always()
|
||||
run: docker compose -f docker-compose.test.yml down -v
|
||||
working-directory: ${{ matrix.node }}/integration/backend
|
||||
|
||||
- name: Display logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== MariaDB logs ==="
|
||||
docker compose -f docker-compose.test.yml logs db-test || true
|
||||
working-directory: ${{ matrix.node }}/integration/backend
|
||||
|
||||
25
.github/workflows/ci.yml
vendored
25
.github/workflows/ci.yml
vendored
|
|
@ -12,10 +12,10 @@ jobs:
|
|||
if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
|
||||
strategy:
|
||||
matrix:
|
||||
node: ["22.14.0"]
|
||||
node: ["24.13.0"]
|
||||
flavor: ["dev", "prod"]
|
||||
fail-fast: false
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: mempool-ci
|
||||
|
||||
name: Backend (${{ matrix.flavor }}) - node ${{ matrix.node }}
|
||||
steps:
|
||||
|
|
@ -63,7 +63,7 @@ jobs:
|
|||
- name: Install ${{ steps.gettoolchain.outputs.toolchain }} Rust toolchain
|
||||
# Latest version available on this commit is 1.71.1
|
||||
# Commit date is Aug 3, 2023
|
||||
uses: dtolnay/rust-toolchain@d8352f6b1d2e870bc5716e7a6d9b65c4cc244a1a
|
||||
uses: dtolnay/rust-toolchain@f7ccc83f9ed1e5b9c81d8a67d7ad1a747e22a561
|
||||
with:
|
||||
toolchain: ${{ steps.gettoolchain.outputs.toolchain }}
|
||||
|
||||
|
|
@ -96,8 +96,8 @@ jobs:
|
|||
name: "Cache assets for builds"
|
||||
strategy:
|
||||
matrix:
|
||||
node: ["22.14.0"]
|
||||
runs-on: ubuntu-latest
|
||||
node: ["24.13.0"]
|
||||
runs-on: mempool-ci
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
|
@ -202,10 +202,10 @@ jobs:
|
|||
if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
|
||||
strategy:
|
||||
matrix:
|
||||
node: ["22.14.0"]
|
||||
node: ["24.13.0"]
|
||||
flavor: ["dev", "prod"]
|
||||
fail-fast: false
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: mempool-ci
|
||||
|
||||
name: Frontend (${{ matrix.flavor }}) - node ${{ matrix.node }}
|
||||
steps:
|
||||
|
|
@ -299,11 +299,12 @@ jobs:
|
|||
|
||||
e2e:
|
||||
if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: mempool-ci
|
||||
needs: frontend
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
node: ["24.13.0"]
|
||||
module: ["mempool", "liquid", "testnet4"]
|
||||
|
||||
name: E2E tests for ${{ matrix.module }}
|
||||
|
|
@ -316,7 +317,7 @@ jobs:
|
|||
- name: Setup node
|
||||
uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: 22
|
||||
node-version: ${{ matrix.node }}
|
||||
cache: "npm"
|
||||
cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json
|
||||
|
||||
|
|
@ -324,9 +325,9 @@ jobs:
|
|||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ${{ matrix.module }}/frontend/node_modules
|
||||
key: ${{ runner.os }}-e2e-${{ matrix.module }}-node-22-${{ hashFiles('${{ matrix.module }}/frontend/package-lock.json') }}
|
||||
key: ${{ runner.os }}-e2e-${{ matrix.module }}-node-${{ matrix.node }}-${{ hashFiles('${{ matrix.module }}/frontend/package-lock.json') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-e2e-${{ matrix.module }}-node-22-
|
||||
${{ runner.os }}-e2e-${{ matrix.module }}-node-${{ matrix.node }}-
|
||||
${{ runner.os }}-e2e-${{ matrix.module }}-
|
||||
|
||||
- name: Restore cached mining pool assets
|
||||
|
|
@ -439,7 +440,7 @@ jobs:
|
|||
CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
|
||||
validate_docker_json:
|
||||
if: "(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')) || github.event_name == 'push'"
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: mempool-ci
|
||||
name: Validate generated backend Docker JSON
|
||||
|
||||
steps:
|
||||
|
|
|
|||
399
.github/workflows/docker.yml
vendored
Normal file
399
.github/workflows/docker.yml
vendored
Normal file
|
|
@ -0,0 +1,399 @@
|
|||
name: Docker build on tag
|
||||
env:
|
||||
DOCKER_CLI_EXPERIMENTAL: enabled
|
||||
TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$"
|
||||
DOCKER_BUILDKIT: 1 # Enable BuildKit for better performance
|
||||
COMPOSE_DOCKER_CLI_BUILD: 0
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- v[0-9]+.[0-9]+.[0-9]+
|
||||
- v[0-9]+.[0-9]+.[0-9]+-*
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened, labeled]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
test-images:
|
||||
# Always run on tag pushes and all pull requests
|
||||
runs-on: mempool-ci
|
||||
timeout-minutes: 30
|
||||
name: Test built Docker images
|
||||
steps:
|
||||
- name: Checkout project
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Add SHORT_SHA env property with commit short sha
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
SHA="${{ github.event.pull_request.head.sha }}"
|
||||
else
|
||||
SHA="${GITHUB_SHA}"
|
||||
fi
|
||||
echo "SHORT_SHA=${SHA:0:8}" >> $GITHUB_ENV
|
||||
|
||||
- name: Set TAG from pushed tag or package.json
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "push" ]; then
|
||||
TAG="${GITHUB_REF/refs\/tags\//}"
|
||||
else
|
||||
FRONTEND_VERSION=$(jq -r '.version' frontend/package.json)
|
||||
BACKEND_VERSION=$(jq -r '.version' backend/package.json)
|
||||
if [ "$FRONTEND_VERSION" != "$BACKEND_VERSION" ]; then
|
||||
echo "Error: Frontend version ($FRONTEND_VERSION) and backend version ($BACKEND_VERSION) do not match"
|
||||
exit 1
|
||||
fi
|
||||
TAG="v${FRONTEND_VERSION}-${SHORT_SHA}"
|
||||
fi
|
||||
echo "TAG=${TAG}" >> $GITHUB_ENV
|
||||
|
||||
- name: Show set environment variables
|
||||
run: |
|
||||
printf " TAG: %s\n" "$TAG"
|
||||
printf " SHORT_SHA: %s\n" "$SHORT_SHA"
|
||||
|
||||
- name: Init repo for Dockerization
|
||||
run: docker/init.sh "$TAG"
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build frontend image locally
|
||||
run: |
|
||||
docker buildx build \
|
||||
--tag test-frontend:$TAG \
|
||||
--build-arg commitHash=$SHORT_SHA \
|
||||
--load \
|
||||
--platform linux/amd64 \
|
||||
./frontend/
|
||||
|
||||
- name: Build backend image locally
|
||||
run: |
|
||||
docker buildx build \
|
||||
--tag test-backend:$TAG \
|
||||
--build-context rustgbt=./rust \
|
||||
--build-context backend=./backend \
|
||||
--build-arg commitHash=$SHORT_SHA \
|
||||
--load \
|
||||
--platform linux/amd64 \
|
||||
./backend/
|
||||
|
||||
- name: Prepare docker-compose test file
|
||||
run: |
|
||||
cat > /tmp/modify_compose.py << 'SCRIPT_END'
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Read the base docker-compose file
|
||||
with open('docker/docker-compose.yml', 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Get TAG from environment
|
||||
tag = os.environ.get('TAG', '')
|
||||
|
||||
# Replace image names with locally built test images
|
||||
content = content.replace('image: mempool/frontend:latest', f'image: test-frontend:{tag}')
|
||||
content = content.replace('image: mempool/backend:latest', f'image: test-backend:{tag}')
|
||||
|
||||
# Change web port mapping from 80:8080 to 8080:8080
|
||||
content = content.replace('- 80:8080', '- 8080:8080')
|
||||
|
||||
# Remove volumes from api service
|
||||
content = re.sub(r' volumes:\n - \.\/data:\/backend\/cache\n', '', content)
|
||||
|
||||
# For db service: remove user and volumes, add tmpfs and healthcheck
|
||||
# Remove user line from db service (only the one in db service)
|
||||
lines = content.split('\n')
|
||||
in_db_service = False
|
||||
new_lines = []
|
||||
for i, line in enumerate(lines):
|
||||
if line.strip().startswith('db:'):
|
||||
in_db_service = True
|
||||
elif line.strip() and not line.startswith(' ') and not line.startswith('\t'):
|
||||
in_db_service = False
|
||||
if in_db_service and line.strip() == 'user: "1000:1000"':
|
||||
continue
|
||||
new_lines.append(line)
|
||||
content = '\n'.join(new_lines)
|
||||
|
||||
# Remove volumes section from db service
|
||||
content = re.sub(r' volumes:\n - \.\/mysql\/data:\/var\/lib\/mysql\n', '', content)
|
||||
|
||||
# Add tmpfs after stop_grace_period in db service (healthcheck already exists in base file)
|
||||
db_stop_grace = ' stop_grace_period: 1m'
|
||||
db_additions = ' stop_grace_period: 1m\n tmpfs:\n - /var/lib/mysql'
|
||||
content = content.replace(db_stop_grace, db_additions, 1)
|
||||
|
||||
# Add depends_on to web service after ports
|
||||
web_ports = ' ports:\n - 8080:8080'
|
||||
web_with_depends = ' ports:\n - 8080:8080\n depends_on:\n - api\n - db'
|
||||
content = content.replace(web_ports, web_with_depends, 1)
|
||||
|
||||
# Add depends_on to api service after command
|
||||
api_command = ' command: "./wait-for-it.sh db:3306 --timeout=720 --strict -- ./start.sh"'
|
||||
api_with_depends = ' command: "./wait-for-it.sh db:3306 --timeout=720 --strict -- ./start.sh"\n depends_on:\n - db'
|
||||
content = content.replace(api_command, api_with_depends, 1)
|
||||
|
||||
# Write the modified content
|
||||
with open('docker-compose.test.yml', 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
print("Generated docker-compose.test.yml")
|
||||
SCRIPT_END
|
||||
python3 /tmp/modify_compose.py
|
||||
cat docker-compose.test.yml
|
||||
|
||||
- name: Start containers
|
||||
run: |
|
||||
docker compose -f docker-compose.test.yml up -d
|
||||
|
||||
- name: Wait for services to be ready
|
||||
run: |
|
||||
echo "Waiting for all services (web, api, db) to be healthy..."
|
||||
timeout=120
|
||||
elapsed=0
|
||||
while [ $elapsed -lt $timeout ]; do
|
||||
# Check health status for all services
|
||||
PS_OUTPUT=$(docker compose -f docker-compose.test.yml ps)
|
||||
HEALTHY_COUNT=$(echo "$PS_OUTPUT" | grep -c "(healthy)" || true)
|
||||
if [ "$HEALTHY_COUNT" -ge 3 ]; then
|
||||
echo "All services are healthy!"
|
||||
echo "$PS_OUTPUT"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for services to be healthy... (${elapsed}s/${timeout}s)"
|
||||
echo "$PS_OUTPUT"
|
||||
sleep 2
|
||||
elapsed=$((elapsed + 2))
|
||||
done
|
||||
if [ $elapsed -ge $timeout ]; then
|
||||
echo "Services did not become healthy in time"
|
||||
docker compose -f docker-compose.test.yml ps
|
||||
docker compose -f docker-compose.test.yml logs
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Verify containers are healthy
|
||||
run: |
|
||||
echo "Checking container health status..."
|
||||
PS_OUTPUT=$(docker compose -f docker-compose.test.yml ps)
|
||||
echo "$PS_OUTPUT"
|
||||
|
||||
# Check that all three services (web, api, db) are healthy
|
||||
HEALTHY_COUNT=$(echo "$PS_OUTPUT" | grep -c "(healthy)" || true)
|
||||
if [ "$HEALTHY_COUNT" -lt 3 ]; then
|
||||
echo "Not all containers are healthy. Expected 3 healthy services, found $HEALTHY_COUNT"
|
||||
docker compose -f docker-compose.test.yml logs
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify each service individually for better error messages
|
||||
if ! echo "$PS_OUTPUT" | grep -q "web.*(healthy)"; then
|
||||
echo "Web service is not healthy"
|
||||
docker compose -f docker-compose.test.yml logs web
|
||||
exit 1
|
||||
fi
|
||||
if ! echo "$PS_OUTPUT" | grep -q "api.*(healthy)"; then
|
||||
echo "API service is not healthy"
|
||||
docker compose -f docker-compose.test.yml logs api
|
||||
exit 1
|
||||
fi
|
||||
if ! echo "$PS_OUTPUT" | grep -q "db.*(healthy)"; then
|
||||
echo "Database service is not healthy"
|
||||
docker compose -f docker-compose.test.yml logs db
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "All containers are healthy!"
|
||||
|
||||
- name: Show container logs
|
||||
if: failure()
|
||||
run: |
|
||||
docker compose -f docker-compose.test.yml logs
|
||||
|
||||
- name: Clean up containers
|
||||
if: always()
|
||||
run: |
|
||||
docker compose -f docker-compose.test.yml down -v
|
||||
|
||||
build:
|
||||
needs: test-images
|
||||
# Run on tag pushes OR on PRs with "docker-push" label (after test-images passes)
|
||||
if: |
|
||||
needs.test-images.result == 'success' &&
|
||||
(github.event_name == 'push' ||
|
||||
(github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'docker-push')))
|
||||
strategy:
|
||||
matrix:
|
||||
service:
|
||||
- frontend
|
||||
- backend
|
||||
runs-on: mempool-ci
|
||||
timeout-minutes: 120
|
||||
name: Build and push to DockerHub
|
||||
outputs:
|
||||
image-digest-frontend: ${{ matrix.service == 'frontend' && steps.docker-build.outputs.digest || '' }}
|
||||
image-digest-backend: ${{ matrix.service == 'backend' && steps.docker-build.outputs.digest || '' }}
|
||||
tag: ${{ matrix.service == 'frontend' && (steps.set-tag-push.outputs.tag || steps.set-tag-pr.outputs.tag) || '' }}
|
||||
steps:
|
||||
- name: Replace the current swap file
|
||||
shell: bash
|
||||
run: |
|
||||
sudo swapoff /mnt/swapfile || true
|
||||
sudo rm -f /mnt/swapfile
|
||||
sudo fallocate -l 16G /mnt/swapfile
|
||||
sudo chmod 600 /mnt/swapfile
|
||||
sudo mkswap /mnt/swapfile
|
||||
sudo swapon /mnt/swapfile
|
||||
|
||||
- name: Show current memory and swap status
|
||||
shell: bash
|
||||
run: |
|
||||
sudo free -h
|
||||
echo
|
||||
sudo swapon --show
|
||||
|
||||
- name: Mount a tmpfs over /var/lib/docker
|
||||
shell: bash
|
||||
run: |
|
||||
if [ ! -d "/var/lib/docker" ]; then
|
||||
echo "Directory '/var/lib/docker' not found"
|
||||
exit 1
|
||||
fi
|
||||
sudo mount -t tmpfs -o size=12G tmpfs /var/lib/docker
|
||||
sudo systemctl restart docker
|
||||
sudo df -h | grep docker
|
||||
|
||||
# Only for tag pushes: use the Git tag as TAG
|
||||
- name: Set TAG from pushed tag
|
||||
if: github.event_name == 'push'
|
||||
id: set-tag-push
|
||||
run: |
|
||||
TAG="${GITHUB_REF/refs\/tags\//}"
|
||||
echo "TAG=${TAG}" >> $GITHUB_ENV
|
||||
echo "tag=${TAG}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Add SHORT_SHA env property with commit short sha
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
SHA="${{ github.event.pull_request.head.sha }}"
|
||||
else
|
||||
SHA="${GITHUB_SHA}"
|
||||
fi
|
||||
echo "SHORT_SHA=${SHA:0:8}" >> $GITHUB_ENV
|
||||
|
||||
|
||||
- name: Login to Docker for building
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Checkout project
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# For PRs: use package.json version + short sha as TAG
|
||||
- name: Set TAG from service package.json for pull requests
|
||||
if: github.event_name == 'pull_request'
|
||||
id: set-tag-pr
|
||||
run: |
|
||||
if [ "${{ matrix.service }}" = "frontend" ]; then
|
||||
VERSION=$(jq -r '.version' frontend/package.json)
|
||||
else
|
||||
VERSION=$(jq -r '.version' backend/package.json)
|
||||
fi
|
||||
TAG="v${VERSION}-${SHORT_SHA}"
|
||||
echo "TAG=${TAG}" >> $GITHUB_ENV
|
||||
echo "tag=${TAG}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Show set environment variables
|
||||
run: |
|
||||
printf " TAG: %s\n" "$TAG"
|
||||
printf " SHORT_SHA: %s\n" "$SHORT_SHA"
|
||||
|
||||
- name: Init repo for Dockerization
|
||||
run: docker/init.sh "$TAG"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
id: qemu
|
||||
|
||||
- name: Setup Docker buildx action
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
driver-opts: |
|
||||
network=host
|
||||
id: buildx
|
||||
|
||||
- name: Available platforms
|
||||
run: echo ${{ steps.buildx.outputs.platforms }}
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v3
|
||||
id: cache
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-${{ matrix.service }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-buildx-${{ matrix.service }}-
|
||||
|
||||
- name: Run Docker buildx for ${{ matrix.service }} against tag
|
||||
id: docker-build
|
||||
run: |
|
||||
docker buildx build \
|
||||
--cache-from "type=local,src=/tmp/.buildx-cache" \
|
||||
--cache-to "type=local,dest=/tmp/.buildx-cache,mode=max" \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:$TAG \
|
||||
--build-context rustgbt=./rust \
|
||||
--build-context backend=./backend \
|
||||
--output "type=registry,push=true" \
|
||||
--build-arg commitHash=$SHORT_SHA \
|
||||
./${{ matrix.service }}/
|
||||
|
||||
tag-latest:
|
||||
needs: build
|
||||
# Only for successful tag pushes (not PRs with docker-push label) and only for "plain" versions (no '-')
|
||||
if: ${{ needs.build.result == 'success' && github.event_name == 'push' && !contains(github.ref_name, '-') }}
|
||||
runs-on: mempool-ci
|
||||
timeout-minutes: 30
|
||||
name: Tag release build as latest
|
||||
strategy:
|
||||
matrix:
|
||||
service:
|
||||
- frontend
|
||||
- backend
|
||||
steps:
|
||||
- name: Set env variables
|
||||
run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
|
||||
- name: Setup Docker buildx action
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
driver-opts: |
|
||||
network=host
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Tag as latest for ${{ matrix.service }}
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
--tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:latest \
|
||||
${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:$TAG
|
||||
8
.github/workflows/e2e_parameterized.yml
vendored
8
.github/workflows/e2e_parameterized.yml
vendored
|
|
@ -22,7 +22,7 @@ on:
|
|||
jobs:
|
||||
cache:
|
||||
name: "Cache assets for builds"
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: mempool-ci
|
||||
steps:
|
||||
- name: Determine checkout ref
|
||||
id: determine-ref
|
||||
|
|
@ -43,7 +43,7 @@ jobs:
|
|||
- name: Setup Node
|
||||
uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: 22.14.0
|
||||
node-version: 24.13.0
|
||||
registry-url: "https://registry.npmjs.org"
|
||||
|
||||
- name: Install (Prod dependencies only)
|
||||
|
|
@ -123,7 +123,7 @@ jobs:
|
|||
key: promo-video-assets-cache
|
||||
|
||||
e2e:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: mempool-ci
|
||||
needs: cache
|
||||
strategy:
|
||||
fail-fast: false
|
||||
|
|
@ -151,7 +151,7 @@ jobs:
|
|||
- name: Setup node
|
||||
uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: 22.14.0
|
||||
node-version: 24.13.0
|
||||
cache: "npm"
|
||||
cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ on: [workflow_dispatch]
|
|||
|
||||
jobs:
|
||||
print-backend-sha:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: mempool-ci
|
||||
name: Get block height
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
|
|
|||
2
.github/workflows/get_backend_hash.yml
vendored
2
.github/workflows/get_backend_hash.yml
vendored
|
|
@ -4,7 +4,7 @@ on: [workflow_dispatch]
|
|||
|
||||
jobs:
|
||||
print-backend-sha:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: mempool-ci
|
||||
name: Print backend hashes
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
|
|
|||
2
.github/workflows/get_image_digest.yml
vendored
2
.github/workflows/get_image_digest.yml
vendored
|
|
@ -10,7 +10,7 @@ on:
|
|||
type: string
|
||||
jobs:
|
||||
print-images-sha:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: mempool-ci
|
||||
name: Print digest for images
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
|
|
|||
152
.github/workflows/on-tag.yml
vendored
152
.github/workflows/on-tag.yml
vendored
|
|
@ -1,152 +0,0 @@
|
|||
name: Docker build on tag
|
||||
env:
|
||||
DOCKER_CLI_EXPERIMENTAL: enabled
|
||||
TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$"
|
||||
DOCKER_BUILDKIT: 1 # Enable BuildKit for better performance
|
||||
COMPOSE_DOCKER_CLI_BUILD: 0
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- v[0-9]+.[0-9]+.[0-9]+
|
||||
- v[0-9]+.[0-9]+.[0-9]+-*
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
strategy:
|
||||
matrix:
|
||||
service:
|
||||
- frontend
|
||||
- backend
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 120
|
||||
name: Build and push to DockerHub
|
||||
outputs:
|
||||
image-digest-frontend: ${{ matrix.service == 'frontend' && steps.docker-build.outputs.digest || '' }}
|
||||
image-digest-backend: ${{ matrix.service == 'backend' && steps.docker-build.outputs.digest || '' }}
|
||||
steps:
|
||||
- name: Replace the current swap file
|
||||
shell: bash
|
||||
run: |
|
||||
sudo swapoff /mnt/swapfile || true
|
||||
sudo rm -f /mnt/swapfile
|
||||
sudo fallocate -l 16G /mnt/swapfile
|
||||
sudo chmod 600 /mnt/swapfile
|
||||
sudo mkswap /mnt/swapfile
|
||||
sudo swapon /mnt/swapfile
|
||||
|
||||
- name: Show current memory and swap status
|
||||
shell: bash
|
||||
run: |
|
||||
sudo free -h
|
||||
echo
|
||||
sudo swapon --show
|
||||
|
||||
- name: Mount a tmpfs over /var/lib/docker
|
||||
shell: bash
|
||||
run: |
|
||||
if [ ! -d "/var/lib/docker" ]; then
|
||||
echo "Directory '/var/lib/docker' not found"
|
||||
exit 1
|
||||
fi
|
||||
sudo mount -t tmpfs -o size=12G tmpfs /var/lib/docker
|
||||
sudo systemctl restart docker
|
||||
sudo df -h | grep docker
|
||||
|
||||
- name: Set env variables
|
||||
run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
|
||||
|
||||
- name: Show set environment variables
|
||||
run: |
|
||||
printf " TAG: %s\n" "$TAG"
|
||||
|
||||
- name: Add SHORT_SHA env property with commit short sha
|
||||
run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV
|
||||
|
||||
- name: Login to Docker for building
|
||||
run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
|
||||
|
||||
- name: Checkout project
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Init repo for Dockerization
|
||||
run: docker/init.sh "$TAG"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
id: qemu
|
||||
|
||||
- name: Setup Docker buildx action
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
driver-opts: |
|
||||
network=host
|
||||
id: buildx
|
||||
|
||||
- name: Available platforms
|
||||
run: echo ${{ steps.buildx.outputs.platforms }}
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v3
|
||||
id: cache
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-${{ matrix.service }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-buildx-${{ matrix.service }}-
|
||||
|
||||
- name: Run Docker buildx for ${{ matrix.service }} against tag
|
||||
id: docker-build
|
||||
run: |
|
||||
docker buildx build \
|
||||
--cache-from "type=local,src=/tmp/.buildx-cache" \
|
||||
--cache-to "type=local,dest=/tmp/.buildx-cache,mode=max" \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:$TAG \
|
||||
--build-context rustgbt=./rust \
|
||||
--build-context backend=./backend \
|
||||
--output "type=registry,push=true" \
|
||||
--build-arg commitHash=$SHORT_SHA \
|
||||
./${{ matrix.service }}/
|
||||
|
||||
tag-latest:
|
||||
needs: build
|
||||
if: ${{ needs.build.result == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
name: Tag images as latest
|
||||
strategy:
|
||||
matrix:
|
||||
service:
|
||||
- frontend
|
||||
- backend
|
||||
steps:
|
||||
- name: Set env variables
|
||||
run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
|
||||
- name: Setup Docker buildx action
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
driver-opts: |
|
||||
network=host
|
||||
|
||||
- name: Login to Docker Hub
|
||||
run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
|
||||
|
||||
- name: Tag multi-arch image as latest for ${{ matrix.service }}
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
--tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:latest \
|
||||
${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:$TAG
|
||||
131
.github/workflows/project-review-status.yml
vendored
Normal file
131
.github/workflows/project-review-status.yml
vendored
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
# Workflow: Automate project board management
|
||||
# - Add newly created issues to project #8
|
||||
# - Set status to "Review Needed" when a reviewer is requested on a non-draft PR
|
||||
name: Project Board Automation
|
||||
|
||||
# Triggers: Review requested on PRs, or new issues opened
|
||||
on:
|
||||
pull_request:
|
||||
types: [review_requested]
|
||||
issues:
|
||||
types: [opened]
|
||||
|
||||
jobs:
|
||||
manage-project-board:
|
||||
runs-on: mempool-ci
|
||||
steps:
|
||||
- name: Update Project Board
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
# Use the PAT stored in repository secrets (has project write access)
|
||||
github-token: ${{ secrets.PROJECT_TOKEN }}
|
||||
script: |
|
||||
// Skip draft PRs
|
||||
if (context.eventName === 'pull_request' && context.payload.pull_request.draft) {
|
||||
console.log('PR is a draft, skipping Review Needed status...');
|
||||
return;
|
||||
}
|
||||
|
||||
// Handle new issues - add to project
|
||||
if (context.eventName === 'issues') {
|
||||
const addMutation = `
|
||||
mutation($projectId: ID!, $contentId: ID!) {
|
||||
addProjectV2ItemById(input: {projectId: $projectId, contentId: $contentId}) {
|
||||
item { id }
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
try {
|
||||
await github.graphql(addMutation, {
|
||||
projectId: "${{ secrets.PROJECT_ID }}",
|
||||
contentId: context.payload.issue.node_id
|
||||
});
|
||||
|
||||
console.log('Successfully added issue to project #8');
|
||||
} catch (error) {
|
||||
// Handle case where the issue is already in the project, or log other failures
|
||||
const errors = error && error.errors ? error.errors : [];
|
||||
const alreadyInProject = errors.some(e =>
|
||||
typeof e.message === 'string' &&
|
||||
e.message.toLowerCase().includes('already') &&
|
||||
e.message.toLowerCase().includes('project')
|
||||
);
|
||||
|
||||
if (alreadyInProject) {
|
||||
console.log('Issue is already in project #8, skipping add.');
|
||||
} else {
|
||||
console.error('Failed to add issue to project #8:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Handle PR review_requested - update status to "Review Needed"
|
||||
// GraphQL query to find the PR's project items
|
||||
// This fetches all projects the PR is linked to
|
||||
const query = `
|
||||
query($owner: String!, $repo: String!, $pr: Int!) {
|
||||
repository(owner: $owner, name: $repo) {
|
||||
pullRequest(number: $pr) {
|
||||
projectItems(first: 10) {
|
||||
nodes {
|
||||
id
|
||||
project {
|
||||
number
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
// Execute the query with current repo/PR context
|
||||
const result = await github.graphql(query, {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
pr: context.payload.pull_request.number
|
||||
});
|
||||
|
||||
// Find the project item that belongs to project #8
|
||||
const projectItems = result.repository.pullRequest.projectItems.nodes;
|
||||
const projectItem = projectItems.find(item => item.project.number === 8);
|
||||
|
||||
// Exit early if PR isn't in project #8
|
||||
if (!projectItem) {
|
||||
console.log('PR is not in project #8, skipping...');
|
||||
return;
|
||||
}
|
||||
|
||||
// GraphQL mutation to update the Status field
|
||||
const mutation = `
|
||||
mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!) {
|
||||
updateProjectV2ItemFieldValue(
|
||||
input: {
|
||||
projectId: $projectId
|
||||
itemId: $itemId
|
||||
fieldId: $fieldId
|
||||
value: { singleSelectOptionId: $optionId }
|
||||
}
|
||||
) {
|
||||
projectV2Item {
|
||||
id
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
// Execute the mutation using IDs stored in repository variables
|
||||
// PROJECT_ID: The project's unique identifier
|
||||
// STATUS_FIELD_ID: The "Status" field's unique identifier
|
||||
// REVIEW_NEEDED_OPTION_ID: The "Review Needed" option's unique identifier
|
||||
await github.graphql(mutation, {
|
||||
projectId: "${{ secrets.PROJECT_ID }}",
|
||||
itemId: projectItem.id,
|
||||
fieldId: "${{ secrets.STATUS_FIELD_ID }}",
|
||||
optionId: "${{ secrets.REVIEW_NEEDED_OPTION_ID }}"
|
||||
});
|
||||
|
||||
console.log('Successfully updated project status to Review Needed');
|
||||
2
.nvmrc
2
.nvmrc
|
|
@ -1 +1 @@
|
|||
v22
|
||||
v24.13.0
|
||||
|
|
|
|||
|
|
@ -1,2 +1,4 @@
|
|||
node_modules
|
||||
dist
|
||||
dist
|
||||
eslint-local-rules
|
||||
.eslintrc.js
|
||||
|
|
@ -1,8 +1,13 @@
|
|||
{
|
||||
module.exports = {
|
||||
"root": true,
|
||||
"parser": "@typescript-eslint/parser",
|
||||
"parserOptions": {
|
||||
"project": "./tsconfig.json",
|
||||
"tsconfigRootDir": __dirname
|
||||
},
|
||||
"plugins": [
|
||||
"@typescript-eslint"
|
||||
"@typescript-eslint",
|
||||
"local-rules"
|
||||
],
|
||||
"extends": [
|
||||
"eslint:recommended",
|
||||
|
|
@ -10,6 +15,16 @@
|
|||
"plugin:@typescript-eslint/recommended",
|
||||
"prettier"
|
||||
],
|
||||
"ignorePatterns": ["dist", "eslint-local-rules", ".eslintrc.js", "testSetup*.ts", "jest.integration.*.ts", "__tests__", "*.config.ts"],
|
||||
"overrides": [
|
||||
{
|
||||
"files": ["src/__integration_tests__/**/*"],
|
||||
"rules": {
|
||||
"@typescript-eslint/no-floating-promises": "off",
|
||||
"local-rules/no-unhandled-await": "off"
|
||||
}
|
||||
}
|
||||
],
|
||||
"rules": {
|
||||
"@typescript-eslint/ban-ts-comment": 1,
|
||||
"@typescript-eslint/ban-types": 1,
|
||||
|
|
@ -21,6 +36,8 @@
|
|||
"@typescript-eslint/no-var-requires": 1,
|
||||
"@typescript-eslint/explicit-function-return-type": 1,
|
||||
"@typescript-eslint/no-unused-vars": 1,
|
||||
"@typescript-eslint/no-floating-promises": "error",
|
||||
"local-rules/no-unhandled-await": "error",
|
||||
"no-console": 1,
|
||||
"no-constant-condition": 1,
|
||||
"no-dupe-else-if": 1,
|
||||
21
backend/docker-compose.test.yml
Normal file
21
backend/docker-compose.test.yml
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
version: "3.7"
|
||||
|
||||
services:
|
||||
db-test:
|
||||
image: mariadb:10.5.21
|
||||
environment:
|
||||
MYSQL_DATABASE: "mempool_test"
|
||||
MYSQL_USER: "mempool_test"
|
||||
MYSQL_PASSWORD: "mempool_test"
|
||||
MYSQL_ROOT_PASSWORD: "admin"
|
||||
MARIADB_AUTO_UPGRADE: "1"
|
||||
ports:
|
||||
- "33306:3306"
|
||||
healthcheck:
|
||||
test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "mempool_test", "-pmempool_test"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
tmpfs:
|
||||
- /var/lib/mysql
|
||||
|
||||
398
backend/eslint-local-rules/index.js
Normal file
398
backend/eslint-local-rules/index.js
Normal file
|
|
@ -0,0 +1,398 @@
|
|||
'use strict';
|
||||
|
||||
module.exports = {
|
||||
'no-unhandled-await': {
|
||||
meta: {
|
||||
type: 'problem',
|
||||
docs: {
|
||||
description: 'forbid unhandled await unless callee is @asyncSafe, context is @asyncUnsafe, or rejection is explicitly handled',
|
||||
},
|
||||
schema: [{
|
||||
type: 'object',
|
||||
properties: {
|
||||
safeTag: { type: 'string' }, // jsdoc tag that marks a callee safe (default '@asyncSafe')
|
||||
unsafeTag: { type: 'string' }, // comment/jsdoc that marks a context unsafe (default '@asyncUnsafe')
|
||||
allowAllSettled: { type: 'boolean' },
|
||||
allowCatchMethod: { type: 'boolean' },
|
||||
allowThenWithTwoArgs:{ type: 'boolean' },
|
||||
},
|
||||
additionalProperties: false,
|
||||
}],
|
||||
messages: {
|
||||
unhandled:
|
||||
'await of non-@asyncSafe callee in @asyncSafe context; use try/catch or annotate callee (@asyncSafe) or context (@asyncUnsafe)',
|
||||
unhandledVoid:
|
||||
'void of non-@asyncSafe callee; annotate callee with @asyncSafe or handle the promise properly',
|
||||
},
|
||||
},
|
||||
|
||||
create(context) {
|
||||
const src = context.getSourceCode();
|
||||
const opt = Object.assign(
|
||||
{
|
||||
safeTag: '@asyncSafe',
|
||||
unsafeTag: '@asyncUnsafe',
|
||||
allowAllSettled: true,
|
||||
allowCatchMethod: true,
|
||||
allowThenWithTwoArgs: true,
|
||||
},
|
||||
(context.options && context.options[0]) || {}
|
||||
);
|
||||
|
||||
// optional typescript API (for cross-file/class resolution)
|
||||
let ts = null, services = null, checker = null, es2ts = null;
|
||||
try {
|
||||
// eslint will only populate parserServices if @typescript-eslint/parser + parserOptions.project are set
|
||||
services = context.parserServices || null;
|
||||
// @ts-ignore
|
||||
if (services && (services.program || services.esTreeNodeToTSNodeMap)) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
ts = require('typescript');
|
||||
// @ts-ignore
|
||||
const program = services.program;
|
||||
// @ts-ignore
|
||||
es2ts = services.esTreeNodeToTSNodeMap;
|
||||
checker = program?.getTypeChecker?.();
|
||||
}
|
||||
} catch { /* noop */ }
|
||||
|
||||
const hasRange = (n) => n && Array.isArray(n.range);
|
||||
const inside = (n, o) => hasRange(n) && hasRange(o) && n.range[0] >= o.range[0] && n.range[1] <= o.range[1];
|
||||
const before = (a, b) => hasRange(a) && hasRange(b) && a.range[0] < b.range[0];
|
||||
|
||||
const isFnNode = (n) =>
|
||||
n &&
|
||||
(n.type === 'FunctionDeclaration' ||
|
||||
n.type === 'FunctionExpression' ||
|
||||
n.type === 'ArrowFunctionExpression' ||
|
||||
n.type === 'MethodDefinition');
|
||||
|
||||
const isCommentWith = (c, tag) => typeof c?.value === 'string' && c.value.includes(tag);
|
||||
|
||||
// --- jsdoc tag utils ----------------------------------------------------
|
||||
const stripAt = (s) => (s || '').replace(/^@/, '');
|
||||
|
||||
function tsNodeHasJsDocTag(tsNode, tag) {
|
||||
if (!ts || !tsNode) return false;
|
||||
try {
|
||||
const want = stripAt(tag);
|
||||
const tags = ts.getJSDocTags(tsNode) || [];
|
||||
return tags.some((t) => {
|
||||
const n = t.tagName && (t.tagName.escapedText || t.tagName.getText?.());
|
||||
return String(n) === want;
|
||||
});
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
function leadingCommentsHaveTag(node, tag) {
|
||||
if (!node) return false;
|
||||
const lead = src.getCommentsBefore(node) || [];
|
||||
return lead.some((c) => isCommentWith(c, tag));
|
||||
}
|
||||
|
||||
function isExportWrapper(node) {
|
||||
return node?.type === 'ExportNamedDeclaration' || node?.type === 'ExportDefaultDeclaration';
|
||||
}
|
||||
|
||||
// does a given function *definition* carry tag in its leading comments?
|
||||
function fnHasTag(fnNode, tag) {
|
||||
if (!fnNode) return false;
|
||||
|
||||
// 1) method definitions: jsdoc sits on the MethodDefinition
|
||||
if (fnNode.type === 'MethodDefinition') {
|
||||
return leadingCommentsHaveTag(fnNode, tag);
|
||||
}
|
||||
|
||||
// 2) function declarations (also handle `export` wrappers)
|
||||
if (fnNode.type === 'FunctionDeclaration') {
|
||||
if (leadingCommentsHaveTag(fnNode, tag)) return true;
|
||||
const p = fnNode.parent;
|
||||
if (isExportWrapper(p) && leadingCommentsHaveTag(p, tag)) return true;
|
||||
const gp = p && p.parent;
|
||||
if (isExportWrapper(gp) && leadingCommentsHaveTag(gp, tag)) return true; // belt & suspenders
|
||||
return false;
|
||||
}
|
||||
|
||||
// 3) function/arrow expressions
|
||||
if (fnNode.type === 'FunctionExpression' || fnNode.type === 'ArrowFunctionExpression') {
|
||||
// tag directly on the expression
|
||||
if (leadingCommentsHaveTag(fnNode, tag)) return true;
|
||||
|
||||
const p = fnNode.parent;
|
||||
|
||||
// tag on class members that wrap the fn expr (class fields or methods-as-values)
|
||||
if (
|
||||
p?.type === 'MethodDefinition' ||
|
||||
p?.type === 'PropertyDefinition' || // ts/estree: class field
|
||||
p?.type === 'ClassProperty' // older @typescript-eslint
|
||||
) {
|
||||
if (leadingCommentsHaveTag(p, tag)) return true;
|
||||
}
|
||||
|
||||
// tag on a variable declarator (const fn = async () => {})
|
||||
if (p?.type === 'VariableDeclarator') {
|
||||
if (leadingCommentsHaveTag(p, tag)) return true;
|
||||
if (p.parent && leadingCommentsHaveTag(p.parent, tag)) return true; // VariableDeclaration
|
||||
// handle: export const fn = async () => {}
|
||||
const exp = p.parent && p.parent.parent;
|
||||
if (isExportWrapper(exp) && leadingCommentsHaveTag(exp, tag)) return true;
|
||||
}
|
||||
|
||||
// handle: export default (async () => {...}) or export default (async function(){})
|
||||
if (isExportWrapper(p) && leadingCommentsHaveTag(p, tag)) return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// nearest class body ancestor (if any)
|
||||
function nearestClassBody() {
|
||||
const anc = context.getAncestors();
|
||||
for (let i = anc.length - 1; i >= 0; i--) {
|
||||
if (anc[i]?.type === 'ClassBody') return anc[i];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// within the same class, find a method by name
|
||||
function findMethodInCurrentClass(propertyName) {
|
||||
const body = nearestClassBody();
|
||||
if (!body) return null;
|
||||
for (const el of body.body || []) {
|
||||
if (el?.type === 'MethodDefinition') {
|
||||
// only handle simple identifiers (not computed) rn
|
||||
if (el.key?.type === 'Identifier' && el.key.name === propertyName) return el;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// nearest function ancestor
|
||||
function nearestFn() {
|
||||
const anc = context.getAncestors();
|
||||
for (let i = anc.length - 1; i >= 0; i--) if (isFnNode(anc[i])) return anc[i];
|
||||
return null;
|
||||
}
|
||||
|
||||
// nearest block or program ancestor
|
||||
function nearestBlockOrProgram() {
|
||||
const anc = context.getAncestors();
|
||||
for (let i = anc.length - 1; i >= 0; i--) {
|
||||
const a = anc[i];
|
||||
if (a?.type === 'BlockStatement' || a?.type === 'Program') return a;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// context is @asyncUnsafe if the function has the tag OR there is a tagged comment earlier in the same block/program
|
||||
function contextIsAnnotatedUnsafe(node) {
|
||||
const fn = nearestFn();
|
||||
if (fnHasTag(fn, opt.unsafeTag)) return true;
|
||||
|
||||
const blk = nearestBlockOrProgram();
|
||||
if (!blk) return false;
|
||||
const lead = src.getCommentsBefore(node) || [];
|
||||
return lead.some((c) => isCommentWith(c, opt.unsafeTag) && inside(c, blk) && before(c, node));
|
||||
}
|
||||
|
||||
// in try { ... } ?
|
||||
function inTryBlock(node) {
|
||||
const anc = context.getAncestors();
|
||||
for (let i = anc.length - 1; i >= 0; i--) {
|
||||
const a = anc[i];
|
||||
if (a?.type === 'TryStatement' && a.block && inside(node, a.block)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
const unwrapChain = (e) => (e && e.type === 'ChainExpression' ? e.expression : e);
|
||||
|
||||
function isHandledAwaitArg(node) {
|
||||
const arg = unwrapChain(node.argument);
|
||||
if (!arg) return false;
|
||||
|
||||
// await Promise.allSettled(...)
|
||||
if (
|
||||
opt.allowAllSettled &&
|
||||
arg.type === 'CallExpression' &&
|
||||
arg.callee?.type === 'MemberExpression' &&
|
||||
arg.callee.object?.type === 'Identifier' &&
|
||||
arg.callee.object.name === 'Promise' &&
|
||||
arg.callee.property?.type === 'Identifier' &&
|
||||
arg.callee.property.name === 'allSettled'
|
||||
) return true;
|
||||
|
||||
// await p.catch(...)
|
||||
if (
|
||||
opt.allowCatchMethod &&
|
||||
arg.type === 'CallExpression' &&
|
||||
arg.callee?.type === 'MemberExpression' &&
|
||||
arg.callee.property?.type === 'Identifier' &&
|
||||
arg.callee.property.name === 'catch'
|
||||
) return true;
|
||||
|
||||
// await p.then(onFulfilled, onRejected)
|
||||
if (
|
||||
opt.allowThenWithTwoArgs &&
|
||||
arg.type === 'CallExpression' &&
|
||||
arg.callee?.type === 'MemberExpression' &&
|
||||
arg.callee.property?.type === 'Identifier' &&
|
||||
arg.callee.property.name === 'then' &&
|
||||
Array.isArray(arg.arguments) &&
|
||||
arg.arguments.length >= 2
|
||||
) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// resolve identifier → local function def in scope (best-effort)
|
||||
function resolveFnFromIdentifier(id) {
|
||||
const name = id?.name;
|
||||
if (!name) return null;
|
||||
let scope = context.getScope();
|
||||
while (scope) {
|
||||
const v = (scope.set && scope.set.get(name)) || scope.variables?.find((vv) => vv.name === name);
|
||||
if (v && v.defs && v.defs.length) {
|
||||
for (const d of v.defs) {
|
||||
const dn = d.node;
|
||||
if (!dn) continue;
|
||||
if (dn.type === 'FunctionDeclaration') return dn;
|
||||
if (dn.type === 'VariableDeclarator') {
|
||||
const init = dn.init;
|
||||
if (init && (init.type === 'ArrowFunctionExpression' || init.type === 'FunctionExpression')) {
|
||||
return init;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
scope = scope.upper;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// typescript-powered resolution: member call callee → ts declarations → jsdoc tags
|
||||
function tsMemberIsAnnotatedSafe(memberExpr) {
|
||||
if (!ts || !checker || !es2ts) return false;
|
||||
const prop = memberExpr.property;
|
||||
if (!prop || prop.type !== 'Identifier') return false; // skip computed/strings rn
|
||||
|
||||
try {
|
||||
const tsObj = es2ts.get(unwrapChain(memberExpr.object));
|
||||
if (!tsObj) return false;
|
||||
let type = checker.getTypeAtLocation(tsObj);
|
||||
if (!type) return false;
|
||||
// normalize to apparent type (unions, etc.)
|
||||
const apparent = checker.getApparentType ? checker.getApparentType(type) : type;
|
||||
const name = prop.name;
|
||||
|
||||
// ts <5 vs >=5 api differences
|
||||
const sym = (apparent.getProperty && apparent.getProperty(name)) ||
|
||||
(checker.getPropertyOfType && checker.getPropertyOfType(apparent, name));
|
||||
if (!sym || !Array.isArray(sym.declarations)) return false;
|
||||
|
||||
for (const decl of sym.declarations) {
|
||||
// method, function, property with function type — accept any with @asyncSafe
|
||||
if (tsNodeHasJsDocTag(decl, opt.safeTag)) return true;
|
||||
// for class methods, also check the parent (sometimes the tag is on the signature)
|
||||
if (decl.parent && tsNodeHasJsDocTag(decl.parent, opt.safeTag)) return true;
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
return false;
|
||||
}
|
||||
|
||||
// estree-only: this.method() inside same class
|
||||
function thisMethodIsAnnotatedSafe(memberExpr) {
|
||||
if (memberExpr.object?.type !== 'ThisExpression') return false;
|
||||
const prop = memberExpr.property;
|
||||
if (!prop || prop.type !== 'Identifier') return false;
|
||||
const m = findMethodInCurrentClass(prop.name);
|
||||
return fnHasTag(m, opt.safeTag);
|
||||
}
|
||||
|
||||
function calleeIsAnnotatedSafe(callExpr) {
|
||||
const arg = unwrapChain(callExpr);
|
||||
if (!arg) return false;
|
||||
|
||||
if (arg.type === 'CallExpression') {
|
||||
const c = unwrapChain(arg.callee);
|
||||
if (!c) return false;
|
||||
|
||||
// direct identifier call
|
||||
if (c.type === 'Identifier') {
|
||||
const def = resolveFnFromIdentifier(c);
|
||||
if (fnHasTag(def, opt.safeTag)) return true;
|
||||
|
||||
// ts fallback for imported funcs
|
||||
if (ts && checker && es2ts) {
|
||||
try {
|
||||
const tsCallee = es2ts.get(c);
|
||||
const sym = checker.getSymbolAtLocation?.(tsCallee);
|
||||
const decls = sym?.declarations || [];
|
||||
for (const d of decls) {
|
||||
if (tsNodeHasJsDocTag(d, opt.safeTag) || (d.parent && tsNodeHasJsDocTag(d.parent, opt.safeTag))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
} catch { /* noop */ }
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// member call: this.m(), obj.m()
|
||||
if (c.type === 'MemberExpression') {
|
||||
// 1) easy path: this.method inside same class
|
||||
if (thisMethodIsAnnotatedSafe(c)) return true;
|
||||
|
||||
// 2) ts-powered cross-file/class/instance resolution
|
||||
if (tsMemberIsAnnotatedSafe(c)) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// function expressions / arrow directly inline
|
||||
if (c.type === 'FunctionExpression' || c.type === 'ArrowFunctionExpression') {
|
||||
return fnHasTag(c, opt.safeTag);
|
||||
}
|
||||
|
||||
// dynamic/new/etc → treat as unsafe
|
||||
return false;
|
||||
}
|
||||
|
||||
// awaiting a non-call promise value → treat as unsafe
|
||||
return false;
|
||||
}
|
||||
|
||||
// --- main ---------------------------------------------------------------
|
||||
return {
|
||||
AwaitExpression(node) {
|
||||
// handled patterns → ok
|
||||
if (inTryBlock(node) || isHandledAwaitArg(node)) return;
|
||||
|
||||
// callee carries @asyncSafe? → ok anywhere
|
||||
if (calleeIsAnnotatedSafe(node.argument)) return;
|
||||
|
||||
// context explicitly @asyncUnsafe? → ok to bubble
|
||||
if (contextIsAnnotatedUnsafe(node)) return;
|
||||
|
||||
// default: context is safe, callee is unsafe → error
|
||||
context.report({ node, messageId: 'unhandled' });
|
||||
},
|
||||
|
||||
// void someAsyncCall() — only allowed if callee is @asyncSafe
|
||||
UnaryExpression(node) {
|
||||
if (node.operator !== 'void') return;
|
||||
|
||||
const arg = unwrapChain(node.argument);
|
||||
if (!arg || arg.type !== 'CallExpression') return;
|
||||
|
||||
// callee carries @asyncSafe? → ok
|
||||
if (calleeIsAnnotatedSafe(node.argument)) return;
|
||||
|
||||
// void of non-safe callee → error
|
||||
context.report({ node, messageId: 'unhandledVoid' });
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
};
|
||||
|
|
@ -1,20 +1,24 @@
|
|||
import type { Config } from "@jest/types"
|
||||
import type { Config } from '@jest/types';
|
||||
|
||||
const config: Config.InitialOptions = {
|
||||
preset: "ts-jest",
|
||||
testEnvironment: "node",
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
verbose: true,
|
||||
automock: false,
|
||||
collectCoverage: true,
|
||||
collectCoverageFrom: ["./src/**/**.ts"],
|
||||
coverageProvider: "v8",
|
||||
collectCoverageFrom: ['./src/**/**.ts'],
|
||||
coverageProvider: 'v8',
|
||||
coverageThreshold: {
|
||||
global: {
|
||||
lines: 1
|
||||
}
|
||||
},
|
||||
setupFiles: [
|
||||
"./testSetup.ts",
|
||||
'./testSetup.ts',
|
||||
],
|
||||
}
|
||||
testPathIgnorePatterns: [
|
||||
'/node_modules/',
|
||||
'/__integration_tests__/',
|
||||
],
|
||||
};
|
||||
export default config;
|
||||
|
|
|
|||
21
backend/jest.integration.config.ts
Normal file
21
backend/jest.integration.config.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import type { Config } from '@jest/types';
|
||||
|
||||
const config: Config.InitialOptions = {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
verbose: true,
|
||||
automock: false,
|
||||
collectCoverage: false,
|
||||
coverageProvider: 'v8',
|
||||
testMatch: [
|
||||
'**/__integration_tests__/**/*.test.ts'
|
||||
],
|
||||
globalSetup: './jest.integration.setup.ts', // Start database before all tests
|
||||
setupFiles: [
|
||||
'./testSetup.integration.ts',
|
||||
],
|
||||
globalTeardown: './jest.integration.teardown.ts', // Stop database after all tests
|
||||
maxWorkers: 1, // Force sequential execution
|
||||
};
|
||||
export default config;
|
||||
|
||||
72
backend/jest.integration.setup.ts
Normal file
72
backend/jest.integration.setup.ts
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
// Setup that runs BEFORE setupFiles
|
||||
// This ensures MEMPOOL_CONFIG_FILE is set before any modules are loaded
|
||||
import * as path from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
|
||||
// Set the config file path if not already set
|
||||
if (!process.env.MEMPOOL_CONFIG_FILE) {
|
||||
process.env.MEMPOOL_CONFIG_FILE = path.join(__dirname, 'mempool-config.test.json');
|
||||
}
|
||||
|
||||
// Helper to get docker compose command (v1 or v2)
|
||||
function getDockerComposeCmd(): string {
|
||||
try {
|
||||
execSync('docker compose version', { stdio: 'pipe' });
|
||||
return 'docker compose';
|
||||
} catch {
|
||||
try {
|
||||
execSync('docker-compose version', { stdio: 'pipe' });
|
||||
return 'docker-compose';
|
||||
} catch {
|
||||
throw new Error('Neither "docker compose" nor "docker-compose" is available');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Start the Docker test database container
|
||||
module.exports = async () => {
|
||||
// Skip if SKIP_DB_SETUP is set (e.g., when test-with-db.sh manages the database)
|
||||
if (process.env.SKIP_DB_SETUP) {
|
||||
console.log('Skipping database setup (managed externally)');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('Starting test database container...');
|
||||
try {
|
||||
const composeFile = path.join(__dirname, 'docker-compose.test.yml');
|
||||
const dockerComposeCmd = getDockerComposeCmd();
|
||||
|
||||
// Start the container
|
||||
execSync(`${dockerComposeCmd} -f "${composeFile}" up -d`, {
|
||||
stdio: 'inherit',
|
||||
cwd: __dirname
|
||||
});
|
||||
|
||||
// Wait for database to be ready
|
||||
console.log('Waiting for database to be ready...');
|
||||
let attempts = 0;
|
||||
const maxAttempts = 30;
|
||||
|
||||
while (attempts < maxAttempts) {
|
||||
try {
|
||||
execSync(`${dockerComposeCmd} -f "${composeFile}" exec -T db-test mysqladmin ping -h localhost -u mempool_test -pmempool_test --silent`, {
|
||||
cwd: __dirname,
|
||||
stdio: 'pipe'
|
||||
});
|
||||
console.log('Database is ready!');
|
||||
break;
|
||||
} catch (e) {
|
||||
attempts++;
|
||||
if (attempts >= maxAttempts) {
|
||||
throw new Error('Database did not start in time');
|
||||
}
|
||||
// Wait 1 second before retrying
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to start test database:', error instanceof Error ? error.message : error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
102
backend/jest.integration.teardown.ts
Normal file
102
backend/jest.integration.teardown.ts
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
import DB from './src/database';
|
||||
import logger from './src/logger';
|
||||
import mempool from './src/api/mempool';
|
||||
import { execSync } from 'child_process';
|
||||
import * as path from 'path';
|
||||
|
||||
// Helper to get docker compose command (v1 or v2)
|
||||
function getDockerComposeCmd(): string {
|
||||
try {
|
||||
execSync('docker compose version', { stdio: 'pipe' });
|
||||
return 'docker compose';
|
||||
} catch {
|
||||
try {
|
||||
execSync('docker-compose version', { stdio: 'pipe' });
|
||||
return 'docker-compose';
|
||||
} catch {
|
||||
throw new Error('Neither "docker compose" nor "docker-compose" is available');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = async () => {
|
||||
try {
|
||||
// Final cleanup after all tests
|
||||
const tables = [
|
||||
'blocks_audits',
|
||||
'blocks_summaries',
|
||||
'blocks_prices',
|
||||
'blocks_templates',
|
||||
'cpfp_clusters',
|
||||
'blocks',
|
||||
'difficulty_adjustments',
|
||||
'hashrates',
|
||||
'prices',
|
||||
'node_records',
|
||||
'nodes_sockets',
|
||||
'nodes',
|
||||
'lightning_stats',
|
||||
'transactions',
|
||||
'elements_pegs',
|
||||
'federation_txos',
|
||||
'pools'
|
||||
];
|
||||
|
||||
await DB.query('SET FOREIGN_KEY_CHECKS = 0');
|
||||
|
||||
for (const table of tables) {
|
||||
try {
|
||||
// Use 'silent' error logging to avoid noise for optional tables that don't exist
|
||||
await DB.query(`TRUNCATE TABLE ${table}`, [], 'silent');
|
||||
} catch (e) {
|
||||
// Table might not exist - silently ignore
|
||||
}
|
||||
}
|
||||
|
||||
await DB.query('SET FOREIGN_KEY_CHECKS = 1');
|
||||
|
||||
logger.info('Integration tests cleanup completed');
|
||||
|
||||
// Close the database connection pool to prevent Jest from hanging
|
||||
await DB.close();
|
||||
logger.info('Database connection pool closed');
|
||||
|
||||
// Clean up singleton resources that have timers or sockets
|
||||
mempool.destroy();
|
||||
logger.info('Mempool resources cleaned up');
|
||||
|
||||
// Close logger's UDP socket last (after all logging is done)
|
||||
logger.close();
|
||||
|
||||
// Stop and remove the Docker test database container
|
||||
// Skip if SKIP_DB_TEARDOWN is set (e.g., when test-with-db.sh manages the database)
|
||||
if (!process.env.SKIP_DB_TEARDOWN) {
|
||||
try {
|
||||
const composeFile = path.join(__dirname, 'docker-compose.test.yml');
|
||||
const dockerComposeCmd = getDockerComposeCmd();
|
||||
execSync(`${dockerComposeCmd} -f "${composeFile}" down -v`, {
|
||||
stdio: 'inherit',
|
||||
cwd: __dirname
|
||||
});
|
||||
console.log('Test database container stopped and removed');
|
||||
} catch (error) {
|
||||
console.error('Failed to stop Docker container:', error instanceof Error ? error.message : error);
|
||||
}
|
||||
} else {
|
||||
console.log('Skipping Docker cleanup (managed externally)');
|
||||
}
|
||||
} catch (error) {
|
||||
// Use console.error since logger might be closed
|
||||
console.error('Failed to cleanup after integration tests:', error instanceof Error ? error.message : error);
|
||||
} finally {
|
||||
// Ensure we always try to close connections even if cleanup fails
|
||||
try {
|
||||
await DB.close();
|
||||
mempool.destroy();
|
||||
logger.close();
|
||||
} catch (e) {
|
||||
// Ignore errors on close
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
163
backend/mempool-config.test.json
Normal file
163
backend/mempool-config.test.json
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
{
|
||||
"MEMPOOL": {
|
||||
"OFFICIAL": false,
|
||||
"NETWORK": "mainnet",
|
||||
"BACKEND": "none",
|
||||
"ENABLED": true,
|
||||
"HTTP_PORT": 8998,
|
||||
"SPAWN_CLUSTER_PROCS": 0,
|
||||
"API_URL_PREFIX": "/api/v1/",
|
||||
"POLL_RATE_MS": 2000,
|
||||
"CACHE_DIR": "./cache",
|
||||
"CACHE_ENABLED": false,
|
||||
"CLEAR_PROTECTION_MINUTES": 20,
|
||||
"RECOMMENDED_FEE_PERCENTILE": 50,
|
||||
"BLOCK_WEIGHT_UNITS": 4000000,
|
||||
"INITIAL_BLOCKS_AMOUNT": 8,
|
||||
"MEMPOOL_BLOCKS_AMOUNT": 8,
|
||||
"INDEXING_BLOCKS_AMOUNT": 11000,
|
||||
"BLOCKS_SUMMARIES_INDEXING": false,
|
||||
"GOGGLES_INDEXING": false,
|
||||
"USE_SECOND_NODE_FOR_MINFEE": false,
|
||||
"EXTERNAL_ASSETS": [],
|
||||
"EXTERNAL_MAX_RETRY": 1,
|
||||
"EXTERNAL_RETRY_INTERVAL": 0,
|
||||
"USER_AGENT": "mempool",
|
||||
"STDOUT_LOG_MIN_PRIORITY": "debug",
|
||||
"AUTOMATIC_POOLS_UPDATE": false,
|
||||
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
|
||||
"POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
|
||||
"POOLS_UPDATE_DELAY": 604800,
|
||||
"AUDIT": false,
|
||||
"RUST_GBT": true,
|
||||
"LIMIT_GBT": false,
|
||||
"CPFP_INDEXING": false,
|
||||
"DISK_CACHE_BLOCK_INTERVAL": 6,
|
||||
"MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
|
||||
"ALLOW_UNREACHABLE": true,
|
||||
"PRICE_UPDATES_PER_HOUR": 1,
|
||||
"MAX_TRACKED_ADDRESSES": 100,
|
||||
"UNIX_SOCKET_PATH": ""
|
||||
},
|
||||
"CORE_RPC": {
|
||||
"HOST": "127.0.0.1",
|
||||
"PORT": 8332,
|
||||
"USERNAME": "mempool",
|
||||
"PASSWORD": "mempool",
|
||||
"TIMEOUT": 60000,
|
||||
"COOKIE": false,
|
||||
"COOKIE_PATH": "/path/to/bitcoin/.cookie",
|
||||
"DEBUG_LOG_PATH": "/path/to/bitcoin/debug.log"
|
||||
},
|
||||
"ELECTRUM": {
|
||||
"HOST": "127.0.0.1",
|
||||
"PORT": 50002,
|
||||
"TLS_ENABLED": true
|
||||
},
|
||||
"ESPLORA": {
|
||||
"REST_API_URL": "http://127.0.0.1:3000",
|
||||
"UNIX_SOCKET_PATH": "/tmp/esplora-bitcoin-mainnet",
|
||||
"BATCH_QUERY_BASE_SIZE": 1000,
|
||||
"RETRY_UNIX_SOCKET_AFTER": 30000,
|
||||
"REQUEST_TIMEOUT": 10000,
|
||||
"FALLBACK_TIMEOUT": 5000,
|
||||
"FALLBACK": [],
|
||||
"MAX_BEHIND_TIP": 2
|
||||
},
|
||||
"SECOND_CORE_RPC": {
|
||||
"HOST": "127.0.0.1",
|
||||
"PORT": 8332,
|
||||
"USERNAME": "mempool",
|
||||
"PASSWORD": "mempool",
|
||||
"TIMEOUT": 60000,
|
||||
"COOKIE": false,
|
||||
"COOKIE_PATH": "/path/to/bitcoin/.cookie"
|
||||
},
|
||||
"DATABASE": {
|
||||
"ENABLED": true,
|
||||
"HOST": "127.0.0.1",
|
||||
"PORT": 33306,
|
||||
"SOCKET": "",
|
||||
"DATABASE": "mempool_test",
|
||||
"USERNAME": "mempool_test",
|
||||
"PASSWORD": "mempool_test",
|
||||
"TIMEOUT": 180000,
|
||||
"PID_DIR": ""
|
||||
},
|
||||
"SYSLOG": {
|
||||
"ENABLED": false,
|
||||
"HOST": "127.0.0.1",
|
||||
"PORT": 514,
|
||||
"MIN_PRIORITY": "info",
|
||||
"FACILITY": "local7"
|
||||
},
|
||||
"STATISTICS": {
|
||||
"ENABLED": true,
|
||||
"TX_PER_SECOND_SAMPLE_PERIOD": 150
|
||||
},
|
||||
"MAXMIND": {
|
||||
"ENABLED": false,
|
||||
"GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
|
||||
"GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
|
||||
"GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
|
||||
},
|
||||
"LIGHTNING": {
|
||||
"ENABLED": false,
|
||||
"BACKEND": "lnd",
|
||||
"STATS_REFRESH_INTERVAL": 600,
|
||||
"GRAPH_REFRESH_INTERVAL": 600,
|
||||
"LOGGER_UPDATE_INTERVAL": 30,
|
||||
"FORENSICS_INTERVAL": 43200,
|
||||
"FORENSICS_RATE_LIMIT": 20
|
||||
},
|
||||
"LND": {
|
||||
"TLS_CERT_PATH": "tls.cert",
|
||||
"MACAROON_PATH": "readonly.macaroon",
|
||||
"REST_API_URL": "https://localhost:8080",
|
||||
"TIMEOUT": 10000
|
||||
},
|
||||
"CLIGHTNING": {
|
||||
"SOCKET": "lightning-rpc"
|
||||
},
|
||||
"SOCKS5PROXY": {
|
||||
"ENABLED": false,
|
||||
"USE_ONION": true,
|
||||
"HOST": "127.0.0.1",
|
||||
"PORT": 9050,
|
||||
"USERNAME": "",
|
||||
"PASSWORD": ""
|
||||
},
|
||||
"EXTERNAL_DATA_SERVER": {
|
||||
"MEMPOOL_API": "https://mempool.space/api/v1",
|
||||
"MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1",
|
||||
"LIQUID_API": "https://liquid.network/api/v1",
|
||||
"LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1"
|
||||
},
|
||||
"REDIS": {
|
||||
"ENABLED": false,
|
||||
"UNIX_SOCKET_PATH": "/tmp/redis.sock",
|
||||
"BATCH_QUERY_BASE_SIZE": 5000
|
||||
},
|
||||
"REPLICATION": {
|
||||
"ENABLED": false,
|
||||
"AUDIT": false,
|
||||
"AUDIT_START_HEIGHT": 774000,
|
||||
"STATISTICS": false,
|
||||
"STATISTICS_START_TIME": 1481932800,
|
||||
"SERVERS": []
|
||||
},
|
||||
"MEMPOOL_SERVICES": {
|
||||
"API": "https://mempool.space/api/v1/services",
|
||||
"ACCELERATIONS": false
|
||||
},
|
||||
"STRATUM": {
|
||||
"ENABLED": false,
|
||||
"API": "http://localhost:1234"
|
||||
},
|
||||
"FIAT_PRICE": {
|
||||
"ENABLED": false,
|
||||
"PAID": false,
|
||||
"API_KEY": ""
|
||||
}
|
||||
}
|
||||
|
||||
9312
backend/package-lock.json
generated
9312
backend/package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
|
@ -34,6 +34,8 @@
|
|||
"reindex-all-blocks": "npm run start-production --update-pools --reindex-blocks",
|
||||
"test": "./node_modules/.bin/jest --coverage",
|
||||
"test:ci": "CI=true ./node_modules/.bin/jest --coverage",
|
||||
"test:integration": "./node_modules/.bin/jest --config=jest.integration.config.ts --runInBand --forceExit",
|
||||
"test:with-db": "bash ./scripts/test-with-db.sh",
|
||||
"lint": "./node_modules/.bin/eslint . --ext .ts",
|
||||
"lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
|
||||
"prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
|
||||
|
|
@ -41,31 +43,36 @@
|
|||
"dependencies": {
|
||||
"@mempool/electrum-client": "1.1.9",
|
||||
"@types/node": "^18.15.3",
|
||||
"axios": "1.12.2",
|
||||
"axios": "1.13.5",
|
||||
"bitcoinjs-lib": "~6.1.3",
|
||||
"crypto-js": "~4.2.0",
|
||||
"express": "~4.21.1",
|
||||
"express": "~4.22.1",
|
||||
"maxmind": "~4.3.11",
|
||||
"mysql2": "~3.14.1",
|
||||
"mysql2": "~3.17.1",
|
||||
"rust-gbt": "file:./rust-gbt",
|
||||
"redis": "^4.7.0",
|
||||
"socks-proxy-agent": "~7.0.0",
|
||||
"typescript": "~4.9.3",
|
||||
"ws": "~8.18.0"
|
||||
"ws": "~8.19.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/compression": "^1.7.2",
|
||||
"@types/crypto-js": "^4.1.1",
|
||||
"@types/express": "^4.17.17",
|
||||
"@types/jest": "^29.5.0",
|
||||
"@types/ws": "~8.5.10",
|
||||
"@types/jest": "^30.0.0",
|
||||
"@types/ws": "~8.18.1",
|
||||
"@typescript-eslint/eslint-plugin": "^5.55.0",
|
||||
"@typescript-eslint/parser": "^5.55.0",
|
||||
"eslint": "^8.36.0",
|
||||
"eslint-config-prettier": "^8.8.0",
|
||||
"jest": "^29.5.0",
|
||||
"eslint-plugin-local-rules": "^3.0.2",
|
||||
"jest": "^30.0.0",
|
||||
"prettier": "^3.0.0",
|
||||
"ts-jest": "^29.1.1",
|
||||
"ts-jest": "^29.4.5",
|
||||
"ts-node": "^10.9.1"
|
||||
},
|
||||
"overrides": {
|
||||
"js-yaml": "^4.1.1",
|
||||
"glob": "^11.1.0"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
40
backend/scripts/debug-integration-tests.sh
Executable file
40
backend/scripts/debug-integration-tests.sh
Executable file
|
|
@ -0,0 +1,40 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Script to debug integration tests
|
||||
# Usage: ./scripts/debug-integration-tests.sh [test-file-pattern]
|
||||
|
||||
set -e
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
# Get the directory of this script
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
BACKEND_DIR="$( cd "$SCRIPT_DIR/.." && pwd )"
|
||||
|
||||
cd "$BACKEND_DIR"
|
||||
|
||||
# Check if MariaDB is running
|
||||
if ! docker-compose -f docker-compose.test.yml ps | grep -q "Up"; then
|
||||
echo -e "${YELLOW}Starting MariaDB container...${NC}"
|
||||
docker-compose -f docker-compose.test.yml up -d
|
||||
|
||||
# Wait for MariaDB
|
||||
echo -e "${YELLOW}Waiting for MariaDB...${NC}"
|
||||
sleep 5
|
||||
fi
|
||||
|
||||
# Run tests with pattern if provided
|
||||
if [ -n "$1" ]; then
|
||||
echo -e "${GREEN}Running tests matching: $1${NC}"
|
||||
MEMPOOL_CONFIG_FILE="$BACKEND_DIR/mempool-config.test.json" npx jest --config=jest.integration.config.ts --runInBand --verbose "$1"
|
||||
else
|
||||
echo -e "${GREEN}Running all integration tests${NC}"
|
||||
MEMPOOL_CONFIG_FILE="$BACKEND_DIR/mempool-config.test.json" npm run test:integration
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}Tests completed${NC}"
|
||||
|
||||
111
backend/scripts/test-with-db.sh
Executable file
111
backend/scripts/test-with-db.sh
Executable file
|
|
@ -0,0 +1,111 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo -e "${GREEN}Starting integration tests with MariaDB...${NC}"
|
||||
|
||||
# Get the directory of this script
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
BACKEND_DIR="$( cd "$SCRIPT_DIR/.." && pwd )"
|
||||
|
||||
cd "$BACKEND_DIR"
|
||||
|
||||
# Detect docker compose command (v1 or v2)
|
||||
if docker compose version &> /dev/null; then
|
||||
DOCKER_COMPOSE="docker compose"
|
||||
elif docker-compose version &> /dev/null; then
|
||||
DOCKER_COMPOSE="docker-compose"
|
||||
else
|
||||
echo -e "${RED}Error: Neither 'docker compose' nor 'docker-compose' is available${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${YELLOW}Using: ${DOCKER_COMPOSE}${NC}"
|
||||
|
||||
# Function to cleanup on exit
|
||||
cleanup() {
|
||||
echo -e "\n${YELLOW}Cleaning up...${NC}"
|
||||
# Kill the backend server if it's running
|
||||
if [ ! -z "$SERVER_PID" ]; then
|
||||
kill $SERVER_PID 2>/dev/null || true
|
||||
fi
|
||||
# Stop containers and remove volumes, but don't fail on network errors
|
||||
$DOCKER_COMPOSE -f docker-compose.test.yml down -v 2>&1 | grep -v "Resource is still in use" || true
|
||||
}
|
||||
|
||||
# Set trap to cleanup on exit
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
# Stop any existing test containers
|
||||
echo -e "${YELLOW}Stopping any existing test containers...${NC}"
|
||||
$DOCKER_COMPOSE -f docker-compose.test.yml down -v 2>/dev/null || true
|
||||
|
||||
# Start MariaDB container
|
||||
echo -e "${GREEN}Starting MariaDB container...${NC}"
|
||||
$DOCKER_COMPOSE -f docker-compose.test.yml up -d
|
||||
|
||||
# Wait for MariaDB to be ready
|
||||
echo -e "${YELLOW}Waiting for MariaDB to be ready...${NC}"
|
||||
max_attempts=30
|
||||
attempt=0
|
||||
while [ $attempt -lt $max_attempts ]; do
|
||||
if $DOCKER_COMPOSE -f docker-compose.test.yml exec -T db-test mysqladmin ping -h localhost -u mempool_test -pmempool_test --silent 2>/dev/null; then
|
||||
echo -e "${GREEN}MariaDB is ready!${NC}"
|
||||
break
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
echo -e "${YELLOW}Attempt $attempt/$max_attempts - waiting for MariaDB...${NC}"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if [ $attempt -eq $max_attempts ]; then
|
||||
echo -e "${RED}MariaDB did not start in time${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Additional wait to ensure MariaDB is fully initialized
|
||||
sleep 3
|
||||
|
||||
# Build the backend
|
||||
echo -e "${GREEN}Building backend...${NC}"
|
||||
npm run build
|
||||
|
||||
# Run integration tests with absolute path to config
|
||||
# SKIP_DB_SETUP=1 and SKIP_DB_TEARDOWN=1 tell Jest that we're managing the database lifecycle
|
||||
echo -e "${GREEN}Running integration tests...${NC}"
|
||||
export MEMPOOL_CONFIG_FILE="$BACKEND_DIR/mempool-config.test.json"
|
||||
export SKIP_DB_SETUP=1
|
||||
export SKIP_DB_TEARDOWN=1
|
||||
npm run test:integration
|
||||
|
||||
# Start the backend server in the background
|
||||
# MEMPOOL_CONFIG_FILE is already exported above
|
||||
echo -e "${GREEN}Starting backend server...${NC}"
|
||||
node dist/index.js &
|
||||
SERVER_PID=$!
|
||||
|
||||
# Wait for server to start and verify connection
|
||||
echo -e "${YELLOW}Waiting for server to start and connect to database...${NC}"
|
||||
sleep 10
|
||||
|
||||
# Check if server is still running
|
||||
if ps -p $SERVER_PID > /dev/null 2>&1; then
|
||||
echo -e "${GREEN}Server started successfully and connected to database!${NC}"
|
||||
|
||||
# Kill the server (it will be in the cleanup function too, but do it here as well)
|
||||
kill $SERVER_PID 2>/dev/null || true
|
||||
wait $SERVER_PID 2>/dev/null || true
|
||||
SERVER_PID=""
|
||||
else
|
||||
echo -e "${RED}Server failed to start or exited prematurely${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}All tests passed successfully!${NC}"
|
||||
|
||||
134
backend/src/__integration_tests__/blocks-repository.test.ts
Normal file
134
backend/src/__integration_tests__/blocks-repository.test.ts
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
import BlocksRepository from '../repositories/BlocksRepository';
|
||||
import { setupTestDatabase, waitForDatabase, cleanupTestData, insertTestPool, insertTestBlock } from './test-helpers';
|
||||
|
||||
describe('BlocksRepository Integration Tests', () => {
|
||||
let defaultPoolId: number;
|
||||
|
||||
beforeAll(async () => {
|
||||
await waitForDatabase();
|
||||
await setupTestDatabase();
|
||||
}, 120000);
|
||||
|
||||
beforeEach(async () => {
|
||||
await cleanupTestData();
|
||||
// Create a default pool for all blocks
|
||||
defaultPoolId = await insertTestPool({
|
||||
name: 'Unknown',
|
||||
slug: 'unknown',
|
||||
addresses: '[]',
|
||||
regexes: '[]'
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await cleanupTestData();
|
||||
});
|
||||
|
||||
test('should insert and retrieve a block', async () => {
|
||||
const blockHash = '00000000000000000001a0e3e9b2e6d6e8a4b0e9b2e6d6e8a4b0e9b2e6d6e8a4';
|
||||
const height = 800000;
|
||||
|
||||
await insertTestBlock({
|
||||
height: height,
|
||||
hash: blockHash,
|
||||
blockTimestamp: new Date('2023-07-16T00:00:00Z'),
|
||||
size: 1500000,
|
||||
weight: 3999000,
|
||||
tx_count: 3000,
|
||||
difficulty: 53911173001054.59,
|
||||
poolId: defaultPoolId
|
||||
});
|
||||
|
||||
const block = await BlocksRepository.$getBlockByHeight(height);
|
||||
|
||||
expect(block).toBeDefined();
|
||||
expect(block!.height).toBe(height);
|
||||
expect(block!.id).toBe(blockHash);
|
||||
});
|
||||
|
||||
test('should get block by hash', async () => {
|
||||
const blockHash = '00000000000000000002b0e3e9b2e6d6e8a4b0e9b2e6d6e8a4b0e9b2e6d6e8a4';
|
||||
const height = 800001;
|
||||
|
||||
await insertTestBlock({
|
||||
height: height,
|
||||
hash: blockHash,
|
||||
tx_count: 2500,
|
||||
poolId: defaultPoolId
|
||||
});
|
||||
|
||||
const block = await BlocksRepository.$getBlockByHash(blockHash);
|
||||
|
||||
expect(block).toBeDefined();
|
||||
expect(block!.id).toBe(blockHash);
|
||||
expect(block!.height).toBe(height);
|
||||
});
|
||||
|
||||
test('should handle non-existent block', async () => {
|
||||
const block = await BlocksRepository.$getBlockByHeight(999999);
|
||||
expect(block).toBeNull();
|
||||
});
|
||||
|
||||
test('should check for missing blocks in range', async () => {
|
||||
// Insert blocks with a gap
|
||||
await insertTestBlock({
|
||||
height: 800100,
|
||||
hash: '0000000000000000000100000000000000000000000000000000000000000001',
|
||||
poolId: defaultPoolId
|
||||
});
|
||||
await insertTestBlock({
|
||||
height: 800102,
|
||||
hash: '0000000000000000000100000000000000000000000000000000000000000003',
|
||||
poolId: defaultPoolId
|
||||
});
|
||||
|
||||
const missingBlocks = await BlocksRepository.$getMissingBlocksBetweenHeights(800100, 800102);
|
||||
|
||||
expect(missingBlocks).toContain(800101);
|
||||
});
|
||||
|
||||
test('should get latest block height', async () => {
|
||||
await insertTestBlock({
|
||||
height: 800200,
|
||||
hash: '0000000000000000000200000000000000000000000000000000000000000001',
|
||||
poolId: defaultPoolId
|
||||
});
|
||||
await insertTestBlock({
|
||||
height: 800201,
|
||||
hash: '0000000000000000000200000000000000000000000000000000000000000002',
|
||||
poolId: defaultPoolId
|
||||
});
|
||||
|
||||
const height = await BlocksRepository.$mostRecentBlockHeight();
|
||||
|
||||
expect(height).toBe(800201);
|
||||
});
|
||||
|
||||
test('should handle block with pool association', async () => {
|
||||
// Insert a pool and get its auto-generated ID
|
||||
const testPoolId = await insertTestPool({
|
||||
name: 'Test Pool',
|
||||
slug: 'test-pool',
|
||||
addresses: '[]',
|
||||
regexes: '[]'
|
||||
});
|
||||
|
||||
const blockHash = '0000000000000000000300000000000000000000000000000000000000000001';
|
||||
await insertTestBlock({
|
||||
height: 800300,
|
||||
hash: blockHash,
|
||||
poolId: testPoolId
|
||||
});
|
||||
|
||||
const block = await BlocksRepository.$getBlockByHash(blockHash);
|
||||
|
||||
expect(block).toBeDefined();
|
||||
expect(block).not.toBeNull();
|
||||
// The pool should be populated with the test pool's data
|
||||
if (block && block.extras?.pool) {
|
||||
expect(block.extras.pool.name).toBe('Test Pool');
|
||||
expect(block.extras.pool.slug).toBe('test-pool');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
import DB from '../database';
|
||||
import config from '../config';
|
||||
import { setupTestDatabase, waitForDatabase, getTestDatabaseConfig } from './test-helpers';
|
||||
|
||||
describe('Database Connection Integration Tests', () => {
|
||||
beforeAll(async () => {
|
||||
// Wait for database to be ready
|
||||
await waitForDatabase();
|
||||
}, 60000);
|
||||
|
||||
test('should connect to the test database', async () => {
|
||||
const dbConfig = getTestDatabaseConfig();
|
||||
expect(dbConfig.enabled).toBe(true);
|
||||
expect(dbConfig.database).toBe('mempool_test');
|
||||
expect(dbConfig.port).toBe(33306);
|
||||
});
|
||||
|
||||
test('should execute a simple query', async () => {
|
||||
const [result] = await DB.query<any>('SELECT 1 as value');
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].value).toBe(1);
|
||||
});
|
||||
|
||||
test('should execute a query with parameters', async () => {
|
||||
const [result] = await DB.query<any>('SELECT ? as sum', [42]);
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].sum).toBe(42);
|
||||
});
|
||||
|
||||
test('should check database connection', async () => {
|
||||
await expect(DB.checkDbConnection()).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
test('should handle query timeout configuration', async () => {
|
||||
expect(config.DATABASE.TIMEOUT).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('should have correct database configuration', () => {
|
||||
expect(config.DATABASE.HOST).toBe('127.0.0.1');
|
||||
expect(config.DATABASE.USERNAME).toBe('mempool_test');
|
||||
expect(config.DATABASE.PASSWORD).toBe('mempool_test');
|
||||
expect(config.DATABASE.DATABASE).toBe('mempool_test');
|
||||
});
|
||||
});
|
||||
|
||||
97
backend/src/__integration_tests__/database-migration.test.ts
Normal file
97
backend/src/__integration_tests__/database-migration.test.ts
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
import DB from '../database';
|
||||
import { setupTestDatabase, waitForDatabase } from './test-helpers';
|
||||
|
||||
describe('Database Migration Integration Tests', () => {
|
||||
beforeAll(async () => {
|
||||
await waitForDatabase();
|
||||
await setupTestDatabase();
|
||||
}, 120000);
|
||||
|
||||
test('should create state table', async () => {
|
||||
const [result] = await DB.query<any>(
|
||||
`SELECT COUNT(*) as count
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'mempool_test'
|
||||
AND table_name = 'state'`
|
||||
);
|
||||
expect(result[0].count).toBe(1);
|
||||
});
|
||||
|
||||
test('should have schema version in state table', async () => {
|
||||
const [result] = await DB.query<any>('SELECT number FROM state WHERE name = \'schema_version\'');
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].number).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('should create blocks table', async () => {
|
||||
const [result] = await DB.query<any>(
|
||||
`SELECT COUNT(*) as count
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'mempool_test'
|
||||
AND table_name = 'blocks'`
|
||||
);
|
||||
expect(result[0].count).toBe(1);
|
||||
});
|
||||
|
||||
test('should create pools table', async () => {
|
||||
const [result] = await DB.query<any>(
|
||||
`SELECT COUNT(*) as count
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'mempool_test'
|
||||
AND table_name = 'pools'`
|
||||
);
|
||||
expect(result[0].count).toBe(1);
|
||||
});
|
||||
|
||||
test('should create hashrates table', async () => {
|
||||
const [result] = await DB.query<any>(
|
||||
`SELECT COUNT(*) as count
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'mempool_test'
|
||||
AND table_name = 'hashrates'`
|
||||
);
|
||||
expect(result[0].count).toBe(1);
|
||||
});
|
||||
|
||||
test('should create prices table', async () => {
|
||||
const [result] = await DB.query<any>(
|
||||
`SELECT COUNT(*) as count
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'mempool_test'
|
||||
AND table_name = 'prices'`
|
||||
);
|
||||
expect(result[0].count).toBe(1);
|
||||
});
|
||||
|
||||
test('blocks table should have required columns', async () => {
|
||||
const [columns] = await DB.query<any>(
|
||||
`SELECT COLUMN_NAME
|
||||
FROM information_schema.COLUMNS
|
||||
WHERE TABLE_SCHEMA = 'mempool_test'
|
||||
AND TABLE_NAME = 'blocks'`
|
||||
);
|
||||
|
||||
const columnNames = columns.map((col: any) => col.COLUMN_NAME);
|
||||
expect(columnNames).toContain('height');
|
||||
expect(columnNames).toContain('hash');
|
||||
expect(columnNames).toContain('blockTimestamp');
|
||||
expect(columnNames).toContain('size');
|
||||
expect(columnNames).toContain('weight');
|
||||
expect(columnNames).toContain('tx_count');
|
||||
});
|
||||
|
||||
test('pools table should have required columns', async () => {
|
||||
const [columns] = await DB.query<any>(
|
||||
`SELECT COLUMN_NAME
|
||||
FROM information_schema.COLUMNS
|
||||
WHERE TABLE_SCHEMA = 'mempool_test'
|
||||
AND TABLE_NAME = 'pools'`
|
||||
);
|
||||
|
||||
const columnNames = columns.map((col: any) => col.COLUMN_NAME);
|
||||
expect(columnNames).toContain('id');
|
||||
expect(columnNames).toContain('name');
|
||||
expect(columnNames).toContain('slug');
|
||||
});
|
||||
});
|
||||
|
||||
122
backend/src/__integration_tests__/pools-repository.test.ts
Normal file
122
backend/src/__integration_tests__/pools-repository.test.ts
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
import PoolsRepository from '../repositories/PoolsRepository';
|
||||
import { setupTestDatabase, waitForDatabase, cleanupTestData, insertTestPool } from './test-helpers';
|
||||
|
||||
describe('PoolsRepository Integration Tests', () => {
|
||||
beforeAll(async () => {
|
||||
await waitForDatabase();
|
||||
await setupTestDatabase();
|
||||
}, 120000);
|
||||
|
||||
beforeEach(async () => {
|
||||
await cleanupTestData();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await cleanupTestData();
|
||||
});
|
||||
|
||||
test('should insert and retrieve a pool', async () => {
|
||||
const poolData = {
|
||||
name: 'Foundry USA',
|
||||
slug: 'foundryusa',
|
||||
link: 'https://foundrydigital.com',
|
||||
addresses: JSON.stringify(['bc1qxhmdufsvnuaaaer4ynz88fspdsxq2h9e9cetdj']),
|
||||
regexes: JSON.stringify(['/Foundry USA Pool/'])
|
||||
};
|
||||
|
||||
const poolId = await insertTestPool(poolData);
|
||||
expect(poolId).toBeGreaterThan(0);
|
||||
|
||||
const pools = await PoolsRepository.$getPools();
|
||||
const insertedPool = pools.find(p => p.id === poolId);
|
||||
|
||||
expect(insertedPool).toBeDefined();
|
||||
expect(insertedPool?.name).toBe(poolData.name);
|
||||
expect(insertedPool?.slug).toBe(poolData.slug);
|
||||
});
|
||||
|
||||
test('should get pool by slug', async () => {
|
||||
await insertTestPool({
|
||||
name: 'AntPool',
|
||||
slug: 'antpool',
|
||||
link: 'https://antpool.com'
|
||||
});
|
||||
|
||||
const pool = await PoolsRepository.$getPool('antpool');
|
||||
|
||||
expect(pool).toBeDefined();
|
||||
expect(pool!.name).toBe('AntPool');
|
||||
expect(pool!.slug).toBe('antpool');
|
||||
});
|
||||
|
||||
test('should get all pools', async () => {
|
||||
await insertTestPool({
|
||||
name: 'Pool 1',
|
||||
slug: 'pool-1'
|
||||
});
|
||||
await insertTestPool({
|
||||
name: 'Pool 2',
|
||||
slug: 'pool-2'
|
||||
});
|
||||
await insertTestPool({
|
||||
name: 'Pool 3',
|
||||
slug: 'pool-3'
|
||||
});
|
||||
|
||||
const pools = await PoolsRepository.$getPools();
|
||||
|
||||
expect(pools.length).toBeGreaterThanOrEqual(3);
|
||||
});
|
||||
|
||||
test('should handle pool with addresses', async () => {
|
||||
const addresses = ['bc1qtest1', 'bc1qtest2', '3TestAddress'];
|
||||
await insertTestPool({
|
||||
name: 'Multi Address Pool',
|
||||
slug: 'multi-address-pool',
|
||||
addresses: JSON.stringify(addresses)
|
||||
});
|
||||
|
||||
const pool = await PoolsRepository.$getPool('multi-address-pool', false);
|
||||
|
||||
expect(pool).toBeDefined();
|
||||
const poolAddresses = JSON.parse(pool!.addresses);
|
||||
expect(poolAddresses).toHaveLength(3);
|
||||
expect(poolAddresses).toContain('bc1qtest1');
|
||||
});
|
||||
|
||||
test('should handle pool with regexes', async () => {
|
||||
const regexes = ['/Pool Name/', '/Alternative Name/'];
|
||||
await insertTestPool({
|
||||
name: 'Regex Pool',
|
||||
slug: 'regex-pool',
|
||||
regexes: JSON.stringify(regexes)
|
||||
});
|
||||
|
||||
const pool = await PoolsRepository.$getPool('regex-pool', false);
|
||||
|
||||
expect(pool).toBeDefined();
|
||||
const poolRegexes = JSON.parse(pool!.regexes);
|
||||
expect(poolRegexes).toHaveLength(2);
|
||||
expect(poolRegexes[0]).toBe('/Pool Name/');
|
||||
});
|
||||
|
||||
test('should handle non-existent pool', async () => {
|
||||
const pool = await PoolsRepository.$getPool('non-existent-pool-slug');
|
||||
expect(pool).toBeNull();
|
||||
});
|
||||
|
||||
test('should update pool information', async () => {
|
||||
const poolDbId = await insertTestPool({
|
||||
name: 'Original Pool Name',
|
||||
slug: 'original-pool'
|
||||
});
|
||||
|
||||
// Update the pool name
|
||||
await PoolsRepository.$renameMiningPool(poolDbId, 'updated-pool', 'Updated Pool Name');
|
||||
|
||||
const pool = await PoolsRepository.$getPool('updated-pool');
|
||||
expect(pool).toBeDefined();
|
||||
expect(pool!.name).toBe('Updated Pool Name');
|
||||
});
|
||||
});
|
||||
|
||||
190
backend/src/__integration_tests__/test-helpers.ts
Normal file
190
backend/src/__integration_tests__/test-helpers.ts
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
import DB from '../database';
|
||||
import config from '../config';
|
||||
import logger from '../logger';
|
||||
import databaseMigration from '../api/database-migration';
|
||||
|
||||
/**
|
||||
* Initialize the test database with schema migrations
|
||||
*/
|
||||
export async function setupTestDatabase(): Promise<void> {
|
||||
try {
|
||||
await DB.checkDbConnection();
|
||||
await databaseMigration.$initializeOrMigrateDatabase();
|
||||
} catch (error) {
|
||||
logger.err('Failed to setup test database: ' + (error instanceof Error ? error.message : error));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up all data from test tables (but preserve schema)
|
||||
* This runs between each test to ensure isolation
|
||||
*/
|
||||
export async function cleanupTestData(): Promise<void> {
|
||||
// Order matters: delete child tables before parent tables
|
||||
const tables = [
|
||||
'blocks_audits',
|
||||
'blocks_summaries',
|
||||
'blocks_prices',
|
||||
'blocks_templates',
|
||||
'cpfp_clusters',
|
||||
'blocks', // blocks references pools
|
||||
'difficulty_adjustments',
|
||||
'hashrates',
|
||||
'prices',
|
||||
'node_records',
|
||||
'nodes_sockets',
|
||||
'nodes',
|
||||
'lightning_stats',
|
||||
'transactions',
|
||||
'elements_pegs',
|
||||
'federation_txos',
|
||||
'pools'
|
||||
];
|
||||
|
||||
try {
|
||||
// Disable foreign key checks temporarily for faster cleanup
|
||||
await DB.query('SET FOREIGN_KEY_CHECKS = 0');
|
||||
|
||||
for (const table of tables) {
|
||||
try {
|
||||
// Use 'silent' error logging to avoid noise for optional tables that don't exist
|
||||
await DB.query(`TRUNCATE TABLE ${table}`, [], 'silent');
|
||||
} catch (e) {
|
||||
// Table might not exist, that's okay for optional tables
|
||||
// Silently ignore - no need to log since these are expected for optional features
|
||||
}
|
||||
}
|
||||
|
||||
// Re-enable foreign key checks
|
||||
await DB.query('SET FOREIGN_KEY_CHECKS = 1');
|
||||
} catch (error) {
|
||||
// Try to re-enable foreign keys even if cleanup failed
|
||||
try {
|
||||
await DB.query('SET FOREIGN_KEY_CHECKS = 1');
|
||||
} catch (e) {
|
||||
// Ignore
|
||||
}
|
||||
logger.err('Failed to cleanup test data: ' + (error instanceof Error ? error.message : error));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for database to be ready
|
||||
*/
|
||||
export async function waitForDatabase(maxRetries = 30, retryInterval = 1000): Promise<void> {
|
||||
for (let i = 0; i < maxRetries; i++) {
|
||||
try {
|
||||
await DB.query('SELECT 1');
|
||||
logger.info('Database is ready');
|
||||
return;
|
||||
} catch (error) {
|
||||
logger.debug(`Waiting for database... attempt ${i + 1}/${maxRetries}`);
|
||||
await new Promise(resolve => setTimeout(resolve, retryInterval));
|
||||
}
|
||||
}
|
||||
throw new Error('Database did not become ready in time');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get database configuration for tests
|
||||
*/
|
||||
export function getTestDatabaseConfig() {
|
||||
return {
|
||||
host: config.DATABASE.HOST,
|
||||
port: config.DATABASE.PORT,
|
||||
database: config.DATABASE.DATABASE,
|
||||
username: config.DATABASE.USERNAME,
|
||||
enabled: config.DATABASE.ENABLED
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert a test pool into the database
|
||||
*/
|
||||
export async function insertTestPool(poolData: {
|
||||
id?: number;
|
||||
name: string;
|
||||
link?: string;
|
||||
slug: string;
|
||||
addresses?: string;
|
||||
regexes?: string;
|
||||
}) {
|
||||
const [result] = await DB.query<any>(
|
||||
`INSERT INTO pools (unique_id, name, link, slug, addresses, regexes)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`,
|
||||
[
|
||||
poolData.id || -1,
|
||||
poolData.name,
|
||||
poolData.link || '',
|
||||
poolData.slug,
|
||||
poolData.addresses || '[]',
|
||||
poolData.regexes || '[]'
|
||||
]
|
||||
);
|
||||
return result.insertId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert a test block into the database
|
||||
*/
|
||||
export async function insertTestBlock(blockData: {
|
||||
height: number;
|
||||
hash: string;
|
||||
blockTimestamp?: Date;
|
||||
size?: number;
|
||||
weight?: number;
|
||||
tx_count?: number;
|
||||
difficulty?: number;
|
||||
poolId?: number | null;
|
||||
}) {
|
||||
const timestamp = blockData.blockTimestamp || new Date();
|
||||
const size = blockData.size || 1000000;
|
||||
const weight = blockData.weight || 4000000;
|
||||
const txCount = blockData.tx_count || 2000;
|
||||
|
||||
await DB.query(
|
||||
`INSERT INTO blocks (
|
||||
height, hash, blockTimestamp, size, weight, tx_count,
|
||||
difficulty, pool_id, version, bits, nonce, merkle_root,
|
||||
previous_block_hash, median_timestamp, stale,
|
||||
fees, fee_span, median_fee,
|
||||
avg_tx_size, total_inputs, total_outputs, total_output_amt,
|
||||
segwit_total_txs, segwit_total_size, segwit_total_weight,
|
||||
header, utxoset_change
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
[
|
||||
blockData.height,
|
||||
blockData.hash,
|
||||
timestamp,
|
||||
size,
|
||||
weight,
|
||||
txCount,
|
||||
blockData.difficulty || 1.0,
|
||||
blockData.poolId !== undefined ? blockData.poolId : null,
|
||||
0x20000000,
|
||||
0x1d00ffff,
|
||||
0,
|
||||
'0000000000000000000000000000000000000000000000000000000000000000',
|
||||
'0000000000000000000000000000000000000000000000000000000000000000',
|
||||
timestamp,
|
||||
0, // stale = false
|
||||
// Required fields with defaults
|
||||
50000000, // fees (in sats)
|
||||
JSON.stringify([0, 0, 0, 0, 0, 0, 0]), // fee_span (JSON array)
|
||||
10000, // median_fee (in sats)
|
||||
size / txCount, // avg_tx_size
|
||||
txCount * 2, // total_inputs (estimate)
|
||||
txCount * 2, // total_outputs (estimate)
|
||||
2100000000000000, // total_output_amt (21M BTC in sats, estimate)
|
||||
txCount, // segwit_total_txs (assume all segwit for test)
|
||||
size, // segwit_total_size
|
||||
weight, // segwit_total_weight
|
||||
'00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', // header (160 chars)
|
||||
0 // utxoset_change
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -30,7 +30,7 @@ describe('Common', () => {
|
|||
expect(Common.isNonStandard(tx)).toEqual(true);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
test('should not misclassify as nonstandard transactions', () => {
|
||||
randomTransactions.forEach((tx) => {
|
||||
expect(Common.isNonStandard(tx)).toEqual(false);
|
||||
|
|
|
|||
|
|
@ -1,52 +1,55 @@
|
|||
[
|
||||
{
|
||||
"txid": "50136231cb7eeeffb17fc41d1cca213426abe5bf3760e3d6421cad0c0edad367",
|
||||
"version": 1,
|
||||
"locktime": 0,
|
||||
"vin": [
|
||||
{
|
||||
"txid": "c7f86fb7b830124057475b282809f3474ef3565daa3de0b599980fb9e84ab019",
|
||||
"vout": 4217,
|
||||
"prevout": {
|
||||
"scriptpubkey": "001466197b5eadd8067ec194a457e1044b6d1fbdd3b3",
|
||||
"scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 66197b5eadd8067ec194a457e1044b6d1fbdd3b3",
|
||||
"scriptpubkey_type": "v0_p2wpkh",
|
||||
"scriptpubkey_address": "bc1qvcvhkh4dmqr8asv553t7zpztd50mm5ang4na33",
|
||||
"value": 106
|
||||
},
|
||||
"scriptsig": "",
|
||||
"scriptsig_asm": "",
|
||||
"witness": [
|
||||
"3043021f2af6060a142c6cfd7428adad6a50745d2424813d7ced5c0bbcca85e70de1be022021440ca1c8c3ed49ecd1b64dca6911adcd430c5d3dd60d77ffe0072953999f5b01",
|
||||
"02ead5c34e3d2c506574b562f857576e11380b6ba15d9f0ad7b7303fdaa9c1513d"
|
||||
],
|
||||
"is_coinbase": false,
|
||||
"sequence": 4294967295
|
||||
}
|
||||
],
|
||||
"vout": [
|
||||
{
|
||||
"scriptpubkey": "6a023a29",
|
||||
"scriptpubkey_asm": "OP_RETURN OP_PUSHBYTES_2 3a29",
|
||||
"scriptpubkey_type": "op_return",
|
||||
"value": 0
|
||||
{
|
||||
"txid": "f859a4e6692c3f15a279449eac191ecb9d17d2d6c003bbae9a55e5da07147eab",
|
||||
"version": 1,
|
||||
"locktime": 0,
|
||||
"size": 170,
|
||||
"weight": 371,
|
||||
"fee": 11586,
|
||||
"vin": [
|
||||
{
|
||||
"is_coinbase": false,
|
||||
"prevout": {
|
||||
"value": 11586,
|
||||
"scriptpubkey": "512088939b42f2ed113dd4756055b179cd784c2cfa52cad29046411c28335aa2055c",
|
||||
"scriptpubkey_address": "bc1p3zfekshja5gnm4r4vp2mz7wd0pxze7jjetffq3jprs5rxk4zq4wqesykl2",
|
||||
"scriptpubkey_asm": "OP_PUSHNUM_1 OP_PUSHBYTES_32 88939b42f2ed113dd4756055b179cd784c2cfa52cad29046411c28335aa2055c",
|
||||
"scriptpubkey_type": "v1_p2tr"
|
||||
},
|
||||
"scriptsig": "",
|
||||
"scriptsig_asm": "",
|
||||
"sequence": 4294967295,
|
||||
"txid": "9b93dab94a3334b6119f75d107da6bcef9435fa52ec6c0de14aa8a094794551a",
|
||||
"vout": 0,
|
||||
"witness": [
|
||||
"6840b6fa27a00ba001cc92797ce4f3ab7b7a32c21d1fce49e893b42e506bd92e8db187966a84ef799915cf671334cc59779915b192bfb66b2afcf384bb61d0f4",
|
||||
"500049276d20616e20616e6e6578212041726520796f7520616e20616e6e65783f00"
|
||||
],
|
||||
"inner_redeemscript_asm": "",
|
||||
"inner_witnessscript_asm": ""
|
||||
}
|
||||
],
|
||||
"vout": [
|
||||
{
|
||||
"value": 0,
|
||||
"scriptpubkey": "6a05616e6e6578",
|
||||
"scriptpubkey_address": "",
|
||||
"scriptpubkey_asm": "OP_RETURN OP_PUSHBYTES_5 616e6e6578",
|
||||
"scriptpubkey_type": "op_return"
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"confirmed": true,
|
||||
"block_height": 896088,
|
||||
"block_hash": "0000000000000000000148370c6ad0eceb23d0de54ea86a362679cee7fcd3f4a",
|
||||
"block_time": 1746868490
|
||||
},
|
||||
{
|
||||
"scriptpubkey": "6a036d7648",
|
||||
"scriptpubkey_asm": "OP_RETURN OP_PUSHBYTES_3 6d7648",
|
||||
"scriptpubkey_type": "op_return",
|
||||
"value": 0
|
||||
}
|
||||
],
|
||||
"size": 186,
|
||||
"weight": 420,
|
||||
"sigops": 1,
|
||||
"fee": 106,
|
||||
"status": {
|
||||
"confirmed": true,
|
||||
"block_height": 836361,
|
||||
"block_hash": "0000000000000000000341cc26cda4af82cd25f7063c448772228cbf2836915b",
|
||||
"block_time": 1711448028
|
||||
"order": 2877166599,
|
||||
"vsize": 93,
|
||||
"adjustedVsize": 92.75,
|
||||
"sigops": 0,
|
||||
"feePerVsize": 124.91644204851752,
|
||||
"adjustedFeePerVsize": 124.91644204851752,
|
||||
"effectiveFeePerVsize": 124.91644204851752
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
@ -51,7 +51,7 @@ describe('Mempool Backend Config', () => {
|
|||
MAX_PUSH_TX_SIZE_WEIGHT: 400000,
|
||||
ALLOW_UNREACHABLE: true,
|
||||
PRICE_UPDATES_PER_HOUR: 1,
|
||||
MAX_TRACKED_ADDRESSES: 1,
|
||||
MAX_TRACKED_ADDRESSES: 1
|
||||
});
|
||||
|
||||
expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
|
||||
|
|
@ -144,7 +144,7 @@ describe('Mempool Backend Config', () => {
|
|||
});
|
||||
|
||||
expect(config.MEMPOOL_SERVICES).toStrictEqual({
|
||||
API: "",
|
||||
API: '',
|
||||
ACCELERATIONS: false,
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { Application } from "express";
|
||||
import config from "../config";
|
||||
import axios from "axios";
|
||||
import logger from "../logger";
|
||||
import { Application } from 'express';
|
||||
import config from '../config';
|
||||
import axios from 'axios';
|
||||
import logger from '../logger';
|
||||
|
||||
class AboutRoutes {
|
||||
public initRoutes(app: Application) {
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ class AccelerationRoutes {
|
|||
public initRoutes(app: Application): void {
|
||||
app
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations', this.$getAcceleratorAccelerations.bind(this))
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/:txid', this.$getAcceleratorAcceleration.bind(this))
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/history', this.$getAcceleratorAccelerationsHistory.bind(this))
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/history/aggregated', this.$getAcceleratorAccelerationsHistoryAggregated.bind(this))
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/stats', this.$getAcceleratorAccelerationsStats.bind(this))
|
||||
|
|
@ -23,6 +24,21 @@ class AccelerationRoutes {
|
|||
res.status(200).send(Object.values(accelerations));
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
private async $getAcceleratorAcceleration(req: Request, res: Response): Promise<void> {
|
||||
if (req.params.txid) {
|
||||
const acceleration = await AccelerationRepository.$getAccelerationInfoForTxid(req.params.txid);
|
||||
if (acceleration) {
|
||||
res.status(200).send(acceleration);
|
||||
} else {
|
||||
res.status(404).send('Acceleration not found');
|
||||
}
|
||||
} else {
|
||||
res.status(400).send('txid is required');
|
||||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
private async $getAcceleratorAccelerationsHistory(req: Request, res: Response): Promise<void> {
|
||||
const history = await AccelerationRepository.$getAccelerationInfo(null, req.query.blockHeight ? parseInt(req.query.blockHeight as string, 10) : null);
|
||||
res.status(200).send(history.map(accel => ({
|
||||
|
|
|
|||
|
|
@ -3,9 +3,12 @@ import path from 'path';
|
|||
import os from 'os';
|
||||
import { IBackendInfo } from '../mempool.interfaces';
|
||||
import config from '../config';
|
||||
import bitcoinClient from './bitcoin/bitcoin-client';
|
||||
import logger from '../logger';
|
||||
|
||||
class BackendInfo {
|
||||
private backendInfo: IBackendInfo;
|
||||
private timer;
|
||||
|
||||
constructor() {
|
||||
// This file is created by ./fetch-version.ts during building
|
||||
|
|
@ -26,7 +29,28 @@ class BackendInfo {
|
|||
gitCommit: versionInfo.gitCommit,
|
||||
lightning: config.LIGHTNING.ENABLED,
|
||||
backend: config.MEMPOOL.BACKEND,
|
||||
coreVersion: '?',
|
||||
osVersion: `${os.type()} ${os.release()}`,
|
||||
};
|
||||
|
||||
this.timer = setInterval(async () => {
|
||||
try {
|
||||
await this.$updateCoreVersion();
|
||||
} catch (e) {
|
||||
logger.err(`Exception in $updateCoreVersion. Reason: ${(e instanceof Error ? e.message : e)}`);
|
||||
}
|
||||
}, 10 * 60 * 1000); // every 10 minutes
|
||||
void this.$updateCoreVersion(); // starting immediately
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
private async $updateCoreVersion(): Promise<void> {
|
||||
try {
|
||||
const networkInfo = await bitcoinClient.getNetworkInfo();
|
||||
this.backendInfo.coreVersion = networkInfo.subversion;
|
||||
} catch (e) {
|
||||
logger.err(`Exception in $updateCoreVersion. Reason: ${(e instanceof Error ? e.message : e)}`);
|
||||
}
|
||||
}
|
||||
|
||||
public getBackendInfo(): IBackendInfo {
|
||||
|
|
|
|||
|
|
@ -165,44 +165,44 @@ export namespace IBitcoinApi {
|
|||
timeout: number; // (numeric) the median time past of a block at which the deployment is considered failed if not yet locked in
|
||||
since: number; // (numeric) height of the first block to which the status applies
|
||||
statistics: { // (object) numeric statistics about BIP9 signalling for a softfork (only for started status)
|
||||
period: number; // (numeric) the length in blocks of the BIP9 signalling period
|
||||
threshold: number; // (numeric) the number of blocks with the version bit set required to activate the feature
|
||||
elapsed: number; // (numeric) the number of blocks elapsed since the beginning of the current period
|
||||
count: number; // (numeric) the number of blocks with the version bit set in the current period
|
||||
possible: boolean; // (boolean) returns false if there are not enough blocks left in this period to pass activation threshold
|
||||
period: number; // (numeric) the length in blocks of the BIP9 signalling period
|
||||
threshold: number; // (numeric) the number of blocks with the version bit set required to activate the feature
|
||||
elapsed: number; // (numeric) the number of blocks elapsed since the beginning of the current period
|
||||
count: number; // (numeric) the number of blocks with the version bit set in the current period
|
||||
possible: boolean; // (boolean) returns false if there are not enough blocks left in this period to pass activation threshold
|
||||
}
|
||||
}
|
||||
|
||||
export interface BlockStats {
|
||||
"avgfee": number;
|
||||
"avgfeerate": number;
|
||||
"avgtxsize": number;
|
||||
"blockhash": string;
|
||||
"feerate_percentiles": [number, number, number, number, number];
|
||||
"height": number;
|
||||
"ins": number;
|
||||
"maxfee": number;
|
||||
"maxfeerate": number;
|
||||
"maxtxsize": number;
|
||||
"medianfee": number;
|
||||
"mediantime": number;
|
||||
"mediantxsize": number;
|
||||
"minfee": number;
|
||||
"minfeerate": number;
|
||||
"mintxsize": number;
|
||||
"outs": number;
|
||||
"subsidy": number;
|
||||
"swtotal_size": number;
|
||||
"swtotal_weight": number;
|
||||
"swtxs": number;
|
||||
"time": number;
|
||||
"total_out": number;
|
||||
"total_size": number;
|
||||
"total_weight": number;
|
||||
"totalfee": number;
|
||||
"txs": number;
|
||||
"utxo_increase": number;
|
||||
"utxo_size_inc": number;
|
||||
'avgfee': number;
|
||||
'avgfeerate': number;
|
||||
'avgtxsize': number;
|
||||
'blockhash': string;
|
||||
'feerate_percentiles': [number, number, number, number, number];
|
||||
'height': number;
|
||||
'ins': number;
|
||||
'maxfee': number;
|
||||
'maxfeerate': number;
|
||||
'maxtxsize': number;
|
||||
'medianfee': number;
|
||||
'mediantime': number;
|
||||
'mediantxsize': number;
|
||||
'minfee': number;
|
||||
'minfeerate': number;
|
||||
'mintxsize': number;
|
||||
'outs': number;
|
||||
'subsidy': number;
|
||||
'swtotal_size': number;
|
||||
'swtotal_weight': number;
|
||||
'swtxs': number;
|
||||
'time': number;
|
||||
'total_out': number;
|
||||
'total_size': number;
|
||||
'total_weight': number;
|
||||
'totalfee': number;
|
||||
'txs': number;
|
||||
'utxo_increase': number;
|
||||
'utxo_size_inc': number;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -213,26 +213,26 @@ export interface TestMempoolAcceptResult {
|
|||
vsize?: number,
|
||||
fees?: {
|
||||
base: number,
|
||||
"effective-feerate": number,
|
||||
"effective-includes": string[],
|
||||
'effective-feerate': number,
|
||||
'effective-includes': string[],
|
||||
},
|
||||
['reject-reason']?: string,
|
||||
}
|
||||
|
||||
export interface SubmitPackageResult {
|
||||
package_msg: string;
|
||||
"tx-results": { [wtxid: string]: TxResult };
|
||||
"replaced-transactions"?: string[];
|
||||
'tx-results': { [wtxid: string]: TxResult };
|
||||
'replaced-transactions'?: string[];
|
||||
}
|
||||
|
||||
export interface TxResult {
|
||||
txid: string;
|
||||
"other-wtxid"?: string;
|
||||
'other-wtxid'?: string;
|
||||
vsize?: number;
|
||||
fees?: {
|
||||
base: number;
|
||||
"effective-feerate"?: number;
|
||||
"effective-includes"?: string[];
|
||||
'effective-feerate'?: number;
|
||||
'effective-includes'?: string[];
|
||||
};
|
||||
error?: string;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -112,6 +112,7 @@ class BitcoinApi implements AbstractBitcoinApi {
|
|||
.then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx);
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $getTxsForBlock(hash: string, fallbackToCore = false): Promise<IEsploraApi.Transaction[]> {
|
||||
const verboseBlock: IBitcoinApi.VerboseBlock = await this.bitcoindClient.getBlock(hash, 2);
|
||||
const transactions: IEsploraApi.Transaction[] = [];
|
||||
|
|
@ -130,7 +131,7 @@ class BitcoinApi implements AbstractBitcoinApi {
|
|||
|
||||
$getRawBlock(hash: string): Promise<Buffer> {
|
||||
return this.bitcoindClient.getBlock(hash, 0)
|
||||
.then((raw: string) => Buffer.from(raw, "hex"));
|
||||
.then((raw: string) => Buffer.from(raw, 'hex'));
|
||||
}
|
||||
|
||||
$getBlockHash(height: number): Promise<string> {
|
||||
|
|
@ -219,6 +220,7 @@ class BitcoinApi implements AbstractBitcoinApi {
|
|||
return this.bitcoindClient.submitPackage(rawTransactions, maxfeerate ?? undefined, maxburnamount ?? undefined);
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
|
||||
const txOut = await this.bitcoindClient.getTxOut(txId, vout, false);
|
||||
return {
|
||||
|
|
@ -229,6 +231,7 @@ class BitcoinApi implements AbstractBitcoinApi {
|
|||
};
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
|
||||
const outSpends: IEsploraApi.Outspend[] = [];
|
||||
const tx = await this.$getRawTransaction(txId, true, false);
|
||||
|
|
@ -247,6 +250,7 @@ class BitcoinApi implements AbstractBitcoinApi {
|
|||
return outSpends;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
|
||||
const outspends: IEsploraApi.Outspend[][] = [];
|
||||
for (const tx of txId) {
|
||||
|
|
@ -260,6 +264,7 @@ class BitcoinApi implements AbstractBitcoinApi {
|
|||
return this.$getBatchedOutspends(txId);
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]> {
|
||||
const outspends: IEsploraApi.Outspend[] = [];
|
||||
for (const outpoint of outpoints) {
|
||||
|
|
@ -269,6 +274,7 @@ class BitcoinApi implements AbstractBitcoinApi {
|
|||
return outspends;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $getCoinbaseTx(blockhash: string): Promise<IEsploraApi.Transaction> {
|
||||
const txids = await this.$getTxIdsForBlock(blockhash);
|
||||
return this.$getRawTransaction(txids[0]);
|
||||
|
|
@ -283,6 +289,7 @@ class BitcoinApi implements AbstractBitcoinApi {
|
|||
return this.bitcoindClient.getNetworkHashPs(120, blockHeight);
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean, lazyPrevouts = false, allowMissingPrevouts = false): Promise<IEsploraApi.Transaction> {
|
||||
let esploraTransaction: IEsploraApi.Transaction = {
|
||||
txid: transaction.txid,
|
||||
|
|
@ -367,6 +374,7 @@ class BitcoinApi implements AbstractBitcoinApi {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
private async $appendMempoolFeeData(transaction: IEsploraApi.Transaction): Promise<IEsploraApi.Transaction> {
|
||||
if (transaction.fee) {
|
||||
return transaction;
|
||||
|
|
@ -384,6 +392,7 @@ class BitcoinApi implements AbstractBitcoinApi {
|
|||
return transaction;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
protected async $addPrevouts(transaction: TransactionExtended): Promise<TransactionExtended> {
|
||||
let addedPrevouts = false;
|
||||
for (const vin of transaction.vin) {
|
||||
|
|
@ -423,6 +432,7 @@ class BitcoinApi implements AbstractBitcoinApi {
|
|||
}
|
||||
|
||||
|
||||
/** @asyncUnsafe */
|
||||
private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean, lazyPrevouts: boolean): Promise<IEsploraApi.Transaction> {
|
||||
if (transaction.vin[0].is_coinbase) {
|
||||
transaction.fee = 0;
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ class BitcoinRoutes {
|
|||
.get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.$getCpfpInfo)
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', this.getDifficultyChange)
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', this.getRecommendedFees)
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'fees/precise', this.getPreciseRecommendedFees)
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', this.getMempoolBlocks)
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', this.getBackendInfo)
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'init-data', this.getInitData)
|
||||
|
|
@ -122,6 +123,16 @@ class BitcoinRoutes {
|
|||
res.json(result);
|
||||
}
|
||||
|
||||
private getPreciseRecommendedFees(req: Request, res: Response) {
|
||||
if (!mempool.isInSync()) {
|
||||
res.statusCode = 503;
|
||||
res.send('Service Unavailable');
|
||||
return;
|
||||
}
|
||||
const result = feeApi.getPreciseRecommendedFee();
|
||||
res.json(result);
|
||||
}
|
||||
|
||||
private getMempoolBlocks(req: Request, res: Response) {
|
||||
try {
|
||||
const result = mempoolBlocks.getMempoolBlocks();
|
||||
|
|
@ -132,17 +143,14 @@ class BitcoinRoutes {
|
|||
}
|
||||
|
||||
private getTransactionTimes(req: Request, res: Response) {
|
||||
if (!Array.isArray(req.query.txId)) {
|
||||
handleError(req, res, 500, 'Not an array');
|
||||
if (!req.query.txId || typeof req.query.txId !== 'object') {
|
||||
handleError(req, res, 500, 'invalid txId format');
|
||||
return;
|
||||
}
|
||||
const txIds: string[] = [];
|
||||
for (const _txId in req.query.txId) {
|
||||
if (typeof req.query.txId[_txId] === 'string') {
|
||||
const txid = req.query.txId[_txId].toString();
|
||||
if (TXID_REGEX.test(txid)) {
|
||||
txIds.push(txid);
|
||||
}
|
||||
for (const txid of Object.values(req.query.txId)) {
|
||||
if (typeof txid === 'string' && TXID_REGEX.test(txid)) {
|
||||
txIds.push(txid);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -475,7 +483,7 @@ class BitcoinRoutes {
|
|||
|
||||
private async getBlocks(req: Request, res: Response) {
|
||||
try {
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
|
||||
const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
|
||||
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
|
||||
res.json(await blocks.$getBlocks(height, 15));
|
||||
|
|
@ -489,7 +497,7 @@ class BitcoinRoutes {
|
|||
|
||||
private async getBlocksByBulk(req: Request, res: Response) {
|
||||
try {
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid - Not implemented
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid - Not implemented
|
||||
handleError(req, res, 404, `This API is only available for Bitcoin networks`);
|
||||
return;
|
||||
}
|
||||
|
|
@ -531,7 +539,7 @@ class BitcoinRoutes {
|
|||
|
||||
private async getChainTips(req: Request, res: Response) {
|
||||
try {
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
|
||||
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
|
||||
const tips = await chainTips.getChainTips();
|
||||
if (tips.length > 0) {
|
||||
|
|
@ -551,7 +559,7 @@ class BitcoinRoutes {
|
|||
|
||||
private async getStaleTips(req: Request, res: Response) {
|
||||
try {
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
|
||||
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
|
||||
const tips = await chainTips.getStaleTips();
|
||||
if (tips.length > 0) {
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import { IEsploraApi } from './esplora-api.interface';
|
|||
import { IElectrumApi } from './electrum-api.interface';
|
||||
import BitcoinApi from './bitcoin-api';
|
||||
import logger from '../../logger';
|
||||
import crypto from "crypto-js";
|
||||
import crypto from 'crypto-js';
|
||||
import loadingIndicators from '../loading-indicators';
|
||||
import memoryCache from '../memory-cache';
|
||||
|
||||
|
|
@ -40,6 +40,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
|
|||
});
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $getAddress(address: string): Promise<IEsploraApi.Address> {
|
||||
const addressInfo = await this.bitcoindClient.validateAddress(address);
|
||||
if (!addressInfo || !addressInfo.isvalid) {
|
||||
|
|
@ -91,6 +92,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]> {
|
||||
const addressInfo = await this.bitcoindClient.validateAddress(address);
|
||||
if (!addressInfo || !addressInfo.isvalid) {
|
||||
|
|
@ -160,6 +162,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $getAddressUtxos(address: string): Promise<IEsploraApi.UTXO[]> {
|
||||
const addressInfo = await this.bitcoindClient.validateAddress(address);
|
||||
if (!addressInfo || !addressInfo.isvalid) {
|
||||
|
|
@ -206,10 +209,11 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $getScriptHashUtxos(scripthash: string): Promise<IEsploraApi.UTXO[]> {
|
||||
const utxos = await this.$getScriptHashUnspent(scripthash);
|
||||
const result: IEsploraApi.UTXO[] = [];
|
||||
for(let utxo of utxos) {
|
||||
for(const utxo of utxos) {
|
||||
if(utxo.height===0) {
|
||||
//Unconfirmed
|
||||
result.push({
|
||||
|
|
@ -244,6 +248,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
|
|||
return this.electrumClient.blockchainScripthash_listunspent(scriptHash);
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $getTransactionMerkleProof(txId: string): Promise<IEsploraApi.MerkleProof> {
|
||||
const tx = await this.$getRawTransaction(txId);
|
||||
return this.electrumClient.blockchainTransaction_getMerkle(txId, tx.status.block_height);
|
||||
|
|
|
|||
|
|
@ -28,6 +28,9 @@ interface FailoverHost {
|
|||
hybrid?: string,
|
||||
backend?: string,
|
||||
electrs?: string,
|
||||
ssr?: string,
|
||||
core?: string,
|
||||
os?: string,
|
||||
lastUpdated: number,
|
||||
}
|
||||
}
|
||||
|
|
@ -35,7 +38,7 @@ interface FailoverHost {
|
|||
class FailoverRouter {
|
||||
activeHost: FailoverHost;
|
||||
fallbackHost: FailoverHost;
|
||||
maxSlippage: number = config.ESPLORA.MAX_BEHIND_TIP ?? 2;
|
||||
maxSlippage: number = config.ESPLORA.MAX_BEHIND_TIP ?? (Common.isLiquid() ? 8 : 2);
|
||||
maxHeight: number = 0;
|
||||
hosts: FailoverHost[];
|
||||
multihost: boolean;
|
||||
|
|
@ -98,11 +101,12 @@ class FailoverRouter {
|
|||
});
|
||||
|
||||
if (this.multihost) {
|
||||
this.pollHosts();
|
||||
void this.pollHosts();
|
||||
}
|
||||
}
|
||||
|
||||
// start polling hosts to measure availability & rtt
|
||||
/** @asyncSafe */
|
||||
private async pollHosts(): Promise<void> {
|
||||
if (this.pollTimer) {
|
||||
clearTimeout(this.pollTimer);
|
||||
|
|
@ -145,7 +149,8 @@ class FailoverRouter {
|
|||
if (Date.now() - host.hashes.lastUpdated > this.gitHashInterval) {
|
||||
await Promise.all([
|
||||
this.$updateFrontendGitHash(host),
|
||||
this.$updateBackendGitHash(host),
|
||||
this.$updateBackendVersions(host),
|
||||
this.$updateSSRGitHash(host),
|
||||
config.MEMPOOL.OFFICIAL ? this.$updateHybridGitHash(host) : Promise.resolve(),
|
||||
]);
|
||||
host.hashes.lastUpdated = Date.now();
|
||||
|
|
@ -191,7 +196,7 @@ class FailoverRouter {
|
|||
|
||||
const elapsed = Date.now() - start;
|
||||
|
||||
this.pollTimer = setTimeout(() => { this.pollHosts(); }, Math.max(1, this.pollInterval - elapsed));
|
||||
this.pollTimer = setTimeout(() => { void this.pollHosts(); }, Math.max(1, this.pollInterval - elapsed));
|
||||
}
|
||||
|
||||
private formatRanking(index: number, host: FailoverHost, active: FailoverHost, maxHeight: number): string {
|
||||
|
|
@ -250,7 +255,12 @@ class FailoverRouter {
|
|||
private async $updateFrontendGitHash(host: FailoverHost): Promise<void> {
|
||||
try {
|
||||
const url = `${host.publicDomain}/resources/config.js`;
|
||||
const response = await this.pollConnection.get<string>(url, { timeout: config.ESPLORA.FALLBACK_TIMEOUT });
|
||||
const response = await this.pollConnection.get<string>(
|
||||
url, {
|
||||
timeout: config.ESPLORA.FALLBACK_TIMEOUT,
|
||||
headers: Common.isLiquid() ? { 'Host': 'liquid.network' } : undefined
|
||||
}
|
||||
);
|
||||
const match = response.data.match(/GIT_COMMIT_HASH\s*=\s*['"](.*?)['"]/);
|
||||
if (match && match[1]?.length) {
|
||||
host.hashes.frontend = match[1];
|
||||
|
|
@ -273,7 +283,7 @@ class FailoverRouter {
|
|||
path: '/en-US/resources/config.js',
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Host': 'mempool.space'
|
||||
'Host': Common.isLiquid() ? 'liquid.network' : 'mempool.space'
|
||||
},
|
||||
timeout: config.ESPLORA.FALLBACK_TIMEOUT,
|
||||
}, (res) => {
|
||||
|
|
@ -301,18 +311,46 @@ class FailoverRouter {
|
|||
}
|
||||
}
|
||||
|
||||
private async $updateBackendGitHash(host: FailoverHost): Promise<void> {
|
||||
private async $updateBackendVersions(host: FailoverHost): Promise<void> {
|
||||
try {
|
||||
const url = `${host.publicDomain}/api/v1/backend-info`;
|
||||
const response = await this.pollConnection.get<any>(url, { timeout: config.ESPLORA.FALLBACK_TIMEOUT });
|
||||
const response = await this.pollConnection.get<any>(
|
||||
url, {
|
||||
timeout: config.ESPLORA.FALLBACK_TIMEOUT,
|
||||
headers: Common.isLiquid() ? { 'Host': 'liquid.network' } : undefined
|
||||
}
|
||||
);
|
||||
if (response.data?.gitCommit) {
|
||||
host.hashes.backend = response.data.gitCommit;
|
||||
}
|
||||
if (response.data?.coreVersion) {
|
||||
host.hashes.core = response.data.coreVersion;
|
||||
}
|
||||
if (response.data?.osVersion) {
|
||||
host.hashes.os = response.data.osVersion;
|
||||
}
|
||||
} catch (e) {
|
||||
// failed to get backend build hash - do nothing
|
||||
}
|
||||
}
|
||||
|
||||
private async $updateSSRGitHash(host: FailoverHost): Promise<void> {
|
||||
try {
|
||||
const url = `${host.publicDomain}/ssr/api/status`;
|
||||
const response = await this.pollConnection.get<any>(
|
||||
url, {
|
||||
timeout: config.ESPLORA.FALLBACK_TIMEOUT,
|
||||
headers: Common.isLiquid() ? { 'Host': 'liquid.network' } : undefined
|
||||
}
|
||||
);
|
||||
if (response.data?.gitHash) {
|
||||
host.hashes.ssr = response.data.gitHash;
|
||||
}
|
||||
} catch (e) {
|
||||
// failed to get ssr build hash - do nothing
|
||||
}
|
||||
}
|
||||
|
||||
// returns the public mempool domain corresponding to an esplora server url
|
||||
// (a bit of a hack to avoid manually specifying frontend & backend URLs for each esplora server)
|
||||
private extractPublicDomain(url: string): string {
|
||||
|
|
@ -409,6 +447,7 @@ class ElectrsApi implements AbstractBitcoinApi {
|
|||
return this.failoverRouter.$get<string>('/blocks/tip/hash');
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $getTxIdsForBlock(hash: string, fallbackToCore = false): Promise<string[]> {
|
||||
try {
|
||||
const txids = await this.failoverRouter.$get<string[]>('/block/' + hash + '/txids');
|
||||
|
|
@ -425,6 +464,7 @@ class ElectrsApi implements AbstractBitcoinApi {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $getTxsForBlock(hash: string, fallbackToCore = false): Promise<IEsploraApi.Transaction[]> {
|
||||
try {
|
||||
const txs = await this.failoverRouter.$get<IEsploraApi.Transaction[]>('/internal/block/' + hash + '/txs');
|
||||
|
|
@ -518,6 +558,7 @@ class ElectrsApi implements AbstractBitcoinApi {
|
|||
return this.failoverRouter.$post<IEsploraApi.Outspend[]>('/internal/txs/outspends/by-outpoint', outpoints.map(out => `${out.txid}:${out.vout}`), 'json');
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $getCoinbaseTx(blockhash: string): Promise<IEsploraApi.Transaction> {
|
||||
const txid = await this.failoverRouter.$get<string>(`/block/${blockhash}/txid/0`);
|
||||
return this.failoverRouter.$get<IEsploraApi.Transaction>('/tx/' + txid);
|
||||
|
|
|
|||
|
|
@ -87,6 +87,8 @@ class Blocks {
|
|||
* @param quiet - don't print non-essential logs
|
||||
* @param addMempoolData - calculate sigops etc
|
||||
* @returns Promise<TransactionExtended[]>
|
||||
*
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
private async $getTransactionsExtended(
|
||||
blockHash: string,
|
||||
|
|
@ -108,7 +110,7 @@ class Blocks {
|
|||
const mempool = memPool.getMempool();
|
||||
let foundInMempool = 0;
|
||||
let totalFound = 0;
|
||||
let missing = 0;
|
||||
const missing = 0;
|
||||
|
||||
// Copy existing transactions from the mempool
|
||||
if (!onlyCoinbase) {
|
||||
|
|
@ -247,6 +249,8 @@ class Blocks {
|
|||
* @param block
|
||||
* @param transactions
|
||||
* @returns BlockExtended
|
||||
*
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
|
||||
const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
|
||||
|
|
@ -329,7 +333,7 @@ class Blocks {
|
|||
extras.totalInputAmt = null;
|
||||
}
|
||||
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
|
||||
let pool: PoolTag;
|
||||
if (coinbaseTx !== undefined) {
|
||||
pool = await this.$findBlockMiner(coinbaseTx);
|
||||
|
|
@ -458,6 +462,8 @@ class Blocks {
|
|||
* Try to find which miner found the block
|
||||
* @param txMinerInfo
|
||||
* @returns
|
||||
*
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
private async $findBlockMiner(txMinerInfo: TransactionMinerInfo | undefined): Promise<PoolTag> {
|
||||
if (txMinerInfo === undefined || txMinerInfo.vout.length < 1) {
|
||||
|
|
@ -556,6 +562,7 @@ class Blocks {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $indexBlockSummary(hash: string, height: number, stale?: boolean): Promise<void> {
|
||||
if (config.MEMPOOL.BACKEND === 'esplora') {
|
||||
const txs = (await bitcoinApi.$getTxsForBlock(hash, stale)).map(tx => transactionUtils.extendMempoolTransaction(tx));
|
||||
|
|
@ -619,6 +626,8 @@ class Blocks {
|
|||
|
||||
/**
|
||||
* [INDEXING] Index expected fees & weight for all audited blocks
|
||||
*
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async $generateAuditStats(): Promise<void> {
|
||||
const blockIds = await BlocksAuditsRepository.$getBlocksWithoutSummaries();
|
||||
|
|
@ -659,6 +668,8 @@ class Blocks {
|
|||
|
||||
/**
|
||||
* [INDEXING] Index transaction classification flags for Goggles
|
||||
*
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $classifyBlocks(): Promise<void> {
|
||||
if (this.classifyingBlocks) {
|
||||
|
|
@ -839,6 +850,7 @@ class Blocks {
|
|||
|
||||
/**
|
||||
* [INDEXING] Index all blocks metadata for the mining dashboard
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $generateBlockDatabase(): Promise<boolean> {
|
||||
try {
|
||||
|
|
@ -918,6 +930,8 @@ class Blocks {
|
|||
|
||||
/**
|
||||
* [INDEXING] Index all blocks first seen time from Bitcoin Core debug logs
|
||||
*
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async $indexBlocksFirstSeen(): Promise<void> {
|
||||
const previous = this.oldestCoreLogTimestamp;
|
||||
|
|
@ -953,6 +967,7 @@ class Blocks {
|
|||
logger.debug(`Indexed ${foundCount} / ${blocks.length} block first seen times in ${((Date.now() - startedAt) / 1000).toFixed(2)} seconds`);
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $updateBlocks(): Promise<number> {
|
||||
// warn if this run stalls the main loop for more than 2 minutes
|
||||
const timer = this.startTimer();
|
||||
|
|
@ -1081,7 +1096,7 @@ class Blocks {
|
|||
this.updateTimerProgress(timer, `saved block summary for ${this.currentBlockHeight}`);
|
||||
}
|
||||
if (config.MEMPOOL.CPFP_INDEXING) {
|
||||
this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary);
|
||||
void this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary);
|
||||
this.updateTimerProgress(timer, `saved cpfp for ${this.currentBlockHeight}`);
|
||||
}
|
||||
}
|
||||
|
|
@ -1149,7 +1164,7 @@ class Blocks {
|
|||
this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
|
||||
}
|
||||
if (config.MEMPOOL.CACHE_ENABLED && !memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
|
||||
diskCache.$saveCacheToDisk();
|
||||
void diskCache.$saveCacheToDisk();
|
||||
}
|
||||
|
||||
// Update Redis cache
|
||||
|
|
@ -1192,6 +1207,7 @@ class Blocks {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
private async updateQuarterEpochBlockTime(): Promise<void> {
|
||||
if (this.currentBlockHeight >= 503) {
|
||||
try {
|
||||
|
|
@ -1205,6 +1221,10 @@ class Blocks {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Index a block if it's missing from the database. Returns the block after indexing
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async $indexBlockByHeight(height: number, skipDb = false): Promise<BlockExtended> {
|
||||
if (Common.indexingEnabled() && !skipDb) {
|
||||
const dbBlock = await blocksRepository.$getBlockByHeight(height);
|
||||
|
|
@ -1217,6 +1237,7 @@ class Blocks {
|
|||
return this.$indexBlock(hash);
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
private async $handleReorgs(blockExtended: BlockExtended, timer: any): Promise<void> {
|
||||
let forkTail = blockExtended;
|
||||
let currentlyIndexed = await blocksRepository.$getBlockByHeight(forkTail.height - 1);
|
||||
|
|
@ -1288,6 +1309,8 @@ class Blocks {
|
|||
|
||||
/**
|
||||
* Index a block if it's missing from the database. Returns the block after indexing
|
||||
*
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async $indexBlock(hash: string, block?: IEsploraApi.Block, skipDb = false): Promise<BlockExtended> {
|
||||
if (Common.indexingEnabled() && !skipDb) {
|
||||
|
|
@ -1317,16 +1340,19 @@ class Blocks {
|
|||
|
||||
/**
|
||||
* Get one block by its hash
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async $getBlock(hash: string): Promise<BlockExtended | IEsploraApi.Block> {
|
||||
public async $getBlock(hash: string, skipMemoryCache: boolean = false): Promise<BlockExtended | IEsploraApi.Block> {
|
||||
// Check the memory cache
|
||||
const blockByHash = this.getBlocks().find((b) => b.id === hash);
|
||||
if (blockByHash) {
|
||||
return blockByHash;
|
||||
if (!skipMemoryCache) {
|
||||
const blockByHash = this.getBlocks().find((b) => b.id === hash);
|
||||
if (blockByHash) {
|
||||
return blockByHash;
|
||||
}
|
||||
}
|
||||
|
||||
// Not Bitcoin network, return the block as it from the bitcoin backend
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) === false) {
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) === false) {
|
||||
return await bitcoinCoreApi.$getBlock(hash);
|
||||
}
|
||||
|
||||
|
|
@ -1334,6 +1360,7 @@ class Blocks {
|
|||
return await this.$indexBlock(hash);
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,
|
||||
skipDBLookup = false, cpfpSummary?: CpfpSummary, blockHeight?: number): Promise<TransactionClassified[]>
|
||||
{
|
||||
|
|
@ -1402,6 +1429,7 @@ class Blocks {
|
|||
return summary.transactions;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $getSingleTxFromSummary(hash: string, txid: string): Promise<TransactionClassified | null> {
|
||||
const txs = await this.$getStrippedBlockTransactions(hash);
|
||||
return txs.find(tx => tx.txid === txid) || null;
|
||||
|
|
@ -1409,15 +1437,16 @@ class Blocks {
|
|||
|
||||
/**
|
||||
* Get 15 blocks
|
||||
*
|
||||
*
|
||||
* Internally this function uses two methods to get the blocks, and
|
||||
* the method is automatically selected:
|
||||
* - Using previous block hash links
|
||||
* - Using block height
|
||||
*
|
||||
* @param fromHeight
|
||||
* @param limit
|
||||
* @returns
|
||||
*
|
||||
* @param fromHeight
|
||||
* @param limit
|
||||
* @returns
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
|
||||
let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight;
|
||||
|
|
@ -1449,9 +1478,10 @@ class Blocks {
|
|||
|
||||
/**
|
||||
* Used for bulk block data query
|
||||
*
|
||||
* @param fromHeight
|
||||
* @param toHeight
|
||||
*
|
||||
* @param fromHeight
|
||||
* @param toHeight
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async $getBlocksBetweenHeight(fromHeight: number, toHeight: number): Promise<any> {
|
||||
if (!Common.indexingEnabled()) {
|
||||
|
|
@ -1569,7 +1599,7 @@ class Blocks {
|
|||
}
|
||||
|
||||
public async $getBlockAuditSummary(hash: string): Promise<BlockAudit | null> {
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) && Common.auditIndexingEnabled()) {
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) && Common.auditIndexingEnabled()) {
|
||||
return BlocksAuditsRepository.$getBlockAudit(hash);
|
||||
} else {
|
||||
return null;
|
||||
|
|
@ -1577,7 +1607,7 @@ class Blocks {
|
|||
}
|
||||
|
||||
public async $getBlockTxAuditSummary(hash: string, txid: string): Promise<TransactionAudit | null> {
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) && Common.auditIndexingEnabled()) {
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) && Common.auditIndexingEnabled()) {
|
||||
return BlocksAuditsRepository.$getBlockTxAudit(hash, txid);
|
||||
} else {
|
||||
return null;
|
||||
|
|
@ -1600,6 +1630,7 @@ class Blocks {
|
|||
return this.currentBlockHeight;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $indexCPFP(hash: string, height: number, txs?: MempoolTransactionExtended[], stale?: boolean): Promise<CpfpSummary | null> {
|
||||
let transactions = txs;
|
||||
if (!transactions) {
|
||||
|
|
@ -1632,6 +1663,7 @@ class Blocks {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise<void> {
|
||||
try {
|
||||
const result = await cpfpRepository.$batchSaveClusters(cpfpSummary.clusters);
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import config from '../config';
|
||||
import logger from '../logger';
|
||||
import { BlockExtended } from '../mempool.interfaces';
|
||||
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
|
||||
import { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
|
||||
import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
|
||||
import bitcoinClient from './bitcoin/bitcoin-client';
|
||||
import { IEsploraApi } from './bitcoin/esplora-api.interface';
|
||||
import blocks from './blocks';
|
||||
|
|
@ -38,10 +39,18 @@ class ChainTips {
|
|||
private staleTipsCacheSize = 50;
|
||||
private maxIndexingQueueSize = 100;
|
||||
|
||||
/** @asyncSafe */
|
||||
public async updateOrphanedBlocks(): Promise<void> {
|
||||
try {
|
||||
this.chainTips = await bitcoinClient.getChainTips();
|
||||
|
||||
const activeTipHeight = this.chainTips.find(tip => tip.status === 'active')?.height || (await bitcoinApi.$getBlockHeightTip());
|
||||
let minIndexHeight = 0;
|
||||
const indexedBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, activeTipHeight);
|
||||
if (indexedBlockAmount > 0) {
|
||||
minIndexHeight = Math.max(0, activeTipHeight - indexedBlockAmount + 1);
|
||||
}
|
||||
|
||||
const start = Date.now();
|
||||
const breakAt = start + 10000;
|
||||
let newOrphans = 0;
|
||||
|
|
@ -64,11 +73,24 @@ class ChainTips {
|
|||
prevhash: block.previousblockhash,
|
||||
};
|
||||
this.blockCache[hash] = orphan;
|
||||
if (this.indexingQueue.length < this.maxIndexingQueueSize) {
|
||||
this.indexingQueue.push({ block, tip: orphan });
|
||||
} else {
|
||||
// re-fetch blocks lazily if the queue is big to keep memory usage sane
|
||||
this.indexingQueue.push({ blockhash: hash, tip: orphan });
|
||||
// don't index stale blocks below the INDEXING_BLOCKS_AMOUNT cutoff
|
||||
if (block.height >= minIndexHeight) {
|
||||
if (this.indexingQueue.length < this.maxIndexingQueueSize) {
|
||||
this.indexingQueue.push({ block, tip: orphan });
|
||||
} else {
|
||||
// re-fetch blocks lazily if the queue is big to keep memory usage sane
|
||||
this.indexingQueue.push({ blockhash: hash, tip: orphan });
|
||||
}
|
||||
}
|
||||
// make sure the cached canonical block at this height is correct & up to date
|
||||
if (block.height >= (activeTipHeight - (config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4))) {
|
||||
const cachedBlocks = blocks.getBlocks();
|
||||
for (const cachedBlock of cachedBlocks) {
|
||||
if (cachedBlock.height === block.height) {
|
||||
// ensure this stale block is included in the orphans list
|
||||
cachedBlock.extras.orphans = Array.from(new Set([...(cachedBlock.extras.orphans || []), orphan]));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -114,6 +136,7 @@ class ChainTips {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
private async $indexOrphanedBlocks(): Promise<void> {
|
||||
if (this.indexingOrphanedBlocks) {
|
||||
return;
|
||||
|
|
@ -125,13 +148,13 @@ class ChainTips {
|
|||
if (!block && !blockhash) {
|
||||
continue;
|
||||
}
|
||||
if (blockhash && !block) {
|
||||
block = await bitcoinCoreApi.$getBlock(blockhash);
|
||||
}
|
||||
if (!block) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
if (blockhash && !block) {
|
||||
block = await bitcoinCoreApi.$getBlock(blockhash);
|
||||
}
|
||||
if (!block) {
|
||||
continue;
|
||||
}
|
||||
let staleBlock: BlockExtended | undefined;
|
||||
const alreadyIndexed = await BlocksSummariesRepository.$isSummaryIndexed(block.id);
|
||||
const needToCache = Object.keys(this.staleTips).length < this.staleTipsCacheSize || block.height > Object.keys(this.staleTips).map(Number).sort((a, b) => b - a)[this.staleTipsCacheSize - 1];
|
||||
|
|
@ -141,7 +164,7 @@ class ChainTips {
|
|||
// don't DDOS core by indexing too fast
|
||||
await Common.sleep$(5000);
|
||||
} else if (needToCache) {
|
||||
staleBlock = await blocks.$getBlock(block.id) as BlockExtended;
|
||||
staleBlock = await blocks.$getBlock(block.id, true) as BlockExtended;
|
||||
}
|
||||
|
||||
if (staleBlock && needToCache) {
|
||||
|
|
@ -157,7 +180,7 @@ class ChainTips {
|
|||
this.trimStaleTipsCache();
|
||||
}
|
||||
} catch (e) {
|
||||
logger.err(`Failed to index orphaned block ${block.id} at height ${block.height}. Reason: ${e instanceof Error ? e.message : e}`);
|
||||
logger.err(`Failed to index orphaned block ${block?.id} at height ${block?.height}. Reason: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
}
|
||||
this.indexingOrphanedBlocks = false;
|
||||
|
|
|
|||
|
|
@ -24,15 +24,15 @@ const MAX_STANDARD_SCRIPTSIG_SIZE = 1650;
|
|||
const DUST_RELAY_TX_FEE = 3;
|
||||
const MAX_OP_RETURN_RELAY = 83;
|
||||
const DEFAULT_PERMIT_BAREMULTISIG = true;
|
||||
const MAX_TX_LEGACY_SIGOPS = 2_500 * 4; // witness-adjusted sigops
|
||||
|
||||
export class Common {
|
||||
static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ?
|
||||
'144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49'
|
||||
: '6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d';
|
||||
static _isLiquid = config.MEMPOOL.NETWORK === 'liquid' || config.MEMPOOL.NETWORK === 'liquidtestnet';
|
||||
|
||||
static isLiquid(): boolean {
|
||||
return this._isLiquid;
|
||||
return config?.MEMPOOL?.NETWORK === 'liquid' || config?.MEMPOOL?.NETWORK === 'liquidtestnet';
|
||||
}
|
||||
|
||||
static median(numbers: number[]) {
|
||||
|
|
@ -225,6 +225,11 @@ export class Common {
|
|||
return true;
|
||||
}
|
||||
|
||||
// legacy sigops
|
||||
if (this.isNonStandardLegacySigops(tx, height)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// input validation
|
||||
for (const vin of tx.vin) {
|
||||
if (vin.is_coinbase) {
|
||||
|
|
@ -236,7 +241,7 @@ export class Common {
|
|||
return true;
|
||||
}
|
||||
// scriptsig-not-pushonly
|
||||
if (vin.scriptsig_asm) {
|
||||
if (vin.scriptsig_asm?.length) {
|
||||
for (const op of vin.scriptsig_asm.split(' ')) {
|
||||
if (opcodes[op] && opcodes[op] > opcodes['OP_16']) {
|
||||
return true;
|
||||
|
|
@ -286,6 +291,7 @@ export class Common {
|
|||
|
||||
// output validation
|
||||
let opreturnCount = 0;
|
||||
let opreturnBytes = 0;
|
||||
for (const vout of tx.vout) {
|
||||
// scriptpubkey
|
||||
if (['nonstandard', 'provably_unspendable', 'empty'].includes(vout.scriptpubkey_type)) {
|
||||
|
|
@ -309,10 +315,7 @@ export class Common {
|
|||
}
|
||||
} else if (vout.scriptpubkey_type === 'op_return') {
|
||||
opreturnCount++;
|
||||
if ((vout.scriptpubkey.length / 2) > MAX_OP_RETURN_RELAY) {
|
||||
// over default datacarrier limit
|
||||
return true;
|
||||
}
|
||||
opreturnBytes += vout.scriptpubkey.length / 2;
|
||||
}
|
||||
// dust
|
||||
// (we could probably hardcode this for the different output types...)
|
||||
|
|
@ -334,9 +337,11 @@ export class Common {
|
|||
}
|
||||
}
|
||||
|
||||
// multi-op-return
|
||||
if (opreturnCount > 1) {
|
||||
return true;
|
||||
// op_return
|
||||
if (opreturnCount > 0) {
|
||||
if (!this.isStandardOpReturn(opreturnBytes, opreturnCount, height)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: non-mandatory-script-verify-flag
|
||||
|
|
@ -371,6 +376,7 @@ export class Common {
|
|||
'testnet4': 42_000,
|
||||
'testnet': 2_900_000,
|
||||
'signet': 211_000,
|
||||
'regtest': 0,
|
||||
'': 863_500,
|
||||
};
|
||||
static isNonStandardVersion(tx: TransactionExtended, height?: number): boolean {
|
||||
|
|
@ -394,6 +400,7 @@ export class Common {
|
|||
'testnet4': 42_000,
|
||||
'testnet': 2_900_000,
|
||||
'signet': 211_000,
|
||||
'regtest': 0,
|
||||
'': 863_500,
|
||||
};
|
||||
static isNonStandardAnchor(vin: IEsploraApi.Vin, height?: number): boolean {
|
||||
|
|
@ -414,6 +421,7 @@ export class Common {
|
|||
'testnet4': 90_500,
|
||||
'testnet': 4_550_000,
|
||||
'signet': 260_000,
|
||||
'regtest': 0,
|
||||
'': 905_000,
|
||||
};
|
||||
static isStandardEphemeralDust(tx: TransactionExtended, height?: number): boolean {
|
||||
|
|
@ -429,6 +437,51 @@ export class Common {
|
|||
return false;
|
||||
}
|
||||
|
||||
// OP_RETURN size & count limits were lifted in v28.3/v29.2/v30.0
|
||||
static OP_RETURN_STANDARDNESS_ACTIVATION_HEIGHT = {
|
||||
'testnet4': 108_000,
|
||||
'testnet': 4_750_000,
|
||||
'signet': 276_500,
|
||||
'regtest': 0,
|
||||
'': 921_000,
|
||||
};
|
||||
static MAX_DATACARRIER_BYTES = 83;
|
||||
static isStandardOpReturn(bytes: number, outputs: number,height?: number): boolean {
|
||||
if (
|
||||
(height == null || (
|
||||
this.OP_RETURN_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
|
||||
&& height >= this.OP_RETURN_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
|
||||
)) // limits lifted
|
||||
|| // OR
|
||||
(bytes <= this.MAX_DATACARRIER_BYTES && outputs <= 1) // below old limits
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// New legacy sigops limit started to be enforced in v30.0
|
||||
static LEGACY_SIGOPS_STANDARDNESS_ACTIVATION_HEIGHT = {
|
||||
'testnet4': 108_000,
|
||||
'testnet': 4_750_000,
|
||||
'signet': 276_500,
|
||||
'regtest': 0,
|
||||
'': 921_000,
|
||||
};
|
||||
static isNonStandardLegacySigops(tx: TransactionExtended, height?: number): boolean {
|
||||
if (
|
||||
height == null || (
|
||||
this.LEGACY_SIGOPS_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
|
||||
&& height >= this.LEGACY_SIGOPS_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
|
||||
)
|
||||
) {
|
||||
if (!transactionUtils.checkSigopsBIP54(tx, MAX_TX_LEGACY_SIGOPS)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static getNonWitnessSize(tx: TransactionExtended): number {
|
||||
let weight = tx.weight;
|
||||
let hasWitness = false;
|
||||
|
|
@ -460,7 +513,7 @@ export class Common {
|
|||
}
|
||||
|
||||
static setLegacySighashFlags(flags: bigint, scriptsig_asm: string): bigint {
|
||||
for (const item of scriptsig_asm.split(' ')) {
|
||||
for (const item of scriptsig_asm?.split(' ') ?? []) {
|
||||
// skip op_codes
|
||||
if (item.startsWith('OP_')) {
|
||||
continue;
|
||||
|
|
@ -749,6 +802,7 @@ export class Common {
|
|||
return txs.map(Common.stripTransaction);
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
static sleep$(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(() => {
|
||||
|
|
@ -806,7 +860,7 @@ export class Common {
|
|||
|
||||
static indexingEnabled(): boolean {
|
||||
return (
|
||||
['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) &&
|
||||
['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) &&
|
||||
config.DATABASE.ENABLED === true &&
|
||||
config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== 0
|
||||
);
|
||||
|
|
@ -863,7 +917,7 @@ export class Common {
|
|||
if (id.indexOf('/') !== -1) {
|
||||
id = id.slice(0, -2);
|
||||
}
|
||||
|
||||
|
||||
if (id.indexOf('x') !== -1) { // Already a short id
|
||||
return id;
|
||||
}
|
||||
|
|
@ -885,6 +939,13 @@ export class Common {
|
|||
}
|
||||
|
||||
static findSocketNetwork(addr: string): {network: string | null, url: string} {
|
||||
if (!addr?.length) {
|
||||
return {
|
||||
network: null,
|
||||
url: ''
|
||||
};
|
||||
}
|
||||
|
||||
let network: string | null = null;
|
||||
let url: string = addr;
|
||||
|
||||
|
|
@ -892,7 +953,7 @@ export class Common {
|
|||
url = addr.split('://')[1];
|
||||
}
|
||||
|
||||
if (!url) {
|
||||
if (!url?.length) {
|
||||
return {
|
||||
network: null,
|
||||
url: addr,
|
||||
|
|
@ -918,7 +979,15 @@ export class Common {
|
|||
};
|
||||
}
|
||||
} else if (addr.indexOf('ipv6') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && url.indexOf(']:'))) {
|
||||
url = url.split('[')[1].split(']')[0];
|
||||
const parts = url.split('[');
|
||||
if (parts.length < 2) {
|
||||
return {
|
||||
network: null,
|
||||
url: addr,
|
||||
};
|
||||
} else {
|
||||
url = parts[1].split(']')[0];
|
||||
}
|
||||
const ipv = isIP(url);
|
||||
if (ipv === 6) {
|
||||
const parts = addr.split(':');
|
||||
|
|
@ -1018,7 +1087,7 @@ export class Common {
|
|||
}
|
||||
|
||||
static getTransactionFromRequest(req: Request, form: boolean): string {
|
||||
let rawTx: any = typeof req.body === 'object' && form
|
||||
const rawTx: any = typeof req.body === 'object' && form
|
||||
? Object.values(req.body)[0] as any
|
||||
: req.body;
|
||||
if (typeof rawTx !== 'string') {
|
||||
|
|
@ -1119,7 +1188,7 @@ export class Common {
|
|||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
// Pass through the input string untouched
|
||||
|
|
@ -1157,14 +1226,14 @@ export class Common {
|
|||
/**
|
||||
* Class to calculate average fee rates of a list of transactions
|
||||
* at certain weight percentiles, in a single pass
|
||||
*
|
||||
*
|
||||
* init with:
|
||||
* maxWeight - the total weight to measure percentiles relative to (e.g. 4MW for a single block)
|
||||
* percentileBandWidth - how many weight units to average over for each percentile (as a % of maxWeight)
|
||||
* percentiles - an array of weight percentiles to compute, in %
|
||||
*
|
||||
*
|
||||
* then call .processNext(tx) for each transaction, in descending order
|
||||
*
|
||||
*
|
||||
* retrieve the final results with .getFeeStats()
|
||||
*/
|
||||
export class OnlineFeeStatsCalculator {
|
||||
|
|
|
|||
|
|
@ -236,7 +236,7 @@ export function calculateMempoolTxCpfp(tx: MempoolTransactionExtended, mempool:
|
|||
/**
|
||||
* Given a root transaction and a list of in-mempool ancestors,
|
||||
* Calculate the CPFP cluster
|
||||
*
|
||||
*
|
||||
* @param tx
|
||||
* @param ancestors
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
|
|||
import { RowDataPacket } from 'mysql2';
|
||||
|
||||
class DatabaseMigration {
|
||||
private static currentVersion = 104;
|
||||
private static currentVersion = 106;
|
||||
private queryTimeout = 3600_000;
|
||||
private statisticsAddedIndexed = false;
|
||||
private uniqueLogs: string[] = [];
|
||||
|
|
@ -28,6 +28,7 @@ class DatabaseMigration {
|
|||
|
||||
/**
|
||||
* Entry point
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async $initializeOrMigrateDatabase(): Promise<void> {
|
||||
logger.debug('MIGRATIONS: Running migrations');
|
||||
|
|
@ -100,11 +101,12 @@ class DatabaseMigration {
|
|||
|
||||
/**
|
||||
* Create all missing tables
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
private async $createMissingTablesAndIndexes(databaseSchemaVersion: number) {
|
||||
await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion);
|
||||
|
||||
const isBitcoin = ['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK);
|
||||
const isBitcoin = ['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK);
|
||||
|
||||
await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
|
||||
await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
|
||||
|
|
@ -566,8 +568,8 @@ class DatabaseMigration {
|
|||
await this.$executeQuery('ALTER TABLE `blocks_templates` ADD INDEX `version` (`version`)');
|
||||
await this.updateToSchemaVersion(67);
|
||||
}
|
||||
|
||||
if (databaseSchemaVersion < 68 && config.MEMPOOL.NETWORK === "liquid") {
|
||||
|
||||
if (databaseSchemaVersion < 68 && config.MEMPOOL.NETWORK === 'liquid') {
|
||||
await this.$executeQuery('TRUNCATE TABLE elements_pegs');
|
||||
await this.$executeQuery('ALTER TABLE elements_pegs ADD PRIMARY KEY (txid, txindex);');
|
||||
await this.$executeQuery(`UPDATE state SET number = 0 WHERE name = 'last_elements_block';`);
|
||||
|
|
@ -931,24 +933,24 @@ class DatabaseMigration {
|
|||
|
||||
// Version 34
|
||||
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_tor_nodes int(11) NOT NULL DEFAULT "0"');
|
||||
|
||||
|
||||
// Version 35
|
||||
await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"');
|
||||
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);');
|
||||
|
||||
// Version 36
|
||||
await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"');
|
||||
|
||||
|
||||
// Version 37
|
||||
await this.$executeQuery(this.getCreateLNNodesSocketsTableQuery(), await this.$checkIfTableExists('nodes_sockets'));
|
||||
|
||||
|
||||
// Version 38
|
||||
await this.$executeQuery(`TRUNCATE lightning_stats`);
|
||||
await this.$executeQuery(`TRUNCATE node_stats`);
|
||||
await this.$executeQuery('ALTER TABLE `lightning_stats` CHANGE `added` `added` timestamp NULL');
|
||||
await this.$executeQuery('ALTER TABLE `node_stats` CHANGE `added` `added` timestamp NULL');
|
||||
await this.updateToSchemaVersion(38);
|
||||
|
||||
|
||||
// Version 39
|
||||
await this.$executeQuery('ALTER TABLE `nodes` ADD alias_search TEXT NULL DEFAULT NULL AFTER `alias`');
|
||||
await this.$executeQuery('ALTER TABLE nodes ADD FULLTEXT(alias_search)');
|
||||
|
|
@ -963,7 +965,7 @@ class DatabaseMigration {
|
|||
|
||||
// Version 42
|
||||
await this.$executeQuery('ALTER TABLE `channels` ADD closing_resolved tinyint(1) DEFAULT 0');
|
||||
|
||||
|
||||
// Version 43
|
||||
await this.$executeQuery(this.getCreateLNNodeRecordsTableQuery(), await this.$checkIfTableExists('nodes_records'));
|
||||
|
||||
|
|
@ -972,7 +974,7 @@ class DatabaseMigration {
|
|||
|
||||
// Version 45
|
||||
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fresh_txs JSON DEFAULT "[]"');
|
||||
|
||||
|
||||
// Version 48
|
||||
await this.$executeQuery('ALTER TABLE `channels` ADD source_checked tinyint(1) DEFAULT 0');
|
||||
await this.$executeQuery('ALTER TABLE `channels` ADD closing_fee bigint(20) unsigned DEFAULT 0');
|
||||
|
|
@ -1002,13 +1004,13 @@ class DatabaseMigration {
|
|||
// Version 62
|
||||
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD expected_fees BIGINT UNSIGNED DEFAULT NULL');
|
||||
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD expected_weight BIGINT UNSIGNED DEFAULT NULL');
|
||||
|
||||
|
||||
// Version 63
|
||||
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fullrbf_txs JSON DEFAULT "[]"');
|
||||
|
||||
|
||||
// Version 64
|
||||
await this.$executeQuery('ALTER TABLE `nodes` ADD features text NULL');
|
||||
|
||||
|
||||
// Version 65
|
||||
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD accelerated_txs JSON DEFAULT "[]"');
|
||||
|
||||
|
|
@ -1044,8 +1046,8 @@ class DatabaseMigration {
|
|||
ADD INDEX \`closing_reason\` (\`closing_reason\`),
|
||||
ADD INDEX \`closing_resolved\` (\`closing_resolved\`)
|
||||
`);
|
||||
|
||||
// Version 86
|
||||
|
||||
// Version 86
|
||||
await this.$executeQuery(`
|
||||
ALTER TABLE \`nodes\`
|
||||
ADD INDEX \`status\` (\`status\`),
|
||||
|
|
@ -1058,20 +1060,20 @@ class DatabaseMigration {
|
|||
// Version 87
|
||||
await this.$executeQuery('ALTER TABLE `nodes_sockets` ADD INDEX `type` (`type`)');
|
||||
await this.updateToSchemaVersion(87);
|
||||
|
||||
|
||||
// Version 88
|
||||
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD INDEX `added` (`added`)');
|
||||
|
||||
|
||||
// Version 89
|
||||
await this.$executeQuery('ALTER TABLE `geo_names` ADD INDEX `names` (`names`)');
|
||||
|
||||
|
||||
// Version 90
|
||||
await this.$executeQuery('ALTER TABLE `hashrates` ADD INDEX `type` (`type`)');
|
||||
|
||||
// Version 91
|
||||
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD INDEX `time` (`time`)');
|
||||
}
|
||||
|
||||
|
||||
if (config.MEMPOOL.NETWORK !== 'liquid') {
|
||||
// Apply all the liquid specific migrations to all other networks
|
||||
// Version 68
|
||||
|
|
@ -1093,7 +1095,7 @@ class DatabaseMigration {
|
|||
ADD INDEX \`bitcoinaddress\` (\`bitcoinaddress\`),
|
||||
ADD INDEX \`bitcointxid\` (\`bitcointxid\`)
|
||||
`);
|
||||
|
||||
|
||||
// Version 93
|
||||
await this.$executeQuery(`
|
||||
ALTER TABLE \`federation_txos\`
|
||||
|
|
@ -1178,6 +1180,48 @@ class DatabaseMigration {
|
|||
await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `stale` (`stale`)');
|
||||
await this.updateToSchemaVersion(103);
|
||||
}
|
||||
|
||||
// reindex liquid federation addresses and txos when needed, and add hardcoded federation addresses
|
||||
// (safe to make this conditional on the network since it doesn't change the database schema)
|
||||
if (databaseSchemaVersion < 105 && config.MEMPOOL.NETWORK === 'liquid') {
|
||||
// Hardcoded federation addresses
|
||||
await this.$executeQuery(`INSERT IGNORE INTO federation_addresses (bitcoinaddress) VALUES ('3G6neksSBMp51kHJ2if8SeDUrzT8iVETWT')`);
|
||||
await this.$executeQuery(`INSERT IGNORE INTO federation_addresses (bitcoinaddress) VALUES ('bc1qwnevjp8nsq7adu3hxlvdvslrf242q4vuavfg0y929jp2zntp3vgq7cq6z2')`);
|
||||
|
||||
// Rollback only on up to date instances
|
||||
const [stateRows]: any[] = await DB.query(`SELECT name, number FROM state WHERE name IN ('last_elements_block', 'last_bitcoin_block_audit')`);
|
||||
const lastElementsBlock = Number(stateRows?.find((row: any) => row.name === 'last_elements_block')?.number ?? 0);
|
||||
const lastBlockAudit = Number(stateRows?.find((row: any) => row.name === 'last_bitcoin_block_audit')?.number ?? 0);
|
||||
if (lastElementsBlock > 3686608 && lastBlockAudit > 929700) {
|
||||
await this.$executeQuery('DELETE FROM elements_pegs WHERE block > 3686608');
|
||||
await this.$executeQuery('DELETE FROM federation_txos WHERE blocknumber > 929701');
|
||||
await this.$executeQuery(`UPDATE federation_txos SET lastblockupdate = 929700 WHERE unspent = 1;`);
|
||||
await this.$executeQuery(`UPDATE state SET number = 3686608 WHERE name = 'last_elements_block';`);
|
||||
await this.$executeQuery(`UPDATE state SET number = 929700 WHERE name = 'last_bitcoin_block_audit';`);
|
||||
}
|
||||
await this.updateToSchemaVersion(105);
|
||||
}
|
||||
|
||||
// another liquid failure, fix bad timelocks on federation txos
|
||||
// (safe to make this conditional on the network since it doesn't change the database schema)
|
||||
if (databaseSchemaVersion < 106 && config.MEMPOOL.NETWORK === 'liquid') {
|
||||
// In a specific setup it's possible that 3G6neksSBMp51kHJ2if8SeDUrzT8iVETWT and bc1qwnevjp8nsq7adu3hxlvdvslrf242q4vuavfg0y929jp2zntp3vgq7cq6z2
|
||||
// were set with a timelock of 2016 instead of 4032
|
||||
// This rollbacks the tables to before bc1qwnevjp8nsq7adu3hxlvdvslrf242q4vuavfg0y929jp2zntp3vgq7cq6z2 is used, and
|
||||
// manually fixes the timelock for 3G6neksSBMp51kHJ2if8SeDUrzT8iVETWT
|
||||
const [stateRows]: any[] = await DB.query(`SELECT name, number FROM state WHERE name IN ('last_elements_block', 'last_bitcoin_block_audit')`);
|
||||
const lastElementsBlock = Number(stateRows?.find((row: any) => row.name === 'last_elements_block')?.number ?? 0);
|
||||
const lastBlockAudit = Number(stateRows?.find((row: any) => row.name === 'last_bitcoin_block_audit')?.number ?? 0);
|
||||
if (lastElementsBlock > 3686608 && lastBlockAudit > 929700) {
|
||||
await this.$executeQuery('DELETE FROM elements_pegs WHERE block > 3686608');
|
||||
await this.$executeQuery('DELETE FROM federation_txos WHERE blocknumber > 929701');
|
||||
await this.$executeQuery(`UPDATE federation_txos SET lastblockupdate = 929700 WHERE unspent = 1;`);
|
||||
await this.$executeQuery(`UPDATE federation_txos SET timelock = 4032 WHERE bitcoinaddress = '3G6neksSBMp51kHJ2if8SeDUrzT8iVETWT';`);
|
||||
await this.$executeQuery(`UPDATE state SET number = 3686608 WHERE name = 'last_elements_block';`);
|
||||
await this.$executeQuery(`UPDATE state SET number = 929700 WHERE name = 'last_bitcoin_block_audit';`);
|
||||
}
|
||||
await this.updateToSchemaVersion(106);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1224,6 +1268,7 @@ class DatabaseMigration {
|
|||
|
||||
/**
|
||||
* Check if 'table' exists in the database
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
private async $checkIfTableExists(table: string): Promise<boolean> {
|
||||
const query = `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '${config.DATABASE.DATABASE}' AND TABLE_NAME = '${table}'`;
|
||||
|
|
@ -1233,6 +1278,7 @@ class DatabaseMigration {
|
|||
|
||||
/**
|
||||
* Get current database version
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
private async $getSchemaVersionFromDatabase(): Promise<number> {
|
||||
const query = `SELECT number FROM state WHERE name = 'schema_version';`;
|
||||
|
|
@ -1242,6 +1288,7 @@ class DatabaseMigration {
|
|||
|
||||
/**
|
||||
* Create the `state` table
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
private async $createMigrationStateTable(): Promise<void> {
|
||||
const query = `CREATE TABLE IF NOT EXISTS state (
|
||||
|
|
@ -1259,6 +1306,7 @@ class DatabaseMigration {
|
|||
|
||||
/**
|
||||
* We actually execute the migrations queries here
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
private async $migrateTableSchemaFromVersion(version: number): Promise<void> {
|
||||
const transactionQueries: string[] = [];
|
||||
|
|
@ -1286,7 +1334,7 @@ class DatabaseMigration {
|
|||
*/
|
||||
private getMigrationQueriesFromVersion(version: number): string[] {
|
||||
const queries: string[] = [];
|
||||
const isBitcoin = ['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK);
|
||||
const isBitcoin = ['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK);
|
||||
|
||||
if (version < 1) {
|
||||
if (config.MEMPOOL.NETWORK !== 'liquid' && config.MEMPOOL.NETWORK !== 'liquidtestnet') {
|
||||
|
|
@ -1325,11 +1373,13 @@ class DatabaseMigration {
|
|||
|
||||
/**
|
||||
* Save the schema version in the database
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
private getUpdateToLatestSchemaVersionQuery(): string {
|
||||
return `UPDATE state SET number = ${DatabaseMigration.currentVersion} WHERE name = 'schema_version';`;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
private async updateToSchemaVersion(version): Promise<void> {
|
||||
await this.$executeQuery(`UPDATE state SET number = ${version} WHERE name = 'schema_version';`);
|
||||
}
|
||||
|
|
@ -1456,7 +1506,7 @@ class DatabaseMigration {
|
|||
pegtxid varchar(65) NOT NULL,
|
||||
pegindex int(11) NOT NULL,
|
||||
pegblocktime int(11) unsigned NOT NULL,
|
||||
PRIMARY KEY (txid, txindex),
|
||||
PRIMARY KEY (txid, txindex),
|
||||
FOREIGN KEY (bitcoinaddress) REFERENCES federation_addresses (bitcoinaddress)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
|
||||
}
|
||||
|
|
@ -1748,6 +1798,7 @@ class DatabaseMigration {
|
|||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $blocksReindexingTruncate(): Promise<void> {
|
||||
logger.warn(`Truncating pools, blocks, hashrates and difficulty_adjustments tables for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
|
||||
await Common.sleep$(5000);
|
||||
|
|
|
|||
|
|
@ -33,11 +33,12 @@ class DiskCache {
|
|||
return;
|
||||
}
|
||||
process.on('SIGINT', (e) => {
|
||||
this.$saveCacheToDisk(true);
|
||||
void this.$saveCacheToDisk(true);
|
||||
process.exit(0);
|
||||
});
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
async $saveCacheToDisk(sync: boolean = false): Promise<void> {
|
||||
if (!cluster.isPrimary || !config.MEMPOOL.CACHE_ENABLED) {
|
||||
return;
|
||||
|
|
@ -174,6 +175,7 @@ class DiskCache {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
async $loadMempoolCache(): Promise<void> {
|
||||
if (!config.MEMPOOL.CACHE_ENABLED || !fs.existsSync(DiskCache.FILE_NAME)) {
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ class ChannelsApi {
|
|||
GROUP BY nodes_1.public_key, nodes_2.public_key
|
||||
ORDER BY channels.capacity DESC
|
||||
LIMIT 10000
|
||||
`;
|
||||
`;
|
||||
}
|
||||
|
||||
const [rows]: any = await DB.query(query, params);
|
||||
|
|
@ -241,10 +241,10 @@ class ChannelsApi {
|
|||
let [feeRates2]: any = await DB.query(query);
|
||||
feeRates2 = feeRates2.map(rate => rate.node2_fee_rate);
|
||||
|
||||
let feeRates = (feeRates1.concat(feeRates2)).sort((a, b) => a - b);
|
||||
const feeRates = (feeRates1.concat(feeRates2)).sort((a, b) => a - b);
|
||||
let avgFeeRate = 0;
|
||||
for (const rate of feeRates) {
|
||||
avgFeeRate += rate;
|
||||
avgFeeRate += rate;
|
||||
}
|
||||
avgFeeRate /= feeRates.length;
|
||||
const medianFeeRate = feeRates[Math.floor(feeRates.length / 2)];
|
||||
|
|
@ -257,14 +257,14 @@ class ChannelsApi {
|
|||
let [baseFees2]: any = await DB.query(query);
|
||||
baseFees2 = baseFees2.map(rate => rate.node2_base_fee_mtokens);
|
||||
|
||||
let baseFees = (baseFees1.concat(baseFees2)).sort((a, b) => a - b);
|
||||
const baseFees = (baseFees1.concat(baseFees2)).sort((a, b) => a - b);
|
||||
let avgBaseFee = 0;
|
||||
for (const fee of baseFees) {
|
||||
avgBaseFee += fee;
|
||||
avgBaseFee += fee;
|
||||
}
|
||||
avgBaseFee /= baseFees.length;
|
||||
const medianBaseFee = feeRates[Math.floor(baseFees.length / 2)];
|
||||
|
||||
|
||||
return {
|
||||
avgCapacity: parseInt(avgCapacity[0].avgCapacity, 10),
|
||||
avgFeeRate: avgFeeRate,
|
||||
|
|
@ -272,7 +272,7 @@ class ChannelsApi {
|
|||
medianCapacity: medianCapacity,
|
||||
medianFeeRate: medianFeeRate,
|
||||
medianBaseFee: medianBaseFee,
|
||||
}
|
||||
};
|
||||
|
||||
} catch (e) {
|
||||
logger.err(`Cannot calculate channels statistics. Reason: ${e instanceof Error ? e.message : e}`);
|
||||
|
|
@ -298,6 +298,7 @@ class ChannelsApi {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getChannelByClosingId(transactionId: string): Promise<any> {
|
||||
try {
|
||||
const query = `
|
||||
|
|
@ -338,6 +339,7 @@ class ChannelsApi {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $updateClosingInfo(channelInfo: { id: string, node1_closing_balance: number, node2_closing_balance: number, closed_by: string | null, closing_fee: number, outputs: ILightningApi.ForensicOutput[]}): Promise<void> {
|
||||
try {
|
||||
const query = `
|
||||
|
|
@ -363,6 +365,7 @@ class ChannelsApi {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $updateOpeningInfo(channelInfo: { id: string, node1_funding_balance: number, node2_funding_balance: number, funding_ratio: number, single_funded: boolean | void }): Promise<void> {
|
||||
try {
|
||||
const query = `
|
||||
|
|
@ -456,7 +459,7 @@ class ChannelsApi {
|
|||
allChannels = allChannels.slice(0, 1000);
|
||||
}
|
||||
|
||||
const channels: any[] = []
|
||||
const channels: any[] = [];
|
||||
for (const row of allChannels) {
|
||||
let channel;
|
||||
if (index >= 0) {
|
||||
|
|
@ -578,8 +581,12 @@ class ChannelsApi {
|
|||
|
||||
/**
|
||||
* Save or update a channel present in the graph
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async $saveChannel(channel: ILightningApi.Channel, status = 1): Promise<void> {
|
||||
if (!channel.chan_point?.length) {
|
||||
return;
|
||||
}
|
||||
const [ txid, vout ] = channel.chan_point.split(':');
|
||||
|
||||
const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
|
||||
|
|
@ -714,6 +721,7 @@ class ChannelsApi {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getLatestChannelUpdateForNode(publicKey: string): Promise<number> {
|
||||
try {
|
||||
const query = `
|
||||
|
|
|
|||
|
|
@ -78,17 +78,14 @@ class ChannelsRoutes {
|
|||
|
||||
private async $getChannelsByTransactionIds(req: Request, res: Response): Promise<void> {
|
||||
try {
|
||||
if (!Array.isArray(req.query.txId)) {
|
||||
handleError(req, res, 400, 'Not an array');
|
||||
if (!req.query.txId || typeof req.query.txId !== 'object') {
|
||||
handleError(req, res, 400, 'invalid txId format');
|
||||
return;
|
||||
}
|
||||
const txIds: string[] = [];
|
||||
for (const _txId in req.query.txId) {
|
||||
if (typeof req.query.txId[_txId] === 'string') {
|
||||
const txid = req.query.txId[_txId].toString();
|
||||
if (TXID_REGEX.test(txid)) {
|
||||
txIds.push(txid);
|
||||
}
|
||||
for (const txid of Object.values(req.query.txId)) {
|
||||
if (typeof txid === 'string' && TXID_REGEX.test(txid)) {
|
||||
txIds.push(txid);
|
||||
}
|
||||
}
|
||||
const channels = await channelsApi.$getChannelsByTransactionId(txIds);
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ class NodesApi {
|
|||
`;
|
||||
|
||||
const [maximums]: any[] = await DB.query(query);
|
||||
|
||||
|
||||
return {
|
||||
maxLiquidity: maximums[0].maxLiquidity,
|
||||
maxChannels: maximums[0].maxChannels,
|
||||
|
|
@ -78,7 +78,7 @@ class NodesApi {
|
|||
node.city = JSON.parse(node.city);
|
||||
node.country = JSON.parse(node.country);
|
||||
|
||||
// Features
|
||||
// Features
|
||||
node.features = JSON.parse(node.features);
|
||||
node.featuresBits = null;
|
||||
if (node.features) {
|
||||
|
|
@ -87,7 +87,7 @@ class NodesApi {
|
|||
maxBit = Math.max(maxBit, feature.bit);
|
||||
}
|
||||
maxBit = Math.ceil(maxBit / 4) * 4 - 1;
|
||||
|
||||
|
||||
node.featuresBits = new Array(maxBit + 1).fill(0);
|
||||
for (const feature of node.features) {
|
||||
node.featuresBits[feature.bit] = 1;
|
||||
|
|
@ -143,6 +143,7 @@ class NodesApi {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $getActiveChannelsStats(node_public_key: string): Promise<unknown> {
|
||||
const query = `
|
||||
SELECT count(short_id) as active_channel_count, sum(capacity) as capacity
|
||||
|
|
@ -394,7 +395,7 @@ class NodesApi {
|
|||
try {
|
||||
const publicKeySearch = search.replace(/[^a-zA-Z0-9]/g, '') + '%';
|
||||
const aliasSearch = search
|
||||
.replace(/[-_.]/g, ' ') // Replace all -_. characters with empty space. Eg: "ln.nicehash" becomes "ln nicehash".
|
||||
.replace(/[-_.]/g, ' ') // Replace all -_. characters with empty space. Eg: "ln.nicehash" becomes "ln nicehash".
|
||||
.replace(/[^a-zA-Z0-9 ]/g, '') // Remove all special characters and keep just A to Z, 0 to 9.
|
||||
.split(' ')
|
||||
.filter(key => key.length)
|
||||
|
|
@ -455,7 +456,7 @@ class NodesApi {
|
|||
} else if (ispList[isp2].ids.includes(channel.isp2ID) === false) {
|
||||
ispList[isp2].ids.push(channel.isp2ID);
|
||||
}
|
||||
|
||||
|
||||
ispList[isp1].capacity += channel.capacity;
|
||||
ispList[isp1].channels += 1;
|
||||
ispList[isp1].nodes[channel.node1PublicKey] = true;
|
||||
|
|
@ -463,7 +464,7 @@ class NodesApi {
|
|||
ispList[isp2].channels += 1;
|
||||
ispList[isp2].nodes[channel.node2PublicKey] = true;
|
||||
}
|
||||
|
||||
|
||||
const ispRanking: any[] = [];
|
||||
for (const isp of Object.keys(ispList)) {
|
||||
ispRanking.push([
|
||||
|
|
@ -494,7 +495,7 @@ class NodesApi {
|
|||
`;
|
||||
const [clearnetCapacity]: any = await DB.query(query);
|
||||
|
||||
// Get the total capacity of all channels which have both nodes on Tor
|
||||
// Get the total capacity of all channels which have both nodes on Tor
|
||||
query = `
|
||||
SELECT SUM(capacity) as capacity
|
||||
FROM (
|
||||
|
|
@ -642,11 +643,11 @@ class NodesApi {
|
|||
for (const country of nodesCountPerCountry) {
|
||||
nodesPerCountry.push({
|
||||
name: JSON.parse(country.names),
|
||||
iso: country.iso_code,
|
||||
iso: country.iso_code,
|
||||
count: country.nodesCount,
|
||||
share: Math.floor(country.nodesCount / nodesWithAS[0].total * 10000) / 100,
|
||||
capacity: country.capacity,
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
return nodesPerCountry;
|
||||
|
|
@ -658,6 +659,7 @@ class NodesApi {
|
|||
|
||||
/**
|
||||
* Save or update a node present in the graph
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $saveNode(node: ILightningApi.Node): Promise<void> {
|
||||
try {
|
||||
|
|
@ -665,7 +667,7 @@ class NodesApi {
|
|||
if ((node.last_update ?? 0) < 1514736061) { // January 1st 2018
|
||||
node.last_update = null;
|
||||
}
|
||||
|
||||
|
||||
const uniqueAddr = [...new Set(node.addresses?.map(a => a.addr))];
|
||||
const formattedSockets = (uniqueAddr.join(',')) ?? '';
|
||||
|
||||
|
|
@ -727,6 +729,7 @@ class NodesApi {
|
|||
|
||||
/**
|
||||
* Set all nodes not in `nodesPubkeys` as inactive (status = 0)
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $setNodesInactive(graphNodesPubkeys: string[]): Promise<void> {
|
||||
if (graphNodesPubkeys.length === 0) {
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ class NodesRoutes {
|
|||
private async $getNodeGroup(req: Request, res: Response) {
|
||||
try {
|
||||
let nodesList;
|
||||
let nodes: any[] = [];
|
||||
const nodes: any[] = [];
|
||||
switch (config.MEMPOOL.NETWORK) {
|
||||
case 'testnet':
|
||||
nodesList = [
|
||||
|
|
@ -174,7 +174,7 @@ class NodesRoutes {
|
|||
];
|
||||
}
|
||||
|
||||
for (let pubKey of nodesList) {
|
||||
for (const pubKey of nodesList) {
|
||||
try {
|
||||
const node = await nodesApi.$getNode(pubKey);
|
||||
if (node) {
|
||||
|
|
@ -354,7 +354,7 @@ class NodesRoutes {
|
|||
return;
|
||||
}
|
||||
|
||||
const nodes = await nodesApi.$getNodesPerISP(req.params.isp);
|
||||
const nodes = await nodesApi.$getNodesPerISP(req.params.isp || '');
|
||||
res.header('Pragma', 'public');
|
||||
res.header('Cache-control', 'public');
|
||||
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
|
||||
|
|
|
|||
|
|
@ -17,8 +17,10 @@ interface RecommendedFees {
|
|||
class FeeApi {
|
||||
constructor() { }
|
||||
|
||||
defaultFee = isLiquid ? 0.1 : 1;
|
||||
minimumIncrement = isLiquid ? 0.1 : 1;
|
||||
minFastestFee = isLiquid ? 0.1 : 1;
|
||||
minHalfHourFee = isLiquid ? 0.1 : 0.5;
|
||||
priorityFactor = isLiquid ? 0 : 0.5;
|
||||
|
||||
public getRecommendedFee(): RecommendedFees {
|
||||
const pBlocks = projectedBlocks.getMempoolBlocks();
|
||||
|
|
@ -27,24 +29,46 @@ class FeeApi {
|
|||
return this.calculateRecommendedFee(pBlocks, mPool);
|
||||
}
|
||||
|
||||
public calculateRecommendedFee(pBlocks: MempoolBlock[], mPool: IBitcoinApi.MempoolInfo): RecommendedFees {
|
||||
const minimumFee = this.roundUpToNearest(mPool.mempoolminfee * 100000, this.minimumIncrement);
|
||||
const defaultMinFee = Math.max(minimumFee, this.defaultFee);
|
||||
public getPreciseRecommendedFee(): RecommendedFees {
|
||||
const pBlocks = projectedBlocks.getMempoolBlocks();
|
||||
const mPool = mempool.getMempoolInfo();
|
||||
|
||||
// minimum non-zero minrelaytxfee / incrementalrelayfee is 1 sat/kvB = 0.001 sat/vB
|
||||
const recommendations = this.calculateRecommendedFee(pBlocks, mPool, 0.001);
|
||||
// enforce floor & offset for highest priority recommendations while <100% hashrate accepts sub-sat fees
|
||||
recommendations.fastestFee = Math.max(recommendations.fastestFee + this.priorityFactor, this.minFastestFee);
|
||||
recommendations.halfHourFee = Math.max(recommendations.halfHourFee + (this.priorityFactor / 2), this.minHalfHourFee);
|
||||
return {
|
||||
'fastestFee': Math.round(recommendations.fastestFee * 1000) / 1000,
|
||||
'halfHourFee': Math.round(recommendations.halfHourFee * 1000) / 1000,
|
||||
'hourFee': Math.round(recommendations.hourFee * 1000) / 1000,
|
||||
'economyFee': Math.round(recommendations.economyFee * 1000) / 1000,
|
||||
'minimumFee': Math.round(recommendations.minimumFee * 1000) / 1000,
|
||||
};
|
||||
}
|
||||
|
||||
public calculateRecommendedFee(pBlocks: MempoolBlock[], mPool: IBitcoinApi.MempoolInfo, minIncrement: number = this.minimumIncrement): RecommendedFees {
|
||||
const purgeRate = this.roundUpToNearest(mPool.mempoolminfee * 100000, minIncrement);
|
||||
const minimumFee = Math.max(purgeRate, minIncrement);
|
||||
|
||||
if (!pBlocks.length) {
|
||||
return {
|
||||
'fastestFee': defaultMinFee,
|
||||
'halfHourFee': defaultMinFee,
|
||||
'hourFee': defaultMinFee,
|
||||
'fastestFee': minimumFee,
|
||||
'halfHourFee': minimumFee,
|
||||
'hourFee': minimumFee,
|
||||
'economyFee': minimumFee,
|
||||
'minimumFee': minimumFee,
|
||||
};
|
||||
}
|
||||
|
||||
const firstMedianFee = this.optimizeMedianFee(pBlocks[0], pBlocks[1]);
|
||||
const secondMedianFee = pBlocks[1] ? this.optimizeMedianFee(pBlocks[1], pBlocks[2], firstMedianFee) : this.defaultFee;
|
||||
const thirdMedianFee = pBlocks[2] ? this.optimizeMedianFee(pBlocks[2], pBlocks[3], secondMedianFee) : this.defaultFee;
|
||||
const firstMedianFee = this.optimizeMedianFee(pBlocks[0], pBlocks[1], undefined, minimumFee, minIncrement);
|
||||
const secondMedianFee = pBlocks[1] ? this.optimizeMedianFee(pBlocks[1], pBlocks[2], firstMedianFee, minimumFee, minIncrement) : minimumFee;
|
||||
const thirdMedianFee = pBlocks[2] ? this.optimizeMedianFee(pBlocks[2], pBlocks[3], secondMedianFee, minimumFee, minIncrement) : minimumFee;
|
||||
|
||||
// explicitly enforce a minimum of ceil(mempoolminfee) on all recommendations.
|
||||
// simply rounding up recommended rates is insufficient, as the purging rate
|
||||
// can exceed the median rate of projected blocks in some extreme scenarios
|
||||
// (see https://bitcoin.stackexchange.com/a/120024)
|
||||
let fastestFee = Math.max(minimumFee, firstMedianFee);
|
||||
let halfHourFee = Math.max(minimumFee, secondMedianFee);
|
||||
let hourFee = Math.max(minimumFee, thirdMedianFee);
|
||||
|
|
@ -55,33 +79,39 @@ class FeeApi {
|
|||
halfHourFee = Math.max(halfHourFee, hourFee, economyFee);
|
||||
hourFee = Math.max(hourFee, economyFee);
|
||||
|
||||
// explicitly enforce a minimum of ceil(mempoolminfee) on all recommendations.
|
||||
// simply rounding up recommended rates is insufficient, as the purging rate
|
||||
// can exceed the median rate of projected blocks in some extreme scenarios
|
||||
// (see https://bitcoin.stackexchange.com/a/120024)
|
||||
return {
|
||||
'fastestFee': fastestFee,
|
||||
'halfHourFee': halfHourFee,
|
||||
'hourFee': hourFee,
|
||||
'economyFee': economyFee,
|
||||
'minimumFee': minimumFee,
|
||||
'fastestFee': this.roundToNearest(fastestFee, minIncrement),
|
||||
'halfHourFee': this.roundToNearest(halfHourFee, minIncrement),
|
||||
'hourFee': this.roundToNearest(hourFee, minIncrement),
|
||||
'economyFee': this.roundToNearest(economyFee, minIncrement),
|
||||
'minimumFee': this.roundToNearest(minimumFee, minIncrement),
|
||||
};
|
||||
}
|
||||
|
||||
private optimizeMedianFee(pBlock: MempoolBlock, nextBlock: MempoolBlock | undefined, previousFee?: number): number {
|
||||
private optimizeMedianFee(pBlock: MempoolBlock, nextBlock: MempoolBlock | undefined, previousFee: number | undefined, minFee: number, minIncrement: number = this.minimumIncrement): number {
|
||||
const useFee = previousFee ? (pBlock.medianFee + previousFee) / 2 : pBlock.medianFee;
|
||||
if (pBlock.blockVSize <= 500000 || pBlock.medianFee < 1) {
|
||||
return this.defaultFee;
|
||||
if (pBlock.blockVSize <= 500000 || pBlock.medianFee < minFee) {
|
||||
return minFee;
|
||||
}
|
||||
if (pBlock.blockVSize <= 950000 && !nextBlock) {
|
||||
const multiplier = (pBlock.blockVSize - 500000) / 500000;
|
||||
return Math.max(Math.round(useFee * multiplier), this.defaultFee);
|
||||
return Math.max(this.roundToNearest(useFee * multiplier, minIncrement), minFee);
|
||||
}
|
||||
return this.roundUpToNearest(useFee, this.minimumIncrement);
|
||||
return Math.max(this.roundUpToNearest(useFee, minIncrement), minFee);
|
||||
}
|
||||
|
||||
private roundUpToNearest(value: number, nearest: number): number {
|
||||
return Math.ceil(value / nearest) * nearest;
|
||||
if (nearest !== 0) {
|
||||
return Math.ceil(value / nearest) * nearest;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private roundToNearest(value: number, nearest: number): number {
|
||||
if (nearest !== 0) {
|
||||
return Math.round(value / nearest) * nearest;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import fs from 'fs';
|
||||
import path from "path";
|
||||
import path from 'path';
|
||||
const { spawnSync } = require('child_process');
|
||||
|
||||
function getVersion(): string {
|
||||
|
|
@ -29,9 +29,9 @@ function getGitCommit(): string {
|
|||
const versionInfo = {
|
||||
version: getVersion(),
|
||||
gitCommit: getGitCommit()
|
||||
}
|
||||
};
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(__dirname, 'version.json'),
|
||||
JSON.stringify(versionInfo, null, 2) + "\n"
|
||||
JSON.stringify(versionInfo, null, 2) + '\n'
|
||||
);
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ class LightningError extends Error {
|
|||
|
||||
const defaultRpcPath = path.join(homedir(), '.lightning')
|
||||
, fStat = (...p) => statSync(path.join(...p))
|
||||
, fExists = (...p) => existsSync(path.join(...p))
|
||||
, fExists = (...p) => existsSync(path.join(...p));
|
||||
|
||||
export default class CLightningClient extends EventEmitter implements AbstractLightningApi {
|
||||
private rpcPath: string;
|
||||
|
|
@ -141,9 +141,9 @@ export default class CLightningClient extends EventEmitter implements AbstractLi
|
|||
// main data directory provided, default to using the bitcoin mainnet subdirectory
|
||||
// to be removed in v0.2.0
|
||||
else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) {
|
||||
logger.warn(`${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`, logger.tags.ln)
|
||||
logger.warn(`specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`, logger.tags.ln)
|
||||
rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc')
|
||||
logger.warn(`${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`, logger.tags.ln);
|
||||
logger.warn(`specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`, logger.tags.ln);
|
||||
rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc');
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -249,6 +249,7 @@ export default class CLightningClient extends EventEmitter implements AbstractLi
|
|||
}));
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
|
||||
const listnodes: any[] = await this.call('listnodes');
|
||||
const listchannels: any[] = await this.call('listchannels');
|
||||
|
|
|
|||
|
|
@ -156,6 +156,7 @@ export function convertNode(clNode: any): ILightningApi.Node {
|
|||
|
||||
/**
|
||||
* Convert clightning "listchannels" response to lnd "describegraph.edges" format
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
export async function convertAndmergeBidirectionalChannels(clChannels: any[]): Promise<ILightningApi.Channel[]> {
|
||||
logger.debug(`Converting clightning nodes and channels to lnd graph format`, logger.tags.ln);
|
||||
|
|
@ -212,6 +213,7 @@ export async function convertAndmergeBidirectionalChannels(clChannels: any[]): P
|
|||
/**
|
||||
* Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format
|
||||
* In this case, clightning knows the channel policy for both nodes
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILightningApi.Channel | null> {
|
||||
const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0);
|
||||
|
|
@ -238,6 +240,7 @@ async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILigh
|
|||
/**
|
||||
* Convert one clightning "getchannels" entry into a full a lnd "describegraph.edges" format
|
||||
* In this case, clightning knows the channel policy of only one node
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Channel | null> {
|
||||
const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannel.short_channel_id);
|
||||
|
|
|
|||
|
|
@ -40,16 +40,17 @@ class LndApi implements AbstractLightningApi {
|
|||
.then((response) => response.data);
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
|
||||
const graph = await axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
|
||||
.then((response) => response.data);
|
||||
|
||||
for (const node of graph.nodes) {
|
||||
const nodeFeatures: ILightningApi.Feature[] = [];
|
||||
for (const bit in node.features) {
|
||||
for (const bit in node.features) {
|
||||
nodeFeatures.push({
|
||||
bit: parseInt(bit, 10),
|
||||
name: node.features[bit].name,
|
||||
name: node.features[bit].name,
|
||||
is_required: node.features[bit].is_required,
|
||||
is_known: node.features[bit].is_known,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import { Common } from '../common';
|
|||
import DB from '../../database';
|
||||
import logger from '../../logger';
|
||||
|
||||
const federationChangeAddresses = ['bc1qxvay4an52gcghxq5lavact7r6qe9l4laedsazz8fj2ee2cy47tlqff4aj4', '3EiAcrzq1cELXScc98KeCswGWZaPGceT1d'];
|
||||
const federationChangeAddresses = ['bc1qxvay4an52gcghxq5lavact7r6qe9l4laedsazz8fj2ee2cy47tlqff4aj4', '3EiAcrzq1cELXScc98KeCswGWZaPGceT1d', '3G6neksSBMp51kHJ2if8SeDUrzT8iVETWT', 'bc1qwnevjp8nsq7adu3hxlvdvslrf242q4vuavfg0y929jp2zntp3vgq7cq6z2'];
|
||||
const auditBlockOffsetWithTip = 1; // Wait for 1 block confirmation before processing the block in the audit process to reduce the risk of reorgs
|
||||
|
||||
class ElementsParser {
|
||||
|
|
@ -36,6 +36,7 @@ class ElementsParser {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
protected async $parseBlock(block: IBitcoinApi.Block) {
|
||||
for (const tx of block.tx) {
|
||||
await this.$parseInputs(tx, block);
|
||||
|
|
@ -43,6 +44,7 @@ class ElementsParser {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
protected async $parseInputs(tx: IBitcoinApi.Transaction, block: IBitcoinApi.Block) {
|
||||
for (const [index, input] of tx.vin.entries()) {
|
||||
if (input.is_pegin) {
|
||||
|
|
@ -51,6 +53,7 @@ class ElementsParser {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
protected async $parsePegIn(input: IBitcoinApi.Vin, vindex: number, txid: string, block: IBitcoinApi.Block) {
|
||||
const bitcoinTx: IBitcoinApi.Transaction = await bitcoinSecondClient.getRawTransaction(input.txid, true);
|
||||
const bitcoinBlock: IBitcoinApi.Block = await bitcoinSecondClient.getBlock(bitcoinTx.blockhash);
|
||||
|
|
@ -60,6 +63,7 @@ class ElementsParser {
|
|||
outputAddress, bitcoinTx.txid, prevout.n, bitcoinBlock.height, bitcoinBlock.time, 1);
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
protected async $parseOutputs(tx: IBitcoinApi.Transaction, block: IBitcoinApi.Block) {
|
||||
for (const output of tx.vout) {
|
||||
if (output.scriptPubKey.pegout_chain) {
|
||||
|
|
@ -74,6 +78,7 @@ class ElementsParser {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string,
|
||||
txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, bitcoinblock: number, bitcoinBlockTime: number, final_tx: number): Promise<void> {
|
||||
const query = `INSERT IGNORE INTO elements_pegs(
|
||||
|
|
@ -87,7 +92,7 @@ class ElementsParser {
|
|||
logger.debug(`Saved L-BTC peg from Liquid block height #${height} with TXID ${txid}.`);
|
||||
|
||||
if (amount > 0) { // Peg-in
|
||||
|
||||
|
||||
// Add the address to the federation addresses table
|
||||
await DB.query(`INSERT IGNORE INTO federation_addresses (bitcoinaddress) VALUES (?)`, [bitcoinaddress]);
|
||||
|
||||
|
|
@ -95,19 +100,21 @@ class ElementsParser {
|
|||
const query_utxos = `INSERT IGNORE INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate, timelock, expiredAt, emergencyKey, pegtxid, pegindex, pegblocktime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
|
||||
const params_utxos: (string | number)[] = [bitcointxid, bitcoinindex, bitcoinaddress, amount, bitcoinblock, bitcoinBlockTime, 1, bitcoinblock - 1, 0, 4032, 0, 0, txid, txindex, blockTime];
|
||||
await DB.query(query_utxos, params_utxos);
|
||||
const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`)
|
||||
const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`);
|
||||
await this.$saveLastBlockAuditToDatabase(minBlockUpdate[0]['lastblockupdate']);
|
||||
logger.debug(`Saved new Federation UTXO ${bitcointxid}:${bitcoinindex} belonging to ${bitcoinaddress} to federation txos`);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
protected async $getLatestBlockHeightFromDatabase(): Promise<number> {
|
||||
const query = `SELECT number FROM state WHERE name = 'last_elements_block'`;
|
||||
const [rows] = await DB.query(query);
|
||||
return rows[0]['number'];
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
protected async $saveLatestBlockToDatabase(blockHeight: number) {
|
||||
const query = `UPDATE state SET number = ? WHERE name = 'last_elements_block'`;
|
||||
await DB.query(query, [blockHeight]);
|
||||
|
|
@ -174,7 +181,7 @@ class ElementsParser {
|
|||
const runningFor = (Date.now() / 1000) - startedAt;
|
||||
const blockPerSeconds = indexedThisRun / elapsedSeconds;
|
||||
indexingSpeeds.push(blockPerSeconds);
|
||||
if (indexingSpeeds.length > 100) indexingSpeeds.shift(); // Keep the length of the up to 100 last indexing speeds
|
||||
if (indexingSpeeds.length > 100) {indexingSpeeds.shift();} // Keep the length of the up to 100 last indexing speeds
|
||||
const meanIndexingSpeed = indexingSpeeds.reduce((a, b) => a + b, 0) / indexingSpeeds.length;
|
||||
const eta = (auditProgress.confirmedTip - auditProgress.lastBlockAudit) / meanIndexingSpeed;
|
||||
logger.debug(`Scanning ${utxos.length} Federation UTXOs and ${redeemAddresses.length} Peg-Out Addresses at Bitcoin block height #${auditProgress.lastBlockAudit} / #${auditProgress.confirmedTip} | ~${meanIndexingSpeed.toFixed(2)} blocks/sec | elapsed: ${(runningFor / 60).toFixed(0)} minutes | ETA: ${(eta / 60).toFixed(0)} minutes`);
|
||||
|
|
@ -189,7 +196,7 @@ class ElementsParser {
|
|||
await this.$parseBitcoinBlock(block, spentAsTip, unspentAsTip, auditProgress.confirmedTip, redeemAddresses);
|
||||
|
||||
// Finally, update the lastblockupdate of the remaining UTXOs and save to the database
|
||||
const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`)
|
||||
const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`);
|
||||
await this.$saveLastBlockAuditToDatabase(minBlockUpdate[0]['lastblockupdate']);
|
||||
|
||||
auditProgress = await this.$getAuditProgress();
|
||||
|
|
@ -201,17 +208,19 @@ class ElementsParser {
|
|||
} catch (e) {
|
||||
this.isUtxosUpdatingRunning = false;
|
||||
throw new Error(e instanceof Error ? e.message : 'Error');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get the UTXOs that need to be scanned in block height (UTXOs that were last updated in the block height - 1)
|
||||
protected async $getFederationUtxosToScan(height: number) {
|
||||
/** @asyncUnsafe */
|
||||
protected async $getFederationUtxosToScan(height: number) {
|
||||
const query = `SELECT txid, txindex, bitcoinaddress, amount, blocknumber, timelock, expiredAt FROM federation_txos WHERE lastblockupdate = ? AND unspent = 1`;
|
||||
const [rows] = await DB.query(query, [height - 1]);
|
||||
return rows as any[];
|
||||
}
|
||||
|
||||
// Returns the UTXOs that are spent as of tip and need to be scanned
|
||||
/** @asyncUnsafe */
|
||||
protected async $getFederationUtxosToParse(utxos: any[]): Promise<any> {
|
||||
const spentAsTip: any[] = [];
|
||||
const unspentAsTip: any[] = [];
|
||||
|
|
@ -220,10 +229,11 @@ class ElementsParser {
|
|||
const result = await bitcoinSecondClient.getTxOut(utxo.txid, utxo.txindex, false);
|
||||
result ? unspentAsTip.push(utxo) : spentAsTip.push(utxo);
|
||||
}
|
||||
|
||||
|
||||
return {spentAsTip, unspentAsTip};
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
protected async $parseBitcoinBlock(block: IBitcoinApi.Block, spentAsTip: any[], unspentAsTip: any[], confirmedTip: number, redeemAddressesData: any[] = []) {
|
||||
const redeemAddresses: string[] = redeemAddressesData.map(redeemAddress => redeemAddress.bitcoinaddress);
|
||||
for (const tx of block.tx) {
|
||||
|
|
@ -255,7 +265,7 @@ class ElementsParser {
|
|||
// Check that the UTXO was not already added in the DB by previous scans
|
||||
const [rows_check] = await DB.query(`SELECT txid FROM federation_txos WHERE txid = ? AND txindex = ?`, [tx.txid, output.n]) as any[];
|
||||
if (rows_check.length === 0) {
|
||||
const timelock = output.scriptPubKey.address === federationChangeAddresses[0] ? 4032 : 2016; // P2WSH change address has a 4032 timelock, P2SH change address has a 2016 timelock
|
||||
const timelock = output.scriptPubKey.address === federationChangeAddresses[1] ? 2016 : 4032; // hardcode timelock for 3EiAcrzq... This will be addressed better in the future
|
||||
const query_utxos = `INSERT INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate, timelock, expiredAt, emergencyKey, pegtxid, pegindex, pegblocktime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
|
||||
const params_utxos: (string | number)[] = [tx.txid, output.n, output.scriptPubKey.address, output.value * 100000000, block.height, block.time, 1, block.height, 0, timelock, 0, 0, '', 0, 0];
|
||||
await DB.query(query_utxos, params_utxos);
|
||||
|
|
@ -296,7 +306,7 @@ class ElementsParser {
|
|||
}
|
||||
}
|
||||
|
||||
for (const utxo of spentAsTip) {
|
||||
for (const utxo of spentAsTip) {
|
||||
if (utxo.expiredAt === 0 && block.height >= utxo.blocknumber + utxo.timelock) { // The UTXO is expiring in this block
|
||||
await DB.query(`UPDATE federation_txos SET lastblockupdate = ?, expiredAt = ? WHERE txid = ? AND txindex = ?`, [block.height, block.time, utxo.txid, utxo.txindex]);
|
||||
} else {
|
||||
|
|
@ -306,21 +316,23 @@ class ElementsParser {
|
|||
|
||||
for (const utxo of unspentAsTip) {
|
||||
if (utxo.expiredAt === 0 && block.height >= utxo.blocknumber + utxo.timelock) { // The UTXO is expiring in this block
|
||||
await DB.query(`UPDATE federation_txos SET unspent = 0, lastblockupdate = ?, expiredAt = ? WHERE txid = ? AND txindex = ?`, [confirmedTip, block.time, utxo.txid, utxo.txindex]);
|
||||
await DB.query(`UPDATE federation_txos SET lastblockupdate = ?, expiredAt = ? WHERE txid = ? AND txindex = ?`, [confirmedTip, block.time, utxo.txid, utxo.txindex]);
|
||||
} else if (utxo.expiredAt === 0 && confirmedTip >= utxo.blocknumber + utxo.timelock) { // The UTXO is expiring before the tip: we need to keep track of it
|
||||
await DB.query(`UPDATE federation_txos SET lastblockupdate = ? WHERE txid = ? AND txindex = ?`, [utxo.blocknumber + utxo.timelock - 1, utxo.txid, utxo.txindex]);
|
||||
await DB.query(`UPDATE federation_txos SET lastblockupdate = ? WHERE txid = ? AND txindex = ?`, [utxo.blocknumber + utxo.timelock - 1, utxo.txid, utxo.txindex]);
|
||||
} else {
|
||||
await DB.query(`UPDATE federation_txos SET lastblockupdate = ? WHERE txid = ? AND txindex = ?`, [confirmedTip, utxo.txid, utxo.txindex]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
protected async $saveLastBlockAuditToDatabase(blockHeight: number) {
|
||||
const query = `UPDATE state SET number = ? WHERE name = 'last_bitcoin_block_audit'`;
|
||||
await DB.query(query, [blockHeight]);
|
||||
}
|
||||
|
||||
// Get the bitcoin block where the audit process was last updated
|
||||
/** @asyncUnsafe */
|
||||
protected async $getAuditProgress(): Promise<any> {
|
||||
const lastblockaudit = await this.$getLastBlockAudit();
|
||||
const bitcoinBlocksToSync = await this.$getBitcoinBlockchainState();
|
||||
|
|
@ -331,20 +343,23 @@ class ElementsParser {
|
|||
}
|
||||
|
||||
// Get the bitcoin blocks remaining to be synced
|
||||
/** @asyncUnsafe */
|
||||
protected async $getBitcoinBlockchainState(): Promise<any> {
|
||||
const result = await bitcoinSecondClient.getBlockchainInfo();
|
||||
return {
|
||||
bitcoinBlocks: result.blocks,
|
||||
bitcoinHeaders: result.headers,
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
protected async $getLastBlockAudit(): Promise<number> {
|
||||
const query = `SELECT number FROM state WHERE name = 'last_bitcoin_block_audit'`;
|
||||
const [rows] = await DB.query(query);
|
||||
return rows[0]['number'];
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
protected async $getRedeemAddressesToScan(): Promise<any[]> {
|
||||
const query = `SELECT datetime, amount, bitcoinaddress FROM elements_pegs where amount < 0 AND bitcoinaddress != '' AND bitcointxid = '';`;
|
||||
const [rows]: any[] = await DB.query(query);
|
||||
|
|
@ -357,6 +372,7 @@ class ElementsParser {
|
|||
|
||||
///////////// DATA QUERY //////////////
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $getAuditStatus(): Promise<any> {
|
||||
const lastBlockAudit = await this.$getLastBlockAudit();
|
||||
const bitcoinBlocksToSync = await this.$getBitcoinBlockchainState();
|
||||
|
|
@ -368,12 +384,14 @@ class ElementsParser {
|
|||
};
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $getPegDataByMonth(): Promise<any> {
|
||||
const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`;
|
||||
const [rows] = await DB.query(query);
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $getFederationReservesByMonth(): Promise<any> {
|
||||
const query = `
|
||||
SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(blocktime), '%Y-%m-01') AS date FROM federation_txos
|
||||
|
|
@ -384,12 +402,13 @@ class ElementsParser {
|
|||
AND
|
||||
(expiredAt = 0 OR expiredAt > UNIX_TIMESTAMP(LAST_DAY(FROM_UNIXTIME(blocktime)) + INTERVAL 1 DAY))
|
||||
GROUP BY
|
||||
date;`;
|
||||
date;`;
|
||||
const [rows] = await DB.query(query);
|
||||
return rows;
|
||||
}
|
||||
|
||||
// Get the current L-BTC pegs and the last Liquid block it was updated
|
||||
/** @asyncUnsafe */
|
||||
public async $getCurrentLbtcSupply(): Promise<any> {
|
||||
const [rows] = await DB.query(`SELECT SUM(amount) AS LBTC_supply FROM elements_pegs;`);
|
||||
const lastblockupdate = await this.$getLatestBlockHeightFromDatabase();
|
||||
|
|
@ -402,6 +421,7 @@ class ElementsParser {
|
|||
}
|
||||
|
||||
// Get the current reserves of the federation and the last Bitcoin block it was updated
|
||||
/** @asyncUnsafe */
|
||||
public async $getCurrentFederationReserves(): Promise<any> {
|
||||
const [rows] = await DB.query(`SELECT SUM(amount) AS total_balance FROM federation_txos WHERE unspent = 1 AND expiredAt = 0;`);
|
||||
const lastblockaudit = await this.$getLastBlockAudit();
|
||||
|
|
@ -414,6 +434,7 @@ class ElementsParser {
|
|||
}
|
||||
|
||||
// Get all of the federation addresses, most balances first
|
||||
/** @asyncUnsafe */
|
||||
public async $getFederationAddresses(): Promise<any> {
|
||||
const query = `SELECT bitcoinaddress, SUM(amount) AS balance FROM federation_txos WHERE unspent = 1 AND expiredAt = 0 GROUP BY bitcoinaddress ORDER BY balance DESC;`;
|
||||
const [rows] = await DB.query(query);
|
||||
|
|
@ -421,6 +442,7 @@ class ElementsParser {
|
|||
}
|
||||
|
||||
// Get all of the UTXOs held by the federation, most recent first
|
||||
/** @asyncUnsafe */
|
||||
public async $getFederationUtxos(): Promise<any> {
|
||||
const query = `SELECT txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, pegtxid, pegindex, pegblocktime, timelock, expiredAt FROM federation_txos WHERE unspent = 1 AND expiredAt = 0 ORDER BY blocktime DESC;`;
|
||||
const [rows] = await DB.query(query);
|
||||
|
|
@ -428,6 +450,7 @@ class ElementsParser {
|
|||
}
|
||||
|
||||
// Get expired UTXOs, most recent first
|
||||
/** @asyncUnsafe */
|
||||
public async $getExpiredUtxos(): Promise<any> {
|
||||
const query = `SELECT txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, pegtxid, pegindex, pegblocktime, timelock, expiredAt FROM federation_txos WHERE unspent = 1 AND expiredAt > 0 ORDER BY blocktime DESC;`;
|
||||
const [rows]: any[] = await DB.query(query);
|
||||
|
|
@ -439,13 +462,15 @@ class ElementsParser {
|
|||
}
|
||||
|
||||
// Get utxos that were spent using emergency keys
|
||||
/** @asyncUnsafe */
|
||||
public async $getEmergencySpentUtxos(): Promise<any> {
|
||||
const query = `SELECT txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, pegtxid, pegindex, pegblocktime, timelock, expiredAt FROM federation_txos WHERE emergencyKey = 1 ORDER BY blocktime DESC;`;
|
||||
const [rows] = await DB.query(query);
|
||||
return rows;
|
||||
}
|
||||
|
||||
|
||||
// Get the total number of federation addresses
|
||||
/** @asyncUnsafe */
|
||||
public async $getFederationAddressesNumber(): Promise<any> {
|
||||
const query = `SELECT COUNT(DISTINCT bitcoinaddress) AS address_count FROM federation_txos WHERE unspent = 1 AND expiredAt = 0;`;
|
||||
const [rows] = await DB.query(query);
|
||||
|
|
@ -453,6 +478,7 @@ class ElementsParser {
|
|||
}
|
||||
|
||||
// Get the total number of federation utxos
|
||||
/** @asyncUnsafe */
|
||||
public async $getFederationUtxosNumber(): Promise<any> {
|
||||
const query = `SELECT COUNT(*) AS utxo_count FROM federation_txos WHERE unspent = 1 AND expiredAt = 0;`;
|
||||
const [rows] = await DB.query(query);
|
||||
|
|
@ -460,6 +486,7 @@ class ElementsParser {
|
|||
}
|
||||
|
||||
// Get the total number of emergency spent utxos and their total amount
|
||||
/** @asyncUnsafe */
|
||||
public async $getEmergencySpentUtxosStats(): Promise<any> {
|
||||
const query = `SELECT COUNT(*) AS utxo_count, SUM(amount) AS total_amount FROM federation_txos WHERE emergencyKey = 1;`;
|
||||
const [rows] = await DB.query(query);
|
||||
|
|
@ -467,6 +494,7 @@ class ElementsParser {
|
|||
}
|
||||
|
||||
// Get recent pegs in / out
|
||||
/** @asyncUnsafe */
|
||||
public async $getPegsList(count: number = 0): Promise<any> {
|
||||
const query = `SELECT txid, txindex, amount, bitcoinaddress, bitcointxid, bitcoinindex, datetime AS blocktime FROM elements_pegs ORDER BY block DESC LIMIT 15 OFFSET ?;`;
|
||||
const [rows] = await DB.query(query, [count]);
|
||||
|
|
@ -474,6 +502,7 @@ class ElementsParser {
|
|||
}
|
||||
|
||||
// Get all peg in / out from the last month
|
||||
/** @asyncUnsafe */
|
||||
public async $getPegsVolumeDaily(): Promise<any> {
|
||||
const pegInQuery = await DB.query(`SELECT SUM(amount) AS volume, COUNT(*) AS number FROM elements_pegs WHERE amount > 0 and datetime > UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -1, CURRENT_TIMESTAMP()));`);
|
||||
const pegOutQuery = await DB.query(`SELECT SUM(amount) AS volume, COUNT(*) AS number FROM elements_pegs WHERE amount < 0 and datetime > UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -1, CURRENT_TIMESTAMP()));`);
|
||||
|
|
@ -484,6 +513,7 @@ class ElementsParser {
|
|||
}
|
||||
|
||||
// Get the total pegs number
|
||||
/** @asyncUnsafe */
|
||||
public async $getPegsCount(): Promise<any> {
|
||||
const [rows] = await DB.query(`SELECT COUNT(*) AS pegs_count FROM elements_pegs;`);
|
||||
return rows[0];
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ class LiquidRoutes {
|
|||
.get(config.MEMPOOL.API_URL_PREFIX + 'asset/:assetId/icon', this.getLiquidIcon)
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'assets/group/:id', this.$getAssetGroup)
|
||||
;
|
||||
|
||||
|
||||
if (config.DATABASE.ENABLED) {
|
||||
app
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs', this.$getElementsPegs)
|
||||
|
|
@ -262,7 +262,7 @@ class LiquidRoutes {
|
|||
res.header('Pragma', 'public');
|
||||
res.header('Cache-control', 'public');
|
||||
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
|
||||
if (['testnet', 'signet', 'liquidtestnet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
|
||||
if (['testnet', 'signet', 'liquidtestnet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
|
||||
handleError(req, res, 400, 'Prices are not available on testnets.');
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,8 +45,9 @@ class MempoolBlocks {
|
|||
return this.mempoolBlockDeltas;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async updatePools$(): Promise<void> {
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) === false) {
|
||||
if (['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) === false) {
|
||||
this.pools = {};
|
||||
return;
|
||||
}
|
||||
|
|
@ -98,6 +99,7 @@ class MempoolBlocks {
|
|||
return mempoolBlockDeltas;
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $makeBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
|
||||
const start = Date.now();
|
||||
|
||||
|
|
@ -172,6 +174,7 @@ class MempoolBlocks {
|
|||
return this.mempoolBlocks;
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $updateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise<void> {
|
||||
if (!this.txSelectionWorker) {
|
||||
// need to reset the worker
|
||||
|
|
@ -228,11 +231,13 @@ class MempoolBlocks {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
private resetRustGbt(): void {
|
||||
this.rustInitialized = false;
|
||||
this.rustGbtGenerator = new GbtGenerator(config.MEMPOOL.BLOCK_WEIGHT_UNITS, config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT);
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
|
||||
const start = Date.now();
|
||||
|
||||
|
|
@ -285,10 +290,12 @@ class MempoolBlocks {
|
|||
return this.mempoolBlocks;
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $oneOffRustBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
|
||||
return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, false, useAccelerations, accelerationPool);
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
|
||||
// GBT optimization requires that uids never get too sparse
|
||||
// as a sanity check, we should also explicitly prevent uint32 uid overflow
|
||||
|
|
|
|||
|
|
@ -21,8 +21,7 @@ class Mempool {
|
|||
private mempoolCandidates: { [txid: string ]: boolean } = {};
|
||||
private spendMap = new Map<string, MempoolTransactionExtended>();
|
||||
private recentlyDeleted: MempoolTransactionExtended[][] = []; // buffer of transactions deleted in recent mempool updates
|
||||
private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
|
||||
maxmempool: 300000000, mempoolminfee: Common.isLiquid() ? 0.00000100 : 0.00001000, minrelaytxfee: Common.isLiquid() ? 0.00000100 : 0.00001000 };
|
||||
private mempoolInfo: IBitcoinApi.MempoolInfo;
|
||||
private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[],
|
||||
deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[]) => void) | undefined;
|
||||
private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[],
|
||||
|
|
@ -39,16 +38,41 @@ class Mempool {
|
|||
private mempoolProtection = 0;
|
||||
private latestTransactions: any[] = [];
|
||||
|
||||
private ESPLORA_MISSING_TX_WARNING_THRESHOLD = 100;
|
||||
private ESPLORA_MISSING_TX_WARNING_THRESHOLD = 100;
|
||||
private SAMPLE_TIME = 10000; // In ms
|
||||
private timer = new Date().getTime();
|
||||
private missingTxCount = 0;
|
||||
private mainLoopTimeout: number = 120000;
|
||||
private txPerSecondInterval: NodeJS.Timeout | null = null;
|
||||
|
||||
public limitGBT = config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE && config.MEMPOOL.LIMIT_GBT;
|
||||
|
||||
constructor() {
|
||||
setInterval(this.updateTxPerSecond.bind(this), 1000);
|
||||
// Initialize mempoolInfo here to avoid circular dependency issues
|
||||
// Use config directly instead of Common.isLiquid() to break circular dependency
|
||||
const isLiquid = config.MEMPOOL.NETWORK === 'liquid' || config.MEMPOOL.NETWORK === 'liquidtestnet';
|
||||
this.mempoolInfo = {
|
||||
loaded: false,
|
||||
size: 0,
|
||||
bytes: 0,
|
||||
usage: 0,
|
||||
total_fee: 0,
|
||||
maxmempool: 300000000,
|
||||
mempoolminfee: isLiquid ? 0.00000100 : 0.00001000,
|
||||
minrelaytxfee: isLiquid ? 0.00000100 : 0.00001000
|
||||
};
|
||||
this.txPerSecondInterval = setInterval(this.updateTxPerSecond.bind(this), 1000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup resources (timers, etc.)
|
||||
* This should only be called when shutting down or in test teardown
|
||||
*/
|
||||
public destroy(): void {
|
||||
if (this.txPerSecondInterval) {
|
||||
clearInterval(this.txPerSecondInterval);
|
||||
this.txPerSecondInterval = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -98,6 +122,7 @@ class Mempool {
|
|||
return this.spendMap.get(`${txid}:${index}`);
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $setMempool(mempoolData: { [txId: string]: MempoolTransactionExtended }) {
|
||||
this.mempoolCache = mempoolData;
|
||||
let count = 0;
|
||||
|
|
@ -179,6 +204,7 @@ class Mempool {
|
|||
return this.mempoolCandidates;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $updateMemPoolInfo() {
|
||||
this.mempoolInfo = await this.$getMempoolInfo();
|
||||
}
|
||||
|
|
@ -208,6 +234,7 @@ class Mempool {
|
|||
return txTimes;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $updateMempool(transactions: string[], accelerations: Record<string, Acceleration> | null, minFeeMempool: string[], minFeeTip: number, pollRate: number): Promise<void> {
|
||||
logger.debug(`Updating mempool...`);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
import { Application, Request, Response } from 'express';
|
||||
import config from "../../config";
|
||||
import config from '../../config';
|
||||
import logger from '../../logger';
|
||||
import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
|
||||
import BlocksRepository from '../../repositories/BlocksRepository';
|
||||
import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
|
||||
import HashratesRepository from '../../repositories/HashratesRepository';
|
||||
import bitcoinClient from '../bitcoin/bitcoin-client';
|
||||
import mining from "./mining";
|
||||
import mining from './mining';
|
||||
import PricesRepository from '../../repositories/PricesRepository';
|
||||
import AccelerationRepository from '../../repositories/AccelerationRepository';
|
||||
import accelerationApi from '../services/acceleration';
|
||||
|
|
@ -53,7 +53,7 @@ class MiningRoutes {
|
|||
res.header('Pragma', 'public');
|
||||
res.header('Cache-control', 'public');
|
||||
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
|
||||
if (['testnet', 'signet', 'liquidtestnet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
|
||||
if (['testnet', 'signet', 'liquidtestnet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
|
||||
handleError(req, res, 400, 'Prices are not available on testnets.');
|
||||
return;
|
||||
}
|
||||
|
|
@ -394,7 +394,7 @@ class MiningRoutes {
|
|||
res.header('Pragma', 'public');
|
||||
res.header('Cache-control', 'public');
|
||||
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
|
||||
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
|
||||
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
|
||||
handleError(req, res, 400, 'Acceleration data is not available.');
|
||||
return;
|
||||
}
|
||||
|
|
@ -409,7 +409,7 @@ class MiningRoutes {
|
|||
res.header('Pragma', 'public');
|
||||
res.header('Cache-control', 'public');
|
||||
res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
|
||||
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
|
||||
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
|
||||
handleError(req, res, 400, 'Acceleration data is not available.');
|
||||
return;
|
||||
}
|
||||
|
|
@ -425,7 +425,7 @@ class MiningRoutes {
|
|||
res.header('Pragma', 'public');
|
||||
res.header('Cache-control', 'public');
|
||||
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
|
||||
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
|
||||
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
|
||||
handleError(req, res, 400, 'Acceleration data is not available.');
|
||||
return;
|
||||
}
|
||||
|
|
@ -440,7 +440,7 @@ class MiningRoutes {
|
|||
res.header('Pragma', 'public');
|
||||
res.header('Cache-control', 'public');
|
||||
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
|
||||
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
|
||||
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
|
||||
handleError(req, res, 400, 'Acceleration data is not available.');
|
||||
return;
|
||||
}
|
||||
|
|
@ -455,7 +455,7 @@ class MiningRoutes {
|
|||
res.header('Pragma', 'public');
|
||||
res.header('Cache-control', 'public');
|
||||
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
|
||||
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
|
||||
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
|
||||
handleError(req, res, 400, 'Acceleration data is not available.');
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ class Mining {
|
|||
private blocksPriceIndexingRunning = false;
|
||||
public lastHashrateIndexingDate: number | null = null;
|
||||
public lastWeeklyHashrateIndexingDate: number | null = null;
|
||||
|
||||
|
||||
public reindexHashrateRequested = false;
|
||||
public reindexDifficultyAdjustmentRequested = false;
|
||||
|
||||
|
|
@ -66,7 +66,7 @@ class Mining {
|
|||
{from, to}
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get historical block rewards
|
||||
*/
|
||||
|
|
@ -175,8 +175,8 @@ class Mining {
|
|||
const blockCount1w: number = await BlocksRepository.$blockCount(pool.id, '1w');
|
||||
const totalBlock1w: number = await BlocksRepository.$blockCount(null, '1w');
|
||||
|
||||
const avgHealth = await BlocksRepository.$getAvgBlockHealthPerPoolId(pool.id);
|
||||
const totalReward = await BlocksRepository.$getTotalRewardForPoolId(pool.id);
|
||||
const avgHealth = await BlocksRepository.$getAvgBlockHealthPerPoolId(pool.id);
|
||||
const totalReward = await BlocksRepository.$getTotalRewardForPoolId(pool.id);
|
||||
|
||||
let currentEstimatedHashrate = 0;
|
||||
try {
|
||||
|
|
@ -218,7 +218,7 @@ class Mining {
|
|||
const now = new Date();
|
||||
|
||||
// Run only if:
|
||||
// * this.lastWeeklyHashrateIndexingDate is set to null (node backend restart, reorg)
|
||||
// * this.lastWeeklyHashrateIndexingDate is set to null (node backend restart, reorg, or re-indexing was requested after mining pools update)
|
||||
// * we started a new week (around Monday midnight)
|
||||
const runIndexing = this.lastWeeklyHashrateIndexingDate === null ||
|
||||
now.getUTCDay() === 1 && this.lastWeeklyHashrateIndexingDate !== now.getUTCDate();
|
||||
|
|
@ -235,7 +235,7 @@ class Mining {
|
|||
|
||||
const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
|
||||
const hashrates: any[] = [];
|
||||
|
||||
|
||||
const lastMonday = new Date(now.setDate(now.getDate() - (now.getDay() + 6) % 7));
|
||||
const lastMondayMidnight = this.getDateMidnight(lastMonday);
|
||||
let toTimestamp = lastMondayMidnight.getTime();
|
||||
|
|
@ -326,6 +326,7 @@ class Mining {
|
|||
|
||||
/**
|
||||
* Generate daily hashrate data
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async $generateNetworkHashrateHistory(): Promise<void> {
|
||||
// If a re-index was requested, truncate first
|
||||
|
|
@ -333,6 +334,7 @@ class Mining {
|
|||
logger.notice(`hashrates will now be re-indexed`);
|
||||
await database.query(`TRUNCATE hashrates`);
|
||||
this.lastHashrateIndexingDate = 0;
|
||||
this.lastWeeklyHashrateIndexingDate = null;
|
||||
this.reindexHashrateRequested = false;
|
||||
}
|
||||
|
||||
|
|
@ -439,6 +441,7 @@ class Mining {
|
|||
|
||||
/**
|
||||
* Index difficulty adjustments
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async $indexDifficultyAdjustments(): Promise<void> {
|
||||
// If a re-index was requested, truncate first
|
||||
|
|
@ -528,6 +531,8 @@ class Mining {
|
|||
|
||||
/**
|
||||
* Create a link between blocks and the latest price at when they were mined
|
||||
*
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $indexBlockPrices(): Promise<void> {
|
||||
if (this.blocksPriceIndexingRunning === true) {
|
||||
|
|
@ -537,7 +542,7 @@ class Mining {
|
|||
|
||||
let totalInserted = 0;
|
||||
try {
|
||||
const prices: any[] = await PricesRepository.$getPricesTimesAndId();
|
||||
const prices: any[] = await PricesRepository.$getPricesTimesAndId();
|
||||
const blocksWithoutPrices: any[] = await BlocksRepository.$getBlocksWithoutPrice();
|
||||
|
||||
const blocksPrices: BlockPrice[] = [];
|
||||
|
|
@ -598,6 +603,8 @@ class Mining {
|
|||
|
||||
/**
|
||||
* Index core coinstatsindex
|
||||
*
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async $indexCoinStatsIndex(): Promise<void> {
|
||||
let timer = new Date().getTime() / 1000;
|
||||
|
|
@ -609,11 +616,11 @@ class Mining {
|
|||
while (currentBlockHeight > 0) {
|
||||
const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex(
|
||||
currentBlockHeight, currentBlockHeight - 10000);
|
||||
|
||||
|
||||
for (const block of indexedBlocks) {
|
||||
const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
|
||||
await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts,
|
||||
Math.round(txoutset.block_info.prevout_spent * 100000000));
|
||||
Math.round(txoutset.block_info.prevout_spent * 100000000));
|
||||
++totalIndexed;
|
||||
|
||||
const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
|
||||
|
|
@ -635,6 +642,7 @@ class Mining {
|
|||
|
||||
/**
|
||||
* List existing mining pools
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async $listPools(): Promise<{name: string, slug: string, unique_id: number}[] | null> {
|
||||
const [rows] = await database.query(`
|
||||
|
|
@ -688,7 +696,7 @@ class Mining {
|
|||
default: return 1 * scale;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Finds the oldest block in a consecutive chain back from the tip
|
||||
// assumes `blocks` is sorted in ascending height order
|
||||
|
|
@ -701,6 +709,7 @@ class Mining {
|
|||
return blocks[0];
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
private async getGenesisData(): Promise<{timestamp: number, bits: number, difficulty: number}> {
|
||||
if (this.genesisData == null) {
|
||||
const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0));
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ class PoolsParser {
|
|||
/**
|
||||
* Populate our db with updated mining pool definition
|
||||
* @param pools
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async migratePoolsJson(): Promise<void> {
|
||||
// We also need to wipe the backend cache to make sure we don't serve blocks with
|
||||
|
|
@ -126,8 +127,8 @@ class PoolsParser {
|
|||
block.extras.pool = reindexedBlock.extras.pool;
|
||||
}
|
||||
// update persistent cache with the reindexed data
|
||||
diskCache.$saveCacheToDisk();
|
||||
redisCache.$updateBlocks(blocks.getBlocks());
|
||||
void diskCache.$saveCacheToDisk();
|
||||
void redisCache.$updateBlocks(blocks.getBlocks());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -159,6 +160,7 @@ class PoolsParser {
|
|||
|
||||
/**
|
||||
* Manually add the 'unknown pool'
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $insertUnknownPool(): Promise<void> {
|
||||
if (!config.DATABASE.ENABLED) {
|
||||
|
|
@ -190,12 +192,13 @@ class PoolsParser {
|
|||
* re-index pool assignment for blocks previously associated with pool
|
||||
*
|
||||
* @param pool local id of existing pool to reindex
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
private async $reindexBlocksForPool(poolId: number): Promise<void> {
|
||||
let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52
|
||||
if (config.MEMPOOL.NETWORK === 'testnet') {
|
||||
firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581
|
||||
} else if (['signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
|
||||
} else if (['signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
|
||||
firstKnownBlockPool = 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,11 @@
|
|||
import { Application, Request, Response } from 'express';
|
||||
import config from '../../config';
|
||||
import pricesUpdater from '../../tasks/price-updater';
|
||||
import logger from '../../logger';
|
||||
import PricesRepository from '../../repositories/PricesRepository';
|
||||
|
||||
class PricesRoutes {
|
||||
public initRoutes(app: Application): void {
|
||||
app
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'prices', this.$getCurrentPrices.bind(this))
|
||||
.get(config.MEMPOOL.API_URL_PREFIX + 'internal/usd-price-history', this.$getAllPrices.bind(this))
|
||||
;
|
||||
}
|
||||
|
||||
|
|
@ -19,23 +16,6 @@ class PricesRoutes {
|
|||
|
||||
res.json(pricesUpdater.getLatestPrices());
|
||||
}
|
||||
|
||||
private async $getAllPrices(req: Request, res: Response): Promise<void> {
|
||||
res.header('Pragma', 'public');
|
||||
res.header('Cache-control', 'public');
|
||||
res.setHeader('Expires', new Date(Date.now() + 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR).toUTCString());
|
||||
|
||||
try {
|
||||
const usdPriceHistory = await PricesRepository.$getPricesTimesAndId();
|
||||
const responseData = usdPriceHistory.map(p => {
|
||||
return { time: p.time, USD: p.USD };
|
||||
});
|
||||
res.status(200).json(responseData);
|
||||
} catch (e: any) {
|
||||
logger.err(`Exception ${e} in PricesRoutes::$getAllPrices. Code: ${e.code}. Message: ${e.message}`);
|
||||
res.status(403).send();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new PricesRoutes();
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import config from "../config";
|
||||
import logger from "../logger";
|
||||
import { MempoolTransactionExtended, TransactionStripped } from "../mempool.interfaces";
|
||||
import config from '../config';
|
||||
import logger from '../logger';
|
||||
import { MempoolTransactionExtended, TransactionStripped } from '../mempool.interfaces';
|
||||
import bitcoinApi from './bitcoin/bitcoin-api-factory';
|
||||
import { IEsploraApi } from "./bitcoin/esplora-api.interface";
|
||||
import { Common } from "./common";
|
||||
import redisCache from "./redis-cache";
|
||||
import { IEsploraApi } from './bitcoin/esplora-api.interface';
|
||||
import { Common } from './common';
|
||||
import redisCache from './redis-cache';
|
||||
|
||||
export interface RbfTransaction extends TransactionStripped {
|
||||
rbf?: boolean;
|
||||
|
|
@ -407,6 +407,7 @@ class RbfCache {
|
|||
};
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async load({ txs, trees, expiring, mempool, spendMap }): Promise<void> {
|
||||
try {
|
||||
txs.forEach(txEntry => {
|
||||
|
|
|
|||
|
|
@ -37,11 +37,12 @@ class RedisCache {
|
|||
},
|
||||
database: NetworkDB[config.MEMPOOL.NETWORK],
|
||||
};
|
||||
this.$ensureConnected();
|
||||
setInterval(() => { this.$ensureConnected(); }, 10000);
|
||||
void this.$ensureConnected();
|
||||
setInterval(() => { void this.$ensureConnected(); }, 10000);
|
||||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
private async $ensureConnected(): Promise<boolean> {
|
||||
if (!this.connected && config.REDIS.ENABLED) {
|
||||
try {
|
||||
|
|
@ -95,6 +96,7 @@ class RedisCache {
|
|||
await this.$flushRbfQueues();
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
async $updateBlocks(blocks: BlockExtended[]): Promise<void> {
|
||||
if (!config.REDIS.ENABLED) {
|
||||
return;
|
||||
|
|
@ -127,6 +129,7 @@ class RedisCache {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
async $addTransaction(tx: MempoolTransactionExtended): Promise<void> {
|
||||
if (!config.REDIS.ENABLED) {
|
||||
return;
|
||||
|
|
@ -139,6 +142,7 @@ class RedisCache {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
async $flushTransactions(): Promise<void> {
|
||||
if (!config.REDIS.ENABLED) {
|
||||
return;
|
||||
|
|
@ -178,6 +182,7 @@ class RedisCache {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
async $removeTransactions(transactions: string[]): Promise<void> {
|
||||
if (!config.REDIS.ENABLED) {
|
||||
return;
|
||||
|
|
@ -206,6 +211,7 @@ class RedisCache {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
async $setRbfEntry(type: string, txid: string, value: any): Promise<void> {
|
||||
if (!config.REDIS.ENABLED) {
|
||||
return;
|
||||
|
|
@ -222,6 +228,7 @@ class RedisCache {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
async $removeRbfEntry(type: string, txid: string): Promise<void> {
|
||||
if (!config.REDIS.ENABLED) {
|
||||
return;
|
||||
|
|
@ -238,6 +245,7 @@ class RedisCache {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
private async $flushRbfQueues(): Promise<void> {
|
||||
if (!config.REDIS.ENABLED) {
|
||||
return;
|
||||
|
|
@ -263,6 +271,7 @@ class RedisCache {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
async $getBlocks(): Promise<BlockExtended[]> {
|
||||
if (!config.REDIS.ENABLED) {
|
||||
return [];
|
||||
|
|
@ -280,6 +289,7 @@ class RedisCache {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
async $getBlockSummaries(): Promise<BlockSummary[]> {
|
||||
if (!config.REDIS.ENABLED) {
|
||||
return [];
|
||||
|
|
@ -297,6 +307,7 @@ class RedisCache {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
async $getMempool(): Promise<{ [txid: string]: MempoolTransactionExtended }> {
|
||||
if (!config.REDIS.ENABLED) {
|
||||
return {};
|
||||
|
|
@ -320,6 +331,7 @@ class RedisCache {
|
|||
return {};
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
async $getRbfEntries(type: string): Promise<any[]> {
|
||||
if (!config.REDIS.ENABLED) {
|
||||
return [];
|
||||
|
|
@ -337,6 +349,7 @@ class RedisCache {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $loadCache(): Promise<void> {
|
||||
if (!config.REDIS.ENABLED) {
|
||||
return;
|
||||
|
|
@ -385,12 +398,14 @@ class RedisCache {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
private async scanKeys<T>(pattern): Promise<{ key: string, value: T }[]> {
|
||||
logger.info(`loading Redis entries for ${pattern}`);
|
||||
let keys: string[] = [];
|
||||
const result: { key: string, value: T }[] = [];
|
||||
const patternLength = pattern.length - 1;
|
||||
let count = 0;
|
||||
/** @asyncUnsafe */
|
||||
const processValues = async (keys): Promise<void> => {
|
||||
const values = await this.client.MGET(keys);
|
||||
for (let i = 0; i < values.length; i++) {
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { BlockExtended } from '../../mempool.interfaces';
|
|||
import axios from 'axios';
|
||||
import mempool from '../mempool';
|
||||
import websocketHandler from '../websocket-handler';
|
||||
import { Common } from '../common';
|
||||
|
||||
type MyAccelerationStatus = 'requested' | 'accelerating' | 'done';
|
||||
|
||||
|
|
@ -74,6 +75,7 @@ class AccelerationApi {
|
|||
this.forcePoll = true;
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
private async $fetchAccelerations(): Promise<Acceleration[] | null> {
|
||||
try {
|
||||
const response = await axios.get(this.apiPath, { responseType: 'json', timeout: 10000 });
|
||||
|
|
@ -238,6 +240,7 @@ class AccelerationApi {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async connectWebsocket(): Promise<void> {
|
||||
if (this.startedWebsocketLoop) {
|
||||
return;
|
||||
|
|
@ -314,7 +317,7 @@ class AccelerationApi {
|
|||
}
|
||||
}
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 5000));
|
||||
await Common.sleep$(5000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { WebSocket } from 'ws';
|
|||
import logger from '../../logger';
|
||||
import config from '../../config';
|
||||
import websocketHandler from '../websocket-handler';
|
||||
import { Common } from '../common';
|
||||
|
||||
export interface StratumJob {
|
||||
pool: number;
|
||||
|
|
@ -58,6 +59,7 @@ class StratumApi {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async connectWebsocket(): Promise<void> {
|
||||
if (!config.STRATUM.ENABLED) {
|
||||
return;
|
||||
|
|
@ -97,7 +99,7 @@ class StratumApi {
|
|||
}
|
||||
});
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 5000));
|
||||
await Common.sleep$(5000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -56,10 +56,11 @@ class WalletApi {
|
|||
|
||||
// Load cache on startup
|
||||
if (config.WALLETS.ENABLED) {
|
||||
this.$loadCache();
|
||||
void this.$loadCache();
|
||||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
private async $loadCache(): Promise<void> {
|
||||
try {
|
||||
const cacheData = await fsPromises.readFile(WalletApi.FILE_NAME, 'utf8');
|
||||
|
|
@ -148,6 +149,7 @@ class WalletApi {
|
|||
}
|
||||
|
||||
// resync wallet addresses from the services backend
|
||||
/** @asyncSafe */
|
||||
async $syncWallets(): Promise<void> {
|
||||
if (!config.WALLETS.ENABLED || this.syncing) {
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -514,7 +514,7 @@ class StatisticsApi {
|
|||
vsize_1600: completeVsizes[36],
|
||||
vsize_1800: completeVsizes[37],
|
||||
vsize_2000: completeVsizes[38],
|
||||
}
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,13 +22,14 @@ class Statistics {
|
|||
const difference = nextInterval.getTime() - now.getTime();
|
||||
|
||||
setTimeout(() => {
|
||||
this.runStatistics();
|
||||
void this.runStatistics();
|
||||
this.intervalTimer = setInterval(() => {
|
||||
this.runStatistics(true);
|
||||
void this.runStatistics(true);
|
||||
}, 1 * 60 * 1000);
|
||||
}, difference);
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async runStatistics(skipIfRecent = false): Promise<void> {
|
||||
if (!memPool.isInSync()) {
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ class TransactionUtils {
|
|||
* @param addPrevouts
|
||||
* @param lazyPrevouts
|
||||
* @param forceCore - See https://github.com/mempool/mempool/issues/2904
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false, forceCore = false, addMempoolData = false): Promise<TransactionExtended> {
|
||||
let transaction: IEsploraApi.Transaction;
|
||||
|
|
@ -69,10 +70,12 @@ class TransactionUtils {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $getMempoolTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise<MempoolTransactionExtended> {
|
||||
return (await this.$getTransactionExtended(txId, addPrevouts, lazyPrevouts, forceCore, true)) as MempoolTransactionExtended;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $getMempoolTransactionsExtended(txids: string[], addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise<MempoolTransactionExtended[]> {
|
||||
if (forceCore || config.MEMPOOL.BACKEND !== 'esplora') {
|
||||
const limiter = pLimit(8); // Run 8 requests at a time
|
||||
|
|
@ -116,7 +119,7 @@ class TransactionUtils {
|
|||
public extendMempoolTransaction(transaction: IEsploraApi.Transaction): MempoolTransactionExtended {
|
||||
const vsize = Math.ceil(transaction.weight / 4);
|
||||
const fractionalVsize = (transaction.weight / 4);
|
||||
let sigops = Common.isLiquid() ? 0 : (transaction.sigops != null ? transaction.sigops : this.countSigops(transaction));
|
||||
const sigops = Common.isLiquid() ? 0 : (transaction.sigops != null ? transaction.sigops : this.countSigops(transaction));
|
||||
// https://github.com/bitcoin/bitcoin/blob/e9262ea32a6e1d364fb7974844fadc36f931f8c6/src/policy/policy.cpp#L295-L298
|
||||
const adjustedVsize = Math.max(fractionalVsize, sigops * 5); // adjusted vsize = Max(weight, sigops * bytes_per_sigop) / witness_scale_factor
|
||||
const feePerVbytes = (transaction.fee || 0) / fractionalVsize;
|
||||
|
|
@ -145,6 +148,9 @@ class TransactionUtils {
|
|||
return str;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the witness-adjusted sigops cost of an asm script
|
||||
*/
|
||||
public countScriptSigops(script: string, isRawScript: boolean = false, witness: boolean = false): number {
|
||||
if (!script?.length) {
|
||||
return 0;
|
||||
|
|
@ -213,6 +219,41 @@ class TransactionUtils {
|
|||
return sigops;
|
||||
}
|
||||
|
||||
/**
|
||||
* see https://github.com/bitcoin/bitcoin/blob/25c45bb0d0bd6618ec9296a1a43605657124e5de/src/policy/policy.cpp#L166-L193
|
||||
* returns true if the transactions is permitted under bip54 sigops rules
|
||||
*
|
||||
* "Unlike the existing block wide sigop limit which counts sigops present in the block
|
||||
* itself (including the scriptPubKey which is not executed until spending later), BIP54
|
||||
* counts sigops in the block where they are potentially executed (only).
|
||||
* This means sigops in the spent scriptPubKey count toward the limit.
|
||||
* `fAccurate` means correctly accounting sigops for CHECKMULTISIGs(VERIFY) with 16 pubkeys
|
||||
* or fewer. This method of accounting was introduced by BIP16, and BIP54 reuses it.
|
||||
* The GetSigOpCount call on the previous scriptPubKey counts both bare and P2SH sigops."
|
||||
*/
|
||||
public checkSigopsBIP54(tx: TransactionExtended, limit): boolean {
|
||||
let sigops = 0;
|
||||
for (const input of tx.vin) {
|
||||
if (input.scriptsig_asm) {
|
||||
sigops += this.countScriptSigops(input.scriptsig_asm);
|
||||
}
|
||||
if (input.prevout) {
|
||||
// P2SH redeem script
|
||||
if (input.prevout.scriptpubkey_type === 'p2sh' && input.inner_redeemscript_asm) {
|
||||
sigops += this.countScriptSigops(input.inner_redeemscript_asm);
|
||||
} else {
|
||||
// prevout scriptpubkey
|
||||
sigops += this.countScriptSigops(input.prevout.scriptpubkey_asm);
|
||||
}
|
||||
}
|
||||
|
||||
if (sigops > limit) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// returns the most significant 4 bytes of the txid as an integer
|
||||
public txidToOrdering(txid: string): number {
|
||||
return parseInt(
|
||||
|
|
@ -229,7 +270,7 @@ class TransactionUtils {
|
|||
return;
|
||||
}
|
||||
|
||||
if (vin.prevout.scriptpubkey_type === 'p2sh') {
|
||||
if (vin.prevout.scriptpubkey_type === 'p2sh' && vin.scriptsig_asm?.length) {
|
||||
const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0];
|
||||
vin.inner_redeemscript_asm = this.convertScriptSigAsm(redeemScript);
|
||||
if (vin.witness && vin.witness.length > 2) {
|
||||
|
|
@ -262,15 +303,15 @@ class TransactionUtils {
|
|||
if (op >= 0x01 && op <= 0x4e) {
|
||||
i++;
|
||||
let push: number;
|
||||
if (op === 0x4c) {
|
||||
if (op === 0x4c && buf.length > i) {
|
||||
push = buf.readUInt8(i);
|
||||
b.push('OP_PUSHDATA1');
|
||||
i += 1;
|
||||
} else if (op === 0x4d) {
|
||||
} else if (op === 0x4d && buf.length > i + 1) {
|
||||
push = buf.readUInt16LE(i);
|
||||
b.push('OP_PUSHDATA2');
|
||||
i += 2;
|
||||
} else if (op === 0x4e) {
|
||||
} else if (op === 0x4e && buf.length > i + 3) {
|
||||
push = buf.readUInt32LE(i);
|
||||
b.push('OP_PUSHDATA4');
|
||||
i += 4;
|
||||
|
|
@ -279,13 +320,15 @@ class TransactionUtils {
|
|||
b.push('OP_PUSHBYTES_' + push);
|
||||
}
|
||||
|
||||
const data = buf.slice(i, i + push);
|
||||
if (i >= buf.length) {
|
||||
break;
|
||||
}
|
||||
const data = buf.subarray(i, Math.min(i + push, buf.length));
|
||||
b.push(data.toString('hex'));
|
||||
i += data.length;
|
||||
if (data.length !== push) {
|
||||
break;
|
||||
}
|
||||
|
||||
b.push(data.toString('hex'));
|
||||
i += data.length;
|
||||
} else {
|
||||
if (op === 0x00) {
|
||||
b.push('OP_0');
|
||||
|
|
@ -325,7 +368,7 @@ class TransactionUtils {
|
|||
* the script item if it is a script spend.
|
||||
*/
|
||||
public witnessToP2TRScript(witness: string[]): string | null {
|
||||
if (witness.length < 2) return null;
|
||||
if (witness.length < 2) {return null;}
|
||||
// Note: see BIP341 for parsing details of witness stack
|
||||
|
||||
// If there are at least two witness elements, and the first byte of the
|
||||
|
|
@ -335,7 +378,7 @@ class TransactionUtils {
|
|||
// If there are at least two witness elements left, script path spending is used.
|
||||
// Call the second-to-last stack element s, the script.
|
||||
// (Note: this phrasing from BIP341 assumes we've *removed* the annex from the stack)
|
||||
if (hasAnnex && witness.length < 3) return null;
|
||||
if (hasAnnex && witness.length < 3) {return null;}
|
||||
const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2;
|
||||
return witness[positionOfScript];
|
||||
}
|
||||
|
|
@ -442,7 +485,7 @@ class TransactionUtils {
|
|||
return 'unknown';
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
export default new TransactionUtils();
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ if (parentPort) {
|
|||
mempool.delete(uid);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
const { blocks, rates, clusters } = makeBlockTemplates(mempool);
|
||||
|
||||
// return the result to main thread.
|
||||
|
|
@ -38,7 +38,7 @@ function makeBlockTemplates(mempool: Map<number, CompactThreadTransaction>)
|
|||
const auditPool: Map<number, AuditTransaction> = new Map();
|
||||
const mempoolArray: AuditTransaction[] = [];
|
||||
const cpfpClusters: Map<number, number[]> = new Map();
|
||||
|
||||
|
||||
mempool.forEach(tx => {
|
||||
tx.dirty = false;
|
||||
// initializing everything up front helps V8 optimize property access later
|
||||
|
|
@ -85,7 +85,7 @@ function makeBlockTemplates(mempool: Map<number, CompactThreadTransaction>)
|
|||
// (i.e. the package rooted in the transaction with the best ancestor score)
|
||||
const blocks: number[][] = [];
|
||||
let blockWeight = 4000;
|
||||
let blockSigops = 0;
|
||||
const blockSigops = 0;
|
||||
let transactions: AuditTransaction[] = [];
|
||||
const modified: PairingHeap<AuditTransaction> = new PairingHeap((a, b): boolean => {
|
||||
if (a.score === b.score) {
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ class WebsocketHandler {
|
|||
'backendInfo': backendInfo.getBackendInfo(),
|
||||
'loadingIndicators': loadingIndicators.getLoadingIndicators(),
|
||||
'da': da?.previousTime ? da : undefined,
|
||||
'fees': feeApi.getRecommendedFee(),
|
||||
'fees': feeApi.getPreciseRecommendedFee(),
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -637,7 +637,7 @@ class WebsocketHandler {
|
|||
}
|
||||
memPool.removeFromSpendMap(deletedTransactions);
|
||||
memPool.addToSpendMap(newTransactions);
|
||||
const recommendedFees = feeApi.getRecommendedFee();
|
||||
const recommendedFees = feeApi.getPreciseRecommendedFee();
|
||||
|
||||
const latestTransactions = memPool.getLatestTransactions();
|
||||
|
||||
|
|
@ -1000,7 +1000,8 @@ class WebsocketHandler {
|
|||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async handleNewBlock(block: BlockExtended, txIds: string[], transactions: MempoolTransactionExtended[]): Promise<void> {
|
||||
if (!this.webSocketServers.length) {
|
||||
throw new Error('No WebSocket.Server have been set');
|
||||
|
|
@ -1053,7 +1054,7 @@ class WebsocketHandler {
|
|||
totalWeight += (tx.vsize * 4);
|
||||
}
|
||||
|
||||
BlocksSummariesRepository.$saveTemplate({
|
||||
void BlocksSummariesRepository.$saveTemplate({
|
||||
height: block.height,
|
||||
template: {
|
||||
id: block.id,
|
||||
|
|
@ -1062,7 +1063,7 @@ class WebsocketHandler {
|
|||
version: 1,
|
||||
});
|
||||
|
||||
BlocksAuditsRepository.$saveAudit({
|
||||
void BlocksAuditsRepository.$saveAudit({
|
||||
version: 1,
|
||||
time: block.timestamp,
|
||||
height: block.height,
|
||||
|
|
@ -1125,7 +1126,7 @@ class WebsocketHandler {
|
|||
const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
|
||||
|
||||
const da = difficultyAdjustment.getDifficultyAdjustment();
|
||||
const fees = feeApi.getRecommendedFee();
|
||||
const fees = feeApi.getPreciseRecommendedFee();
|
||||
const mempoolInfo = memPool.getMempoolInfo();
|
||||
|
||||
// pre-compute address transactions
|
||||
|
|
@ -1473,6 +1474,7 @@ class WebsocketHandler {
|
|||
return addressCache;
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
private async getFullTransactions(transactions: MempoolTransactionExtended[]): Promise<MempoolTransactionExtended[]> {
|
||||
for (let i = 0; i < transactions.length; i++) {
|
||||
try {
|
||||
|
|
@ -1506,7 +1508,7 @@ class WebsocketHandler {
|
|||
if (client['track-rbf']) {
|
||||
numRbfSubs++;
|
||||
}
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
let count = 0;
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ interface IConfig {
|
|||
MEMPOOL: {
|
||||
ENABLED: boolean;
|
||||
OFFICIAL: boolean;
|
||||
NETWORK: 'mainnet' | 'testnet' | 'signet' | 'liquid' | 'liquidtestnet';
|
||||
NETWORK: 'mainnet' | 'testnet' | 'testnet4' | 'signet' | 'liquid' | 'liquidtestnet' | 'regtest';
|
||||
BACKEND: 'esplora' | 'electrum' | 'none';
|
||||
HTTP_PORT: number;
|
||||
UNIX_SOCKET_PATH: string;
|
||||
|
|
@ -398,7 +398,7 @@ class Config implements IConfig {
|
|||
});
|
||||
return next;
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export default new Config();
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import { execSync } from 'child_process';
|
|||
timezone: '+00:00',
|
||||
};
|
||||
|
||||
/** @asyncUnsafe */
|
||||
private checkDBFlag() {
|
||||
if (config.DATABASE.ENABLED === false) {
|
||||
const stack = new Error().stack;
|
||||
|
|
@ -32,6 +33,7 @@ import { execSync } from 'child_process';
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async query<T extends RowDataPacket[][] | RowDataPacket[] | OkPacket |
|
||||
OkPacket[] | ResultSetHeader>(query, params?, errorLogLevel: LogLevel | 'silent' = 'debug', connection?: PoolConnection): Promise<[T, FieldPacket[]]>
|
||||
{
|
||||
|
|
@ -76,6 +78,7 @@ import { execSync } from 'child_process';
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
private async $rollbackAtomic(connection: PoolConnection): Promise<void> {
|
||||
try {
|
||||
await connection.rollback();
|
||||
|
|
@ -85,6 +88,7 @@ import { execSync } from 'child_process';
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $atomicQuery<T extends RowDataPacket[][] | RowDataPacket[] | OkPacket |
|
||||
OkPacket[] | ResultSetHeader>(queries: { query, params }[], errorLogLevel: LogLevel | 'silent' = 'debug'): Promise<[T, FieldPacket[]][]>
|
||||
{
|
||||
|
|
@ -116,6 +120,8 @@ import { execSync } from 'child_process';
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
/** @asyncSafe */
|
||||
public async checkDbConnection() {
|
||||
this.checkDBFlag();
|
||||
try {
|
||||
|
|
@ -171,15 +177,34 @@ import { execSync } from 'child_process';
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
private async getPool(): Promise<Pool> {
|
||||
if (this.pool === null) {
|
||||
this.pool = createPool(this.poolConfig);
|
||||
this.pool.on('connection', function (newConnection: PoolConnection) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-floating-promises -- callback API, not a promise despite types
|
||||
newConnection.query(`SET time_zone='+00:00'`);
|
||||
});
|
||||
}
|
||||
return this.pool;
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the database connection pool
|
||||
* This should only be called when the application is shutting down
|
||||
* or at the end of test suites
|
||||
*/
|
||||
public async close(): Promise<void> {
|
||||
if (this.pool !== null) {
|
||||
try {
|
||||
await this.pool.end();
|
||||
} catch (e) {
|
||||
logger.err(`Exception in close. Reason: ${(e instanceof Error ? e.message : e)}`);
|
||||
}
|
||||
this.pool = null;
|
||||
logger.debug('Database connection pool closed');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new DB();
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ class Server {
|
|||
this.app = express();
|
||||
|
||||
if (!config.MEMPOOL.SPAWN_CLUSTER_PROCS) {
|
||||
this.startServer();
|
||||
void this.startServer();
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -92,10 +92,11 @@ class Server {
|
|||
}, 10000);
|
||||
});
|
||||
} else {
|
||||
this.startServer(true);
|
||||
void this.startServer(true);
|
||||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
async startServer(worker = false): Promise<void> {
|
||||
logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
|
||||
|
||||
|
|
@ -142,12 +143,13 @@ class Server {
|
|||
res.setHeader('Access-Control-Expose-Headers', 'X-Total-Count,X-Mempool-Auth');
|
||||
next();
|
||||
})
|
||||
.use(express.urlencoded({ extended: true }))
|
||||
.use(express.text({ type: ['text/plain', 'application/base64'] }))
|
||||
.use(express.json())
|
||||
.use(express.urlencoded({ extended: true, limit: '10mb' }))
|
||||
.use(express.text({ type: ['text/plain', 'application/base64'], limit: '10mb' }))
|
||||
.use(express.json({ limit: '10mb' }))
|
||||
;
|
||||
|
||||
if (config.DATABASE.ENABLED && config.FIAT_PRICE.ENABLED) {
|
||||
/** @asyncUnsafe */
|
||||
await priceUpdater.$initializeLatestPriceWithDb();
|
||||
}
|
||||
|
||||
|
|
@ -161,19 +163,21 @@ class Server {
|
|||
this.setUpWebsocketHandling();
|
||||
|
||||
await poolsUpdater.updatePoolsJson(); // Needs to be done before loading the disk cache because we sometimes wipe it
|
||||
if (config.DATABASE.ENABLED === true && config.MEMPOOL.ENABLED && ['mainnet', 'testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) && !poolsUpdater.currentSha) {
|
||||
if (config.DATABASE.ENABLED === true && config.MEMPOOL.ENABLED && ['mainnet', 'testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) && !poolsUpdater.currentSha) {
|
||||
logger.err(`Failed to retreive pools-v2.json sha, cannot run block indexing. Please make sure you've set valid urls in your mempool-config.json::MEMPOOL::POOLS_JSON_URL and mempool-config.json::MEMPOOL::POOLS_JSON_TREE_UR, aborting now`);
|
||||
return process.exit(1);
|
||||
}
|
||||
|
||||
await syncAssets.syncAssets$();
|
||||
if (config.DATABASE.ENABLED) {
|
||||
/** @asyncUnsafe */
|
||||
await mempoolBlocks.updatePools$();
|
||||
}
|
||||
if (config.MEMPOOL.ENABLED) {
|
||||
if (config.MEMPOOL.CACHE_ENABLED) {
|
||||
await diskCache.$loadMempoolCache();
|
||||
} else if (config.REDIS.ENABLED) {
|
||||
/** @asyncUnsafe */
|
||||
await redisCache.$loadCache();
|
||||
}
|
||||
}
|
||||
|
|
@ -197,20 +201,20 @@ class Server {
|
|||
}
|
||||
|
||||
if (config.FIAT_PRICE.ENABLED) {
|
||||
priceUpdater.$run();
|
||||
void priceUpdater.$run();
|
||||
}
|
||||
await chainTips.updateOrphanedBlocks();
|
||||
|
||||
this.setUpHttpApiRoutes();
|
||||
|
||||
if (config.MEMPOOL.ENABLED) {
|
||||
this.runMainUpdateLoop();
|
||||
void this.runMainUpdateLoop();
|
||||
}
|
||||
|
||||
setInterval(() => { this.healthCheck(); }, 2500);
|
||||
|
||||
if (config.LIGHTNING.ENABLED) {
|
||||
this.$runLightningBackend();
|
||||
void this.$runLightningBackend();
|
||||
}
|
||||
|
||||
this.server.listen(config.MEMPOOL.HTTP_PORT, () => {
|
||||
|
|
@ -231,9 +235,10 @@ class Server {
|
|||
});
|
||||
}
|
||||
|
||||
poolsUpdater.$startService();
|
||||
void poolsUpdater.$startService();
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
async runMainUpdateLoop(): Promise<void> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
|
|
@ -256,13 +261,13 @@ class Server {
|
|||
if (numHandledBlocks === 0) {
|
||||
await memPool.$updateMempool(newMempool, latestAccelerations, minFeeMempool, minFeeTip, pollRate);
|
||||
}
|
||||
indexer.$run();
|
||||
void indexer.$run();
|
||||
if (config.WALLETS.ENABLED) {
|
||||
// might take a while, so run in the background
|
||||
walletApi.$syncWallets();
|
||||
void walletApi.$syncWallets();
|
||||
}
|
||||
if (config.FIAT_PRICE.ENABLED) {
|
||||
priceUpdater.$run();
|
||||
void priceUpdater.$run();
|
||||
}
|
||||
|
||||
// rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS
|
||||
|
|
@ -294,6 +299,7 @@ class Server {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
async $runLightningBackend(): Promise<void> {
|
||||
try {
|
||||
await fundingTxFetcher.$init();
|
||||
|
|
@ -303,7 +309,7 @@ class Server {
|
|||
} catch(e) {
|
||||
logger.err(`Exception in $runLightningBackend. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
|
||||
await Common.sleep$(1000 * 60);
|
||||
this.$runLightningBackend();
|
||||
void this.$runLightningBackend();
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -336,9 +342,9 @@ class Server {
|
|||
}
|
||||
loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler));
|
||||
|
||||
accelerationApi.connectWebsocket();
|
||||
void accelerationApi.connectWebsocket();
|
||||
if (config.STRATUM.ENABLED) {
|
||||
stratumApi.connectWebsocket();
|
||||
void stratumApi.connectWebsocket();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ class Indexer {
|
|||
private indexerRunning = false;
|
||||
private tasksRunning: { [key in TaskName]?: boolean; } = {};
|
||||
private tasksScheduled: { [key in TaskName]?: NodeJS.Timeout; } = {};
|
||||
private reindexTimeout: NodeJS.Timeout | undefined;
|
||||
private coreIndexes: CoreIndex[] = [];
|
||||
|
||||
public indexerIsRunning(): boolean {
|
||||
|
|
@ -34,6 +35,8 @@ class Indexer {
|
|||
|
||||
/**
|
||||
* Check which core index is available for indexing
|
||||
*
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async checkAvailableCoreIndexes(): Promise<void> {
|
||||
const updatedCoreIndexes: CoreIndex[] = [];
|
||||
|
|
@ -45,13 +48,13 @@ class Indexer {
|
|||
synced: indexes[indexName].synced,
|
||||
best_block_height: indexes[indexName].best_block_height,
|
||||
};
|
||||
logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`);
|
||||
logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`);
|
||||
updatedCoreIndexes.push(newState);
|
||||
|
||||
if (indexName === 'coinstatsindex' && newState.synced === true) {
|
||||
const previousState = this.isCoreIndexReady('coinstatsindex');
|
||||
// if (!previousState || previousState.synced === false) {
|
||||
this.runSingleTask('coinStatsIndex');
|
||||
void this.runSingleTask('coinStatsIndex');
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
|
@ -61,9 +64,9 @@ class Indexer {
|
|||
|
||||
/**
|
||||
* Return the best block height if a core index is available, or 0 if not
|
||||
*
|
||||
* @param name
|
||||
* @returns
|
||||
*
|
||||
* @param name
|
||||
* @returns
|
||||
*/
|
||||
public isCoreIndexReady(name: string): CoreIndex | null {
|
||||
for (const index of this.coreIndexes) {
|
||||
|
|
@ -76,10 +79,23 @@ class Indexer {
|
|||
|
||||
public reindex(): void {
|
||||
if (Common.indexingEnabled()) {
|
||||
if (this.reindexTimeout) {
|
||||
clearTimeout(this.reindexTimeout);
|
||||
this.reindexTimeout = undefined;
|
||||
}
|
||||
this.runIndexer = true;
|
||||
}
|
||||
}
|
||||
|
||||
private scheduleNextRun(timeout: number): void {
|
||||
if (!this.reindexTimeout) { // Only one future run should be planned, ignore if already scheduled
|
||||
this.reindexTimeout = setTimeout(() => {
|
||||
this.reindexTimeout = undefined;
|
||||
this.reindex();
|
||||
}, timeout);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* schedules a single task to run in `timeout` ms
|
||||
* only one task of each type may be scheduled
|
||||
|
|
@ -111,6 +127,8 @@ class Indexer {
|
|||
* Runs a single task immediately
|
||||
*
|
||||
* (use `scheduleSingleTask` instead to queue a task to run after some timeout)
|
||||
*
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async runSingleTask(task: TaskName): Promise<void> {
|
||||
if (!Common.indexingEnabled() || this.tasksRunning[task]) {
|
||||
|
|
@ -120,7 +138,7 @@ class Indexer {
|
|||
|
||||
switch (task) {
|
||||
case 'blocksPrices': {
|
||||
if (!['testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK) && config.FIAT_PRICE.ENABLED) {
|
||||
if (!['testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) && config.FIAT_PRICE.ENABLED) {
|
||||
let lastestPriceId;
|
||||
try {
|
||||
lastestPriceId = await PricesRepository.$getLatestPriceId();
|
||||
|
|
@ -149,6 +167,7 @@ class Indexer {
|
|||
this.tasksRunning[task] = false;
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $run(): Promise<void> {
|
||||
if (!Common.indexingEnabled() || this.runIndexer === false ||
|
||||
this.indexerRunning === true || mempool.hasPriority()
|
||||
|
|
@ -156,38 +175,44 @@ class Indexer {
|
|||
return;
|
||||
}
|
||||
|
||||
if (config.FIAT_PRICE.ENABLED) {
|
||||
try {
|
||||
await priceUpdater.$run();
|
||||
} catch (e) {
|
||||
logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e));
|
||||
}
|
||||
}
|
||||
|
||||
// Do not attempt to index anything unless Bitcoin Core is fully synced
|
||||
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
|
||||
if (blockchainInfo.blocks !== blockchainInfo.headers) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.runIndexer = false;
|
||||
this.indexerRunning = true;
|
||||
|
||||
logger.debug(`Running mining indexer`);
|
||||
|
||||
await this.checkAvailableCoreIndexes();
|
||||
const retryDelay = 10000;
|
||||
const runEvery = 1000 * 3600; // 1 hour
|
||||
let nextRunDelay = runEvery;
|
||||
let runSuccessful = false;
|
||||
|
||||
try {
|
||||
if (config.FIAT_PRICE.ENABLED) {
|
||||
try {
|
||||
await priceUpdater.$run();
|
||||
} catch (e) {
|
||||
logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e));
|
||||
}
|
||||
}
|
||||
|
||||
// Do not attempt to index anything unless Bitcoin Core is fully synced
|
||||
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
|
||||
if (blockchainInfo.blocks !== blockchainInfo.headers) {
|
||||
logger.debug(`Bitcoin Core not fully synced, retrying index run in 10 seconds.`);
|
||||
nextRunDelay = retryDelay;
|
||||
return;
|
||||
}
|
||||
|
||||
logger.debug(`Running mining indexer`);
|
||||
|
||||
await this.checkAvailableCoreIndexes();
|
||||
|
||||
const chainValid = await blocks.$generateBlockDatabase();
|
||||
if (chainValid === false) {
|
||||
// Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration
|
||||
logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`, logger.tags.mining);
|
||||
setTimeout(() => this.reindex(), 10000);
|
||||
this.indexerRunning = false;
|
||||
nextRunDelay = retryDelay;
|
||||
return;
|
||||
}
|
||||
|
||||
this.runSingleTask('blocksPrices');
|
||||
void this.runSingleTask('blocksPrices');
|
||||
await blocks.$indexCoinbaseAddresses();
|
||||
await mining.$indexDifficultyAdjustments();
|
||||
await mining.$generateNetworkHashrateHistory();
|
||||
|
|
@ -202,20 +227,21 @@ class Indexer {
|
|||
await BlocksAuditsRepository.$migrateAuditsV0toV1();
|
||||
await BlocksRepository.$migrateBlocks();
|
||||
// do not wait for classify blocks to finish
|
||||
blocks.$classifyBlocks();
|
||||
void blocks.$classifyBlocks();
|
||||
runSuccessful = true;
|
||||
} catch (e) {
|
||||
this.indexerRunning = false;
|
||||
nextRunDelay = retryDelay;
|
||||
logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));
|
||||
setTimeout(() => this.reindex(), 10000);
|
||||
} finally {
|
||||
this.indexerRunning = false;
|
||||
return;
|
||||
const nextRunAt = new Date(Date.now() + nextRunDelay).toUTCString();
|
||||
if (runSuccessful) {
|
||||
logger.debug(`Indexing completed. Next run planned at ${nextRunAt}`);
|
||||
} else {
|
||||
logger.debug(`Indexing did not complete, next run planned at ${nextRunAt}`);
|
||||
}
|
||||
this.scheduleNextRun(nextRunDelay);
|
||||
}
|
||||
|
||||
this.indexerRunning = false;
|
||||
|
||||
const runEvery = 1000 * 3600; // 1 hour
|
||||
logger.debug(`Indexing completed. Next run planned at ${new Date(new Date().getTime() + runEvery).toUTCString()}`);
|
||||
setTimeout(() => this.reindex(), runEvery);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ class Logger {
|
|||
mining: 'Mining',
|
||||
ln: 'Lightning',
|
||||
goggles: 'Goggles',
|
||||
};
|
||||
};
|
||||
|
||||
// @ts-ignore
|
||||
public emerg: ((msg: string, tag?: string) => void);
|
||||
|
|
@ -67,6 +67,8 @@ class Logger {
|
|||
}
|
||||
}
|
||||
this.client = dgram.createSocket('udp4');
|
||||
// Unref the socket so it doesn't prevent Node.js from exiting
|
||||
this.client.unref();
|
||||
this.network = this.getNetwork();
|
||||
}
|
||||
|
||||
|
|
@ -84,7 +86,7 @@ class Logger {
|
|||
|
||||
private getNetwork(): string {
|
||||
if (config.LIGHTNING.ENABLED) {
|
||||
return config.MEMPOOL.NETWORK === 'mainnet' ? 'lightning' : `${config.MEMPOOL.NETWORK}-lightning`;
|
||||
return config.MEMPOOL.NETWORK === 'mainnet' ? 'lightning' : `${config.MEMPOOL.NETWORK}-lightning`;
|
||||
}
|
||||
if (config.MEMPOOL.NETWORK && config.MEMPOOL.NETWORK !== 'mainnet') {
|
||||
return config.MEMPOOL.NETWORK;
|
||||
|
|
@ -153,6 +155,20 @@ class Logger {
|
|||
months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
|
||||
return months[month] + ' ' + day + ' ' + hours + ':' + minutes + ':' + seconds;
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the UDP socket used for syslog
|
||||
* This should only be called when shutting down or in test teardown
|
||||
*/
|
||||
public close(): void {
|
||||
if (this.client) {
|
||||
// Unref allows Node.js to exit even if the socket is open
|
||||
this.client.unref();
|
||||
this.client.close(() => {
|
||||
// Socket closed callback
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export type LogLevel = 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
|
||||
|
|
|
|||
|
|
@ -504,9 +504,36 @@ export interface IBackendInfo {
|
|||
gitCommit: string;
|
||||
version: string;
|
||||
lightning: boolean;
|
||||
coreVersion: string;
|
||||
osVersion: string;
|
||||
backend: 'esplora' | 'electrum' | 'none';
|
||||
}
|
||||
|
||||
export interface INetworkInfo {
|
||||
version: number;
|
||||
subversion: string;
|
||||
protocolversion: number;
|
||||
localservices: string;
|
||||
localrelay: boolean;
|
||||
timeoffset: number;
|
||||
networkactive: boolean;
|
||||
networks: {
|
||||
name: string;
|
||||
limited: boolean;
|
||||
reachable: boolean;
|
||||
proxy: string;
|
||||
proxy_randomize_credentials: boolean;
|
||||
}[];
|
||||
relayfee: number;
|
||||
incrementalfee: number;
|
||||
localaddresses: {
|
||||
address: string;
|
||||
port: number;
|
||||
score: number;
|
||||
}[];
|
||||
warnings: string;
|
||||
}
|
||||
|
||||
export interface IDifficultyAdjustment {
|
||||
progressPercent: number;
|
||||
difficultyChange: number;
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ class AuditReplication {
|
|||
inProgress: boolean = false;
|
||||
skip: Set<string> = new Set();
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $sync(): Promise<void> {
|
||||
if (!config.REPLICATION.ENABLED || !config.REPLICATION.AUDIT) {
|
||||
// replication not enabled
|
||||
|
|
@ -54,6 +55,7 @@ class AuditReplication {
|
|||
this.inProgress = false;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
private async $syncAudit(hash: string): Promise<boolean> {
|
||||
if (this.skip.has(hash)) {
|
||||
// we already know none of our trusted servers have this audit
|
||||
|
|
@ -77,6 +79,8 @@ class AuditReplication {
|
|||
return success;
|
||||
}
|
||||
|
||||
|
||||
/** @asyncSafe */
|
||||
private async $getMissingAuditBlocks(): Promise<string[]> {
|
||||
try {
|
||||
const startHeight = config.REPLICATION.AUDIT_START_HEIGHT || 0;
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ const steps = {
|
|||
class StatisticsReplication {
|
||||
inProgress: boolean = false;
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $sync(): Promise<void> {
|
||||
if (!config.REPLICATION.ENABLED || !config.REPLICATION.STATISTICS || !config.STATISTICS.ENABLED) {
|
||||
// replication not enabled, or statistics not enabled
|
||||
|
|
@ -51,12 +52,12 @@ class StatisticsReplication {
|
|||
logger.info(`Statistics table is complete, no replication needed`, 'Replication');
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
for (const interval of missingIntervals) {
|
||||
logger.debug(`Missing ${missingStatistics[interval].size} statistics rows in '${interval}' timespan`, 'Replication');
|
||||
}
|
||||
logger.debug(`Fetching ${missingIntervals.join(', ')} statistics endpoints from trusted servers to fill ${totalMissing} rows missing in statistics`, 'Replication');
|
||||
|
||||
|
||||
let totalSynced = 0;
|
||||
let totalMissed = 0;
|
||||
|
||||
|
|
@ -74,16 +75,17 @@ class StatisticsReplication {
|
|||
this.inProgress = false;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
private async $syncStatistics(interval: string, missingTimes: Set<number>): Promise<any> {
|
||||
|
||||
|
||||
let success = false;
|
||||
let synced = 0;
|
||||
let missed = new Set(missingTimes);
|
||||
const missed = new Set(missingTimes);
|
||||
const syncResult = await $sync(`/api/v1/statistics/${interval}`);
|
||||
if (syncResult && syncResult.data?.length) {
|
||||
success = true;
|
||||
logger.info(`Fetched /api/v1/statistics/${interval} from ${syncResult.server}`);
|
||||
|
||||
logger.info(`Fetched /api/v1/statistics/${interval} from ${syncResult.server}`);
|
||||
|
||||
for (const stat of syncResult.data) {
|
||||
const time = this.roundToNearestStep(stat.added, steps[interval]);
|
||||
if (missingTimes.has(time)) {
|
||||
|
|
@ -105,6 +107,8 @@ class StatisticsReplication {
|
|||
return { success, synced, missed: missed.size };
|
||||
}
|
||||
|
||||
|
||||
/** @asyncUnsafe */
|
||||
private async $getMissingStatistics(): Promise<MissingStatistics> {
|
||||
try {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
|
|
@ -129,7 +133,7 @@ class StatisticsReplication {
|
|||
startTime < now - day * 30 ? [now - day * 90, now - day * 30, '3m' ] : null, // from 3 months ago to 1 month ago = 2 hours granularity
|
||||
startTime < now - day * 90 ? [now - day * 180, now - day * 90, '6m' ] : null, // from 6 months ago to 3 months ago = 3 hours granularity
|
||||
startTime < now - day * 180 ? [now - day * 365 * 2, now - day * 180, '2y' ] : null, // from 2 years ago to 6 months ago = 8 hours granularity
|
||||
startTime < now - day * 365 * 2 ? [startTime, now - day * 365 * 2, 'all'] : null, // from start of statistics to 2 years ago = 12 hours granularity
|
||||
startTime < now - day * 365 * 2 ? [startTime, now - day * 365 * 2, 'all'] : null, // from start of statistics to 2 years ago = 12 hours granularity
|
||||
];
|
||||
|
||||
for (const interval of intervals) {
|
||||
|
|
@ -138,7 +142,7 @@ class StatisticsReplication {
|
|||
}
|
||||
missingStatistics[interval[2] as string] = await this.$getMissingStatisticsInterval(interval, startTime);
|
||||
}
|
||||
|
||||
|
||||
return missingStatistics;
|
||||
} catch (e: any) {
|
||||
logger.err(`Cannot fetch missing statistics times from db. Reason: ` + (e instanceof Error ? e.message : e));
|
||||
|
|
@ -146,6 +150,7 @@ class StatisticsReplication {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
private async $getMissingStatisticsInterval(interval: any, startTime: number): Promise<Set<number>> {
|
||||
try {
|
||||
const start = interval[0];
|
||||
|
|
@ -169,17 +174,17 @@ class StatisticsReplication {
|
|||
if (timeSteps.length === 0) {
|
||||
return new Set<number>();
|
||||
}
|
||||
|
||||
|
||||
const roundedTimesAlreadyHere: number[] = Array.from(new Set(rows.map(row => this.roundToNearestStep(row.added, step))));
|
||||
|
||||
const missingTimes = timeSteps.filter(time => !roundedTimesAlreadyHere.includes(time)).filter((time, i, arr) => {
|
||||
// Remove outsiders
|
||||
if (i === 0) {
|
||||
return arr[i + 1] === time + step
|
||||
return arr[i + 1] === time + step;
|
||||
} else if (i === arr.length - 1) {
|
||||
return arr[i - 1] === time - step;
|
||||
}
|
||||
return (arr[i + 1] === time + step) && (arr[i - 1] === time - step)
|
||||
return (arr[i + 1] === time + step) && (arr[i - 1] === time - step);
|
||||
});
|
||||
|
||||
// Don't bother fetching if very few rows are missing
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import axios, { AxiosResponse } from 'axios';
|
|||
import { SocksProxyAgent } from 'socks-proxy-agent';
|
||||
import * as https from 'https';
|
||||
|
||||
/** @asyncSafe */
|
||||
export async function $sync(path): Promise<{ data?: any, exists: boolean, server?: string }> {
|
||||
// start with a random server so load is uniformly spread
|
||||
let allMissing = true;
|
||||
|
|
@ -14,7 +15,7 @@ export async function $sync(path): Promise<{ data?: any, exists: boolean, server
|
|||
if (server === backendInfo.getBackendInfo().hostname) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
try {
|
||||
const result = await query(`https://${server}${path}`);
|
||||
if (result) {
|
||||
|
|
@ -33,6 +34,7 @@ export async function $sync(path): Promise<{ data?: any, exists: boolean, server
|
|||
return { exists: !allMissing };
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
export async function query(path): Promise<object> {
|
||||
type axiosOptions = {
|
||||
headers: {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ export interface PublicAcceleration {
|
|||
class AccelerationRepository {
|
||||
private bidBoostV2Activated = 831580;
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $saveAcceleration(acceleration: AccelerationInfo, block: IEsploraApi.Block, pool_id: number, accelerationData: Acceleration[]): Promise<void> {
|
||||
const accelerationMap: { [txid: string]: Acceleration } = {};
|
||||
for (const acc of accelerationData) {
|
||||
|
|
@ -60,6 +61,38 @@ class AccelerationRepository {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getAccelerationInfoForTxid(txid: string): Promise<PublicAcceleration | null> {
|
||||
try {
|
||||
const [rows] = await DB.query(`
|
||||
SELECT *, UNIX_TIMESTAMP(requested) as requested_timestamp, UNIX_TIMESTAMP(added) as block_timestamp FROM accelerations
|
||||
JOIN pools on pools.unique_id = accelerations.pool
|
||||
WHERE txid = ?
|
||||
`, [txid]) as RowDataPacket[][];
|
||||
if (rows?.length) {
|
||||
const row = rows[0];
|
||||
return {
|
||||
txid: row.txid,
|
||||
height: row.height,
|
||||
added: row.requested_timestamp || row.block_timestamp,
|
||||
pool: {
|
||||
id: row.id,
|
||||
slug: row.slug,
|
||||
name: row.name,
|
||||
},
|
||||
effective_vsize: row.effective_vsize,
|
||||
effective_fee: row.effective_fee,
|
||||
boost_rate: row.boost_rate,
|
||||
boost_cost: row.boost_cost,
|
||||
};
|
||||
}
|
||||
} catch (e: any) {
|
||||
logger.err(`Cannot get acceleration info for txid ${txid}. Reason: ` + (e instanceof Error ? e.message : e));
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public async $getAccelerationInfo(poolSlug: string | null = null, height: number | null = null, interval: string | null = null): Promise<PublicAcceleration[]> {
|
||||
if (!interval || !['24h', '3d', '1w', '1m'].includes(interval)) {
|
||||
interval = '1m';
|
||||
|
|
@ -74,7 +107,7 @@ class AccelerationRepository {
|
|||
SELECT *, UNIX_TIMESTAMP(requested) as requested_timestamp, UNIX_TIMESTAMP(added) as block_timestamp FROM accelerations
|
||||
JOIN pools on pools.unique_id = accelerations.pool
|
||||
`;
|
||||
let params: any[] = [];
|
||||
const params: any[] = [];
|
||||
let hasFilter = false;
|
||||
|
||||
if (interval && height === null) {
|
||||
|
|
@ -137,7 +170,7 @@ class AccelerationRepository {
|
|||
SELECT SUM(boost_cost) as total_cost, COUNT(txid) as count FROM accelerations
|
||||
JOIN pools on pools.unique_id = accelerations.pool
|
||||
`;
|
||||
let params: any[] = [];
|
||||
const params: any[] = [];
|
||||
let hasFilter = false;
|
||||
|
||||
if (interval) {
|
||||
|
|
@ -165,6 +198,7 @@ class AccelerationRepository {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getLastSyncedHeight(): Promise<number> {
|
||||
try {
|
||||
const [rows] = await DB.query(`
|
||||
|
|
@ -180,6 +214,7 @@ class AccelerationRepository {
|
|||
return 0;
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
private async $setLastSyncedHeight(height: number): Promise<void> {
|
||||
try {
|
||||
await DB.query(`
|
||||
|
|
@ -193,6 +228,7 @@ class AccelerationRepository {
|
|||
}
|
||||
|
||||
// modifies block transactions
|
||||
/** @asyncSafe */
|
||||
public async $indexAccelerationsForBlock(block: BlockExtended, accelerations: Acceleration[], transactions: MempoolTransactionExtended[]): Promise<void> {
|
||||
const blockTxs: { [txid: string]: MempoolTransactionExtended } = {};
|
||||
for (const tx of transactions) {
|
||||
|
|
@ -211,7 +247,7 @@ class AccelerationRepository {
|
|||
const tx = blockTxs[acc.txid];
|
||||
const accelerationInfo = accelerationCosts.getAccelerationInfo(tx, boostRate, transactions);
|
||||
accelerationInfo.cost = Math.max(0, Math.min(acc.feeDelta, accelerationInfo.cost));
|
||||
this.$saveAcceleration(accelerationInfo, block, block.extras.pool.id, successfulAccelerations);
|
||||
void this.$saveAcceleration(accelerationInfo, block, block.extras.pool.id, successfulAccelerations);
|
||||
}
|
||||
}
|
||||
let anyConfirmed = false;
|
||||
|
|
@ -320,7 +356,7 @@ class AccelerationRepository {
|
|||
const accelerationSummaries = accelerations.map(acc => ({
|
||||
...acc,
|
||||
pools: acc.pools,
|
||||
}))
|
||||
}));
|
||||
for (const acc of accelerations) {
|
||||
if (blockTxs[acc.txid] && acc.pools.includes(block.extras.pool.id)) {
|
||||
const tx = blockTxs[acc.txid];
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ interface MigrationAudit {
|
|||
}
|
||||
|
||||
class BlocksAuditRepositories {
|
||||
/** @asyncSafe */
|
||||
public async $saveAudit(audit: BlockAudit): Promise<void> {
|
||||
try {
|
||||
await DB.query(`INSERT INTO blocks_audits(version, time, height, hash, unseen_txs, missing_txs, added_txs, prioritized_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight)
|
||||
|
|
@ -29,6 +30,7 @@ class BlocksAuditRepositories {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $setSummary(hash: string, expectedFees: number, expectedWeight: number) {
|
||||
try {
|
||||
await DB.query(`
|
||||
|
|
@ -42,6 +44,7 @@ class BlocksAuditRepositories {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getBlocksHealthHistory(div: number, interval: string | null): Promise<any> {
|
||||
try {
|
||||
let query = `SELECT UNIX_TIMESTAMP(time) as time, height, match_rate FROM blocks_audits`;
|
||||
|
|
@ -60,6 +63,7 @@ class BlocksAuditRepositories {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getBlocksHealthCount(): Promise<number> {
|
||||
try {
|
||||
const [rows] = await DB.query(`SELECT count(hash) as count FROM blocks_audits`);
|
||||
|
|
@ -70,6 +74,7 @@ class BlocksAuditRepositories {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getBlockAudit(hash: string): Promise<BlockAudit | null> {
|
||||
try {
|
||||
const [rows]: any[] = await DB.query(
|
||||
|
|
@ -94,7 +99,7 @@ class BlocksAuditRepositories {
|
|||
JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash
|
||||
WHERE blocks_audits.hash = ?
|
||||
`, [hash]);
|
||||
|
||||
|
||||
if (rows.length) {
|
||||
rows[0].unseenTxs = JSON.parse(rows[0].unseenTxs);
|
||||
rows[0].missingTxs = JSON.parse(rows[0].missingTxs);
|
||||
|
|
@ -115,6 +120,7 @@ class BlocksAuditRepositories {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getBlockTxAudit(hash: string, txid: string): Promise<TransactionAudit | null> {
|
||||
try {
|
||||
const blockAudit = await this.$getBlockAudit(hash);
|
||||
|
|
@ -151,6 +157,7 @@ class BlocksAuditRepositories {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getBlockAuditScore(hash: string): Promise<AuditScore> {
|
||||
try {
|
||||
const [rows]: any[] = await DB.query(
|
||||
|
|
@ -165,6 +172,7 @@ class BlocksAuditRepositories {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getBlockAuditScores(maxHeight: number, minHeight: number): Promise<AuditScore[]> {
|
||||
try {
|
||||
const [rows]: any[] = await DB.query(
|
||||
|
|
@ -179,6 +187,7 @@ class BlocksAuditRepositories {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getBlocksWithoutSummaries(): Promise<string[]> {
|
||||
try {
|
||||
const [fromRows]: any[] = await DB.query(`
|
||||
|
|
@ -207,6 +216,7 @@ class BlocksAuditRepositories {
|
|||
|
||||
/**
|
||||
* [INDEXING] Migrate audits from v0 to v1
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $migrateAuditsV0toV1(): Promise<void> {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -114,6 +114,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Save indexed block data in the database
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $saveBlockInDatabase(block: BlockExtended) {
|
||||
const truncatedCoinbaseSignature = block?.extras?.coinbaseSignature?.substring(0, 500);
|
||||
|
|
@ -218,9 +219,10 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Save newly indexed data from core coinstatsindex
|
||||
*
|
||||
* @param utxoSetSize
|
||||
* @param totalInputAmt
|
||||
*
|
||||
* @param utxoSetSize
|
||||
* @param totalInputAmt
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $updateCoinStatsIndexData(blockHash: string, utxoSetSize: number,
|
||||
totalInputAmt: number
|
||||
|
|
@ -246,9 +248,10 @@ class BlocksRepository {
|
|||
/**
|
||||
* Update missing fee amounts fields
|
||||
*
|
||||
* @param blockHash
|
||||
* @param feeAmtPercentiles
|
||||
* @param medianFeeAmt
|
||||
* @param blockHash
|
||||
* @param feeAmtPercentiles
|
||||
* @param medianFeeAmt
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $updateFeeAmounts(blockHash: string, feeAmtPercentiles, medianFeeAmt) : Promise<void> {
|
||||
try {
|
||||
|
|
@ -271,9 +274,14 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get all block height that have not been indexed between [startHeight, endHeight]
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getMissingBlocksBetweenHeights(startHeight: number, endHeight: number): Promise<number[]> {
|
||||
if (startHeight < endHeight) {
|
||||
// Ensure startHeight is the lower value and endHeight is the higher value
|
||||
const minHeight = Math.min(startHeight, endHeight);
|
||||
const maxHeight = Math.max(startHeight, endHeight);
|
||||
|
||||
if (minHeight === maxHeight) {
|
||||
return [];
|
||||
}
|
||||
|
||||
|
|
@ -281,13 +289,13 @@ class BlocksRepository {
|
|||
const [rows]: any[] = await DB.query(`
|
||||
SELECT height
|
||||
FROM blocks
|
||||
WHERE height <= ? AND height >= ? AND stale = 0
|
||||
ORDER BY height DESC;
|
||||
`, [startHeight, endHeight]);
|
||||
WHERE height >= ? AND height <= ? AND stale = 0
|
||||
ORDER BY height ASC;
|
||||
`, [minHeight, maxHeight]);
|
||||
|
||||
const indexedBlockHeights: number[] = [];
|
||||
rows.forEach((row: any) => { indexedBlockHeights.push(row.height); });
|
||||
const seekedBlocks: number[] = Array.from(Array(startHeight - endHeight + 1).keys(), n => n + endHeight).reverse();
|
||||
const seekedBlocks: number[] = Array.from(Array(maxHeight - minHeight + 1).keys(), n => n + minHeight);
|
||||
const missingBlocksHeights = seekedBlocks.filter(x => indexedBlockHeights.indexOf(x) === -1);
|
||||
|
||||
return missingBlocksHeights;
|
||||
|
|
@ -299,6 +307,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get empty blocks for one or all pools
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $countEmptyBlocks(poolId: number | null, interval: string | null = null): Promise<any> {
|
||||
interval = Common.getSqlInterval(interval);
|
||||
|
|
@ -331,6 +340,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Return most recent block height
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $mostRecentBlockHeight(): Promise<number> {
|
||||
try {
|
||||
|
|
@ -344,6 +354,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get blocks count for a period
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $blockCount(poolId: number | null, interval: string | null = null): Promise<number> {
|
||||
interval = Common.getSqlInterval(interval);
|
||||
|
|
@ -377,6 +388,7 @@ class BlocksRepository {
|
|||
* @param from - The oldest timestamp
|
||||
* @param to - The newest timestamp
|
||||
* @returns
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $blockCountBetweenTimestamp(poolId: number | null, from: number, to: number): Promise<number> {
|
||||
const params: any[] = [];
|
||||
|
|
@ -404,10 +416,11 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get blocks count for a period
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $blockCountBetweenHeight(startHeight: number, endHeight: number): Promise<number> {
|
||||
const params: any[] = [];
|
||||
let query = `SELECT count(height) as blockCount
|
||||
const query = `SELECT count(height) as blockCount
|
||||
FROM blocks
|
||||
WHERE height <= ${startHeight} AND height >= ${endHeight} AND stale = 0`;
|
||||
|
||||
|
|
@ -422,6 +435,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get average block health for all blocks for a single pool
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getAvgBlockHealthPerPoolId(poolId: number): Promise<number | null> {
|
||||
const params: any[] = [];
|
||||
|
|
@ -447,6 +461,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get average block health for all blocks for a single pool
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getTotalRewardForPoolId(poolId: number): Promise<number> {
|
||||
const params: any[] = [];
|
||||
|
|
@ -471,6 +486,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get the oldest indexed block
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $oldestBlockTimestamp(): Promise<number> {
|
||||
const query = `SELECT UNIX_TIMESTAMP(blockTimestamp) as blockTimestamp
|
||||
|
|
@ -495,6 +511,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get blocks mined by a specific mining pool
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getBlocksByPool(slug: string, startHeight?: number): Promise<BlockExtended[]> {
|
||||
const pool = await PoolsRepository.$getPool(slug);
|
||||
|
|
@ -535,6 +552,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get one block by height
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getBlockByHeight(height: number): Promise<BlockExtended | null> {
|
||||
try {
|
||||
|
|
@ -583,6 +601,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Return blocks difficulty
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getBlocksDifficulty(): Promise<object[]> {
|
||||
try {
|
||||
|
|
@ -598,6 +617,7 @@ class BlocksRepository {
|
|||
* Get the first block at or directly after a given timestamp
|
||||
* @param timestamp number unix time in seconds
|
||||
* @returns The height and timestamp of a block (timestamp might vary from given timestamp)
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getBlockHeightFromTimestamp(
|
||||
timestamp: number,
|
||||
|
|
@ -626,6 +646,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get general block stats
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getBlockStats(blockCount: number): Promise<any> {
|
||||
try {
|
||||
|
|
@ -649,6 +670,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Check if the canonical chain of blocks is valid and fix it if needed
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $validateChain(): Promise<boolean> {
|
||||
try {
|
||||
|
|
@ -682,7 +704,16 @@ class BlocksRepository {
|
|||
// iterate back to genesis, resetting canonical status where necessary
|
||||
let hash = tip;
|
||||
const tipHeight = blocksByHash[hash].height || (await bitcoinApi.$getBlock(hash))?.height;
|
||||
for (let height = tipHeight; height > 0; height--) {
|
||||
|
||||
// stop at the last canonical block we're supposed to have indexed already
|
||||
let lastIndexedBlockHeight = minHeight;
|
||||
const indexedBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, tipHeight);
|
||||
if (indexedBlockAmount > 0) {
|
||||
lastIndexedBlockHeight = Math.max(0, tipHeight - indexedBlockAmount + 1);
|
||||
}
|
||||
|
||||
|
||||
for (let height = tipHeight; height > lastIndexedBlockHeight; height--) {
|
||||
const block = blocksByHash[hash];
|
||||
if (!block) {
|
||||
// block hasn't been indexed
|
||||
|
|
@ -725,6 +756,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get the historical averaged block fees
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getHistoricalBlockFees(div: number, interval: string | null, timespan?: {from: number, to: number}): Promise<any> {
|
||||
try {
|
||||
|
|
@ -757,6 +789,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get the historical averaged block rewards
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getHistoricalBlockRewards(div: number, interval: string | null): Promise<any> {
|
||||
try {
|
||||
|
|
@ -787,6 +820,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get the historical averaged block fee rate percentiles
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getHistoricalBlockFeeRates(div: number, interval: string | null): Promise<any> {
|
||||
try {
|
||||
|
|
@ -819,6 +853,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get the historical averaged block sizes
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getHistoricalBlockSizes(div: number, interval: string | null): Promise<any> {
|
||||
try {
|
||||
|
|
@ -845,6 +880,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get the historical averaged block weights
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getHistoricalBlockWeights(div: number, interval: string | null): Promise<any> {
|
||||
try {
|
||||
|
|
@ -872,6 +908,7 @@ class BlocksRepository {
|
|||
/**
|
||||
* Get a list of blocks that have been indexed
|
||||
* (includes stale blocks)
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getIndexedBlocks(): Promise<{ height: number, hash: string, stale: boolean }[]> {
|
||||
try {
|
||||
|
|
@ -885,6 +922,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get a list of blocks that have not had CPFP data indexed
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getCPFPUnindexedBlocks(): Promise<number[]> {
|
||||
try {
|
||||
|
|
@ -918,6 +956,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Return the oldest block from a consecutive chain of block from the most recent one
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getOldestConsecutiveBlock(): Promise<any> {
|
||||
try {
|
||||
|
|
@ -936,6 +975,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get all blocks which have not be linked to a price yet
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getBlocksWithoutPrice(): Promise<object[]> {
|
||||
try {
|
||||
|
|
@ -957,6 +997,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Save block price by batch
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $saveBlockPrices(blockPrices: BlockPrice[]): Promise<void> {
|
||||
try {
|
||||
|
|
@ -978,6 +1019,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Get all indexed blocsk with missing coinstatsindex data
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getBlocksMissingCoinStatsIndex(maxHeight: number, minHeight: number): Promise<any> {
|
||||
try {
|
||||
|
|
@ -997,6 +1039,7 @@ class BlocksRepository {
|
|||
/**
|
||||
* Get all indexed blocks with missing coinbase addresses
|
||||
* (includes stale blocks)
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getBlocksWithoutCoinbaseAddresses(): Promise<any> {
|
||||
try {
|
||||
|
|
@ -1016,9 +1059,10 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Save indexed median fee to avoid recomputing it later
|
||||
*
|
||||
* @param id
|
||||
* @param feePercentiles
|
||||
*
|
||||
* @param id
|
||||
* @param feePercentiles
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $saveFeePercentilesForBlockId(id: string, feePercentiles: number[]): Promise<void> {
|
||||
try {
|
||||
|
|
@ -1035,9 +1079,10 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Save indexed effective fee statistics
|
||||
*
|
||||
* @param id
|
||||
* @param feeStats
|
||||
*
|
||||
* @param id
|
||||
* @param feeStats
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $saveEffectiveFeeStats(id: string, feeStats: EffectiveFeeStats): Promise<void> {
|
||||
try {
|
||||
|
|
@ -1054,9 +1099,10 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Save coinbase addresses
|
||||
*
|
||||
*
|
||||
* @param id
|
||||
* @param addresses
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $saveCoinbaseAddresses(id: string, addresses: string[]): Promise<void> {
|
||||
try {
|
||||
|
|
@ -1073,9 +1119,10 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Save pool
|
||||
*
|
||||
*
|
||||
* @param id
|
||||
* @param poolId
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $savePool(id: string, poolId: number): Promise<void> {
|
||||
try {
|
||||
|
|
@ -1092,6 +1139,9 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Save block first seen times
|
||||
*
|
||||
* @param results
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $saveFirstSeenTimes(results: { hash: string; firstSeen: number | null }[]): Promise<void> {
|
||||
if (!results.length) {
|
||||
|
|
@ -1143,7 +1193,7 @@ class BlocksRepository {
|
|||
|
||||
/**
|
||||
* Change which block at a height belongs to the canonical chain
|
||||
*
|
||||
*
|
||||
* @param hash
|
||||
* @param height
|
||||
*/
|
||||
|
|
@ -1172,8 +1222,9 @@ class BlocksRepository {
|
|||
/**
|
||||
* Convert a mysql row block into a BlockExtended. Note that you
|
||||
* must provide the correct field into dbBlk object param
|
||||
*
|
||||
* @param dbBlk
|
||||
*
|
||||
* @param dbBlk
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
private async formatDbBlockIntoExtendedBlock(dbBlk: DatabaseBlock): Promise<BlockExtended> {
|
||||
const blk: Partial<BlockExtended> = {};
|
||||
|
|
@ -1300,6 +1351,7 @@ class BlocksRepository {
|
|||
}
|
||||
|
||||
// migration to fix median fee bug
|
||||
/** @asyncSafe */
|
||||
private async $migrateBlocksToV1(): Promise<number> {
|
||||
let blocksMigrated = 0;
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import logger from '../logger';
|
|||
import { BlockSummary, TransactionClassified } from '../mempool.interfaces';
|
||||
|
||||
class BlocksSummariesRepository {
|
||||
/** @asyncSafe */
|
||||
public async $getByBlockId(id: string): Promise<BlockSummary | undefined> {
|
||||
try {
|
||||
const [summary]: any[] = await DB.query(`SELECT * from blocks_summaries WHERE id = ?`, [id]);
|
||||
|
|
@ -19,6 +20,7 @@ class BlocksSummariesRepository {
|
|||
return undefined;
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $saveTransactions(blockHeight: number, blockId: string, transactions: TransactionClassified[], version: number): Promise<void> {
|
||||
try {
|
||||
const transactionsStr = JSON.stringify(transactions);
|
||||
|
|
@ -33,6 +35,7 @@ class BlocksSummariesRepository {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $saveTemplate(params: { height: number, template: BlockSummary, version: number}): Promise<void> {
|
||||
const blockId = params.template?.id;
|
||||
try {
|
||||
|
|
@ -53,6 +56,7 @@ class BlocksSummariesRepository {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getTemplate(id: string): Promise<BlockSummary | undefined> {
|
||||
try {
|
||||
const [templates]: any[] = await DB.query(`SELECT * from blocks_templates WHERE id = ?`, [id]);
|
||||
|
|
@ -69,6 +73,7 @@ class BlocksSummariesRepository {
|
|||
return undefined;
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getIndexedSummariesId(): Promise<string[]> {
|
||||
try {
|
||||
const [rows] = await DB.query(`SELECT id from blocks_summaries`) as RowDataPacket[][];
|
||||
|
|
@ -80,6 +85,7 @@ class BlocksSummariesRepository {
|
|||
return [];
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getSummariesWithVersion(version: number): Promise<{ height: number, id: string }[]> {
|
||||
try {
|
||||
const [rows]: any[] = await DB.query(`
|
||||
|
|
@ -97,6 +103,7 @@ class BlocksSummariesRepository {
|
|||
return [];
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getTemplatesWithVersion(version: number): Promise<{ height: number, id: string }[]> {
|
||||
try {
|
||||
const [rows]: any[] = await DB.query(`
|
||||
|
|
@ -115,6 +122,7 @@ class BlocksSummariesRepository {
|
|||
return [];
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getSummariesBelowVersion(version: number): Promise<{ height: number, id: string, version: number }[]> {
|
||||
try {
|
||||
const [rows]: any[] = await DB.query(`
|
||||
|
|
@ -133,6 +141,7 @@ class BlocksSummariesRepository {
|
|||
return [];
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getTemplatesBelowVersion(version: number): Promise<{ height: number, id: string, version: number }[]> {
|
||||
try {
|
||||
const [rows]: any[] = await DB.query(`
|
||||
|
|
@ -154,8 +163,9 @@ class BlocksSummariesRepository {
|
|||
|
||||
/**
|
||||
* Get the fee percentiles if the block has already been indexed, [] otherwise
|
||||
*
|
||||
* @param id
|
||||
*
|
||||
* @param id
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getFeePercentilesByBlockId(id: string): Promise<number[] | null> {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -73,6 +73,8 @@ class CpfpRepository {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $getCluster(clusterRoot: string): Promise<CpfpCluster | void> {
|
||||
const [clusterRows]: any = await DB.query(
|
||||
`
|
||||
|
|
@ -91,6 +93,7 @@ class CpfpRepository {
|
|||
return;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $getClustersAt(height: number): Promise<CpfpCluster[]> {
|
||||
const [clusterRows]: any = await DB.query(
|
||||
`
|
||||
|
|
|
|||
|
|
@ -215,7 +215,7 @@ class HashratesRepository {
|
|||
logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Delete hashrates from the database from timestamp
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ export interface NodeRecord {
|
|||
}
|
||||
|
||||
class NodesRecordsRepository {
|
||||
/** @asyncSafe */
|
||||
public async $saveRecord(record: NodeRecord): Promise<void> {
|
||||
try {
|
||||
const payloadBytes = Buffer.from(record.payload, 'base64');
|
||||
|
|
@ -26,6 +27,7 @@ class NodesRecordsRepository {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getRecordTypes(publicKey: string): Promise<any> {
|
||||
try {
|
||||
const query = `
|
||||
|
|
@ -40,6 +42,7 @@ class NodesRecordsRepository {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $deleteUnusedRecords(publicKey: string, recordTypes: number[]): Promise<number> {
|
||||
try {
|
||||
let query;
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ export interface NodeSocket {
|
|||
}
|
||||
|
||||
class NodesSocketsRepository {
|
||||
/** @asyncSafe */
|
||||
public async $saveSocket(socket: NodeSocket): Promise<void> {
|
||||
try {
|
||||
await DB.query(`
|
||||
|
|
@ -23,6 +24,7 @@ class NodesSocketsRepository {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $deleteUnusedSockets(publicKey: string, addresses: string[]): Promise<number> {
|
||||
if (addresses.length === 0) {
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import { PoolInfo, PoolTag } from '../mempool.interfaces';
|
|||
class PoolsRepository {
|
||||
/**
|
||||
* Get all pools tagging info
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async $getPools(): Promise<PoolTag[]> {
|
||||
const [rows] = await DB.query('SELECT id, unique_id as uniqueId, name, addresses, regexes, slug FROM pools');
|
||||
|
|
@ -16,6 +17,7 @@ class PoolsRepository {
|
|||
|
||||
/**
|
||||
* Get unknown pool tagging info
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async $getUnknownPool(): Promise<PoolTag> {
|
||||
let [rows]: any[] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"');
|
||||
|
|
@ -28,6 +30,7 @@ class PoolsRepository {
|
|||
|
||||
/**
|
||||
* Get basic pool info and block count
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getPoolsInfo(interval: string | null = null): Promise<PoolInfo[]> {
|
||||
interval = Common.getSqlInterval(interval);
|
||||
|
|
@ -66,6 +69,7 @@ class PoolsRepository {
|
|||
|
||||
/**
|
||||
* Get basic pool info and block count between two timestamp
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
|
||||
const query = `SELECT COUNT(height) as blockCount, pools.id as poolId, pools.name as poolName
|
||||
|
|
@ -85,6 +89,7 @@ class PoolsRepository {
|
|||
|
||||
/**
|
||||
* Get a mining pool info
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getPool(slug: string, parse: boolean = true): Promise<PoolTag | null> {
|
||||
const query = `
|
||||
|
|
@ -102,7 +107,7 @@ class PoolsRepository {
|
|||
if (parse) {
|
||||
rows[0].regexes = JSON.parse(rows[0].regexes);
|
||||
}
|
||||
if (['testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
|
||||
if (['testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
|
||||
rows[0].addresses = []; // pools-v2.json only contains mainnet addresses
|
||||
} else if (parse) {
|
||||
rows[0].addresses = JSON.parse(rows[0].addresses);
|
||||
|
|
@ -117,6 +122,7 @@ class PoolsRepository {
|
|||
|
||||
/**
|
||||
* Get a mining pool info by its unique id
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $getPoolByUniqueId(id: number, parse: boolean = true): Promise<PoolTag | null> {
|
||||
const query = `
|
||||
|
|
@ -134,7 +140,7 @@ class PoolsRepository {
|
|||
if (parse) {
|
||||
rows[0].regexes = JSON.parse(rows[0].regexes);
|
||||
}
|
||||
if (['testnet', 'signet', 'testnet4'].includes(config.MEMPOOL.NETWORK)) {
|
||||
if (['testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
|
||||
rows[0].addresses = []; // pools.json only contains mainnet addresses
|
||||
} else if (parse) {
|
||||
rows[0].addresses = JSON.parse(rows[0].addresses);
|
||||
|
|
@ -149,8 +155,9 @@ class PoolsRepository {
|
|||
|
||||
/**
|
||||
* Insert a new mining pool in the database
|
||||
*
|
||||
* @param pool
|
||||
*
|
||||
* @param pool
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $insertNewMiningPool(pool: any, slug: string): Promise<void> {
|
||||
try {
|
||||
|
|
@ -166,10 +173,11 @@ class PoolsRepository {
|
|||
|
||||
/**
|
||||
* Rename an existing mining pool
|
||||
*
|
||||
*
|
||||
* @param dbId
|
||||
* @param newSlug
|
||||
* @param newName
|
||||
* @param newName
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $renameMiningPool(dbId: number, newSlug: string, newName: string): Promise<void> {
|
||||
try {
|
||||
|
|
@ -186,9 +194,10 @@ class PoolsRepository {
|
|||
|
||||
/**
|
||||
* Update an exisiting mining pool link
|
||||
*
|
||||
* @param dbId
|
||||
* @param newLink
|
||||
*
|
||||
* @param dbId
|
||||
* @param newLink
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $updateMiningPoolLink(dbId: number, newLink: string): Promise<void> {
|
||||
try {
|
||||
|
|
@ -206,10 +215,11 @@ class PoolsRepository {
|
|||
|
||||
/**
|
||||
* Update an existing mining pool addresses or coinbase tags
|
||||
*
|
||||
* @param dbId
|
||||
* @param addresses
|
||||
* @param regexes
|
||||
*
|
||||
* @param dbId
|
||||
* @param addresses
|
||||
* @param regexes
|
||||
* @asyncSafe
|
||||
*/
|
||||
public async $updateMiningPoolTags(dbId: number, addresses: string, regexes: string): Promise<void> {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -179,7 +179,7 @@ class PricesRepository {
|
|||
prices[currency] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
try {
|
||||
if (!config.FIAT_PRICE.API_KEY) { // Store only the 7 main currencies
|
||||
await DB.query(`
|
||||
|
|
@ -191,8 +191,8 @@ class PricesRepository {
|
|||
await DB.query(`
|
||||
INSERT INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY, BGN, BRL, CNY, CZK, DKK, HKD, HRK, HUF, IDR, ILS, INR, ISK, KRW, MXN, MYR, NOK, NZD, PHP, PLN, RON, RUB, SEK, SGD, THB, TRY, ZAR)
|
||||
VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ? , ?, ?, ?, ?, ?, ?, ?, ? , ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? , ? )`,
|
||||
[time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY, prices.BGN, prices.BRL, prices.CNY, prices.CZK, prices.DKK,
|
||||
prices.HKD, prices.HRK, prices.HUF, prices.IDR, prices.ILS, prices.INR, prices.ISK, prices.KRW, prices.MXN, prices.MYR, prices.NOK, prices.NZD,
|
||||
[time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY, prices.BGN, prices.BRL, prices.CNY, prices.CZK, prices.DKK,
|
||||
prices.HKD, prices.HRK, prices.HUF, prices.IDR, prices.ILS, prices.INR, prices.ISK, prices.KRW, prices.MXN, prices.MYR, prices.NOK, prices.NZD,
|
||||
prices.PHP, prices.PLN, prices.RON, prices.RUB, prices.SEK, prices.SGD, prices.THB, prices.TRY, prices.ZAR]
|
||||
);
|
||||
}
|
||||
|
|
@ -224,6 +224,7 @@ class PricesRepository {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $getOldestPriceTime(): Promise<number> {
|
||||
const [oldestRow] = await DB.query(`
|
||||
SELECT UNIX_TIMESTAMP(time) AS time
|
||||
|
|
@ -234,6 +235,7 @@ class PricesRepository {
|
|||
return oldestRow[0] ? oldestRow[0].time : 0;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $getLatestPriceId(): Promise<number | null> {
|
||||
const [oldestRow] = await DB.query(`
|
||||
SELECT id
|
||||
|
|
@ -244,6 +246,7 @@ class PricesRepository {
|
|||
return oldestRow[0] ? oldestRow[0].id : null;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $getLatestPriceTime(): Promise<number> {
|
||||
const [oldestRow] = await DB.query(`
|
||||
SELECT UNIX_TIMESTAMP(time) AS time
|
||||
|
|
@ -254,6 +257,7 @@ class PricesRepository {
|
|||
return oldestRow[0] ? oldestRow[0].time : 0;
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $getPricesTimes(): Promise<number[]> {
|
||||
const [times] = await DB.query(`
|
||||
SELECT UNIX_TIMESTAMP(time) AS time
|
||||
|
|
@ -267,6 +271,7 @@ class PricesRepository {
|
|||
return times.map(time => time.time);
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $getPricesTimesWithMissingFields(): Promise<{time: number, USD: number, eur_missing: boolean, gbp_missing: boolean, cad_missing: boolean, chf_missing: boolean, aud_missing: boolean, jpy_missing: boolean}[]> {
|
||||
const [times] = await DB.query(`
|
||||
SELECT UNIX_TIMESTAMP(time) AS time,
|
||||
|
|
@ -289,6 +294,7 @@ class PricesRepository {
|
|||
return times as {time: number, USD: number, eur_missing: boolean, gbp_missing: boolean, cad_missing: boolean, chf_missing: boolean, aud_missing: boolean, jpy_missing: boolean}[];
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $getPricesTimesAndId(): Promise<{time: number, id: number, USD: number}[]> {
|
||||
const [times] = await DB.query(`
|
||||
SELECT
|
||||
|
|
@ -302,6 +308,7 @@ class PricesRepository {
|
|||
return times as {time: number, id: number, USD: number}[];
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $getLatestConversionRates(): Promise<ApiPrice> {
|
||||
const [rates] = await DB.query(`
|
||||
SELECT ${ApiPriceFields}
|
||||
|
|
@ -317,6 +324,7 @@ class PricesRepository {
|
|||
return rates[0] as ApiPrice;
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getNearestHistoricalPrice(timestamp: number | undefined, currency?: string): Promise<Conversion | null> {
|
||||
try {
|
||||
const [rates] = await DB.query(`
|
||||
|
|
@ -341,8 +349,8 @@ class PricesRepository {
|
|||
`);
|
||||
if (!Array.isArray(latestPrices)) {
|
||||
throw Error(`Cannot get single historical price from the database`);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Compute fiat exchange rates
|
||||
let latestPrice = latestPrices[0] as ApiPrice;
|
||||
if (!latestPrice || latestPrice.USD === -1) {
|
||||
|
|
@ -350,8 +358,8 @@ class PricesRepository {
|
|||
}
|
||||
|
||||
const computeFx = (usd: number, other: number): number => usd <= 0.05 ? 0 : Math.round(Math.max(other, 0) / usd * 100) / 100;
|
||||
|
||||
const exchangeRates: ExchangeRates = config.FIAT_PRICE.API_KEY ?
|
||||
|
||||
const exchangeRates: ExchangeRates = config.FIAT_PRICE.API_KEY ?
|
||||
{
|
||||
USDEUR: computeFx(latestPrice.USD, latestPrice.EUR),
|
||||
USDGBP: computeFx(latestPrice.USD, latestPrice.GBP),
|
||||
|
|
@ -428,6 +436,7 @@ class PricesRepository {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
public async $getHistoricalPrices(currency?: string): Promise<Conversion | null> {
|
||||
try {
|
||||
const [rates] = await DB.query(`
|
||||
|
|
@ -446,10 +455,10 @@ class PricesRepository {
|
|||
latestPrice = priceUpdater.getEmptyPricesObj();
|
||||
}
|
||||
|
||||
const computeFx = (usd: number, other: number): number =>
|
||||
const computeFx = (usd: number, other: number): number =>
|
||||
usd <= 0 ? 0 : Math.round(Math.max(other, 0) / usd * 100) / 100;
|
||||
|
||||
const exchangeRates: ExchangeRates = config.FIAT_PRICE.API_KEY ?
|
||||
|
||||
const exchangeRates: ExchangeRates = config.FIAT_PRICE.API_KEY ?
|
||||
{
|
||||
USDEUR: computeFx(latestPrice.USD, latestPrice.EUR),
|
||||
USDGBP: computeFx(latestPrice.USD, latestPrice.GBP),
|
||||
|
|
|
|||
|
|
@ -1,61 +1,61 @@
|
|||
var commands = require('./commands')
|
||||
var rpc = require('./jsonrpc')
|
||||
const commands = require('./commands');
|
||||
const rpc = require('./jsonrpc');
|
||||
|
||||
// ===----------------------------------------------------------------------===//
|
||||
// JsonRPC
|
||||
// ===----------------------------------------------------------------------===//
|
||||
function Client (opts) {
|
||||
// @ts-ignore
|
||||
this.rpc = new rpc.JsonRPC(opts)
|
||||
this.rpc = new rpc.JsonRPC(opts);
|
||||
}
|
||||
|
||||
// ===----------------------------------------------------------------------===//
|
||||
// cmd
|
||||
// ===----------------------------------------------------------------------===//
|
||||
Client.prototype.cmd = function () {
|
||||
var args = [].slice.call(arguments)
|
||||
var cmd = args.shift()
|
||||
const args = [].slice.call(arguments);
|
||||
const cmd = args.shift();
|
||||
|
||||
callRpc(cmd, args, this.rpc)
|
||||
}
|
||||
callRpc(cmd, args, this.rpc);
|
||||
};
|
||||
|
||||
// ===----------------------------------------------------------------------===//
|
||||
// callRpc
|
||||
// ===----------------------------------------------------------------------===//
|
||||
function callRpc (cmd, args, rpc) {
|
||||
var fn = args[args.length - 1]
|
||||
let fn = args[args.length - 1];
|
||||
|
||||
// If the last argument is a callback, pop it from the args list
|
||||
if (typeof fn === 'function') {
|
||||
args.pop()
|
||||
args.pop();
|
||||
} else {
|
||||
fn = function () {}
|
||||
fn = function () {};
|
||||
}
|
||||
|
||||
return rpc.call(cmd, args, function () {
|
||||
var args = [].slice.call(arguments)
|
||||
const args = [].slice.call(arguments);
|
||||
// @ts-ignore
|
||||
args.unshift(null)
|
||||
args.unshift(null);
|
||||
// @ts-ignore
|
||||
fn.apply(this, args)
|
||||
fn.apply(this, args);
|
||||
}, function (err) {
|
||||
fn(err)
|
||||
})
|
||||
fn(err);
|
||||
});
|
||||
}
|
||||
|
||||
// ===----------------------------------------------------------------------===//
|
||||
// Initialize wrappers
|
||||
// ===----------------------------------------------------------------------===//
|
||||
(function () {
|
||||
for (var protoFn in commands) {
|
||||
for (const protoFn in commands) {
|
||||
(function (protoFn) {
|
||||
Client.prototype[protoFn] = function () {
|
||||
var args = [].slice.call(arguments)
|
||||
return callRpc(commands[protoFn], args, this.rpc)
|
||||
}
|
||||
})(protoFn)
|
||||
const args = [].slice.call(arguments);
|
||||
return callRpc(commands[protoFn], args, this.rpc);
|
||||
};
|
||||
})(protoFn);
|
||||
}
|
||||
})()
|
||||
})();
|
||||
|
||||
// Export!
|
||||
module.exports.Client = Client;
|
||||
|
|
|
|||
|
|
@ -1,43 +1,43 @@
|
|||
var http = require('http')
|
||||
var https = require('https')
|
||||
const http = require('http');
|
||||
const https = require('https');
|
||||
import { readFileSync } from 'fs';
|
||||
|
||||
var JsonRPC = function (opts) {
|
||||
const JsonRPC = function (opts) {
|
||||
// @ts-ignore
|
||||
this.opts = opts || {}
|
||||
this.opts = opts || {};
|
||||
// @ts-ignore
|
||||
this.http = this.opts.ssl ? https : http
|
||||
}
|
||||
this.http = this.opts.ssl ? https : http;
|
||||
};
|
||||
|
||||
JsonRPC.prototype.call = function (method, params) {
|
||||
return new Promise((resolve, reject) => {
|
||||
var time = Date.now()
|
||||
var requestJSON
|
||||
const time = Date.now();
|
||||
let requestJSON;
|
||||
|
||||
if (Array.isArray(method)) {
|
||||
// multiple rpc batch call
|
||||
requestJSON = []
|
||||
requestJSON = [];
|
||||
method.forEach(function (batchCall, i) {
|
||||
requestJSON.push({
|
||||
id: time + '-' + i,
|
||||
method: batchCall.method,
|
||||
params: batchCall.params
|
||||
})
|
||||
})
|
||||
});
|
||||
});
|
||||
} else {
|
||||
// single rpc call
|
||||
requestJSON = {
|
||||
id: time,
|
||||
method: method,
|
||||
params: params
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// First we encode the request into JSON
|
||||
requestJSON = JSON.stringify(requestJSON)
|
||||
requestJSON = JSON.stringify(requestJSON);
|
||||
|
||||
// prepare request options
|
||||
var requestOptions = {
|
||||
const requestOptions = {
|
||||
host: this.opts.host || 'localhost',
|
||||
port: this.opts.port || 8332,
|
||||
method: 'POST',
|
||||
|
|
@ -48,11 +48,11 @@ JsonRPC.prototype.call = function (method, params) {
|
|||
},
|
||||
agent: false,
|
||||
rejectUnauthorized: this.opts.ssl && this.opts.sslStrict !== false
|
||||
}
|
||||
};
|
||||
|
||||
if (this.opts.ssl && this.opts.sslCa) {
|
||||
// @ts-ignore
|
||||
requestOptions.ca = this.opts.sslCa
|
||||
// @ts-ignore
|
||||
requestOptions.ca = this.opts.sslCa;
|
||||
}
|
||||
|
||||
// use HTTP auth if user and password set
|
||||
|
|
@ -64,61 +64,61 @@ JsonRPC.prototype.call = function (method, params) {
|
|||
requestOptions.auth = this.cachedCookie;
|
||||
} else if (this.opts.user && this.opts.pass) {
|
||||
// @ts-ignore
|
||||
requestOptions.auth = this.opts.user + ':' + this.opts.pass
|
||||
requestOptions.auth = this.opts.user + ':' + this.opts.pass;
|
||||
}
|
||||
|
||||
// Now we'll make a request to the server
|
||||
var cbCalled = false
|
||||
var request = this.http.request(requestOptions)
|
||||
let cbCalled = false;
|
||||
const request = this.http.request(requestOptions);
|
||||
|
||||
// start request timeout timer
|
||||
var reqTimeout = setTimeout(function () {
|
||||
if (cbCalled) return
|
||||
cbCalled = true
|
||||
request.abort()
|
||||
var err = new Error('ETIMEDOUT')
|
||||
const reqTimeout = setTimeout(function () {
|
||||
if (cbCalled) {return;}
|
||||
cbCalled = true;
|
||||
request.abort();
|
||||
const err = new Error('ETIMEDOUT');
|
||||
// @ts-ignore
|
||||
err.code = 'ETIMEDOUT'
|
||||
reject(err)
|
||||
}, this.opts.timeout || 30000)
|
||||
err.code = 'ETIMEDOUT';
|
||||
reject(err);
|
||||
}, this.opts.timeout || 30000);
|
||||
|
||||
// set additional timeout on socket in case of remote freeze after sending headers
|
||||
request.setTimeout(this.opts.timeout || 30000, function () {
|
||||
if (cbCalled) return
|
||||
cbCalled = true
|
||||
request.abort()
|
||||
var err = new Error('ESOCKETTIMEDOUT')
|
||||
if (cbCalled) {return;}
|
||||
cbCalled = true;
|
||||
request.abort();
|
||||
const err = new Error('ESOCKETTIMEDOUT');
|
||||
// @ts-ignore
|
||||
err.code = 'ESOCKETTIMEDOUT'
|
||||
reject(err)
|
||||
})
|
||||
err.code = 'ESOCKETTIMEDOUT';
|
||||
reject(err);
|
||||
});
|
||||
|
||||
request.on('error', function (err) {
|
||||
if (cbCalled) return
|
||||
cbCalled = true
|
||||
clearTimeout(reqTimeout)
|
||||
reject(err)
|
||||
})
|
||||
if (cbCalled) {return;}
|
||||
cbCalled = true;
|
||||
clearTimeout(reqTimeout);
|
||||
reject(err);
|
||||
});
|
||||
|
||||
request.on('response', (response) => {
|
||||
clearTimeout(reqTimeout)
|
||||
clearTimeout(reqTimeout);
|
||||
|
||||
// We need to buffer the response chunks in a nonblocking way.
|
||||
var buffer = ''
|
||||
let buffer = '';
|
||||
response.on('data', function (chunk) {
|
||||
buffer = buffer + chunk
|
||||
})
|
||||
buffer = buffer + chunk;
|
||||
});
|
||||
// When all the responses are finished, we decode the JSON and
|
||||
// depending on whether it's got a result or an error, we call
|
||||
// emitSuccess or emitError on the promise.
|
||||
response.on('end', () => {
|
||||
var err
|
||||
let err;
|
||||
|
||||
if (cbCalled) return
|
||||
cbCalled = true
|
||||
if (cbCalled) {return;}
|
||||
cbCalled = true;
|
||||
|
||||
try {
|
||||
var decoded = JSON.parse(buffer)
|
||||
var decoded = JSON.parse(buffer);
|
||||
} catch (e) {
|
||||
// if we authenticated using a cookie and it failed, read the cookie file again
|
||||
if (
|
||||
|
|
@ -129,19 +129,19 @@ JsonRPC.prototype.call = function (method, params) {
|
|||
}
|
||||
|
||||
if (response.statusCode !== 200) {
|
||||
err = new Error('Invalid params, response status code: ' + response.statusCode)
|
||||
err.code = -32602
|
||||
reject(err)
|
||||
err = new Error('Invalid params, response status code: ' + response.statusCode);
|
||||
err.code = -32602;
|
||||
reject(err);
|
||||
} else {
|
||||
err = new Error('Problem parsing JSON response from server')
|
||||
err.code = -32603
|
||||
reject(err)
|
||||
err = new Error('Problem parsing JSON response from server');
|
||||
err.code = -32603;
|
||||
reject(err);
|
||||
}
|
||||
return
|
||||
return;
|
||||
}
|
||||
|
||||
if (!Array.isArray(decoded)) {
|
||||
decoded = [decoded]
|
||||
decoded = [decoded];
|
||||
}
|
||||
|
||||
// iterate over each response, normally there will be just one
|
||||
|
|
@ -149,29 +149,29 @@ JsonRPC.prototype.call = function (method, params) {
|
|||
decoded.forEach(function (decodedResponse, i) {
|
||||
if (decodedResponse.hasOwnProperty('error') && decodedResponse.error != null) {
|
||||
if (reject) {
|
||||
err = new Error(decodedResponse.error.message || '')
|
||||
err = new Error(decodedResponse.error.message || '');
|
||||
if (decodedResponse.error.code) {
|
||||
err.code = decodedResponse.error.code
|
||||
err.code = decodedResponse.error.code;
|
||||
}
|
||||
reject(err)
|
||||
reject(err);
|
||||
}
|
||||
} else if (decodedResponse.hasOwnProperty('result')) {
|
||||
// @ts-ignore
|
||||
resolve(decodedResponse.result, response.headers)
|
||||
resolve(decodedResponse.result, response.headers);
|
||||
} else {
|
||||
if (reject) {
|
||||
err = new Error(decodedResponse.error.message || '')
|
||||
err = new Error(decodedResponse.error.message || '');
|
||||
if (decodedResponse.error.code) {
|
||||
err.code = decodedResponse.error.code
|
||||
err.code = decodedResponse.error.code;
|
||||
}
|
||||
reject(err)
|
||||
reject(err);
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
});
|
||||
});
|
||||
});
|
||||
request.end(requestJSON);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
module.exports.JsonRPC = JsonRPC
|
||||
module.exports.JsonRPC = JsonRPC;
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ const PATH = './';
|
|||
class SyncAssets {
|
||||
constructor() { }
|
||||
|
||||
/** @asyncSafe */
|
||||
public async syncAssets$() {
|
||||
for (const url of config.MEMPOOL.EXTERNAL_ASSETS) {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ class ForensicsService {
|
|||
await this.$runTasks();
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
private async $runTasks(): Promise<void> {
|
||||
try {
|
||||
logger.debug(`Running forensics scans`);
|
||||
|
|
@ -36,7 +37,7 @@ class ForensicsService {
|
|||
logger.err('ForensicsService.$runTasks() error: ' + (e instanceof Error ? e.message : e));
|
||||
}
|
||||
|
||||
setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.FORENSICS_INTERVAL);
|
||||
setTimeout(() => { void this.$runTasks(); }, 1000 * config.LIGHTNING.FORENSICS_INTERVAL);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -340,6 +341,7 @@ class ForensicsService {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
private async $attributeChannelBalances(
|
||||
prevChannel, openChannel, input: IEsploraApi.Vin, openContribution: number | null = null,
|
||||
initiator: 'remote' | 'local' | null = null, linkedOpenings: boolean = false
|
||||
|
|
@ -449,7 +451,7 @@ class ForensicsService {
|
|||
const initiatorSide = initiator === 'remote' ? prevRemote : prevLocal;
|
||||
prevChannel.closed_by = prevChannel[`node${initiatorSide}_public_key`];
|
||||
}
|
||||
|
||||
|
||||
// save changes to the closing channel
|
||||
await channelsApi.$updateClosingInfo(prevChannel);
|
||||
} else {
|
||||
|
|
@ -465,6 +467,7 @@ class ForensicsService {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
async fetchTransaction(txid: string, temp: boolean = false): Promise<IEsploraApi.Transaction | null> {
|
||||
let tx = this.txCache[txid];
|
||||
if (!tx) {
|
||||
|
|
@ -485,6 +488,7 @@ class ForensicsService {
|
|||
|
||||
// fetches a batch of transactions and adds them to the txCache
|
||||
// the returned list of txs does *not* preserve ordering or number
|
||||
/** @asyncSafe */
|
||||
async fetchTransactions(txids, temp: boolean = false): Promise<(IEsploraApi.Transaction | null)[]> {
|
||||
// deduplicate txids
|
||||
const uniqueTxids = [...new Set<string>(txids)];
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ class NetworkSyncService {
|
|||
await this.$runTasks();
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
private async $runTasks(): Promise<void> {
|
||||
const taskStartTime = Date.now();
|
||||
try {
|
||||
|
|
@ -37,7 +38,7 @@ class NetworkSyncService {
|
|||
const networkGraph = await lightningApi.$getNetworkGraph();
|
||||
if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) {
|
||||
logger.info(`LN Network graph is empty, retrying in 10 seconds`, logger.tags.ln);
|
||||
setTimeout(() => { this.$runTasks(); }, 10000);
|
||||
setTimeout(() => { void this.$runTasks(); }, 10000);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -47,7 +48,7 @@ class NetworkSyncService {
|
|||
await this.$lookUpCreationDateFromChain();
|
||||
await this.$updateNodeFirstSeen();
|
||||
await this.$scanForClosedChannels();
|
||||
|
||||
|
||||
if (config.MEMPOOL.BACKEND === 'esplora') {
|
||||
// run forensics on new channels only
|
||||
await forensicsService.$runClosedChannelsForensics(true);
|
||||
|
|
@ -57,7 +58,7 @@ class NetworkSyncService {
|
|||
logger.err(`$runTasks() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
|
||||
}
|
||||
|
||||
setTimeout(() => { this.$runTasks(); }, Math.max(1, (1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL) - (Date.now() - taskStartTime)));
|
||||
setTimeout(() => { void this.$runTasks(); }, Math.max(1, (1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL) - (Date.now() - taskStartTime)));
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -111,7 +112,9 @@ class NetworkSyncService {
|
|||
await nodesApi.$setNodesInactive(graphNodesPubkeys);
|
||||
|
||||
if (config.MAXMIND.ENABLED) {
|
||||
$lookupNodeLocation();
|
||||
$lookupNodeLocation().catch((e) => {
|
||||
logger.err(`Error in $lookupNodeLocation: ${e instanceof Error ? e.message : e}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -226,7 +229,7 @@ class NetworkSyncService {
|
|||
|
||||
if (channels.length > 0) {
|
||||
logger.debug(`Updated ${channels.length} channels' creation date`, logger.tags.ln);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
logger.err(`$lookUpCreationDateFromChain() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,17 +9,19 @@ class LightningStatsUpdater {
|
|||
logger.info(`Starting Lightning Stats service`, logger.tags.ln);
|
||||
|
||||
await this.$runTasks();
|
||||
LightningStatsImporter.$run();
|
||||
void LightningStatsImporter.$run();
|
||||
}
|
||||
|
||||
/** @asyncSafe */
|
||||
private async $runTasks(): Promise<void> {
|
||||
await this.$logStatsDaily();
|
||||
|
||||
setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL);
|
||||
setTimeout(() => { void this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the latest entry for each node every config.LIGHTNING.STATS_REFRESH_INTERVAL seconds
|
||||
* @asyncSafe
|
||||
*/
|
||||
private async $logStatsDaily(): Promise<void> {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import logger from '../../../logger';
|
|||
|
||||
const fsPromises = promises;
|
||||
|
||||
const BLOCKS_CACHE_MAX_SIZE = 100;
|
||||
const BLOCKS_CACHE_MAX_SIZE = 100;
|
||||
const CACHE_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/ln-funding-txs-cache.json';
|
||||
|
||||
class FundingTxFetcher {
|
||||
|
|
@ -28,12 +28,13 @@ class FundingTxFetcher {
|
|||
}
|
||||
}
|
||||
|
||||
/** @asyncUnsafe */
|
||||
async $fetchChannelsFundingTxs(channelIds: string[]): Promise<void> {
|
||||
if (this.running) {
|
||||
return;
|
||||
}
|
||||
this.running = true;
|
||||
|
||||
|
||||
const globalTimer = new Date().getTime() / 1000;
|
||||
let cacheTimer = new Date().getTime() / 1000;
|
||||
let loggerTimer = new Date().getTime() / 1000;
|
||||
|
|
@ -57,7 +58,9 @@ class FundingTxFetcher {
|
|||
elapsedSeconds = Math.round((new Date().getTime() / 1000) - cacheTimer);
|
||||
if (elapsedSeconds > 60) {
|
||||
logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`, logger.tags.ln);
|
||||
fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
|
||||
fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache)).catch((e) => {
|
||||
logger.err(`Error saving funding txs cache to disk: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
|
||||
});
|
||||
cacheTimer = new Date().getTime() / 1000;
|
||||
}
|
||||
}
|
||||
|
|
@ -65,20 +68,27 @@ class FundingTxFetcher {
|
|||
if (this.channelNewlyProcessed > 0) {
|
||||
logger.info(`Indexed ${this.channelNewlyProcessed} additional channels funding tx`, logger.tags.ln);
|
||||
logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`, logger.tags.ln);
|
||||
fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
|
||||
fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache)).catch((e) => {
|
||||
logger.err(`Error saving funding txs cache to disk: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
|
||||
});
|
||||
}
|
||||
|
||||
this.running = false;
|
||||
}
|
||||
|
||||
|
||||
/** @asyncUnsafe */
|
||||
public async $fetchChannelOpenTx(channelId: string): Promise<{timestamp: number, txid: string, value: number} | null> {
|
||||
channelId = Common.channelIntegerIdToShortId(channelId);
|
||||
|
||||
if (!channelId?.length) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (this.fundingTxCache[channelId]) {
|
||||
return this.fundingTxCache[channelId];
|
||||
}
|
||||
|
||||
const parts = channelId.split('x');
|
||||
const parts = channelId?.split('x') ?? [];
|
||||
if (parts.length < 3) {
|
||||
logger.debug(`Channel ID ${channelId} does not seem valid, should contains at least 3 parts separated by 'x'`, logger.tags.ln);
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import { ResultSetHeader } from 'mysql2';
|
|||
import * as IPCheck from '../../../utils/ipcheck.js';
|
||||
import { Reader } from 'mmdb-lib';
|
||||
|
||||
/** @asyncSafe */
|
||||
export async function $lookupNodeLocation(): Promise<void> {
|
||||
let loggerTimer = new Date().getTime() / 1000;
|
||||
let progress = 0;
|
||||
|
|
@ -25,7 +26,7 @@ export async function $lookupNodeLocation(): Promise<void> {
|
|||
} catch (e) { }
|
||||
|
||||
for (const node of nodes) {
|
||||
const sockets: string[] = node.sockets.split(',');
|
||||
const sockets: string[] = node.sockets?.split(',') ?? [];
|
||||
for (const socket of sockets) {
|
||||
const ip = socket.substring(0, socket.lastIndexOf(':')).replace('[', '').replace(']', '');
|
||||
const hasClearnet = [4, 6].includes(net.isIP(ip));
|
||||
|
|
@ -60,13 +61,13 @@ export async function $lookupNodeLocation(): Promise<void> {
|
|||
|
||||
if (city && (asn || isp)) {
|
||||
const query = `
|
||||
UPDATE nodes SET
|
||||
as_number = ?,
|
||||
city_id = ?,
|
||||
country_id = ?,
|
||||
subdivision_id = ?,
|
||||
longitude = ?,
|
||||
latitude = ?,
|
||||
UPDATE nodes SET
|
||||
as_number = ?,
|
||||
city_id = ?,
|
||||
country_id = ?,
|
||||
subdivision_id = ?,
|
||||
longitude = ?,
|
||||
latitude = ?,
|
||||
accuracy_radius = ?
|
||||
WHERE public_key = ?
|
||||
`;
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ const fsPromises = promises;
|
|||
class LightningStatsImporter {
|
||||
topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER;
|
||||
|
||||
/** @asyncSafe */
|
||||
async $run(): Promise<void> {
|
||||
try {
|
||||
const [channels]: any[] = await DB.query('SELECT short_id from channels;');
|
||||
|
|
@ -33,6 +34,7 @@ class LightningStatsImporter {
|
|||
|
||||
/**
|
||||
* Generate LN network stats for one day
|
||||
* @asyncUnsafe
|
||||
*/
|
||||
public async computeNetworkStats(timestamp: number,
|
||||
networkGraph: ILightningApi.NetworkGraph, isHistorical: boolean = false): Promise<unknown> {
|
||||
|
|
@ -99,7 +101,7 @@ class LightningStatsImporter {
|
|||
const feeRates: number[] = [];
|
||||
const baseFees: number[] = [];
|
||||
const alreadyCountedChannels = {};
|
||||
|
||||
|
||||
const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id FROM channels`);
|
||||
const channelsInDb = {};
|
||||
for (const channel of channelsInDbRaw) {
|
||||
|
|
@ -108,6 +110,9 @@ class LightningStatsImporter {
|
|||
|
||||
for (const channel of networkGraph.edges) {
|
||||
const short_id = Common.channelIntegerIdToShortId(channel.channel_id);
|
||||
if (!short_id?.length) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const tx = await fundingTxFetcher.$fetchChannelOpenTx(short_id);
|
||||
if (!tx) {
|
||||
|
|
@ -142,7 +147,7 @@ class LightningStatsImporter {
|
|||
channels: 0,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
if (!alreadyCountedChannels[short_id]) {
|
||||
capacity += Math.round(tx.value * 100000000);
|
||||
capacities.push(Math.round(tx.value * 100000000));
|
||||
|
|
@ -159,7 +164,7 @@ class LightningStatsImporter {
|
|||
if (policy && parseInt(policy.fee_rate_milli_msat, 10) < 5000) {
|
||||
avgFeeRate += parseInt(policy.fee_rate_milli_msat, 10);
|
||||
feeRates.push(parseInt(policy.fee_rate_milli_msat, 10));
|
||||
}
|
||||
}
|
||||
if (policy && parseInt(policy.fee_base_msat, 10) < 5000) {
|
||||
avgBaseFee += parseInt(policy.fee_base_msat, 10);
|
||||
baseFees.push(parseInt(policy.fee_base_msat, 10));
|
||||
|
|
@ -385,7 +390,7 @@ class LightningStatsImporter {
|
|||
totalProcessed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
if (this.isIncorrectSnapshot(timestamp, graph)) {
|
||||
logger.debug(`Ignoring ${this.topologiesFolder}/${filename}, because we defined it as an incorrect snapshot`);
|
||||
++totalProcessed;
|
||||
|
|
@ -396,7 +401,7 @@ class LightningStatsImporter {
|
|||
logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`, logger.tags.ln);
|
||||
logStarted = true;
|
||||
}
|
||||
|
||||
|
||||
const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`;
|
||||
logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`, logger.tags.ln);
|
||||
|
||||
|
|
@ -472,7 +477,7 @@ class LightningStatsImporter {
|
|||
fee_rate_milli_msat: edge.fee_proportional_millionths,
|
||||
max_htlc_msat: edge.htlc_maximum_msat,
|
||||
last_update: edge.timestamp,
|
||||
disabled: false,
|
||||
disabled: false,
|
||||
},
|
||||
node2_policy: null,
|
||||
});
|
||||
|
|
@ -542,7 +547,7 @@ class LightningStatsImporter {
|
|||
// UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 OR
|
||||
// UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 OR
|
||||
// UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 OR
|
||||
// UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000
|
||||
// UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000
|
||||
// )
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue