From 404772372f4f68cd39583c47af5f2e64f23178a3 Mon Sep 17 00:00:00 2001 From: Jonathan Zernik Date: Sat, 18 Jul 2020 17:19:43 -0700 Subject: [PATCH] Add btcd credentials to sqkserver config (#81) * Add btcd credentials and rpc cert to sqkserver configs * Start the squeak block verifier worker thread on startup * Btcd request still failing because of cert * Got btcd client working * Use valid block hash and height in itest --- docker/btcd/Dockerfile | 4 +- docker/sqkserver/Dockerfile | 4 +- docker/sqkserver/start-sqkserver.sh | 18 +++++++ init.sql | 3 +- itests/config.ini | 10 ++-- itests/docker-compose.yml | 3 ++ itests/test.py | 16 ++++-- requirements.txt | 1 + squeakserver/blockchain/__init__.py | 0 .../blockchain/bitcoin_blockchain_client.py | 54 +++++++++++++++++++ .../node/squeak_block_periodic_worker.py | 26 +++++++++ squeakserver/node/squeak_block_verifier.py | 30 +++++++++++ squeakserver/server/main.py | 26 +++++++++ squeakserver/server/postgres_db.py | 15 ++++++ 14 files changed, 197 insertions(+), 13 deletions(-) create mode 100644 docker/sqkserver/start-sqkserver.sh create mode 100644 squeakserver/blockchain/__init__.py create mode 100644 squeakserver/blockchain/bitcoin_blockchain_client.py create mode 100644 squeakserver/node/squeak_block_periodic_worker.py create mode 100644 squeakserver/node/squeak_block_verifier.py diff --git a/docker/btcd/Dockerfile b/docker/btcd/Dockerfile index 13998e48..adc78005 100644 --- a/docker/btcd/Dockerfile +++ b/docker/btcd/Dockerfile @@ -36,6 +36,8 @@ COPY --from=builder /go/bin/gencerts /bin/ COPY "docker/btcd/start-btcctl.sh" . COPY "docker/btcd/start-btcd.sh" . +RUN "/bin/gencerts" -h + RUN apk add --no-cache \ bash \ ca-certificates \ @@ -45,7 +47,7 @@ RUN apk add --no-cache \ && chmod +x start-btcd.sh \ # Manually generate certificate and add all domains, it is needed to connect # "btcctl" and "lnd" to "btcd" over docker links. -&& "/bin/gencerts" --host="*" --directory="/rpc" --force +&& "/bin/gencerts" --host="btcd" --host="rpcserver" --host="blockchain" --directory="/rpc" --force # Create a volume to house pregenerated RPC credentials. This will be # shared with any lnd, btcctl containers so they can securely query btcd's RPC diff --git a/docker/sqkserver/Dockerfile b/docker/sqkserver/Dockerfile index 67113fa8..cabf7977 100644 --- a/docker/sqkserver/Dockerfile +++ b/docker/sqkserver/Dockerfile @@ -34,4 +34,6 @@ RUN python3 setup.py install EXPOSE 8774 -CMD ["runsqueakserver", "--config", "config.ini", "--log-level", "DEBUG", "run-server"] +# Copy the entrypoint script. +COPY "docker/sqkserver/start-sqkserver.sh" . +RUN chmod +x start-sqkserver.sh diff --git a/docker/sqkserver/start-sqkserver.sh b/docker/sqkserver/start-sqkserver.sh new file mode 100644 index 00000000..dfa3fc23 --- /dev/null +++ b/docker/sqkserver/start-sqkserver.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# exit from script if error was raised. +set -e + +# Add btcd's RPC TLS certificate to system Certificate Authority list. exec runsqueak \ +cp /rpc/rpc.cert /usr/share/ca-certificates/btcd.crt +echo btcd.crt >> /etc/ca-certificates.conf +update-ca-certificates + +# To make python requests use the system ca-certificates bundle, it +# needs to be told to use it over its own embedded bundle +export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt + +exec runsqueakserver \ + --config config.ini \ + --log-level DEBUG \ + run-server diff --git a/init.sql b/init.sql index c4831f8d..c072ab55 100644 --- a/init.sql +++ b/init.sql @@ -15,7 +15,8 @@ CREATE TABLE IF NOT EXISTS squeak ( encContent CHAR(2272) NOT NULL, -- Encrypted content length is always 1136 bytes (2272 hex characters). vchScriptSig bytea NOT NULL, address VARCHAR(35) NOT NULL, -- Maximum length of a bitcoin address is 35. - vchDecryptionKey bytea NOT NULL + vchDecryptionKey bytea NOT NULL, + block_header bytea ); CREATE INDEX IF NOT EXISTS idx_squeak_address diff --git a/itests/config.ini b/itests/config.ini index b634e375..311bf3d6 100644 --- a/itests/config.ini +++ b/itests/config.ini @@ -9,11 +9,11 @@ rpc_port=10009 tls_cert_path=/root/.lnd/tls.cert macaroon_path=/root/.lnd/data/chain/bitcoin/simnet/admin.macaroon -[electrum] -rpc_host= -rpc_port= -rpc_user= -rpc_pass= +[bitcoin] +rpc_host=btcd +rpc_port=18556 +rpc_user=devuser +rpc_pass=devpass [server] rpc_host=0.0.0.0 diff --git a/itests/docker-compose.yml b/itests/docker-compose.yml index a1d29649..2b85a29c 100644 --- a/itests/docker-compose.yml +++ b/itests/docker-compose.yml @@ -97,14 +97,17 @@ services: context: ../ dockerfile: docker/sqkserver/Dockerfile volumes: + - test_shared:/rpc - test_lnd_server_dir:/root/.lnd - ./config.ini:/app/config.ini links: + - "btcd:btcd" - "lnd_server:lnd" depends_on: - db sysctls: - net.ipv6.conf.all.disable_ipv6=0 + entrypoint: ["./start-sqkserver.sh"] test: image: test diff --git a/itests/test.py b/itests/test.py index 29da91d2..d57cbc3e 100644 --- a/itests/test.py +++ b/itests/test.py @@ -59,9 +59,14 @@ def get_address(signing_key): return str(address) -def make_squeak(signing_key: CSigningKey, content: str, reply_to: bytes = b'\x00'*HASH_LENGTH): - block_height = 0 - block_hash = lx('4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b') +def get_latest_block_info(lightning_client): + get_info_response = lightning_client.get_info() + block_hash = bytes.fromhex(get_info_response.block_hash) + block_height = get_info_response.block_height + return block_hash, block_height + + +def make_squeak(signing_key: CSigningKey, content: str, block_height, block_hash, reply_to: bytes = b'\x00'*HASH_LENGTH): timestamp = int(time.time()) return MakeSqueakFromStr( signing_key, @@ -123,7 +128,8 @@ def run(): # Post a squeak with a direct request to the server signing_key = generate_signing_key() - squeak = make_squeak(signing_key, 'hello from itest!') + block_height, block_hash = get_latest_block_info(lnd_lightning_client) + squeak = make_squeak(signing_key, 'hello from itest!', block_hash, block_height) squeak_hash = get_hash(squeak) squeak_msg = build_squeak_msg(squeak) @@ -208,7 +214,7 @@ def run(): # Open channel to the server lightning node pubkey_bytes = string_to_hex(buy_response.offer.pubkey) open_channel_response = lnd_lightning_client.open_channel(pubkey_bytes, 1000000) - print("Server open channel response: " + str(open_channel_response)) + print("Opening channel...") for update in open_channel_response: if update.HasField('chan_open'): channel_point = update.chan_open.channel_point diff --git a/requirements.txt b/requirements.txt index bbdffb61..b13d3f35 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ googleapis-common-protos grpcio grpcio-tools psycopg2 +requests diff --git a/squeakserver/blockchain/__init__.py b/squeakserver/blockchain/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/squeakserver/blockchain/bitcoin_blockchain_client.py b/squeakserver/blockchain/bitcoin_blockchain_client.py new file mode 100644 index 00000000..d77c490d --- /dev/null +++ b/squeakserver/blockchain/bitcoin_blockchain_client.py @@ -0,0 +1,54 @@ +import json +from typing import Optional + +import requests + + +class BitcoinBlockchainClient: + """Access a bitcoin daemon using RPC.""" + + def __init__( + self, + host: str, + port: int, + rpc_user: str, + rpc_password: str, + ) -> None: + self.url = f'https://{rpc_user}:{rpc_password}@{host}:{port}' + self.headers = {'content-type': 'application/json'} + + def get_block_hash(self, block_height: int) -> Optional[bytes]: + # return self.access.getblockhash(block_height) + payload = { + "method": "getblockhash", + "params": [block_height], + "jsonrpc": "2.0", + "id": 0, + } + response = requests.post( + self.url, + data=json.dumps(payload), + headers=self.headers, + ).json() + + result = response["result"] + block_hash = bytes.fromhex(result) + return block_hash + + # def get_block_hash(self, block_height: int) -> Optional[bytes]: + # # return self.access.getblockhash(block_height) + # payload = { + # "method": "getblockhash", + # "params": [block_height], + # "jsonrpc": "2.0", + # "id": 0, + # } + # response = requests.post( + # self.url, + # data=json.dumps(payload), + # headers=self.headers, + # ).json() + + # result = response["result"] + # block_hash = bytes.fromhex(result) + # return block_hash diff --git a/squeakserver/node/squeak_block_periodic_worker.py b/squeakserver/node/squeak_block_periodic_worker.py new file mode 100644 index 00000000..062392a0 --- /dev/null +++ b/squeakserver/node/squeak_block_periodic_worker.py @@ -0,0 +1,26 @@ +import logging +import threading +import time + + +logger = logging.getLogger(__name__) + + +VERIFY_UPDATE_INTERVAL_S = 10.0 + + +class SqueakBlockPeriodicWorker: + + def __init__(self, squeak_block_verifier, postgres_db): + self.squeak_block_verifier = squeak_block_verifier + self.postgres_db = postgres_db + + def verify_squeaks(self): + logger.info('Calling verify_squeaks.') + squeaks_to_verify = self.postgres_db.get_unverified_block_squeaks() + for squeak_hash in squeaks_to_verify: + self.squeak_block_verifier.verify_squeak_block(squeak_hash) + + def start_running(self): + threading.Timer(VERIFY_UPDATE_INTERVAL_S, self.start_running).start() + self.verify_squeaks() diff --git a/squeakserver/node/squeak_block_verifier.py b/squeakserver/node/squeak_block_verifier.py new file mode 100644 index 00000000..aab2a9ee --- /dev/null +++ b/squeakserver/node/squeak_block_verifier.py @@ -0,0 +1,30 @@ +import logging + + +logger = logging.getLogger(__name__) + + +class SqueakBlockVerifier: + + def __init__(self, postgres_db, blockchain_client): + self.postgres_db = postgres_db + self.blockchain_client = blockchain_client + + def verify_squeak_block(self, squeak_hash): + logger.info('Verifying squeak hash: {}'.format(squeak_hash)) + squeak = self._get_squeak(squeak_hash) + block_info = self._get_block_info(squeak) + + def _get_squeak(self, squeak_hash): + return self.postgres_db.get_squeak(squeak_hash) + + def _mark_squeak_verified(self, squeak_hash): + # TODO + pass + + def _get_block_info(self, squeak): + block_height = squeak.nBlockHeight + block_hash = self.blockchain_client.get_block_hash(block_height) + logger.info('Got block hash from blockchain: {}'.format(block_hash)) + logger.info('Got block hash from squeak: {}'.format(squeak.hashBlock)) + logger.info('Is block hash correct: {}'.format(squeak.hashBlock == block_hash)) diff --git a/squeakserver/server/main.py b/squeakserver/server/main.py index ec61051f..6cab7eec 100644 --- a/squeakserver/server/main.py +++ b/squeakserver/server/main.py @@ -18,6 +18,9 @@ from squeakserver.server.squeak_server_servicer import SqueakServerServicer from squeakserver.server.squeak_server_handler import SqueakServerHandler from squeakserver.server.db_params import parse_db_params from squeakserver.server.postgres_db import PostgresDb +from squeakserver.node.squeak_block_verifier import SqueakBlockVerifier +from squeakserver.node.squeak_block_periodic_worker import SqueakBlockPeriodicWorker +from squeakserver.blockchain.bitcoin_blockchain_client import BitcoinBlockchainClient logger = logging.getLogger(__name__) @@ -94,6 +97,23 @@ def load_postgres_db(config): return PostgresDb(db_params) +def load_blockchain_client(config): + return BitcoinBlockchainClient( + config['bitcoin']['rpc_host'], + config['bitcoin']['rpc_port'], + config['bitcoin']['rpc_user'], + config['bitcoin']['rpc_pass'], + ) + + +def load_squeak_block_verifier(postgres_db, blockchain_client): + return SqueakBlockVerifier(postgres_db, blockchain_client) + + +def load_squeak_block_periodic_worker(squeak_block_verifier, postgres_db): + return SqueakBlockPeriodicWorker(squeak_block_verifier, postgres_db) + + def sigterm_handler(_signo, _stack_frame): # Raises SystemExit(0): sys.exit(0) @@ -183,6 +203,12 @@ def run_server(config): lightning_client = load_lightning_client(config) lightning_host_port = load_lightning_host_port(config) + # Start the squeak block verifier + bitcoin_blockchain_client = load_blockchain_client(config) + squeak_block_verifier = load_squeak_block_verifier(postgres_db, bitcoin_blockchain_client) + squeak_block_periodic_worker = load_squeak_block_periodic_worker(squeak_block_verifier, postgres_db) + squeak_block_periodic_worker.start_running() + # start admin rpc server admin_handler = load_admin_handler(lightning_client, postgres_db) admin_rpc_server = load_admin_rpc_server(config, admin_handler) diff --git a/squeakserver/server/postgres_db.py b/squeakserver/server/postgres_db.py index a6ae30ea..68a753b4 100644 --- a/squeakserver/server/postgres_db.py +++ b/squeakserver/server/postgres_db.py @@ -170,3 +170,18 @@ class PostgresDb(): following=row[6], ) return squeak_profile + + def get_unverified_block_squeaks(self): + """ Lookup squeaks. """ + sql = """ + SELECT hash FROM squeak + WHERE block_header IS NULL; + """ + with self.get_cursor() as curs: + curs.execute(sql) + rows = curs.fetchall() + hashes = [ + bytes.fromhex(row[0]) + for row in rows + ] + return hashes