Add btcd credentials to sqkserver config (#81)

* Add btcd credentials and rpc cert to sqkserver configs

* Start the squeak block verifier worker thread on startup

* Btcd request still failing because of cert

* Got btcd client working

* Use valid block hash and height in itest

docker/btcd/Dockerfile

@@ -36,6 +36,8 @@ COPY --from=builder /go/bin/gencerts /bin/
 COPY "docker/btcd/start-btcctl.sh" .
 COPY "docker/btcd/start-btcd.sh" .
 
+RUN "/bin/gencerts" -h
+
 RUN apk add --no-cache \
     bash \
     ca-certificates \
@@ -45,7 +47,7 @@ RUN apk add --no-cache \
 &&  chmod +x start-btcd.sh \
 # Manually generate certificate and add all domains, it is needed to connect
 # "btcctl" and "lnd" to "btcd" over docker links.
-&& "/bin/gencerts" --host="*" --directory="/rpc" --force
+&& "/bin/gencerts" --host="btcd" --host="rpcserver" --host="blockchain" --directory="/rpc" --force
 
 # Create a volume to house pregenerated RPC credentials. This will be
 # shared with any lnd, btcctl containers so they can securely query btcd's RPC

docker/sqkserver/Dockerfile

@@ -34,4 +34,6 @@ RUN python3 setup.py install
 
 EXPOSE 8774
 
-CMD ["runsqueakserver", "--config", "config.ini", "--log-level", "DEBUG", "run-server"]
+# Copy the entrypoint script.
+COPY "docker/sqkserver/start-sqkserver.sh" .
+RUN chmod +x start-sqkserver.sh

docker/sqkserver/start-sqkserver.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# exit from script if error was raised.
+set -e
+
+# Add btcd's RPC TLS certificate to system Certificate Authority list.	exec runsqueak \
+cp /rpc/rpc.cert /usr/share/ca-certificates/btcd.crt
+echo btcd.crt >> /etc/ca-certificates.conf
+update-ca-certificates
+
+# To make python requests use the system ca-certificates bundle, it
+# needs to be told to use it over its own embedded bundle
+export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+
+exec runsqueakserver \
+     --config config.ini \
+     --log-level DEBUG \
+     run-server

init.sql

@@ -15,7 +15,8 @@ CREATE TABLE IF NOT EXISTS squeak (
   encContent CHAR(2272) NOT NULL, -- Encrypted content length is always 1136 bytes (2272 hex characters).
   vchScriptSig bytea NOT NULL,
   address VARCHAR(35) NOT NULL, -- Maximum length of a bitcoin address is 35.
-  vchDecryptionKey bytea NOT NULL
+  vchDecryptionKey bytea NOT NULL,
+  block_header bytea
 );
 
 CREATE INDEX IF NOT EXISTS idx_squeak_address

itests/config.ini

@@ -9,11 +9,11 @@ rpc_port=10009
 tls_cert_path=/root/.lnd/tls.cert
 macaroon_path=/root/.lnd/data/chain/bitcoin/simnet/admin.macaroon
 
-[electrum]
-rpc_host=
-rpc_port=
-rpc_user=
-rpc_pass=
+[bitcoin]
+rpc_host=btcd
+rpc_port=18556
+rpc_user=devuser
+rpc_pass=devpass
 
 [server]
 rpc_host=0.0.0.0

itests/docker-compose.yml

@@ -97,14 +97,17 @@ services:
       context: ../
       dockerfile: docker/sqkserver/Dockerfile
     volumes:
+      - test_shared:/rpc
       - test_lnd_server_dir:/root/.lnd
       - ./config.ini:/app/config.ini
     links:
+      - "btcd:btcd"
       - "lnd_server:lnd"
     depends_on:
       - db
     sysctls:
       - net.ipv6.conf.all.disable_ipv6=0
+    entrypoint: ["./start-sqkserver.sh"]
 
   test:
     image: test

itests/test.py

@@ -59,9 +59,14 @@ def get_address(signing_key):
     return str(address)
 
 
-def make_squeak(signing_key: CSigningKey, content: str, reply_to: bytes = b'\x00'*HASH_LENGTH):
-    block_height = 0
-    block_hash = lx('4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b')
+def get_latest_block_info(lightning_client):
+    get_info_response = lightning_client.get_info()
+    block_hash = bytes.fromhex(get_info_response.block_hash)
+    block_height = get_info_response.block_height
+    return block_hash, block_height
+
+
+def make_squeak(signing_key: CSigningKey, content: str, block_height, block_hash, reply_to: bytes = b'\x00'*HASH_LENGTH):
     timestamp = int(time.time())
     return MakeSqueakFromStr(
         signing_key,
@@ -123,7 +128,8 @@ def run():
 
         # Post a squeak with a direct request to the server
         signing_key = generate_signing_key()
-        squeak = make_squeak(signing_key, 'hello from itest!')
+        block_height, block_hash = get_latest_block_info(lnd_lightning_client)
+        squeak = make_squeak(signing_key, 'hello from itest!', block_hash, block_height)
         squeak_hash = get_hash(squeak)
 
         squeak_msg = build_squeak_msg(squeak)
@@ -208,7 +214,7 @@ def run():
         # Open channel to the server lightning node
         pubkey_bytes = string_to_hex(buy_response.offer.pubkey)
         open_channel_response = lnd_lightning_client.open_channel(pubkey_bytes, 1000000)
-        print("Server open channel response: " + str(open_channel_response))
+        print("Opening channel...")
         for update in open_channel_response:
             if update.HasField('chan_open'):
                 channel_point = update.chan_open.channel_point

requirements.txt

@@ -5,3 +5,4 @@ googleapis-common-protos
 grpcio
 grpcio-tools
 psycopg2
+requests

squeakserver/blockchain/bitcoin_blockchain_client.py

@@ -0,0 +1,54 @@
+import json
+from typing import Optional
+
+import requests
+
+
+class BitcoinBlockchainClient:
+    """Access a bitcoin daemon using RPC."""
+
+    def __init__(
+            self,
+            host: str,
+            port: int,
+            rpc_user: str,
+            rpc_password: str,
+    ) -> None:
+        self.url = f'https://{rpc_user}:{rpc_password}@{host}:{port}'
+        self.headers = {'content-type': 'application/json'}
+
+    def get_block_hash(self, block_height: int) -> Optional[bytes]:
+        # return self.access.getblockhash(block_height)
+        payload = {
+            "method": "getblockhash",
+            "params": [block_height],
+            "jsonrpc": "2.0",
+            "id": 0,
+        }
+        response = requests.post(
+            self.url,
+            data=json.dumps(payload),
+            headers=self.headers,
+        ).json()
+
+        result = response["result"]
+        block_hash = bytes.fromhex(result)
+        return block_hash
+
+    # def get_block_hash(self, block_height: int) -> Optional[bytes]:
+    #     # return self.access.getblockhash(block_height)
+    #     payload = {
+    #         "method": "getblockhash",
+    #         "params": [block_height],
+    #         "jsonrpc": "2.0",
+    #         "id": 0,
+    #     }
+    #     response = requests.post(
+    #         self.url,
+    #         data=json.dumps(payload),
+    #         headers=self.headers,
+    #     ).json()
+
+    #     result = response["result"]
+    #     block_hash = bytes.fromhex(result)
+    #     return block_hash

squeakserver/node/squeak_block_periodic_worker.py

@@ -0,0 +1,26 @@
+import logging
+import threading
+import time
+
+
+logger = logging.getLogger(__name__)
+
+
+VERIFY_UPDATE_INTERVAL_S = 10.0
+
+
+class SqueakBlockPeriodicWorker:
+
+    def __init__(self, squeak_block_verifier, postgres_db):
+        self.squeak_block_verifier = squeak_block_verifier
+        self.postgres_db = postgres_db
+
+    def verify_squeaks(self):
+        logger.info('Calling verify_squeaks.')
+        squeaks_to_verify = self.postgres_db.get_unverified_block_squeaks()
+        for squeak_hash in squeaks_to_verify:
+            self.squeak_block_verifier.verify_squeak_block(squeak_hash)
+
+    def start_running(self):
+        threading.Timer(VERIFY_UPDATE_INTERVAL_S, self.start_running).start()
+        self.verify_squeaks()

squeakserver/node/squeak_block_verifier.py

@@ -0,0 +1,30 @@
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+class SqueakBlockVerifier:
+
+    def __init__(self, postgres_db, blockchain_client):
+        self.postgres_db = postgres_db
+        self.blockchain_client = blockchain_client
+
+    def verify_squeak_block(self, squeak_hash):
+        logger.info('Verifying squeak hash: {}'.format(squeak_hash))
+        squeak = self._get_squeak(squeak_hash)
+        block_info = self._get_block_info(squeak)
+
+    def _get_squeak(self, squeak_hash):
+        return self.postgres_db.get_squeak(squeak_hash)
+
+    def _mark_squeak_verified(self, squeak_hash):
+        # TODO
+        pass
+
+    def _get_block_info(self, squeak):
+        block_height = squeak.nBlockHeight
+        block_hash = self.blockchain_client.get_block_hash(block_height)
+        logger.info('Got block hash from blockchain: {}'.format(block_hash))
+        logger.info('Got block hash from squeak: {}'.format(squeak.hashBlock))
+        logger.info('Is block hash correct: {}'.format(squeak.hashBlock == block_hash))

squeakserver/server/main.py

@@ -18,6 +18,9 @@ from squeakserver.server.squeak_server_servicer import SqueakServerServicer
 from squeakserver.server.squeak_server_handler import SqueakServerHandler
 from squeakserver.server.db_params import parse_db_params
 from squeakserver.server.postgres_db import PostgresDb
+from squeakserver.node.squeak_block_verifier import SqueakBlockVerifier
+from squeakserver.node.squeak_block_periodic_worker import SqueakBlockPeriodicWorker
+from squeakserver.blockchain.bitcoin_blockchain_client import BitcoinBlockchainClient
 
 
 logger = logging.getLogger(__name__)
@@ -94,6 +97,23 @@ def load_postgres_db(config):
     return PostgresDb(db_params)
 
 
+def load_blockchain_client(config):
+    return BitcoinBlockchainClient(
+        config['bitcoin']['rpc_host'],
+        config['bitcoin']['rpc_port'],
+        config['bitcoin']['rpc_user'],
+        config['bitcoin']['rpc_pass'],
+    )
+
+
+def load_squeak_block_verifier(postgres_db, blockchain_client):
+    return SqueakBlockVerifier(postgres_db, blockchain_client)
+
+
+def load_squeak_block_periodic_worker(squeak_block_verifier, postgres_db):
+    return SqueakBlockPeriodicWorker(squeak_block_verifier, postgres_db)
+
+
 def sigterm_handler(_signo, _stack_frame):
     # Raises SystemExit(0):
     sys.exit(0)
@@ -183,6 +203,12 @@ def run_server(config):
     lightning_client = load_lightning_client(config)
     lightning_host_port = load_lightning_host_port(config)
 
+    # Start the squeak block verifier
+    bitcoin_blockchain_client = load_blockchain_client(config)
+    squeak_block_verifier = load_squeak_block_verifier(postgres_db, bitcoin_blockchain_client)
+    squeak_block_periodic_worker = load_squeak_block_periodic_worker(squeak_block_verifier, postgres_db)
+    squeak_block_periodic_worker.start_running()
+
     # start admin rpc server
     admin_handler = load_admin_handler(lightning_client, postgres_db)
     admin_rpc_server = load_admin_rpc_server(config, admin_handler)

squeakserver/server/postgres_db.py

@@ -170,3 +170,18 @@ class PostgresDb():
                 following=row[6],
             )
             return squeak_profile
+
+    def get_unverified_block_squeaks(self):
+        """ Lookup squeaks. """
+        sql = """
+        SELECT hash FROM squeak
+        WHERE block_header IS NULL;
+        """
+        with self.get_cursor() as curs:
+            curs.execute(sql)
+            rows = curs.fetchall()
+            hashes = [
+                bytes.fromhex(row[0])
+                for row in rows
+            ]
+            return hashes