Add clightning backend support (#2205)

* Add c-lightning support

* Got subscribe paid invoices working with c-lightning

* Remove log lines

* Use a single config section for all lightning configs

* Fix itest config

* Update configuration doc with new confis

README.md

@@ -39,10 +39,10 @@ The protocol is defined [here](https://github.com/yzernik/squeak/blob/master/doc
 [node]
 network=mainnet
 
-[lnd]
+[lightning]
-host=localhost
+backend=lnd
-tls_cert_path=~/.lnd/tls.cert
+lnd_tls_cert_path=~/.lnd/tls.cert
-macaroon_path=~/.lnd/data/chain/bitcoin/mainnet/admin.macaroon
+lnd_macaroon_path=~/.lnd/data/chain/bitcoin/testnet/admin.macaroon
 
 [bitcoin]
 rpc_host=localhost
@@ -60,6 +60,14 @@ username=devuser
 password=devpass
 ```
 
+If you want to use [c-lightning](https://github.com/ElementsProject/lightning) as the Lightning Network node backend instead of [LND](https://github.com/lightningnetwork/lnd), replace the `lightning` section with the following:
+
+```
+[lightning]
+backend=clightning
+clightning_rpc_file=~/.lightning/lightning-rpc
+```
+
 Add any other [configs](docs/configuration.md) that you need.
 
 ### Step 2. Install squeaknode:

clightning-config.ini

@@ -0,0 +1,21 @@
+[node]
+network=testnet
+
+[lightning]
+backend=clightning
+
+[bitcoin]
+rpc_host=localhost
+rpc_port=18332
+rpc_user=devuser
+rpc_pass=devpass
+zeromq_hashblock_port=28334
+
+[tor]
+proxy_ip=localhost
+proxy_port=9150
+
+[webadmin]
+enabled=true
+username=devuser
+password=devpass

config.ini

@@ -1,10 +1,10 @@
 [node]
 network=testnet
 
-[lnd]
+[lightning]
-host=localhost
+external_host=localhost
-tls_cert_path=~/.lnd/tls.cert
+lnd_tls_cert_path=~/.lnd/tls.cert
-macaroon_path=~/.lnd/data/chain/bitcoin/testnet/admin.macaroon
+lnd_macaroon_path=~/.lnd/data/chain/bitcoin/testnet/admin.macaroon
 
 [bitcoin]
 rpc_host=localhost

docker/config-clightning.ini

@@ -0,0 +1,26 @@
+[node]
+network=testnet
+
+[lightning]
+backend=clightning
+external_host=clightning_server
+external_port=19735
+
+[clightning]
+enabled=true
+
+[bitcoin]
+rpc_host=bitcoin-core
+rpc_port=18332
+rpc_user=devuser
+rpc_pass=devpass
+zeromq_hashblock_port=28334
+
+[tor]
+proxy_ip=tor-node
+proxy_port=9050
+
+[webadmin]
+enabled=true
+username=devuser
+password=devpass

docker/config.ini

@@ -1,10 +1,11 @@
 [node]
 network=testnet
 
-[lnd]
+[lightning]
-host=lnd
+lnd_rpc_host=lnd
-tls_cert_path=~/.lnd/tls.cert
+external_host=lnd
-macaroon_path=~/.lnd/data/chain/bitcoin/testnet/admin.macaroon
+lnd_tls_cert_path=~/.lnd/tls.cert
+lnd_macaroon_path=~/.lnd/data/chain/bitcoin/testnet/admin.macaroon
 
 [bitcoin]
 rpc_host=bitcoin-core

docker/docker-compose.mainnet.yml

@@ -19,6 +19,16 @@ services:
     environment:
       - NETWORK=mainnet
 
+  clightning_client:
+    command:
+      - --bitcoin-rpcconnect=bitcoin-core
+      - --bitcoin-rpcuser=devuser
+      - --bitcoin-rpcpassword=devpass
+      - --network=mainnet
+      - --plugin-dir=/usr/libexec/c-lightning/plugins
+      - --alias=myitestnode
+      - --log-level=debug
+
   # squeaknode:
   #   environment:
   #     - NETWORK=mainnet

docker/docker-compose.override.yml

@@ -15,6 +15,32 @@ services:
       -zmqpubhashblock=tcp://0.0.0.0:28334
       -zmqpubhashtx=tcp://0.0.0.0:28334
 
+  clightning_client:
+    command:
+      - --bitcoin-rpcconnect=bitcoin-core
+      - --bitcoin-rpcuser=devuser
+      - --bitcoin-rpcpassword=devpass
+      - --network=testnet
+      - --plugin-dir=/usr/libexec/c-lightning/plugins
+      - --alias=myclientningclient
+      - --log-level=debug
+      - --rpc-file=/root/.lightning/lightning-rpc
+      - --rpc-file-mode=0777
+      - --proxy=tor-node:9050
+
+  clightning_server:
+    command:
+      - --bitcoin-rpcconnect=bitcoin-core
+      - --bitcoin-rpcuser=devuser
+      - --bitcoin-rpcpassword=devpass
+      - --network=testnet
+      - --plugin-dir=/usr/libexec/c-lightning/plugins
+      - --alias=myclientningserver
+      - --log-level=debug
+      - --rpc-file=/root/.lightning/lightning-rpc
+      - --rpc-file-mode=0777
+      - --proxy=tor-node:9050
+
   # squeaknode:
   #   environment:
   #     - NETWORK=testnet

docker/docker-compose.yml

@@ -57,6 +57,45 @@ services:
       - net.ipv6.conf.all.disable_ipv6=0
     entrypoint: ["./start-lnd.sh"]
 
+  clightning_client:
+    image: elementsproject/lightningd:v0.11.0.1
+    container_name: clightning_client
+    expose:
+      - "9735"
+      - "19735"
+      - "9835"
+      - "10009"
+    ports:
+      - 10010:10009
+      - 9835:9835
+    volumes:
+      - ~/.lightning:/root/.lightning
+      - tor-dir:/var/lib/tor
+    links:
+      - "bitcoin-core:bitcoin-core"
+    depends_on:
+      - "tor-node"
+    sysctls:
+      - net.ipv6.conf.all.disable_ipv6=0
+
+  clightning_server:
+    image: elementsproject/lightningd:v0.11.0.1
+    container_name: clightning_server
+    expose:
+      - "9735"
+      - "19735"
+      - "9835"
+      - "10009"
+    volumes:
+      - clightning_server_dir:/root/.lightning
+      - tor-dir:/var/lib/tor
+    links:
+      - "bitcoin-core:bitcoin-core"
+    depends_on:
+      - "tor-node"
+    sysctls:
+      - net.ipv6.conf.all.disable_ipv6=0
+
   tor-node:
     image: osminogin/tor-simple
     container_name: tor-node
@@ -89,10 +128,36 @@ services:
       - net.ipv6.conf.all.disable_ipv6=0
     entrypoint: ["./entrypoint.sh"]
 
+  squeaknode_clightning:
+    image: squeaknode
+    container_name: squeaknode_clightning
+    build:
+      context: ../
+      dockerfile: Dockerfile
+    depends_on:
+      - "lnd_server"
+    volumes:
+      - clightning_server_dir:/root/.lightning
+      - squeaknode_clightning_dir:/root/.sqk
+      - ./config-clightning.ini:/config.ini
+    ports:
+      - 18558:18555
+      - 12996:12994
+    links:
+      - "bitcoin-core:bitcoin-core"
+      - "lnd_server:lnd"
+    sysctls:
+      - net.ipv6.conf.all.disable_ipv6=0
+    entrypoint: ["./entrypoint.sh"]
+
 volumes:
   lnd_server_dir:
     driver: local
+  clightning_server_dir:
+    driver: local
   squeaknode_dir:
     driver: local
+  squeaknode_clightning_dir:
+    driver: local
   tor-dir:
     driver: local

docs/configuration.md

@@ -27,12 +27,14 @@ bitcoin.rpc_pass | string | | yes | "" | SQUEAKNODE_BITCOIN_RPC_PASS | The passw
 bitcoin.rpc_use_ssl | boolean | [true, false] | yes | false | SQUEAKNODE_BITCOIN_USE_SSL | Use SSL for the connection to the bitcoin node.
 bitcoin.rpc_ssl_cert | str |  | yes | "" | SQUEAKNODE_BITCOIN_SSL_CERT | The path to the SSL cert to use for connection to the bitcoin node, if one is used.
 bitcoin.zeromq_hashblock_port | int | | yes | 28334 | SQUEAKNODE_BITCOIN_ZEROMQ_HASHBLOCK_PORT | The port to use to subscribe with zeromq to new block hashes on the bitcoin node.
-lnd.host | string | | yes | "localhost" | SQUEAKNODE_LND_HOST | The host of the LND node to connect.
+lightning.backend | string | | yes | "lnd" | SQUEAKNODE_LIGHTNING_BACKEND | The Lightning Network node implementation to use.
-lnd.external_host | string | | yes | "" | SQUEAKNODE_LND_EXTERNAL_HOST | The host of the LND node to share with other peers.
+lightning.external_host | string | | yes | "" | SQUEAKNODE_LIGHTNING_EXTERNAL_HOST | The host of the lightning node to share with other peers.
-lnd.port | int | | yes | | SQUEAKNODE_LND_PORT | The port of the LND node to use for LND peer connections.
+lightning.external_port | int | | yes | | SQUEAKNODE_LIGHTNING_EXTERNAL_PORT | The port of the lightning node to share with other peers.
-lnd.rpc_port | int | | yes | | SQUEAKNODE_LND_RPC_PORT | The port of the LND node to use for RPC connections.
+lightning.lnd_rpc_host | string | | yes | "localhost" | SQUEAKNODE_LIGHTNING_LND_RPC_HOST | The host of the LND node to connect.
-lnd.tls_cert_path | string | | yes | "" | SQUEAKNODE_LND_TLS_CERT_PATH | The path to the TLS certificate to use for LND connection.
+lightning.lnd_rpc_port | int | | yes | | SQUEAKNODE_LIGHTNING_LND_RPC_PORT | The port of the LND node to use for RPC connections.
-lnd.macaroon_path | string | | yes | "" | SQUEAKNODE_LND_MACAROON_PATH | The path to the macaroon to use for LND connection.
+lightning.lnd_tls_cert_path | string | | yes | "" | SQUEAKNODE_LIGHTNING_LND_TLS_CERT_PATH | The path to the TLS certificate to use for LND connection.
+lightning.lnd_macaroon_path | string | | yes | "" | SQUEAKNODE_LIGHTNING_LND_MACAROON_PATH | The path to the macaroon to use for LND connection.
+lightning.clightning_rpc_file | string | | yes | "" | SQUEAKNODE_LIGHTNING_CLIGHTNING_RPC_FILE | The path to the file to use for c-lightning RPC.
 tor.proxy_ip | string | | yes | "" | SQUEAKNODE_TOR_PROXY_IP | The ip address or host of the SOCKS5 Tor proxy, if one is used.
 tor.proxy_port | int | | yes | 0 | SQUEAKNODE_TOR_PROXY_PORT | The port of the SOCKS5 Tor proxy, is one is used.
 db.connection_string | string | | yes | "" | SQUEAKNODE_DB_CONNECTION_STRING | The connection string to use to connect to a SQL database. If none is specified, a sqlite database will be used on the local file system.

itests/config.ini

@@ -3,10 +3,10 @@ network=simnet
 price_msat=1000000
 max_squeaks_per_public_key_per_block=50
 
-[lnd]
+[lightning]
-host=lnd
+lnd_rpc_host=lnd
-tls_cert_path=/root/.lnd/tls.cert
+lnd_tls_cert_path=/root/.lnd/tls.cert
-macaroon_path=/root/.lnd/data/chain/bitcoin/simnet/admin.macaroon
+lnd_macaroon_path=/root/.lnd/data/chain/bitcoin/simnet/admin.macaroon
 
 [bitcoin]
 rpc_host=btcd

requirements.txt

@@ -1,4 +1,5 @@
 alembic==1.7.1
+docutils==0.17.1
 Flask==2.0.1
 flask-cors==3.0.10
 flask-login==0.5.0
@@ -7,6 +8,7 @@ googleapis-common-protos==1.53.0
 importlib_resources==1.4.0
 mypy-protobuf==2.9
 protobuf==3.17.3
+pyln-client==0.10.1
 PySocks==1.7.1
 python-bitcoinlib==0.11.0
 pyzmq==22.3.0

squeaknode/config/config.py

@@ -54,11 +54,15 @@ BITCOIN_RPC_PORT = {
     "testnet": 18332,
     "simnet": 18556,
 }
+DEFAULT_LIGHTNING_BACKEND = "lnd"
 DEFAULT_LND_PORT = 9735
 DEFAULT_LND_RPC_PORT = 10009
+DEFAULT_CLIGHTNING_DIR = ".lightning"
+DEFAULT_CLIGHTNING_RPC_FILE = str(
+    Path.home() / DEFAULT_CLIGHTNING_DIR / 'lightning-rpc')
 DEFAULT_SQK_DIR = ".sqk"
 DEFAULT_SQK_DIR_PATH = str(Path.home() / DEFAULT_SQK_DIR)
-DEFAULT_LND_HOST = "localhost"
+DEFAULT_LND_RPC_HOST = "localhost"
 DEFAULT_INTEREST_BLOCK_INTERVAL = 2016
 DEFAULT_SENT_OFFER_RETENTION_S = 86400
 DEFAULT_RECEIVED_OFFER_RETENTION_S = 86400
@@ -82,14 +86,17 @@ class BitcoinConfig(Config):
                                 default=DEFAULT_BITCOIN_ZEROMQ_HASHBLOCK_PORT)
 
 
-@section('lnd')
+@section('lightning')
-class LndConfig(Config):
+class LightningConfig(Config):
-    host = key(cast=str, required=False, default=DEFAULT_LND_HOST)
+    backend = key(cast=str, required=False, default=DEFAULT_LIGHTNING_BACKEND)
     external_host = key(cast=str, required=False, default="")
-    port = key(cast=int, required=False, default=DEFAULT_LND_PORT)
+    external_port = key(cast=int, required=False, default=DEFAULT_LND_PORT)
-    rpc_port = key(cast=int, required=False, default=DEFAULT_LND_RPC_PORT)
+    lnd_rpc_host = key(cast=str, required=False, default=DEFAULT_LND_RPC_HOST)
-    tls_cert_path = key(cast=str, required=False, default="")
+    lnd_rpc_port = key(cast=int, required=False, default=DEFAULT_LND_RPC_PORT)
-    macaroon_path = key(cast=str, required=False, default="")
+    lnd_tls_cert_path = key(cast=str, required=False, default="")
+    lnd_macaroon_path = key(cast=str, required=False, default="")
+    clightning_rpc_file = key(cast=str, required=False,
+                              default=DEFAULT_CLIGHTNING_RPC_FILE)
 
 
 @section('tor')
@@ -172,7 +179,7 @@ class TwitterConfig(Config):
 
 class SqueaknodeConfig(Config):
     bitcoin = group_key(BitcoinConfig)
-    lnd = group_key(LndConfig)
+    lightning = group_key(LightningConfig)
     tor = group_key(TorConfig)
     server = group_key(ServerConfig)
     rpc = group_key(RpcConfig)

squeaknode/lightning/clightning_lightning_client.py

@@ -0,0 +1,140 @@
+# MIT License
+#
+# Copyright (c) 2020 Jonathan Zernik
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import logging
+import time
+import uuid
+
+from pyln.client import LightningRpc
+
+from squeaknode.core.exception import InvoiceSubscriptionError
+from squeaknode.lightning.info import Info
+from squeaknode.lightning.invoice import Invoice
+from squeaknode.lightning.invoice_stream import InvoiceStream
+from squeaknode.lightning.lightning_client import LightningClient
+from squeaknode.lightning.pay_req import PayReq
+from squeaknode.lightning.payment import Payment
+
+
+logger = logging.getLogger(__name__)
+
+
+class CLightningClient(LightningClient):
+    """Access a c-lightning instance using the Python API."""
+
+    def __init__(
+        self,
+        rpc_path: str,
+    ) -> None:
+        self.rpc_path = rpc_path
+
+    def init(self):
+        # Create an instance of the LightningRpc object using the Core Lightning daemon on your computer.
+        logger.info('initializing clightning client: {}'.format(self.rpc_path))
+        self.lrpc = LightningRpc(self.rpc_path)
+
+    def pay_invoice(self, payment_request: str) -> Payment:
+        payment = self.lrpc.pay(payment_request)
+        if payment['status'] == 'complete':
+            return Payment(
+                payment_preimage=bytes.fromhex(payment['payment_preimage']),
+                payment_error='',
+            )
+        else:
+            return Payment(
+                payment_preimage=b'',
+                payment_error='Payment failed.',
+            )
+
+    def get_info(self) -> Info:
+        info = self.lrpc.getinfo()
+        pubkey = info['id']
+        binding = info.get('binding')
+        uris = []
+        if binding:
+            for b in binding:
+                address = b['address']
+                if ':' not in address:  # TODO: Change type of uri to LightningAddress.
+                    port = b['port']
+                    uri = f"{pubkey}@{address}:{port}"
+                    uris.append(uri)
+        return Info(
+            uris=uris,
+        )
+
+    def decode_pay_req(self, payment_request: str) -> PayReq:
+        pay_req = self.lrpc.decodepay(payment_request)
+        return PayReq(
+            payment_hash=bytes.fromhex(pay_req['payment_hash']),
+            payment_point=b'',  # TODO: Use real payment point.
+            num_msat=pay_req['amount_msat'].millisatoshis,
+            destination=pay_req['payee'],
+            timestamp=int(pay_req['created_at']),
+            expiry=int(pay_req['expiry']),
+        )
+
+    def create_invoice(self, preimage: bytes, amount_msat: int) -> Invoice:
+        created_invoice = self.lrpc.invoice(
+            amount_msat,
+            label=str(uuid.uuid4()),
+            description="Squeaknode invoice",
+            preimage=preimage.hex(),
+        )
+        creation_time = int(time.time())
+        expiry = int(created_invoice['expires_at']) - creation_time
+        return Invoice(
+            r_hash=bytes.fromhex(created_invoice['payment_hash']),
+            payment_request=created_invoice['bolt11'],
+            value_msat=amount_msat,
+            settled=False,
+            settle_index=0,
+            creation_date=creation_time,
+            expiry=expiry,
+        )
+
+    def subscribe_invoices(self, settle_index: int) -> InvoiceStream:
+
+        def cancel_fn():
+            return None
+
+        def get_invoice_stream():
+            try:
+                pay_index = settle_index
+                while True:
+                    payment = self.lrpc.waitanyinvoice(lastpay_index=pay_index)
+                    if payment['status'] == 'paid':
+                        pay_index = payment.get('pay_index') or 0
+                        yield Invoice(
+                            r_hash=bytes.fromhex(payment['payment_hash']),
+                            payment_request=payment.get('bolt11') or '',
+                            value_msat=payment.get('value_msat'),
+                            settled=True,
+                            settle_index=pay_index,
+                            creation_date=0,  # This value is ignored.
+                            expiry=payment['expires_at'],
+                        )
+            except Exception:
+                raise InvoiceSubscriptionError()
+
+        return InvoiceStream(
+            cancel=cancel_fn,
+            result_stream=get_invoice_stream(),
+        )

squeaknode/lightning/lightning_client.py

@@ -34,6 +34,13 @@ logger = logging.getLogger(__name__)
 
 class LightningClient(ABC):
 
+    @abstractmethod
+    def init(self):
+        """Initialize the lightning client.
+        Returns:
+            None
+        """
+
     @abstractmethod
     def get_info(self) -> Info:
         """Get info about the lightning node.

squeaknode/lightning/lnd_lightning_client.py

@@ -171,7 +171,15 @@ class LNDLightningClient(LightningClient):
             try:
                 for invoice in subscribe_result:
                     if invoice.settled:
-                        yield invoice
+                        yield Invoice(
+                            r_hash=invoice.r_hash,
+                            payment_request=invoice.payment_request,
+                            value_msat=invoice.value_msat,
+                            settled=invoice.settled,
+                            settle_index=invoice.settle_index,
+                            creation_date=invoice.creation_date,
+                            expiry=invoice.expiry,
+                        )
             except grpc.RpcError as e:
                 if e.code() != grpc.StatusCode.CANCELLED:
                     raise InvoiceSubscriptionError()

squeaknode/node/squeak_controller.py

@@ -105,10 +105,10 @@ class SqueakController:
             peer_address: PeerAddress,
     ) -> Optional[Offer]:
         lnd_external_address: Optional[LightningAddressHostPort] = None
-        if self.config.lnd.external_host:
+        if self.config.lightning.external_host:
             lnd_external_address = LightningAddressHostPort(
-                host=self.config.lnd.external_host,
+                host=self.config.lightning.external_host,
-                port=self.config.lnd.port,
+                port=self.config.lightning.external_port,
             )
         price_msat = self.get_sell_price_msat()
         if price_msat == 0:

squeaknode/node/squeak_node.py

@@ -34,6 +34,7 @@ from squeaknode.core.squeak_core import SqueakCore
 from squeaknode.db.db_engine import get_connection_string
 from squeaknode.db.db_engine import get_engine
 from squeaknode.db.squeak_db import SqueakDb
+from squeaknode.lightning.clightning_lightning_client import CLightningClient
 from squeaknode.lightning.lnd_lightning_client import LNDLightningClient
 from squeaknode.node.node_settings import NodeSettings
 from squeaknode.node.payment_processor import PaymentProcessor
@@ -48,6 +49,7 @@ from squeaknode.server.app import SqueakPeerWebServer
 from squeaknode.server.squeak_peer_server_handler import SqueakPeerServerHandler
 from squeaknode.twitter.twitter_forwarder import TwitterForwarder
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -118,12 +120,21 @@ class SqueakNode:
         self.node_settings = NodeSettings(self.squeak_db)
 
     def create_lightning_client(self):
-        self.lightning_client = LNDLightningClient(
+        if self.config.lightning.backend == 'lnd':
-            self.config.lnd.host,
+            self.lightning_client = LNDLightningClient(
-            self.config.lnd.rpc_port,
+                self.config.lightning.lnd_rpc_host,
-            self.config.lnd.tls_cert_path,
+                self.config.lightning.lnd_rpc_port,
-            self.config.lnd.macaroon_path,
+                self.config.lightning.lnd_tls_cert_path,
-        )
+                self.config.lightning.lnd_macaroon_path,
+            )
+        elif self.config.lightning.backend == 'clightning':
+            self.lightning_client = CLightningClient(
+                self.config.lightning.clightning_rpc_file,
+            )
+        else:
+            raise Exception('Invalid lightning backend: {}'.format(
+                self.config.lightning.backend,
+            ))
 
     def create_bitcoin_client(self):
         self.bitcoin_client = BitcoinCoreClient(

tests/core/test_squeak_core.py

@@ -189,6 +189,9 @@ class MockLightningClient(LightningClient):
         self.payment = payment
         self.invoice_stream = invoice_stream
 
+    def init(self):
+        pass
+
     def get_info(self):
         return self.info