start integration testing with lnregtest

This is the first commit to add dependency on lnregtest
(https://github.com/bitromortac/lnregtest), which is used to create
small dummy lightning networks, whose master node is interfaced with
lndmanage.

.gitignore

@@ -1,11 +1,14 @@
-grpc_compile/googleapis/*
-nodes/*
+# development
 .idea/*
 .vscode/*
 venv/*
-lndmanage.log
-tests/graph_pickles/
+grpc_compile/googleapis/*
+
+# runtime data
+*.log
 __pycache__/*
 config.ini
 *.pyc
-venv/*
+
+# test data
+test/test_data/*

README.md

@@ -192,6 +192,15 @@ $ ./lndmanage.py status
 ```
 If it works, you should see the node status.
 
+Testing
+-------
+Requirements are an installation of [lnregtest](https://github.com/bitromortac/lnregtest)
+and links to bitcoind, bitcoin-cli, lnd, and lncli in the `test/bin` folder.
+
+Tests can be run with
+`python3 -m unittest discover test`
+from the root folder.
+
 Docker
 ------
 

_settings.py

@@ -1,12 +1,5 @@
-import os
 import configparser
 
-config = configparser.ConfigParser()
-dirname = os.path.dirname(__file__)
-filename = os.path.join(dirname, 'config.ini')
-
-config.read(filename)
-
 # -------- graph settings --------
 # accepted age of the network graph
 CACHING_RETENTION_MINUTES = 30
@@ -35,10 +28,7 @@ UNBALANCED_CHANNEL = 0.2
 CHUNK_SIZE = 1.0
 REBALANCING_TRIALS = 30
 
-# -------- logging --------
-# debug level can be INFO or DEBUG
-DEBUG_LEVEL = config['logging']['loglevel']
-
+# logger settings
 logger_config = {
     'version': 1,
     'disable_existing_loggers': False,
@@ -74,3 +64,9 @@ logger_config = {
         },
     }
 }
+
+
+def read_config(config_path):
+    config = configparser.ConfigParser()
+    config.read(config_path)
+    return config

lib/node.py

@@ -51,16 +51,20 @@ class LndNode(Node):
     """
     Implements an interface to an lnd node.
     """
-    def __init__(self):
+    def __init__(self, config_file=None, lnd_home=None, lnd_host=None,
+                 regtest=False):
         super().__init__()
+        self.config_file = config_file
+        self.lnd_home = lnd_home
+        self.lnd_host = lnd_host
+        self.regtest = regtest
         self._stub = self.connect()
         self.network = Network(self)
         self.update_blockheight()
         self.set_info()
         self.public_active_channels = self.get_open_channels(public_only=True, active_only=True)
 
-    @staticmethod
-    def connect():
+    def connect(self):
         """
         Establishes a connection to lnd using the hostname, tls certificate,
         and admin macaroon defined in settings.
@@ -76,10 +80,28 @@ class LndNode(Node):
                                                'ECDHE-ECDSA-AES256-SHA384:' + \
                                                'ECDHE-ECDSA-AES256-GCM-SHA384'
 
-        cert = open(os.path.expanduser(_settings.config['network']['tls_cert_file']), 'rb').read()
+        # if no lnd_home is given, then use the paths from the config,
+        # else override them with default file paths in lnd_home
+        if self.lnd_home is not None:
+            cert_file = os.path.join(self.lnd_home, 'tls.cert')
+            bitcoin_network = 'regtest' if self.regtest else 'mainnet'
+            macaroon_file = os.path.join(
+                self.lnd_home, 'data/chain/bitcoin/',
+                bitcoin_network, 'admin.macaroon')
+            if self.lnd_host is None:
+                raise ValueError('if lnd_home is given, lnd_host must be given also')
+            lnd_host = self.lnd_host
+        else:
+            config = _settings.read_config(self.config_file)
+            cert_file = os.path.expanduser(config['network']['tls_cert_file'])
+            macaroon_file = os.path.expanduser(config['network']['admin_macaroon_file'])
+            lnd_host = config['network']['lnd_grpc_host']
+
+        with open(cert_file, 'rb') as f:
+            cert = f.read()
 
         if macaroons:
-            with open(os.path.expanduser(_settings.config['network']['admin_macaroon_file']), 'rb') as f:
+            with open(macaroon_file, 'rb') as f:
                 macaroon_bytes = f.read()
                 macaroon = codecs.encode(macaroon_bytes, 'hex')
 
@@ -94,7 +116,7 @@ class LndNode(Node):
         else:
             creds = grpc.ssl_channel_credentials(cert)
 
-        channel = grpc.secure_channel(_settings.config['network']['lnd_grpc_host'], creds, options=[
+        channel = grpc.secure_channel(lnd_host, creds, options=[
             ('grpc.max_receive_message_length', 50 * 1024 * 1024)  # necessary to circumvent standard size limitation
         ])
 

lndmanage.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 import argparse
 import time
+import os
 
 from lib.node import LndNode
 from lib.listchannels import ListChannels
@@ -271,7 +272,10 @@ def main():
         # update the loglevel of the stdout handler to the user choice
         logger.handlers[0].setLevel(args.loglevel)
 
-    node = LndNode()
+    # config.ini is expected to be in root directory
+    root_dir = os.path.dirname(os.path.realpath(__file__))
+    config_file = os.path.join(root_dir, 'config.ini')
+    node = LndNode(config_file=config_file)
 
     if args.cmd == 'status':
         node.print_status()

test/bin/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

test/graph_definitions/small_star_ring.py

@@ -0,0 +1,77 @@
+"""
+Implements a lightning network topology:
+"""
+nodes = {
+    'A': {
+        'grpc_port': 11009,
+        'rest_port': 8080,
+        'port': 9735,
+        'base_fee_msat': 1,
+        'fee_rate': 0.000001,
+        'channels': {
+            1: {
+                'to': 'B',
+                'capacity': 1000000,
+                'ratio_local': 10,
+                'ratio_remote': 0,
+            },
+            2: {
+                'to': 'C',
+                'capacity': 1000000,
+                'ratio_local': 5,
+                'ratio_remote': 5,
+            },
+        }
+    },
+    'B': {
+        'grpc_port': 11010,
+        'rest_port': 8081,
+        'port': 9736,
+        'base_fee_msat': 2,
+        'fee_rate': 0.000001,
+        'channels': {
+            3: {
+                'to': 'C',
+                'capacity': 10000000,
+                'ratio_local': 5,
+                'ratio_remote': 5,
+            },
+            4: {
+                'to': 'D',
+                'capacity': 10000000,
+                'ratio_local': 5,
+                'ratio_remote': 5,
+            },
+        }
+    },
+    'C': {
+        'grpc_port': 11011,
+        'rest_port': 8082,
+        'port': 9737,
+        'base_fee_msat': 1,
+        'fee_rate': 0.000003,
+        'channels': {
+            5: {
+                'to': 'D',
+                'capacity': 1000000,
+                'ratio_local': 5,
+                'ratio_remote': 5,
+            },
+        }
+    },
+    'D': {
+        'grpc_port': 11012,
+        'rest_port': 8083,
+        'port': 9738,
+        'base_fee_msat': 1,
+        'fee_rate': 0.000002,
+        'channels': {
+            6: {
+                'to': 'A',
+                'capacity': 1000000,
+                'ratio_local': 10,
+                'ratio_remote': 0,
+            },
+        }
+    },
+}

test/test_rebalance.py

@@ -0,0 +1,96 @@
+import os, time
+from unittest import TestCase
+
+from lib.node import LndNode
+from lib.listchannels import ListChannels
+from lib.rebalance import Rebalancer
+from lnregtest.lib.network import RegtestNetwork
+from lnregtest.lib.utils import format_dict, dict_comparison
+
+import _settings
+import logging.config
+logging.config.dictConfig(_settings.logger_config)
+logger = logging.getLogger(__name__)
+
+test_dir = os.path.dirname(os.path.realpath(__file__))
+bin_dir = os.path.join(test_dir, 'bin')
+graph_definitions = os.path.join(test_dir, 'graph_definitions')
+small_star_ring_location = os.path.join(graph_definitions, 'small_star_ring.py')
+test_data_dir = os.path.join(test_dir, 'test_data')
+
+# important to set the cache time to zero, otherwise one will
+# have unexpected behavior of the tests
+_settings.CACHING_RETENTION_MINUTES = 0
+
+
+class TestRebalance(TestCase):
+    def setUp(self):
+        self.testnet = RegtestNetwork(
+            binary_folder=bin_dir,
+            network_definition_location=small_star_ring_location,
+            nodedata_folder=test_data_dir,
+            node_limit='H',
+            from_scratch=True
+        )
+        # run network and print information
+        self.testnet.run_nocleanup()
+        # to run the lightning network in the background and do some testing
+        # here, run:
+        # $ lnregtest --nodedata_folder /path/to/lndmanage/test/test_data/
+        # self.testnet.run_from_background()
+
+        logger.info("Generated network information:")
+        logger.info(format_dict(self.testnet.node_mapping))
+        logger.info(format_dict(self.testnet.channel_mapping))
+        logger.info(format_dict(self.testnet.assemble_graph()))
+
+        master_node_data_dir = self.testnet.master_node.lnd_data_dir
+        master_node_port = self.testnet.master_node.grpc_port
+
+        # initialize lndnode
+        self.lndnode = LndNode(
+            lnd_home=master_node_data_dir,
+            lnd_host='localhost:' + str(master_node_port),
+            regtest=True
+        )
+        self.lndnode.print_status()
+        logger.info('Initializing done.')
+
+    def test_rebalance_channel_6(self):
+        listchannels = ListChannels(self.lndnode)
+        listchannels.print_all_channels('rev_alias')
+
+        rebalancer = Rebalancer(
+            self.lndnode,
+            max_effective_fee_rate=50,
+            budget_sat=20
+        )
+        # graph state before
+        graph_should = self.testnet.assemble_graph()
+
+        # channel A-B, defined as channel 1
+        # channel A-C, defined as channel 2
+        # channel A-D, defined as channel 6
+        # TODO: test channel 1 and 2, which are currently failing to rebalance
+        test_channel_number = 6
+        channel_id = self.testnet.channel_mapping[
+            test_channel_number]['channel_id']
+        logger.info('Testing rebalancing of channel: {}'.format(channel_id))
+
+        # rebalance channel
+        rebalancer.rebalance(
+            channel_id,
+            dry=False,
+            chunksize=1.0,
+            target=0.0,
+            allow_unbalancing=False
+        )
+        graph_is = self.testnet.assemble_graph()
+
+        dict_comparison(graph_should, graph_is, show_diff=True)
+
+        listchannels = ListChannels(self.lndnode)
+        listchannels.print_all_channels('rev_alias')
+
+    def tearDown(self):
+        self.testnet.cleanup()

tests/graphs.py

@@ -1 +0,0 @@
-# TODO: place here all graphs for testing

tests/test_rebalance.py

@@ -1,33 +0,0 @@
-from unittest import TestCase
-
-import _settings
-from lib.node import LndNode
-from lib.rebalance import Rebalancer, manual_rebalance
-
-import logging.config
-logging.config.dictConfig(_settings.logger_config)
-
-
-class TestRebalance(TestCase):
-    def setUp(self):
-        self.node = LndNode()
-
-    def test_manual_rebalance(self):
-        manual_rebalance(self.node, 000000000000000000, 000000000000000000, amt=141248, number_of_routes=5)
-
-    def test_auto_rebalance(self):
-        rebalancer = Rebalancer(self.node, max_effective_fee_rate=1, budget_sat=10)
-        invoice_r_hash = self.node.get_rebalance_invoice(memo='autorebalance test')
-        rebalancer.rebalance_two_channels(000000000000000000, 000000000000000000, amt_sat=1,
-                                          invoice_r_hash=invoice_r_hash, budget_sat=10)
-
-    def test_rebalance(self):
-        rebalancer = Rebalancer(self.node, max_effective_fee_rate=0.0001, budget_sat=50)
-        channel_id = 000000000000000000
-        fee = rebalancer.rebalance(channel_id)
-        print(fee)
-
-
-if __name__ == "__main__":
-    node = LndNode()
-    result = manual_rebalance(node, 000000000000000000, 000000000000000000, amt=10, number_of_routes=5)