From 8910bdedbfa8a3592d7fe2cf2cc1ff07d0575dab Mon Sep 17 00:00:00 2001 From: bitromortac Date: Thu, 29 Jul 2021 19:53:53 +0200 Subject: [PATCH] listpeers: list aggregated channel data of peers adds the main listpeers command with subcommands * listpeers in: sorted by total incoming funds * listpeers out: sorted by total outgoing funds --- examples/example_node_info.py | 2 +- lndmanage/lib/{types.py => data_types.py} | 21 +- lndmanage/lib/fee_setting.py | 2 +- lndmanage/lib/forwardings.py | 659 +++++++++------- lndmanage/lib/listchannels.py | 598 --------------- lndmanage/lib/listings.py | 866 ++++++++++++++++++++++ lndmanage/lib/node.py | 18 +- lndmanage/lib/openchannels.py | 2 +- lndmanage/lib/recommend_nodes.py | 4 +- lndmanage/lndmanage.py | 52 +- test/test_circle.py | 2 +- test/test_rebalance.py | 2 +- 12 files changed, 1359 insertions(+), 869 deletions(-) rename lndmanage/lib/{types.py => data_types.py} (62%) delete mode 100644 lndmanage/lib/listchannels.py create mode 100644 lndmanage/lib/listings.py diff --git a/examples/example_node_info.py b/examples/example_node_info.py index 5cacd7e..ae4c190 100644 --- a/examples/example_node_info.py +++ b/examples/example_node_info.py @@ -1,4 +1,4 @@ -from lndmanage.lib.listchannels import ListChannels +from lndmanage.lib.listings import ListChannels from lndmanage.lib.node import LndNode from lndmanage import settings diff --git a/lndmanage/lib/types.py b/lndmanage/lib/data_types.py similarity index 62% rename from lndmanage/lib/types.py rename to lndmanage/lib/data_types.py index 50a3c7e..5337ba5 100644 --- a/lndmanage/lib/types.py +++ b/lndmanage/lib/data_types.py @@ -21,11 +21,28 @@ class UTXO: def __hash__(self): return hash(self.txid) + hash(self.output_index) - def __eq__(self, other: 'UTXO'): + def __eq__(self, other: "UTXO"): if self.txid == other.txid and self.output_index == other.output_index: return True else: return False def __str__(self): - return f"{self.txid}:{self.output_index} {self.amount_sat} sat" \ No newline at end of file + return f"{self.txid}:{self.output_index} {self.amount_sat} sat" + + +@dataclass(order=True) +class NodeProperties: + age: int + local_fee_rates: list + local_base_fees: list + local_balances: list + number_active_channels: int + number_channels: int + number_private_channels: int + public_capacities: list + private_capacites: list + remote_fee_rates: list + remote_base_fees: list + remote_balances: list + sent_received_per_week: int diff --git a/lndmanage/lib/fee_setting.py b/lndmanage/lib/fee_setting.py index a2ff552..583af4b 100644 --- a/lndmanage/lib/fee_setting.py +++ b/lndmanage/lib/fee_setting.py @@ -131,7 +131,7 @@ class FeeSetter(object): self.time_end = time.time() self.time_start = self.time_end - from_days_ago * 24 * 60 * 60 self.time_interval_days = from_days_ago - self.forwarding_analyzer.initialize_forwarding_data( + self.forwarding_analyzer.initialize_forwarding_stats( self.time_start, self.time_end ) diff --git a/lndmanage/lib/forwardings.py b/lndmanage/lib/forwardings.py index ee55346..28acddd 100644 --- a/lndmanage/lib/forwardings.py +++ b/lndmanage/lib/forwardings.py @@ -1,15 +1,19 @@ -import logging +"""Module for gathering statistics of channels or nodes.""" from collections import OrderedDict, defaultdict +import logging +from typing import Dict import numpy as np +from lndmanage.lib.data_types import NodeProperties from lndmanage.lib.node import LndNode +from lndmanage.lib.ln_utilities import channel_unbalancedness_and_commit_fee from lndmanage import settings logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) -np.warnings.filterwarnings('ignore') +np.warnings.filterwarnings("ignore") # nearest neighbor weight for flow-analysis ~1/(avg. degree) NEIGHBOR_WEIGHT = 0.1 @@ -17,12 +21,7 @@ NEIGHBOR_WEIGHT = 0.1 NEXT_NEIGHBOR_WEIGHT = 0.02 -def nan_to_zero(number): - """ - Converts float('nan') to 0 - :param number: float - :return: float - """ +def nan_to_zero(number: float) -> float: if number is np.nan or number != number: return 0.0 else: @@ -30,167 +29,163 @@ def nan_to_zero(number): class ForwardingAnalyzer(object): - """ - Analyzes forwardings for single channels. - """ - def __init__(self, node): + """Analyzes forwardings for single channels.""" + + def __init__(self, node: "LndNode"): self.node = node self.forwarding_events = self.node.get_forwarding_events() - self.channels = {} - self.total_forwarding_amount_sat = 0 - self.total_forwarding_fees_msat = 0 - self.forwardings = 0 - self.cumulative_effective_fee = 0 - self.timestamp_first_send = 1E10 # somewhere in future - self.timestamp_last_send = 0 # at beginning of time - self.max_time_interval = None + self.channel_forwarding_stats = {} # type: Dict[str, ForwardingStatistics] + self.node_forwarding_stats = {} # type: Dict[str, ForwardingStatistics] + self.max_time_interval_days = None - def initialize_forwarding_data(self, time_start, time_end): - """ - Initializes the channel statistics objects with data from - the forwardings. + def initialize_forwarding_stats(self, time_start: float, time_end: float): + """Initializes the channel and node statistics objects with data from forwardings. :param time_start: time interval start, unix timestamp :param time_end: time interval end, unix timestamp """ + channel_id_to_node_id = self.node.get_channel_id_to_node_id() + self.channel_forwarding_stats = defaultdict(ForwardingStatistics) + self.node_forwarding_stats = defaultdict(ForwardingStatistics) + + min_timestamp = float("inf") + max_timestamp = 0 for f in self.forwarding_events: - if time_start < f['timestamp'] < time_end: + if time_start < f["timestamp"] < time_end: + # find min and max range of forardings + if f["timestamp"] > max_timestamp: + max_timestamp = f["timestamp"] + if f["timestamp"] < min_timestamp: + min_timestamp = f["timestamp"] # make a dictionary entry for unknown channels - channel_id_in = f['chan_id_in'] - channel_id_out = f['chan_id_out'] + channel_id_in = f["chan_id_in"] + channel_id_out = f["chan_id_out"] + node_id_in = channel_id_to_node_id.get(channel_id_in) + node_id_out = channel_id_to_node_id.get(channel_id_out) - if channel_id_in not in self.channels.keys(): - self.channels[channel_id_in] = ChannelStatistics( - channel_id_in) - if channel_id_out not in self.channels.keys(): - self.channels[channel_id_out] = ChannelStatistics( - channel_id_out) + # node statistics + if node_id_in: + self.node_forwarding_stats[node_id_in].inward_forwardings.append( + f["amt_in"] + ) + if node_id_out: + self.node_forwarding_stats[node_id_out].outward_forwardings.append( + f["amt_out"] + ) + self.node_forwarding_stats[node_id_out].absolute_fees.append( + f["fee_msat"] + ) + self.node_forwarding_stats[node_id_out].effective_fees.append( + f["effective_fee"] + ) + self.node_forwarding_stats[node_id_out].timestamps.append( + f["timestamp"] + ) - self.channels[channel_id_in].inward_forwardings.append( - f['amt_in']) - self.channels[channel_id_out].outward_forwardings.append( - f['amt_out']) - self.channels[channel_id_out].absolute_fees.append( - f['fee_msat']) - self.channels[channel_id_out].effective_fees.append( - f['effective_fee']) - self.channels[channel_id_out].timestamps.append( - f['timestamp']) - - self.total_forwarding_amount_sat += f['amt_in'] - self.total_forwarding_fees_msat += f['fee_msat'] - self.forwardings += 1 - self.cumulative_effective_fee += f['effective_fee'] - - def get_forwarding_statistics_channels(self): - """ - Prepares the forwarding statistics for each channel. - :return: dict: statistics with channel_id as keys - """ - channel_statistics = {} - - for k, c in self.channels.items(): - - try: - timestamp_first_send = min(c.timestamps) - if self.timestamp_first_send > timestamp_first_send: - self.timestamp_first_send = timestamp_first_send - except ValueError: - pass - try: - timestamp_last_send = max(c.timestamps) - if self.timestamp_last_send < timestamp_last_send: - self.timestamp_last_send = timestamp_last_send - except ValueError: - pass - - channel_statistics[k] = { - 'effective_fee': c.effective_fee(), - 'fees_total': c.fees_total(), - 'flow_direction': c.flow_direction(), - 'mean_forwarding_in': c.mean_forwarding_in(), - 'mean_forwarding_out': c.mean_forwarding_out(), - 'median_forwarding_in': c.median_forwarding_in(), - 'median_forwarding_out': c.median_forwarding_out(), - 'number_forwardings': c.number_forwardings(), - 'number_forwardings_out': c.number_forwardings_out(), - 'largest_forwarding_amount_in': - c.largest_forwarding_amount_in(), - 'largest_forwarding_amount_out': - c.largest_forwarding_amount_out(), - 'total_forwarding_in': c.total_forwarding_in(), - 'total_forwarding_out': c.total_forwarding_out(), - } + # channel statistics + self.channel_forwarding_stats[channel_id_in].inward_forwardings.append( + f["amt_in"] + ) + self.channel_forwarding_stats[ + channel_id_out + ].outward_forwardings.append(f["amt_out"]) + self.channel_forwarding_stats[channel_id_out].absolute_fees.append( + f["fee_msat"] + ) + self.channel_forwarding_stats[channel_id_out].effective_fees.append( + f["effective_fee"] + ) + self.channel_forwarding_stats[channel_id_out].timestamps.append( + f["timestamp"] + ) # determine the time interval starting with the first forwarding # to the last forwarding in the analyzed time interval determined # by time_start and time_end - self.max_time_interval = \ - (self.timestamp_last_send - self.timestamp_first_send) \ - / (24 * 60 * 60) + self.max_time_interval_days = (max_timestamp - min_timestamp) / (24 * 60 * 60) + def get_forwarding_statistics_channels(self): + """Prepares the forwarding statistics for each channel. + + :return: dict: statistics with channel_id as keys + """ + channel_statistics = {} + + for k, c in self.channel_forwarding_stats.items(): + channel_statistics[k] = { + "effective_fee": c.effective_fee(), + "fees_total": c.fees_total(), + "flow_direction": c.flow_direction(), + "mean_forwarding_in": c.mean_forwarding_in(), + "mean_forwarding_out": c.mean_forwarding_out(), + "median_forwarding_in": c.median_forwarding_in(), + "median_forwarding_out": c.median_forwarding_out(), + "number_forwardings": c.number_forwardings(), + "number_forwardings_out": c.number_forwardings_out(), + "largest_forwarding_amount_in": c.largest_forwarding_amount_in(), + "largest_forwarding_amount_out": c.largest_forwarding_amount_out(), + "total_forwarding_in": c.total_forwarding_in(), + "total_forwarding_out": c.total_forwarding_out(), + } return channel_statistics - def get_forwarding_statistics_nodes(self, sort_by='total_forwarding'): - """ - Calculates forwarding statistics based on single channels for + def get_forwarding_statistics_nodes(self, sort_by="total_forwarding"): + """Calculates forwarding statistics based on individual channels for their nodes. :param sort_by: str, abbreviation for the dict key :return: """ - channel_statistics = self.get_forwarding_statistics_channels() closed_channels = self.node.get_closed_channels() open_channels = self.node.get_open_channels() - logger.debug(f"Number of channels with known forwardings: " - f"{len(closed_channels) + len(open_channels)} " - f"(thereof {len(closed_channels)} closed channels).") + logger.debug( + f"Number of channels with known forwardings: " + f"{len(closed_channels) + len(open_channels)} " + f"(thereof {len(closed_channels)} closed channels)." + ) - node_statistics = OrderedDict() - - # go through channel statistics and calculate node statistics - for k, n in channel_statistics.items(): - # historic node data can be outdated, so we need to take care - # that the remote pub key is known, - # otherwise it is useless information - - channel_data = open_channels.get(k, None) - if not channel_data: - channel_data = closed_channels.get(k, None) - - if channel_data: - remote_pubkey = channel_data['remote_pubkey'] - if remote_pubkey not in node_statistics.keys(): - node_statistics[remote_pubkey] = { - 'total_forwarding_in': n['total_forwarding_in'], - 'total_forwarding_out': n['total_forwarding_out'], - 'total_forwarding': - n['total_forwarding_in'] + - n['total_forwarding_out'], - } - else: - node_statistics[remote_pubkey]['total_forwarding_in'] \ - += n['total_forwarding_in'] - node_statistics[remote_pubkey]['total_forwarding_out'] \ - += n['total_forwarding_out'] - node_statistics[remote_pubkey]['total_forwarding'] \ - += (n['total_forwarding_in'] + - n['total_forwarding_out']) - - for k, n in node_statistics.items(): - tot_in = n['total_forwarding_in'] - tot_out = n['total_forwarding_out'] - node_statistics[k]['flow_direction'] = \ - (- tot_in + tot_out) / (tot_in + tot_out) + forwarding_stats = defaultdict(dict) + for nid, node_stats in self.node_forwarding_stats.items(): + tot_in = node_stats.total_forwarding_in() + tot_out = node_stats.total_forwarding_out() + forwarding_stats[nid]["effective_fee"] = node_stats.effective_fee() + forwarding_stats[nid]["fees_total"] = node_stats.fees_total() + forwarding_stats[nid]["flow_direction"] = ( + -((float(tot_in) / (tot_in + tot_out)) - 0.5) / 0.5 + ) + forwarding_stats[nid][ + "largest_forwarding_amount_in" + ] = node_stats.largest_forwarding_amount_in() + forwarding_stats[nid][ + "largest_forwarding_amount_out" + ] = node_stats.largest_forwarding_amount_out() + forwarding_stats[nid][ + "mean_forwarding_in" + ] = node_stats.mean_forwarding_in() + forwarding_stats[nid][ + "mean_forwarding_out" + ] = node_stats.mean_forwarding_out() + forwarding_stats[nid][ + "median_forwarding_in" + ] = node_stats.median_forwarding_in() + forwarding_stats[nid][ + "median_forwarding_out" + ] = node_stats.median_forwarding_out() + forwarding_stats[nid][ + "number_forwardings" + ] = node_stats.number_forwardings() + forwarding_stats[nid]["total_forwarding_in"] = tot_in + forwarding_stats[nid]["total_forwarding_out"] = tot_out + forwarding_stats[nid]["total_forwarding"] = tot_in + tot_out sorted_dict = OrderedDict( - sorted(node_statistics.items(), key=lambda x: -x[1][sort_by])) + sorted(forwarding_stats.items(), key=lambda x: -x[1][sort_by]) + ) return sorted_dict def simple_flow_analysis(self, last_forwardings_to_analyze=100): - """ - Takes each forwarding event and determines the set of incoming nodes + """Takes each forwarding event and determines the set of incoming nodes (up to second nearest neighbors) and outgoing nodes (up to second nearest neighbors) and does a frequency analysis of both sets, assigning a probability that a certain node was involved in @@ -207,8 +202,7 @@ class ForwardingAnalyzer(object): number of last forwardings to be analyzed :type last_forwardings_to_analyze: int - :return: sending list, receiving list - """ + :return: sending list, receiving list""" # TODO: refine with channel capacities # TODO: refine with update time # TODO: refine with route calculation @@ -220,65 +214,76 @@ class ForwardingAnalyzer(object): total_incoming_neighbors = defaultdict(float) total_outgoing_neighbors = defaultdict(float) - logger.info(f"Total forwarding events found: " - f"{len(self.forwarding_events)}.") - logger.info(f"Carrying out flow analysis for last " - f"{last_forwardings_to_analyze} forwarding events.") + logger.info( + f"Total forwarding events found: " f"{len(self.forwarding_events)}." + ) + logger.info( + f"Carrying out flow analysis for last " + f"{last_forwardings_to_analyze} forwarding events." + ) number_progress_report = last_forwardings_to_analyze // 10 meaningful_outward_forwardings = 0 - for nf, f in enumerate( - self.forwarding_events[-last_forwardings_to_analyze:]): + for nf, f in enumerate(self.forwarding_events[-last_forwardings_to_analyze:]): # report progress if nf % number_progress_report == 0: logger.info( f"Analysis progress: " - f"{100 * float(nf) / last_forwardings_to_analyze}%") + f"{100 * float(nf) / last_forwardings_to_analyze}%" + ) - chan_id_in = f['chan_id_in'] - chan_id_out = f['chan_id_out'] + chan_id_in = f["chan_id_in"] + chan_id_out = f["chan_id_out"] edge_data_in = self.node.network.edges.get(chan_id_in, None) edge_data_out = self.node.network.edges.get(chan_id_out, None) if edge_data_in is not None and edge_data_out is not None: # determine incoming and outgoing node pub keys - incoming_node_pub_key = edge_data_in['node1_pub'] \ - if edge_data_in['node1_pub'] != self.node.pub_key \ - else edge_data_in['node2_pub'] - outgoing_node_pub_key = edge_data_out['node1_pub'] \ - if edge_data_out['node1_pub'] != self.node.pub_key \ - else edge_data_out['node2_pub'] + incoming_node_pub_key = ( + edge_data_in["node1_pub"] + if edge_data_in["node1_pub"] != self.node.pub_key + else edge_data_in["node2_pub"] + ) + outgoing_node_pub_key = ( + edge_data_out["node1_pub"] + if edge_data_out["node1_pub"] != self.node.pub_key + else edge_data_out["node2_pub"] + ) # nodes involved in the forwarding process should be excluded - excluded_nodes = [self.node.pub_key, incoming_node_pub_key, - outgoing_node_pub_key] + excluded_nodes = [ + self.node.pub_key, + incoming_node_pub_key, + outgoing_node_pub_key, + ] # determine all the nearest and second nearest # neighbors of the incoming/outgoing nodes, # they may appear more than once incoming_neighbors = self.__determine_joined_neighbors( - incoming_node_pub_key, excluded_nodes=excluded_nodes) + incoming_node_pub_key, excluded_nodes=excluded_nodes + ) outgoing_neighbors = self.__determine_joined_neighbors( - outgoing_node_pub_key, excluded_nodes=excluded_nodes) + outgoing_node_pub_key, excluded_nodes=excluded_nodes + ) # do a symmetric difference of node sets with weights symmetric_difference_weights = self.__symmetric_difference( - incoming_neighbors, outgoing_neighbors) + incoming_neighbors, outgoing_neighbors + ) final_outgoing_nodes = self.__filter_nodes( - symmetric_difference_weights, - return_positive_weights=True) + symmetric_difference_weights, return_positive_weights=True + ) final_incoming_nodes = self.__filter_nodes( - symmetric_difference_weights, - return_positive_weights=False) + symmetric_difference_weights, return_positive_weights=False + ) # normalize the weights - normalized_incoming = self.__normalize_neighbors( - final_incoming_nodes) - normalized_outgoing = self.__normalize_neighbors( - final_outgoing_nodes) + normalized_incoming = self.__normalize_neighbors(final_incoming_nodes) + normalized_outgoing = self.__normalize_neighbors(final_outgoing_nodes) # set weight for each forwarding event weight = 1 @@ -296,21 +301,26 @@ class ForwardingAnalyzer(object): # testing whether the forwarding amount in msat has remainder # zero when divided by 1000 or not, provided the sent amount # was larger than 1E6 msat. - if (f['amt_out_msat'] % 1000): + if f["amt_out_msat"] % 1000: meaningful_outward_forwardings += 1 logger.debug( f"Forwarding was not last hop: {f['amt_out_msat']}, " - f"chan_id_out: {chan_id_out}") + f"chan_id_out: {chan_id_out}" + ) for n, nv in normalized_outgoing.items(): total_outgoing_neighbors[n] += nv * weight - logger.info(f"Could use {meaningful_outward_forwardings} " - f"forwardings to estimate targets of payments.") + logger.info( + f"Could use {meaningful_outward_forwardings} " + f"forwardings to estimate targets of payments." + ) # sort according to weights total_incoming_node_dict = self.__weighted_neighbors_to_sorted_dict( - total_incoming_neighbors) + total_incoming_neighbors + ) total_outgoing_node_dict = self.__weighted_neighbors_to_sorted_dict( - total_outgoing_neighbors) + total_outgoing_neighbors + ) return total_incoming_node_dict, total_outgoing_node_dict @@ -327,12 +337,11 @@ class ForwardingAnalyzer(object): node_list = [(n, nv) for n, nv in node_dict.items()] node_list_sorted = sorted(node_list, key=lambda x: x[1], reverse=True) for n, nv in node_list_sorted: - sorted_nodes_dict[n] = {'weight': nv} + sorted_nodes_dict[n] = {"weight": nv} return sorted_nodes_dict def __determine_joined_neighbors(self, node_pub_key, excluded_nodes): - """ - Determines the joined set of nearest and second neighbors and assigns + """Determines the joined set of nearest and second neighbors and assigns a weight to every node dependent how often they appear. :param node_pub_key: str, public key of the home node @@ -342,19 +351,20 @@ class ForwardingAnalyzer(object): """ neighbors = list(self.node.network.neighbors(node_pub_key)) - second_neighbors = list( - self.node.network.second_neighbors(node_pub_key)) + second_neighbors = list(self.node.network.second_neighbors(node_pub_key)) # determine neighbor node_weights neighbor_weights = self.__analyze_neighbors( - neighbors, excluded_nodes=excluded_nodes, weight=NEIGHBOR_WEIGHT) + neighbors, excluded_nodes=excluded_nodes, weight=NEIGHBOR_WEIGHT + ) second_neighbor_weights = self.__analyze_neighbors( - second_neighbors, excluded_nodes=excluded_nodes, - weight=NEXT_NEIGHBOR_WEIGHT) + second_neighbors, excluded_nodes=excluded_nodes, weight=NEXT_NEIGHBOR_WEIGHT + ) # combine nearest and second nearest neighbor node weights joined_neighbors = self.__join_neighbors( - neighbor_weights, second_neighbor_weights) + neighbor_weights, second_neighbor_weights + ) return joined_neighbors @@ -403,8 +413,7 @@ class ForwardingAnalyzer(object): @staticmethod def __join_neighbors(first_neighbor_dict, second_neighbor_dict): - """ - Joins two node weight dicts together. + """Joins two node weight dicts together. :param first_neighbor_dict: dict, keys: node_pub_keys, values: node weights :param second_neighbor_dict: dict, keys: node_pub_keys, @@ -418,7 +427,8 @@ class ForwardingAnalyzer(object): for n, v in second_neighbor_dict.items(): if n in joined_neighbor_dict: joined_neighbor_dict[n] = min( - 1, joined_neighbor_dict[n] + second_neighbor_dict[n]) + 1, joined_neighbor_dict[n] + second_neighbor_dict[n] + ) else: joined_neighbor_dict[n] = second_neighbor_dict[n] @@ -479,132 +489,265 @@ class ForwardingAnalyzer(object): return new_node_weights -class ChannelStatistics(object): - """ - Functionality to analyze the forwardings of a single channel. - """ - def __init__(self, channel_id): - self.channel_id = channel_id +class ForwardingStatistics(object): + """Functionality to analyze the forwardings of a single node/channel.""" + def __init__(self): self.inward_forwardings = [] self.outward_forwardings = [] - self.timestamps = [] self.absolute_fees = [] self.effective_fees = [] - def total_forwarding_in(self): + def total_forwarding_in(self) -> int: return sum(self.inward_forwardings) - def total_forwarding_out(self): + def total_forwarding_out(self) -> int: return sum(self.outward_forwardings) - def mean_forwarding_in(self): + def mean_forwarding_in(self) -> float: return np.mean(self.inward_forwardings) - def mean_forwarding_out(self): + def mean_forwarding_out(self) -> float: return np.mean(self.outward_forwardings) - def median_forwarding_in(self): + def median_forwarding_in(self) -> float: return np.median(self.inward_forwardings) - def median_forwarding_out(self): + def median_forwarding_out(self) -> float: return np.median(self.outward_forwardings) - def fees_total(self): + def fees_total(self) -> int: return sum(self.absolute_fees) - def effective_fee(self): + def effective_fee(self) -> float: return np.mean(self.effective_fees) - def largest_forwarding_amount_out(self): - return max(self.outward_forwardings, default=float('nan')) + def largest_forwarding_amount_out(self) -> int: + return max(self.outward_forwardings, default=float("nan")) - def largest_forwarding_amount_in(self): - return max(self.inward_forwardings, default=float('nan')) + def largest_forwarding_amount_in(self) -> int: + return max(self.inward_forwardings, default=float("nan")) - def flow_direction(self): + def flow_direction(self) -> float: total_in = self.total_forwarding_in() total_out = self.total_forwarding_out() try: - return (- total_in + total_out) / (total_in + total_out) + return (-total_in + total_out) / (total_in + total_out) except ZeroDivisionError: return 0 - def number_forwardings(self): + def number_forwardings(self) -> int: return len(self.inward_forwardings) + len(self.outward_forwardings) def number_forwardings_out(self): return len(self.outward_forwardings) -def get_forwarding_statistics_channels(node, time_interval_start, - time_interval_end): - """ - Joins data from listchannels and fwdinghistory to have a extended - information about a channel. - :param node: :class:`lib.node.Node` - :param time_interval_start: unix timestamp - :param time_interval_end: unix timestamp +def get_node_properites( + node: LndNode, time_interval_start: float, time_interval_end: float +) -> Dict: + """Joins data from channels and fwdinghistory to have extended + information about a node. + + :return: dict of node information with channel_id as keys + """ + forwarding_analyzer = ForwardingAnalyzer(node) + forwarding_analyzer.initialize_forwarding_stats( + time_interval_start, time_interval_end + ) + node_forwarding_statistics = forwarding_analyzer.get_forwarding_statistics_nodes() + logger.debug( + f"Time interval (between first and last forwarding) is " + f"{forwarding_analyzer.max_time_interval_days:6.2f} days." + ) + channel_id_to_node_id = node.get_channel_id_to_node_id(open_only=True) + node_ids_with_open_channels = {nid for nid in channel_id_to_node_id.values()} + open_channels = node.get_open_channels() + + nodes_properties = {} # type: Dict[str, NodeProperties] + + # for each channel, accumulate properties in node properties + for k, c in open_channels.items(): + remote_pubkey = c["remote_pubkey"] + try: + properties = nodes_properties[remote_pubkey] + except KeyError: + nodes_properties[remote_pubkey] = NodeProperties( + age=c["age"], + local_fee_rates=[c["local_fee_rate"]], + local_base_fees=[c["local_base_fee"]], + local_balances=[c["local_balance"]], + number_active_channels=1 if c["active"] else 0, + number_channels=1, + number_private_channels=1 if c["private"] else 0, + remote_fee_rates=[c["peer_fee_rate"]], + remote_base_fees=[c["peer_base_fee"]], + remote_balances=[c["remote_balance"]], + sent_received_per_week=c["sent_received_per_week"], + public_capacities=[c["capacity"]] if not c["private"] else [], + private_capacites=[c["capacity"]] if c["private"] else [], + ) + else: + properties.age = max(c["age"], nodes_properties[c["remote_pubkey"]].age) + properties.local_fee_rates.append(c["local_fee_rate"]) + properties.local_base_fees.append(c["local_base_fee"]) + properties.local_balances.append(c["local_balance"]) + properties.number_active_channels += 1 if c["active"] else 0 + properties.number_channels += 1 + properties.number_private_channels += 1 if c["private"] else 0 + properties.remote_fee_rates.append(c["peer_fee_rate"]) + properties.remote_base_fees.append(c["peer_base_fee"]) + properties.remote_balances.append(c["remote_balance"]) + properties.sent_received_per_week += c["sent_received_per_week"] + if not c["private"]: + properties.public_capacities.append(c["capacity"]) + else: + properties.private_capacites.append(c["capacity"]) + + # unify node properties with forwarding data + node_properties_forwardings = {} + # we start with looping over node properties, as this info is complete + for node_id, properties in nodes_properties.items(): + local_balance = sum(properties.local_balances) + remote_balance = sum(properties.remote_balances) + capacity = sum(properties.private_capacites) + sum(properties.public_capacities) + # there can be old forwarding data, which we neglect + if node_id not in node_ids_with_open_channels: + continue + + # initial data: + node_properties_forwardings[node_id] = { + "age": properties.age, + "alias": node.network.node_alias(node_id), + "local_base_fee": np.median(properties.local_base_fees), + "local_fee_rate": np.median(properties.local_fee_rates), + "local_balance": local_balance, + "max_local_balance": max(properties.local_balances), + "max_remote_balance": max(properties.remote_balances), + "number_channels": properties.number_channels, + "number_active_channels": properties.number_active_channels, + "number_private_channels": properties.number_private_channels, + "node_id": node_id, + "remote_base_fee": np.median(properties.remote_base_fees), + "remote_fee_rate": np.median(properties.remote_fee_rates), + "remote_balance": remote_balance, + "sent_reveived_per_week": properties.sent_received_per_week, + "total_capacity": capacity, + "max_public_capacity": max(properties.public_capacities) + if properties.public_capacities + else 0, + "unbalancedness": channel_unbalancedness_and_commit_fee( + local_balance, capacity, 0, False + )[0], + } + + # add forwarding data if available: + try: + statistics = node_forwarding_statistics[node_id] + except KeyError: # we don't have forwarding data, populate with defaults + node_properties_forwardings[node_id].update( + { + "effective_fee": float("nan"), + "fees_total": 0, + "flow_direction": float("nan"), + "largest_forwarding_amount_in": float("nan"), + "largest_forwarding_amount_out": float("nan"), + "median_forwarding_out": float("nan"), + "median_forwarding_in": float("nan"), + "mean_forwarding_out": float("nan"), + "mean_forwarding_in": float("nan"), + "number_forwardings": 0, + "total_forwarding_in": 0, + "total_forwarding_out": 0, + "total_forwarding": 0, + } + ) + else: + node_properties_forwardings[node_id].update(**statistics) + + try: + node_properties_forwardings[node_id][ + "fees_total_per_week" + ] = node_properties_forwardings[node_id]["fees_total"] / ( + forwarding_analyzer.max_time_interval_days / 7 + ) + except ZeroDivisionError: + node_properties_forwardings[node_id]["fees_total_per_week"] = float("nan") + + return node_properties_forwardings + + +def get_channel_properties( + node: LndNode, time_interval_start: float, time_interval_end: float +) -> Dict: + """Joins data from listchannels and fwdinghistory to have extended + information about channels. + :return: dict of channel information with channel_id as keys """ forwarding_analyzer = ForwardingAnalyzer(node) - forwarding_analyzer.initialize_forwarding_data( - time_interval_start, time_interval_end) + forwarding_analyzer.initialize_forwarding_stats( + time_interval_start, time_interval_end + ) # dict with channel_id keys statistics = forwarding_analyzer.get_forwarding_statistics_channels() - logger.debug(f"Time interval (between first and last forwarding) is " - f"{forwarding_analyzer.max_time_interval:6.2f} days.") + logger.debug( + f"Time interval (between first and last forwarding) is " + f"{forwarding_analyzer.max_time_interval_days:6.2f} days." + ) # join the two data sets: channels = node.get_unbalanced_channels(unbalancedness_greater_than=0.0) for k, c in channels.items(): # we may not have forwarding data for every channel - chan_stats = statistics.get(c['chan_id'], {}) - c['forwardings_per_channel_age'] = chan_stats.get('number_forwardings', 0.01) / c['age'] - c['bandwidth_demand'] = max( - nan_to_zero(chan_stats.get('mean_forwarding_in', 0)), - nan_to_zero(chan_stats.get('mean_forwarding_out', 0)) - ) / c['capacity'] - c['fees_total'] = chan_stats.get('fees_total', 0) + chan_stats = statistics.get(c["chan_id"], {}) + c["forwardings_per_channel_age"] = ( + chan_stats.get("number_forwardings", 0.01) / c["age"] + ) + c["bandwidth_demand"] = ( + max( + nan_to_zero(chan_stats.get("mean_forwarding_in", 0)), + nan_to_zero(chan_stats.get("mean_forwarding_out", 0)), + ) + / c["capacity"] + ) + c["fees_total"] = chan_stats.get("fees_total", 0) try: - c['fees_total_per_week'] = chan_stats.get('fees_total', 0) \ - / (forwarding_analyzer.max_time_interval / 7) + c["fees_total_per_week"] = chan_stats.get("fees_total", 0) / ( + forwarding_analyzer.max_time_interval_days / 7 + ) except ZeroDivisionError: - c['fees_total_per_week'] = float('nan') - c['flow_direction'] = chan_stats.get('flow_direction', float('nan')) - c['median_forwarding_in'] = chan_stats.get('median_forwarding_in', float('nan')) - c['median_forwarding_out'] = chan_stats.get('median_forwarding_out', float('nan')) - c['mean_forwarding_in'] = chan_stats.get('mean_forwarding_in', float('nan')) - c['mean_forwarding_out'] = chan_stats.get('mean_forwarding_out', float('nan')) - c['number_forwardings'] = chan_stats.get('number_forwardings', 0) - c['largest_forwarding_amount_in'] = chan_stats.get('largest_forwarding_amount_in', float('nan')) - c['largest_forwarding_amount_out'] = chan_stats.get('largest_forwarding_amount_out', float('nan')) - c['total_forwarding_in'] = chan_stats.get('total_forwarding_in', 0) - c['total_forwarding_out'] = chan_stats.get('total_forwarding_out', 0) + c["fees_total_per_week"] = float("nan") + c["flow_direction"] = chan_stats.get("flow_direction", float("nan")) + c["median_forwarding_in"] = chan_stats.get("median_forwarding_in", float("nan")) + c["median_forwarding_out"] = chan_stats.get( + "median_forwarding_out", float("nan") + ) + c["mean_forwarding_in"] = chan_stats.get("mean_forwarding_in", float("nan")) + c["mean_forwarding_out"] = chan_stats.get("mean_forwarding_out", float("nan")) + c["number_forwardings"] = chan_stats.get("number_forwardings", 0) + c["largest_forwarding_amount_in"] = chan_stats.get( + "largest_forwarding_amount_in", float("nan") + ) + c["largest_forwarding_amount_out"] = chan_stats.get( + "largest_forwarding_amount_out", float("nan") + ) + c["total_forwarding_in"] = chan_stats.get("total_forwarding_in", 0) + c["total_forwarding_out"] = chan_stats.get("total_forwarding_out", 0) # action required if flow same direction as unbalancedness # or bandwidth demand too high # TODO: refine 'action_required' by better metric - if c['unbalancedness'] * c['flow_direction'] > 0 and abs( - c['unbalancedness']) > settings.UNBALANCED_CHANNEL: - c['action_required'] = True + if ( + c["unbalancedness"] * c["flow_direction"] > 0 + and abs(c["unbalancedness"]) > settings.UNBALANCED_CHANNEL + ): + c["action_required"] = True else: - c['action_required'] = False - if c['bandwidth_demand'] > 0.5: - c['action_required'] = True + c["action_required"] = False + if c["bandwidth_demand"] > 0.5: + c["action_required"] = True - return channels - - -if __name__ == '__main__': - import time - import logging.config - logging.config.dictConfig(settings.logger_config) - logger = logging.getLogger() - - nd = LndNode() - fa = ForwardingAnalyzer(nd) - fa.initialize_forwarding_data(time_start=0, time_end=time.time()) - print(fa.simple_flow_analysis()) \ No newline at end of file + return channels \ No newline at end of file diff --git a/lndmanage/lib/listchannels.py b/lndmanage/lib/listchannels.py deleted file mode 100644 index e765021..0000000 --- a/lndmanage/lib/listchannels.py +++ /dev/null @@ -1,598 +0,0 @@ -""" -Module for printing lightning channels. -""" - -import math -import logging -import time -from collections import OrderedDict - -from lndmanage.lib.forwardings import get_forwarding_statistics_channels -from lndmanage import settings - -logger = logging.getLogger(__name__) -logger.addHandler(logging.NullHandler()) - -# define symbols for bool to string conversion -POSITIVE_MARKER = "\u2713" -NEGATIVE_MARKER = "\u2717" -ALIAS_LENGTH = 25 -ANNOTATION_LENGTH = 25 - - -# define printing abbreviations -# convert key can specify a function, which lets one do unit conversions -PRINT_CHANNELS_FORMAT = { - "act": { - "dict_key": "active", - "description": "channel is active", - "width": 3, - "format": "^3", - "align": ">", - "convert": lambda x: POSITIVE_MARKER if x else NEGATIVE_MARKER, - }, - "age": { - "dict_key": "age", - "description": "channel age [days]", - "width": 5, - "format": "5.0f", - "align": ">", - }, - "alias": { - "dict_key": "alias", - "description": "alias", - "width": ALIAS_LENGTH, - "format": ".<" + str(ALIAS_LENGTH), - "align": "^", - "convert": lambda x: alias_cutoff(x), - }, - "annotation": { - "dict_key": "annotation", - "description": "channel annotation", - "width": ANNOTATION_LENGTH, - "format": ".<" + str(ANNOTATION_LENGTH), - "align": "^", - "convert": lambda x: alias_cutoff(x), - }, - "atb": { - "dict_key": "amount_to_balanced", - "description": "amount to be balanced (local side) [sat]", - "width": 5, - "format": "5.0f", - "align": ">", - }, - "bwd": { - "dict_key": "bandwidth_demand", - "description": "bandwidth demand: capacity / max(mean_in, mean_out)", - "width": 5, - "format": "5.2f", - "align": ">", - }, - "cap": { - "dict_key": "capacity", - "description": "channel capacity [sat]", - "width": 9, - "format": "9d", - "align": ">", - }, - "cid": { - "dict_key": "chan_id", - "description": "channel id", - "width": 18, - "format": "", - "align": "^", - }, - "fees": { - "dict_key": "fees_total", - "description": "total fees [sat]", - "width": 7, - "format": "7.2f", - "align": ">", - "convert": lambda x: float(x) / 1000, - }, - "f/w": { - "dict_key": "fees_total_per_week", - "description": "total fees per week [sat / week]", - "width": 6, - "format": "6.2f", - "align": ">", - "convert": lambda x: float(x) / 1000, - }, - "nfwd/a": { - "dict_key": "forwardings_per_channel_age", - "description": "number of forwardings per channel age in forwarding interval [1 / days]", - "width": 6, - "format": "6.2f", - "align": ">", - }, - "flow": { - "dict_key": "flow_direction", - "description": "flow direction (positive is outwards)", - "width": 5, - "format": "5.2f", - "align": ">", - # 'convert': lambda x: '>'*int(nan_to_zero(x)*10/3.0) if x > 0 else - # '<'*(-int(nan_to_zero(x)*10/3.0)) - }, - "pbf": { - "dict_key": "peer_base_fee", - "description": "peer base fee [msat]", - "width": 5, - "format": "5.0f", - "align": ">", - }, - "pfr": { - "dict_key": "peer_fee_rate", - "description": "peer fee rate", - "width": 8, - "format": "1.6f", - "align": ">", - "convert": lambda x: x / 1e6, - }, - "lbf": { - "dict_key": "local_base_fee", - "description": "local base fee [msat]", - "width": 5, - "format": "5.0f", - "align": ">", - }, - "lfr": { - "dict_key": "local_fee_rate", - "description": "local fee rate", - "width": 8, - "format": "1.6f", - "align": ">", - "convert": lambda x: x / 1e6, - }, - "lup": { - "dict_key": "last_update", - "description": "last update time [days ago]", - "width": 5, - "format": "5.0f", - "align": ">", - }, - "lupp": { - "dict_key": "last_update_peer", - "description": "last update time by peer [days ago]", - "width": 5, - "format": "5.0f", - "align": ">", - }, - "lupl": { - "dict_key": "last_update_local", - "description": "last update time by local [days ago]", - "width": 5, - "format": "5.0f", - "align": ">", - }, - "lb": { - "dict_key": "local_balance", - "description": "local balance [sat]", - "width": 9, - "format": "9d", - "align": ">", - }, - "nfwd": { - "dict_key": "number_forwardings", - "description": "number of forwardings", - "width": 5, - "format": "5.0f", - "align": ">", - }, - "priv": { - "dict_key": "private", - "description": "channel is private", - "width": 5, - "format": "^5", - "align": ">", - "convert": lambda x: POSITIVE_MARKER if x else NEGATIVE_MARKER, - }, - "r": { - "dict_key": "action_required", - "description": "action is required", - "width": 1, - "format": "^1", - "align": ">", - "convert": lambda x: NEGATIVE_MARKER if x else "", - }, - "rb": { - "dict_key": "remote_balance", - "description": "remote balance [sat]", - "width": 9, - "format": "9d", - "align": ">", - }, - "in": { - "dict_key": "total_forwarding_in", - "description": "total forwarding inwards [sat]", - "width": 10, - "format": "10.0f", - "align": ">", - }, - "ini": { - "dict_key": "initiator", - "description": "true if we opened channel", - "width": 3, - "format": "^3", - "align": ">", - "convert": lambda x: POSITIVE_MARKER if x else NEGATIVE_MARKER, - }, - "tot": { - "dict_key": "total_forwarding", - "description": "total forwarding [sat]", - "width": 5, - "format": "5.0f", - "align": ">", - }, - "ub": { - "dict_key": "unbalancedness", - "description": "unbalancedness [-1 ... 1] (0 is 50:50 balanced)", - "width": 5, - "format": "5.2f", - "align": ">", - }, - "ulr": { - "dict_key": "uptime_lifetime_ratio", - "description": "ratio of uptime to lifetime of channel [0 ... 1]", - "width": 5, - "format": "5.2f", - "align": ">", - }, - "imed": { - "dict_key": "median_forwarding_in", - "description": "median forwarding inwards [sat]", - "width": 10, - "format": "10.0f", - "align": ">", - }, - "imean": { - "dict_key": "mean_forwarding_in", - "description": "mean forwarding inwards [sat]", - "width": 10, - "format": "10.0f", - "align": ">", - }, - "imax": { - "dict_key": "largest_forwarding_amount_in", - "description": "largest forwarding inwards [sat]", - "width": 10, - "format": "10.0f", - "align": ">", - }, - "omed": { - "dict_key": "median_forwarding_out", - "description": "median forwarding outwards [sat]", - "width": 10, - "format": "10.0f", - "align": ">", - }, - "omean": { - "dict_key": "mean_forwarding_out", - "description": "mean forwarding outwards [sat]", - "width": 10, - "format": "10.0f", - "align": ">", - }, - "omax": { - "dict_key": "largest_forwarding_amount_out", - "description": "largest forwarding outwards [sat]", - "width": 10, - "format": "10.0f", - "align": ">", - }, - "out": { - "dict_key": "total_forwarding_out", - "description": "total forwarding outwards [sat]", - "width": 10, - "format": "10.0f", - "align": ">", - }, - "sr/w": { - "dict_key": "sent_received_per_week", - "description": "sent and received per week [sat]", - "width": 9, - "format": "9d", - "align": ">", - }, -} - - -def alias_cutoff(alias): - """ - Cuts off the node alias at a certain length and removes unicode - characters from the node alias. - :param alias: str - :return: str - """ - if len(alias) > ALIAS_LENGTH: - return alias[: ALIAS_LENGTH - 3] + "..." - else: - return alias - - -class ListChannels(object): - """ - A class to list lightning channels. - """ - - def __init__(self, node): - """ - :param node: :class:`lib.node.Node` - """ - self.node = node - - def print_all_channels(self, sort_string="rev_alias"): - """ - Prints all active and inactive channels. - - :param sort_string: str - """ - - channels = self._add_channel_annotations(self.node.get_all_channels()) - - sort_string, reverse_sorting = self._sorting_order(sort_string) - sort_dict = { - "function": lambda x: ( - x[1][PRINT_CHANNELS_FORMAT["priv"]["dict_key"]], - x[1][sort_string], - ), - "string": sort_string, - "reverse": reverse_sorting, - } - - self._print_channels( - channels, - columns="cid,priv,act,ub,cap,lb,rb,lbf," "lfr,annotation,alias", - sort_dict=sort_dict, - ) - - def print_channels_unbalanced(self, unbalancedness, sort_string="rev_ub"): - """ - Prints unbalanced channels with - |unbalancedness(channel)| > unbalancedness. - - :param unbalancedness: float - :param sort_string: str - """ - - channels = self._add_channel_annotations( - self.node.get_unbalanced_channels(unbalancedness) - ) - - sort_string, reverse_sorting = self._sorting_order(sort_string) - sort_dict = { - "function": lambda x: x[1][sort_string], - "string": sort_string, - "reverse": reverse_sorting, - } - - self._print_channels( - channels, - columns="cid,ub,cap,lb,rb,pbf,pfr,annotation,alias", - sort_dict=sort_dict, - ) - - def print_channels_inactive(self, sort_string="lupp"): - """ - Prints all inactive channels. - - :param sort_string: str - """ - - channels = self._add_channel_annotations(self.node.get_inactive_channels()) - - sort_string, reverse_sorting = self._sorting_order(sort_string) - sort_dict = { - "function": lambda x: (-x[1]["private"], x[1][sort_string]), - "string": sort_string, - "reverse": reverse_sorting, - } - - self._print_channels( - channels, - columns="cid,lupp,ulr,priv,ini,age,ub,cap,lb,rb," "sr/w,annotation,alias", - sort_dict=sort_dict, - ) - - def print_channels_forwardings( - self, time_interval_start, time_interval_end, sort_string - ): - - """ - Prints forwarding statistics for each channel. - - :param time_interval_start: int - :param time_interval_end: int - :param sort_string: str - """ - - channels = get_forwarding_statistics_channels( - self.node, time_interval_start, time_interval_end - ) - - channels = self._add_channel_annotations(channels) - - sort_string, reverse_sorting = self._sorting_order(sort_string) - sort_dict = { - "function": lambda x: ( - float("inf") if math.isnan(x[1][sort_string]) else x[1][sort_string], - x[1][PRINT_CHANNELS_FORMAT["nfwd"]["dict_key"]], - x[1][PRINT_CHANNELS_FORMAT["ub"]["dict_key"]], - ), - "string": sort_string, - "reverse": reverse_sorting, - } - - self._print_channels( - channels, - columns="cid,nfwd,age,fees,f/w,flow,ub,bwd,r," - "cap,pbf,pfr,annotation,alias", - sort_dict=sort_dict, - ) - - def print_channels_hygiene(self, time_interval_start, sort_string): - """ - Prints hygiene statistics for each channel. - - :param time_interval_start: int - :param time_interval_end: int - :param sort_string: str - """ - time_interval_end = time.time() - channels = get_forwarding_statistics_channels( - self.node, time_interval_start, time_interval_end - ) - - channels = self._add_channel_annotations(channels) - - sort_string, reverse_sorting = self._sorting_order(sort_string) - sort_dict = { - "function": lambda x: x[1][sort_string], - "string": sort_string, - "reverse": reverse_sorting, - } - - self._print_channels( - channels, - columns="cid,age,ini,nfwd/a,nfwd,f/w,ulr,lb,cap,lfr,pfr,annotation,alias", - sort_dict=sort_dict, - ) - - def _add_channel_annotations(self, channels): - """ - Appends metadata to existing channel dicts from the configuration file. - - :param channels: dict - :return: dict - """ - if self.node.config: - logger.debug("Adding annotations from file %s.", self.node.config_file) - # mapping between the channel point and channel id - channel_point_mapping = { - k: v["channel_point"].split(":")[0] for k, v in channels.items() - } - # only read annotations if config file is given - if self.node.config: - annotations = self.node.config["annotations"] - else: - annotations = {} - channel_annotations_funding_id = {} - channel_annotations_channel_id = {} - - for id, annotation in annotations.items(): - if len(id) == 18 and id.isnumeric(): - # valid channel id - channel_annotations_channel_id[int(id)] = annotation - elif len(id) == 64 and id.isalnum(): - # valid funding transaction id - channel_annotations_funding_id[id] = annotation - else: - raise ValueError( - "First part needs to be either a channel id or the " - "funding transaction id. \n" - "The funding transaction id can be found with " - "`lncli listchannels` under the channel point (the " - "characters before the colon)." - ) - - for channel_id, channel_values in channels.items(): - # get the annotation by channel id first - annotation = channel_annotations_channel_id.get(channel_id, None) - # if no channel annotation, try with funding id - if annotation is None: - annotation = channel_annotations_funding_id.get( - channel_point_mapping[channel_id], None - ) - - if annotation is not None: - channels[channel_id]["annotation"] = annotation - else: - channels[channel_id]["annotation"] = "" - - return channels - - def _print_channels(self, channels, columns, sort_dict): - """ - General purpose channel printing. - - :param channels: dict - :param columns: str - :param sort_dict: dict - """ - - if not channels: - logger.info(">>> Did not find any channels.") - - channels = OrderedDict( - sorted( - channels.items(), - key=sort_dict["function"], - reverse=sort_dict["reverse"], - ) - ) - - logger.info("Sorting channels by %s.", sort_dict["string"]) - - logger.info("-------- Description --------") - columns = columns.split(",") - for column in columns: - logger.info( - "%-10s %s", column, PRINT_CHANNELS_FORMAT[column]["description"] - ) - - logger.info("-------- Channels --------") - # prepare the column header - column_header = "" - for column in columns: - column_label = PRINT_CHANNELS_FORMAT[column]["align"] - column_width = PRINT_CHANNELS_FORMAT[column]["width"] - column_header += f"{column:{column_label}{column_width}} " - - # print the channel data - for channel_number, (_, channel_data) in enumerate(channels.items()): - if not channel_number % 20: - logger.info(column_header) - row = self._row_string(channel_data, columns) - logger.info(row) - - @staticmethod - def _row_string(column_values, columns): - """ - Constructs the formatted row string for table printing. - - :param column_values: dict - :param columns: list of str - :return: formatted str - """ - - string = "" - for column in columns: - format_string = PRINT_CHANNELS_FORMAT[column]["format"] - conversion_function = PRINT_CHANNELS_FORMAT[column].get( - "convert", lambda x: x - ) - value = column_values[PRINT_CHANNELS_FORMAT[column]["dict_key"]] - converted_value = conversion_function(value) - string += f"{converted_value:{format_string}} " - - return string - - @staticmethod - def _sorting_order(sort_string): - """ - Determines the sorting string and the sorting order. - - If sort_string starts with 'rev_', the sorting order is reversed. - - :param sort_string: str - :return: bool - """ - - reverse_sorting = True - if sort_string[:4] == "rev_": - reverse_sorting = False - sort_string = sort_string[4:] - - sort_string = PRINT_CHANNELS_FORMAT[sort_string]["dict_key"] - - return sort_string, reverse_sorting diff --git a/lndmanage/lib/listings.py b/lndmanage/lib/listings.py new file mode 100644 index 0000000..93d9abe --- /dev/null +++ b/lndmanage/lib/listings.py @@ -0,0 +1,866 @@ +""" +Module for printing lightning channels. +""" + +import math +import logging +from collections import OrderedDict +from typing import TYPE_CHECKING, Tuple, List, Dict + +if TYPE_CHECKING: + from lndmanage.lib.node import LndNode + +from lndmanage.lib.forwardings import ( + get_channel_properties, + get_node_properites, +) +from lndmanage import settings + +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + +# define symbols for bool to string conversion +POSITIVE_MARKER = "\u2713" +NEGATIVE_MARKER = "\u2717" +ALIAS_LENGTH = 25 +ANNOTATION_LENGTH = 25 + +PRINT_CHANNELS_FORMAT = { + "act": { + "dict_key": "active", + "description": "channel is active", + "width": 3, + "format": "^3", + "align": ">", + "convert": lambda x: POSITIVE_MARKER if x else NEGATIVE_MARKER, + }, + "age": { + "dict_key": "age", + "description": "channel age [days]", + "width": 5, + "format": "5.0f", + "align": ">", + }, + "alias": { + "dict_key": "alias", + "description": "alias", + "width": ALIAS_LENGTH, + "format": ".<" + str(ALIAS_LENGTH), + "align": "^", + "convert": lambda x: alias_cutoff(x), + }, + "annotation": { + "dict_key": "annotation", + "description": "channel annotation", + "width": ANNOTATION_LENGTH, + "format": ".<" + str(ANNOTATION_LENGTH), + "align": "^", + "convert": lambda x: alias_cutoff(x), + }, + "atb": { + "dict_key": "amount_to_balanced", + "description": "amount to be balanced (local side) [sat]", + "width": 5, + "format": "5.0f", + "align": ">", + }, + "bwd": { + "dict_key": "bandwidth_demand", + "description": "bandwidth demand: capacity / max(mean_in, mean_out)", + "width": 5, + "format": "5.2f", + "align": ">", + }, + "cap": { + "dict_key": "capacity", + "description": "channel capacity [sat]", + "width": 9, + "format": "9d", + "align": ">", + }, + "cid": { + "dict_key": "chan_id", + "description": "channel id", + "width": 18, + "format": "", + "align": "^", + }, + "fees": { + "dict_key": "fees_total", + "description": "total fees [sat]", + "width": 7, + "format": "7.2f", + "align": ">", + "convert": lambda x: float(x) / 1000, + }, + "f/w": { + "dict_key": "fees_total_per_week", + "description": "total fees per week [sat / week]", + "width": 6, + "format": "6.2f", + "align": ">", + "convert": lambda x: float(x) / 1000, + }, + "nfwd/a": { + "dict_key": "forwardings_per_channel_age", + "description": "number of forwardings per channel age in forwarding interval [1 / days]", + "width": 6, + "format": "6.2f", + "align": ">", + }, + "flow": { + "dict_key": "flow_direction", + "description": "flow direction (positive is outwards)", + "width": 5, + "format": "5.2f", + "align": ">", + # 'convert': lambda x: '>'*int(nan_to_zero(x)*10/3.0) if x > 0 else + # '<'*(-int(nan_to_zero(x)*10/3.0)) + }, + "pbf": { + "dict_key": "peer_base_fee", + "description": "peer base fee [msat]", + "width": 5, + "format": "5.0f", + "align": ">", + }, + "pfr": { + "dict_key": "peer_fee_rate", + "description": "peer fee rate", + "width": 8, + "format": "1.6f", + "align": ">", + "convert": lambda x: x / 1e6, + }, + "lbf": { + "dict_key": "local_base_fee", + "description": "local base fee [msat]", + "width": 5, + "format": "5.0f", + "align": ">", + }, + "lfr": { + "dict_key": "local_fee_rate", + "description": "local fee rate [sat/sat]", + "width": 8, + "format": "1.6f", + "align": ">", + "convert": lambda x: x / 1e6, + }, + "lup": { + "dict_key": "last_update", + "description": "last update time [days ago]", + "width": 5, + "format": "5.0f", + "align": ">", + }, + "lupp": { + "dict_key": "last_update_peer", + "description": "last update time by peer [days ago]", + "width": 5, + "format": "5.0f", + "align": ">", + }, + "lupl": { + "dict_key": "last_update_local", + "description": "last update time by local [days ago]", + "width": 5, + "format": "5.0f", + "align": ">", + }, + "lb": { + "dict_key": "local_balance", + "description": "local balance [sat]", + "width": 9, + "format": "9d", + "align": ">", + }, + "nfwd": { + "dict_key": "number_forwardings", + "description": "number of forwardings", + "width": 4, + "format": "4.0f", + "align": ">", + }, + "priv": { + "dict_key": "private", + "description": "channel is private", + "width": 5, + "format": "^5", + "align": ">", + "convert": lambda x: POSITIVE_MARKER if x else NEGATIVE_MARKER, + }, + "r": { + "dict_key": "action_required", + "description": "action is required", + "width": 1, + "format": "^1", + "align": ">", + "convert": lambda x: NEGATIVE_MARKER if x else "", + }, + "rb": { + "dict_key": "remote_balance", + "description": "remote balance [sat]", + "width": 9, + "format": "9d", + "align": ">", + }, + "in": { + "dict_key": "total_forwarding_in", + "description": "total forwarding inwards [sat]", + "width": 10, + "format": "10.0f", + "align": ">", + }, + "ini": { + "dict_key": "initiator", + "description": "true if we opened channel", + "width": 3, + "format": "^3", + "align": ">", + "convert": lambda x: POSITIVE_MARKER if x else NEGATIVE_MARKER, + }, + "tot": { + "dict_key": "total_forwarding", + "description": "total forwarding [sat]", + "width": 5, + "format": "5.0f", + "align": ">", + }, + "ub": { + "dict_key": "unbalancedness", + "description": "unbalancedness [-1 ... 1] (0 is 50:50 balanced)", + "width": 5, + "format": "5.2f", + "align": ">", + }, + "ulr": { + "dict_key": "uptime_lifetime_ratio", + "description": "ratio of uptime to lifetime of channel [0 ... 1]", + "width": 5, + "format": "5.2f", + "align": ">", + }, + "imed": { + "dict_key": "median_forwarding_in", + "description": "median forwarding inwards [sat]", + "width": 10, + "format": "10.0f", + "align": ">", + }, + "imean": { + "dict_key": "mean_forwarding_in", + "description": "mean forwarding inwards [sat]", + "width": 10, + "format": "10.0f", + "align": ">", + }, + "imax": { + "dict_key": "largest_forwarding_amount_in", + "description": "largest forwarding inwards [sat]", + "width": 10, + "format": "10.0f", + "align": ">", + }, + "omed": { + "dict_key": "median_forwarding_out", + "description": "median forwarding outwards [sat]", + "width": 10, + "format": "10.0f", + "align": ">", + }, + "omean": { + "dict_key": "mean_forwarding_out", + "description": "mean forwarding outwards [sat]", + "width": 10, + "format": "10.0f", + "align": ">", + }, + "omax": { + "dict_key": "largest_forwarding_amount_out", + "description": "largest forwarding outwards [sat]", + "width": 10, + "format": "10.0f", + "align": ">", + }, + "out": { + "dict_key": "total_forwarding_out", + "description": "total forwarding outwards [sat]", + "width": 10, + "format": "10.0f", + "align": ">", + }, + "sr/w": { + "dict_key": "sent_received_per_week", + "description": "sent and received per week [sat]", + "width": 9, + "format": "9d", + "align": ">", + }, +} + +PRINT_PEERS_FORMAT = { + "alias": { + "dict_key": "alias", + "description": "alias", + "width": ALIAS_LENGTH, + "format": ".<" + str(ALIAS_LENGTH), + "align": "^", + "convert": lambda x: alias_cutoff(x), + }, + "cap": { + "dict_key": "total_capacity", + "description": "total capacity [sat]", + "width": 9, + "format": "9d", + "align": ">", + }, + "mpc": { + "dict_key": "max_public_capacity", + "description": "maximum public channel capacity [sat]", + "width": 9, + "format": "9d", + "align": ">", + }, + "nid": { + "dict_key": "node_id", + "description": "node id", + "width": 66, + "format": "", + "align": "^", + }, + "flow": { + "dict_key": "flow_direction", + "description": "flow direction (positive is outwards)", + "width": 5, + "format": "5.2f", + "align": ">", + }, + "f/w": { + "dict_key": "fees_total_per_week", + "description": "total fees per week [sat / week]", + "width": 6, + "format": "6.2f", + "align": ">", + "convert": lambda x: float(x) / 1000, + }, + "nc": { + "dict_key": "number_channels", + "description": "number of channels", + "width": 2, + "format": "2d", + "align": "<", + }, + "np": { + "dict_key": "number_private_channels", + "description": "number of private channels", + "width": 2, + "format": "2d", + "align": "<", + }, + "na": { + "dict_key": "number_active_channels", + "description": "number of active channels", + "width": 2, + "format": "2d", + "align": "<", + }, + "mlb": { + "dict_key": "max_local_balance", + "description": "maximal local balance [sat]", + "width": 8, + "format": "8d", + "align": ">", + }, + "mrb": { + "dict_key": "max_remote_balance", + "description": "maximal remote balance [sat]", + "width": 8, + "format": "8d", + "align": ">", + }, + "lbf": { + "dict_key": "local_base_fee", + "description": "median local base fee [msat]", + "width": 5, + "format": "5.0f", + "align": ">", + }, + "lfr": { + "dict_key": "local_fee_rate", + "description": "median local fee rate", + "width": 8, + "format": "1.6f", + "align": ">", + "convert": lambda x: x / 1e6, + }, + "lup": { + "dict_key": "last_update", + "description": "last update time [days ago]", + "width": 5, + "format": "5.0f", + "align": ">", + }, + "lupp": { + "dict_key": "last_update_peer", + "description": "last update time by peer [days ago]", + "width": 5, + "format": "5.0f", + "align": ">", + }, + "lupl": { + "dict_key": "last_update_local", + "description": "last update time by local [days ago]", + "width": 5, + "format": "5.0f", + "align": ">", + }, + "lb": { + "dict_key": "local_balance", + "description": "total local balance [sat]", + "width": 9, + "format": "9d", + "align": ">", + }, + "nfwd": { + "dict_key": "number_forwardings", + "description": "number of forwardings", + "width": 4, + "format": "4.0f", + "align": ">", + }, + "priv": { + "dict_key": "private", + "description": "channel is private", + "width": 5, + "format": "^5", + "align": ">", + "convert": lambda x: POSITIVE_MARKER if x else NEGATIVE_MARKER, + }, + "rb": { + "dict_key": "remote_balance", + "description": "total remote balance [sat]", + "width": 9, + "format": "9d", + "align": ">", + }, + "r": { + "dict_key": "routable", + "description": "all channels are routable (active and at least one public channel)", + "width": 1, + "format": "^1", + "align": ">", + "convert": lambda x: NEGATIVE_MARKER if x else "", + }, + "in": { + "dict_key": "total_forwarding_in", + "description": "total forwarding inwards [sat]", + "width": 10, + "format": "10.0f", + "align": ">", + }, + "ini": { + "dict_key": "initiator", + "description": "true if we opened channel", + "width": 3, + "format": "^3", + "align": ">", + "convert": lambda x: POSITIVE_MARKER if x else NEGATIVE_MARKER, + }, + "tot": { + "dict_key": "total_forwarding", + "description": "total forwarding [sat]", + "width": 5, + "format": "5.0f", + "align": ">", + }, + "ub": { + "dict_key": "unbalancedness", + "description": "unbalancedness [-1 ... 1] (0 is 50:50 balanced)", + "width": 5, + "format": "5.2f", + "align": ">", + }, + "ulr": { + "dict_key": "uptime_lifetime_ratio", + "description": "ratio of uptime to lifetime of channel [0 ... 1]", + "width": 5, + "format": "5.2f", + "align": ">", + }, + "imed": { + "dict_key": "median_forwarding_in", + "description": "median forwarding inwards [sat]", + "width": 10, + "format": "10.0f", + "align": ">", + }, + "imean": { + "dict_key": "mean_forwarding_in", + "description": "mean forwarding inwards [sat]", + "width": 10, + "format": "10.0f", + "align": ">", + }, + "imax": { + "dict_key": "largest_forwarding_amount_in", + "description": "largest forwarding inwards [sat]", + "width": 10, + "format": "10.0f", + "align": ">", + }, + "omed": { + "dict_key": "median_forwarding_out", + "description": "median forwarding outwards [sat]", + "width": 10, + "format": "10.0f", + "align": ">", + }, + "omean": { + "dict_key": "mean_forwarding_out", + "description": "mean forwarding outwards [sat]", + "width": 10, + "format": "10.0f", + "align": ">", + }, + "omax": { + "dict_key": "largest_forwarding_amount_out", + "description": "largest forwarding outwards [sat]", + "width": 10, + "format": "10.0f", + "align": ">", + }, + "out": { + "dict_key": "total_forwarding_out", + "description": "total forwarding outwards [sat]", + "width": 10, + "format": "10.0f", + "align": ">", + }, + "sr/w": { + "dict_key": "sent_received_per_week", + "description": "sent and received per week [sat]", + "width": 9, + "format": "9d", + "align": ">", + }, + "rbf": { + "dict_key": "remote_base_fee", + "description": "median remote base fee [msat]", + "width": 5, + "format": "5.0f", + "align": ">", + }, + "rfr": { + "dict_key": "remote_fee_rate", + "description": "median remote fee rate", + "width": 8, + "format": "1.6f", + "align": ">", + "convert": lambda x: x / 1e6, + }, +} + + +def alias_cutoff(alias: str) -> str: + """Cuts off the node alias at a certain length and removes unicode + characters from the node alias.""" + + if len(alias) > ALIAS_LENGTH: + return alias[: ALIAS_LENGTH - 3] + "..." + else: + return alias + + +def _sorting_order(sort_string: str) -> Tuple[str, bool]: + """Determines the sorting string and the sorting order. + + If sort_string starts with 'rev_', the sorting order is reversed.""" + + reverse_sorting = True + if sort_string[:4] == "rev_": + reverse_sorting = False + sort_string = sort_string[4:] + + sort_string = PRINT_CHANNELS_FORMAT[sort_string]["dict_key"] + + return sort_string, reverse_sorting + + +def _row_string(column_values: Dict, print_format: Dict, columns: List[str]): + """Constructs the formatted row string for table printing.""" + + string = "" + for column in columns: + format_string = print_format[column]["format"] + conversion_function = print_format[column].get("convert", lambda x: x) + value = column_values[print_format[column]["dict_key"]] + converted_value = conversion_function(value) + string += f"{converted_value:{format_string}} " + + return string + + +def _print_objects(objects: Dict, print_format: Dict, columns: str, sort_dict: Dict): + """General purpose node/channel printing.""" + + if not objects: + logger.info(">>> Did not find any channels.") + + objects = OrderedDict( + sorted(objects.items(), key=sort_dict["function"], reverse=sort_dict["reverse"]) + ) + + logger.info("Sorting channels by %s.", sort_dict["string"]) + + logger.info("-------- Description --------") + columns = columns.split(",") + for column in columns: + logger.info("%-10s %s", column, print_format[column]["description"]) + logger.info("-----------------------------") + # prepare the column header + column_header = "" + for column in columns: + column_label = print_format[column]["align"] + column_width = print_format[column]["width"] + column_header += f"{column:{column_label}{column_width}} " + + # print the channel data + for channel_number, (_, channel_data) in enumerate(objects.items()): + if not channel_number % 20: + logger.info(column_header) + row = _row_string(channel_data, print_format, columns) + logger.info(row) + + +class ListChannels(object): + """A class to list lightning channels.""" + + def __init__(self, node: "LndNode"): + self.node = node + + def print_all_channels(self, sort_string="rev_alias"): + """Prints all active and inactive channels.""" + channels = self._add_channel_annotations(self.node.get_all_channels()) + sort_string, reverse_sorting = _sorting_order(sort_string) + sort_dict = { + "function": lambda x: ( + x[1][PRINT_CHANNELS_FORMAT["priv"]["dict_key"]], + x[1][sort_string], + ), + "string": sort_string, + "reverse": reverse_sorting, + } + _print_objects( + channels, + PRINT_CHANNELS_FORMAT, + columns="cid,priv,act,ub,cap,lb,rb,lbf,lfr,annotation,alias", + sort_dict=sort_dict, + ) + + def print_channels_unbalanced(self, unbalancedness: float, sort_string="rev_ub"): + """Prints unbalanced channels with |unbalancedness(channel)| > unbalancedness.""" + channels = self._add_channel_annotations( + self.node.get_unbalanced_channels(unbalancedness) + ) + sort_string, reverse_sorting = _sorting_order(sort_string) + sort_dict = { + "function": lambda x: x[1][sort_string], + "string": sort_string, + "reverse": reverse_sorting, + } + _print_objects( + channels, + PRINT_CHANNELS_FORMAT, + columns="cid,ub,cap,lb,rb,pbf,pfr,annotation,alias", + sort_dict=sort_dict, + ) + + def print_channels_inactive(self, sort_string="lupp"): + """Prints all inactive channels.""" + channels = self._add_channel_annotations(self.node.get_inactive_channels()) + sort_string, reverse_sorting = _sorting_order(sort_string) + sort_dict = { + "function": lambda x: (-x[1]["private"], x[1][sort_string]), + "string": sort_string, + "reverse": reverse_sorting, + } + + _print_objects( + channels, + PRINT_CHANNELS_FORMAT, + columns="cid,lupp,ulr,priv,ini,age,ub,cap,lb,rb,sr/w,annotation,alias", + sort_dict=sort_dict, + ) + + def print_channels_forwardings( + self, time_interval_start: float, time_interval_end: float, sort_string: str + ): + """Prints forwarding statistics for each channel. + + :param time_interval_start: int + :param time_interval_end: int + :param sort_string: str + """ + channels = get_channel_properties( + self.node, time_interval_start, time_interval_end + ) + channels = self._add_channel_annotations(channels) + sort_string, reverse_sorting = _sorting_order(sort_string) + sort_dict = { + "function": lambda x: ( + float("inf") if math.isnan(x[1][sort_string]) else x[1][sort_string], + x[1][PRINT_CHANNELS_FORMAT["nfwd"]["dict_key"]], + x[1][PRINT_CHANNELS_FORMAT["ub"]["dict_key"]], + ), + "string": sort_string, + "reverse": reverse_sorting, + } + _print_objects( + channels, + PRINT_CHANNELS_FORMAT, + columns="cid,nfwd,age,fees,f/w,flow,ub,bwd,r," + "cap,pbf,pfr,annotation,alias", + sort_dict=sort_dict, + ) + + def print_channels_hygiene(self, time_interval_start: float, sort_string: str): + """Prints hygiene statistics for each channel.""" + time_interval_end = time.time() + channels = get_channel_properties( + self.node, time_interval_start, time_interval_end + ) + channels = self._add_channel_annotations(channels) + sort_string, reverse_sorting = _sorting_order(sort_string) + sort_dict = { + "function": lambda x: x[1][sort_string], + "string": sort_string, + "reverse": reverse_sorting, + } + _print_objects( + channels, + PRINT_CHANNELS_FORMAT, + columns="cid,age,ini,nfwd/a,nfwd,f/w,ulr,lb,cap,lfr,pfr,annotation,alias", + sort_dict=sort_dict, + ) + + def _add_channel_annotations(self, channels: Dict) -> Dict: + """Appends metadata to existing channel dicts from the configuration file.""" + if self.node.config: + logger.debug("Adding annotations from file %s.", self.node.config_file) + # mapping between the channel point and channel id + channel_point_mapping = { + k: v["channel_point"].split(":")[0] for k, v in channels.items() + } + # only read annotations if config file is given + if self.node.config_file: + config = settings.read_config(self.node.config_file) + annotations = config["annotations"] + else: + annotations = {} + channel_annotations_funding_id = {} + channel_annotations_channel_id = {} + + for chan_id, annotation in annotations.items(): + if len(chan_id) == 18 and chan_id.isnumeric(): + # valid channel id + channel_annotations_channel_id[int(chan_id)] = annotation + elif len(chan_id) == 64 and chan_id.isalnum(): + # valid funding transaction id + channel_annotations_funding_id[chan_id] = annotation + else: + raise ValueError( + "First part needs to be either a channel id or the " + "funding transaction id. \n" + "The funding transaction id can be found with " + "`lncli listchannels` under the channel point (the " + "characters before the colon)." + ) + + for channel_id, channel_values in channels.items(): + # get the annotation by channel id first + annotation = channel_annotations_channel_id.get(channel_id, None) + # if no channel annotation, try with funding id + if annotation is None: + annotation = channel_annotations_funding_id.get( + channel_point_mapping[channel_id], None + ) + + if annotation is not None: + channels[channel_id]["annotation"] = annotation + else: + channels[channel_id]["annotation"] = "" + + return channels + + @staticmethod + def _row_string(column_values, columns): + """ + Constructs the formatted row string for table printing. + + :param column_values: dict + :param columns: list of str + :return: formatted str + """ + + string = "" + for column in columns: + format_string = PRINT_CHANNELS_FORMAT[column]["format"] + conversion_function = PRINT_CHANNELS_FORMAT[column].get( + "convert", lambda x: x + ) + value = column_values[PRINT_CHANNELS_FORMAT[column]["dict_key"]] + converted_value = conversion_function(value) + string += f"{converted_value:{format_string}} " + + +class ListPeers(object): + """A class to list lightning peers (with existing channels).""" + + def __init__(self, node: "LndNode"): + self.node = node + + def print_all_nodes( + self, + time_interval_start: float, + time_interval_end: float, + sort_string: str = "f/w", + ): + """Prints nodes with forwarding statistics.""" + + nodes = get_node_properites(self.node, time_interval_start, time_interval_end) + + sort_string, reverse_sorting = _sorting_order(sort_string) + sort_dict = { + "function": lambda x: ( + x[1][sort_string], + x[1][PRINT_PEERS_FORMAT["mpc"]["dict_key"]], + ), + "string": sort_string, + "reverse": reverse_sorting, + } + + _print_objects( + nodes, + PRINT_PEERS_FORMAT, + columns="nid,nc,na,np,nfwd,flow,ub,f/w,in,out,mpc,lb,mlb,rb,mrb,lfr,rfr,alias", + sort_dict=sort_dict, + ) + + +if __name__ == "__main__": + import time + import logging.config + from lndmanage.lib.node import LndNode + + logging.config.dictConfig(settings.logger_config) + logger = logging.getLogger() + + nd = LndNode("/home/user/.lndmanage/config.ini") + lp = ListPeers(nd) + lc = ListChannels(nd) + lp.print_all_nodes( + time_interval_start=time.time() - 3600 * 24 * 14, + time_interval_end=time.time(), + sort_string="f/w", + ) diff --git a/lndmanage/lib/node.py b/lndmanage/lib/node.py index 68f3091..e6263f3 100644 --- a/lndmanage/lib/node.py +++ b/lndmanage/lib/node.py @@ -4,7 +4,7 @@ from collections import OrderedDict, defaultdict import datetime import os import time -from typing import List, TYPE_CHECKING, Optional +from typing import List, TYPE_CHECKING, Optional, Dict import grpc from grpc._channel import _Rendezvous @@ -28,7 +28,7 @@ from lndmanage.lib.ln_utilities import ( channel_unbalancedness_and_commit_fee ) from lndmanage.lib.psbt import extract_psbt_inputs_outputs -from lndmanage.lib.types import UTXO, AddressType +from lndmanage.lib.data_types import UTXO, AddressType from lndmanage.lib.user import yes_no_question from lndmanage.lib.utilities import convert_dictionary_number_strings_to_ints from lndmanage import settings @@ -311,7 +311,8 @@ class LndNode(Node): if c['private']: self.total_private_channels += 1 - def get_open_channels(self, active_only=False, public_only=False): + def get_open_channels(self, active_only=False, public_only=False) \ + -> Dict[int, Dict]: """ Fetches information (fee settings of the counterparty, channel capacity, balancedness) about this node's open channels and saves @@ -429,6 +430,17 @@ class LndNode(Node): sorted(channels.items(), key=lambda x: x[1]['alias'])) return sorted_dict + def get_channel_id_to_node_id(self, open_only=False) -> Dict[int, str]: + channel_id_to_node_id = {} + closed_channels = self.get_closed_channels() + open_channels = self.get_open_channels() + for cid, c in open_channels.items(): + channel_id_to_node_id[cid] = c['remote_pubkey'] + if not open_only: + for cid, c in closed_channels.items(): + channel_id_to_node_id[cid] = c['remote_pubkey'] + return channel_id_to_node_id + def get_inactive_channels(self): """ Returns all inactive channels. diff --git a/lndmanage/lib/openchannels.py b/lndmanage/lib/openchannels.py index 8970b35..e9aaf38 100644 --- a/lndmanage/lib/openchannels.py +++ b/lndmanage/lib/openchannels.py @@ -5,7 +5,7 @@ from math import ceil import logging from typing import TYPE_CHECKING, List, Optional, Tuple -from lndmanage.lib.types import UTXO, AddressType +from lndmanage.lib.data_types import UTXO, AddressType from lndmanage import settings if TYPE_CHECKING: diff --git a/lndmanage/lib/recommend_nodes.py b/lndmanage/lib/recommend_nodes.py index ef33cf9..ec12a02 100644 --- a/lndmanage/lib/recommend_nodes.py +++ b/lndmanage/lib/recommend_nodes.py @@ -225,7 +225,7 @@ class RecommendNodes(object): """ forwarding_analyzer = ForwardingAnalyzer(self.node) # analyze all historic forwardings - forwarding_analyzer.initialize_forwarding_data(0, time.time()) + forwarding_analyzer.initialize_forwarding_stats(0, time.time()) nodes = forwarding_analyzer.get_forwarding_statistics_nodes() nodes = self.add_metadata_and_remove_pruned(nodes) return nodes @@ -244,7 +244,7 @@ class RecommendNodes(object): """ forwarding_analyzer = ForwardingAnalyzer(self.node) # analyze all historic forwardings - forwarding_analyzer.initialize_forwarding_data(0, time.time()) + forwarding_analyzer.initialize_forwarding_stats(0, time.time()) nodes_in, nodes_out = forwarding_analyzer.simple_flow_analysis( last_forwardings_to_analyze) raw_nodes = nodes_out if out_direction else nodes_in diff --git a/lndmanage/lndmanage.py b/lndmanage/lndmanage.py index 27f81da..a00be23 100755 --- a/lndmanage/lndmanage.py +++ b/lndmanage/lndmanage.py @@ -16,7 +16,7 @@ from lndmanage.lib.exceptions import ( ) from lndmanage.lib.fee_setting import FeeSetter, optimization_parameters from lndmanage.lib.info import Info -from lndmanage.lib.listchannels import ListChannels +from lndmanage.lib.listings import ListChannels, ListPeers from lndmanage.lib.lncli import Lncli from lndmanage.lib.node import LndNode from lndmanage.lib.openchannels import ChannelOpener @@ -131,6 +131,30 @@ class Parser(object): '--sort-by', default='rev_nfwd/a', type=str, help='sort by column (look at description)') + # cmd: listpeers + self.parser_listpeers = subparsers.add_parser( + 'listpeers', + help='lists peers with extended information', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + self.parser_listpeers.add_argument( + '--from-days-ago', default=60, type=int, + help='time interval start (days ago)') + self.parser_listpeers.add_argument( + '--sort-by', default='f/w', type=str, + help='sort by column (look at description)') + listpeers_subparsers = self.parser_listpeers.add_subparsers( + dest='subcmd') + + # cmd: listpeers in + listpeers_subparsers.add_parser( + 'in', + help="displays peers sorted by inward traffic") + + # cmd: listpeers out + listpeers_subparsers.add_parser( + 'out', + help="displays peers sorted by outward traffic") + # cmd: rebalance self.parser_rebalance = subparsers.add_parser( 'rebalance', help='rebalance a channel', @@ -489,6 +513,32 @@ class Parser(object): listchannels.print_channels_hygiene( time_interval_start=time_from, sort_string=args.sort_by) + elif args.cmd == 'listpeers': + listpeers = ListPeers(node) + time_from = time.time() - args.from_days_ago * 24 * 60 * 60 + time_to = time.time() + logger.info( + f"Forwardings from {args.from_days_ago} days ago" + f" to now are included.") + if not args.subcmd: + listpeers.print_all_nodes( + time_interval_start=time_from, + time_interval_end=time_to, + sort_string=args.sort_by, + ) + elif args.subcmd == 'in': + listpeers.print_all_nodes( + time_interval_start=time_from, + time_interval_end=time_to, + sort_string='in', + ) + elif args.subcmd == 'out': + listpeers.print_all_nodes( + time_interval_start=time_from, + time_interval_end=time_to, + sort_string='out', + ) + elif args.cmd == 'rebalance': if args.target: logger.warning("Warning: Target is set, this is still an " diff --git a/test/test_circle.py b/test/test_circle.py index 4f41967..991e1bd 100644 --- a/test/test_circle.py +++ b/test/test_circle.py @@ -3,7 +3,7 @@ Tests for circular self-payments. """ import time -from lndmanage.lib.listchannels import ListChannels +from lndmanage.lib.listings import ListChannels from lndmanage.lib.rebalance import Rebalancer from lndmanage.lib.exceptions import ( RebalanceFailure, diff --git a/test/test_rebalance.py b/test/test_rebalance.py index c46922c..84b6be9 100644 --- a/test/test_rebalance.py +++ b/test/test_rebalance.py @@ -4,7 +4,7 @@ Integration tests for rebalancing of channels. import time from lndmanage import settings -from lndmanage.lib.listchannels import ListChannels +from lndmanage.lib.listings import ListChannels from lndmanage.lib.rebalance import Rebalancer from lndmanage.lib.ln_utilities import channel_unbalancedness_and_commit_fee from lndmanage.lib.exceptions import RebalanceCandidatesExhausted