From fdc5523266321f4f04c85806dccf21604b525d6b Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Tue, 3 Mar 2026 14:30:33 -0800 Subject: [PATCH 1/3] contrib/clboss-forwarding-stats: feemon DB integration and avg liquidity Add an optional --db argument pointing at the feemon sqlite DB. When present, fetch feemon records for each peer over the chosen window (one extra day earlier than window_start_ts, so a boundary balance record exists for the window start) and compute two new per-peer quantities: - avg_to_us_msat: time-weighted average of our-side balance over the window, used as the liquidity divisor for forwarding metrics instead of the instantaneous current balance. - OpDays: operational days estimated from the feemon record count (~1/hour, so count / 24), used to prorate per-peer rates when a channel has only been open for part of the window. Both are surfaced as new columns in the output table. --- contrib/clboss-forwarding-stats | 182 ++++++++++++++++++++++++++++---- 1 file changed, 160 insertions(+), 22 deletions(-) diff --git a/contrib/clboss-forwarding-stats b/contrib/clboss-forwarding-stats index 9e5f860..177e970 100755 --- a/contrib/clboss-forwarding-stats +++ b/contrib/clboss-forwarding-stats @@ -8,6 +8,7 @@ # The channels at the top of the list are good, the ones at the bottom are bad. import argparse +from datetime import datetime, timezone import json import os import subprocess @@ -17,6 +18,7 @@ from tabulate import tabulate from wcwidth import wcswidth from clboss.alias_cache import lookup_alias +from feemon_data import load_merged_records_by_node def now(): now_ts = time.time() @@ -89,40 +91,131 @@ def pad_string(s, width): return s + " " * pad -def calculate_peer_score(peer, channel, daystr): +def calculate_peer_score(peer, avg_to_us_msat, daystr): + if not avg_to_us_msat: + return None + forwarded = peer.get("in_forwarded", 0) + peer.get("out_forwarded", 0) - to_us_msat = channel.get("to_us_msat", 0) age = peer_age_days(peer) window = peer_window(age, daystr) - if to_us_msat == 0: - return 0.0 - - fwd_per_bal = (float(forwarded) / float(to_us_msat)) + fwd_per_bal = float(forwarded) / float(avg_to_us_msat) fwd_per_bal_per_day = fwd_per_bal / window return fwd_per_bal_per_day -def calculate_trcl(peer, channel, daystr): - to_us_msat = channel.get("to_us_msat", 0) +def calculate_trcl(peer, avg_to_us_msat, daystr): + if not avg_to_us_msat: + return None age = peer_age_days(peer) window = peer_window(age, daystr) - if to_us_msat == 0: - return 0.0 - netearnings = peer_net_earnings(peer) - epd_per_bal = (float(netearnings) / float(to_us_msat)) + epd_per_bal = float(netearnings) / float(avg_to_us_msat) epd_per_bal_per_day = epd_per_bal / window # in basis points return epd_per_bal_per_day * 365 * 100 * 100 +def compute_operational_days(records, window_start_ts, now_ts): + """Estimate operational days from feemon record count in the window. + + Feemon records arrive roughly once per hour, so count / 24 + gives the number of days the channel was operational. + """ + count = sum(1 for rec in records + if window_start_ts <= rec["ts"] <= now_ts) + return count / 24.0 + + +def compute_time_weighted_avg_balance(records, window_start_ts, now_ts): + """Compute time-weighted average of balance_our_msat over [window_start_ts, now_ts]. + + Uses sample-and-hold: each balance value holds until the next record. + Records should be pre-filtered to the window period; any record at or + before window_start_ts is used as the boundary balance. + + Returns (avg_msat_int, effective_start_ts) or (None, None) if no data. + """ + if not records: + return (None, None) + + # Extract (ts, balance) pairs, skipping records without balance data + points = [] + for rec in records: + bal = rec["fields"].get("balance_our_msat") + if bal is not None: + points.append((rec["ts"], int(bal))) + + if not points: + return (None, None) + + # Find the boundary balance: last record at or before window_start_ts + boundary_balance = None + first_in_window_idx = None + for i, (ts, bal) in enumerate(points): + if ts <= window_start_ts: + boundary_balance = bal + else: + if first_in_window_idx is None: + first_in_window_idx = i + + if boundary_balance is not None: + # We have data covering the window start + effective_start = window_start_ts + # Build the segments: start with boundary balance from window_start + segments = [] + current_bal = boundary_balance + current_ts = window_start_ts + + # Add segments for each point within the window + for ts, bal in points: + if ts <= window_start_ts: + continue + if ts >= now_ts: + break + segments.append((current_bal, ts - current_ts)) + current_bal = bal + current_ts = ts + + # Final segment to now + segments.append((current_bal, now_ts - current_ts)) + + elif first_in_window_idx is not None: + # No data before window start; average over [first_record_ts, now] + first_ts, first_bal = points[first_in_window_idx] + effective_start = first_ts + segments = [] + current_bal = first_bal + current_ts = first_ts + + for ts, bal in points[first_in_window_idx + 1:]: + if ts >= now_ts: + break + segments.append((current_bal, ts - current_ts)) + current_bal = bal + current_ts = ts + + segments.append((current_bal, now_ts - current_ts)) + else: + # All points are after now_ts (shouldn't happen in practice) + return (None, None) + + total_duration = sum(dur for _, dur in segments) + if total_duration <= 0: + # Single point at now_ts + return (int(points[-1][1]), effective_start) + + integral = sum(bal * dur for bal, dur in segments) + avg = int(integral / total_duration) + return (avg, effective_start) + + def main(): parser = argparse.ArgumentParser( description="Run lightning-cli with specified network" @@ -139,6 +232,10 @@ def main(): type=int, help="Number of days of history to use with clboss-recent-earnings", ) + parser.add_argument( + "--db", default=None, + help="Path to feemon sqlite database (optional; default: API only).", + ) args = parser.parse_args() @@ -235,13 +332,48 @@ def main(): peers[id]["age"] = determine_peer_age( lightning_dir, network_option, id, peer_metrics) + # Fetch feemon data and compute time-weighted average balances + now_ts = now() + peer_ids = list(peers.keys()) + if args.days: + window_start_ts = now_ts - args.days * 86400 + else: + window_start_ts = 0 + + if args.days: + # Fetch one extra day so compute_time_weighted_avg_balance has a + # boundary record at or before window_start_ts. + since_dt = datetime.fromtimestamp(window_start_ts - 86400, tz=timezone.utc) + else: + since_dt = None + + feemon_records_by_node = load_merged_records_by_node( + args.db, peer_ids, api_node_ids=peer_ids, + since_dt=since_dt, before_dt=None, + lightning_dir=lightning_dir, + network_option=network_option, + ) + + avg_to_us_by_peer = {} + op_days_by_peer = {} + for peer_id in peer_ids: + records = feemon_records_by_node.get(peer_id, []) + avg, _ = compute_time_weighted_avg_balance( + records, window_start_ts, now_ts + ) + if avg is not None: + avg_to_us_by_peer[peer_id] = avg + op_days_by_peer[peer_id] = compute_operational_days( + records, window_start_ts, now_ts + ) + sorted_channels = sorted( channels.keys(), - key=lambda cid: -calculate_peer_score( + key=lambda cid: -(calculate_peer_score( peers[channels[cid]["peerid"]], - channels[cid], + avg_to_us_by_peer.get(channels[cid]["peerid"]), args.days - ) + ) or 0) ) # Prepare table data @@ -252,16 +384,20 @@ def main(): for short_channel_id in sorted_channels: peer_id = channels[short_channel_id]["peerid"] peer = peers[peer_id] - fpbpd = calculate_peer_score(peer, channels[short_channel_id], args.days) - fpbpdstr = f"{fpbpd:.5f}" - trcl = calculate_trcl(peer, channels[short_channel_id], args.days) - trclstr = f"{trcl:.3f}" + avg = avg_to_us_by_peer.get(peer_id) + fpbpd = calculate_peer_score(peer, avg, args.days) + fpbpdstr = f"{fpbpd:.5f}" if fpbpd is not None else "--" + trcl = calculate_trcl(peer, avg, args.days) + trclstr = f"{trcl:.3f}" if trcl is not None else "--" fwded = peer.get("in_forwarded", 0) + peer.get("out_forwarded", 0) netearnings = peer_net_earnings(peer) alias = pad_string(peer["alias"], max_alias_length) opener = "L" if channels[short_channel_id]["opener"] == "local" else "R" agestr = str(int(peer_age_days(peer))) + op_days = op_days_by_peer.get(peer_id, 0) + opdaysstr = f"{op_days:.1f}" to_us_msat = f"{channels[short_channel_id]['to_us_msat']:_}" + avg_to_us_str = f"{avg:_}" if avg is not None else "-" forwarded = f"{fwded:_}" netearningsstr = f"{netearnings:_}" table_data.append( @@ -270,7 +406,9 @@ def main(): short_channel_id, opener, to_us_msat, + avg_to_us_str, agestr, + opdaysstr, forwarded, netearningsstr, fpbpdstr, @@ -281,16 +419,16 @@ def main(): # Print the table without grid table_str = tabulate( table_data, - headers=["Alias", "SCID", "O", "to us", "Age", "Forwarded", "NetEarn", "FPBPD", "TRCL*"], + headers=["Alias", "SCID", "O", "to us", "avg_to_us", "Age", "OpDays", "Forwarded", "NetEarn", "FPBPD", "TRAL*"], tablefmt="plain", stralign="left", numalign="right", - colalign=("left", "left", "left", "right", "right", "right", "right", "right", "right"), + colalign=("left", "left", "left", "right", "right", "right", "right", "right", "right", "right", "right"), disable_numparse=True, ) print(table_str) - print("* annualized Trailing Return on Current Liquidity in basis points") + print("* annualized Trailing Return on Average Liquidity in basis points") if __name__ == "__main__": main() From 18273b933c1c0b5566ed7a53ddc99b6a56dabaa5 Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Wed, 4 Mar 2026 12:20:12 -0800 Subject: [PATCH 2/3] contrib/clboss-forwarding-stats: rework FRAL/TRAL on avg liquidity Rename the per-peer forwarding metric from FPBPD to FRAL (Forwarding Rate on Average Liquidity), and TRCL to TRAL (Trailing Return on Average Liquidity). Both are now computed against avg_to_us_msat and OpDays rather than the previous current-balance and clamped peer-age divisors, which gives a more honest per-active-day rate for channels that were closed and reopened. Adds a --sort {fral,tral} option (default fral) so the user can pick the ranking metric, a 1-OpDay floor in both calculations to suppress nonsense ratios from brand-new channels, and a footnote legend at the bottom of the table describing what each metric means. --- contrib/clboss-forwarding-stats | 76 +++++++++++++++------------------ 1 file changed, 34 insertions(+), 42 deletions(-) diff --git a/contrib/clboss-forwarding-stats b/contrib/clboss-forwarding-stats index 177e970..f76114c 100755 --- a/contrib/clboss-forwarding-stats +++ b/contrib/clboss-forwarding-stats @@ -80,46 +80,28 @@ def peer_net_earnings(peer): return earnings - expenditures -def peer_window(age, daystr): - if daystr: - return min(age, int(daystr)) - else: - return age - def pad_string(s, width): pad = width - wcswidth(s) return s + " " * pad -def calculate_peer_score(peer, avg_to_us_msat, daystr): - if not avg_to_us_msat: +def calculate_fral(peer, avg_to_us_msat, op_days): + if not avg_to_us_msat or op_days < 1: return None forwarded = peer.get("in_forwarded", 0) + peer.get("out_forwarded", 0) - age = peer_age_days(peer) - window = peer_window(age, daystr) - - fwd_per_bal = float(forwarded) / float(avg_to_us_msat) - fwd_per_bal_per_day = fwd_per_bal / window - - return fwd_per_bal_per_day + return float(forwarded) / float(avg_to_us_msat) / op_days -def calculate_trcl(peer, avg_to_us_msat, daystr): - if not avg_to_us_msat: +def calculate_tral(peer, avg_to_us_msat, op_days): + if not avg_to_us_msat or op_days < 1: return None - age = peer_age_days(peer) - window = peer_window(age, daystr) - netearnings = peer_net_earnings(peer) - epd_per_bal = float(netearnings) / float(avg_to_us_msat) - epd_per_bal_per_day = epd_per_bal / window - - # in basis points - return epd_per_bal_per_day * 365 * 100 * 100 + # annualized, in basis points + return float(netearnings) / float(avg_to_us_msat) / op_days * 365 * 100 * 100 def compute_operational_days(records, window_start_ts, now_ts): @@ -236,6 +218,10 @@ def main(): "--db", default=None, help="Path to feemon sqlite database (optional; default: API only).", ) + parser.add_argument( + "--sort", choices=["fral", "tral"], default="fral", + help="Column to sort by (default: fral).", + ) args = parser.parse_args() @@ -367,14 +353,19 @@ def main(): records, window_start_ts, now_ts ) - sorted_channels = sorted( - channels.keys(), - key=lambda cid: -(calculate_peer_score( - peers[channels[cid]["peerid"]], - avg_to_us_by_peer.get(channels[cid]["peerid"]), - args.days - ) or 0) - ) + def channel_sort_key(cid): + peer_id = channels[cid]["peerid"] + op_days = op_days_by_peer.get(peer_id, 0) + avg = avg_to_us_by_peer.get(peer_id) + peer = peers[peer_id] + if args.sort == "tral": + val = calculate_tral(peer, avg, op_days) + else: + val = calculate_fral(peer, avg, op_days) + sort_val = val if val is not None else float('-inf') + return -sort_val + + sorted_channels = sorted(channels.keys(), key=channel_sort_key) # Prepare table data table_data = [] @@ -385,16 +376,16 @@ def main(): peer_id = channels[short_channel_id]["peerid"] peer = peers[peer_id] avg = avg_to_us_by_peer.get(peer_id) - fpbpd = calculate_peer_score(peer, avg, args.days) - fpbpdstr = f"{fpbpd:.5f}" if fpbpd is not None else "--" - trcl = calculate_trcl(peer, avg, args.days) - trclstr = f"{trcl:.3f}" if trcl is not None else "--" + op_days = op_days_by_peer.get(peer_id, 0) + fral = calculate_fral(peer, avg, op_days) + fralstr = f"{fral:.5f}" if fral is not None else "--" + tral = calculate_tral(peer, avg, op_days) + tralstr = f"{tral:.3f}" if tral is not None else "--" fwded = peer.get("in_forwarded", 0) + peer.get("out_forwarded", 0) netearnings = peer_net_earnings(peer) alias = pad_string(peer["alias"], max_alias_length) opener = "L" if channels[short_channel_id]["opener"] == "local" else "R" agestr = str(int(peer_age_days(peer))) - op_days = op_days_by_peer.get(peer_id, 0) opdaysstr = f"{op_days:.1f}" to_us_msat = f"{channels[short_channel_id]['to_us_msat']:_}" avg_to_us_str = f"{avg:_}" if avg is not None else "-" @@ -411,15 +402,15 @@ def main(): opdaysstr, forwarded, netearningsstr, - fpbpdstr, - trclstr, + fralstr, + tralstr, ] ) # Print the table without grid table_str = tabulate( table_data, - headers=["Alias", "SCID", "O", "to us", "avg_to_us", "Age", "OpDays", "Forwarded", "NetEarn", "FPBPD", "TRAL*"], + headers=["Alias", "SCID", "O", "curr_to_us", "avg_to_us", "Age", "OpDays", "Forwarded", "NetEarn", "FRAL**", "TRAL*"], tablefmt="plain", stralign="left", numalign="right", @@ -428,7 +419,8 @@ def main(): ) print(table_str) - print("* annualized Trailing Return on Average Liquidity in basis points") + print("** Forwarding Rate on Average Liquidity (forwarded / avg balance / op days)") + print("* Trailing Return on Average Liquidity in basis points, annualized") if __name__ == "__main__": main() From 1a6d0f1974a0d970b3454238d3e8a4310c74b224 Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Thu, 5 Mar 2026 09:36:23 -0800 Subject: [PATCH 3/3] contrib/clboss-forwarding-stats: validate --days and --db CLI inputs - --days must be a strictly positive integer. Reject 0 and negative values which would push window_start_ts into the future or get silently treated as unset. - --db, when provided, must reference an existing file. Raise a clear argparse error at startup instead of bubbling up as a sqlite OperationalError later from inside the feemon loader. --- contrib/clboss-forwarding-stats | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/clboss-forwarding-stats b/contrib/clboss-forwarding-stats index f76114c..3446154 100755 --- a/contrib/clboss-forwarding-stats +++ b/contrib/clboss-forwarding-stats @@ -225,6 +225,12 @@ def main(): args = parser.parse_args() + if args.days is not None and args.days <= 0: + parser.error("--days must be a positive integer") + + if args.db is not None and not os.path.isfile(args.db): + parser.error(f"--db file not found: {args.db}") + # Reconcile network option if args.network: network_option = f"--network={args.network}"