lnd/simulation/evaluate.py

#!/usr/bin/env python3
"""The routesim evaluator: candidate params JSON in, (score, feedback) out.

The score composes the three signals a Lightning sender cares about, in
strict priority order:

  1. success rate (dominant term),
  2. attempts per payment (latency proxy, small penalty),
  3. fees paid in ppm on delivered value (small penalty).

The feedback dict carries the aggregate plus compact traces of the failed
payments (failure code and position per attempt), which is what a reflective
optimizer needs to reason about *why* a parameter set underperforms.
"""

import json
import os
import subprocess
import tempfile

ROUTESIM = os.environ.get("ROUTESIM_BIN", "routesim")

# Score weights: one extra attempt costs 1% success rate; 1000 ppm in fees
# costs 2%. Both penalties saturate so that success rate always dominates:
# the worst possible score is success_rate - 0.25, keeping deltas in
# success rate visible to the optimizer even on pathological runs.
#
# THE 1/N RULE, and do not raise the fee weight without reading it. A scored
# file holds between 6 and 10 payments (gen_scenarios.py draws randint(6, 10)),
# so abandoning one payment in the smallest file costs 1/6 = 0.167 of
# objective. The entire fee term is worth at most FEE_PPM_CAP * FEE_WEIGHT =
# 0.100. The fee term is therefore structurally incapable of paying for
# abandonment: even dropping the most expensive payment in a file and taking
# the fee penalty to zero loses money. That factor of 1.67 is the only thing
# standing between the fee term and the exp-013 give-up attractor.
#
#   The fee term's maximum value must stay strictly below 1/N, where N is the
#   payment count of the smallest scored file.
#
# Doubling FEE_WEIGHT breaks it. Removing the cap breaks it unconditionally.
#
# AND THE RULE APPLIES TO BOTH FEE METRICS, which corrects the exp-023 design
# spec. The spec argued that fee_ppm_attempted escapes the rule because the
# abandoned amount stays in its denominator, so the weight could safely rise
# once the fee term was charged against it. It does not escape. Fixing the
# denominator only stops abandonment from SHRINKING it; the numerator still
# falls, because a payment nobody completes pays no fee. Abandoning a payment
# that costs f and spends s on partial shards moves the ratio from (F+f)/A to
# (F+s)/A, which is a weak improvement for EVERY payment that pays a fee.
# fee_ppm_on_success is at least partly self-limiting by comparison: it only
# improves when the abandoned payment was dearer than the file's average, and
# abandoning a cheap payment makes it worse. Measured on the sealed hard tier,
# switching metric raises the objective of the arm that abandons most by the
# most (exp-023 stage C landing note).
#
# What fee_ppm_attempted is actually for is different and still worth having:
# it counts money that LEFT THE SENDER, including on payments that then failed,
# which fee_ppm_on_success cannot see at all. A router that burns fees on
# partial mpp shards that never complete is invisible to the scored metric and
# obvious in this one. It runs as a pre-registered side-by-side arm scored
# offline, not as a change to what the optimizer maximizes, and the 1/N rule
# above governs it exactly as it governs the scored metric.
ATTEMPT_WEIGHT = 0.01
ATTEMPT_CAP = 15
FEE_WEIGHT = 0.00002
FEE_PPM_CAP = 5_000

# The aggregate field the fee penalty is charged against. FEE_METRIC is what
# every published number in this program was scored with: fees on the payments
# that completed, over the amount they delivered. FEE_METRIC_ATTEMPTED is the
# alternative: fees actually spent, including on payments that failed, over the
# amount the batch was asked to deliver. Both are reported by every run, so the
# alternative arm re-scores archived outputs with no re-execution.
FEE_METRIC = "fee_ppm_on_success"
FEE_METRIC_ATTEMPTED = "fee_ppm_attempted"

# The one sentence about fees that every evaluator says unconditionally, in the
# style the exp-017 rewrite established for the give-up rule. A thresholded
# warning would not work here for the same reason it did not work there: fees
# falling is not by itself evidence of anything.
FEE_HINT = (
    "Fees fall for two reasons, cheaper routes and fewer completed "
    "payments, and only the first is an improvement: read fee_ppm "
    "against success_rate exactly the way attempts are read against it."
)


def capped_fee_ppm(agg: dict, fee_metric: str = FEE_METRIC) -> float:
    """The fee ppm the penalty is charged on, saturated at the cap."""
    return min(agg[fee_metric], FEE_PPM_CAP)


def composite_score(agg: dict, fee_metric: str = FEE_METRIC) -> float:
    """The objective: success rate less the attempt and fee penalties.

    Passing FEE_METRIC_ATTEMPTED scores the pre-registered alternative arm
    off the same aggregate, which is why this lives in one place.
    """
    extra_attempts = min(
        max(agg["attempts_per_scenario"] - 1.0, 0.0), ATTEMPT_CAP,
    )

    return (
        agg["success_rate"]
        - ATTEMPT_WEIGHT * extra_attempts
        - FEE_WEIGHT * capped_fee_ppm(agg, fee_metric)
    )


def run_routesim(params_json: str, scenario_path: str) -> dict:
    """Run one scenario file under the candidate params, returning the
    parsed output."""
    with tempfile.NamedTemporaryFile(
            mode="w", suffix=".json", delete=False) as f:
        f.write(params_json)
        params_path = f.name

    try:
        proc = subprocess.run(
            [ROUTESIM, "--params", params_path,
             "--scenarios", scenario_path],
            capture_output=True, text=True, timeout=300,
        )
        if proc.returncode != 0:
            raise RuntimeError(proc.stderr.strip())
        return json.loads(proc.stdout)
    finally:
        os.unlink(params_path)


def summarize_failures(results: list, limit: int = 5) -> list:
    """Compact per-failure traces for reflection feedback."""
    failures = []
    for result in results:
        if result["success"]:
            continue
        attempts = result.get("attempts") or []
        failures.append({
            "target": result["scenario"]["target"],
            "amt_msat": result["scenario"]["amt_msat"],
            "num_attempts": len(attempts),
            "terminal_error": result.get("error", ""),
            "attempt_failures": [
                {
                    "hops": len(a["hops"]),
                    "failure": a.get("failure", ""),
                    "failed_at_hop": a.get("failure_hop", -1),
                }
                for a in attempts[-4:]  # the last attempts show the endgame
            ],
        })
        if len(failures) >= limit:
            break
    return failures


def evaluate(candidate: str, example) -> tuple[float, dict]:
    """The optimize_anything evaluator contract.

    `example` is the path to one scenario file.
    """
    # Reject malformed candidates with actionable feedback.
    try:
        json.loads(candidate)
    except json.JSONDecodeError as exc:
        return 0.0, {"error": f"candidate is not valid JSON: {exc}"}

    try:
        output = run_routesim(candidate, str(example))
    except Exception as exc:  # simulator rejected the params
        return 0.0, {"error": f"routesim failed: {exc}"}

    agg = output["aggregate"]

    score = composite_score(agg)

    return score, {
        "score": score,
        "aggregate": agg,
        "failed_payments": summarize_failures(output["results"]),
        "hint": (
            "success_rate dominates the score; extra attempts and fee ppm "
            "apply small penalties. Failed payments show per-attempt "
            "failure codes and the hop index where each failed. "
            "TemporaryChannelFailure = liquidity miss (probability model "
            "was wrong), FeeInsufficient/IncorrectCltvExpiry = policy "
            "modeling bug. " + FEE_HINT
        ),
    }


if __name__ == "__main__":
    # Manual check: evaluate the lnd defaults on one example.
    import sys

    defaults = subprocess.run(
        [ROUTESIM, "--dump-defaults"], capture_output=True, text=True,
        check=True,
    ).stdout

    score, info = evaluate(defaults, sys.argv[1])
    print(f"score={score:.4f}")
    print(json.dumps(info["aggregate"], indent=2))
    print(json.dumps(info["failed_payments"], indent=2))