lnd/simulation/evaluate_code.py

#!/usr/bin/env python3
"""Code-mode evaluator: the candidate is the full Go source of
cmd/routesim/candidate_impl.go — an entire routing algorithm.

Each eval compiles a routesim binary with the candidate swapped in via
`go build -overlay` (no working-tree mutation, parallel-safe) and runs it
with --router=candidate. Compile errors come back as feedback, which is
the highest-signal input a reflective proposer can get.
"""

import json
import os
import re
import subprocess
import tempfile
from pathlib import Path

import evaluate as params_eval

REPO = Path(os.environ.get(
    "LND_REPO",
    Path(__file__).resolve().parent.parent,
))
GO = os.environ.get("GO_BIN", "go")

# Tokens that have no business in a routing algorithm and defeat the
# information hiding of the simulator (reward-hack guard). The second
# group enforces the sealed-view invariant in the evaluator itself
# rather than by post-hoc grep: GraphSession callbacks receive the
# sealed view, and any candidate naming the hidden-state surfaces is
# probing for an escape.
BANNED = re.compile(
    r'\b(unsafe|reflect|os/exec|syscall|net/http|io/ioutil)\b|'
    r'"os"|_test\b|'
    r'\b(LocalBalances|AssignLiquidity|BalanceNodeChannels|SendHtlc)\b|'
    r'\b(HoldHtlc|SettleHold|ReleaseHold)\b|'
    r'\*\s*routing\.SimGraph',
)

FENCE = re.compile(r"^```(?:go)?\s*$|^```\s*$", re.MULTILINE)


def extract_source(candidate: str) -> str:
    """Strip markdown fences if the proposer wrapped the file in them."""
    text = candidate.strip()
    if text.startswith("```"):
        text = FENCE.sub("", text).strip()

    # Agentic proposers sometimes prepend prose despite instructions;
    # a Go file must start at its package clause, so slice from there.
    if not text.startswith("package "):
        idx = text.find("package main")
        if idx > 0:
            text = text[idx:]

    return text + "\n"


def compile_candidate(source: str, workdir: Path) -> tuple[Path, str]:
    """Compile a routesim binary with the candidate overlaid. Returns
    (binary path, "") on success or (None, compiler output) on failure."""
    cand_path = workdir / "candidate_impl.go"
    cand_path.write_text(source)

    overlay = workdir / "overlay.json"
    target = str(REPO / "cmd" / "routesim" / "candidate_impl.go")
    overlay.write_text(json.dumps(
        {"Replace": {target: str(cand_path)}},
    ))

    binary = workdir / "routesim"
    try:
        proc = subprocess.run(
            [GO, "build", "-overlay", str(overlay), "-o", str(binary),
             "./cmd/routesim"],
            cwd=REPO, capture_output=True, text=True, timeout=300,
        )
    except subprocess.TimeoutExpired:
        # Under concurrent engines a slow build must score zero, not
        # crash the whole run (raise_on_exception aborts on evaluator
        # exceptions).
        return None, "go build timed out after 300s (host under load?)"
    if proc.returncode != 0:
        return None, proc.stderr[-4000:]

    return binary, ""


def run_compiled(binary, example) -> tuple[float, dict]:
    """Run a compiled candidate binary on one scenario file and score it."""
    # A pathological candidate (infinite loop, quadratic blowup) must
    # score 0, not crash the whole optimization run. 120s is generous:
    # a healthy router does a full scenario batch in well under a
    # second, so a timeout means the candidate is broken.
    try:
        proc = subprocess.run(
            [str(binary), "--scenarios", str(example),
             "--router", "candidate"],
            capture_output=True, text=True, timeout=120,
        )
    except subprocess.TimeoutExpired:
        return 0.0, {
            "error": "timeout: candidate did not finish in 120s",
            "hint": "The router likely loops without making progress "
            "(e.g. RequestRoute never returns an error to terminate "
            "the payment, or splits without shrinking). Ensure every "
            "path terminates and shard amounts strictly decrease.",
        }
    if proc.returncode != 0:
        return 0.0, {
            "error": f"runtime failure: {proc.stderr[-2000:]}",
        }

    try:
        output = json.loads(proc.stdout)
    except json.JSONDecodeError:
        return 0.0, {
            "error": "candidate produced no valid JSON output",
            "stdout_tail": proc.stdout[-1000:],
        }

    try:
        agg = output["aggregate"]
        results = output["results"]
    except (KeyError, TypeError):
        return 0.0, {
            "error": "routesim output missing aggregate/results",
        }

    fee_ppm = params_eval.capped_fee_ppm(agg)
    score = params_eval.composite_score(agg)

    # Separate objective axes for Pareto-frontier preservation
    # (frontier_type="hybrid" in the engine config). Retries and parts
    # are disentangled: a mandatory 3-shard MPP payment is not "2 extra
    # attempts" of waste, while 3 failures before 1 settle are. Axes are
    # oriented so higher is better.
    settled_parts = sum(
        1
        for res in results
        for att in (res.get("attempts") or [])
        if att.get("success")
    )
    total_attempts = agg["total_attempts"]
    retries = max(total_attempts - settled_parts, 0)
    num = max(agg["num_scenarios"], 1)
    scores = {
        "success": agg["success_rate"],
        "retry_efficiency": -min(retries / num, 25.0),
        "fee_efficiency": -fee_ppm / params_eval.FEE_PPM_CAP,
    }

    # Abandonment and background-settle rates are surfaced as top-level
    # side information rather than left buried in the aggregate dict.
    # exp-013 showed the composite objective hides giving up on hard
    # payments inside the same number as attempt efficiency; a candidate
    # at the attempt frontier can only "improve" by abandoning, and the
    # reflection model needs to see that channel explicitly to avoid
    # walking into it.
    #
    # A thresholded warning on give_up_rate alone does not work: exp-017
    # measured that for candidate routers the field equals
    # 1 - success_rate on every tier (a candidate "gives up" whenever it
    # returns failure without exhausting the attempt budget, which is
    # how candidates always fail), so such a warning fires universally
    # and becomes noise. Abandonment is only readable JOINTLY, as low
    # attempts together with low success, and the hint states that rule
    # unconditionally instead.
    give_up_rate = agg.get("give_up_rate", 0.0)
    bg_settle_rate = agg.get("bg_settle_rate", 0.0)

    hint = (
        "success_rate dominates; attempts and fee ppm apply small "
        "penalties. The router only sees gossip (no hidden "
        "balances), its own channel balances, and per-attempt "
        "failure feedback via ReportAttempt. Beware the give-up "
        "attractor: a candidate can cut attempts by abandoning hard "
        "payments, which the composite score partly rewards. Fewer "
        "attempts is only an improvement if success_rate held or "
        "rose; if attempts AND success both fell, the edit taught "
        "the router to quit, not to route. (give_up_rate counts "
        "payments failed without exhausting the attempt budget; for "
        "most candidates it simply equals 1 - success_rate, so read "
        "abandonment off the success/attempts pair, not off that "
        "field alone.) " + params_eval.FEE_HINT + " A payment may also "
        "carry a fee budget (spec.FeeLimitMsat, the total across all "
        "shards, lnwire.MaxMilliSatoshi when there is none): a route "
        "over budget is refused before it is sent, costing an attempt "
        "and teaching nothing, and fee_limit_failures counts those "
        "refusals."
    )

    return score, {
        "score": score,
        "scores": scores,
        "give_up_rate": give_up_rate,
        "bg_settle_rate": bg_settle_rate,
        "aggregate": agg,
        "failed_payments": params_eval.summarize_failures(results),
        "hint": hint,
    }


def evaluate(candidate: str, example) -> tuple[float, dict]:
    """The optimize_anything evaluator contract for code candidates."""
    source = extract_source(candidate)

    banned = BANNED.search(source)
    if banned:
        return 0.0, {
            "error": f"banned identifier {banned.group(0)!r}: candidates "
            "must not use unsafe/reflect/os/exec/net or probe the hidden "
            "simulator state — pure routing logic only.",
        }

    with tempfile.TemporaryDirectory(prefix="routesim-cand-") as tmp:
        workdir = Path(tmp)

        binary, compile_err = compile_candidate(source, workdir)
        if binary is None:
            return 0.0, {
                "error": "compile failed",
                "compiler_output": compile_err,
                "hint": "Return the COMPLETE contents of "
                "candidate_impl.go (package main), defining "
                "newCandidateRouter with the exact contract signature.",
            }

        return run_compiled(binary, example)


def batch_evaluate(pairs) -> list:
    """Batch form of the evaluator contract: one compile per unique
    candidate instead of one per (candidate, example) pair.

    A candidate's valset pass previously ran `go build` once per
    example — eight identical compiles for an eight-file valset. Here
    the pairs are grouped by extracted source, each unique candidate is
    compiled once, and its scenario runs execute against the shared
    binary. Returns results in input order.
    """
    from concurrent.futures import ThreadPoolExecutor

    order = list(pairs)
    by_source: dict = {}
    for idx, (candidate, example) in enumerate(order):
        by_source.setdefault(extract_source(candidate), []).append(
            (idx, candidate, example),
        )

    results: list = [None] * len(order)

    def run_group(group) -> None:
        source, members = group
        banned = BANNED.search(source)
        if banned:
            outcome = (0.0, {
                "error": f"banned identifier {banned.group(0)!r}: "
                "candidates must not use unsafe/reflect/os/exec/net or "
                "probe the hidden simulator state — pure routing logic "
                "only.",
            })
            for idx, _, _ in members:
                results[idx] = outcome
            return

        with tempfile.TemporaryDirectory(prefix="routesim-cand-") as tmp:
            workdir = Path(tmp)
            binary, compile_err = compile_candidate(source, workdir)
            if binary is None:
                outcome = (0.0, {
                    "error": "compile failed",
                    "compiler_output": compile_err,
                    "hint": "Return the COMPLETE contents of "
                    "candidate_impl.go (package main), defining "
                    "newCandidateRouter with the exact contract "
                    "signature.",
                })
                for idx, _, _ in members:
                    results[idx] = outcome
                return

            for idx, _, example in members:
                results[idx] = run_compiled(binary, example)

    with ThreadPoolExecutor(max_workers=4) as pool:
        list(pool.map(run_group, by_source.items()))

    return results


if __name__ == "__main__":
    import sys

    seed_path = REPO / "cmd" / "routesim" / "candidate_impl.go"
    score, info = evaluate(seed_path.read_text(), sys.argv[1])
    print(f"score={score:.4f}")
    print(json.dumps(info.get("aggregate", info), indent=2))