From 820d06d016f2057f0e5217eed8ddc8482cf79cb5 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Mon, 27 Jul 2026 10:53:04 -0700 Subject: [PATCH] simulation: searchers default to high effort, 900s reflections In this commit, we retune the codex searcher defaults with exp-018's measurements in hand: the gepa arm at xhigh lost four of thirteen iterations to the 600s reflection timeout and took nine hours for 150 evals. The evolutionary loop supplies the search, so iteration throughput beats per-proposal depth; the default effort drops back to high with a 900s timeout for headroom on large seeds, and xhigh stays one flag away via codex::xhigh for runs where a deep single proposal is the point. --- simulation/codex_lm.py | 14 +++++++++----- simulation/run_gepa_code.py | 14 +++++++------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/simulation/codex_lm.py b/simulation/codex_lm.py index 209578032..1ac8caaad 100644 --- a/simulation/codex_lm.py +++ b/simulation/codex_lm.py @@ -70,17 +70,21 @@ below asks for, with no preamble and no commentary. class CodexLM: """Callable implementing GEPA's LanguageModel protocol via codex exec.""" - def __init__(self, model: str = "gpt-5.6-sol", timeout: int = 600, - require_marker: str = None, effort: str = "xhigh"): + def __init__(self, model: str = "gpt-5.6-sol", timeout: int = 900, + require_marker: str = None, effort: str = "high"): self.model = model self.timeout = timeout # Reasoning effort for the searcher. Passed as a per-call config # override so it always wins over whatever the harness home's # config.toml was written with (that file is only created on - # first use and never rewritten). Search/reflection agents run - # at xhigh by default; drop to high only if wall-clock per - # proposal becomes the bottleneck. + # first use and never rewritten). The default is high: exp-018's + # gepa arm ran at xhigh and lost 4 of 13 iterations to the 600s + # reflection timeout while taking nine hours for 150 evals — the + # evolutionary loop supplies the search, so iteration throughput + # beats per-proposal depth. Request xhigh explicitly + # (codex::xhigh) when a deep single proposal is the + # point. self.effort = effort # require_marker is a substring every valid completion must contain diff --git a/simulation/run_gepa_code.py b/simulation/run_gepa_code.py index 071da8670..cb3678b19 100644 --- a/simulation/run_gepa_code.py +++ b/simulation/run_gepa_code.py @@ -139,7 +139,7 @@ def main() -> None: help="rotate gepa <-> meta_harness on plateaus") parser.add_argument("--no-adaptive", dest="adaptive", action="store_false") - parser.add_argument("--reflection-timeout", type=int, default=600, + parser.add_argument("--reflection-timeout", type=int, default=900, help="seconds to allow one reflection call. Raise " "it for large seeds, whose reflections are slow.") parser.add_argument("--seed-file", default=None, @@ -168,17 +168,17 @@ def main() -> None: reflection_lm = args.reflection_lm if reflection_lm.startswith("codex:"): # codex:[:] — e.g. codex:gpt-5.6-sol:xhigh. - # Searcher agents default to xhigh reasoning effort; a large - # seed makes reflection slow (a thousand-line candidate takes - # codex well past the ten minute default, and a timeout there - # costs a whole iteration to a stub proposal), so the timeout - # knob matters more at higher effort. + # Searchers default to high effort with a 900s timeout after + # exp-018 measured xhigh at 600s losing roughly a third of its + # iterations to reflection timeouts; a large seed makes + # reflection slow, so the timeout knob matters more at higher + # effort. spec = reflection_lm.split(":") reflection_lm = CodexLM( model=spec[1], require_marker="package main", timeout=args.reflection_timeout, - effort=spec[2] if len(spec) > 2 else "xhigh", + effort=spec[2] if len(spec) > 2 else "high", ) elif reflection_lm.startswith("claude:"): # claude:[:] — e.g. claude:claude-opus-5:medium.