simulation: searchers default to high effort, 900s reflections

In this commit, we retune the codex searcher defaults with exp-018's
measurements in hand: the gepa arm at xhigh lost four of thirteen
iterations to the 600s reflection timeout and took nine hours for 150
evals. The evolutionary loop supplies the search, so iteration
throughput beats per-proposal depth; the default effort drops back to
high with a 900s timeout for headroom on large seeds, and xhigh stays
one flag away via codex:<model>:xhigh for runs where a deep single
proposal is the point.
This commit is contained in:
Olaoluwa Osuntokun 2026-07-27 10:53:04 -07:00
parent 3f228ed3e3
commit 820d06d016
2 changed files with 16 additions and 12 deletions

View file

@ -70,17 +70,21 @@ below asks for, with no preamble and no commentary.
class CodexLM:
"""Callable implementing GEPA's LanguageModel protocol via codex exec."""
def __init__(self, model: str = "gpt-5.6-sol", timeout: int = 600,
require_marker: str = None, effort: str = "xhigh"):
def __init__(self, model: str = "gpt-5.6-sol", timeout: int = 900,
require_marker: str = None, effort: str = "high"):
self.model = model
self.timeout = timeout
# Reasoning effort for the searcher. Passed as a per-call config
# override so it always wins over whatever the harness home's
# config.toml was written with (that file is only created on
# first use and never rewritten). Search/reflection agents run
# at xhigh by default; drop to high only if wall-clock per
# proposal becomes the bottleneck.
# first use and never rewritten). The default is high: exp-018's
# gepa arm ran at xhigh and lost 4 of 13 iterations to the 600s
# reflection timeout while taking nine hours for 150 evals — the
# evolutionary loop supplies the search, so iteration throughput
# beats per-proposal depth. Request xhigh explicitly
# (codex:<model>:xhigh) when a deep single proposal is the
# point.
self.effort = effort
# require_marker is a substring every valid completion must contain

View file

@ -139,7 +139,7 @@ def main() -> None:
help="rotate gepa <-> meta_harness on plateaus")
parser.add_argument("--no-adaptive", dest="adaptive",
action="store_false")
parser.add_argument("--reflection-timeout", type=int, default=600,
parser.add_argument("--reflection-timeout", type=int, default=900,
help="seconds to allow one reflection call. Raise "
"it for large seeds, whose reflections are slow.")
parser.add_argument("--seed-file", default=None,
@ -168,17 +168,17 @@ def main() -> None:
reflection_lm = args.reflection_lm
if reflection_lm.startswith("codex:"):
# codex:<model>[:<effort>] — e.g. codex:gpt-5.6-sol:xhigh.
# Searcher agents default to xhigh reasoning effort; a large
# seed makes reflection slow (a thousand-line candidate takes
# codex well past the ten minute default, and a timeout there
# costs a whole iteration to a stub proposal), so the timeout
# knob matters more at higher effort.
# Searchers default to high effort with a 900s timeout after
# exp-018 measured xhigh at 600s losing roughly a third of its
# iterations to reflection timeouts; a large seed makes
# reflection slow, so the timeout knob matters more at higher
# effort.
spec = reflection_lm.split(":")
reflection_lm = CodexLM(
model=spec[1],
require_marker="package main",
timeout=args.reflection_timeout,
effort=spec[2] if len(spec) > 2 else "xhigh",
effort=spec[2] if len(spec) > 2 else "high",
)
elif reflection_lm.startswith("claude:"):
# claude:<model>[:<effort>] — e.g. claude:claude-opus-5:medium.