From 248f577828690d0cfe749a0283c044338ec79af5 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sun, 23 Aug 2026 18:23:51 +0000 Subject: [PATCH 1/5] feat(v1): split run configs into launch TOML and resolved dir A run dir's configs/ now holds the launch TOML copied verbatim (configs/eval.toml) next to configs/resolved/eval.json, the resolved dump - the same layout prime-rl's launchers write. saved_config_path still finds legacy configs/.json runs. Also drop the startup log of the full eval config: it is persisted to disk now, so the log line was redundant. Co-Authored-By: Claude Fable 5 --- verifiers/v1/cli/eval/main.py | 2 +- verifiers/v1/cli/eval/runner.py | 1 - verifiers/v1/cli/output.py | 55 ++++++++++++++++++++++++--------- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/verifiers/v1/cli/eval/main.py b/verifiers/v1/cli/eval/main.py index dd7c6153dd..dc1ed47cf9 100644 --- a/verifiers/v1/cli/eval/main.py +++ b/verifiers/v1/cli/eval/main.py @@ -32,7 +32,7 @@ USAGE = ( "usage: uv run eval [] [--env.id ] [options] [@ file.toml]\n" - " uv run eval @ /configs/eval.json --resume (re-run the run's missing/errored rollouts)" + " uv run eval @ /configs/resolved/eval.json --resume (re-run the run's missing/errored rollouts)" ) diff --git a/verifiers/v1/cli/eval/runner.py b/verifiers/v1/cli/eval/runner.py index 56043d0000..d5c94f4bf3 100644 --- a/verifiers/v1/cli/eval/runner.py +++ b/verifiers/v1/cli/eval/runner.py @@ -140,7 +140,6 @@ async def run(slot: RunSlot) -> Episode: async def run_eval(config: EvalConfig) -> list[Episode]: from verifiers.v1.utils.loaders import load_environment, load_taskset - logger.info("eval config:\n%s", config.model_dump_json(indent=2)) # The env comes up in this process only for an in-process run; a served run's # workers each load their own, and this process owns just the taskset. env = None if config.serve is not None else load_environment(config.env) diff --git a/verifiers/v1/cli/output.py b/verifiers/v1/cli/output.py index be9842d4a8..e571967415 100644 --- a/verifiers/v1/cli/output.py +++ b/verifiers/v1/cli/output.py @@ -29,9 +29,13 @@ """Filename a run's rollout episodes are written to (one JSON episode per line).""" CONFIG_DIR = "configs" -"""Directory inside a run dir holding its resolved config (one `.json`, re-runnable -via `@ /configs/.json`). Resolved configs are JSON, not TOML: JSON keeps -nulls, so explicit None settings round-trip exactly on re-parse.""" +"""Directory inside a run dir holding its configs: the launch TOML copied verbatim to +`configs/.toml`, and the resolved config at `configs/resolved/.json` +(re-runnable via `@ /configs/resolved/.json`). Resolved configs are JSON, +not TOML: JSON keeps nulls, so explicit None settings round-trip exactly on re-parse.""" + +RESOLVED_DIR = "resolved" +"""Subdirectory of `configs/` holding the resolved per-component JSON dumps.""" def create_attempt_log_dir(run_dir: Path) -> Path: @@ -76,13 +80,13 @@ def config_digest(config: BaseModel) -> str: def saved_config_path(run_dir: Path) -> Path | None: - """The run's saved resolved config (`configs/.json`), None if absent.""" - candidates = ( - sorted((run_dir / CONFIG_DIR).glob("*.json")) - if (run_dir / CONFIG_DIR).is_dir() - else [] - ) - return candidates[0] if candidates else None + """The run's saved resolved config (`configs/resolved/.json`; legacy runs + kept it at `configs/.json`), None if absent.""" + for config_dir in (run_dir / CONFIG_DIR / RESOLVED_DIR, run_dir / CONFIG_DIR): + candidates = sorted(config_dir.glob("*.json")) if config_dir.is_dir() else [] + if candidates: + return candidates[0] + return None # Compiling an adapter is the expensive part; run output reuses only a few model classes. @@ -100,22 +104,45 @@ def output_path(config: EvalConfig) -> Path: def write_config( config: BaseModel, results_dir: Path, filename: str = "eval.json" ) -> Path: - """Write the run's resolved config to `configs/` (re-readable via + """Write the run's resolved config to `configs/resolved/` (re-readable via `@ `); return its path. The full model dump is written, nulls included, so the file round-trips exactly.""" - config_dir = results_dir / CONFIG_DIR + config_dir = results_dir / CONFIG_DIR / RESOLVED_DIR config_dir.mkdir(parents=True, exist_ok=True) config_path = config_dir / filename config_path.write_text(json.dumps(config.model_dump(mode="json"), indent=2)) return config_path +def write_launch_toml(results_dir: Path, name: str = "eval") -> None: + """Copy the launch `@` TOML file(s) verbatim to `configs/.toml`.""" + import sys + + argv = sys.argv[1:] + paths = [] + for i, arg in enumerate(argv): + if arg == "@" and i + 1 < len(argv): + paths.append(Path(argv[i + 1])) + elif arg.startswith("@") and len(arg) > 1: + paths.append(Path(arg[1:])) + texts = [p.read_text() for p in paths if p.is_file()] + if not texts: + return + if len(texts) > 1: + texts = [f"# @ {p}\n{t}" for p, t in zip(paths, texts)] + config_dir = results_dir / CONFIG_DIR + config_dir.mkdir(parents=True, exist_ok=True) + (config_dir / f"{name}.toml").write_text("\n".join(texts)) + + def save_config( config: BaseModel, results_dir: Path, filename: str = "eval.json" ) -> None: - """Set up the run's output dir: write the resolved config and start a fresh (empty) - `traces.jsonl`. Call once up front, before episodes start landing.""" + """Set up the run's output dir: write the resolved config, copy the launch TOML, + and start a fresh (empty) `traces.jsonl`. Call once up front, before episodes start + landing.""" write_config(config, results_dir, filename) + write_launch_toml(results_dir, Path(filename).stem) (results_dir / TRACES_FILE).write_text( "" ) # fresh; appended to as rollouts complete From 0bb06d96042d46a38475eb7a19428ab6bdf0c295 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sun, 23 Aug 2026 20:00:49 +0000 Subject: [PATCH 2/5] fix: launch-TOML copy takes root config references only The argv scan captured nested `--flag @ file` / `--flag @file` references and concatenated them into configs/eval.toml as if they were root launch files - re-running that copy would apply nested keys at root. Only space-form root `@ file` references are copied now, and only .toml sources (a resume from the resolved JSON writes no launch copy). Co-Authored-By: Claude Fable 5 --- verifiers/v1/cli/output.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/verifiers/v1/cli/output.py b/verifiers/v1/cli/output.py index e571967415..bd7aebe4da 100644 --- a/verifiers/v1/cli/output.py +++ b/verifiers/v1/cli/output.py @@ -121,11 +121,11 @@ def write_launch_toml(results_dir: Path, name: str = "eval") -> None: argv = sys.argv[1:] paths = [] for i, arg in enumerate(argv): - if arg == "@" and i + 1 < len(argv): + # root config references only: `@ file`; a `--flag @ file` / `--flag @file` + # is a nested reference and belongs under its flag, not in the launch copy + if arg == "@" and i + 1 < len(argv) and (i == 0 or not argv[i - 1].startswith("--")): paths.append(Path(argv[i + 1])) - elif arg.startswith("@") and len(arg) > 1: - paths.append(Path(arg[1:])) - texts = [p.read_text() for p in paths if p.is_file()] + texts = [p.read_text() for p in paths if p.suffix == ".toml" and p.is_file()] if not texts: return if len(texts) > 1: From 57a13159e54465c25638c6d8b5c22b310ce20b8e Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Mon, 24 Aug 2026 01:13:40 +0000 Subject: [PATCH 3/5] style: ruff format output.py Co-Authored-By: Claude Fable 5 --- verifiers/v1/cli/output.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/verifiers/v1/cli/output.py b/verifiers/v1/cli/output.py index bd7aebe4da..85d9dff19e 100644 --- a/verifiers/v1/cli/output.py +++ b/verifiers/v1/cli/output.py @@ -123,7 +123,11 @@ def write_launch_toml(results_dir: Path, name: str = "eval") -> None: for i, arg in enumerate(argv): # root config references only: `@ file`; a `--flag @ file` / `--flag @file` # is a nested reference and belongs under its flag, not in the launch copy - if arg == "@" and i + 1 < len(argv) and (i == 0 or not argv[i - 1].startswith("--")): + if ( + arg == "@" + and i + 1 < len(argv) + and (i == 0 or not argv[i - 1].startswith("--")) + ): paths.append(Path(argv[i + 1])) texts = [p.read_text() for p in paths if p.suffix == ".toml" and p.is_file()] if not texts: From 2a0b2c68879c0919c97772cb237f7d1cdcb1ecf6 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Mon, 24 Aug 2026 01:28:41 +0000 Subject: [PATCH 4/5] fix: align launch-TOML headers with their filtered sources The multi-file header zip paired the unfiltered path list against the toml-filtered texts, so a non-toml root reference shifted every header onto the wrong file. Co-Authored-By: Claude Fable 5 --- verifiers/v1/cli/output.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/verifiers/v1/cli/output.py b/verifiers/v1/cli/output.py index 068b4f43e3..36cb275d74 100644 --- a/verifiers/v1/cli/output.py +++ b/verifiers/v1/cli/output.py @@ -121,11 +121,10 @@ def write_launch_toml(results_dir: Path, name: str = "eval") -> None: and (i == 0 or not argv[i - 1].startswith("--")) ): paths.append(Path(argv[i + 1])) - texts = [p.read_text() for p in paths if p.suffix == ".toml" and p.is_file()] - if not texts: + tomls = [(p, p.read_text()) for p in paths if p.suffix == ".toml" and p.is_file()] + if not tomls: return - if len(texts) > 1: - texts = [f"# @ {p}\n{t}" for p, t in zip(paths, texts)] + texts = [text for _, text in tomls] if len(tomls) == 1 else [f"# @ {p}\n{text}" for p, text in tomls] config_dir = results_dir / CONFIG_DIR config_dir.mkdir(parents=True, exist_ok=True) (config_dir / f"{name}.toml").write_text("\n".join(texts)) From f7c83add73ab419bdca56852e35c65dd15fcd3bb Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Mon, 24 Aug 2026 01:34:01 +0000 Subject: [PATCH 5/5] style: format with the CI-pinned ruff 0.16.0 Co-Authored-By: Claude Fable 5 --- verifiers/v1/cli/output.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/verifiers/v1/cli/output.py b/verifiers/v1/cli/output.py index 36cb275d74..f651bd5844 100644 --- a/verifiers/v1/cli/output.py +++ b/verifiers/v1/cli/output.py @@ -124,7 +124,11 @@ def write_launch_toml(results_dir: Path, name: str = "eval") -> None: tomls = [(p, p.read_text()) for p in paths if p.suffix == ".toml" and p.is_file()] if not tomls: return - texts = [text for _, text in tomls] if len(tomls) == 1 else [f"# @ {p}\n{text}" for p, text in tomls] + texts = ( + [text for _, text in tomls] + if len(tomls) == 1 + else [f"# @ {p}\n{text}" for p, text in tomls] + ) config_dir = results_dir / CONFIG_DIR config_dir.mkdir(parents=True, exist_ok=True) (config_dir / f"{name}.toml").write_text("\n".join(texts))