feat: split run configs into launch TOML and resolved dir

verifiers/v1/cli/eval/main.py

@@ -31,7 +31,7 @@
 
 USAGE = (
     "usage: uv run eval [<taskset-id>] [--env.id <id>] [options] [@ file.toml]\n"
-    "       uv run eval @ <run-dir>/configs/eval.json --resume   (re-run the run's missing/errored rollouts)"
+    "       uv run eval @ <run-dir>/configs/resolved/eval.json --resume   (re-run the run's missing/errored rollouts)"
 )
 
 

verifiers/v1/cli/eval/runner.py

@@ -140,7 +140,6 @@ async def run(slot: RunSlot) -> Episode:
 async def run_eval(config: EvalConfig) -> list[Episode]:
     from verifiers.v1.utils.loaders import load_environment, load_taskset
 
-    logger.info("eval config:\n%s", config.model_dump_json(indent=2))
     # The env comes up in this process only for an in-process run; a served run's
     # workers each load their own, and this process owns just the taskset.
     env = None if config.serve is not None else load_environment(config.env)

verifiers/v1/cli/output.py

@@ -28,9 +28,13 @@
 """Filename a run's rollout episodes are written to (one JSON episode per line)."""
 
 CONFIG_DIR = "configs"
-"""Directory inside a run dir holding its resolved config (one `<cli>.json`, re-runnable
-via `@ <run-dir>/configs/<cli>.json`). Resolved configs are JSON, not TOML: JSON keeps
-nulls, so explicit None settings round-trip exactly on re-parse."""
+"""Directory inside a run dir holding its configs: the launch TOML copied verbatim to
+`configs/<cli>.toml`, and the resolved config at `configs/resolved/<cli>.json`
+(re-runnable via `@ <run-dir>/configs/resolved/<cli>.json`). Resolved configs are JSON,
+not TOML: JSON keeps nulls, so explicit None settings round-trip exactly on re-parse."""
+
+RESOLVED_DIR = "resolved"
+"""Subdirectory of `configs/` holding the resolved per-component JSON dumps."""
 
 
 def create_attempt_log_dir(run_dir: Path) -> Path:
@@ -68,13 +72,13 @@
 
 
 def saved_config_path(run_dir: Path) -> Path | None:
-    """The run's saved resolved config (`configs/<cli>.json`), None if absent."""
-    candidates = (
-        sorted((run_dir / CONFIG_DIR).glob("*.json"))
-        if (run_dir / CONFIG_DIR).is_dir()
-        else []
-    )
-    return candidates[0] if candidates else None
+    """The run's saved resolved config (`configs/resolved/<cli>.json`; legacy runs
+    kept it at `configs/<cli>.json`), None if absent."""
+    for config_dir in (run_dir / CONFIG_DIR / RESOLVED_DIR, run_dir / CONFIG_DIR):
+        candidates = sorted(config_dir.glob("*.json")) if config_dir.is_dir() else []
+        if candidates:
+            return candidates[0]
+    return None
 
 
 # Compiling an adapter is the expensive part; run output reuses only a few model classes.
@@ -92,22 +96,48 @@
 def write_config(
     config: BaseModel, results_dir: Path, filename: str = "eval.json"
 ) -> Path:
-    """Write the run's resolved config to `configs/<filename>` (re-readable via
+    """Write the run's resolved config to `configs/resolved/<filename>` (re-readable via
     `@ <path>`); return its path. The full model dump is written, nulls included, so the
     file round-trips exactly."""
-    config_dir = results_dir / CONFIG_DIR
+    config_dir = results_dir / CONFIG_DIR / RESOLVED_DIR
     config_dir.mkdir(parents=True, exist_ok=True)
     config_path = config_dir / filename
     config_path.write_text(json.dumps(config.model_dump(mode="json"), indent=2))
     return config_path
 
 
+def write_launch_toml(results_dir: Path, name: str = "eval") -> None:
+    """Copy the launch `@` TOML file(s) verbatim to `configs/<name>.toml`."""
+    import sys
+
+    argv = sys.argv[1:]
+    paths = []
+    for i, arg in enumerate(argv):
+        # root config references only: `@ file`; a `--flag @ file` / `--flag @file`
+        # is a nested reference and belongs under its flag, not in the launch copy
+        if (
+            arg == "@"
+            and i + 1 < len(argv)
+            and (i == 0 or not argv[i - 1].startswith("--"))
+        ):
+            paths.append(Path(argv[i + 1]))
+    tomls = [(p, p.read_text()) for p in paths if p.suffix == ".toml" and p.is_file()]
+    if not tomls:
+        return
+    texts = [text for _, text in tomls] if len(tomls) == 1 else [f"# @ {p}\n{text}" for p, text in tomls]
+    config_dir = results_dir / CONFIG_DIR
+    config_dir.mkdir(parents=True, exist_ok=True)
+    (config_dir / f"{name}.toml").write_text("\n".join(texts))
+
+
 def save_config(
     config: BaseModel, results_dir: Path, filename: str = "eval.json"
 ) -> None:
-    """Set up the run's output dir: write the resolved config and start a fresh (empty)
-    `traces.jsonl`. Call once up front, before episodes start landing."""
+    """Set up the run's output dir: write the resolved config, copy the launch TOML,
+    and start a fresh (empty) `traces.jsonl`. Call once up front, before episodes start
+    landing."""
     write_config(config, results_dir, filename)
+    write_launch_toml(results_dir, Path(filename).stem)
     (results_dir / TRACES_FILE).write_text(
         ""
     )  # fresh; appended to as rollouts complete