diff --git a/verifiers/v1/cli/eval/main.py b/verifiers/v1/cli/eval/main.py index 3af8baf613..b245e42f4a 100644 --- a/verifiers/v1/cli/eval/main.py +++ b/verifiers/v1/cli/eval/main.py @@ -31,7 +31,7 @@ USAGE = ( "usage: uv run eval [] [--env.id ] [options] [@ file.toml]\n" - " uv run eval @ /configs/eval.json --resume (re-run the run's missing/errored rollouts)" + " uv run eval @ /configs/resolved/eval.json --resume (re-run the run's missing/errored rollouts)" ) diff --git a/verifiers/v1/cli/eval/runner.py b/verifiers/v1/cli/eval/runner.py index 2b593fc5de..792a281d79 100644 --- a/verifiers/v1/cli/eval/runner.py +++ b/verifiers/v1/cli/eval/runner.py @@ -140,7 +140,6 @@ async def run(slot: RunSlot) -> Episode: async def run_eval(config: EvalConfig) -> list[Episode]: from verifiers.v1.utils.loaders import load_environment, load_taskset - logger.info("eval config:\n%s", config.model_dump_json(indent=2)) # The env comes up in this process only for an in-process run; a served run's # workers each load their own, and this process owns just the taskset. env = None if config.serve is not None else load_environment(config.env) diff --git a/verifiers/v1/cli/output.py b/verifiers/v1/cli/output.py index cfef9fe48f..f651bd5844 100644 --- a/verifiers/v1/cli/output.py +++ b/verifiers/v1/cli/output.py @@ -28,9 +28,13 @@ """Filename a run's rollout episodes are written to (one JSON episode per line).""" CONFIG_DIR = "configs" -"""Directory inside a run dir holding its resolved config (one `.json`, re-runnable -via `@ /configs/.json`). Resolved configs are JSON, not TOML: JSON keeps -nulls, so explicit None settings round-trip exactly on re-parse.""" +"""Directory inside a run dir holding its configs: the launch TOML copied verbatim to +`configs/.toml`, and the resolved config at `configs/resolved/.json` +(re-runnable via `@ /configs/resolved/.json`). Resolved configs are JSON, +not TOML: JSON keeps nulls, so explicit None settings round-trip exactly on re-parse.""" + +RESOLVED_DIR = "resolved" +"""Subdirectory of `configs/` holding the resolved per-component JSON dumps.""" def create_attempt_log_dir(run_dir: Path) -> Path: @@ -68,13 +72,13 @@ def attempt_log_file(run_dir: Path) -> Path: def saved_config_path(run_dir: Path) -> Path | None: - """The run's saved resolved config (`configs/.json`), None if absent.""" - candidates = ( - sorted((run_dir / CONFIG_DIR).glob("*.json")) - if (run_dir / CONFIG_DIR).is_dir() - else [] - ) - return candidates[0] if candidates else None + """The run's saved resolved config (`configs/resolved/.json`; legacy runs + kept it at `configs/.json`), None if absent.""" + for config_dir in (run_dir / CONFIG_DIR / RESOLVED_DIR, run_dir / CONFIG_DIR): + candidates = sorted(config_dir.glob("*.json")) if config_dir.is_dir() else [] + if candidates: + return candidates[0] + return None # Compiling an adapter is the expensive part; run output reuses only a few model classes. @@ -92,22 +96,52 @@ def output_path(config: EvalConfig) -> Path: def write_config( config: BaseModel, results_dir: Path, filename: str = "eval.json" ) -> Path: - """Write the run's resolved config to `configs/` (re-readable via + """Write the run's resolved config to `configs/resolved/` (re-readable via `@ `); return its path. The full model dump is written, nulls included, so the file round-trips exactly.""" - config_dir = results_dir / CONFIG_DIR + config_dir = results_dir / CONFIG_DIR / RESOLVED_DIR config_dir.mkdir(parents=True, exist_ok=True) config_path = config_dir / filename config_path.write_text(json.dumps(config.model_dump(mode="json"), indent=2)) return config_path +def write_launch_toml(results_dir: Path, name: str = "eval") -> None: + """Copy the launch `@` TOML file(s) verbatim to `configs/.toml`.""" + import sys + + argv = sys.argv[1:] + paths = [] + for i, arg in enumerate(argv): + # root config references only: `@ file`; a `--flag @ file` / `--flag @file` + # is a nested reference and belongs under its flag, not in the launch copy + if ( + arg == "@" + and i + 1 < len(argv) + and (i == 0 or not argv[i - 1].startswith("--")) + ): + paths.append(Path(argv[i + 1])) + tomls = [(p, p.read_text()) for p in paths if p.suffix == ".toml" and p.is_file()] + if not tomls: + return + texts = ( + [text for _, text in tomls] + if len(tomls) == 1 + else [f"# @ {p}\n{text}" for p, text in tomls] + ) + config_dir = results_dir / CONFIG_DIR + config_dir.mkdir(parents=True, exist_ok=True) + (config_dir / f"{name}.toml").write_text("\n".join(texts)) + + def save_config( config: BaseModel, results_dir: Path, filename: str = "eval.json" ) -> None: - """Set up the run's output dir: write the resolved config and start a fresh (empty) - `traces.jsonl`. Call once up front, before episodes start landing.""" + """Set up the run's output dir: write the resolved config, copy the launch TOML, + and start a fresh (empty) `traces.jsonl`. Call once up front, before episodes start + landing.""" write_config(config, results_dir, filename) + write_launch_toml(results_dir, Path(filename).stem) (results_dir / TRACES_FILE).write_text( "" ) # fresh; appended to as rollouts complete