From 3ceaf952c9353f328a6fe18289f34ed334821fbd Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 15 Sep 2026 17:24:50 +0000 Subject: [PATCH 1/2] remove the eval CLI prime-rl's `uv run eval` replaces it: the `eval` console script, its runner, resume, rich dashboard, platform upload and example configs go, along with the `prime-runs` dependency. `RunConfig` moves to `configs/cli/run.py` for the run-directory CLIs that stay (validate, gepa), and the e2e suite runs tasksets in-process. --- AGENTS.md | 4 +- configs/agentic_judge.toml | 17 - configs/alphabet_sort.toml | 18 - configs/code_golf.toml | 12 - configs/deepwiki.toml | 9 - configs/glossary.toml | 9 - configs/gsm8k.toml | 10 - configs/gsm8k_rlm.toml | 21 - configs/harbor.toml | 17 - configs/textarena.toml | 20 - configs/wiki_search.toml | 9 - configs/wiki_search_compact.toml | 13 - configs/wordle.toml | 16 - docs/mint.json | 1 - docs/v1/env.md | 4 +- docs/v1/evaluation.md | 80 --- docs/v1/gepa.md | 6 +- docs/v1/getting_started.md | 2 +- docs/v1/overview.md | 1 - docs/v1/tasksets.md | 8 +- pyproject.toml | 2 - skills/evaluate-environments/SKILL.md | 222 ------ tests/v1/conftest.py | 17 +- tests/v1/runner.py | 76 +++ tests/v1/test_configs.py | 18 +- tests/v1/test_envs.py | 69 +- tests/v1/test_platform_run.py | 104 --- uv.lock | 31 +- verifiers/v1/cli/dashboard/__init__.py | 3 +- verifiers/v1/cli/dashboard/eval.py | 904 ------------------------- verifiers/v1/cli/eval/__init__.py | 0 verifiers/v1/cli/eval/hint.py | 5 - verifiers/v1/cli/eval/main.py | 140 ---- verifiers/v1/cli/eval/resume.py | 58 -- verifiers/v1/cli/eval/runner.py | 166 ----- verifiers/v1/cli/init.py | 4 +- verifiers/v1/cli/output.py | 12 +- verifiers/v1/configs/cli/eval.py | 150 ---- verifiers/v1/configs/cli/run.py | 42 ++ verifiers/v1/configs/cli/validate.py | 2 +- verifiers/v1/configs/client.py | 5 +- verifiers/v1/gepa/adapter.py | 2 +- verifiers/v1/gepa/config.py | 2 +- verifiers/v1/gepa/runner.py | 6 +- verifiers/v1/utils/platform.py | 179 ----- 45 files changed, 178 insertions(+), 2318 deletions(-) delete mode 100644 configs/agentic_judge.toml delete mode 100644 configs/alphabet_sort.toml delete mode 100644 configs/code_golf.toml delete mode 100644 configs/deepwiki.toml delete mode 100644 configs/glossary.toml delete mode 100644 configs/gsm8k.toml delete mode 100644 configs/gsm8k_rlm.toml delete mode 100644 configs/harbor.toml delete mode 100644 configs/textarena.toml delete mode 100644 configs/wiki_search.toml delete mode 100644 configs/wiki_search_compact.toml delete mode 100644 configs/wordle.toml delete mode 100644 docs/v1/evaluation.md delete mode 100644 skills/evaluate-environments/SKILL.md create mode 100644 tests/v1/runner.py delete mode 100644 tests/v1/test_platform_run.py delete mode 100644 verifiers/v1/cli/dashboard/eval.py delete mode 100644 verifiers/v1/cli/eval/__init__.py delete mode 100644 verifiers/v1/cli/eval/hint.py delete mode 100644 verifiers/v1/cli/eval/main.py delete mode 100644 verifiers/v1/cli/eval/resume.py delete mode 100644 verifiers/v1/cli/eval/runner.py delete mode 100644 verifiers/v1/configs/cli/eval.py create mode 100644 verifiers/v1/configs/cli/run.py delete mode 100644 verifiers/v1/utils/platform.py diff --git a/AGENTS.md b/AGENTS.md index 3e9df124c0..af14dd4e24 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,7 @@ ## Running code - **Always use uv**: run code and commands with `uv run`, never raw `python`. Make sure `uv` is installed ([docs](https://docs.astral.sh/uv/getting-started/installation/)). -- **Scaffold environments**: create a new taskset/environment with `uv run vf-init ` (`uv run vf-init -h` lists options like `-T`/`-H`), and run evals with `uv run vf-eval `. +- **Scaffold environments**: create a new taskset/environment with `uv run vf-init ` (`uv run vf-init -h` lists options like `-T`/`-H`), validate them model-free with `uv run vf-validate `, and evaluate them with prime-rl's `uv run eval `. - **Validate TOML first**: validate config `.toml` files before running them. - **Don't add dependencies**: never add dependencies or optional extras to the top-level `pyproject.toml`. @@ -20,7 +20,7 @@ ## Skills -- **Use bundled skills first**: the skills in `skills/` cover the core workflows — `create-environments` (build or migrate a v1 taskset/environment/harness), `evaluate-environments` (configure and run evals), `release` (publish stable versions), and `brainstorm` (ideation and research planning). Reach for them before doing the work by hand. +- **Use bundled skills first**: the skills in `skills/` cover the core workflows — `create-environments` (build or migrate a v1 taskset/environment/harness), `release` (publish stable versions), and `brainstorm` (ideation and research planning). Reach for them before doing the work by hand. ## Testing diff --git a/configs/agentic_judge.toml b/configs/agentic_judge.toml deleted file mode 100644 index decf371139..0000000000 --- a/configs/agentic_judge.toml +++ /dev/null @@ -1,17 +0,0 @@ -[select] -limit = 2 - -[env] -id = "agentic-judge" - -[env.taskset] -id = "gsm8k" - -[env.solver.harness] -id = "bash" - -[env.solver.runtime] -type = "docker" - -[env.judge] -model = "z-ai/glm-5.2" diff --git a/configs/alphabet_sort.toml b/configs/alphabet_sort.toml deleted file mode 100644 index 567c903fa9..0000000000 --- a/configs/alphabet_sort.toml +++ /dev/null @@ -1,18 +0,0 @@ -# alphabet-sort — multi-turn alphabetical sorting, driven by a colocated user simulator. -# -# uv run vf-eval @ configs/alphabet_sort.toml --model -num_rollouts = 2 - -[select] -limit = 5 - -[env.taskset] -id = "alphabet-sort" -min_turns = 2 -max_turns = 2 - -[env.agent] -max_turns = 4 - -[env.agent.harness] -id = "null" diff --git a/configs/code_golf.toml b/configs/code_golf.toml deleted file mode 100644 index 9638b0cce6..0000000000 --- a/configs/code_golf.toml +++ /dev/null @@ -1,12 +0,0 @@ -# code-golf — the taskset ships a recipe env: one episode runs -# `--env.attempts` independent attempts (default 2) and scores them against -# each other (shortest wins), so one rollout per task is a full comparison. -# -# uv run vf-eval code-golf @ configs/code_golf.toml -num_rollouts = 1 - -[select] -limit = 1 - -[env.taskset] -id = "code-golf" diff --git a/configs/deepwiki.toml b/configs/deepwiki.toml deleted file mode 100644 index e8d2a01483..0000000000 --- a/configs/deepwiki.toml +++ /dev/null @@ -1,9 +0,0 @@ -# deepwiki — an existing remote MCP server, reached by URL (nothing to run ourselves). -# -# uv run vf-eval @ configs/deepwiki.toml - -[select] -limit = 1 - -[env.taskset] -id = "deepwiki" diff --git a/configs/glossary.toml b/configs/glossary.toml deleted file mode 100644 index 5b01e22bd7..0000000000 --- a/configs/glossary.toml +++ /dev/null @@ -1,9 +0,0 @@ -# glossary — a colocated MCP tool server (runs inside the harness's own runtime). -# -# uv run vf-eval glossary @ configs/glossary.toml - -[select] -limit = 1 - -[env.taskset] -id = "glossary" diff --git a/configs/gsm8k.toml b/configs/gsm8k.toml deleted file mode 100644 index 6bda1f2b73..0000000000 --- a/configs/gsm8k.toml +++ /dev/null @@ -1,10 +0,0 @@ -# gsm8k — grade-school math, bash harness; a few tasks with several rollouts each. -# -# uv run vf-eval gsm8k @ configs/gsm8k.toml -num_rollouts = 3 - -[select] -limit = 5 - -[env.taskset] -id = "gsm8k" diff --git a/configs/gsm8k_rlm.toml b/configs/gsm8k_rlm.toml deleted file mode 100644 index 35067c97b7..0000000000 --- a/configs/gsm8k_rlm.toml +++ /dev/null @@ -1,21 +0,0 @@ -# The gsm8k taskset run with the rlm harness, configured entirely from TOML. -# -# uv run vf-eval @ configs/gsm8k_rlm.toml -# -# The discriminator ids (which taskset, which harness) come from this file's -# [env.taskset] / [env.agent.harness] `id` — they select the schema, and the rest of the -# file fills the typed config, each side validated against its specific type (GSM8KConfig / -# RLMHarnessConfig). prime-rl consumes the same [env] shape, reading the ids straight from -# its config (TOML-driven there too). - -[select] -limit = 1 - -[env.taskset] -id = "gsm8k" - -[env.agent.harness] -id = "rlm" - -[env.agent.runtime] -type = "docker" diff --git a/configs/harbor.toml b/configs/harbor.toml deleted file mode 100644 index 389ae38484..0000000000 --- a/configs/harbor.toml +++ /dev/null @@ -1,17 +0,0 @@ -# harbor — Terminal-Bench 2, each task run in its own declared container image (needs docker). -# The `bash` harness gives the agent the bash (+ edit) tools it drives the terminal with. -# -# uv run vf-eval @ configs/harbor.toml - -[select] -limit = 10 - -[env.taskset] -id = "harbor" -dataset = "terminal-bench/terminal-bench-2" - -[env.agent] -runtime = { type = "docker" } - -[env.agent.harness] -id = "bash" diff --git a/configs/textarena.toml b/configs/textarena.toml deleted file mode 100644 index a151e397e3..0000000000 --- a/configs/textarena.toml +++ /dev/null @@ -1,20 +0,0 @@ -# textarena — single-player TextArena games, the engine playing the user. -# Supported (--env.taskset.game): Wordle-v0, Wordle-v0-long, Hangman-v0, WordLadder-v0, -# WordSearch-v0. The env's rollout() steps the real engine host-side as the run's user; -# the harness only ever sees the conversation. -# -# uv run vf-eval @ configs/textarena.toml --model -num_rollouts = 2 - -[select] -limit = 5 - -[env.taskset] -id = "textarena" -game = "Wordle-v0" - -[env.player] -max_turns = 15 - -[env.player.harness] -id = "null" diff --git a/configs/wiki_search.toml b/configs/wiki_search.toml deleted file mode 100644 index f7d73af274..0000000000 --- a/configs/wiki_search.toml +++ /dev/null @@ -1,9 +0,0 @@ -# wiki-search — a shared MCP tool server (the corpus is built once, reused across rollouts). -# -# uv run vf-eval @ configs/wiki_search.toml - -[select] -limit = 1 - -[env.taskset] -id = "wiki-search" diff --git a/configs/wiki_search_compact.toml b/configs/wiki_search_compact.toml deleted file mode 100644 index 7c757aca86..0000000000 --- a/configs/wiki_search_compact.toml +++ /dev/null @@ -1,13 +0,0 @@ -# wiki-search with the compact harness — the transcript is rewritten (branched) each turn, -# so the model always sees a compacted context instead of the full history. -# -# uv run vf-eval @ configs/wiki_search_compact.toml - -[select] -limit = 1 - -[env.taskset] -id = "wiki-search" - -[env.agent.harness] -id = "compact" diff --git a/configs/wordle.toml b/configs/wordle.toml deleted file mode 100644 index 9416acedba..0000000000 --- a/configs/wordle.toml +++ /dev/null @@ -1,16 +0,0 @@ -# wordle — the textarena taskset pinned to Wordle, the engine playing the user. -# -# uv run vf-eval @ configs/wordle.toml --model -num_rollouts = 2 - -[select] -limit = 5 - -[env.taskset] -id = "wordle" - -[env.player] -max_turns = 8 - -[env.player.harness] -id = "null" diff --git a/docs/mint.json b/docs/mint.json index e92b16edca..fb67c6ce28 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -12,7 +12,6 @@ "v1/getting_started", "v1/architecture", "v1/tasksets", - "v1/evaluation", "v1/harnesses", "v1/env", "v1/harbor", diff --git a/docs/v1/env.md b/docs/v1/env.md index 33ea518e0f..071e7bd492 100644 --- a/docs/v1/env.md +++ b/docs/v1/env.md @@ -23,11 +23,11 @@ verifiers comes with different pre-built `Env`s to use: ## Isolated deterministic verification Select `--env.id isolated-verifier` when the task's score must not run in the -solver's sandbox. It is still a one-agent run: the environment records one solver +solver's sandbox in a Prime-RL evaluation. It is still a one-agent run: the environment records one solver trace and starts no verifier agent, model, or harness. ```bash -uv run vf-eval my-task --env.id isolated-verifier --env.agent.runtime.type docker +uv run eval my-task --env.id isolated-verifier --env.agent.runtime.type docker ``` Task authors use the existing task API. Declare every solver output the verifier diff --git a/docs/v1/evaluation.md b/docs/v1/evaluation.md deleted file mode 100644 index f1155759aa..0000000000 --- a/docs/v1/evaluation.md +++ /dev/null @@ -1,80 +0,0 @@ -# Evaluation - -To evaluate any taskset, use the `eval` entrypoint: - -```bash -uv run vf-eval primeintellect/terminal-bench-2 -``` - -You can also use `.toml` files for configuration: - -```toml -model = "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B" - -[sampling] -temperature = 1.0 - -[env.taskset] -id = "primeintellect/terminal-bench-2" - -[env.agent.harness] -id = "codex" -version = "0.116.0" - -[env.agent.runtime] -type = "docker" -``` - -Validate the config by using `uv run vf-eval @ config.toml --dry-run`. To run the evaluation, use `uv run vf-eval @ config.toml`. - -Use dotted arguments to set values using the CLI, e.g. `--sampling.temperature 0.5`. CLI arguments overwrite toml arguments when both are present. - -The output from evaluations are written into `outputs/----//` by default, where `` is the taskset, prefixed by the paired env id when `--env.id` sets one (use `output_dir` to overwrite the folder). The folder contains the used `config.toml`, all the episodes in `traces.jsonl`, as well as logs of the run and workers in `logs/attempt_/eval.log` — one directory per launch attempt (a resume starts a new one), with `logs/latest` pointing at the current attempt. - -## Common config values - -- `model` — the model id to evaluate, e.g. `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B` -- `sampling` — generation params passed to the model, e.g. `sampling.temperature` -- `env.taskset.id` — pick the taskset (or the positional `eval `) -- `env.agent.harness.id` — pick the agent's harness (`[env.agent.harness]` in TOML) -- `select` — which tasks to evaluate (see [Selecting tasks](tasksets.md#selecting-tasks)). - `-n` sets `select.limit`: not setting it means all tasks, and an infinite taskset (a - procedural generator, e.g. `wordle`) requires it. `-s` sets `select.shuffle` (fixed seed) -- `num_rollouts` — rollouts per task -- `verbose` — log at debug instead of info -- `rich` — the live dashboard (default); `--no-rich` streams logs to the console and - prints each trace as JSON at the end -- `rich.show_logs` — replace the dashboard's per-rollout rows with a live tail of the - attempt's logs (`logs/latest/eval.log`), the env workers' lines included -- `push` — upload the run to the Prime Intellect platform as it goes (default; `--no-push` - keeps it local). `run.attach ` streams into a run the platform already - created instead of opening a new one — this is how a hosted evaluation's sandbox runs - the same command; it is not something a local run sets by hand - -## Resuming evaluations - -`--resume ` re-runs only the rollouts a previous run left missing or errored, appending to that run's own `traces.jsonl`. It reloads the run's saved `config.toml` verbatim, so it takes no other arguments. Good rollouts are kept, while errored ones are dropped and redone. - -## Disabling tools - -Almost every harness comes with a `disabled_tools` list, which can be used to disable one or multiple tools: - -```toml -[env.agent.harness] -disabled_tools = ["shell_tool"] -``` - -The names of these tools are set by the respective harness. Consult the relevant documentation for the given harness for the relevant name(s). Some harnesses do not offer support to disable tools. - -## Skills - -Harnesses whose program supports SKILL.md skills natively (e.g. Claude Code, Codex) take a `skills` list. A local skill folder is uploaded to `/`. A `{runtime = "..."}` entry copies the contents of a directory already inside the runtime into the program's skill discovery directory: - -```toml -[env.agent.harness] -skills = [{runtime = "/opt/skills"}, "path/to/my-skill"] -``` - -Tasks can supply the same sources through `TaskData.skills`, for example `skills=[{"runtime": "/opt/skills"}]`. Task sources are installed first, followed by harness sources; later files override matching earlier files. Each run gets its own installed skills. Sources are resolved after task setup, and missing source directories fail the run. - -Setting `skills` on a task or harness without native harness skill support fails up front. diff --git a/docs/v1/gepa.md b/docs/v1/gepa.md index 218d8313ab..2abf6ff639 100644 --- a/docs/v1/gepa.md +++ b/docs/v1/gepa.md @@ -35,13 +35,13 @@ Validate the config by using `uv run vf-gepa @ config.toml --dry-run`. To run GE ## Output -Results go under `outputs/----//`, matching `eval`. +Results go under `outputs/------/`. The best system prompt is printed when the run finishes and written to `best_system_prompt.txt` in that folder. -Hand it back to eval or training via the config-layer taskset system prompt: +Hand it back to evaluation or training via the config-layer taskset system prompt (prime-rl's `uv run eval`, or an `[[orchestrator.train.source]]`): ```bash -uv run vf-eval reverse-text \ +uv run eval reverse-text \ --env.taskset.system-prompt outputs//best_system_prompt.txt ``` diff --git a/docs/v1/getting_started.md b/docs/v1/getting_started.md index d8958b6011..1df28edf4f 100644 --- a/docs/v1/getting_started.md +++ b/docs/v1/getting_started.md @@ -9,7 +9,7 @@ cd verifiers uv sync ``` -You can now run tasksets directly, e.g. `uv run vf-eval `, and scaffold new ones with `uv run vf-init `. +Scaffold new tasksets with `uv run vf-init ` and check them model-free with `uv run vf-validate `. Run evaluations through [Prime-RL](https://github.com/PrimeIntellect-ai/prime-rl/blob/main/docs/eval.md) with `uv run eval `. ## Skills diff --git a/docs/v1/overview.md b/docs/v1/overview.md index 295774b153..0b2cae0f9c 100644 --- a/docs/v1/overview.md +++ b/docs/v1/overview.md @@ -38,7 +38,6 @@ A trace records the message graph, rewards, metrics, errors, and one per-call re - [Architecture](architecture.md) — How verifiers works behind-the-scenes - [Tasksets](tasksets.md) — How to create tasksets - [Harbor Tasksets](harbor.md) — How to create Harbor-based tasksets -- [Evaluation](evaluation.md) — How to evaluate tasksets - [Harnesses](harnesses.md) — How to build custom harnesses - [Agent](agent.md) — How to run standalone agents - [Env](env.md) — How to build multi-agent environments diff --git a/docs/v1/tasksets.md b/docs/v1/tasksets.md index be03e6c730..948ee759cf 100644 --- a/docs/v1/tasksets.md +++ b/docs/v1/tasksets.md @@ -126,12 +126,12 @@ taskset.include(idx=["0:100"]).exclude(names=["broken"]).shuffle(seed=0).take(5) `include` keeps and `exclude` drops the tasks named by `idx` (ints and Python slices `start:stop:step`, e.g. `"100:"`, `":50"`, `"::2"`; no negative positions), `ids`, `keys` or `names`; a task matches when any list names it. An entry that matches no task logs a warning once the stream was read. -The eval, debug, validate and GEPA entrypoints take a `select` block and apply it with `taskset.select(config.select)`, which chains the views in a fixed order: `include`, `exclude`, `shuffle`, `skip`, `limit`: +Prime-RL's eval and Verifiers' debug, validate and GEPA entrypoints take a `select` block and apply it with `taskset.select(config.select)`, which chains the views in a fixed order: `include`, `exclude`, `shuffle`, `skip`, `limit`: ```bash -uv run vf-eval gsm8k --select.limit 50 --select.shuffle # 50 random tasks (short: -n 50 -s) -uv run vf-eval gsm8k --select.include.idx 0:100 # the first 100 tasks -uv run vf-eval gsm8k --select.shuffle --select.skip 100 --select.limit 100 # a random split disjoint from the first +uv run eval gsm8k --select.limit 50 --select.shuffle # 50 random tasks (short: -n 50 -s) +uv run eval gsm8k --select.include.idx 0:100 # the first 100 tasks +uv run eval gsm8k --select.shuffle --select.skip 100 --select.limit 100 # a random split disjoint from the first ``` ```toml diff --git a/pyproject.toml b/pyproject.toml index 59641800ff..ea36f53345 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,6 @@ dependencies = [ "openai>=2.54.0,<3.0.0", "prime-tunnel>=0.1.12", "prime-sandboxes>=0.4.2", - "prime-runs>=0.1.4", "pydantic>=2.12.3", "rich>=11.0.0", "tenacity>=8.5.0", @@ -154,7 +153,6 @@ prime-pydantic-config = false renderers = false [project.scripts] -vf-eval = "verifiers.v1.cli.eval.main:main" vf-validate = "verifiers.v1.cli.validate:main" vf-debug = "verifiers.v1.cli.debug:main" vf-replay = "verifiers.v1.cli.replay:main" diff --git a/skills/evaluate-environments/SKILL.md b/skills/evaluate-environments/SKILL.md deleted file mode 100644 index 4848010b62..0000000000 --- a/skills/evaluate-environments/SKILL.md +++ /dev/null @@ -1,222 +0,0 @@ ---- -name: evaluate-environments -description: Run and evaluate verifiers tasksets. Set up the necessary config files and observe the runs and their results. ---- - -# Evaluate Tasksets - -## Goal - -Set up an evaluation for a taskset in the correct way to reproduce results from others or evaluate a model and harness combination on a given taskset. - -## Canonical path - -Use the `eval` entrypoint - -```bash -uv run vf-eval -``` - -## Core workflow - -1. Resolve and validate config without model calls: - -```bash -uv run vf-eval --dry-run -``` - -2. Run model-free validation. Each task gets two checks in independent runtimes: gold (`setup`, then `validate` applies and checks the reference answer; unchecked when the task has no `validate`) and noop (`setup`, then `finalize` and the task's full scoring, configured judges included, on the untouched task; invalid when its reward already reaches 1.0, unchecked when it has no reward). `--only-gold` / `--only-noop` run one; `--only-setup` only checks that `setup` completes: - -```bash -uv run vf-validate --runtime.type subprocess -``` - -3. Do a small run to see whether it works correctly: - -```bash -uv run vf-eval -m deepseek/deepseek-v4-flash -n 3 -r 1 -``` - -4. Inspect successful, zero-reward, and errored traces. -5. Scale only after task loading, harness capability, runtime lifecycle, and scoring are correct. - -When the user requests a full run, do not restrict the number of tasks. Ask for the appropriate harness to use (if not specified) - -## IDs and plugin resolution - -A plugin id names an installed package (e.g. `my-taskset`); verifiers imports it and never installs anything itself. - -The leading ID is shorthand for `--env.taskset.id`. A harness belongs to an agent — `--env.agent.harness.*` on the single-agent env, `--env..harness.*` on a multi-agent one (there is no run-level `--harness.*`): - -```bash -uv run vf-eval my-task-v1 --env.agent.harness.id codex --env.agent.runtime.type prime -``` - -The env — the control flow between agents — owns the whole `[env]` block. Empty `--env.id` -keeps the taskset's own story (its exported `Env` subclass, else the single-agent -env); `--env.id` pairs a reusable env with any taskset, its knobs typed under `--env.*`: - -```bash -uv run vf-eval my-task-v1 --env.id best-of-n --env.n 8 # pass@k / rejection sampling -uv run vf-eval my-task-v1 --env.id agentic-judge \ - --env.judge.runtime.type docker # a judge agent verifies each attempt in a sandbox -``` - -## Disabling tools - -Almost every harness comes with a `disabled_tools` list, which can be used to disable one or multiple tools: - -```toml -[env.agent.harness] -disabled_tools = ["shell_tool"] -``` - -The names of these tools are set by the respective harness. Research the relevant first party documentation for the given harness for the relevant name(s). Some harnesses do not offer support to disable tools. - -## Config discovery - -The CLI help is generated from the current config classes. Include the taskset and env ids you plan to use before `--help` so their concrete config fields are loaded: - -```bash -uv run vf-eval my-task-v1 \ - --env.id best-of-n \ - --help -``` - -For implementation details and defaults, start at `verifiers/v1/configs/cli/eval.py` and follow its fields into `verifiers/v1/configs/`. Client configs live in `verifiers/v1/configs/client.py`, sampling in `verifiers/v1/types.py`, and runtime- and harness-specific configs next to their implementations in `verifiers/v1/runtimes/` and `verifiers/v1/harnesses/`. Custom taskset and env config fields live next to those implementations. - -## Typed taskset overrides - -Taskset settings: - -```bash -uv run vf-eval my-task-v1 --env.taskset.split test --env.taskset.difficulty hard -``` - -Harness and runtime settings: - -```bash -uv run vf-eval my-task-v1 \ - --env.agent.harness.id rlm \ - --env.agent.runtime.type docker \ - --env.agent.runtime.cpu 4 \ - --env.agent.runtime.memory 8 -``` - -Sampling: - -```bash -uv run vf-eval my-task-v1 \ - --sampling.temperature 0.7 \ - --sampling.top-p 0.95 \ - --sampling.max-tokens 2048 \ - --sampling.reasoning-effort medium -``` - -Always research the correct sampling parameters first. This is one of the most important settings, so make sure to find the correct values. For open models, you can find them on Hugging Face in the README and/or in the generation config. - -Your parameter selection or settings should leave room for full runs, and you should not restrict things like tokens or number of turns unless specified by the user. - -Leave optional settings unset unless the user asks for them. Always confirm the harness, runtime, and sampling parameters before running an evaluation. - -## Reproducible TOML - -You can also use a TOML: - -```toml -model = "openai/gpt-5-mini" - -[env.taskset] -id = "my-task-v1" -split = "test" - -[env.agent] -runtime = { type = "subprocess" } - -[env.agent.harness] -id = "bash" - -[sampling] -temperature = 0.7 -``` - -```bash -uv run vf-eval @ configs/my-eval.toml -``` - -## Retries - -Whole-rollout retry is off by default (`max_retries = 0`). Each retry starts a fresh rollout. Setting only `max_retries` retries any captured error up to that cap. Use `env.retries` for whole-episode retries or `env.agent.retries` for the agent alone. - -Ordered rules override the default budget for matching errors. With a zero default, only explicitly enabled errors retry. For example: - -```toml -[env.agent.retries] -max_retries = 0 - -[[env.agent.retries.rules]] -type = "ProviderError" -status_code = [429, "5xx"] -max_retries = 3 - -[[env.agent.retries.rules]] -type = "SandboxError" -message = 'temporarily unavailable|connection reset' -max_retries = 2 -``` - -Fields within a rule must all match. `type` matches the exact recorded exception name, `status_code` matches any listed status or status class, and `message` is a regex search (plain text matches a substring). Invalid regexes fail config validation. Omitted match fields match anything. Each rule must explicitly provide `max_retries`; an omitted budget fails validation. - -The first matching rule wins for each error; zero retries excludes it, and exhausted rules never fall through. Unmatched errors share the default `max_retries` budget. Each retry consumes only the matching rule's budget, or the default budget when no rule matches. Budgets persist across the entire run; the default is not a global cap. An empty rules list uses only the default budget. When an attempt captures multiple errors, the first eligible error triggers the retry; a denied error does not veto other errors. Successful traces' recovered errors do not trigger episode retries. - -## Output and resume - -A run writes to `output_dir / run.dir` (`-o` sets `output_dir`, default `outputs`; `run.dir` defaults to the auto-generated run name): - -```text -outputs/------/ -├── configs/eval.json -├── logs/eval.log -└── traces.jsonl -``` - -`configs/eval.json` is the run's resolved config, re-runnable via `@`. `traces.jsonl` is one **episode** per line — the episode's traces plus their shared standing — appended after each episode finishes, so an episode is durable whole or not at all (a torn last line is the whole episode redone on resume). - -Resume in place by re-running the run's own saved config with `--resume` (it re-runs only the missing/errored rollouts; any config drift from the saved run is refused): - -```bash -uv run vf-eval @ /configs/eval.json --resume -``` - -To overwrite a run dir and start fresh instead, use `--clean`. - -## Trace inspection - -For each representative sample inspect: - -- `task` and prompt fields; -- `branches`, assistant messages, tool messages, and stop condition; -- named `rewards`, aggregate `reward`, and `metrics`; -- persisted `info` artifacts; -- `error`/`errors` and boundary type; -- per-call `calls` records (model, sampling, finish reason, usage, timing, error) linked to the graph; -- usage and stage timing; -- token/mask/logprob fields when using the training client. - -Classify outcomes: - -1. Valid completion and correct reward. -2. Valid completion with low reward (model/task outcome). -3. Truncated completion (budget outcome). -4. Captured rollout error (provider, harness, tool, user, runtime, task, or interception). - -Do not average these categories together without reporting failure rate. - -## Metrics interpretation - -- Binary rewards support solve rate and pass@k-style analysis. -- Continuous rewards need distributions, quantiles, and per-task/group comparisons. -- Group rewards must be interpreted with their comparison rule and group size. -- Always inspect samples before attributing a delta to model quality. -- Keep taskset, harness, runtime, sampling, and selected task indices fixed across variants. -- Do not overinterpret a tiny smoke run. diff --git a/tests/v1/conftest.py b/tests/v1/conftest.py index f33d7f42c7..37a2b69a81 100644 --- a/tests/v1/conftest.py +++ b/tests/v1/conftest.py @@ -5,7 +5,7 @@ not unit tests of individual components. They need a model API key (`PRIME_API_KEY`); without one the `e2e`-marked tests skip (config parsing still runs). -`run_v1` mirrors the eval CLI (`run_eval`, in-process); `run_v1_server` drives the same env +`run_v1` runs the env in-process (`tests/v1/runner.py`); `run_v1_server` drives the same env through an env-server worker pool, the path prime-rl trains through. Placement coverage (harness x harness runtime x tool server runtime) is PAIRWISE, not a full cross product: each test carries a curated list of combinations (in test_e2e.py) that hits every axis value and the cross-boundary pairs with @@ -41,8 +41,7 @@ import pytest import verifiers.v1 as vf -from verifiers.v1.cli.eval.runner import run_eval -from verifiers.v1.configs.cli.eval import EvalConfig +from tests.v1.runner import RunnerConfig, run_episodes from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.trace import Trace from verifiers.v1.utils.loaders import harness_config_type @@ -134,8 +133,8 @@ def _eval_config( runtime: dict | None = None, env: dict | None = None, reasoning_effort: str | None = None, -) -> EvalConfig: - """Build the smallest `EvalConfig` that still exercises the path, shared by the in-process +) -> RunnerConfig: + """Build the smallest `RunnerConfig` that still exercises the path, shared by the in-process (`run_v1`) and env-server (`run_v1_server`) fixtures. `taskset_overrides` merges onto the `{id: ...}` config; `runtime` places the `agent` seat's harness (an agent field, not a harness one). @@ -179,7 +178,7 @@ def _eval_config( seat_cfg.setdefault("timeout", {"rollout": rollout_timeout, "scoring": 60}) # Agent runs retry locally; interactions retry with their whole episode. seat_cfg.setdefault("retries", retries) - return EvalConfig( + return RunnerConfig( env={ "taskset": taskset_cfg, **env_cfg, @@ -190,21 +189,19 @@ def _eval_config( "max_tokens": max_tokens, "reasoning_effort": reasoning_effort, }, - rich=None, output_dir=output_dir.parent, run={"dir": output_dir.name}, model=CI_MODEL, - push=False, ) @pytest.fixture def run_v1(): - """Run a v1 taskset end-to-end in-process (`run_eval`) and return its traces.""" + """Run a v1 taskset end-to-end in-process and return its traces.""" async def _run(taskset: str, **kwargs) -> list[Trace]: config = _eval_config(taskset, **kwargs) - records = await run_eval(config) + records = await run_episodes(config) # The runner answers durability envelopes; the tests assert on traces. return [t for r in records for t in r.traces] diff --git a/tests/v1/runner.py b/tests/v1/runner.py new file mode 100644 index 0000000000..e7235c8874 --- /dev/null +++ b/tests/v1/runner.py @@ -0,0 +1,76 @@ +"""Run an environment in-process for the e2e tests, saving episodes for replay.""" + +import asyncio +from pathlib import Path + +from pydantic import Field, SerializeAsAny, model_validator +from pydantic_config import BaseConfig + +from verifiers.v1.cli.output import output_path, save_config +from verifiers.v1.clients import ClientConfig, EvalClientConfig, ModelContext +from verifiers.v1.configs.cli.env import narrowed_env_annotation, resolve_env_field +from verifiers.v1.configs.cli.run import RunConfig, default_run_name +from verifiers.v1.configs.env import EnvConfig +from verifiers.v1.configs.select import SelectCLIConfig +from verifiers.v1.envs.single_agent import SingleAgentEnvConfig +from verifiers.v1.episode import Episode, EvalRunInfo +from verifiers.v1.types import SamplingConfig +from verifiers.v1.utils.loaders import load_environment +from verifiers.v1.utils.trace_store import append_episode + + +class RunnerConfig(BaseConfig): + """The environment, model, and task selection used by the test fixtures.""" + + env: SerializeAsAny[EnvConfig] = SingleAgentEnvConfig() + run: RunConfig = Field(default_factory=RunConfig) + model: str + client: ClientConfig = EvalClientConfig() + sampling: SamplingConfig = SamplingConfig() + select: SelectCLIConfig = SelectCLIConfig() + num_rollouts: int = 1 + max_concurrent: int | None = 128 + output_dir: Path = Path("outputs") + + @model_validator(mode="before") + @classmethod + def _resolve_env(cls, data): + return resolve_env_field(data, narrowed_env_annotation(cls)) + + @model_validator(mode="after") + def auto_setup_run_name(self): + if self.run.name is None: + self.run.name = default_run_name(self.env, self.model) + if self.run.dir is None: + self.run.dir = self.run.name + return self + + +async def run_episodes(config: RunnerConfig) -> list[Episode]: + """Execute the selected tasks and persist each completed episode.""" + env = load_environment(config.env) + tasks = list(env.taskset.select(config.select)) + out = output_path(config) + save_config(config, out, "run.json") + write_lock = asyncio.Lock() + + async def on_complete(episode: Episode) -> None: + episode.record_run(EvalRunInfo(id=config.run.id, name=config.run.name)) + await append_episode(out, episode, write_lock) + + ctx = ModelContext( + client=config.client, model=config.model, sampling=config.sampling + ) + semaphore = ( + asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None + ) + async with env.serving(): + return list( + await asyncio.gather( + *( + env.run_slot(slot, ctx, semaphore, on_complete) + for task in tasks + for slot in env.slots(task, config.num_rollouts) + ) + ) + ) diff --git a/tests/v1/test_configs.py b/tests/v1/test_configs.py index 2239ab2baf..4a95f039a0 100644 --- a/tests/v1/test_configs.py +++ b/tests/v1/test_configs.py @@ -1,26 +1,10 @@ -"""Every checked-in v1 eval config parses. - -Mirrors prime-rl's config test: glob the configs and assert each validates into its config -type. The root `configs/*.toml` are the `uv run vf-eval @ ` v1 configs (EvalConfig). -""" - -import tomllib -from pathlib import Path +"""Runtime configuration validation.""" import pytest -from verifiers.v1.configs.cli.eval import EvalConfig from verifiers.v1.runtimes import E2BConfig from verifiers.v1.runtimes.e2b import _egress_update -CONFIGS = sorted((Path(__file__).resolve().parents[2] / "configs").glob("*.toml")) - - -@pytest.mark.parametrize("path", CONFIGS, ids=lambda p: p.name) -def test_eval_config_parses(path: Path) -> None: - config = EvalConfig.model_validate(tomllib.load(path.open("rb"))) - assert config.env.taskset.id - @pytest.mark.parametrize( "rule", diff --git a/tests/v1/test_envs.py b/tests/v1/test_envs.py index f159bc877f..86e76e5209 100644 --- a/tests/v1/test_envs.py +++ b/tests/v1/test_envs.py @@ -1,34 +1,22 @@ -"""Smoke-eval every example taskset in `environments/` through the `eval` CLI. +"""Smoke-eval every example taskset in `environments/` in-process. -Each taskset runs with its required harness for one short, capped rollout, so a broken -example taskset fails CI. `compact` is excluded (it's a harness, not a taskset); -SWE/container tasksets need a docker/prime runtime and are covered by dedicated V1 e2e tests. +Each taskset runs with its required harness for one short, capped rollout through the +shared e2e runner, so a broken example taskset fails CI. `compact` is excluded (it's a +harness, not a taskset); SWE/container tasksets need a docker/prime runtime and are +covered by dedicated V1 e2e tests. """ -import os -import subprocess from pathlib import Path import pytest pytestmark = pytest.mark.e2e -EVAL_TIMEOUT = 600 # 10 minutes for a capped eval (-n 1 -r 1) - ENVIRONMENTS = Path(__file__).parent.parent.parent / "environments" # V1 tasksets that aren't part of the default CI install. SKIP_EVAL = {"nemo_gym_weather"} -# Per-run caps are seat fields; recipe envs name their own seats. -SEATS: dict[str, tuple[str, ...]] = { - "code_golf": ("golfer",), - "kuhn_poker": ("player0", "player1"), - "openenv_wordle": ("player",), - "proposer_solver": ("proposer", "solver"), - "wordle": ("player",), -} - def v1_tasksets() -> list[str]: if not ENVIRONMENTS.is_dir(): @@ -41,42 +29,19 @@ def v1_tasksets() -> list[str]: @pytest.mark.parametrize("taskset", v1_tasksets()) -def test_eval(taskset: str): +async def test_eval(run_v1, taskset: str, tmp_path: Path): """Run one capped rollout of `taskset`; a taskset that bundles a harness uses it by default.""" if taskset in SKIP_EVAL: pytest.skip(f"{taskset} can't run a plain-CI smoke eval") - if os.getenv("PRIME_API_KEY"): - model = [ - "-m", "openai/gpt-5.6-luna", - "--client.base-url", "https://api.pinference.ai/api/v1", - "--client.api-key-var", "PRIME_API_KEY", - ] # fmt: skip - elif os.getenv("OPENAI_API_KEY"): - model = [ - "-m", "gpt-4.1-mini", - "--client.base-url", "https://api.openai.com/v1", - "--client.api-key-var", "OPENAI_API_KEY", - ] # fmt: skip - else: - pytest.skip("no model API key configured") - - caps = [ - flag - for seat in SEATS.get(taskset, ("agent",)) - for flag in (f"--env.{seat}.max-turns", "4") - ] - cmd = [ - "uv", "run", "--no-sync", "vf-eval", taskset, - *model, - "-n", "1", "-r", "1", *caps, - "--sampling.max-tokens", "512", "--no-rich", "--no-push", - ] # fmt: skip - try: - proc = subprocess.run( - cmd, capture_output=True, text=True, timeout=EVAL_TIMEOUT, check=False - ) - except subprocess.TimeoutExpired: - pytest.fail(f"Timed out after {EVAL_TIMEOUT}s evaluating {taskset}") - assert proc.returncode == 0, ( - f"eval {taskset} failed: {(proc.stderr or proc.stdout)[-2000:]}" + # `harness=None`: every seat keeps the taskset's own harness; the runner caps each + # seat's turns. + traces = await run_v1( + taskset, + output_dir=tmp_path / taskset, + harness=None, + max_tokens=512, + rollout_timeout=600, ) + assert traces, f"{taskset} produced no trace" + for trace in traces: + assert trace.ok, f"{taskset}: {trace.errors}" diff --git a/tests/v1/test_platform_run.py b/tests/v1/test_platform_run.py deleted file mode 100644 index c6d5dec2e6..0000000000 --- a/tests/v1/test_platform_run.py +++ /dev/null @@ -1,104 +0,0 @@ -"""`open_run`: the eval's run on the platform, and `run.attach` for hosted evaluations.""" - -import prime_runs as pr -import pytest - -from verifiers.v1.configs.cli.eval import EvalConfig -from verifiers.v1.utils import platform -from verifiers.v1.utils.platform import PushState, open_run - - -class _FakeRun: - """Stands in for `pr.Run`: an id, a URL, nothing else the tests reach for.""" - - def __init__(self, run_id: str, mode: str) -> None: - self.id = run_id - self.mode = mode - self.url = ( - None if mode == "disabled" else f"https://app.example/evaluations/{run_id}" - ) - self.finished = False - - -@pytest.fixture -def init_calls(monkeypatch): - """Record every `pr.init` call; the fake returns a run keyed by the attach id.""" - calls: list[dict] = [] - - def fake_init(**kwargs): - calls.append(kwargs) - return _FakeRun(kwargs.get("id") or "eval-created", kwargs["mode"]) - - monkeypatch.setattr(platform.pr, "init", fake_init) - monkeypatch.delenv(pr.MODE_ENV, raising=False) - return calls - - -def test_attach_needs_push(): - with pytest.raises(ValueError, match="needs push"): - EvalConfig(run={"attach": "eval-hosted"}, push=False) - - -def test_a_run_without_attach_is_created_on_the_platform(init_calls): - config = EvalConfig(push=True) - state = PushState() - - run = open_run(config, state, num_examples=3) - - assert init_calls[0]["mode"] == "online" - assert init_calls[0]["id"] is None - assert run.id == config.run.id == "eval-created" - - -def test_attach_hands_the_launchers_id_to_the_sdk(init_calls): - config = EvalConfig(run={"attach": "eval-hosted"}) - state = PushState() - - run = open_run(config, state, num_examples=3) - - assert init_calls[0]["mode"] == "online" - assert init_calls[0]["id"] == "eval-hosted" - # The launcher's id is the run's one id: traces and the run dir carry it too. - assert run.id == config.run.id == "eval-hosted" - assert state.error is None - - -def test_a_failed_attach_fails_the_eval_instead_of_falling_back(monkeypatch): - def refuse(**kwargs): - raise pr.UnauthorizedError("bad key") - - monkeypatch.setattr(platform.pr, "init", refuse) - monkeypatch.delenv(pr.MODE_ENV, raising=False) - config = EvalConfig(run={"attach": "eval-hosted"}) - - with pytest.raises(RuntimeError, match="could not attach to run 'eval-hosted'"): - open_run(config, PushState(), num_examples=3) - - -def test_a_failed_open_without_attach_still_falls_back_to_a_local_run(monkeypatch): - calls: list[dict] = [] - - def flaky(**kwargs): - calls.append(kwargs) - if kwargs["mode"] == "online": - raise pr.UnauthorizedError("bad key") - return _FakeRun("local-run", kwargs["mode"]) - - monkeypatch.setattr(platform.pr, "init", flaky) - monkeypatch.delenv(pr.MODE_ENV, raising=False) - state = PushState() - - run = open_run(EvalConfig(push=True), state, num_examples=3) - - assert [call["mode"] for call in calls] == ["online", "disabled"] - assert run.mode == "disabled" - assert state.error is not None and "UnauthorizedError" in state.error - - -def test_the_sdk_kill_switch_refuses_an_attach(init_calls, monkeypatch): - monkeypatch.setenv(pr.MODE_ENV, "disabled") - config = EvalConfig(run={"attach": "eval-hosted"}) - - with pytest.raises(RuntimeError, match="PRIME_RUNS_MODE=disabled"): - open_run(config, PushState(), num_examples=3) - assert init_calls == [] diff --git a/uv.lock b/uv.lock index d9f244ed3b..4d8902a34e 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 4 +revision = 5 requires-python = ">=3.11, <3.15" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -23,7 +23,6 @@ exclude-newer-span = "P7D" [options.exclude-newer-package] harbor = "2026-08-11T00:00:00Z" prime-pydantic-config = false -prime-runs = false prime-sandboxes = false prime-tunnel = false renderers = false @@ -3511,19 +3510,6 @@ toml = [ { name = "tomli" }, ] -[[package]] -name = "prime-runs" -version = "0.1.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx" }, - { name = "prime-traces" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6c/de/6fb503ce854315d096982b79340b02eb5a978b139a7c719d39df77e7f211/prime_runs-0.1.4.tar.gz", hash = "sha256:f06c2bc7db28f0ed99ca6b06aa5186245ca94009872eb861ab0be7cff25484bd", size = 67282, upload-time = "2026-10-08T00:23:59.354Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ba/51/8f300ac057c95afa9df591172b2e8374743afdeeae300771fd78c91a5f75/prime_runs-0.1.4-py3-none-any.whl", hash = "sha256:f33b477c2d53eca08524a3d469810943eee31af8f60ceffc38ef2b9c90596d1b", size = 46253, upload-time = "2026-10-08T00:23:57.736Z" }, -] - [[package]] name = "prime-sandboxes" version = "0.4.2" @@ -3543,19 +3529,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/b5/6a9fe6ae1342872c1c0cc45072cc0c58a5fa2783c4c45770e585f2535c8f/prime_sandboxes-0.4.2-py3-none-any.whl", hash = "sha256:ed59f001c7575339d6c903166c5468f783eda0997586ec6476fd8b35f65746f9", size = 74970, upload-time = "2026-10-02T22:38:29.565Z" }, ] -[[package]] -name = "prime-traces" -version = "0.0.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx" }, - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8c/77/13fa99c0fb71909f2f35d048d9724efb9ae432484e47f12386b1542815e8/prime_traces-0.0.3.tar.gz", hash = "sha256:4071833db77a606ccbb718e5c73f2dfaf63982fe801aa096daed0bc605da554f", size = 50874, upload-time = "2026-08-27T19:46:44.672Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/9a/5a917de29a3a839c0ba36d445e3af562f744d9716b509571be643a01476c/prime_traces-0.0.3-py3-none-any.whl", hash = "sha256:0838f60651f1ed38371e383bd88b954e461d20ac424874e694bc83358934f5f3", size = 38345, upload-time = "2026-08-27T19:46:43.396Z" }, -] - [[package]] name = "prime-tunnel" version = "0.1.12" @@ -5421,6 +5394,7 @@ wheels = [ [[package]] name = "verifiers" source = { editable = "." } +default-groups = ["dev", "examples"] dependencies = [ { name = "aiohttp" }, { name = "anthropic" }, @@ -5436,7 +5410,6 @@ dependencies = [ { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "openai" }, { name = "prime-pydantic-config", extra = ["toml"] }, - { name = "prime-runs" }, { name = "prime-sandboxes" }, { name = "prime-tunnel" }, { name = "pydantic" }, diff --git a/verifiers/v1/cli/dashboard/__init__.py b/verifiers/v1/cli/dashboard/__init__.py index 21e473e157..f08f021af3 100644 --- a/verifiers/v1/cli/dashboard/__init__.py +++ b/verifiers/v1/cli/dashboard/__init__.py @@ -1,4 +1,3 @@ -from verifiers.v1.cli.dashboard.eval import dashboard from verifiers.v1.cli.dashboard.validate import TaskProgress, validate_dashboard -__all__ = ["TaskProgress", "dashboard", "validate_dashboard"] +__all__ = ["TaskProgress", "validate_dashboard"] diff --git a/verifiers/v1/cli/dashboard/eval.py b/verifiers/v1/cli/dashboard/eval.py deleted file mode 100644 index 3326164182..0000000000 --- a/verifiers/v1/cli/dashboard/eval.py +++ /dev/null @@ -1,904 +0,0 @@ -"""Live eval dashboard.""" - -import contextlib -import time -from collections import deque -from pathlib import Path -from typing import TYPE_CHECKING - -from pydantic import BaseModel -from rich import get_console -from rich.console import Group -from rich.markup import escape -from rich.progress_bar import ProgressBar -from rich.rule import Rule -from rich.table import Table -from rich.text import Text - -from verifiers.v1.cli.dashboard.base import live_view -from verifiers.v1.cli.eval.hint import PRIME_RL_HINT -from verifiers.v1.cli.output import attempt_log_file, output_path -from verifiers.v1.configs.cli.eval import EvalConfig -from verifiers.v1.env import RunSlot -from verifiers.v1.trace import Trace -from verifiers.v1.types import Usage -from verifiers.v1.utils.format import ( - format_count, - format_mean, - format_override, - format_time, -) -from verifiers.v1.utils.interrupt import cleaning_up - -if TYPE_CHECKING: - from verifiers.v1.utils.platform import PushState - -# For sizing pages to the terminal: detects the real terminal height/width each access (the live -# view writes to the same terminal). Reused so we don't rebuild it every refresh tick. -_CONSOLE = get_console() -_PAGE_SECONDS = 5.0 # rotate to the next page of rollouts this often when they overflow -# The under-bar breakdown pads its label column to the Overview's widest label, so the two -# `label value` grids (above and below the progress bar) line their values up. -_LABEL_WIDTH = len("timeouts") - -_STYLE = { - "pending": "dim", - "boot": "orange3", - "build": "dark_orange", - "setup": "yellow", - "running": "cyan", - "finalize": "magenta", - "scoring": "blue", - "success": "green", - "error": "red", -} -_MARK_LABEL = { - "pending": "pending", - "boot": "boot", - "build": "build", - "setup": "setup", - "running": "rollout", - "finalize": "finalize", - "scoring": "scoring", - "success": "success", - "error": "error", -} -_MARK_WIDTH = max(len(label) for label in _MARK_LABEL.values()) -# Each label padded to a common width and bracketed, so the `[ ]` line up in a column down the -# left edge (label left-aligned inside) — the current phase reads at a glance. `escape` keeps the -# brackets literal: Rich parses `[label]` in a cell as markup and would otherwise drop it. -_MARK = { - state: escape(f"[{label:<{_MARK_WIDTH}}]") for state, label in _MARK_LABEL.items() -} - - -def format_cost_usd(cost_usd: float) -> str: - if cost_usd >= 1: - return f"${cost_usd:.2f}" - return f"${cost_usd:.4f}" - - -def _seat_value(config: EvalConfig, read): - """A cap as the overview shows it: the declared seats' shared value, the - string 'per-seat' when they disagree (caps live on the seats).""" - from verifiers.v1.configs.agent import agent_config_fields - - values = {read(spec) for spec in agent_config_fields(config.env).values()} - return values.pop() if len(values) == 1 else "per-seat" - - -def _limits(config: EvalConfig) -> list[str]: - """Per-run caps for the overview (concurrency first, then turns, tokens), read - off the declared seats. An unset cap reads as 'no ...' rather than being hidden.""" - toks = [] - for label, field in ( - ("in", "max_input_tokens"), - ("out", "max_output_tokens"), - ("total", "max_total_tokens"), - ): - v = _seat_value(config, lambda spec, field=field: getattr(spec, field)) - if v == "per-seat": - toks.append(f"{label} per-seat") - elif v: - toks.append(f"{label}≤{v}") - turns = _seat_value(config, lambda spec: spec.max_turns) - return [ - f"≤{config.max_concurrent} episodes" - if config.max_concurrent - else "no concurrency cap", - "per-seat turn caps" - if turns == "per-seat" - else (f"{turns} turns" if turns else "no turn cap"), - f"{', '.join(toks)} tokens" if toks else "no token cap", - ] - - -def _timeouts(config: EvalConfig) -> list[str]: - """Per-stage run timeouts for the overview (each seat's own), plus the env's - finalize() bound (unset → 'no timeout').""" - rows = [] - for stage in ("setup", "rollout", "finalize", "scoring"): - v = _seat_value(config, lambda spec, stage=stage: getattr(spec.timeout, stage)) - if v == "per-seat": - rows.append(f"{stage} per-seat") - elif v: - rows.append(f"{stage} {v:g}s") - else: - rows.append(f"no {stage} timeout") - finalize = config.env.timeout.finalize - rows.append( - f"env finalize {finalize:g}s" if finalize else "no env finalize timeout" - ) - return rows - - -def _aligned(rows: list[list[str]]) -> list[str]: - """Join each row's `·`-separated segments, padding shared columns to a common width so the - separators line up across rows (each row's last segment is left ragged).""" - widths: dict[int, int] = {} - for row in rows: - for i, seg in enumerate(row): - widths[i] = max(widths.get(i, 0), len(seg)) - return [ - " · ".join( - seg.ljust(widths[i]) if i < len(row) - 1 else seg - for i, seg in enumerate(row) - ) - for row in rows - ] - - -def _interrupt_footer() -> Group | None: - """The graceful-shutdown notice under the rollouts once Ctrl-C has begun teardown — the - on-screen echo of the warning (console logging is silenced in rich mode); a further Ctrl-C - is ignored while it shows. Sits beside the `--push` line, mirroring its placement.""" - if not cleaning_up(): - return None - return Group( - Rule(style="dim"), - Text( - "interrupted — cleaning up, tearing down containers/sandboxes. " - "please wait; a further ctrl-c is ignored.", - style="yellow", - ), - ) - - -def _warning(config: EvalConfig) -> Text | None: - """A local-runtime caution when any code-running seat resolves to the subprocess - runtime (the tool-less chat loops are exempt), shown above the overview rather - than as a row in it.""" - from verifiers.v1.utils.loaders import harness_class - - if any( - getattr(config.env, role).runtime.type == "subprocess" - and harness_class(h.id).EXECUTES_CODE - for role, h in config.env.agent_harnesses().items() - ): - return Text( - "warning Runs on the local system; local files and settings may affect this " - "evaluation. Use subprocess only for debugging, or use docker or prime for an " - "isolated run.", - style="yellow", - ) - return None - - -def overrides( - config: BaseModel, - default: BaseModel | None = None, - skip: frozenset[str] = frozenset(), -) -> list[str]: - """`field=value` segments for the fields the *user* customized, sorted, diffed against each - field's declared default. Not `model_fields_set`: a `--resume` run reloads the full saved - JSON config, so `model_fields_set` would flag every field. `default` is the reference instance, - threaded through recursion so a pinned nested default (`taskset.task.tools`) reads as - unchanged. `skip` holds dotted paths (`runtime.type`).""" - segments: list[str] = [] - fields = type(config).model_fields - for field in sorted(fields): - if field in skip: - continue - value = getattr(config, field) - # `get_default` returns `PydanticUndefined` for a required field, so it always reads as set. - field_def = ( - fields[field].get_default(call_default_factory=True) - if default is None - else getattr(default, field, None) - ) - if isinstance(value, BaseModel): # nested config: flatten as `field.` - child_skip = frozenset( - s[len(field) + 1 :] for s in skip if s.startswith(f"{field}.") - ) - # A switched discriminator (e.g. subprocess→docker) is an override the per-field diff - # misses — within the new class `type` equals its own default — so surface it. - class_changed = type(value) is not type(field_def) - if ( - class_changed - and "type" not in child_skip - and (discriminator := getattr(value, "type", None)) is not None - ): - segments.append(f"{field}.type={format_override(discriminator)}") - # A switched class is a new shape, so diff against its own defaults, not the instance. - segments.extend( - f"{field}.{seg}" - for seg in overrides( - value, default=None if class_changed else field_def, skip=child_skip - ) - ) - elif value != field_def: - segments.append(f"{field}={format_override(value)}") - return segments - - -def Overview(config: EvalConfig) -> Table: - sampling = ", ".join( - f"{k}={v}" for k, v in config.sampling.model_dump(exclude_none=True).items() - ) - grid = Table.grid(padding=(0, 2)) - grid.add_column(style="dim") - grid.add_column() - seats = config.env.agent_harnesses() - taskset = config.env.taskset - env_label = taskset.name if taskset.id else "no taskset" - if config.env.id: - env_label = f"{config.env.id}+{env_label}" - # One seat story when every seat resolves the same way (the common case); one - # row per seat when they diverge (a judge on its own harness/runtime). - runtimes = {role: getattr(config.env, role).runtime.type for role in seats} - stories = list( - dict.fromkeys( - f"{h.name} harness · {runtimes[role]} runtime" - for role, h in seats.items() - ) - ) - if len(stories) == 1: - grid.add_row("env", f"{env_label} · {stories[0]}") - else: - grid.add_row("env", env_label) - for role, h in seats.items(): - grid.add_row(f" {role}", f"{h.name} harness · {runtimes[role]} runtime") - model = f"{config.model} ({sampling})" if sampling else config.model - grid.add_row("model", f"{model} via {config.client.base_url}") - # Non-default knobs the user set, one row each when non-empty. `escape` the cell: an override - # value (or our `[...]`/`{...}` delimiters) can carry Rich markup that would otherwise be - # parsed as styling and dropped. `id` is in the `env` row; the seat's `runtime.type` too - # (hidden here), but only for the seat — `taskset.task.tools.runtime.type` has no other display. - if taskset_over := overrides(taskset, skip=frozenset({"id"})): - grid.add_row("taskset", escape(" · ".join(taskset_over))) - for role, h in seats.items(): - if harness_over := overrides(h, skip=frozenset({"id"})): - label = f"{role}.harness" if len(seats) > 1 else "harness" - grid.add_row(label, escape(" · ".join(harness_over))) - runtime = getattr(config.env, role).runtime - if runtime_over := overrides(runtime, skip=frozenset({"type"})): - label = f"{role}.runtime" if len(seats) > 1 else "runtime" - grid.add_row(label, escape(" · ".join(runtime_over))) - limits, timeouts = _aligned([_limits(config), _timeouts(config)]) - grid.add_row("limits", limits) - grid.add_row("timeouts", timeouts) - grid.add_row("output", Text(str(output_path(config)), overflow="fold")) - return grid - - -def _prime_rl_footer() -> Group: - return Group(Text(""), Text(PRIME_RL_HINT, style="dim", overflow="fold")) - - -def _push_footer(push: "PushState | None") -> Group | None: - """The `--push` line under the rollouts: dim with the run's URL while it streams, - white once pushed, red when it failed. `None` when `--push` is off or the run stayed - local.""" - if push is None or not push.started: - return None - if push.error and push.url: - # The run exists and holds what streamed up; only closing it out failed. - line = Text(f"Traces pushed ({push.url})", style="white", overflow="fold") - line.append(f" not closed out: {push.error}", style="red") - elif push.error: - line = Text(f"Trace push failed ({push.error})", style="red", overflow="fold") - elif (incomplete := push.incomplete) and push.url: - label = "Traces pushed" if push.finished else "Pushing traces" - line = Text(f"{label} ({push.url})", style="white", overflow="fold") - line.append(f" incomplete: {incomplete}", style="red") - elif not push.finished: - line = Text(f"Pushing traces ({push.url})", style="dim", overflow="fold") - else: - line = Text(f"Traces pushed ({push.url})", style="white", overflow="fold") - return Group(Rule(style="dim"), line) - - -class TraceStats: - """Resource totals for a dashboard frame, retained once the episode is final.""" - - def __init__(self, trace: Trace) -> None: - self.num_input_tokens = trace.num_input_tokens - self.num_output_tokens = trace.num_output_tokens - self.num_turns = trace.num_turns - self.num_branches = trace.num_branches - self.usage = trace.usage - self.judge_usage = Usage.aggregate(trace.extra_usage) - - -def Progress( - slots: list[RunSlot], - start: float, - completed: dict[str, TraceStats], - page: tuple[int, int] | None = None, -) -> Group: - # On resume, `slots` includes the previous session's kept rollouts (as finished slots), so - # progress, reward, err, and the breakdown cover the whole run, not just this session's. - done = [s for s in slots if s.done] # fully scored episodes - done_traces = [t for s in done for t in s.traces] - # Score aggregates read the policy's traces: auxiliary roles (a judge's verdict - # run, a modeled user) are `trainable=False` and carry no rewards, so counting - # them dilutes every mean with structural zeros. An all-untrainable run (every - # role frozen) falls back to all traces rather than showing nothing. - scored = [t for t in done_traces if t.agent.trainable] or done_traces - total = len(slots) - # Headline reward = mean over non-errored traces; when any errored, `format_mean` appends - # the global avg (errored count as 0) in parens. `err` is the share of episodes that - # ended not-ok (a trace errored, or the env's rollout()/score() hook itself failed). - reward = format_mean(scored, lambda t: t.reward) - err = ( - f"{sum(s.episode is None or not s.episode.ok for s in done) / len(done):.2f}" - if done - else "—" - ) - stats = ( - f"{len(done)}/{total} · {format_time(time.time() - start)} · " - f"reward {reward} · err {err}" - ) - if page is not None: # overflowing — show which page, and that the arrows page - stats += f" (page {page[0]}/{page[1]} ◄ ►)" - row = Table.grid(expand=True, padding=(0, 1)) - row.add_column(ratio=1) # bar stretches to fill the width left of the stats - row.add_column(justify="right", no_wrap=True) - row.add_row( - ProgressBar(total=total or 1, completed=len(done)), - Text(stats), - ) - breakdown = _breakdown(scored, done_traces, completed) - return Group(row, breakdown) if breakdown is not None else Group(row) - - -def _score_segments(traces: list[Trace], source: str) -> str | None: - """`name mean` segments for every reward/metric key seen across `traces`, - first-seen order (a trace carries only its own task's signals — unscored ones - seeded as `None` — so keys can vary); None when no trace recorded anything.""" - names: list[str] = [] - for trace in traces: - names.extend(n for n in getattr(trace, source) if n not in names) - if not names: - return None - segments = [] - for name in names: - mean = format_mean(traces, lambda t, n=name, s=source: _score(t, s, n)) - segments.append(f"{name} {mean}") - return " · ".join(segments) - - -def _score(trace: Trace, source: str, name: str) -> float: - """Rewards carry raw score + weight; the breakdown shows the raw score. A missing - or unscored (seeded `None`) entry reads as 0.0.""" - if source == "rewards": - reward = trace.rewards.get(name) - return reward.score if reward is not None else 0.0 - value = trace.metrics.get(name) - return value if value is not None else 0.0 - - -def _breakdown( - scored: list[Trace], done: list[Trace], completed: dict[str, TraceStats] -) -> Table | None: - """Score rows read the policy view (`scored` — trainable traces); with several - roles in play they split per role, each role averaging over its OWN traces (no - dilution, so the split covers every role — an untrainable seat's received - rewards stay visible). Resource rows read every completed trace: a judge's - tokens were spent regardless of trainability.""" - grid = Table.grid(padding=(0, 2)) - grid.add_column(style="dim", min_width=_LABEL_WIDTH) - grid.add_column() - # rewards/metrics are error-corrected means — skip when every rollout errored (no clean mean to - # show); usage/time below still cover errored rollouts (their resources were spent regardless). - has_clean = any(not t.has_error for t in done) - score_rows = (("rewards", "rewards"), ("metrics", "metrics")) if has_clean else () - by_agent: dict[str, list[Trace]] = {} - for trace in done: - by_agent.setdefault(trace.agent.name, []).append(trace) - for label, source in score_rows: - if len(by_agent) > 1: - segments = [ - f"[dim]{name}:[/dim] {means}" - for name, traces in by_agent.items() - if (means := _score_segments(traces, source)) is not None - ] - if segments: - grid.add_row(label, " ".join(segments)) - elif (means := _score_segments(scored, source)) is not None: - grid.add_row(label, means) - - # Resource use over every completed rollout (errored ones still spent tokens/time): tokens and - # cost are summed; each timing phase is averaged over the rollouts that have it timed (averaged - # per phase rather than summed, since phases run concurrently across rollouts). - total_in = total_out = total_cached = total_reasoning = 0 - total_judge_in = total_judge_out = 0 - total_cost = total_judge_cost = 0.0 - have_cost = have_cached = have_reasoning = have_judge = False - phase_secs: dict[str, float] = {} - phase_count: dict[str, int] = {} - model_secs = harness_secs = 0.0 - for trace in done: - stats = completed[trace.id] - total_in += stats.num_input_tokens - total_out += stats.num_output_tokens - usage = stats.usage - if usage is not None: - if usage.cached_input_tokens is not None: - total_cached += usage.cached_input_tokens - have_cached = True - if usage.reasoning_tokens is not None: - total_reasoning += usage.reasoning_tokens - have_reasoning = True - if usage.cost is not None: - total_cost += usage.cost - have_cost = True - # Judge / auxiliary scoring calls (off the message graph) shown separately from the agent's. - judge = stats.judge_usage - if judge is not None: - total_judge_in += judge.input_tokens - total_judge_out += judge.completion_tokens - if judge.cost is not None: - total_judge_cost += judge.cost - have_judge = True - for phase in ("boot", "setup", "agent", "finalize", "scoring"): - span = getattr(trace.timing, phase) - if span.end: # phase was timed for this rollout - phase_secs[phase] = phase_secs.get(phase, 0.0) + span.duration - phase_count[phase] = phase_count.get(phase, 0) + 1 - model_secs += trace.timing.agent.model.duration - harness_secs += trace.timing.agent.harness.duration - if ( - total_in - or total_out - or have_cost - or have_cached - or have_reasoning - or have_judge - ): - tokens = f"{format_count(total_in)}/{format_count(total_out)} tokens" - details = [] - if have_cached: - details.append(f"{format_count(total_cached)} cached") - if have_reasoning: - details.append(f"{format_count(total_reasoning)} reasoning") - if details: - tokens += f" ({', '.join(details)})" - usage = [tokens] - if have_judge: - usage.append( - f"+ {format_count(total_judge_in)}/{format_count(total_judge_out)} judge" - ) - if have_cost: - cost = format_cost_usd(total_cost) - if total_judge_cost: - cost += f" + {format_cost_usd(total_judge_cost)} judge" - usage.append(cost) - grid.add_row("usage", " · ".join(usage)) - time_segments = [] - for phase in ("boot", "setup", "agent", "finalize", "scoring"): - count = phase_count.get(phase) - if not count: - continue - segment = f"{phase} {format_time(phase_secs[phase] / count)}" - if phase == "agent": - segment += ( - f" (model {format_time(model_secs / count)}" - f" + harness {format_time(harness_secs / count)})" - ) - time_segments.append(segment) - if time_segments: - grid.add_row("time", " · ".join(time_segments)) - return grid if grid.row_count else None - - -def _stage(trace: Trace) -> str: - """The stage a live (not-yet-done) rollout is in, derived from its trace's timing - spans — the engine opens and closes each span exactly at the stage transitions, so - the current stage is the latest span started but not yet ended. A completed trace - whose slot isn't done is waiting on its episode's other traces (and the env's - `score()`) — that's scoring.""" - if trace.is_completed: - return "scoring" - for stage, span in ( - ("scoring", trace.timing.scoring), - ("finalize", trace.timing.finalize), - ("running", trace.timing.agent), - ("setup", trace.timing.setup), - ("boot", trace.timing.boot), - ): - if span.start and not span.end: - break - else: - stage = "boot" # trace minted, first span not yet opened (an instant) - # A boot stuck on a first-use platform image build reads differently from a - # normal boot — it can sit there for ~10 minutes (prime runtime only). - if stage == "boot" and getattr(trace.agent.runtime, "image_cached", None) is False: - return "build" - return stage - - -def _started(slot: RunSlot) -> float: - # Sort key: when a rollout began (its first trace's boot start; setup for - # pre-boot-span traces on resume; the slot's own dispatch stamp on a served run). - # A still-pending rollout has neither, so it sorts last (+inf) — behind everything - # already in flight, in task order. - if not slot.traces: - return slot.started if slot.started is not None else float("inf") - return min(t.timing.boot.start or t.timing.setup.start for t in slot.traces) - - -def _groups(slots: list[RunSlot]) -> list[list[RunSlot]]: - # The n rollouts of each task, grouped together (so they sit adjacent); groups ordered by - # earliest start, rollouts within a group by start. Every slot carries its `task` from - # construction, so ones still queued behind the concurrency cap (no trace yet) are grouped - # and shown too — as `[pending]`. Finished ones stay (never removed). - by_task: dict[int, list[RunSlot]] = {} - for slot in slots: - by_task.setdefault(slot.task.data.idx, []).append(slot) - groups = list(by_task.values()) - for group in groups: - group.sort(key=_started) - groups.sort(key=lambda g: _started(g[0])) - return groups - - -def _brace(i: int, size: int) -> str: - if size == 1: - return " " - return "╭" if i == 0 else "╰" if i == size - 1 else "│" - - -def Rows( - groups: list[list[RunSlot]], - now: float, - runtime_type: str, - completed: dict[str, TraceStats], -) -> Table: - # (brace, state, left sections, result, time); a slot contributes one row per live - # trace (a multi-agent episode shows each role's trace), braced per task. - rows: list[tuple[str, str, list[str], str, str]] = [] - for group in groups: - group_rows: list[tuple[str, list[str], str, str]] = [] - for slot in group: - task = slot.task.data - base = f"name={task.name[:32]}" if task.name else f"idx={task.idx}" - if not slot.traces: - if slot.done: # the env's rollout() itself failed before any trace - error = ( - slot.episode.last_error if slot.episode is not None else None - ) - group_rows.append( - ( - "error", - [f"task {base}", *[""] * 7], - error.type if error is not None else "error", - "", - ) - ) - elif slot.started is not None: - # Dispatched to an env server: running, but with no live trace to - # read stages/turns from — detail lands when the episode completes. - elapsed = format_time(now - slot.started) - group_rows.append( - ("running", [f"task {base}", *[""] * 7], "", elapsed) - ) - else: # queued behind the concurrency cap — only its task is known yet - group_rows.append(("pending", [f"task {base}", *[""] * 7], "", "")) - continue - for t in slot.traces: - label = f"{base} agent={t.agent.name}" - if slot.done: # fully scored — reward is final - state = "error" if t.has_error else "success" - # A trace that recorded nothing shows no reward: a judge or - # modeled-user seat's `reward=0.00` would read as a score. - result = ( - t.last_error.type - if t.has_error and t.last_error - else (f"reward={t.reward:.2f}" if t.rewards else "") - ) - if t.has_error: - stop = "" # error shown instead - else: - stop = t.stop_condition or "" - if ( - t.is_truncated - ): # flag a clipped rollout next to its stop condition - stop = f"{stop} (truncated)".strip() - elif t.is_completed and (err := t.last_error) is not None: - # An errored trace whose episode is still running its other - # traces (or `score()`) is already a failure — show it, don't - # let it sit as "scoring" until the whole episode lands. - state, result, stop = "error", err.type, "" - else: - state, result, stop = _stage(t), "", "" - # The trace's own stamp, not the run-level runtime: a role's harness - # may resolve elsewhere (the judge env's sandboxed judge on a - # subprocess run). - if t.agent.runtime is not None: - rt = t.agent.runtime - runtime = f"{rt.type}({rt.id})" if rt.id else rt.type - else: - runtime = runtime_type - stats = completed[t.id] if slot.done else TraceStats(t) - turns = stats.num_turns - start = t.timing.boot.start or t.timing.setup.start - end = ( - t.timing.scoring.end - or t.timing.finalize.end - or t.timing.agent.end - # a rollout that errored in boot/setup has only that span's end — freeze there - # once done, else (still running) the timer would grow off `now` forever - or ( - (t.timing.setup.end or t.timing.boot.end) - if t.is_completed - else 0 - ) - or now - ) - prompt, completion = stats.num_input_tokens, stats.num_output_tokens - nbranches = stats.num_branches - usage = stats.usage - cached = usage.cached_input_tokens if usage else None - reasoning = usage.reasoning_tokens if usage else None - cost = usage.cost if usage else None - tokens = "" - if prompt or completion: - tokens = f"{format_count(prompt)}/{format_count(completion)} tokens" - details = [] - if cached is not None: - details.append(f"{format_count(cached)} cached") - if reasoning is not None: - details.append(f"{format_count(reasoning)} reasoning") - if details: - tokens += f" ({', '.join(details)})" - left = [ - f"task {label}", - t.id[:8], - runtime, - f"{turns} turn{'s' * (turns != 1)}", - f"{nbranches} branch{'es' * (nbranches != 1)}", - tokens, - f"{format_cost_usd(cost)}" if cost is not None else "", - stop, # stop condition (agent_completed / max_turns / error), once done - ] - # No start time yet (queued, not generating) → blank, not `now - 0` (~56 years). - elapsed = format_time(end - start) if start else "" - group_rows.append((state, left, result, elapsed)) - for i, (state, left, result, elapsed) in enumerate(group_rows): - rows.append((_brace(i, len(group_rows)), state, left, result, elapsed)) - grid = Table.grid(expand=True, padding=(0, 1)) - grid.add_column(ratio=1, no_wrap=True) # brace + mark + dot-separated sections - grid.add_column(justify="right", no_wrap=True) # result (reward / error) - grid.add_column(justify="right", no_wrap=True) # time - if not rows: - return grid - # Pad each left section to its max width across rows (drop all-empty ones) so they align, - # then join with " · ". Text sections left-justified, numeric right. - pad = ( - str.ljust, - str.ljust, - str.ljust, - str.rjust, - str.rjust, - str.ljust, - str.rjust, - str.ljust, - ) - widths = [max(len(left[i]) for _, _, left, _, _ in rows) for i in range(8)] - for brace, state, left, result, elapsed in rows: - sections = [pad[i](left[i], widths[i]) for i in range(8) if widths[i]] - grid.add_row( - f"{brace} {_MARK[state]} " + " · ".join(sections), - f"{result} ·" if result else "", # trailing dot only when there's a result - elapsed, - style=_STYLE[state], - ) - return grid - - -class Pager: - def __init__(self) -> None: - self.page = 0 - self.manual = False - self.count = 1 - self.origin: float | None = None - - def on_key(self, key: str) -> None: - if key in ("left", "right") and self.count > 1: - self.manual = True - self.page = (self.page + (1 if key == "right" else -1)) % self.count - - def index(self, now: float) -> int: - # Track the auto page while it drives, so the first arrow continues from what's on screen - # rather than jumping back to page 1. Clamp in manual mode (count can shrink on resize). - if not self.manual: - # Anchor the timer to the first paged frame, so paging opens on page 1 then rotates. - if self.origin is None: - self.origin = now - self.page = int((now - self.origin) / _PAGE_SECONDS) % self.count - else: - self.page = max(0, min(self.page, self.count - 1)) - return self.page - - -_TAIL_BYTES = 256 * 1024 -"""How far back into a pre-existing log file the tail's first read reaches.""" - - -class LogTail: - """Incremental tail of the run's log file for `--rich.show-logs`: read only what - was appended since the last frame, keep a bounded window of whole lines.""" - - def __init__(self, path: Path, max_lines: int = 1000) -> None: - self.path = path - self.lines: deque[str] = deque(maxlen=max_lines) - self._pos: int | None = None - self._partial = "" - - def _poll(self) -> None: - try: - size = self.path.stat().st_size - except OSError: # not written yet — the first record creates it - return - if ( - self._pos is not None and size < self._pos - ): # truncated/replaced — start over - self._pos, self._partial = None, "" - if size == self._pos: - return - # The first read starts at most _TAIL_BYTES from the end: a pre-existing file - # (a reused run dir) must not be read whole on the render path. - first = self._pos is None - start = max(0, size - _TAIL_BYTES) if first else self._pos - with self.path.open("rb") as file: - file.seek(start) - data = file.read() - self._pos = file.tell() - chunk = data.decode("utf-8", errors="replace") - *complete, self._partial = (self._partial + chunk).split("\n") - if first and start: # landed mid-line — drop the cut first line - complete = complete[1:] - self.lines.extend(complete) - - def view(self, height: int) -> Group: - """The newest lines that fit, one Text per line (no markup parsing — log - content stays literal), each cropped to the terminal width.""" - self._poll() - window = list(self.lines)[-height:] - return Group( - *(Text(line, no_wrap=True, overflow="ellipsis") for line in window) - ) - - -def _rows_of(group: list[RunSlot]) -> int: - """How many display rows a task's slots take: one per trace, one for a slot with - none yet (pending) or none at all (the env's hook failed before any trace).""" - return sum(max(1, len(slot.traces)) for slot in group) - - -def _paginate( - groups: list[list[RunSlot]], rows_per_page: int, pager: Pager, now: float -) -> tuple[list[list[RunSlot]], int, int]: - """Pack groups (a task's rollouts kept together) into pages of at most `rows_per_page` rows, - selecting the one `pager` points at. Returns (this page's groups, 0-based index, page count) — - a single page when everything already fits.""" - if sum(_rows_of(g) for g in groups) <= rows_per_page: - # Everything fits: arrows stay inert, and clear the anchor so paging re-opens on page 1 if - # rollouts overflow again later (e.g. a terminal resize) rather than off the stale origin. - pager.count = 1 - pager.origin = None - return groups, 0, 1 - pages: list[list[list[RunSlot]]] = [] - current: list[list[RunSlot]] = [] - used = 0 - for group in groups: - if current and used + _rows_of(group) > rows_per_page: - pages.append(current) - current, used = [], 0 - current.append(group) - used += _rows_of(group) - if current: - pages.append(current) - pager.count = len(pages) - index = pager.index(now) - return pages[index], index, len(pages) - - -def _render( - slots: list[RunSlot], - start: float, - pager: Pager, - header: Group | Table, - runtime_type: str, - completed: dict[str, TraceStats], - push: "PushState | None" = None, - tail: LogTail | None = None, -) -> Group: - now = time.time() - # A trace can finish before other agents and env scoring update the episode. - # Only done slots are final; resumed episodes enter the view already done. - completed.update( - (trace.id, TraceStats(trace)) - for slot in slots - if slot.done - for trace in slot.traces - if trace.id not in completed - ) - # The --push status line (and, on Ctrl-C, the cleanup notice) appear under the rollouts. Measure - # the fixed top (header + progress + rule) and the footer so the rollout rows fill what's left; - # page through them (timer / arrows) when they'd overflow (else rich truncates). - footers = [ - f - for f in (_push_footer(push), _interrupt_footer(), _prime_rl_footer()) - if f is not None - ] - footer = Group(*footers) - progress = Progress(slots, start, completed) - top = Group(header, progress, Rule(style="dim")) - reserved = len(_CONSOLE.render_lines(top)) + len(_CONSOLE.render_lines(footer)) - rows_per_page = max(1, _CONSOLE.size.height - reserved - 1) - if tail is not None: # --show-logs: the run's log stream in place of rollout rows - parts = [ - header, - progress, - Rule(style="dim"), - tail.view(rows_per_page), - footer, - ] - return Group(*parts) - page_groups, index, count = _paginate(_groups(slots), rows_per_page, pager, now) - if count > 1: - progress = Progress(slots, start, completed, page=(index + 1, count)) - parts = [ - header, - progress, - Rule(style="dim"), - Rows(page_groups, now, runtime_type, completed), - footer, - ] - return Group(*parts) - - -@contextlib.asynccontextmanager -async def dashboard( - slots: list[RunSlot], - config: EvalConfig, - start: float, - push: "PushState | None" = None, -): - pager = Pager() - completed: dict[str, TraceStats] = {} - warning = _warning(config) - header = Group(warning, Text(""), Overview(config)) if warning else Overview(config) - runtime_type = ( - "/".join( - dict.fromkeys( - getattr(config.env, role).runtime.type - for role in config.env.agent_harnesses() - ) - ) - or "subprocess" - ) - tail = ( - LogTail(attempt_log_file(output_path(config))) - if config.rich is not None and config.rich.show_logs - else None - ) - async with live_view( - lambda: _render( - slots, start, pager, header, runtime_type, completed, push, tail - ), - on_key=None if tail is not None else pager.on_key, - ): - yield diff --git a/verifiers/v1/cli/eval/__init__.py b/verifiers/v1/cli/eval/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/verifiers/v1/cli/eval/hint.py b/verifiers/v1/cli/eval/hint.py deleted file mode 100644 index 4523b19511..0000000000 --- a/verifiers/v1/cli/eval/hint.py +++ /dev/null @@ -1,5 +0,0 @@ -PRIME_RL_HINT = ( - "vf-eval will be deprecated - move to prime-rl's `uv run eval`: it streams traces " - "live into a dashboard, evaluates several envs in one run and scales through env " - "servers. https://github.com/PrimeIntellect-ai/prime-rl" -) diff --git a/verifiers/v1/cli/eval/main.py b/verifiers/v1/cli/eval/main.py deleted file mode 100644 index e66101a8bb..0000000000 --- a/verifiers/v1/cli/eval/main.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Eval CLI entrypoint.""" - -import asyncio -import json -import logging -import shutil -import sys - -from pydantic_config import cli - -from verifiers.v1.cli.eval.runner import run_eval -from verifiers.v1.cli.output import ( - create_attempt_log_dir, - output_path, - saved_config_path, - write_config, -) -from verifiers.v1.cli.resolve import ( - extract_id, - narrow_config, - plugin_errors, - references_config_file, - with_positional_taskset, -) -from verifiers.v1.configs.cli.eval import EvalConfig -from verifiers.v1.utils.interrupt import install_interrupt -from verifiers.v1.utils.logging import setup_logging -from verifiers.v1.utils.trace_store import TRACES_FILE - -logger = logging.getLogger(__name__) - -USAGE = ( - "usage: uv run vf-eval [] [--env.id ] [options] [@ file.toml]\n" - " uv run vf-eval @ /configs/resolved/eval.json --resume (re-run the run's missing/errored rollouts)" -) - - -def main(argv: list[str] | None = None) -> None: - argv = with_positional_taskset(list(sys.argv[1:]) if argv is None else list(argv)) - - if not argv or any(arg in ("-h", "--help") for arg in argv): - print(USAGE) - sys.argv = [sys.argv[0], "--help"] - with plugin_errors(): - cli( - narrow_config(EvalConfig, argv) - ) # full option help, narrowed to the given ids - return - # An env-block flag skips the usage gate so the typed parse renders its - # did-you-mean instead of a bare usage line. - typed_axis = any(a.startswith("--env.") for a in argv) - if ( - not extract_id(argv, "env.taskset") - and not references_config_file(argv) - and not typed_axis - ): - raise SystemExit( - USAGE - ) # need a taskset (positional / --env.taskset.id) or a @ file.toml - - with plugin_errors(): - config_type = narrow_config(EvalConfig, argv) - sys.argv = [ - sys.argv[0], - *argv, - ] # let prime-pydantic-config render help/errors - config = cli(config_type) - # A named run directory is re-entered only by `--resume` or wiped by `--clean`: any - # other write into it — the dry-run config included, which would clobber the - # config a resume typically re-runs — would overwrite the previous run. - run_path = output_path(config) - if config.clean and not config.resume and run_path.exists(): - output_dir = config.output_dir.resolve() - resolved_run_path = run_path.resolve() - if resolved_run_path == output_dir or not resolved_run_path.is_relative_to( - output_dir - ): - raise SystemExit("--clean requires run.dir to name a child of output_dir") - shutil.rmtree(run_path) - traces_file = run_path / TRACES_FILE - if not config.resume and traces_file.exists() and traces_file.stat().st_size > 0: - raise SystemExit( - f"run directory {run_path} already contains results - append --resume to " - "re-run its missing/errored rollouts, overwrite it with --clean, or pick " - "another --run.name" - ) - if config.resume: - # A resumed eval is only trustworthy under the exact config that produced the - # existing rollouts — anything else silently mixes incomparable results. Saved - # configs are full JSON dumps (nulls included), so the comparison is exact. - saved_path = saved_config_path(run_path) - if saved_path is None: - raise SystemExit( - f"--resume: no saved config under {run_path} - not a run dir" - ) - with plugin_errors(): - saved = config_type.model_validate_json(saved_path.read_text()) - saved_dump = saved.model_dump(mode="json") - current_dump = config.model_dump(mode="json") - saved_json = json.dumps(saved_dump, sort_keys=True, separators=(",", ":")) - current_json = json.dumps(current_dump, sort_keys=True, separators=(",", ":")) - if saved_json != current_json: - changed = sorted( - key - for key in set(saved_dump) | set(current_dump) - if saved_dump.get(key) != current_dump.get(key) - ) - raise SystemExit( - f"--resume requires the exact config the run was started with - it " - f"differs in [{', '.join(changed)}]. Resumed rollouts would not be " - f"comparable; re-run with `uv run vf-eval @ {saved_path} --resume`, or " - "start a fresh run" - ) - if config.dry_run: # resolved + validated; write it to the output dir and exit - setup_logging("DEBUG" if config.verbose else "INFO") - logger.info("wrote config to %s", write_config(config, run_path)) - return - # Always tee this attempt's logs to `logs/attempt_/eval.log` (`logs/latest` - # points there); `--rich.show-logs` tails it live. - log_file = str(create_attempt_log_dir(run_path) / "eval.log") - level = "DEBUG" if config.verbose else "INFO" - setup_logging(level, log_file=log_file, console=config.rich is None) - # First Ctrl-C / SIGTERM warns and raises KeyboardInterrupt so a killed/timed-out eval still - # runs each rollout's `finally` (tears down containers/sandboxes) and any worker pool it - # spawned; further signals during that cleanup are swallowed so an impatient second Ctrl-C - # can't orphan those resources. - install_interrupt() - - try: - episodes = asyncio.run(run_eval(config)) - except KeyboardInterrupt: - # Graceful cleanup has already run (each rollout's `finally`); partial results are on - # disk. Exit on the conventional Ctrl-C code without a traceback. - raise SystemExit(130) - if ( - config.rich is None - ): # --rich is the whole output; otherwise dump each trace as JSON - for episode in episodes: - for trace in episode.traces: - print(trace.model_dump_json(indent=2, exclude_none=True)) diff --git a/verifiers/v1/cli/eval/resume.py b/verifiers/v1/cli/eval/resume.py deleted file mode 100644 index 940ece9ba5..0000000000 --- a/verifiers/v1/cli/eval/resume.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Resume an interrupted eval: reload its finished rollouts and run only the rest. - -`load` validates saved episodes, applies the environment's completion verdict, and -uses the shared rollout planner to retain only those matching the current tasks. -""" - -from collections.abc import Callable -from pathlib import Path - -from pydantic_core import from_json - -from verifiers.v1.episode import WireEpisode -from verifiers.v1.task import TaskT -from verifiers.v1.utils.eval import plan_rollouts -from verifiers.v1.utils.trace_store import TRACES_FILE - - -def load( - resume_dir: Path, - tasks: list[TaskT], - num_rollouts: int, - complete: Callable[[WireEpisode], bool] | None = None, -) -> list[tuple[TaskT, list[WireEpisode], int]]: - """Plan the remaining rollouts and atomically rewrite the stream to kept rows. - - ``complete`` is the keep-verdict (default ``episode.ok``). Torn or malformed - rows are discarded; retained rows preserve their original serialized fields. - """ - path = resume_dir / TRACES_FILE - verdict = complete if complete is not None else (lambda episode: episode.ok) - episodes: list[WireEpisode] = [] - lines: dict[str, bytes] = {} - if path.exists(): - with path.open("rb") as results: - for line in results: - if not line.strip(): - continue - try: - row = from_json(line) - if "traces" not in row: - continue - episode = WireEpisode.model_validate(row) - if not verdict(episode): - continue - # A malformed row from any task/episode plugin is owed again. - except Exception: # noqa: BLE001, S112 - continue - episodes.append(episode) - lines.setdefault( - episode.id, line if line.endswith(b"\n") else line + b"\n" - ) - plan = plan_rollouts(tasks, num_rollouts, episodes) - tmp = path.with_suffix(".jsonl.tmp") - tmp.write_bytes( - b"".join(lines[episode.id] for _, kept, _ in plan for episode in kept) - ) - tmp.replace(path) - return plan diff --git a/verifiers/v1/cli/eval/runner.py b/verifiers/v1/cli/eval/runner.py deleted file mode 100644 index 22f348db35..0000000000 --- a/verifiers/v1/cli/eval/runner.py +++ /dev/null @@ -1,166 +0,0 @@ -"""The eval runner: fan episodes out with bounded concurrency. - -Rollouts run in-process: the env comes up once in this process and each slot becomes -one episode through `env.run_slot`. The dashboard watches the same `RunSlot`s the env -fills with live traces. Serving an env to many consumers is prime-rl's job (its `eval` -runs env servers); this CLI is the quick local path. -""" - -import asyncio -import contextlib -import logging -import os -import time -from collections.abc import AsyncIterator, Awaitable, Callable, Iterable -from typing import TypeVar, cast - -from verifiers.v1.cli.dashboard import dashboard -from verifiers.v1.cli.eval import resume -from verifiers.v1.cli.eval.hint import PRIME_RL_HINT -from verifiers.v1.cli.output import output_path, save_config -from verifiers.v1.clients import ModelContext -from verifiers.v1.configs.cli.eval import EvalConfig -from verifiers.v1.env import Env, RunSlot -from verifiers.v1.episode import Episode, EvalRunInfo -from verifiers.v1.utils.aio import run_shielded -from verifiers.v1.utils.eval import plan_rollouts -from verifiers.v1.utils.platform import ( - PushState, - abort_run, - finish_run, - log_episodes, - open_run, -) -from verifiers.v1.utils.trace_store import append_episode - -logger = logging.getLogger(__name__) - -T = TypeVar("T") - - -async def gather_rollouts(rollouts: Iterable[Awaitable[T]]) -> list[T]: - """`asyncio.gather`, but one rollout failing cancels the rest and waits for them - to unwind, so nothing keeps uploading into a run the caller is already closing. - Not a `TaskGroup`: that wraps errors in an `ExceptionGroup`, and `main` would no - longer see a `KeyboardInterrupt` as Ctrl-C.""" - tasks = [asyncio.ensure_future(rollout) for rollout in rollouts] - try: - return await asyncio.gather(*tasks) - except BaseException: - for task in tasks: - task.cancel() - # return_exceptions: wait for every task, not just the first to cancel. - await asyncio.gather(*tasks, return_exceptions=True) - raise - - -RunSlotFn = Callable[[RunSlot], Awaitable[Episode]] -OnComplete = Callable[[Episode], Awaitable[None]] - - -@contextlib.asynccontextmanager -async def _in_process( - env: Env, - config: EvalConfig, - semaphore: asyncio.Semaphore | None, - on_complete: OnComplete, -) -> AsyncIterator[RunSlotFn]: - """Run slots in-process: serving resources (shared tool servers, interception) - come up once for the run; the env's agents borrow them.""" - ctx = ModelContext( - client=config.client, model=config.model, sampling=config.sampling - ) - - async def run(slot: RunSlot) -> Episode: - return await env.run_slot(slot, ctx, semaphore, on_complete) - - async with env.serving(): - yield run - - -async def run_eval(config: EvalConfig) -> list[Episode]: - from verifiers.v1.utils.loaders import load_environment - - env = load_environment(config.env) - taskset = env.taskset.select(config.select) - if not taskset.bounded: - raise ValueError( - f"{type(taskset).__name__} is infinite - bound the run with -n" - ) - tasks = list(taskset) - out = output_path(config) - if config.resume: - # the env's own keep-verdict decides what resumes - plan = resume.load( - out, - tasks, - config.num_rollouts, - lambda episode: env.complete(cast(Episode, episode)), - ) - if not any(n for _, _, n in plan): - print( - f"nothing to resume in {out}: all {len(tasks)}x{config.num_rollouts} " - "rollouts already completed without error" - ) - raise SystemExit(0) - logger.info( - "resuming %s: %d task(s), %d rollout(s) owed", - out, - sum(n > 0 for _, _, n in plan), - sum(n for _, _, n in plan), - ) - else: - plan = plan_rollouts(tasks, config.num_rollouts) - save_config(config, out) - logger.info( - "running %dx%d rollouts on %s", len(plan), config.num_rollouts, config.model - ) - logger.info(PRIME_RL_HINT) - finished = [cast(Episode, episode) for _, kept, _ in plan for episode in kept] - start = time.time() - logger.info("results: %s", out) - - semaphore = ( - asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None - ) - write_lock = asyncio.Lock() - push_state = PushState() - - # Opened before the first rollout so every episode streams as it lands. - run = open_run(config, push_state, num_examples=len(tasks)) - # The run identity: every process this run spawns inherits it. - os.environ.setdefault("VF_RUN_ID", config.run.id) - # Resumed rollouts are part of this run too. - log_episodes(run, finished) - - async def on_complete(episode: Episode) -> None: - episode.record_run(EvalRunInfo(id=config.run.id, name=config.run.name)) - await append_episode(out, episode, write_lock) - await asyncio.to_thread(log_episodes, run, [episode]) - - # The run is closed out whatever breaks, env setup and teardown included. - try: - async with _in_process(env, config, semaphore, on_complete) as run_slot: - # the env's own slots: it fills their live traces as the rollouts run - planned = [slot for task, _, n in plan if n for slot in env.slots(task, n)] - slots = [RunSlot.finished(episode) for episode in finished] + planned - display = ( - dashboard(slots, config, start, push=push_state) - if config.rich is not None - else contextlib.nullcontext() - ) - async with display: - results = await gather_rollouts(run_slot(slot) for slot in planned) - episodes = finished + list(results) - # Drain and close out off the event loop so the view keeps refreshing. - # Shielded: a Ctrl-C here must not cancel the close-out before the - # worker picks it up (a cancelled executor item never runs), so it - # runs to completion first and the interrupt is re-raised after — - # by which point the run is finished and `abort_run` has nothing to do. - await run_shielded( - asyncio.to_thread(finish_run, run, episodes, push_state) - ) - except BaseException as e: - await asyncio.to_thread(abort_run, run, e, push_state) - raise - return episodes diff --git a/verifiers/v1/cli/init.py b/verifiers/v1/cli/init.py index ac631d4d4f..1fdda629ca 100644 --- a/verifiers/v1/cli/init.py +++ b/verifiers/v1/cli/init.py @@ -191,7 +191,7 @@ def _readme(dash: str, pkg: str, *, add_tool: bool, add_harness: bool) -> str: ```bash uv pip install -e . # install this package (or register it in your project) -uv run vf-eval {dash} -n 3 # evaluate a few tasks with the bash harness +uv run vf-validate {dash} # model-free gold checks ``` ## Layout @@ -228,7 +228,7 @@ def scaffold(config: InitConfig) -> Path: _write(pkg_dir / "servers" / "__init__.py", "") _write(pkg_dir / "servers" / "tool.py", _tool_py(stem, prefix)) - print(f"\ndone. next:\n uv pip install -e {env_dir}\n uv run vf-eval {dash} -n 3") + print(f"\ndone. next:\n uv pip install -e {env_dir}\n uv run vf-validate {dash}") return env_dir diff --git a/verifiers/v1/cli/output.py b/verifiers/v1/cli/output.py index 72fdf681da..0df052578f 100644 --- a/verifiers/v1/cli/output.py +++ b/verifiers/v1/cli/output.py @@ -3,10 +3,11 @@ import json import os from pathlib import Path +from typing import Protocol from pydantic import BaseModel -from verifiers.v1.configs.cli.eval import EvalConfig +from verifiers.v1.configs.cli.run import RunConfig from verifiers.v1.utils.trace_store import TRACES_FILE CONFIG_DIR = "configs" @@ -60,7 +61,14 @@ def saved_config_path(run_dir: Path) -> Path | None: return candidates[0] if candidates else None -def output_path(config: EvalConfig) -> Path: +class RunOutput(Protocol): + """A config that owns a run directory: `output_dir / run.dir`.""" + + output_dir: Path + run: RunConfig + + +def output_path(config: RunOutput) -> Path: """Where this run writes: `output_dir / run.dir` — the same grouping convention as training. The run directory defaults to the auto-generated run name (`------`).""" diff --git a/verifiers/v1/configs/cli/eval.py b/verifiers/v1/configs/cli/eval.py deleted file mode 100644 index 944bd4b38f..0000000000 --- a/verifiers/v1/configs/cli/eval.py +++ /dev/null @@ -1,150 +0,0 @@ -"""The `EvalConfig`: the single config object the eval CLI parses.""" - -from pathlib import Path -from uuid import uuid4 - -from pydantic import AliasChoices, Field, PrivateAttr, SerializeAsAny, model_validator -from pydantic_config import BaseConfig - -from verifiers.v1.clients import ClientConfig, EvalClientConfig -from verifiers.v1.configs.cli.env import narrowed_env_annotation, resolve_env_field -from verifiers.v1.configs.env import EnvConfig -from verifiers.v1.configs.select import SelectCLIConfig -from verifiers.v1.envs.single_agent import SingleAgentEnvConfig -from verifiers.v1.types import SamplingConfig - - -def default_run_name(env: EnvConfig, model: str) -> str: - """The auto-generated run name: `------`, a - descriptive leaf for the run directory `output_dir / run.name`. The short-id - suffix keeps repeated invocations from colliding.""" - taskset = env.taskset - name = taskset.name if taskset.id else "no-taskset" - if taskset.id and env.id: - # Same compounding as `EnvConfig.env_id`: a `best-of-n+gsm8k` run must - # not share a name with a plain `gsm8k` one. - name = f"{env.id}+{name}" - # Every seat's resolved harness, distinct, in role order. - harness = "+".join(dict.fromkeys(h.name for h in env.agent_harnesses().values())) - slug = ( - f"{name}--{model.replace('/', '--')}--{harness or 'default'}--{uuid4().hex[:8]}" - ) - return slug.lower() - - -class RichConfig(BaseConfig): - """The live dashboard.""" - - show_logs: bool = False - """Replace the dashboard's per-rollout rows with a live tail of the attempt's log - file (`logs/latest/eval.log`) — the env's own log lines included.""" - - -class RunConfig(BaseConfig): - name: str | None = None - """Run name. Auto-generated as `------` when unset.""" - - dir: str | None = None - """Run directory name — the run writes to `output_dir / dir`. Defaults to `run.name`; - set it only when the directory should differ from the display name.""" - - attach: str | None = None - """Stream into a run the launcher already created on the platform instead of opening a - new one — its evaluation id. Hosted evaluations pass the sandbox's `$EVALUATION_ID` - here; the platform owns that run's record and status, and a run that cannot be - attached to is an error rather than a local fallback. Requires `push`.""" - - _id: str | None = PrivateAttr(default=None) - - @property - def id(self) -> str: - """The run's one id, assigned by `open_run` from the prime-runs handle: the - platform's evaluation id online, the SDK's local id otherwise. The run mints - none of its own, so the run dir, every trace and the dashboard agree.""" - if self._id is None: - raise RuntimeError("the run has no id until `open_run` has opened it") - return self._id - - def assign_id(self, run_id: str) -> None: - """Called once by `open_run`, before the first rollout.""" - if self._id is not None and self._id != run_id: - raise RuntimeError(f"the run already has id {self._id!r}") - self._id = run_id - - -class EvalConfig(BaseConfig): - env: SerializeAsAny[EnvConfig] = SingleAgentEnvConfig() - """The environment — which env, its seed taskset, each agent, its knobs. Narrowed to - the selected env's config class by the env id, else the taskset id.""" - run: RunConfig = Field(default_factory=RunConfig) - """Run identity: `run.name` is the display name, `run.dir` names the directory - under `output_dir`, and `run.id` is stamped on traces.""" - model: str = Field( - "deepseek/deepseek-v4-flash", validation_alias=AliasChoices("model", "m") - ) - """Model id.""" - client: ClientConfig = EvalClientConfig() - sampling: SamplingConfig = SamplingConfig() - select: SelectCLIConfig = SelectCLIConfig() - """Which of the taskset's tasks to evaluate, under `--select.*` (`-n` sets - `select.limit`, `-s` sets `select.shuffle`).""" - num_rollouts: int = Field( - 1, - ge=1, - validation_alias=AliasChoices( - "group_size", "rollouts_per_example", "num_rollouts", "r" - ), - ) - """Independent episodes per task — the trainer's group size.""" - max_concurrent: int | None = Field( - 128, ge=1, validation_alias=AliasChoices("max_concurrent", "c") - ) - """Episodes in flight at once, `None` for no limit. An episode plays its agents one - at a time, so this is the live agent runs too — until `--env.max-concurrent-agents` - says otherwise.""" - verbose: bool = Field(False, validation_alias=AliasChoices("verbose", "v")) - """Log at debug level instead of the default info.""" - dry_run: bool = Field(False, exclude=True) - """Resolve + validate the config and dump it, then exit. Excluded from the saved - config so re-running `@ configs/eval.json` (or resuming/replaying the dir) actually runs.""" - clean: bool = Field(False, exclude=True) - """Delete the run directory (`output_dir / run.dir`) before running, overwriting a - previous run's results. Excluded from the saved config.""" - rich: RichConfig | None = Field(default_factory=RichConfig) - """The live dashboard (on by default; `--no-rich` streams logs to the console - instead); `--rich.show-logs` swaps the rollout rows for the run's logs.""" - push: bool = True - """Upload the finished run to the Prime Intellect platform (the private Evaluations - tab) at the end of the eval. On by default; disable with `--no-push`. Needs - `$PRIME_API_KEY` or `prime login`.""" - output_dir: Path = Field( - Path("outputs"), validation_alias=AliasChoices("output_dir", "o") - ) - """Directory that groups related runs. The run itself (`configs/eval.json` + - `traces.jsonl`) writes to `output_dir / run.dir`.""" - resume: bool = Field(False, exclude=True) - """Re-run the run's missing/errored rollouts in place instead of starting fresh. The - run dir comes from the resolved config (`output_dir / run.dir`), so resume with the - run's own config — e.g. `uv run vf-eval @ /configs/eval.json --resume`. Excluded - from the saved config.""" - - @model_validator(mode="before") - @classmethod - def _resolve_env(cls, data): - return resolve_env_field(data, narrowed_env_annotation(cls)) - - @model_validator(mode="after") - def auto_setup_run_name(self): - if self.run.name is None: - self.run.name = default_run_name(self.env, self.model) - if self.run.dir is None: - self.run.dir = self.run.name - return self - - @model_validator(mode="after") - def attach_needs_push(self): - if self.run.attach and not self.push: - raise ValueError( - "run.attach names a run on the platform, so it needs push (drop --no-push)" - ) - return self diff --git a/verifiers/v1/configs/cli/run.py b/verifiers/v1/configs/cli/run.py new file mode 100644 index 0000000000..f5ba38cf08 --- /dev/null +++ b/verifiers/v1/configs/cli/run.py @@ -0,0 +1,42 @@ +"""Run identity shared by the CLIs that write a run directory (`validate`, `gepa`).""" + +from uuid import uuid4 + +from pydantic import PrivateAttr +from pydantic_config import BaseConfig + +from verifiers.v1.configs.env import EnvConfig + + +def default_run_name(env: EnvConfig, model: str) -> str: + """The auto-generated run name: `------`, a + descriptive leaf for the run directory `output_dir / run.name`. The short-id + suffix keeps repeated invocations from colliding.""" + taskset = env.taskset + name = taskset.name if taskset.id else "no-taskset" + if taskset.id and env.id: + # Same compounding as `EnvConfig.env_id`: a `best-of-n+gsm8k` run must + # not share a name with a plain `gsm8k` one. + name = f"{env.id}+{name}" + # Every seat's resolved harness, distinct, in role order. + harness = "+".join(dict.fromkeys(h.name for h in env.agent_harnesses().values())) + slug = ( + f"{name}--{model.replace('/', '--')}--{harness or 'default'}--{uuid4().hex[:8]}" + ) + return slug.lower() + + +class RunConfig(BaseConfig): + name: str | None = None + """Run name. Auto-generated as `------` when unset.""" + + dir: str | None = None + """Run directory name — the run writes to `output_dir / dir`. Defaults to `run.name`; + set it only when the directory should differ from the display name.""" + + # TODO: fetch the id from the Prime SDK once runs are registered there. + _id: str = PrivateAttr(default_factory=lambda: str(uuid4())) + + @property + def id(self) -> str: + return self._id diff --git a/verifiers/v1/configs/cli/validate.py b/verifiers/v1/configs/cli/validate.py index faaa2f5f0b..2ba542b28a 100644 --- a/verifiers/v1/configs/cli/validate.py +++ b/verifiers/v1/configs/cli/validate.py @@ -6,7 +6,7 @@ from pydantic import AliasChoices, Field, SerializeAsAny, model_validator from pydantic_config import BaseConfig -from verifiers.v1.configs.cli.eval import RunConfig +from verifiers.v1.configs.cli.run import RunConfig from verifiers.v1.configs.select import SelectCLIConfig from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.runtimes import PrimeConfig, RuntimeConfig diff --git a/verifiers/v1/configs/client.py b/verifiers/v1/configs/client.py index 0ef124bac0..231370dcd5 100644 --- a/verifiers/v1/configs/client.py +++ b/verifiers/v1/configs/client.py @@ -4,9 +4,8 @@ + extra headers); `clients.resolve_client` turns one into a live `Client` — the interception server builds one per distinct config and shares it across the rollouts it multiplexes. The default Prime endpoint, API key, and team fall back to -the active Prime CLI config, so direct `uv run vf-eval` calls behave like `prime eval`. -Both the eval entrypoint (its model client) and in-env LLM calls (e.g. a judge reward) -build clients from these. `ClientConfig` is the CLI-selectable discriminated union +the active Prime CLI config. Both an eval's model client and in-env LLM calls (e.g. a +judge reward) build clients from these. `ClientConfig` is the CLI-selectable discriminated union (eval | train). """ diff --git a/verifiers/v1/gepa/adapter.py b/verifiers/v1/gepa/adapter.py index 20fc95254e..adf3c6fe51 100644 --- a/verifiers/v1/gepa/adapter.py +++ b/verifiers/v1/gepa/adapter.py @@ -40,7 +40,7 @@ class GEPAAdapter: semaphore: asyncio.Semaphore | None = None on_complete: Callable[[Episode], Awaitable[None]] | None = None """Called with each rollout's episode as it finalizes — the runner's persist hook that - streams episodes to `traces.jsonl`, exactly as `run_eval` does.""" + streams episodes to `traces.jsonl`.""" reflection_columns: list[str] = field(default_factory=list) propose_new_texts: Callable[..., Candidate] | None = None """Part of GEPA's adapter protocol — its proposer reads this attribute on every reflection diff --git a/verifiers/v1/gepa/config.py b/verifiers/v1/gepa/config.py index 0aa5b2a993..364e5fcbdc 100644 --- a/verifiers/v1/gepa/config.py +++ b/verifiers/v1/gepa/config.py @@ -15,7 +15,7 @@ from verifiers.v1.clients import EvalClientConfig from verifiers.v1.configs.cli.env import narrowed_env_annotation, resolve_env_field -from verifiers.v1.configs.cli.eval import RunConfig, default_run_name +from verifiers.v1.configs.cli.run import RunConfig, default_run_name from verifiers.v1.configs.env import EnvConfig from verifiers.v1.configs.select import SelectCLIConfig from verifiers.v1.envs.single_agent import SingleAgentEnvConfig diff --git a/verifiers/v1/gepa/runner.py b/verifiers/v1/gepa/runner.py index 39ad185677..af43866adf 100644 --- a/verifiers/v1/gepa/runner.py +++ b/verifiers/v1/gepa/runner.py @@ -51,12 +51,12 @@ def run_gepa(env: Env, config: GEPAConfig) -> GEPAResult: if run_dir is not None: save_config( config, run_dir, "gepa.json" - ) # resolved config + a fresh traces.jsonl (like run_eval) + ) # resolved config + a fresh traces.jsonl logger.info("results: %s", run_dir) # optimize() is synchronous and blocking, so it drives the run from this (main) thread. We # own one event loop: `env.serving()` (shared tool servers + interception pool, built once - # like run_eval) is entered on it, each rollout batch runs on it via `loop.run_until_complete` + # once) is entered on it, each rollout batch runs on it via `loop.run_until_complete` # (GEPAAdapter.evaluate), and it's all torn down in `finally`. Keeping optimize() on the # main thread means a Ctrl-C raises straight through it into this teardown. loop = asyncio.new_event_loop() @@ -64,7 +64,7 @@ def run_gepa(env: Env, config: GEPAConfig) -> GEPAResult: asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None ) # Stream every rollout's episode to traces.jsonl as it finalizes — the same persist hook - # run_eval passes to `env.run_slot` (each trace records its candidate prompt). + # `env.run_slot` takes (each trace records its candidate prompt). write_lock = asyncio.Lock() async def on_complete(episode: Episode) -> None: diff --git a/verifiers/v1/utils/platform.py b/verifiers/v1/utils/platform.py deleted file mode 100644 index 5362ba4b46..0000000000 --- a/verifiers/v1/utils/platform.py +++ /dev/null @@ -1,179 +0,0 @@ -"""The eval's run on the Prime Intellect platform (`--no-push` to keep it local).""" - -import asyncio -import logging -import os -from collections.abc import Mapping -from dataclasses import dataclass -from typing import Any - -import prime_runs as pr -from prime_runs.projection import ( - build_samples, # noqa: F401 - prime-rl imports it from here -) - -from verifiers.v1.configs.cli.eval import EvalConfig -from verifiers.v1.episode import Episode - -logger = logging.getLogger(__name__) - - -@dataclass -class PushState: - """The dashboard's view of the run: reads through to it, owns no I/O.""" - - run: pr.Run | None = None - error: str | None = None - - @property - def incomplete(self) -> str | None: - """Current upload errors and record losses, including while the run is active.""" - return _losses(self.run) if self.run is not None else None - - @property - def url(self) -> str | None: - return self.run.url if self.run is not None else None - - @property - def finished(self) -> bool: - return self.run is not None and self.run.finished - - @property - def started(self) -> bool: - """A live run, or a reason there isn't one.""" - return self.error is not None or self.url is not None - - -def open_run(config: EvalConfig, state: PushState, *, num_examples: int) -> pr.Run: - """Open the run this eval streams into, before the first rollout, and give the - config the run's id. A run that cannot be opened is logged and replaced by a - disabled one; the eval goes on. With `run.attach` the run already exists on the - platform (a hosted evaluation's launcher created it and is waiting on it), so - there is no local fallback: failing to attach fails the eval.""" - attach = config.run.attach - identity: dict[str, Any] = { - "name": config.run.name, - # Resolved by name via the hub's get-or-create; no taskset, nothing to attach to. - "environments": [config.env.taskset.id] if config.env.taskset.id else [], - "model": config.model, - "framework": "verifiers", - # The v0 keys the dashboard's lists read. The config itself is a follow-up: - # a dump or the launched file can carry credentials and needs masking first. - "config": { - "model": config.model, - "num_examples": num_examples, - "rollouts_per_example": config.num_rollouts, - }, - } - if config.push and os.getenv(pr.MODE_ENV, "").strip().lower() == "disabled": - if attach: - raise RuntimeError( - f"run.attach={attach!r} names a run on the platform, but " - f"{pr.MODE_ENV}=disabled would keep this eval local" - ) - # The SDK's own kill switch; the explicit `mode="online"` below would override it. - logger.info("--push: %s=disabled; running without a platform run", pr.MODE_ENV) - elif config.push: - try: - state.run = pr.init(mode="online", id=attach, **identity) - except Exception as e: - if attach: - # The launcher's run would sit at running until it times out; a local - # run nobody reads is not a substitute. - raise RuntimeError( - f"--run.attach: could not attach to run {attach!r} ({type(e).__name__}: {e})" - ) from e - logger.warning( - "--push: could not open the run (%s: %s); running without it", - type(e).__name__, - e, - ) - state.error = f"{type(e).__name__}: {e}" - if state.run is None: - state.run = pr.init(mode="disabled", **identity) - # The run's one id: the platform's when online, the SDK's local one otherwise. - # The SDK keys every upload to it regardless; this is for the local records. - config.run.assign_id(state.run.id) - return state.run - - -def log_episodes(run: pr.Run, episodes: list[Episode]) -> None: - """Hand finished episodes to the run, best effort: the SDK already keeps upload - failures on its own thread, so this only guards the hand-off itself. A problem - here is the platform's, never the eval's.""" - if not episodes: - return - try: - run.log_episodes(episodes) - except Exception as e: # noqa: BLE001 - the rollouts are on disk; report, don't raise - logger.warning( - "--push: could not queue %d episode(s) (%s: %s)", - len(episodes), - type(e).__name__, - e, - ) - - -def finish_run(run: pr.Run, episodes: list[Episode], state: PushState) -> None: - """Drain, write the run's aggregates, close it out. Blocking: call it off the loop.""" - try: - summary = pr.metrics.from_episodes(episodes) - except Exception as e: # noqa: BLE001 - close the run even without its headline - logger.warning( - "--push: could not aggregate the run's metrics (%s: %s)", - type(e).__name__, - e, - ) - summary = None - _close(run, state, summary=summary) - - -def abort_run(run: pr.Run, error: BaseException, state: PushState) -> None: - """Close the run out after the eval broke, so it doesn't sit at running. Not - for a break during `finish_run`: that close-out completes on its own thread - and the SDK lets the first `finish()` decide the status — it sets `run.status` - as it starts, so an in-flight one is visible here before it is `finished`.""" - if run.finished or run.status is not pr.RunStatus.RUNNING: - return - if isinstance(error, (KeyboardInterrupt, asyncio.CancelledError)): - status, message = pr.RunStatus.CANCELLED, "interrupted" - else: - status, message = pr.RunStatus.FAILED, f"{type(error).__name__}: {error}" - _close(run, state, status=status, error=message) - - -def _close( - run: pr.Run, - state: PushState, - summary: Mapping[str, Any] | None = None, - status: pr.RunStatus = pr.RunStatus.COMPLETED, - error: str | None = None, -) -> None: - """`run.finish()`, best effort: the results are on disk, so nothing here may raise.""" - try: - run.finish(summary, status=status, error=error) - except Exception as e: # noqa: BLE001 - the run is over; report, don't raise - logger.warning( - "--push: could not close out the run (%s: %s)", type(e).__name__, e - ) - if state.error is None: - state.error = f"{type(e).__name__}: {e}" - else: - if incomplete := state.incomplete: - logger.warning("--push: %s, but %s", status.value, incomplete) - if run.url: - # The run's own status: `finish()` is a no-op once another caller closed it. - logger.info("--push: %s -> %s", run.status.value, run.url) - - -def _losses(run: pr.Run) -> str | None: - """What the run could not finish or store, or `None`.""" - parts = list(run.errors) - parts.extend( - f"{count} record(s) not stored by the {sink} sink" - for sink, count in sorted(run.failed_records.items()) - if count - ) - if run.dropped_records: - parts.append(f"{run.dropped_records} record(s) never queued (uploader overrun)") - return "; ".join(parts) or None From 37b6e11f2bd941a3924d436cd0b8a07d9aa8f983 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 15 Sep 2026 21:59:54 +0000 Subject: [PATCH 2/2] scaffold README points eval at prime-rl The scaffold installs only verifiers, which has no eval executable since the eval CLI moved to prime-rl. Co-Authored-By: Claude Fable 5.1 --- verifiers/v1/cli/init.py | 3 +++ verifiers/v1/gepa/runner.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/verifiers/v1/cli/init.py b/verifiers/v1/cli/init.py index 1fdda629ca..ca3e0a892a 100644 --- a/verifiers/v1/cli/init.py +++ b/verifiers/v1/cli/init.py @@ -194,6 +194,9 @@ def _readme(dash: str, pkg: str, *, add_tool: bool, add_harness: bool) -> str: uv run vf-validate {dash} # model-free gold checks ``` +3. Evaluate from a [prime-rl](https://github.com/PrimeIntellect-ai/prime-rl) checkout + that has this package installed: `uv run eval {dash} -n 3`. + ## Layout {layout_block} diff --git a/verifiers/v1/gepa/runner.py b/verifiers/v1/gepa/runner.py index af43866adf..2b5c91e998 100644 --- a/verifiers/v1/gepa/runner.py +++ b/verifiers/v1/gepa/runner.py @@ -55,7 +55,7 @@ def run_gepa(env: Env, config: GEPAConfig) -> GEPAResult: logger.info("results: %s", run_dir) # optimize() is synchronous and blocking, so it drives the run from this (main) thread. We - # own one event loop: `env.serving()` (shared tool servers + interception pool, built once + # own one event loop: `env.serving()` (shared tool servers + interception pool, built # once) is entered on it, each rollout batch runs on it via `loop.run_until_complete` # (GEPAAdapter.evaluate), and it's all torn down in `finally`. Keeping optimize() on the # main thread means a Ctrl-C raises straight through it into this teardown.