diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml index a8b6b18204..963acf71af 100644 --- a/.markdownlint-cli2.yaml +++ b/.markdownlint-cli2.yaml @@ -28,11 +28,6 @@ overrides: MD007: false MD029: false combine: merge - - filter: - - skills/evaluate-environments/SKILL.md - config: - MD029: false - combine: merge - filter: - verifiers/envs/experimental/composable/tasksets/swe/README.md config: diff --git a/AGENTS.md b/AGENTS.md index d7cbe7cbcb..ff18014eb4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,7 @@ ## Running code - **Always use uv**: run code and commands with `uv run`, never raw `python`. Make sure `uv` is installed ([docs](https://docs.astral.sh/uv/getting-started/installation/)). -- **Run evals**: run a taskset with `uv run vf-eval `. +- **Check tasksets**: check a taskset without a model with `uv run vf-validate `. Run a few episodes with `uv run vf-run -n 1 -r 1`. Run full model evals with prime-rl's `uv run eval`. - **Validate TOML first**: validate config `.toml` files before running them. - **Don't add dependencies**: never add dependencies or optional extras to the top-level `pyproject.toml`. diff --git a/docs/building-environments.md b/docs/building-environments.md index b178173956..83477b44ff 100644 --- a/docs/building-environments.md +++ b/docs/building-environments.md @@ -6,7 +6,7 @@ An environment package defines the tasks an agent receives, the files and tools A verifiers evaluation spans up to four execution boundaries: -- **The evaluator** (the machine running `vf-eval` or an environment worker) loads datasets, runs Python task hooks (`setup`, `finalize`, `@vf.reward`), coordinates agents, and records traces. +- **The evaluator** (the machine running the environment server) loads datasets, runs Python task hooks (`setup`, `finalize`, `@vf.reward`), coordinates agents, and records traces. - **The agent sandbox** (the agent's configured [runtime](runtimes.md#capability-matrix)) runs the harness, holds the task workspace, and executes any commands the agent invokes. - **The grading sandbox** is a fresh container started after the agent finishes when you use isolated scoring (`ScoringConfig(mode="isolated")`), giving graders a clean filesystem containing only declared agent outputs and private test assets. - **Task tools** run in their own configured runtime by default, inside the agent sandbox when configured with `colocated = true`, inside a named task service when configured with `service = ""`, or on a Prime VM of their own, under its own egress policy, when their `runtime` is a Prime runtime beside a Prime workspace. @@ -188,9 +188,7 @@ Install your new package into the CLI environment and run a quick smoke test bef ```bash uv pip install -e environments/file_sum -uv run vf-eval @ file-sum.toml --dry-run uv run vf-validate file-sum --runtime.type docker -n 1 -uv run vf-eval @ file-sum.toml -n 1 -r 1 -c 1 ``` -`--dry-run` validates the TOML config and taskset resolution without starting containers. Next, `vf-validate` runs your task's `validate` hook and a no-op check inside a single sandbox to confirm that the gold answer scores `1.0` while an untouched workspace scores `0.0`. Because `vf-validate` exercises `Task` directly in one sandbox, it does not transfer artifacts or call `stage_verifier` automatically; if your gold check needs to exercise the full isolated-grading flow without a model, request `trace` by name in `validate`, finalize the submission, and call `await vf.grade_submission(self, trace, runtime)` (or call `await self.stage_verifier(runtime)` directly). Finally, running a single rollout with `vf-eval -n 1 -r 1 -c 1` exercises the full solver-to-verifier transfer lifecycle; see [debugging](debugging.md) for inspecting saved traces and diagnosing setup or grading errors. +`vf-validate` runs your task's `validate` hook and a no-op check inside a single sandbox to confirm that the gold answer scores `1.0` while an untouched workspace scores `0.0`. Because `vf-validate` exercises `Task` directly in one sandbox, it does not transfer artifacts or call `stage_verifier` automatically; if your gold check needs to exercise the full isolated-grading flow without a model, request `trace` by name in `validate`, finalize the submission, and call `await vf.grade_submission(self, trace, runtime)` (or call `await self.stage_verifier(runtime)` directly). Finally, running a single rollout with `uv run vf-run file-sum -n 1 -r 1` exercises the full solver-to-verifier transfer lifecycle; see [debugging](debugging.md) for inspecting saved traces and diagnosing setup or grading errors. diff --git a/docs/debugging.md b/docs/debugging.md index 15036396cc..8429b4d647 100644 --- a/docs/debugging.md +++ b/docs/debugging.md @@ -4,25 +4,22 @@ verifiers includes CLI tools to validate task setup and scoring before spending ## Before running an agent -Before launching a full evaluation, you can verify your configuration, task setup, reference solutions, and scoring logic directly against a runtime: +Before launching a full evaluation, you can verify your task setup, reference solutions, and scoring logic directly against a runtime: ```bash -uv run vf-eval my-task --dry-run uv run vf-validate my-task -n 1 --only-setup uv run vf-validate my-task -n 1 --only-gold uv run vf-validate my-task -n 1 --only-noop uv run vf-debug my-task -n 1 --command 'pwd && ls -la' ``` -`vf-eval --dry-run` resolves and saves the configuration without loading dataset rows or starting sandboxes. - `vf-validate` boots the configured runtime and runs task checks without an agent: - `--only-setup` runs `Task.setup(runtime)` to verify that containers, files, and background services start cleanly. - `--only-gold` runs `setup` followed by `Task.validate(runtime)` to check a reference answer or solution; `validate` can also request the check's `trace` by name. `validate` returns `True` (valid), `False` (invalid), or `None` when the task defines no gold check (reported as `unchecked` rather than a pass). - `--only-noop` runs `setup`, `finalize`, and task scoring on the untouched environment. Any task that already scores `1.0` or higher with no agent actions is flagged as `invalid` (lower partial rewards pass the check and are recorded in the results, while tasks with no reward are marked `unchecked`). -When you run `vf-validate` without an `--only-*` flag, it runs both the gold and noop checks in separate runtimes. Because validation runs directly against the task rather than through a harness or `Env`, it does not apply harness network restrictions, transfer artifacts, or invoke `stage_verifier`. If your gold check relies on private tests, call your staging helper inside `Task.validate`, and use a real `vf-eval` run to test the full isolated-grading lifecycle. +When you run `vf-validate` without an `--only-*` flag, it runs both the gold and noop checks in separate runtimes. Because validation runs directly against the task rather than through a harness or `Env`, it does not apply harness network restrictions, transfer artifacts, or invoke `stage_verifier`. If your gold check relies on private tests, call your staging helper inside `Task.validate`, and use a real `vf-run` rollout to test the full isolated-grading lifecycle. Validation writes `results.jsonl`, `summary.json`, `logs/validate.log`, and `configs/resolved/validate.json`. You can re-run missing, errored, or timed-out checks in place with `--resume`: @@ -30,7 +27,7 @@ Validation writes `results.jsonl`, `summary.json`, `logs/validate.log`, and `con uv run vf-validate @ /configs/resolved/validate.json --resume ``` -For interactive troubleshooting, `vf-debug` runs `Task.setup` and then executes a one-off shell `--command` or uploaded `--script-path check.sh` inside the runtime before tearing it down. Because neither `vf-validate` nor `vf-debug` wraps the task in an `Env`, both take top-level `--taskset.*` and `--runtime.*` flags (rather than `vf-eval`'s `--env.taskset.*` and `--env.agent.runtime.*`), alongside [`--select.*`](tasksets.md#selecting-tasks) to filter tasks. Note that both commands execute real task setup (which may spin up paid sandboxes), and `vf-validate`'s noop check will call any configured judges. +For interactive troubleshooting, `vf-debug` runs `Task.setup` and then executes a one-off shell `--command` or uploaded `--script-path check.sh` inside the runtime before tearing it down. Because neither `vf-validate` nor `vf-debug` wraps the task in an `Env`, both take top-level `--taskset.*` and `--runtime.*` flags, alongside [`--select.*`](tasksets.md#selecting-tasks) to filter tasks. Note that both commands execute real task setup (which may spin up paid sandboxes), and `vf-validate`'s noop check will call any configured judges. ## Re-score without rerunning the agent diff --git a/docs/evaluation.md b/docs/evaluation.md deleted file mode 100644 index 71d71e0052..0000000000 --- a/docs/evaluation.md +++ /dev/null @@ -1,131 +0,0 @@ -# Evaluation - -For new model evaluations, use [`uv run eval` in prime-rl](https://github.com/PrimeIntellect-ai/prime-rl/blob/main/docs/eval.md). - -To evaluate an installed taskset, run `vf-eval`: - -```bash -uv run vf-eval primeintellect/terminal-bench-2 -``` - -The taskset package must already be installed in your environment (passing an ID like `owner/name@version` loads the installed `name` package rather than downloading or pinning that version). Because tasksets and harnesses expose their own CLI flags, include their IDs when passing `--help` to see all available options: - -```bash -uv run vf-eval my-task --env.agent.harness.id codex --help -``` - -You can also configure a run with a `.toml` file: - -```toml -model = "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B" - -[sampling] -temperature = 1.0 - -[env.taskset] -id = "primeintellect/terminal-bench-2" - -[env.agent.harness] -id = "codex" -version = "0.116.0" - -[env.agent.runtime] -type = "docker" -``` - -Validate the config with `uv run vf-eval @ config.toml --dry-run`, and run the evaluation with `uv run vf-eval @ config.toml`. Dotted CLI flags such as `--sampling.temperature 0.5` override values from the `.toml` file when both are present. - -By default, evaluation outputs are written to `output_dir / run.dir`, which resolves to `outputs/------/` (where `` is the taskset name, prefixed by the paired env ID when `--env.id` sets one). Each run directory contains: - -```text -configs/eval.toml # launch TOML, when supplied -configs/resolved/eval.json # full resolved config, usable with @ -traces.jsonl # one complete episode per line -logs/attempt_1/eval.log # run and worker logs for this launch -logs/latest # symlink to the latest attempt -``` - -## Model endpoints - -By default, the client connects to Prime Inference and authenticates with `PRIME_API_KEY`, falling back to your active Prime CLI credentials. To point `vf-eval` at another compatible endpoint, set `base_url` and the name of the environment variable holding your API key: - -```toml -[client] -base_url = "http://localhost:8000/v1" -api_key_var = "MODEL_API_KEY" -``` - -The endpoint must support whichever API the harness uses. In multi-agent runs, each agent can override `model`, `client`, and `sampling` under `env.`, while task [judges](tasksets.md#using-judges) configure their own client settings independently. - -To pass secret headers from environment variables or customize HTTP timeouts, use `client.headers_from_env` and `client.timeout`: - -```toml -[client.headers_from_env] -"X-Project-Token" = "PROJECT_TOKEN" - -[client.timeout] -connect = 30 -read = 300 -``` - -Unlike static `client.headers` (which are written into the saved resolved config), `headers_from_env` reads environment variables when the client starts and overrides any matching static headers. HTTP timeouts are in seconds: `connect` defaults to `30`, while `read`, `write`, and `pool` have no limit by default. Because `read` only bounds the interval between received chunks rather than the total duration of a response, use agent stage timeouts to cap a full rollout. - -## Sampling settings - -Shared generation parameters go under `[sampling]`, with per-agent overrides under `[env..sampling]`. Overrides merge field by field, so setting an agent's `temperature` preserves the shared token limit and provider options (pass `None` in Python or `null` in JSON to clear an inherited field). An agent that sets its own `client` does not inherit `[sampling]`, because shared options can be specific to the run's endpoint. Give that agent all of its sampling under `[env..sampling]`. - -Alongside standard parameters like `temperature`, `top_p`, `reasoning_effort`, and `max_tokens` (or its alias `max_completion_tokens`), you can pass provider-specific options either directly in `[sampling]` or inside `extra_body`. - -## Resuming evaluations - -To resume an interrupted or partially failed evaluation, re-run its saved resolved config with `--resume`: - -```bash -uv run vf-eval @ /configs/resolved/eval.json --resume -``` - -Completed episodes are preserved in the run directory, while missing, errored, or malformed episodes run again in place. To discard the existing run directory and start over instead, pass `--clean` (or write to a new directory to keep the previous results). - -Because `--resume` matches completed episodes to tasks by their `TaskData` hash (filling each occurrence's `num_rollouts` slots in order when identical task data appears more than once), the installed taskset, code, and dataset must stay unchanged across runs. Overriding `Task.key` does not replace hash matching: any change to `TaskData`, including its assigned `idx`, produces a new hash and causes the task to run again. - -## Timeouts and retries - -Full-rollout retries are disabled by default (separate from HTTP retries inside the provider SDK). You can configure retry budgets per error type under `[env.agent.retries]`, or under `[env.retries]` for environments that manage shared resources such as Harbor Compose: - -```toml -[env.agent.retries] -max_retries = 0 - -[[env.agent.retries.rules]] -type = "ProviderError" -status_code = [429, "5xx"] -max_retries = 2 - -[[env.agent.retries.rules]] -type = "SandboxError" -max_retries = 2 -``` - -When an agent rollout fails, `rules` are checked in order and the first rule where all fields match sets the retry budget for that error; unmatched errors use the top-level `max_retries` budget. Each retry starts a fresh attempt from the beginning. To bound how long individual stages or rollouts can run, configure stage timeouts and token limits on the [runtime](runtimes.md); see [debugging](debugging.md) for diagnosing failed episodes. - -## Disabling tools - -Most harnesses expose a `disabled_tools` list that you can use to turn off specific built-in tools: - -```toml -[env.agent.harness] -disabled_tools = ["shell_tool"] -``` - -Tool names and support for disabling them depend on the harness; check the harness documentation for the exact names it accepts. - -## Skills - -Harnesses whose underlying program natively supports `SKILL.md` files (such as Claude Code and Codex) accept a `skills` list. Passing a local skill directory uploads it to the runtime, while `{runtime = "..."}` copies skills from a directory already inside the runtime into the harness's skill discovery path: - -```toml -[env.agent.harness] -skills = [{runtime = "/opt/skills"}, "path/to/my-skill"] -``` - -Tasks can also declare skills via `TaskData.skills` (for example, `skills=[{"runtime": "/opt/skills"}]`). Task skills are installed first, followed by harness skills, with later files overwriting earlier ones at the same path. Each rollout gets its own copy after task setup completes, and missing source directories (or configuring `skills` on a harness that does not support them) will fail the run. diff --git a/docs/gepa.md b/docs/gepa.md index 46c34ea155..9a58f649df 100644 --- a/docs/gepa.md +++ b/docs/gepa.md @@ -8,7 +8,7 @@ uv run vf-gepa reverse-text `vf-gepa` alternates between running rollouts on the taskset and asking a reflection LLM to review the trajectories and propose improvements to `TaskData.system_prompt`. -GEPA reuses the same `env` (taskset + agent), `client`, and `sampling` configuration as `vf-eval`, so its `.toml` config looks almost identical: +GEPA takes the same `env` (taskset + agent), `client`, and `sampling` configuration as `vf-run`: ```toml model = "deepseek/deepseek-v4-flash" @@ -27,7 +27,7 @@ Validate the config with `uv run vf-gepa @ config.toml --dry-run`, and run optim ## Common config values -- `model` / `-m` — model for the rollouts under optimization (default: `deepseek/deepseek-v4-flash`, same as `vf-eval`) +- `model` / `-m` — model for the rollouts under optimization (default: `deepseek/deepseek-v4-flash`) - `reflection_model` / `reflection_client` — teacher model and endpoint that propose new prompts (default: reuse `model` and `client`) - `select` — [task selection](tasksets.md#selecting-tasks) applied before splitting into train and validation sets (shuffling is off by default) - `num_train` / `num_val` — train tasks for reflection minibatches and validation tasks for scoring candidates on the Pareto frontier (defaults: `100` / `50`) @@ -38,14 +38,9 @@ Task selection runs before the train/val split: the first `num_train` selected t ## Output -Results are written to `output_dir / run.dir`, which defaults to `outputs/------/` just like `vf-eval`. The resolved config is saved at `configs/resolved/gepa.json`, and when the run finishes, the best system prompt is printed to the console and written to `best_system_prompt.txt`. +Results are written to `output_dir / run.dir`, which defaults to `outputs/------/`. The resolved config is saved at `configs/resolved/gepa.json`, and when the run finishes, the best system prompt is printed to the console and written to `best_system_prompt.txt`. -You can pass that file directly back into evaluation or training via `--env.taskset.system-prompt`: - -```bash -uv run vf-eval reverse-text \ - --env.taskset.system-prompt outputs//best_system_prompt.txt -``` +Pass that file back into evaluation or training through the taskset's `system_prompt` config field. ## Limitations diff --git a/docs/getting_started.md b/docs/getting_started.md index c32bbb37cc..092344a7a3 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -9,28 +9,26 @@ cd verifiers uv sync ``` -The checkout installs the example tasksets right away, so you can verify your setup with a local dry run that doesn't call a model: +The checkout installs the example tasksets right away, so you can verify your setup with a local check that doesn't call a model: ```bash -uv run vf-eval reverse-text --env.agent.harness.id null \ - --env.agent.runtime.type subprocess --dry-run +uv run vf-validate reverse-text -n 1 ``` -For full model evaluations, we recommend [`uv run eval` in prime-rl](https://github.com/PrimeIntellect-ai/prime-rl/blob/main/docs/eval.md), which is replacing the bundled `vf-eval` CLI. +To run a few episodes against a model, use `uv run vf-run reverse-text -n 2 -r 1`. For full model evaluations, use [`uv run eval` in prime-rl](https://github.com/PrimeIntellect-ai/prime-rl/blob/main/docs/eval.md). To work with other tasksets, including packages you build yourself following [building environments](building-environments.md#create-a-package), install them into the environment first (for example, `uv pip install -e environments/`) so the CLI can resolve their IDs. Harbor-based tasksets require Python 3.12+ and the `harbor` extra (`uv sync --extra harbor`). ## Commands -verifiers installs six CLI commands, which you run with `uv run` in the environment where verifiers and your taskset package are installed: +verifiers installs five CLI commands, which you run with `uv run` in the environment where verifiers and your taskset package are installed: | Command | Use | | --- | --- | | [`vf-validate `](debugging.md#before-running-an-agent) | Check task setup, a known gold solution, or the untouched task | | [`vf-debug `](debugging.md#before-running-an-agent) | Run a command or script inside the prepared task runtime | -| [`vf-replay `](debugging.md#re-score-without-rerunning-the-agent) | Re-score saved `vf-eval` traces without recreating their sandbox | +| [`vf-replay `](debugging.md#re-score-without-rerunning-the-agent) | Re-score saved traces without recreating their sandbox | | `vf-run ` | Run a few episodes in-process and save their traces, with a one-line summary | | [`vf-gepa `](gepa.md) | Optimize a system prompt against task rewards | -| [`vf-eval `](evaluation.md) | Run the bundled evaluator | -Pass `--help` to any command to inspect its flags. Because tasksets and harnesses register their own options dynamically, include the taskset and harness IDs before `--help` when checking evaluation flags (note that `vf-validate` and `vf-debug` take `--taskset.*` and `--runtime.*`, whereas `vf-eval` nests them under `--env.taskset.*` and `--env..runtime.*`). +Pass `--help` to any command to inspect its flags. Because tasksets and harnesses register their own options dynamically, include the taskset ID before `--help` to see the taskset's flags. diff --git a/docs/harnesses.md b/docs/harnesses.md index 17e5e385a2..54b1f418a1 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -1,6 +1,6 @@ # Harnesses -verifiers ships with a range of harnesses out of the box, including Claude Code, Codex, Pi, Prime-Agent, the tool-enabled `bash` harness, the CDP-driven `browser-use` harness, and the minimal `null` harness (which has no built-in shell tools of its own, though it can still call task MCP tools). You can inspect the settings for any harness with `uv run vf-eval my-task --env.agent.harness.id codex --help`. Most third-party CLI harnesses must run inside a container runtime, and individual harnesses vary in which optional features they support (such as MCP tools, follow-up messages, image input, and skills). +verifiers ships with a range of harnesses out of the box, including Claude Code, Codex, Pi, Prime-Agent, the tool-enabled `bash` harness, the CDP-driven `browser-use` harness, and the minimal `null` harness (which has no built-in shell tools of its own, though it can still call task MCP tools). Most third-party CLI harnesses must run inside a container runtime, and individual harnesses vary in which optional features they support (such as MCP tools, follow-up messages, image input, and skills). ## Bash harness diff --git a/docs/mint.json b/docs/mint.json index 478acee8d3..7897c0abc4 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -9,7 +9,6 @@ "architecture", "building-environments", "tasksets", - "evaluation", "runtimes", "debugging", "harnesses", diff --git a/docs/overview.md b/docs/overview.md index eee33d2ae1..939baca438 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -39,7 +39,6 @@ A trace records the message graph, rewards, metrics, errors, and one per-call re - [Building environments](building-environments.md) — Build a package and separate agent execution from grading - [Tasksets](tasksets.md) — How to create tasksets - [Harbor Tasksets](harbor.md) — How to create Harbor-based tasksets -- [Evaluation](evaluation.md) — How to evaluate tasksets - [Runtimes](runtimes.md) — Containers, files, resources, deployments, and network access - [Debugging](debugging.md) — Check setup, diagnose failures, and re-score saved runs - [Harnesses](harnesses.md) — How to build custom harnesses diff --git a/docs/tasksets.md b/docs/tasksets.md index 89d94bb45e..ee9d617220 100644 --- a/docs/tasksets.md +++ b/docs/tasksets.md @@ -70,7 +70,7 @@ Then install the package into the Python environment running the CLI and verify ```bash uv pip install -e environments/addition -uv run vf-eval addition --dry-run +uv run vf-validate addition -n 1 --only-setup ``` ## Data and configuration @@ -191,12 +191,12 @@ taskset.include(idx=["0:100"]).exclude(names=["broken"]).shuffle(seed=0).take(5) `include` keeps and `exclude` drops tasks matching any of `idx` (integers or non-negative Python slice strings like `"0:100"`, `"100:"`, or `"::2"`), `ids`, `keys`, or `names`; a task matches when any provided list names it. Any filter entry that matches nothing logs a warning once the stream has been consumed. -The eval, debug, validate, and GEPA entrypoints all accept a `select` config and apply it via `taskset.select(config.select)`, which chains the views in a fixed order: `include`, `exclude`, `shuffle`, `skip`, `limit`: +The debug, validate, and GEPA entrypoints all accept a `select` config and apply it via `taskset.select(config.select)`, which chains the views in a fixed order: `include`, `exclude`, `shuffle`, `skip`, `limit`: ```bash -uv run vf-eval gsm8k --select.include.idx 0:100 # the first 100 tasks -uv run vf-eval gsm8k -s -n 50 # 50 random tasks (--select.shuffle --select.limit 50) -uv run vf-eval gsm8k -s --select.skip 50 -n 50 # the next 50 tasks from the same shuffled order +uv run vf-validate gsm8k --select.include.idx 0:100 # the first 100 tasks +uv run vf-validate gsm8k -s -n 50 # 50 random tasks (--select.shuffle --select.limit 50) +uv run vf-validate gsm8k -s --select.skip 50 -n 50 # the next 50 tasks from the same shuffled order ``` ```toml @@ -211,7 +211,7 @@ Because `shuffle` (seeded by `select.seed`, which defaults to `0`) runs before ` ## Lazy and infinite tasksets -`load()` can be a generator that yields tasks one at a time instead of returning a list. Because consumers iterate the taskset lazily, `vf-eval -n 5` only builds five tasks (unless filtering skips items or `shuffle` needs to materialize the candidate pool), which pays off whenever constructing a task is expensive. +`load()` can be a generator that yields tasks one at a time instead of returning a list. Because consumers iterate the taskset lazily, `-n 5` only builds five tasks (unless filtering skips items or `shuffle` needs to materialize the candidate pool), which pays off whenever constructing a task is expensive. A procedural taskset can also yield tasks forever. Declare `INFINITE = True` on the class so consumers know the stream has no end — infinity is a property of the taskset, while how many tasks a run takes (`-n`) is up to the caller: @@ -236,7 +236,7 @@ Two rules follow from an infinite stream: 1. Every run must bound the taskset with `take` (`-n` / `--select.limit` on the CLI) or an `include` consisting only of closed `idx` ranges; omitting both raises an error. 2. `shuffle` needs a finite bound _before_ it runs, since you cannot shuffle an infinite stream. Because `select` applies `limit` after `shuffle`, shuffling an infinite taskset from the CLI requires a closed `include.idx` range first (for example, `--select.include.idx 0:1000 -s -n 50`); in Python, `taskset.take(1000).shuffle()` works as well. -The generator runs once on the client (the eval entrypoint or prime-rl orchestrator pulls tasks from `load()` and sends each task's `TaskData` to the environment worker), so workers never need to reproduce the generator sequence. Keep `load()` deterministic if you want `--resume` to regenerate the same first `n` tasks across runs (see `alphabet_sort`, `color_codeword`, or the built-in `textarena` taskset). +The generator runs once on the client (`vf-run` or the prime-rl orchestrator pulls tasks from `load()` and sends each task's `TaskData` to the environment worker), so workers never need to reproduce the generator sequence. Keep `load()` deterministic if you want `--resume` to regenerate the same first `n` tasks across runs (see `alphabet_sort`, `color_codeword`, or the built-in `textarena` taskset). ## Adding Tools diff --git a/environments/proposer_solver/proposer_solver/taskset.py b/environments/proposer_solver/proposer_solver/taskset.py index badc08072a..e103ab81a0 100644 --- a/environments/proposer_solver/proposer_solver/taskset.py +++ b/environments/proposer_solver/proposer_solver/taskset.py @@ -12,9 +12,8 @@ Seats are deliberately heterogeneous: point the proposer at a code-running harness in a real sandbox and keep the solvers on a cheap tool-less chat loop — - uv run vf-eval proposer-solver -n 4 \ - --env.proposer.harness.id codex --env.proposer.runtime.type prime \ - --env.solver.harness.id null + --env.proposer.harness.id codex --env.proposer.runtime.type prime \ + --env.solver.harness.id null Train-side, the seats flip independently per run (`--env.train_solver false` trains only on proposer rollouts; both default trainable, late-bound to the run's diff --git a/environments/scratchpad/scratchpad/taskset.py b/environments/scratchpad/scratchpad/taskset.py index a106f53725..d4062dbaf6 100644 --- a/environments/scratchpad/scratchpad/taskset.py +++ b/environments/scratchpad/scratchpad/taskset.py @@ -3,7 +3,7 @@ Each rollout is assigned a unique word and asked to round-trip it through the shared scratchpad server, then report what came back. The reward is 1 iff the model reports its own word. The server is taskset-scoped (one instance per environment worker) yet writable, so this -is a direct test of per-rollout isolation via `self.state`: `uv run vf-eval scratchpad -n 8 -r 1` +is a direct test of per-rollout isolation via `self.state`: a run of 8 tasks scores a mean reward of 1.0 because each rollout's write to `self.state` stays isolated even though every rollout handled by that worker shares its server process. """ diff --git a/pyproject.toml b/pyproject.toml index 67de5e4fb7..8ca0536ea8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -150,7 +150,6 @@ prime-pydantic-config = false renderers = false [project.scripts] -vf-eval = "verifiers.v1.cli.eval.main:main" vf-run = "verifiers.v1.cli.run:main" vf-validate = "verifiers.v1.cli.validate:main" vf-debug = "verifiers.v1.cli.debug:main" diff --git a/tests/unit/cli/test_replay.py b/tests/unit/cli/test_replay.py index 244c3dd1de..a7933f4965 100644 --- a/tests/unit/cli/test_replay.py +++ b/tests/unit/cli/test_replay.py @@ -10,7 +10,6 @@ import verifiers.v1 as vf from verifiers.v1.cli.output import CONFIG_DIR, RESOLVED_DIR, saved_config_path from verifiers.v1.cli.replay import run_replay -from verifiers.v1.configs.cli.eval import EvalConfig from verifiers.v1.configs.cli.replay import ReplayConfig from verifiers.v1.graph import MessageNode from verifiers.v1.types import AssistantMessage, UserMessage @@ -18,12 +17,12 @@ async def saved_run(run_dir: Path) -> vf.Trace: - """A one-trace reverse-text run on disk, as `run_eval` leaves it: the resolved - config under `configs/resolved/` and a scored trace in `traces.jsonl`.""" - config = EvalConfig(env={"taskset": {"id": "reverse-text"}}, rich=None) + """A one-trace reverse-text run on disk: the resolved config under + `configs/resolved/` and a scored trace in `traces.jsonl`.""" + config = {"env": {"taskset": {"id": "reverse-text"}}} resolved = run_dir / CONFIG_DIR / RESOLVED_DIR resolved.mkdir(parents=True) - (resolved / "eval.json").write_text(json.dumps(config.model_dump(mode="json"))) + (resolved / "eval.json").write_text(json.dumps(config)) data = ReverseTextData(idx=0, prompt="abc", answer="cba") task = ReverseTextTask(data) trace = vf.Trace( diff --git a/verifiers/v1/cli/dashboard/__init__.py b/verifiers/v1/cli/dashboard/__init__.py index 21e473e157..f08f021af3 100644 --- a/verifiers/v1/cli/dashboard/__init__.py +++ b/verifiers/v1/cli/dashboard/__init__.py @@ -1,4 +1,3 @@ -from verifiers.v1.cli.dashboard.eval import dashboard from verifiers.v1.cli.dashboard.validate import TaskProgress, validate_dashboard -__all__ = ["TaskProgress", "dashboard", "validate_dashboard"] +__all__ = ["TaskProgress", "validate_dashboard"] diff --git a/verifiers/v1/cli/dashboard/eval.py b/verifiers/v1/cli/dashboard/eval.py deleted file mode 100644 index 41f78bc77c..0000000000 --- a/verifiers/v1/cli/dashboard/eval.py +++ /dev/null @@ -1,882 +0,0 @@ -"""Live eval dashboard.""" - -import contextlib -import time -from collections import deque -from pathlib import Path - -from pydantic import BaseModel -from rich import get_console -from rich.console import Group -from rich.markup import escape -from rich.progress_bar import ProgressBar -from rich.rule import Rule -from rich.table import Table -from rich.text import Text - -from verifiers.v1.cli.dashboard.base import live_view -from verifiers.v1.cli.eval.hint import PRIME_RL_HINT -from verifiers.v1.cli.output import attempt_log_file, output_path -from verifiers.v1.configs.cli.eval import EvalConfig -from verifiers.v1.env import RunSlot -from verifiers.v1.trace import Trace -from verifiers.v1.types import Usage -from verifiers.v1.utils.format import ( - format_count, - format_mean, - format_override, - format_time, -) -from verifiers.v1.utils.interrupt import cleaning_up - -# For sizing pages to the terminal: detects the real terminal height/width each access (the live -# view writes to the same terminal). Reused so we don't rebuild it every refresh tick. -_CONSOLE = get_console() -_PAGE_SECONDS = 5.0 # rotate to the next page of rollouts this often when they overflow -# The under-bar breakdown pads its label column to the Overview's widest label, so the two -# `label value` grids (above and below the progress bar) line their values up. -_LABEL_WIDTH = len("timeouts") - -_STYLE = { - "pending": "dim", - "boot": "orange3", - "build": "dark_orange", - "setup": "yellow", - "running": "cyan", - "finalize": "magenta", - "scoring": "blue", - "success": "green", - "dropped": "grey62", - "error": "red", -} -_MARK_LABEL = { - "pending": "pending", - "boot": "boot", - "build": "build", - "setup": "setup", - "running": "rollout", - "finalize": "finalize", - "scoring": "scoring", - "success": "success", - "dropped": "dropped", - "error": "error", -} -_MARK_WIDTH = max(len(label) for label in _MARK_LABEL.values()) -# Each label padded to a common width and bracketed, so the `[ ]` line up in a column down the -# left edge (label left-aligned inside) — the current phase reads at a glance. `escape` keeps the -# brackets literal: Rich parses `[label]` in a cell as markup and would otherwise drop it. -_MARK = { - state: escape(f"[{label:<{_MARK_WIDTH}}]") for state, label in _MARK_LABEL.items() -} - - -def format_cost_usd(cost_usd: float) -> str: - if cost_usd >= 1: - return f"${cost_usd:.2f}" - return f"${cost_usd:.4f}" - - -def _seat_value(config: EvalConfig, read): - """A cap as the overview shows it: the declared seats' shared value, the - string 'per-seat' when they disagree (caps live on the seats).""" - from verifiers.v1.configs.agent import agent_config_fields - - values = {read(spec) for spec in agent_config_fields(config.env).values()} - return values.pop() if len(values) == 1 else "per-seat" - - -def _limits(config: EvalConfig) -> list[str]: - """Per-run caps for the overview (concurrency first, then turns, tokens), read - off the declared seats. An unset cap reads as 'no ...' rather than being hidden.""" - toks = [] - for label, field in ( - ("in", "max_input_tokens"), - ("out", "max_output_tokens"), - ("total", "max_total_tokens"), - ): - v = _seat_value(config, lambda spec, field=field: getattr(spec, field)) - if v == "per-seat": - toks.append(f"{label} per-seat") - elif v: - toks.append(f"{label}≤{v}") - turns = _seat_value(config, lambda spec: spec.max_turns) - return [ - f"≤{config.max_concurrent} episodes" - if config.max_concurrent - else "no concurrency cap", - "per-seat turn caps" - if turns == "per-seat" - else (f"{turns} turns" if turns else "no turn cap"), - f"{', '.join(toks)} tokens" if toks else "no token cap", - ] - - -def _timeouts(config: EvalConfig) -> list[str]: - """Per-stage run timeouts for the overview (each seat's own), plus the env's - finalize() bound (unset → 'no timeout').""" - rows = [] - for stage in ("setup", "rollout", "finalize", "scoring"): - v = _seat_value(config, lambda spec, stage=stage: getattr(spec.timeout, stage)) - if v == "per-seat": - rows.append(f"{stage} per-seat") - elif v: - rows.append(f"{stage} {v:g}s") - else: - rows.append(f"no {stage} timeout") - finalize = config.env.timeout.finalize - rows.append( - f"env finalize {finalize:g}s" if finalize else "no env finalize timeout" - ) - return rows - - -def _aligned(rows: list[list[str]]) -> list[str]: - """Join each row's `·`-separated segments, padding shared columns to a common width so the - separators line up across rows (each row's last segment is left ragged).""" - widths: dict[int, int] = {} - for row in rows: - for i, seg in enumerate(row): - widths[i] = max(widths.get(i, 0), len(seg)) - return [ - " · ".join( - seg.ljust(widths[i]) if i < len(row) - 1 else seg - for i, seg in enumerate(row) - ) - for row in rows - ] - - -def _interrupt_footer() -> Group | None: - """The graceful-shutdown notice under the rollouts once Ctrl-C has begun teardown — the - on-screen echo of the warning (console logging is silenced in rich mode); a further Ctrl-C - is ignored while it shows.""" - if not cleaning_up(): - return None - return Group( - Rule(style="dim"), - Text( - "interrupted — cleaning up, tearing down containers/sandboxes. " - "please wait; a further ctrl-c is ignored.", - style="yellow", - ), - ) - - -def _warning(config: EvalConfig) -> Text | None: - """A local-runtime caution when any code-running seat resolves to the subprocess - runtime (the tool-less chat loops are exempt), shown above the overview rather - than as a row in it.""" - from verifiers.v1.utils.loaders import harness_class - - if any( - getattr(config.env, role).runtime.type == "subprocess" - and harness_class(h.id).EXECUTES_CODE - for role, h in config.env.agent_harnesses().items() - ): - return Text( - "warning Runs on the local system; local files and settings may affect this " - "evaluation. Use subprocess only for debugging, or use docker or prime for an " - "isolated run.", - style="yellow", - ) - return None - - -def overrides( - config: BaseModel, - default: BaseModel | None = None, - skip: frozenset[str] = frozenset(), -) -> list[str]: - """`field=value` segments for the fields the *user* customized, sorted, diffed against each - field's declared default. Not `model_fields_set`: a `--resume` run reloads the full saved - JSON config, so `model_fields_set` would flag every field. `default` is the reference instance, - threaded through recursion so a pinned nested default (`taskset.task.tools`) reads as - unchanged. `skip` holds dotted paths (`runtime.type`).""" - segments: list[str] = [] - fields = type(config).model_fields - for field in sorted(fields): - if field in skip: - continue - value = getattr(config, field) - # `get_default` returns `PydanticUndefined` for a required field, so it always reads as set. - field_def = ( - fields[field].get_default(call_default_factory=True) - if default is None - else getattr(default, field, None) - ) - if isinstance(value, BaseModel): # nested config: flatten as `field.` - child_skip = frozenset( - s[len(field) + 1 :] for s in skip if s.startswith(f"{field}.") - ) - # A switched discriminator (e.g. subprocess→docker) is an override the per-field diff - # misses — within the new class `type` equals its own default — so surface it. - class_changed = type(value) is not type(field_def) - if ( - class_changed - and "type" not in child_skip - and (discriminator := getattr(value, "type", None)) is not None - ): - segments.append(f"{field}.type={format_override(discriminator)}") - # A switched class is a new shape, so diff against its own defaults, not the instance. - segments.extend( - f"{field}.{seg}" - for seg in overrides( - value, default=None if class_changed else field_def, skip=child_skip - ) - ) - elif value != field_def: - segments.append(f"{field}={format_override(value)}") - return segments - - -def Overview(config: EvalConfig) -> Table: - sampling = ", ".join( - f"{k}={v}" for k, v in config.sampling.model_dump(exclude_none=True).items() - ) - grid = Table.grid(padding=(0, 2)) - grid.add_column(style="dim") - grid.add_column() - seats = config.env.agent_harnesses() - taskset = config.env.taskset - env_label = taskset.name if taskset.id else "no taskset" - if config.env.id: - env_label = f"{config.env.id}+{env_label}" - # One seat story when every seat resolves the same way (the common case); one - # row per seat when they diverge (a judge on its own harness/runtime). - runtimes = {role: getattr(config.env, role).runtime.type for role in seats} - stories = list( - dict.fromkeys( - f"{h.name} harness · {runtimes[role]} runtime" - for role, h in seats.items() - ) - ) - if len(stories) == 1: - grid.add_row("env", f"{env_label} · {stories[0]}") - else: - grid.add_row("env", env_label) - for role, h in seats.items(): - grid.add_row(f" {role}", f"{h.name} harness · {runtimes[role]} runtime") - model = f"{config.model} ({sampling})" if sampling else config.model - grid.add_row("model", f"{model} via {config.client.base_url}") - # Non-default knobs the user set, one row each when non-empty. `escape` the cell: an override - # value (or our `[...]`/`{...}` delimiters) can carry Rich markup that would otherwise be - # parsed as styling and dropped. `id` is in the `env` row; the seat's `runtime.type` too - # (hidden here), but only for the seat — `taskset.task.tools.runtime.type` has no other display. - if taskset_over := overrides(taskset, skip=frozenset({"id"})): - grid.add_row("taskset", escape(" · ".join(taskset_over))) - for role, h in seats.items(): - if harness_over := overrides(h, skip=frozenset({"id"})): - label = f"{role}.harness" if len(seats) > 1 else "harness" - grid.add_row(label, escape(" · ".join(harness_over))) - runtime = getattr(config.env, role).runtime - if runtime_over := overrides(runtime, skip=frozenset({"type"})): - label = f"{role}.runtime" if len(seats) > 1 else "runtime" - grid.add_row(label, escape(" · ".join(runtime_over))) - limits, timeouts = _aligned([_limits(config), _timeouts(config)]) - grid.add_row("limits", limits) - grid.add_row("timeouts", timeouts) - grid.add_row("output", Text(str(output_path(config)), overflow="fold")) - return grid - - -def _prime_rl_footer() -> Group: - return Group(Text(""), Text(PRIME_RL_HINT, style="dim", overflow="fold")) - - -class TraceStats: - """Resource totals for a dashboard frame, retained once the episode is final.""" - - def __init__(self, trace: Trace) -> None: - self.num_input_tokens = trace.num_input_tokens - self.num_output_tokens = trace.num_output_tokens - self.num_turns = trace.num_turns - self.num_branches = trace.num_branches - self.usage = trace.usage - self.judge_usage = Usage.aggregate(trace.extra_usage) - - -def Progress( - slots: list[RunSlot], - start: float, - completed: dict[str, TraceStats], - page: tuple[int, int] | None = None, -) -> Group: - # On resume, `slots` includes the previous session's kept rollouts (as finished slots), so - # progress, reward, err, and the breakdown cover the whole run, not just this session's. - done = [s for s in slots if s.done] # fully scored episodes - done_traces = [t for s in done for t in s.traces] - # Score aggregates read the policy's traces: auxiliary roles (a judge's verdict - # run, a modeled user) are `trainable=False` and carry no rewards, so counting - # them dilutes every mean with structural zeros. An all-untrainable run (every - # role frozen) falls back to all traces rather than showing nothing. - scored = [t for t in done_traces if t.agent.trainable] or done_traces - total = len(slots) - # Headline reward = mean over non-errored, non-dropped traces; when any errored, - # `format_mean` appends the global avg (errored count as 0) in parens. `err` is the share - # of episodes that ended not-ok (a trace errored, or the env's rollout()/score() hook - # itself failed); `dropped` the share of policy traces left out of every mean. - reward = format_mean(scored, lambda t: t.reward) - err = ( - f"{sum(s.episode is None or not s.episode.ok for s in done) / len(done):.2f}" - if done - else "—" - ) - stats = ( - f"{len(done)}/{total} · {format_time(time.time() - start)} · " - f"reward {reward} · err {err}" - ) - if dropped := sum(t.is_dropped for t in scored): - stats += f" · dropped {dropped / len(scored):.2f}" - if page is not None: # overflowing — show which page, and that the arrows page - stats += f" (page {page[0]}/{page[1]} ◄ ►)" - row = Table.grid(expand=True, padding=(0, 1)) - row.add_column(ratio=1) # bar stretches to fill the width left of the stats - row.add_column(justify="right", no_wrap=True) - row.add_row( - ProgressBar(total=total or 1, completed=len(done)), - Text(stats), - ) - breakdown = _breakdown(scored, done_traces, completed) - return Group(row, breakdown) if breakdown is not None else Group(row) - - -def _score_segments(traces: list[Trace], source: str) -> str | None: - """`name mean` segments for every reward/metric key seen across `traces`, - first-seen order (a trace carries only its own task's signals — unscored ones - seeded as `None` — so keys can vary); None when no trace recorded anything.""" - names: list[str] = [] - for trace in traces: - names.extend(n for n in getattr(trace, source) if n not in names) - if not names: - return None - segments = [] - for name in names: - mean = format_mean(traces, lambda t, n=name, s=source: _score(t, s, n)) - segments.append(f"{name} {mean}") - return " · ".join(segments) - - -def _score(trace: Trace, source: str, name: str) -> float: - """Rewards carry raw score + weight; the breakdown shows the raw score. A missing - or unscored (seeded `None`) entry reads as 0.0.""" - if source == "rewards": - reward = trace.rewards.get(name) - return reward.score if reward is not None else 0.0 - value = trace.metrics.get(name) - return value if value is not None else 0.0 - - -def _breakdown( - scored: list[Trace], done: list[Trace], completed: dict[str, TraceStats] -) -> Table | None: - """Score rows read the policy view (`scored` — trainable traces); with several - roles in play they split per role, each role averaging over its OWN traces (no - dilution, so the split covers every role — an untrainable seat's received - rewards stay visible). Resource rows read every completed trace: a judge's - tokens were spent regardless of trainability.""" - grid = Table.grid(padding=(0, 2)) - grid.add_column(style="dim", min_width=_LABEL_WIDTH) - grid.add_column() - # rewards/metrics are error-corrected means — skip when every rollout errored (no clean mean to - # show); usage/time below still cover errored rollouts (their resources were spent regardless). - has_clean = any(not t.has_error for t in done) - score_rows = (("rewards", "rewards"), ("metrics", "metrics")) if has_clean else () - by_agent: dict[str, list[Trace]] = {} - for trace in done: - by_agent.setdefault(trace.agent.name, []).append(trace) - for label, source in score_rows: - if len(by_agent) > 1: - segments = [ - f"[dim]{name}:[/dim] {means}" - for name, traces in by_agent.items() - if (means := _score_segments(traces, source)) is not None - ] - if segments: - grid.add_row(label, " ".join(segments)) - elif (means := _score_segments(scored, source)) is not None: - grid.add_row(label, means) - - # Resource use over every completed rollout (errored ones still spent tokens/time): tokens and - # cost are summed; each timing phase is averaged over the rollouts that have it timed (averaged - # per phase rather than summed, since phases run concurrently across rollouts). - total_in = total_out = total_cached = total_reasoning = 0 - total_judge_in = total_judge_out = 0 - total_cost = total_judge_cost = 0.0 - have_cost = have_cached = have_reasoning = have_judge = False - phase_secs: dict[str, float] = {} - phase_count: dict[str, int] = {} - model_secs = harness_secs = 0.0 - for trace in done: - stats = completed[trace.id] - total_in += stats.num_input_tokens - total_out += stats.num_output_tokens - usage = stats.usage - if usage is not None: - if usage.cached_input_tokens is not None: - total_cached += usage.cached_input_tokens - have_cached = True - if usage.reasoning_tokens is not None: - total_reasoning += usage.reasoning_tokens - have_reasoning = True - if usage.cost is not None: - total_cost += usage.cost - have_cost = True - # Judge / auxiliary scoring calls (off the message graph) shown separately from the agent's. - judge = stats.judge_usage - if judge is not None: - total_judge_in += judge.input_tokens - total_judge_out += judge.completion_tokens - if judge.cost is not None: - total_judge_cost += judge.cost - have_judge = True - for phase in ("boot", "setup", "agent", "finalize", "scoring"): - span = getattr(trace.timing, phase) - if span.end: # phase was timed for this rollout - phase_secs[phase] = phase_secs.get(phase, 0.0) + span.duration - phase_count[phase] = phase_count.get(phase, 0) + 1 - model_secs += trace.timing.agent.model.duration - harness_secs += trace.timing.agent.harness.duration - if ( - total_in - or total_out - or have_cost - or have_cached - or have_reasoning - or have_judge - ): - tokens = f"{format_count(total_in)}/{format_count(total_out)} tokens" - details = [] - if have_cached: - details.append(f"{format_count(total_cached)} cached") - if have_reasoning: - details.append(f"{format_count(total_reasoning)} reasoning") - if details: - tokens += f" ({', '.join(details)})" - usage = [tokens] - if have_judge: - usage.append( - f"+ {format_count(total_judge_in)}/{format_count(total_judge_out)} judge" - ) - if have_cost: - cost = format_cost_usd(total_cost) - if total_judge_cost: - cost += f" + {format_cost_usd(total_judge_cost)} judge" - usage.append(cost) - grid.add_row("usage", " · ".join(usage)) - time_segments = [] - for phase in ("boot", "setup", "agent", "finalize", "scoring"): - count = phase_count.get(phase) - if not count: - continue - segment = f"{phase} {format_time(phase_secs[phase] / count)}" - if phase == "agent": - segment += ( - f" (model {format_time(model_secs / count)}" - f" + harness {format_time(harness_secs / count)})" - ) - time_segments.append(segment) - if time_segments: - grid.add_row("time", " · ".join(time_segments)) - return grid if grid.row_count else None - - -def _stage(trace: Trace) -> str: - """The stage a live (not-yet-done) rollout is in, derived from its trace's timing - spans — the engine opens and closes each span exactly at the stage transitions, so - the current stage is the latest span started but not yet ended. A completed trace - whose slot isn't done is waiting on its episode's other traces (and the env's - `score()`) — that's scoring.""" - if trace.is_completed: - return "scoring" - for stage, span in ( - ("scoring", trace.timing.scoring), - ("finalize", trace.timing.finalize), - ("running", trace.timing.agent), - ("setup", trace.timing.setup), - ("boot", trace.timing.boot), - ): - if span.start and not span.end: - break - else: - stage = "boot" # trace minted, first span not yet opened (an instant) - # A boot stuck on a first-use platform image build reads differently from a - # normal boot — it can sit there for ~10 minutes (prime runtime only). - if stage == "boot" and getattr(trace.agent.runtime, "image_cached", None) is False: - return "build" - return stage - - -def _started(slot: RunSlot) -> float: - # Sort key: when a rollout began (its first trace's boot start; setup for - # pre-boot-span traces on resume; the slot's own dispatch stamp on a served run). - # A still-pending rollout has neither, so it sorts last (+inf) — behind everything - # already in flight, in task order. - if not slot.traces: - return slot.started if slot.started is not None else float("inf") - return min(t.timing.boot.start or t.timing.setup.start for t in slot.traces) - - -def _groups(slots: list[RunSlot]) -> list[list[RunSlot]]: - # The n rollouts of each task, grouped together (so they sit adjacent); groups ordered by - # earliest start, rollouts within a group by start. Every slot carries its `task` from - # construction, so ones still queued behind the concurrency cap (no trace yet) are grouped - # and shown too — as `[pending]`. Finished ones stay (never removed). - by_task: dict[int, list[RunSlot]] = {} - for slot in slots: - by_task.setdefault(slot.task.data.idx, []).append(slot) - groups = list(by_task.values()) - for group in groups: - group.sort(key=_started) - groups.sort(key=lambda g: _started(g[0])) - return groups - - -def _brace(i: int, size: int) -> str: - if size == 1: - return " " - return "╭" if i == 0 else "╰" if i == size - 1 else "│" - - -def Rows( - groups: list[list[RunSlot]], - now: float, - runtime_type: str, - completed: dict[str, TraceStats], -) -> Table: - # (brace, state, left sections, result, time); a slot contributes one row per live - # trace (a multi-agent episode shows each role's trace), braced per task. - rows: list[tuple[str, str, list[str], str, str]] = [] - for group in groups: - group_rows: list[tuple[str, list[str], str, str]] = [] - for slot in group: - task = slot.task.data - base = f"name={task.name[:32]}" if task.name else f"idx={task.idx}" - if not slot.traces: - if slot.done: # the env's rollout() itself failed before any trace - error = ( - slot.episode.last_error if slot.episode is not None else None - ) - group_rows.append( - ( - "error", - [f"task {base}", *[""] * 7], - error.type if error is not None else "error", - "", - ) - ) - elif slot.started is not None: - # Dispatched to an env server: running, but with no live trace to - # read stages/turns from — detail lands when the episode completes. - elapsed = format_time(now - slot.started) - group_rows.append( - ("running", [f"task {base}", *[""] * 7], "", elapsed) - ) - else: # queued behind the concurrency cap — only its task is known yet - group_rows.append(("pending", [f"task {base}", *[""] * 7], "", "")) - continue - for t in slot.traces: - label = f"{base} agent={t.agent.name}" - if slot.done: # fully scored — reward is final - state = ( - "error" - if t.has_error - else "dropped" - if t.is_dropped - else "success" - ) - # A trace that recorded nothing shows no reward: a judge or - # modeled-user seat's `reward=0.00` would read as a score, and - # a dropped trace's reward counts nowhere, so it shows its reason. - if t.has_error: - result = t.last_error.type if t.last_error else "" - elif t.is_dropped: - result = t.dropped - else: - result = f"reward={t.reward:.2f}" if t.rewards else "" - if t.has_error: - stop = "" # error shown instead - else: - stop = t.stop_condition or "" - if ( - t.is_truncated - ): # flag a clipped rollout next to its stop condition - stop = f"{stop} (truncated)".strip() - elif t.is_completed and (err := t.last_error) is not None: - # An errored trace whose episode is still running its other - # traces (or `score()`) is already a failure — show it, don't - # let it sit as "scoring" until the whole episode lands. - state, result, stop = "error", err.type, "" - else: - state, result, stop = _stage(t), "", "" - # The trace's own stamp, not the run-level runtime: a role's harness - # may resolve elsewhere (the judge env's sandboxed judge on a - # subprocess run). - if t.agent.runtime is not None: - rt = t.agent.runtime - runtime = f"{rt.type}({rt.id})" if rt.id else rt.type - else: - runtime = runtime_type - stats = completed[t.id] if slot.done else TraceStats(t) - turns = stats.num_turns - start = t.timing.boot.start or t.timing.setup.start - end = ( - t.timing.scoring.end - or t.timing.finalize.end - or t.timing.agent.end - # a rollout that errored in boot/setup has only that span's end — freeze there - # once done, else (still running) the timer would grow off `now` forever - or ( - (t.timing.setup.end or t.timing.boot.end) - if t.is_completed - else 0 - ) - or now - ) - prompt, completion = stats.num_input_tokens, stats.num_output_tokens - nbranches = stats.num_branches - usage = stats.usage - cached = usage.cached_input_tokens if usage else None - reasoning = usage.reasoning_tokens if usage else None - cost = usage.cost if usage else None - tokens = "" - if prompt or completion: - tokens = f"{format_count(prompt)}/{format_count(completion)} tokens" - details = [] - if cached is not None: - details.append(f"{format_count(cached)} cached") - if reasoning is not None: - details.append(f"{format_count(reasoning)} reasoning") - if details: - tokens += f" ({', '.join(details)})" - left = [ - f"task {label}", - t.id[:8], - runtime, - f"{turns} turn{'s' * (turns != 1)}", - f"{nbranches} branch{'es' * (nbranches != 1)}", - tokens, - f"{format_cost_usd(cost)}" if cost is not None else "", - stop, # stop condition (agent_completed / max_turns / error), once done - ] - # No start time yet (queued, not generating) → blank, not `now - 0` (~56 years). - elapsed = format_time(end - start) if start else "" - group_rows.append((state, left, result, elapsed)) - for i, (state, left, result, elapsed) in enumerate(group_rows): - rows.append((_brace(i, len(group_rows)), state, left, result, elapsed)) - grid = Table.grid(expand=True, padding=(0, 1)) - grid.add_column(ratio=1, no_wrap=True) # brace + mark + dot-separated sections - grid.add_column(justify="right", no_wrap=True) # result (reward / error) - grid.add_column(justify="right", no_wrap=True) # time - if not rows: - return grid - # Pad each left section to its max width across rows (drop all-empty ones) so they align, - # then join with " · ". Text sections left-justified, numeric right. - pad = ( - str.ljust, - str.ljust, - str.ljust, - str.rjust, - str.rjust, - str.ljust, - str.rjust, - str.ljust, - ) - widths = [max(len(left[i]) for _, _, left, _, _ in rows) for i in range(8)] - for brace, state, left, result, elapsed in rows: - sections = [pad[i](left[i], widths[i]) for i in range(8) if widths[i]] - grid.add_row( - f"{brace} {_MARK[state]} " + " · ".join(sections), - f"{result} ·" if result else "", # trailing dot only when there's a result - elapsed, - style=_STYLE[state], - ) - return grid - - -class Pager: - def __init__(self) -> None: - self.page = 0 - self.manual = False - self.count = 1 - self.origin: float | None = None - - def on_key(self, key: str) -> None: - if key in ("left", "right") and self.count > 1: - self.manual = True - self.page = (self.page + (1 if key == "right" else -1)) % self.count - - def index(self, now: float) -> int: - # Track the auto page while it drives, so the first arrow continues from what's on screen - # rather than jumping back to page 1. Clamp in manual mode (count can shrink on resize). - if not self.manual: - # Anchor the timer to the first paged frame, so paging opens on page 1 then rotates. - if self.origin is None: - self.origin = now - self.page = int((now - self.origin) / _PAGE_SECONDS) % self.count - else: - self.page = max(0, min(self.page, self.count - 1)) - return self.page - - -_TAIL_BYTES = 256 * 1024 -"""How far back into a pre-existing log file the tail's first read reaches.""" - - -class LogTail: - """Incremental tail of the run's log file for `--rich.show-logs`: read only what - was appended since the last frame, keep a bounded window of whole lines.""" - - def __init__(self, path: Path, max_lines: int = 1000) -> None: - self.path = path - self.lines: deque[str] = deque(maxlen=max_lines) - self._pos: int | None = None - self._partial = "" - - def _poll(self) -> None: - try: - size = self.path.stat().st_size - except OSError: # not written yet — the first record creates it - return - if ( - self._pos is not None and size < self._pos - ): # truncated/replaced — start over - self._pos, self._partial = None, "" - if size == self._pos: - return - # The first read starts at most _TAIL_BYTES from the end: a pre-existing file - # (a reused run dir) must not be read whole on the render path. - first = self._pos is None - start = max(0, size - _TAIL_BYTES) if first else self._pos - with self.path.open("rb") as file: - file.seek(start) - data = file.read() - self._pos = file.tell() - chunk = data.decode("utf-8", errors="replace") - *complete, self._partial = (self._partial + chunk).split("\n") - if first and start: # landed mid-line — drop the cut first line - complete = complete[1:] - self.lines.extend(complete) - - def view(self, height: int) -> Group: - """The newest lines that fit, one Text per line (no markup parsing — log - content stays literal), each cropped to the terminal width.""" - self._poll() - window = list(self.lines)[-height:] - return Group( - *(Text(line, no_wrap=True, overflow="ellipsis") for line in window) - ) - - -def _rows_of(group: list[RunSlot]) -> int: - """How many display rows a task's slots take: one per trace, one for a slot with - none yet (pending) or none at all (the env's hook failed before any trace).""" - return sum(max(1, len(slot.traces)) for slot in group) - - -def _paginate( - groups: list[list[RunSlot]], rows_per_page: int, pager: Pager, now: float -) -> tuple[list[list[RunSlot]], int, int]: - """Pack groups (a task's rollouts kept together) into pages of at most `rows_per_page` rows, - selecting the one `pager` points at. Returns (this page's groups, 0-based index, page count) — - a single page when everything already fits.""" - if sum(_rows_of(g) for g in groups) <= rows_per_page: - # Everything fits: arrows stay inert, and clear the anchor so paging re-opens on page 1 if - # rollouts overflow again later (e.g. a terminal resize) rather than off the stale origin. - pager.count = 1 - pager.origin = None - return groups, 0, 1 - pages: list[list[list[RunSlot]]] = [] - current: list[list[RunSlot]] = [] - used = 0 - for group in groups: - if current and used + _rows_of(group) > rows_per_page: - pages.append(current) - current, used = [], 0 - current.append(group) - used += _rows_of(group) - if current: - pages.append(current) - pager.count = len(pages) - index = pager.index(now) - return pages[index], index, len(pages) - - -def _render( - slots: list[RunSlot], - start: float, - pager: Pager, - header: Group | Table, - runtime_type: str, - completed: dict[str, TraceStats], - tail: LogTail | None = None, -) -> Group: - now = time.time() - # A trace can finish before other agents and env scoring update the episode. - # Only done slots are final; resumed episodes enter the view already done. - completed.update( - (trace.id, TraceStats(trace)) - for slot in slots - if slot.done - for trace in slot.traces - if trace.id not in completed - ) - # The Ctrl-C cleanup notice and the prime-rl hint appear under the rollouts. Measure - # the fixed top (header + progress + rule) and the footer so the rollout rows fill what's left; - # page through them (timer / arrows) when they'd overflow (else rich truncates). - footers = [f for f in (_interrupt_footer(), _prime_rl_footer()) if f is not None] - footer = Group(*footers) - progress = Progress(slots, start, completed) - top = Group(header, progress, Rule(style="dim")) - reserved = len(_CONSOLE.render_lines(top)) + len(_CONSOLE.render_lines(footer)) - rows_per_page = max(1, _CONSOLE.size.height - reserved - 1) - if tail is not None: # --show-logs: the run's log stream in place of rollout rows - parts = [ - header, - progress, - Rule(style="dim"), - tail.view(rows_per_page), - footer, - ] - return Group(*parts) - page_groups, index, count = _paginate(_groups(slots), rows_per_page, pager, now) - if count > 1: - progress = Progress(slots, start, completed, page=(index + 1, count)) - parts = [ - header, - progress, - Rule(style="dim"), - Rows(page_groups, now, runtime_type, completed), - footer, - ] - return Group(*parts) - - -@contextlib.asynccontextmanager -async def dashboard( - slots: list[RunSlot], - config: EvalConfig, - start: float, -): - pager = Pager() - completed: dict[str, TraceStats] = {} - warning = _warning(config) - header = Group(warning, Text(""), Overview(config)) if warning else Overview(config) - runtime_type = ( - "/".join( - dict.fromkeys( - getattr(config.env, role).runtime.type - for role in config.env.agent_harnesses() - ) - ) - or "subprocess" - ) - tail = ( - LogTail(attempt_log_file(output_path(config))) - if config.rich is not None and config.rich.show_logs - else None - ) - async with live_view( - lambda: _render(slots, start, pager, header, runtime_type, completed, tail), - on_key=None if tail is not None else pager.on_key, - ): - yield diff --git a/verifiers/v1/cli/eval/__init__.py b/verifiers/v1/cli/eval/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/verifiers/v1/cli/eval/hint.py b/verifiers/v1/cli/eval/hint.py deleted file mode 100644 index 4523b19511..0000000000 --- a/verifiers/v1/cli/eval/hint.py +++ /dev/null @@ -1,5 +0,0 @@ -PRIME_RL_HINT = ( - "vf-eval will be deprecated - move to prime-rl's `uv run eval`: it streams traces " - "live into a dashboard, evaluates several envs in one run and scales through env " - "servers. https://github.com/PrimeIntellect-ai/prime-rl" -) diff --git a/verifiers/v1/cli/eval/main.py b/verifiers/v1/cli/eval/main.py deleted file mode 100644 index e66101a8bb..0000000000 --- a/verifiers/v1/cli/eval/main.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Eval CLI entrypoint.""" - -import asyncio -import json -import logging -import shutil -import sys - -from pydantic_config import cli - -from verifiers.v1.cli.eval.runner import run_eval -from verifiers.v1.cli.output import ( - create_attempt_log_dir, - output_path, - saved_config_path, - write_config, -) -from verifiers.v1.cli.resolve import ( - extract_id, - narrow_config, - plugin_errors, - references_config_file, - with_positional_taskset, -) -from verifiers.v1.configs.cli.eval import EvalConfig -from verifiers.v1.utils.interrupt import install_interrupt -from verifiers.v1.utils.logging import setup_logging -from verifiers.v1.utils.trace_store import TRACES_FILE - -logger = logging.getLogger(__name__) - -USAGE = ( - "usage: uv run vf-eval [] [--env.id ] [options] [@ file.toml]\n" - " uv run vf-eval @ /configs/resolved/eval.json --resume (re-run the run's missing/errored rollouts)" -) - - -def main(argv: list[str] | None = None) -> None: - argv = with_positional_taskset(list(sys.argv[1:]) if argv is None else list(argv)) - - if not argv or any(arg in ("-h", "--help") for arg in argv): - print(USAGE) - sys.argv = [sys.argv[0], "--help"] - with plugin_errors(): - cli( - narrow_config(EvalConfig, argv) - ) # full option help, narrowed to the given ids - return - # An env-block flag skips the usage gate so the typed parse renders its - # did-you-mean instead of a bare usage line. - typed_axis = any(a.startswith("--env.") for a in argv) - if ( - not extract_id(argv, "env.taskset") - and not references_config_file(argv) - and not typed_axis - ): - raise SystemExit( - USAGE - ) # need a taskset (positional / --env.taskset.id) or a @ file.toml - - with plugin_errors(): - config_type = narrow_config(EvalConfig, argv) - sys.argv = [ - sys.argv[0], - *argv, - ] # let prime-pydantic-config render help/errors - config = cli(config_type) - # A named run directory is re-entered only by `--resume` or wiped by `--clean`: any - # other write into it — the dry-run config included, which would clobber the - # config a resume typically re-runs — would overwrite the previous run. - run_path = output_path(config) - if config.clean and not config.resume and run_path.exists(): - output_dir = config.output_dir.resolve() - resolved_run_path = run_path.resolve() - if resolved_run_path == output_dir or not resolved_run_path.is_relative_to( - output_dir - ): - raise SystemExit("--clean requires run.dir to name a child of output_dir") - shutil.rmtree(run_path) - traces_file = run_path / TRACES_FILE - if not config.resume and traces_file.exists() and traces_file.stat().st_size > 0: - raise SystemExit( - f"run directory {run_path} already contains results - append --resume to " - "re-run its missing/errored rollouts, overwrite it with --clean, or pick " - "another --run.name" - ) - if config.resume: - # A resumed eval is only trustworthy under the exact config that produced the - # existing rollouts — anything else silently mixes incomparable results. Saved - # configs are full JSON dumps (nulls included), so the comparison is exact. - saved_path = saved_config_path(run_path) - if saved_path is None: - raise SystemExit( - f"--resume: no saved config under {run_path} - not a run dir" - ) - with plugin_errors(): - saved = config_type.model_validate_json(saved_path.read_text()) - saved_dump = saved.model_dump(mode="json") - current_dump = config.model_dump(mode="json") - saved_json = json.dumps(saved_dump, sort_keys=True, separators=(",", ":")) - current_json = json.dumps(current_dump, sort_keys=True, separators=(",", ":")) - if saved_json != current_json: - changed = sorted( - key - for key in set(saved_dump) | set(current_dump) - if saved_dump.get(key) != current_dump.get(key) - ) - raise SystemExit( - f"--resume requires the exact config the run was started with - it " - f"differs in [{', '.join(changed)}]. Resumed rollouts would not be " - f"comparable; re-run with `uv run vf-eval @ {saved_path} --resume`, or " - "start a fresh run" - ) - if config.dry_run: # resolved + validated; write it to the output dir and exit - setup_logging("DEBUG" if config.verbose else "INFO") - logger.info("wrote config to %s", write_config(config, run_path)) - return - # Always tee this attempt's logs to `logs/attempt_/eval.log` (`logs/latest` - # points there); `--rich.show-logs` tails it live. - log_file = str(create_attempt_log_dir(run_path) / "eval.log") - level = "DEBUG" if config.verbose else "INFO" - setup_logging(level, log_file=log_file, console=config.rich is None) - # First Ctrl-C / SIGTERM warns and raises KeyboardInterrupt so a killed/timed-out eval still - # runs each rollout's `finally` (tears down containers/sandboxes) and any worker pool it - # spawned; further signals during that cleanup are swallowed so an impatient second Ctrl-C - # can't orphan those resources. - install_interrupt() - - try: - episodes = asyncio.run(run_eval(config)) - except KeyboardInterrupt: - # Graceful cleanup has already run (each rollout's `finally`); partial results are on - # disk. Exit on the conventional Ctrl-C code without a traceback. - raise SystemExit(130) - if ( - config.rich is None - ): # --rich is the whole output; otherwise dump each trace as JSON - for episode in episodes: - for trace in episode.traces: - print(trace.model_dump_json(indent=2, exclude_none=True)) diff --git a/verifiers/v1/cli/eval/resume.py b/verifiers/v1/cli/eval/resume.py deleted file mode 100644 index 940ece9ba5..0000000000 --- a/verifiers/v1/cli/eval/resume.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Resume an interrupted eval: reload its finished rollouts and run only the rest. - -`load` validates saved episodes, applies the environment's completion verdict, and -uses the shared rollout planner to retain only those matching the current tasks. -""" - -from collections.abc import Callable -from pathlib import Path - -from pydantic_core import from_json - -from verifiers.v1.episode import WireEpisode -from verifiers.v1.task import TaskT -from verifiers.v1.utils.eval import plan_rollouts -from verifiers.v1.utils.trace_store import TRACES_FILE - - -def load( - resume_dir: Path, - tasks: list[TaskT], - num_rollouts: int, - complete: Callable[[WireEpisode], bool] | None = None, -) -> list[tuple[TaskT, list[WireEpisode], int]]: - """Plan the remaining rollouts and atomically rewrite the stream to kept rows. - - ``complete`` is the keep-verdict (default ``episode.ok``). Torn or malformed - rows are discarded; retained rows preserve their original serialized fields. - """ - path = resume_dir / TRACES_FILE - verdict = complete if complete is not None else (lambda episode: episode.ok) - episodes: list[WireEpisode] = [] - lines: dict[str, bytes] = {} - if path.exists(): - with path.open("rb") as results: - for line in results: - if not line.strip(): - continue - try: - row = from_json(line) - if "traces" not in row: - continue - episode = WireEpisode.model_validate(row) - if not verdict(episode): - continue - # A malformed row from any task/episode plugin is owed again. - except Exception: # noqa: BLE001, S112 - continue - episodes.append(episode) - lines.setdefault( - episode.id, line if line.endswith(b"\n") else line + b"\n" - ) - plan = plan_rollouts(tasks, num_rollouts, episodes) - tmp = path.with_suffix(".jsonl.tmp") - tmp.write_bytes( - b"".join(lines[episode.id] for _, kept, _ in plan for episode in kept) - ) - tmp.replace(path) - return plan diff --git a/verifiers/v1/cli/eval/runner.py b/verifiers/v1/cli/eval/runner.py deleted file mode 100644 index 2eeeeb91cc..0000000000 --- a/verifiers/v1/cli/eval/runner.py +++ /dev/null @@ -1,139 +0,0 @@ -"""The eval runner: fan episodes out with bounded concurrency. - -Rollouts run in-process: the env comes up once in this process and each slot becomes -one episode through `env.run_slot`. The dashboard watches the same `RunSlot`s the env -fills with live traces. Serving an env to many consumers is prime-rl's job (its `eval` -runs env servers); this CLI is the quick local path. -""" - -import asyncio -import contextlib -import logging -import os -import time -from collections.abc import AsyncIterator, Awaitable, Callable, Iterable -from typing import TypeVar, cast - -from verifiers.v1.cli.dashboard import dashboard -from verifiers.v1.cli.eval import resume -from verifiers.v1.cli.eval.hint import PRIME_RL_HINT -from verifiers.v1.cli.output import output_path, save_config -from verifiers.v1.clients import ModelContext -from verifiers.v1.configs.cli.eval import EvalConfig -from verifiers.v1.env import Env, RunSlot -from verifiers.v1.episode import Episode -from verifiers.v1.trace import EvalRunInfo -from verifiers.v1.utils.eval import plan_rollouts -from verifiers.v1.utils.trace_store import append_episode - -logger = logging.getLogger(__name__) - -T = TypeVar("T") - - -async def gather_rollouts(rollouts: Iterable[Awaitable[T]]) -> list[T]: - """`asyncio.gather`, but one rollout failing cancels the rest and waits for them - to unwind, so no rollout outlives the eval it belongs to. - Not a `TaskGroup`: that wraps errors in an `ExceptionGroup`, and `main` would no - longer see a `KeyboardInterrupt` as Ctrl-C.""" - tasks = [asyncio.ensure_future(rollout) for rollout in rollouts] - try: - return await asyncio.gather(*tasks) - except BaseException: - for task in tasks: - task.cancel() - # return_exceptions: wait for every task, not just the first to cancel. - await asyncio.gather(*tasks, return_exceptions=True) - raise - - -RunSlotFn = Callable[[RunSlot], Awaitable[Episode]] -OnComplete = Callable[[Episode], Awaitable[None]] - - -@contextlib.asynccontextmanager -async def _in_process( - env: Env, - config: EvalConfig, - semaphore: asyncio.Semaphore | None, - on_complete: OnComplete, -) -> AsyncIterator[RunSlotFn]: - """Run slots in-process: serving resources (shared tool servers, interception) - come up once for the run; the env's agents borrow them.""" - ctx = ModelContext( - client=config.client, model=config.model, sampling=config.sampling - ) - - async def run(slot: RunSlot) -> Episode: - return await env.run_slot(slot, ctx, semaphore, on_complete) - - async with env.serving(): - yield run - - -async def run_eval(config: EvalConfig) -> list[Episode]: - from verifiers.v1.utils.loaders import load_environment - - env = load_environment(config.env) - taskset = env.taskset.select(config.select) - if not taskset.bounded: - raise ValueError( - f"{type(taskset).__name__} is infinite - bound the run with -n" - ) - tasks = list(taskset) - out = output_path(config) - if config.resume: - # the env's own keep-verdict decides what resumes - plan = resume.load( - out, - tasks, - config.num_rollouts, - lambda episode: env.complete(cast(Episode, episode)), - ) - if not any(n for _, _, n in plan): - print( - f"nothing to resume in {out}: all {len(tasks)}x{config.num_rollouts} " - "rollouts already completed without error" - ) - raise SystemExit(0) - logger.info( - "resuming %s: %d task(s), %d rollout(s) owed", - out, - sum(n > 0 for _, _, n in plan), - sum(n for _, _, n in plan), - ) - else: - plan = plan_rollouts(tasks, config.num_rollouts) - save_config(config, out) - logger.info( - "running %dx%d rollouts on %s", len(plan), config.num_rollouts, config.model - ) - logger.info(PRIME_RL_HINT) - finished = [cast(Episode, episode) for _, kept, _ in plan for episode in kept] - start = time.time() - logger.info("results: %s", out) - - semaphore = ( - asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None - ) - write_lock = asyncio.Lock() - - # The run identity: every process this run spawns inherits it. - os.environ.setdefault("VF_RUN_ID", config.run.id) - - async def on_complete(episode: Episode) -> None: - episode.record_run(EvalRunInfo(id=config.run.id, name=config.run.name)) - await append_episode(out, episode, write_lock) - - async with _in_process(env, config, semaphore, on_complete) as run_slot: - # the env's own slots: it fills their live traces as the rollouts run - planned = [slot for task, _, n in plan if n for slot in env.slots(task, n)] - slots = [RunSlot.finished(episode) for episode in finished] + planned - display = ( - dashboard(slots, config, start) - if config.rich is not None - else contextlib.nullcontext() - ) - async with display: - results = await gather_rollouts(run_slot(slot) for slot in planned) - return finished + list(results) diff --git a/verifiers/v1/cli/output.py b/verifiers/v1/cli/output.py index 0df052578f..b4c38f82cc 100644 --- a/verifiers/v1/cli/output.py +++ b/verifiers/v1/cli/output.py @@ -1,7 +1,6 @@ -"""CLI output directories, saved launch/config files, and per-attempt log paths.""" +"""CLI output directories and saved launch/config files.""" import json -import os from pathlib import Path from typing import Protocol @@ -20,40 +19,6 @@ """Subdirectory of `configs/` holding the resolved per-component JSON dumps.""" -def create_attempt_log_dir(run_dir: Path) -> Path: - """Create `logs/attempt_` for this launch attempt and atomically repoint the - relative `logs/latest` symlink at it (prime-rl's log layout). Every launch — - fresh or `--resume` — gets its own numbered log directory, so a resume never - appends to an earlier attempt's logs.""" - logs_dir = run_dir / "logs" - logs_dir.mkdir(parents=True, exist_ok=True) - attempts = ( - int(p.name.removeprefix("attempt_")) - for p in logs_dir.glob("attempt_*") - if p.name.removeprefix("attempt_").isdigit() - ) - attempt_dir = logs_dir / f"attempt_{1 + max(attempts, default=0)}" - attempt_dir.mkdir() - # Atomically repoint the relative `latest` symlink: create a temp link, then rename. - tmp_link = logs_dir / f".{attempt_dir.name}" - if tmp_link.is_symlink() or tmp_link.exists(): - tmp_link.unlink() - os.symlink(attempt_dir.name, tmp_link) - os.replace(tmp_link, logs_dir / "latest") - return attempt_dir - - -def attempt_log_file(run_dir: Path) -> Path: - """The current attempt's `eval.log`. The CLI creates the attempt dir once at - startup; everyone after it — the runner's worker spawn, the dashboard's log - tail — resolves through `logs/latest`, so the whole invocation shares one file. - A direct `run_eval` call (no CLI) creates the first attempt itself.""" - latest = run_dir / "logs" / "latest" - if not latest.exists(): - create_attempt_log_dir(run_dir) - return latest / "eval.log" - - def saved_config_path(run_dir: Path) -> Path | None: """The run's saved resolved config (`configs/resolved/.json`), None if absent.""" config_dir = run_dir / CONFIG_DIR / RESOLVED_DIR diff --git a/verifiers/v1/configs/cli/eval.py b/verifiers/v1/configs/cli/eval.py deleted file mode 100644 index 8dbd53c2e7..0000000000 --- a/verifiers/v1/configs/cli/eval.py +++ /dev/null @@ -1,87 +0,0 @@ -"""The `EvalConfig`: the single config object the eval CLI parses.""" - -from pathlib import Path - -from pydantic import AliasChoices, Field, SerializeAsAny, model_validator -from pydantic_config import BaseConfig - -from verifiers.v1.configs.cli.env import narrowed_env_annotation, resolve_env_field -from verifiers.v1.configs.cli.run import RunConfig, default_run_name -from verifiers.v1.configs.client import ClientConfig, EvalClientConfig -from verifiers.v1.configs.env import EnvConfig, SingleAgentEnvConfig -from verifiers.v1.configs.sampling import SamplingConfig -from verifiers.v1.configs.select import SelectCLIConfig - - -class RichConfig(BaseConfig): - """The live dashboard.""" - - show_logs: bool = False - """Replace the dashboard's per-rollout rows with a live tail of the attempt's log - file (`logs/latest/eval.log`) — the env's own log lines included.""" - - -class EvalConfig(BaseConfig): - env: SerializeAsAny[EnvConfig] = SingleAgentEnvConfig() - """The environment — which env, its seed taskset, each agent, its knobs. Narrowed to - the selected env's config class by the env id, else the taskset id.""" - run: RunConfig = Field(default_factory=RunConfig) - """Run identity: `run.name` is the display name, `run.dir` names the directory - under `output_dir`, and `run.id` is stamped on traces.""" - model: str = Field( - "deepseek/deepseek-v4-flash", validation_alias=AliasChoices("model", "m") - ) - """Model id.""" - client: ClientConfig = EvalClientConfig() - sampling: SamplingConfig = SamplingConfig() - select: SelectCLIConfig = SelectCLIConfig() - """Which of the taskset's tasks to evaluate, under `--select.*` (`-n` sets - `select.limit`, `-s` sets `select.shuffle`).""" - num_rollouts: int = Field( - 1, - ge=1, - validation_alias=AliasChoices( - "group_size", "rollouts_per_example", "num_rollouts", "r" - ), - ) - """Independent episodes per task — the trainer's group size.""" - max_concurrent: int | None = Field( - 128, ge=1, validation_alias=AliasChoices("max_concurrent", "c") - ) - """Episodes in flight at once, `None` for no limit. An episode plays its agents one - at a time, so this is the live agent runs too — until `--env.max-concurrent-agents` - says otherwise.""" - verbose: bool = Field(False, validation_alias=AliasChoices("verbose", "v")) - """Log at debug level instead of the default info.""" - dry_run: bool = Field(False, exclude=True) - """Resolve + validate the config and dump it, then exit. Excluded from the saved - config so re-running `@ configs/eval.json` (or resuming/replaying the dir) actually runs.""" - clean: bool = Field(False, exclude=True) - """Delete the run directory (`output_dir / run.dir`) before running, overwriting a - previous run's results. Excluded from the saved config.""" - rich: RichConfig | None = Field(default_factory=RichConfig) - """The live dashboard (on by default; `--no-rich` streams logs to the console - instead); `--rich.show-logs` swaps the rollout rows for the run's logs.""" - output_dir: Path = Field( - Path("outputs"), validation_alias=AliasChoices("output_dir", "o") - ) - """Directory that groups related runs. The run itself (`configs/eval.json` + - `traces.jsonl`) writes to `output_dir / run.dir`.""" - resume: bool = Field(False, exclude=True) - """Re-run the run's missing/errored rollouts in place instead of starting fresh. The - run dir comes from the resolved config (`output_dir / run.dir`), so resume with the - run's own config — e.g. `uv run vf-eval @ /configs/eval.json --resume`. Excluded - from the saved config.""" - - @model_validator(mode="before") - @classmethod - def _resolve_env(cls, data): - return resolve_env_field(data, narrowed_env_annotation(cls)) - - @model_validator(mode="after") - def auto_setup_run_name(self): - if self.run.name is None: - self.run.name = default_run_name(self.env, self.model) - if self.run.dir is None: - self.run.dir = self.run.name - return self diff --git a/verifiers/v1/configs/client.py b/verifiers/v1/configs/client.py index 0ef124bac0..231370dcd5 100644 --- a/verifiers/v1/configs/client.py +++ b/verifiers/v1/configs/client.py @@ -4,9 +4,8 @@ + extra headers); `clients.resolve_client` turns one into a live `Client` — the interception server builds one per distinct config and shares it across the rollouts it multiplexes. The default Prime endpoint, API key, and team fall back to -the active Prime CLI config, so direct `uv run vf-eval` calls behave like `prime eval`. -Both the eval entrypoint (its model client) and in-env LLM calls (e.g. a judge reward) -build clients from these. `ClientConfig` is the CLI-selectable discriminated union +the active Prime CLI config. Both an eval's model client and in-env LLM calls (e.g. a +judge reward) build clients from these. `ClientConfig` is the CLI-selectable discriminated union (eval | train). """ diff --git a/verifiers/v1/gepa/adapter.py b/verifiers/v1/gepa/adapter.py index 66f48b572a..9f8e555492 100644 --- a/verifiers/v1/gepa/adapter.py +++ b/verifiers/v1/gepa/adapter.py @@ -40,7 +40,7 @@ class GEPAAdapter: semaphore: asyncio.Semaphore | None = None on_complete: Callable[[Episode], Awaitable[None]] | None = None """Called with each rollout's episode as it finalizes — the runner's persist hook that - streams episodes to `traces.jsonl`, exactly as `run_eval` does.""" + streams episodes to `traces.jsonl`.""" reflection_columns: list[str] = field(default_factory=list) propose_new_texts: Callable[..., Candidate] | None = None """Part of GEPA's adapter protocol — its proposer reads this attribute on every reflection diff --git a/verifiers/v1/gepa/config.py b/verifiers/v1/gepa/config.py index 49912bc9ee..b65186237d 100644 --- a/verifiers/v1/gepa/config.py +++ b/verifiers/v1/gepa/config.py @@ -2,7 +2,7 @@ GEPA optimizes one taskset's `Task.system_prompt` by alternating rollouts (`evaluate`) with a teacher LM reflecting on the reflective dataset (`make_reflective_dataset`) — see -`verifiers.v1.gepa.adapter.GEPAAdapter`. Like `EvalConfig`, it owns an `env` field (the +`verifiers.v1.gepa.adapter.GEPAAdapter`. It owns an `env` field (the environment: its taskset, seats, limits) and adds the optimization loop's own knobs (model, reflection model, train/val split, budget). There is no `[serve]` block here — GEPA always runs in-process, since its adapter protocol is itself synchronous (see @@ -25,7 +25,7 @@ class GEPAConfig(BaseConfig): """The GEPA run plus its environment. `model` runs the rollouts under optimization; `reflection_model` (defaults to `model`) proposes new system prompts from the reflective - dataset. `model` defaults to the same id as `EvalConfig`.""" + dataset.""" env: SerializeAsAny[EnvConfig] = SingleAgentEnvConfig() """The environment under optimization — the same `[env]` block as an eval's diff --git a/verifiers/v1/gepa/runner.py b/verifiers/v1/gepa/runner.py index 39ad185677..2b5c91e998 100644 --- a/verifiers/v1/gepa/runner.py +++ b/verifiers/v1/gepa/runner.py @@ -51,12 +51,12 @@ def run_gepa(env: Env, config: GEPAConfig) -> GEPAResult: if run_dir is not None: save_config( config, run_dir, "gepa.json" - ) # resolved config + a fresh traces.jsonl (like run_eval) + ) # resolved config + a fresh traces.jsonl logger.info("results: %s", run_dir) # optimize() is synchronous and blocking, so it drives the run from this (main) thread. We - # own one event loop: `env.serving()` (shared tool servers + interception pool, built once - # like run_eval) is entered on it, each rollout batch runs on it via `loop.run_until_complete` + # own one event loop: `env.serving()` (shared tool servers + interception pool, built + # once) is entered on it, each rollout batch runs on it via `loop.run_until_complete` # (GEPAAdapter.evaluate), and it's all torn down in `finally`. Keeping optimize() on the # main thread means a Ctrl-C raises straight through it into this teardown. loop = asyncio.new_event_loop() @@ -64,7 +64,7 @@ def run_gepa(env: Env, config: GEPAConfig) -> GEPAResult: asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None ) # Stream every rollout's episode to traces.jsonl as it finalizes — the same persist hook - # run_eval passes to `env.run_slot` (each trace records its candidate prompt). + # `env.run_slot` takes (each trace records its candidate prompt). write_lock = asyncio.Lock() async def on_complete(episode: Episode) -> None: