From da6982f0132d56683897f50f9e452a5c90e3e325 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:12:53 +0000 Subject: [PATCH 01/28] chore: final touches - harness configs no longer pin `id`; the caller supplies it (--harness.id / toml / a taskset's bundled harness), mirroring tasksets. Update the base HarnessConfig docstring and the `init` scaffold to match. - refine the v1 user guide (GUIDE.md, README.md) - register the reverse-text v0 env as an editable dep for the `eval --id reverse-text` legacy-bridge example Co-Authored-By: Claude Opus 4.8 (1M context) --- environments/compact/compact/harness.py | 2 -- .../tau2_bench_v1/tau2_bench_v1/harness.py | 2 -- packages/harnesses/harnesses/bash/harness.py | 2 -- packages/harnesses/harnesses/codex/harness.py | 1 - packages/harnesses/harnesses/default/harness.py | 2 -- .../harnesses/harnesses/kimi_code/harness.py | 1 - .../harnesses/mini_swe_agent/harness.py | 1 - packages/harnesses/harnesses/rlm/harness.py | 2 -- .../harnesses/harnesses/terminus_2/harness.py | 1 - pyproject.toml | 2 ++ uv.lock | 17 +++++++++++++++++ verifiers/v1/GUIDE.md | 14 ++++++-------- verifiers/v1/README.md | 2 +- verifiers/v1/cli/init.py | 6 +++--- verifiers/v1/harness.py | 7 ++++--- 15 files changed, 33 insertions(+), 29 deletions(-) diff --git a/environments/compact/compact/harness.py b/environments/compact/compact/harness.py index 902b50fbf1..382012e807 100644 --- a/environments/compact/compact/harness.py +++ b/environments/compact/compact/harness.py @@ -23,8 +23,6 @@ class CompactingHarnessConfig(HarnessConfig): """A context-rewrite harness: it rebuilds its prompt from carried-over notes each compaction instead of appending, so the trajectory branches at every compaction.""" - id: str = "compact" - class CompactingHarness(Harness[CompactingHarnessConfig]): SUPPORTS_TASK_TOOLS = True diff --git a/environments/tau2_bench_v1/tau2_bench_v1/harness.py b/environments/tau2_bench_v1/tau2_bench_v1/harness.py index cb8cc16b5c..4c20847d53 100644 --- a/environments/tau2_bench_v1/tau2_bench_v1/harness.py +++ b/environments/tau2_bench_v1/tau2_bench_v1/harness.py @@ -2,7 +2,6 @@ import os import sys from pathlib import Path -from typing import Literal os.environ.setdefault( "TAU2_DATA_DIR", str(Path.home() / ".cache" / "tau2-bench-v1" / "data") @@ -47,7 +46,6 @@ def _flip_roles(self: UserState): class Tau2HarnessConfig(vf.HarnessConfig): - id: Literal["tau2-bench-v1"] = "tau2-bench-v1" # The runner imports this installed module, so it must share the host interpreter. runtime: vf.SubprocessConfig = vf.SubprocessConfig() diff --git a/packages/harnesses/harnesses/bash/harness.py b/packages/harnesses/harnesses/bash/harness.py index a39fb3d1b7..2eeb9f8cc0 100644 --- a/packages/harnesses/harnesses/bash/harness.py +++ b/packages/harnesses/harnesses/bash/harness.py @@ -25,8 +25,6 @@ class BashHarnessConfig(HarnessConfig): """The built-in bash harness. A uv script (deps: openai, mcp), so it runs in any runtime that has `uv` (the harness bootstraps it) with no other setup.""" - id: str = "bash" - class BashHarness(Harness[BashHarnessConfig]): APPENDS_SYSTEM_PROMPT = True diff --git a/packages/harnesses/harnesses/codex/harness.py b/packages/harnesses/harnesses/codex/harness.py index 68ea51341b..057a8075a0 100644 --- a/packages/harnesses/harnesses/codex/harness.py +++ b/packages/harnesses/harnesses/codex/harness.py @@ -39,7 +39,6 @@ class CodexHarnessConfig(HarnessConfig): """The Codex CLI harness — which codex release to install in the runtime.""" - id: str = "codex" version: str = "0.137.0" """Codex release to install (the `rust-v` GitHub release); pinned for reproducibility.""" diff --git a/packages/harnesses/harnesses/default/harness.py b/packages/harnesses/harnesses/default/harness.py index 8a6f1f04c4..d5e2fbaf63 100644 --- a/packages/harnesses/harnesses/default/harness.py +++ b/packages/harnesses/harnesses/default/harness.py @@ -22,8 +22,6 @@ class DefaultHarnessConfig(HarnessConfig): """The built-in harness. A uv script (deps: openai, mcp), so it runs in any runtime that has `uv` (the harness bootstraps it) with no other setup.""" - id: str = "default" - class DefaultHarness(Harness[DefaultHarnessConfig]): APPENDS_SYSTEM_PROMPT = True diff --git a/packages/harnesses/harnesses/kimi_code/harness.py b/packages/harnesses/harnesses/kimi_code/harness.py index 1fe257d7ca..8fc4111f8c 100644 --- a/packages/harnesses/harnesses/kimi_code/harness.py +++ b/packages/harnesses/harnesses/kimi_code/harness.py @@ -39,7 +39,6 @@ class KimiCodeHarnessConfig(HarnessConfig): """The Kimi Code CLI harness.""" - id: str = "kimi-code" version: str = "0.14.3" """Kimi Code release to install, pinned for reproducibility.""" diff --git a/packages/harnesses/harnesses/mini_swe_agent/harness.py b/packages/harnesses/harnesses/mini_swe_agent/harness.py index 8735c606ac..acc3914ad9 100644 --- a/packages/harnesses/harnesses/mini_swe_agent/harness.py +++ b/packages/harnesses/harnesses/mini_swe_agent/harness.py @@ -13,7 +13,6 @@ class MiniSWEAgentHarnessConfig(HarnessConfig): """The mini-swe-agent CLI harness.""" - id: str = "mini-swe-agent" version: str = "2.2.8" """mini-swe-agent release to install, pinned for reproducibility.""" diff --git a/packages/harnesses/harnesses/rlm/harness.py b/packages/harnesses/harnesses/rlm/harness.py index 49ecad3f26..edc433de6f 100644 --- a/packages/harnesses/harnesses/rlm/harness.py +++ b/packages/harnesses/harnesses/rlm/harness.py @@ -27,8 +27,6 @@ class RLMHarnessConfig(HarnessConfig): """The rlm CLI harness — how to install rlm and how it should run.""" - id: str = "rlm" - version: str = "main" """Git ref (branch, tag, or commit) of rlm to install.""" max_depth: int = 0 diff --git a/packages/harnesses/harnesses/terminus_2/harness.py b/packages/harnesses/harnesses/terminus_2/harness.py index e7aa09fa63..71f88fa8a0 100644 --- a/packages/harnesses/harnesses/terminus_2/harness.py +++ b/packages/harnesses/harnesses/terminus_2/harness.py @@ -15,7 +15,6 @@ class Terminus2HarnessConfig(HarnessConfig): """The Harbor Terminus 2 harness.""" - id: str = "terminus-2" version: str = "0.14.0" """Harbor release to install, pinned for reproducibility.""" diff --git a/pyproject.toml b/pyproject.toml index 8a1507a75d..6ae4cd0991 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,7 @@ examples = [ "wordle-v1", "terminal-bench-2-v1", "alphabet-sort-v1", "r2e-gym-v1", "scaleswe-v1", "swelego-v1", "scratchpad-v1", "general-agent-v1", "swebench-verified-v1", "tau2-bench-v1", + "reverse-text", # v0 env for the `eval --id reverse-text` legacy-bridge example ] [project.optional-dependencies] @@ -149,6 +150,7 @@ wiki-search-v1 = { path = "environments/wiki_search_v1", editable = true } gsm8k-v1 = { path = "environments/gsm8k_v1", editable = true } code-golf-v1 = { path = "environments/code_golf_v1", editable = true } reverse-text-v1 = { path = "environments/reverse_text_v1", editable = true } +reverse-text = { path = "environments/reverse_text", editable = true } wordle-v1 = { path = "environments/wordle_v1", editable = true } terminal-bench-2-v1 = { path = "environments/terminal_bench_2_v1", editable = true } math-env-v1 = { path = "environments/math_env_v1", editable = true } diff --git a/uv.lock b/uv.lock index 2e6dfeb35d..8711d49cc4 100644 --- a/uv.lock +++ b/uv.lock @@ -5075,6 +5075,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, ] +[[package]] +name = "reverse-text" +version = "0.1.4" +source = { editable = "environments/reverse_text" } +dependencies = [ + { name = "datasets" }, + { name = "verifiers" }, +] + +[package.metadata] +requires-dist = [ + { name = "datasets" }, + { name = "verifiers", specifier = ">=0.1.5.post0" }, +] + [[package]] name = "reverse-text-v1" version = "0.1.0" @@ -6337,6 +6352,7 @@ examples = [ { name = "gsm8k-v1" }, { name = "math-env-v1" }, { name = "r2e-gym-v1" }, + { name = "reverse-text" }, { name = "reverse-text-v1" }, { name = "scaleswe-v1" }, { name = "scratchpad-v1" }, @@ -6437,6 +6453,7 @@ examples = [ { name = "gsm8k-v1", editable = "environments/gsm8k_v1" }, { name = "math-env-v1", editable = "environments/math_env_v1" }, { name = "r2e-gym-v1", editable = "environments/r2e_gym_v1" }, + { name = "reverse-text", editable = "environments/reverse_text" }, { name = "reverse-text-v1", editable = "environments/reverse_text_v1" }, { name = "scaleswe-v1", editable = "environments/scaleswe_v1" }, { name = "scratchpad-v1", editable = "environments/scratchpad_v1" }, diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index e8130093ad..9b6ac05bd4 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -5,13 +5,12 @@ why, see the [README](README.md); this guide is the longer-form how-to. `import ## Mental model -A v1 environment is three decoupled pieces, each selected by `id` and configured by typed -config. You'll work with them in very different proportions: +A v1 environment is three decoupled pieces, each packagable and configured through typed +config. You'll likely work with them in different proportions: - **Taskset** — the data and the scoring: it produces typed `Task`s and owns every `@reward` / `@metric`, plus any tools and a user simulator (*what* the model is asked and *how* it's - graded). **This is what you author** — for almost every environment it's the only piece you - write. + graded). For many environments, this is the only piece you write. - **Harness** — the program that drives the rollout turn to turn, a chat loop or an agent CLI (*how* the model is called). **Usually you just pick a built-in** (`default` / `rlm` / `codex`); you only write your own if you need a custom rollout loop. With some exceptions, any @@ -42,10 +41,9 @@ flag names are the dotted config path (`--harness.runtime.type docker`). See the # Authoring a taskset -A taskset is a package selected by `id`. Scaffold one with `uv run init my-task-v1` (add +A taskset is a package exporting a `vf.Taskset`. Scaffold one with `uv run init my-task-v1` (add `--add-tool` / `--add-user` / `--add-harness` for more pieces, `--v0` for a legacy environment), -or copy the closest `environments/_v1` and edit. The scaffold runs out of the box; replace -`load_tasks` and the `@reward`. The whole minimal shape: +or copy the closest `environments/_v1` and edit. The minimal shape is: ```python import verifiers.v1 as vf @@ -112,7 +110,7 @@ from verifiers.v1 import TaskTimeout, TaskResources MyTask( idx=0, prompt=..., timeout=TaskTimeout(setup=300, harness=1200, scoring=120), # seconds; per stage, None = no limit - resources=TaskResources(cpu=4, memory=8, gpu="A100:2", disk=20), # Modal units; None = runtime default + resources=TaskResources(cpu=4, memory=8, gpu="A100:2", disk=20), # None = runtime default ) ``` diff --git a/verifiers/v1/README.md b/verifiers/v1/README.md index 5248ba496a..1253124886 100644 --- a/verifiers/v1/README.md +++ b/verifiers/v1/README.md @@ -259,6 +259,6 @@ bridge — its rollouts mapped to v1 `Trace`s. Set `--id` (instead of a `taskset ```bash uv run eval --id reverse-text -n 2 # eval a v0 env -uv run eval --id reverse-text --args.num_train_examples 50 \ +uv run eval --id reverse-text --args.dataset_split train \ --extra-env-kwargs.max-total-completion-tokens 256 # construction + post-load kwargs ``` diff --git a/verifiers/v1/cli/init.py b/verifiers/v1/cli/init.py index 1225d31bb9..d6ade854da 100644 --- a/verifiers/v1/cli/init.py +++ b/verifiers/v1/cli/init.py @@ -178,13 +178,13 @@ async def respond(self, message: str) -> vf.Messages: """ -def _harness_py(dash: str, prefix: str) -> str: +def _harness_py(prefix: str) -> str: return f'''\ import verifiers.v1 as vf class {prefix}HarnessConfig(vf.HarnessConfig): - id: str = "{dash}" + """Run knobs for this harness. Add fields here (e.g. a CLI version to install).""" class {prefix}Harness(vf.Harness[{prefix}HarnessConfig]): @@ -280,7 +280,7 @@ def scaffold(config: InitConfig) -> Path: config.force, ) if config.add_harness: - _write(pkg_dir / "harness.py", _harness_py(dash, prefix), config.force) + _write(pkg_dir / "harness.py", _harness_py(prefix), config.force) if config.add_tool or config.add_user: _write(pkg_dir / "servers" / "__init__.py", "", config.force) if config.add_tool: diff --git a/verifiers/v1/harness.py b/verifiers/v1/harness.py index 7e55bca233..60b34da604 100644 --- a/verifiers/v1/harness.py +++ b/verifiers/v1/harness.py @@ -33,9 +33,10 @@ class HarnessConfig(BaseConfig): - """A harness's config — subclass per harness (each pins `id` to the harness id). Mirrors - `TasksetConfig`: the base type names the field, the concrete subclass is resolved by id - (no closed union).""" + """A harness's config — subclass per harness to add run knobs. Mirrors `TasksetConfig`: the + base type names the field, the concrete subclass is resolved by id (no closed union) — the + id is supplied by the caller (`--harness.id` / toml / a taskset's bundled harness), never + pinned on the subclass.""" id: EnvId = "default" """The harness id, which selects this harness: a local package, or an From 175600b75bb27143ffe799baa669a57a31504227 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:13:54 +0000 Subject: [PATCH 02/28] docs: don't use auto-tracked num_turns as the @vf.metric example trace.num_turns is recorded by the framework; show a custom signal instead. Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index 9b6ac05bd4..0301ee87d4 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -154,7 +154,7 @@ you name** — declare any subset of `task` / `trace` / `runtime` and you get ex async def correct(self, task, trace) -> float: ... @vf.metric() # recorded, not summed — return a float or a dict to merge -async def num_turns(self, trace) -> float: ... +async def format_ok(self, trace) -> float: ... @vf.group_reward(weight=1.0) # scores a task's N rollouts together async def best_of_n(self, traces: list[vf.Trace]) -> list[float]: ... From fea7e8b6f1379fb00429f77e2a7c680bccf88f0b Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:15:07 +0000 Subject: [PATCH 03/28] docs: move the load_tasks example into the Loading tasks section Keep the config code block focused on the config class. Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index 0301ee87d4..9f1528b62d 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -126,11 +126,6 @@ CLI flags (and TOML keys), and the instance reaches the taskset as `self.config` ```python class GSM8KConfig(vf.TasksetConfig): split: Literal["train", "test"] = "test" # --taskset.split test - -class GSM8KTaskset(vf.Taskset[GSM8KTask, GSM8KConfig]): - def load_tasks(self): - rows = load_dataset("gsm8k", split=self.config.split) # read knobs off self.config - ... ``` Nested configs nest the flag path: a `tools: vf.ToolsetConfig` field is set with @@ -144,6 +139,13 @@ able to change. The base `TasksetConfig` carries `id` (the taskset's id, set via rollout), so do dataset loading / filtering / slicing here off `self.config`. Return your typed `Task` subclass instances. +```python +class GSM8KTaskset(vf.Taskset[GSM8KTask, GSM8KConfig]): + def load_tasks(self): + rows = load_dataset("gsm8k", split=self.config.split) # read knobs off self.config + ... +``` + ## Scoring — rewards, metrics, group rewards Rewards and metrics are decorated `async` methods. The framework **injects whichever arguments From a73e7aa7a05fd6d5fdde09d383fbe91156e74648 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:15:47 +0000 Subject: [PATCH 04/28] docs: show full GSM8KTask construction in the load_tasks example Build typed task instances from the rows instead of stopping at the dataset load. Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index 9f1528b62d..ad225d1ede 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -141,9 +141,16 @@ rollout), so do dataset loading / filtering / slicing here off `self.config`. Re ```python class GSM8KTaskset(vf.Taskset[GSM8KTask, GSM8KConfig]): - def load_tasks(self): - rows = load_dataset("gsm8k", split=self.config.split) # read knobs off self.config - ... + def load_tasks(self) -> list[GSM8KTask]: + rows = load_dataset("openai/gsm8k", "main", split=self.config.split) # read knobs off self.config + return [ + GSM8KTask( + idx=i, + prompt=row["question"], + answer=row["answer"].split("####")[-1].strip(), + ) + for i, row in enumerate(rows) + ] ``` ## Scoring — rewards, metrics, group rewards From 5c6ac617f307cd97a142946c233b81517fd5792d Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:19:01 +0000 Subject: [PATCH 05/28] feat: allow @reward to return dict[str, float] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A reward can now report a family of named contributions, like @metric already can; each entry is recorded under its own key and scaled by the function's weight before being summed into trace.reward. This aligns the reward/metric return contracts — the only remaining (and intended) differences are that rewards are summed and carry a weight. Documented in the v1 guide. Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 14 ++++++++------ verifiers/v1/decorators.py | 6 ++++-- verifiers/v1/taskset.py | 12 +++++++++--- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index ad225d1ede..cf3947ffdb 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -159,7 +159,7 @@ Rewards and metrics are decorated `async` methods. The framework **injects which you name** — declare any subset of `task` / `trace` / `runtime` and you get exactly those: ```python -@vf.reward(weight=1.0) # summed (weighted) into trace.reward +@vf.reward(weight=1.0) # summed (weighted) into trace.reward — a float or a dict to merge async def correct(self, task, trace) -> float: ... @vf.metric() # recorded, not summed — return a float or a dict to merge @@ -173,7 +173,7 @@ The decorators and what each can receive: | decorator | params | optional kwargs | returns | | --- | --- | --- | --- | -| `@vf.reward` | `task`, `trace`, `runtime` | `weight=1.0`, `priority=0` | `float` (× weight → summed into `trace.reward`) | +| `@vf.reward` | `task`, `trace`, `runtime` | `weight=1.0`, `priority=0` | `float`, **or a `dict[str, float]`** (each × weight → summed into `trace.reward`) | | `@vf.metric` | `task`, `trace`, `runtime` | `priority=0` | `float`, **or a `dict[str, float]`** merged into `trace.metrics` | | `@vf.group_reward` | `task`, `traces` | `weight=1.0`, `priority=0` | `list[float]`, one per trace | | `@vf.stop` | `trace` | `priority=0` | `bool` | @@ -184,10 +184,12 @@ Notes that bite if missed: after the per-rollout runtimes are gone). To compare a runtime-derived signal across a task's rollouts, record it per-rollout as a `@metric`/`@reward` first, then read it off each trace in the group reward. Group rewards need `-r/--num-rollouts ≥ 2`. -- **`@metric` returning a dict** lets one method report a whole family of numbers (each key merged - into `trace.metrics`). A scalar is recorded under the method name. -- **`weight`** scales a reward's contribution (`trace.reward = Σ value·weight`); each reward is - keyed by its method name, so two rewards (or a reward and a group reward) sharing a name clobber. +- **Returning a dict** (`@reward` or `@metric`) lets one method report a whole family of values — + each key is recorded under its own name (rewards into `trace.rewards`, metrics into + `trace.metrics`); a scalar is recorded under the method name. A reward's `weight` scales every + entry it returns. +- **`weight`** scales a reward's contribution (`trace.reward = Σ value·weight`); each contribution is + keyed by the method name (or, for a dict return, its keys), so two sharing a name clobber. - **`priority`** orders execution within a kind (higher first, then by name). It mostly matters for `@stop` — the highest-priority stop that fires sets the stop reason. diff --git a/verifiers/v1/decorators.py b/verifiers/v1/decorators.py index 131d6c6f11..80b1b1c3ce 100644 --- a/verifiers/v1/decorators.py +++ b/verifiers/v1/decorators.py @@ -101,8 +101,10 @@ def reward( def reward( func: F | None = None, weight: float = 1.0, priority: int = 0 ) -> F | Callable[[F], F]: - """Mark a per-rollout reward (summed into trace.reward). Declare any of - `task`/`trace`/`runtime`; they're injected by name.""" + """Mark a per-rollout reward (weighted, summed into trace.reward). Declare any of + `task`/`trace`/`runtime`; they're injected by name. Return a `float` (recorded under + the method name) or a `dict[str, float]` (each entry under its own key); every + contribution is scaled by `weight` before it's summed into `trace.reward`.""" decorator = mark("reward", reward_priority=priority, _vf_weight=weight) return decorator if func is None else decorator(func) diff --git a/verifiers/v1/taskset.py b/verifiers/v1/taskset.py index e8c8c9cbb5..fc00d27ef1 100644 --- a/verifiers/v1/taskset.py +++ b/verifiers/v1/taskset.py @@ -107,8 +107,9 @@ async def validate(self, task: TaskT, runtime: Runtime) -> bool: async def score(self, trace: Trace, runtime: Runtime) -> None: """Score one rollout: run all `@metric` then `@reward` over its trace, concurrently within each phase. Each metric is recorded in `trace.metrics` - (a number, or a mapping merged in); each reward (weighted) in `trace.rewards`, - which `trace.reward` sums. Signals declare what they need — `task`, `trace`, + (a number, or a mapping merged in); each reward (weighted — likewise a number or a + mapping merged in) in `trace.rewards`, which `trace.reward` sums. Signals declare + what they need — `task`, `trace`, `runtime` — so a reward is either a pure function of the trace or runs read/write/exec in that (still-live) runtime, e.g. a verifier script.""" available = {"task": trace.task, "trace": trace, "runtime": runtime} @@ -127,7 +128,12 @@ async def score(self, trace: Trace, runtime: Runtime) -> None: rewards, await asyncio.gather(*(invoke(fn, available) for fn in rewards)), ): - trace.record_reward(fn.__name__, result, getattr(fn, "_vf_weight", 1.0)) + weight = getattr(fn, "_vf_weight", 1.0) + if isinstance(result, Mapping): + for name, value in result.items(): + trace.record_reward(name, value, weight) + else: + trace.record_reward(fn.__name__, result, weight) async def score_group(self, traces: list[Trace]) -> None: """Score a group of rollouts of one task: run every `@group_reward` over all From b17549236f2d8dee61c53f51c31f394ecad05d7b Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:19:07 +0000 Subject: [PATCH 06/28] docs: drop the score/score_group override note Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index cf3947ffdb..798c09e925 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -193,9 +193,6 @@ Notes that bite if missed: - **`priority`** orders execution within a kind (higher first, then by name). It mostly matters for `@stop` — the highest-priority stop that fires sets the stop reason. -You normally never override `score` / `score_group` — those are the dispatch machinery that finds -and runs your decorators. - ### Reading the trace A reward reads the finished trajectory off `trace`. The most useful read-only members: From 4c46f8241b800b5baa71a07bfae497b668fa8eca Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:20:18 +0000 Subject: [PATCH 07/28] docs: livelier scoring examples + reframe the notes heading Use an enthusiasm metric and a brevity (length-penalty) group reward instead of stubs, and reword "Notes that bite if missed" to "Good to know". Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index 798c09e925..e9b8b373f4 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -162,11 +162,14 @@ you name** — declare any subset of `task` / `trace` / `runtime` and you get ex @vf.reward(weight=1.0) # summed (weighted) into trace.reward — a float or a dict to merge async def correct(self, task, trace) -> float: ... -@vf.metric() # recorded, not summed — return a float or a dict to merge -async def format_ok(self, trace) -> float: ... - -@vf.group_reward(weight=1.0) # scores a task's N rollouts together -async def best_of_n(self, traces: list[vf.Trace]) -> list[float]: ... +@vf.metric() # recorded, not summed — a float or a dict to merge +async def enthusiasm(self, trace) -> float: + return float(trace.assistant_messages[-1].content.count("!")) + +@vf.group_reward(weight=0.1) # compares a task's N rollouts — here, a length penalty +async def brevity(self, traces: list[vf.Trace]) -> list[float]: + longest = max(t.completion_len for t in traces) or 1 + return [-t.completion_len / longest for t in traces] # longest gets -1, shorter ones less ``` The decorators and what each can receive: @@ -178,18 +181,12 @@ The decorators and what each can receive: | `@vf.group_reward` | `task`, `traces` | `weight=1.0`, `priority=0` | `list[float]`, one per trace | | `@vf.stop` | `trace` | `priority=0` | `bool` | -Notes that bite if missed: +Good to know: - **`@group_reward` gets no `runtime` and no single `trace`** — only `task` and `traces` (it runs after the per-rollout runtimes are gone). To compare a runtime-derived signal across a task's rollouts, record it per-rollout as a `@metric`/`@reward` first, then read it off each trace in the group reward. Group rewards need `-r/--num-rollouts ≥ 2`. -- **Returning a dict** (`@reward` or `@metric`) lets one method report a whole family of values — - each key is recorded under its own name (rewards into `trace.rewards`, metrics into - `trace.metrics`); a scalar is recorded under the method name. A reward's `weight` scales every - entry it returns. -- **`weight`** scales a reward's contribution (`trace.reward = Σ value·weight`); each contribution is - keyed by the method name (or, for a dict return, its keys), so two sharing a name clobber. - **`priority`** orders execution within a kind (higher first, then by name). It mostly matters for `@stop` — the highest-priority stop that fires sets the stop reason. From c5ed34b452a22b029cfde057fc3e2206be07e618 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:21:42 +0000 Subject: [PATCH 08/28] docs: restructure the trace-reading table and reframe in-runtime scoring Group the trace members by area and fix the table: drop the nonexistent trace.messages (use trace.branches[-1].messages), add usage / has_response / id, and note trace.nodes. Reframe in-runtime scoring around when to use the runtime object (host-unsafe/heavy computation, or runtime-only information). Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 53 +++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index e9b8b373f4..52a4fd9182 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -164,12 +164,9 @@ async def correct(self, task, trace) -> float: ... @vf.metric() # recorded, not summed — a float or a dict to merge async def enthusiasm(self, trace) -> float: - return float(trace.assistant_messages[-1].content.count("!")) @vf.group_reward(weight=0.1) # compares a task's N rollouts — here, a length penalty async def brevity(self, traces: list[vf.Trace]) -> list[float]: - longest = max(t.completion_len for t in traces) or 1 - return [-t.completion_len / longest for t in traces] # longest gets -1, shorter ones less ``` The decorators and what each can receive: @@ -192,31 +189,57 @@ Good to know: ### Reading the trace -A reward reads the finished trajectory off `trace`. The most useful read-only members: +A reward reads the finished trajectory off `trace`. The most useful read-only members, by area: + +**Task & messages** | member | type | what | | --- | --- | --- | | `trace.task` | `TaskT` | the typed task (your subclass) | | `trace.assistant_messages` | `list[AssistantMessage]` | the model's responses in order (excludes prompt-supplied messages) | -| `trace.messages` | `Messages` | the full conversation (main branch) | -| `trace.tool_messages` | `list[ToolMessage]` | tool results | -| `trace.reward` / `trace.rewards` | `float` / `dict` | summed reward / per-function contributions | +| `trace.tool_messages` | `list[ToolMessage]` | tool results (main branch) | +| `trace.branches[-1].messages` | `Messages` | the full conversation of the main (last) branch | + +**Scoring** (rewards/metrics come from your decorator returns; `info`/`state` you set yourself) + +| member | type | what | +| --- | --- | --- | +| `trace.reward` / `trace.rewards` | `float` / `dict[str, float]` | summed reward / per-key contributions | | `trace.metrics` | `dict[str, float]` | recorded metrics | | `trace.info` | `dict` | free-form persisted artifact bag (see below) | | `trace.state` | `StateT` | transient per-rollout state (see [State](#per-rollout-state)) | + +**Status & lifecycle** + +| member | type | what | +| --- | --- | --- | +| `trace.is_completed` / `trace.stop_condition` | `bool` / `str \| None` | whether / why the rollout ended | +| `trace.is_truncated` | `bool` | ended by hitting a turn/token/length cap | +| `trace.has_response` | `bool` | the last turn produced non-empty content | +| `trace.has_error` / `trace.error` / `trace.errors` | `bool` / `Error \| None` / `list[Error]` | error state (most recent / all attempts) | + +**Counts, tokens & timing** + +| member | type | what | +| --- | --- | --- | | `trace.num_turns` | `int` | sampled model turns | -| `trace.num_branches` / `trace.branches` | `int` / `list` | branch count / the branches (compaction, retokenization) | -| `trace.is_truncated` | `bool` | hit a turn/token/length cap | -| `trace.stop_condition` / `trace.is_completed` | `str \| None` / `bool` | why/whether the rollout ended | -| `trace.has_error` / `trace.error` / `trace.errors` | `bool` / … | error state | -| `trace.prompt_len` / `completion_len` / `total_tokens` | `int` | token counts | +| `trace.num_branches` / `trace.branches` | `int` / `list[Branch]` | branch count / the branches (>1 under compaction or subagents) | +| `trace.prompt_len` / `trace.completion_len` / `trace.total_tokens` | `int` | token counts (summed over branches) | +| `trace.usage` | `Usage \| None` | provider-reported token usage | | `trace.timing` | `Timing` | per-stage durations | +| `trace.id` | `str` | unique rollout id | + +The raw message graph is `trace.nodes` (each message stored once); `branches` is the friendly view +over it, so you rarely touch `nodes` directly. ### In-runtime scoring -To score with a dependency the eval process shouldn't have (e.g. `math-verify`), run it as a uv -script *in the rollout's runtime* — the dep resolves inside the runtime and never touches the eval -process: +Declare `runtime` on a `@reward`/`@metric` when scoring needs the rollout's runtime — either +because it requires **heavy computation that shouldn't run on the host** (e.g. a verifier with its +own dependencies like `math-verify`), or because it needs **information that only lives in the +agent's runtime** (files the agent wrote, command output, container state). The `runtime` object +gives you read/write/exec in there; a common pattern is a uv script whose PEP 723 deps resolve +inside the runtime and never touch the eval process: ```python VERIFY = (Path(__file__).parent / "verify.py").read_text() # PEP 723 header declares its deps From e5e33813dc59feabf5be06196890701f73a980d4 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:23:17 +0000 Subject: [PATCH 09/28] docs: separate scoring outputs from read inputs; say "host" not "eval process" The trace-reading table now lists only what a reward reads; reward/rewards/ metrics are called out as outputs you shouldn't read mid-scoring. Rename that group to "Carried state" (info/state). Use "host" instead of "eval process". Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index 52a4fd9182..ba9da626f3 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -189,7 +189,7 @@ Good to know: ### Reading the trace -A reward reads the finished trajectory off `trace`. The most useful read-only members, by area: +A reward reads the finished trajectory off `trace`. The most useful members, by area: **Task & messages** @@ -200,12 +200,10 @@ A reward reads the finished trajectory off `trace`. The most useful read-only me | `trace.tool_messages` | `list[ToolMessage]` | tool results (main branch) | | `trace.branches[-1].messages` | `Messages` | the full conversation of the main (last) branch | -**Scoring** (rewards/metrics come from your decorator returns; `info`/`state` you set yourself) +**Carried state** (set during the rollout / `finalize`, read back here) | member | type | what | | --- | --- | --- | -| `trace.reward` / `trace.rewards` | `float` / `dict[str, float]` | summed reward / per-key contributions | -| `trace.metrics` | `dict[str, float]` | recorded metrics | | `trace.info` | `dict` | free-form persisted artifact bag (see below) | | `trace.state` | `StateT` | transient per-rollout state (see [State](#per-rollout-state)) | @@ -229,8 +227,9 @@ A reward reads the finished trajectory off `trace`. The most useful read-only me | `trace.timing` | `Timing` | per-stage durations | | `trace.id` | `str` | unique rollout id | -The raw message graph is `trace.nodes` (each message stored once); `branches` is the friendly view -over it, so you rarely touch `nodes` directly. +`trace.reward` / `trace.rewards` / `trace.metrics` are scoring *outputs*, filled in during the +scoring pass — don't read them from inside a `@reward`/`@metric`; a `@group_reward` reads metrics +off each finished trace instead. ### In-runtime scoring @@ -239,13 +238,13 @@ because it requires **heavy computation that shouldn't run on the host** (e.g. a own dependencies like `math-verify`), or because it needs **information that only lives in the agent's runtime** (files the agent wrote, command output, container state). The `runtime` object gives you read/write/exec in there; a common pattern is a uv script whose PEP 723 deps resolve -inside the runtime and never touch the eval process: +inside the runtime and never touch the host: ```python VERIFY = (Path(__file__).parent / "verify.py").read_text() # PEP 723 header declares its deps @vf.reward() -async def verified(self, task, trace, runtime) -> float: +async def verify(self, task, trace, runtime) -> float: r = await runtime.run_uv_script(VERIFY, args=[task.answer, trace.assistant_messages[-1].content]) return float(r.stdout.strip() == "1.0") ``` @@ -268,8 +267,7 @@ trace (above) or from per-rollout state set by a tool / user sim (see [State](#p ## Lifecycle hooks -A rollout runs **`setup → harness → finalize → scoring`**, each independently timeout-bounded -(`--timeout.{setup,rollout,finalize,scoring}`, or per-task `TaskTimeout`). A taskset can hook any +A rollout runs **`setup → harness → finalize → scoring`**. A taskset can hook any stage; all are `async`: | hook | signature | when | gets runtime? | @@ -560,7 +558,7 @@ user message). **Two program styles.** A self-contained chat loop is usually a single-file uv script (`runtime.run_uv_script`, so the harness needs only `uv` in the runtime — its inline deps resolve -there, never in the eval process; identical scripts share one content-addressed uv env). An agent +there, never on the host; identical scripts share one content-addressed uv env). An agent CLI / binary is installed and launched with `runtime.run(...)`. Either way, harness-owned env vars (`OPENAI_BASE_URL` / `OPENAI_API_KEY` / `OPENAI_MODEL`, …) are spread *after* `self.config.env`, so they take precedence over any collision. From 1607fb01f529947692b242089a9517ae6b9c64e3 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:24:07 +0000 Subject: [PATCH 10/28] docs: drop validate from the lifecycle-hooks table validate isn't part of the setup -> harness -> finalize -> scoring rollout loop (it's run only by `uv run validate`, documented in the CLI reference). Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index ba9da626f3..cbcc01c937 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -267,19 +267,16 @@ trace (above) or from per-rollout state set by a tool / user sim (see [State](#p ## Lifecycle hooks -A rollout runs **`setup → harness → finalize → scoring`**. A taskset can hook any -stage; all are `async`: +A rollout runs **`setup → harness → finalize → scoring`**. A taskset can hook any stage: | hook | signature | when | gets runtime? | | --- | --- | --- | --- | | `setup` | `(self, task, runtime)` | per-task prep before the harness (clone a repo, start a service) — the trace doesn't exist yet | ✓ | | `finalize` | `(self, task, trace, runtime)` | after the harness, before scoring — apply a diff, snapshot, scrape artifacts into `trace.info` | ✓ | -| `validate` | `(self, task, runtime) -> bool` | model-free gold check (does the reference solution pass?), run only by `uv run validate` | ✓ | | `tools` | `(self, task) -> list[vf.Toolset]` | per task, before the harness — the task's tool servers | ✗ | | `user` | `(self, task) -> vf.User \| None` | per task, before the harness — the user simulator | ✗ | -`setup`/`finalize`/`validate` errors fail the rollout legibly (captured onto the trace, not a -crash). +`setup`/`finalize` errors fail the rollout legibly (captured onto the trace, not a crash). ## Runtime access From bb8e299f945d5489935b99c423a90dd0ce2ea73b Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:26:42 +0000 Subject: [PATCH 11/28] docs: give info its own section, contrasted with state Lift the trace.info paragraph into a "Persisted info" section beside "Per-rollout state", with a table contrasting the two per-rollout stores (persisted vs transient). Cross-link both ways. Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index cbcc01c937..c20493a5a7 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -204,7 +204,7 @@ A reward reads the finished trajectory off `trace`. The most useful members, by | member | type | what | | --- | --- | --- | -| `trace.info` | `dict` | free-form persisted artifact bag (see below) | +| `trace.info` | `dict` | free-form persisted artifact bag (see [Persisted info](#persisted-info)) | | `trace.state` | `StateT` | transient per-rollout state (see [State](#per-rollout-state)) | **Status & lifecycle** @@ -316,16 +316,32 @@ class SWETaskset(vf.Taskset[SWETask, SWEConfig]): return 1.0 if result.exit_code == 0 else 0.0 ``` -`trace.info` is a free-form, **JSON-serializable, persisted** dict for anything that isn't a -reward or metric — runtime artifacts (the diff above, captured logs, command output) you want in -`results.jsonl` for inspection. Use `metrics` for numbers that aggregate, `trace.info` for -everything else (a non-serializable value fails the dump). +## Persisted info + +`trace.info` is a free-form, **JSON-serializable** dict for per-rollout artifacts that are neither a +reward nor a metric — the diff above, captured logs, command output, file paths. Write to it from +`finalize` or a `@reward`/`@metric` by assigning into the dict; it is persisted with the trace +(dumped to `results.jsonl` and sent over the wire), so every value must be JSON-serializable — a +non-serializable value fails the trace dump rather than being silently dropped. + +```python +trace.info["build_log"] = result.stdout +``` + +It pairs with [`trace.state`](#per-rollout-state) — the two per-rollout stores are opposites: + +| | `trace.info` | `trace.state` | +| --- | --- | --- | +| lifetime | **persisted** (dumped + sent over the wire) | **transient** (never dumped or sent) | +| for | artifacts to inspect after the run | live state the rollout reads and acts on | +| shape | free-form `dict[str, Any]`, JSON-serializable | typed `vf.State` subclass | +| written by | `finalize` / `@reward` / `@metric` | tools / user sim (`self.state`) + scoring | ## Per-rollout state `trace.state` is the complementary **transient** store: a typed, mutable `vf.State` shared across the rollout's tool servers, user simulator, and scoring — the one place per-rollout *runtime* state -lives (counters, game progress, your own end-of-trajectory flag). Unlike `info` it is **never** +lives (counters, game progress, your own end-of-trajectory flag). Unlike [`info`](#persisted-info) it is **never** persisted to disk or sent over the wire. Subclass `vf.State` to declare typed fields (each needs a default) and parameterize the taskset and any stateful server on it: @@ -362,10 +378,10 @@ Who touches it how: ## Tools -A tool server is a **vf-native class** (not raw MCP, no FastMCP boilerplate) authored from a -config — the same shape as a taskset. Define `@vf.tool` methods on a `vf.Toolset[ConfigT]` (or -`vf.Toolset[ConfigT, StateT]` for one that shares state); the model sees `_` -and the docstring is the description: +A tool server is a **vf-native class** authored from a config — the same shape +as a taskset. Define `@vf.tool` methods on a `vf.Toolset[ConfigT]` (or +`vf.Toolset[ConfigT, StateT]` for one that shares state); the model sees +`_` and the docstring is the description: ```python class GlossaryToolset(vf.Toolset[GlossaryToolsetConfig]): From 6ce36f29a858005e3fd247dd62eb4ecf2d1863a1 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:27:05 +0000 Subject: [PATCH 12/28] docs: note that a Toolset wraps an MCP server @vf.tool methods are served as MCP tools the harness connects to. Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index c20493a5a7..9254693468 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -378,17 +378,17 @@ Who touches it how: ## Tools -A tool server is a **vf-native class** authored from a config — the same shape -as a taskset. Define `@vf.tool` methods on a `vf.Toolset[ConfigT]` (or -`vf.Toolset[ConfigT, StateT]` for one that shares state); the model sees -`_` and the docstring is the description: +A tool server is a **vf-native class** that wraps an **MCP server** — authored from a config, the +same shape as a taskset. Define `@vf.tool` methods on a `vf.Toolset[ConfigT]` (or +`vf.Toolset[ConfigT, StateT]` for one that shares state); the framework serves them as MCP tools the +harness connects to, so the model sees `_` and the docstring is the description: ```python class GlossaryToolset(vf.Toolset[GlossaryToolsetConfig]): TOOL_PREFIX = "glossary" # model sees glossary_lookup (empty → class name snake-cased) async def setup(self) -> None: - self.facts = _load_facts() # task-agnostic, runs once per server process + self.facts = load_facts() # task-agnostic, runs once per server process @vf.tool def lookup(self, name: str) -> str: # typed params the model fills; docstring → description From 2f0374bad0c993a6e79361532a04b5be4a6a1a7b Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:28:45 +0000 Subject: [PATCH 13/28] docs: show the self-launching __main__ line in the tool/user examples Each tool/user server is its own self-launching module under servers/. Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index 9254693468..e22bf3c83e 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -394,6 +394,10 @@ class GlossaryToolset(vf.Toolset[GlossaryToolsetConfig]): def lookup(self, name: str) -> str: # typed params the model fills; docstring → description """Look up a glossary term.""" return self.facts.get(name.lower(), "unknown") + + +if __name__ == "__main__": + GlossaryToolset.run() # self-launching module under servers/ (see below) ``` Two setup hooks, plus the shared state: @@ -418,9 +422,13 @@ class HagglerUser(vf.User[vf.UserConfig, HagglerState]): if done: self.state.deal_closed = True # end via a @vf.stop the taskset declares return [{"role": "user", "content": reply}] + + +if __name__ == "__main__": + HagglerUser.run() # self-launching module under servers/ (see below) ``` -The framework calls `respond` after each assistant turn and injects the reply as the next user +The framework calls `respond` after each agent turn and injects the reply as the next user message; it's consumed by the framework, never shown to the model. A taskset supplies one via `user(task) -> vf.User | None`. If a task carries **no prompt** (`prompt=None`), the simulator also **opens the conversation**: the framework calls `respond("")` once before the first model turn and From e61da2a489abb4e0e7c7ed49ddde0346a963f81c Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:29:03 +0000 Subject: [PATCH 14/28] docs: merge own-host/own-sandbox into a single "own runtime" placement row Both are the same placement (a per-rollout runtime); subprocess on the host by default, a docker/prime sandbox when runtime.type is set. Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index e22bf3c83e..94289de2a8 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -449,8 +449,7 @@ the default is the cheapest correct thing; the rest trade setup cost for isolati | mode | config | runs | pros | cons | | --- | --- | --- | --- | --- | -| **own host** *(default)* | *(nothing)* | own `subprocess` runtime on the host, one per rollout | cheapest launch; full per-rollout isolation | pays `setup` every rollout | -| **own sandbox** | `runtime = {type = "docker"\|"prime"}` | own sandbox per rollout, over a tunnel | isolates untrusted tool code / deps / network | sandbox spin-up + env install + `setup`, every rollout | +| **own runtime** *(default)* | *(nothing; `runtime = {type = "docker"\|"prime"}` for a sandbox)* | own runtime per rollout — `subprocess` on the host (default), or a `docker`/`prime` sandbox over a tunnel | full per-rollout isolation; a sandbox also isolates untrusted code / deps / network | pays `setup` every rollout (a sandbox adds spin-up + env install) | | **colocated** | `colocated = true` | inside the harness's runtime, one per rollout (no tunnel) | no extra runtime/tunnel; can touch the harness's filesystem | couples to the harness; `setup` per rollout | | **shared** | `shared = true` | one instance for the whole eval | `setup` once; writable per-rollout if state lives in `self.state` | state outside `self.state` corrupts across rollouts; `setup_task` skipped | | **shared + fork** | `shared = true, fork = true` | warm parent + forked child per rollout (copy-on-write) | `setup` once **and** isolates arbitrary in-process/on-disk state; runs `setup_task` per child | a process per concurrent rollout; Linux only | From 571bbbc0ea0324b67d47e412148409df81dc6748 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:30:18 +0000 Subject: [PATCH 15/28] docs: add the missing reference examples general-agent-v1 (per-task dynamic tools), tau2-bench-v1 (tools + user sim + bundled harness), and swebench-verified-v1 (SWE-bench on prebuilt images) were the only *_v1 envs absent from the example table. Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index 94289de2a8..35c0d98a86 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -478,11 +478,14 @@ The `*_v1` tasksets under `environments/` are the reference library — each sho | `alphabet-sort-v1` | multi-turn, stateful, driven by a `vf.User` simulator | | `glossary-v1` | the simplest tool server (own host runtime) | | `wikispeedia-v1` | a stateful tool server (global `setup` + per-task `setup_task`) | +| `general-agent-v1` | per-task tools loaded dynamically (each task ships its own `tools.py`) + gold tool-call-chain scoring | | `wiki-search-v1` | a shared, read-only tool server (built once) + an LLM judge | | `scratchpad-v1` | a shared, **writable** tool server — per-rollout state isolated via `self.state` | | `deepwiki-v1` | an existing remote tool server, by URL | +| `tau2-bench-v1` | **both** tools and a user simulator at once, with a bundled harness (the tau2 benchmark) | | `color-codeword-v1` | a multimodal (image) task | | `scaleswe-v1`, `swelego-v1`, `r2e-gym-v1` | containerized SWE tasks (rlm harness, prime runtime) | +| `swebench-verified-v1` | SWE-bench Verified via `harbor-v1` on prime's prebuilt images (no Dockerfile build) | | `wordle-v1`, `terminal-bench-2-v1` | thin configs over the shipped `textarena-v1` / `harbor-v1` integrations | --- From 4366186bf72a7154b8a200cc9b5a6c7ba70a08d3 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:30:38 +0000 Subject: [PATCH 16/28] docs: reframe the custom-harness intro and list mini-swe-agent/kimi-code Frame authoring a harness as "when you need rollout logic the built-ins can't express" rather than "you rarely need this", and add the mini-swe-agent and kimi-code built-ins to the table. Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index 35c0d98a86..9c56c18927 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -492,8 +492,9 @@ The `*_v1` tasksets under `environments/` are the reference library — each sho # Authoring a harness -You rarely need this — a custom harness is for a rollout loop the built-ins can't express -(context compaction, subagents, a bespoke agent CLI). Built-ins, selected with `--harness.id`: +If you need to customize the rollout logic beyond what the built-in harnesses provide — a loop they +can't express (context compaction, subagents, a bespoke agent CLI) — author a custom harness. +Otherwise pick a built-in, selected with `--harness.id`: | id | what it is | | --- | --- | @@ -501,6 +502,8 @@ You rarely need this — a custom harness is for a rollout loop the built-ins ca | `bash` | the `default` chat loop plus a local `bash` tool, for shell-driving agents | | `rlm` | the RLM CLI agent | | `codex` | the Codex CLI (Responses dialect + SSE relay) | +| `mini-swe-agent` | the mini-swe-agent CLI (a minimal SWE agent) | +| `kimi-code` | the Kimi Code CLI agent | ```bash uv run eval gsm8k-v1 -n 1 # default harness From 32d2bcb7d3beb1c7f1b50715cb85eb5dc9cbfa8a Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:31:17 +0000 Subject: [PATCH 17/28] docs: surface the NEEDS_CONTAINER taskset capability flag Mention it in the taskset authoring intro alongside the generic params. Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index 9c56c18927..b1dafb65f8 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -80,6 +80,11 @@ these off the generic bases to type `self.config`, `trace.task`, and `trace.stat The taskset module must export its `Taskset` subclass via `__all__` — the loader walks the exported names and finds the single `Taskset` subclass. +**Capability flag.** A taskset has one class var, `NEEDS_CONTAINER` (default `False`); set it `True` +to declare the taskset only runs in a container runtime (`docker` / `prime`), so the framework +refuses the subprocess runtime up front — the taskset-wide counterpart to a task's per-row `image` +(see [Runtimes](#runtimes)). + ## The task `vf.Task` is a frozen pydantic model. Subclass it to add typed, task-specific fields (the From fcac141bf66af37cabb4f5fe540f538cdbbcec07 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:31:57 +0000 Subject: [PATCH 18/28] refactor: rename harness capability flag SUPPORTS_TASK_TOOLS -> SUPPORTS_MCP Rename the flag on the base Harness and every harness that sets it (built-ins + tau2/compact env harnesses), the Environment check, and align the user-facing error message and the README/GUIDE docs. Co-Authored-By: Claude Opus 4.8 (1M context) --- environments/compact/compact/harness.py | 2 +- environments/tau2_bench_v1/tau2_bench_v1/harness.py | 2 +- packages/harnesses/harnesses/codex/harness.py | 2 +- packages/harnesses/harnesses/kimi_code/harness.py | 2 +- packages/harnesses/harnesses/mini_swe_agent/harness.py | 2 +- packages/harnesses/harnesses/rlm/harness.py | 2 +- packages/harnesses/harnesses/terminus_2/harness.py | 2 +- verifiers/v1/GUIDE.md | 4 ++-- verifiers/v1/README.md | 2 +- verifiers/v1/env.py | 6 +++--- verifiers/v1/harness.py | 2 +- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/environments/compact/compact/harness.py b/environments/compact/compact/harness.py index 382012e807..a75b8ab4a4 100644 --- a/environments/compact/compact/harness.py +++ b/environments/compact/compact/harness.py @@ -25,7 +25,7 @@ class CompactingHarnessConfig(HarnessConfig): class CompactingHarness(Harness[CompactingHarnessConfig]): - SUPPORTS_TASK_TOOLS = True + SUPPORTS_MCP = True async def setup(self, runtime: Runtime) -> None: await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.env) diff --git a/environments/tau2_bench_v1/tau2_bench_v1/harness.py b/environments/tau2_bench_v1/tau2_bench_v1/harness.py index 4c20847d53..3df68ffc55 100644 --- a/environments/tau2_bench_v1/tau2_bench_v1/harness.py +++ b/environments/tau2_bench_v1/tau2_bench_v1/harness.py @@ -51,7 +51,7 @@ class Tau2HarnessConfig(vf.HarnessConfig): class Tau2Harness(vf.Harness[Tau2HarnessConfig]): - SUPPORTS_TASK_TOOLS = False + SUPPORTS_MCP = False async def launch( self, diff --git a/packages/harnesses/harnesses/codex/harness.py b/packages/harnesses/harnesses/codex/harness.py index 057a8075a0..f6f742035a 100644 --- a/packages/harnesses/harnesses/codex/harness.py +++ b/packages/harnesses/harnesses/codex/harness.py @@ -45,7 +45,7 @@ class CodexHarnessConfig(HarnessConfig): class CodexHarness(Harness[CodexHarnessConfig]): APPENDS_SYSTEM_PROMPT = False # TODO - SUPPORTS_TASK_TOOLS = False # TODO + SUPPORTS_MCP = False # TODO async def setup(self, runtime: Runtime) -> None: logger.info("codex: ensuring codex %s is installed", self.config.version) diff --git a/packages/harnesses/harnesses/kimi_code/harness.py b/packages/harnesses/harnesses/kimi_code/harness.py index 8fc4111f8c..c7e55fc74c 100644 --- a/packages/harnesses/harnesses/kimi_code/harness.py +++ b/packages/harnesses/harnesses/kimi_code/harness.py @@ -45,7 +45,7 @@ class KimiCodeHarnessConfig(HarnessConfig): class KimiCodeHarness(Harness[KimiCodeHarnessConfig]): APPENDS_SYSTEM_PROMPT = False - SUPPORTS_TASK_TOOLS = True + SUPPORTS_MCP = True async def setup(self, runtime: Runtime) -> None: logger.info( diff --git a/packages/harnesses/harnesses/mini_swe_agent/harness.py b/packages/harnesses/harnesses/mini_swe_agent/harness.py index acc3914ad9..d68fbe6b18 100644 --- a/packages/harnesses/harnesses/mini_swe_agent/harness.py +++ b/packages/harnesses/harnesses/mini_swe_agent/harness.py @@ -19,7 +19,7 @@ class MiniSWEAgentHarnessConfig(HarnessConfig): class MiniSWEAgentHarness(Harness[MiniSWEAgentHarnessConfig]): APPENDS_SYSTEM_PROMPT = False - SUPPORTS_TASK_TOOLS = False + SUPPORTS_MCP = False async def setup(self, runtime: Runtime) -> None: source = PROGRAM_SOURCE.replace("{version}", self.config.version) diff --git a/packages/harnesses/harnesses/rlm/harness.py b/packages/harnesses/harnesses/rlm/harness.py index edc433de6f..812bc4e184 100644 --- a/packages/harnesses/harnesses/rlm/harness.py +++ b/packages/harnesses/harnesses/rlm/harness.py @@ -37,7 +37,7 @@ class RLMHarnessConfig(HarnessConfig): class RLMHarness(Harness[RLMHarnessConfig]): APPENDS_SYSTEM_PROMPT = True - SUPPORTS_TASK_TOOLS = False + SUPPORTS_MCP = False async def setup(self, runtime: Runtime) -> None: # install.sh fetches curl/uv itself; add git only when the image lacks it. diff --git a/packages/harnesses/harnesses/terminus_2/harness.py b/packages/harnesses/harnesses/terminus_2/harness.py index 71f88fa8a0..4f05241c26 100644 --- a/packages/harnesses/harnesses/terminus_2/harness.py +++ b/packages/harnesses/harnesses/terminus_2/harness.py @@ -21,7 +21,7 @@ class Terminus2HarnessConfig(HarnessConfig): class Terminus2Harness(Harness[Terminus2HarnessConfig]): APPENDS_SYSTEM_PROMPT = True - SUPPORTS_TASK_TOOLS = False + SUPPORTS_MCP = False async def setup(self, runtime: Runtime) -> None: source = PROGRAM_SOURCE.replace("{version}", self.config.version) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index b1dafb65f8..e111d180cd 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -522,7 +522,7 @@ load instead of mis-running: | flag | default | gates | | --- | --- | --- | -| `SUPPORTS_TASK_TOOLS` | `True` | exposes the task's MCP tools to the model (set `False` for a harness with no MCP client) | +| `SUPPORTS_MCP` | `True` | exposes the task's MCP tools to the model (set `False` for a harness with no MCP client) | | `SUPPORTS_USER_SIM` | `False` | drives a task's user simulator (multi-turn user injection) | | `SUPPORTS_MESSAGE_PROMPT` | `False` | accepts a `Messages`-list `task.prompt` (e.g. image-bearing) | | `APPENDS_SYSTEM_PROMPT` | `False` | emits `task.system_prompt` as a real system message (else it's folded into the user prompt with a warning) | @@ -545,7 +545,7 @@ class MyHarnessConfig(vf.HarnessConfig): class MyHarness(vf.Harness[MyHarnessConfig]): - SUPPORTS_TASK_TOOLS = True + SUPPORTS_MCP = True SUPPORTS_USER_SIM = True async def launch(self, ctx, trace, runtime, endpoint, secret, mcp_urls) -> vf.ProgramResult: diff --git a/verifiers/v1/README.md b/verifiers/v1/README.md index 1253124886..92679a4d0a 100644 --- a/verifiers/v1/README.md +++ b/verifiers/v1/README.md @@ -139,7 +139,7 @@ guaranteed cleanup of its resources, even on exit/interrupt. A taskset may expose task-specific tools beyond the tools shipping natively with the harness as MCP servers. Its placement (separate runtime or colocated with harness) is configurable on `taskset.tools` and reachability is handled resolved -automatically. Tools only run under a harness with `SUPPORTS_TASK_TOOLS` (the `default` +automatically. Tools only run under a harness with `SUPPORTS_MCP` (the `default` harness has it; `rlm` doesn't) — an incompatible pairing is refused at load. The tool examples each show one placement: diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index f9d66aa788..7113b2af81 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -237,13 +237,13 @@ def __init__(self, config: EnvConfig) -> None: self.taskset = load_taskset(config.taskset) self.harness = load_harness(config.harness) if ( - not self.harness.SUPPORTS_TASK_TOOLS + not self.harness.SUPPORTS_MCP and type(self.taskset).tools is not Taskset.tools ): raise ValueError( - f"Harness {self.harness.config.id!r} does not support task tools, but taskset " + f"Harness {self.harness.config.id!r} does not support MCP tools, but taskset " f"{self.taskset.config.id!r} exposes tool servers (MCP). Run it with a harness " - f"that supports task tools (e.g. --harness.id default), or use a taskset without tools." + f"that supports MCP (e.g. --harness.id default), or use a taskset without tools." ) if ( not self.harness.SUPPORTS_USER_SIM diff --git a/verifiers/v1/harness.py b/verifiers/v1/harness.py index 60b34da604..70f5831280 100644 --- a/verifiers/v1/harness.py +++ b/verifiers/v1/harness.py @@ -69,7 +69,7 @@ class Harness(ABC, Generic[ConfigT]): APPENDS_SYSTEM_PROMPT: ClassVar[bool] = False """Emit task.system_prompt as a system message. If False, a task that sets a system_prompt is rejected.""" - SUPPORTS_TASK_TOOLS: ClassVar[bool] = True + SUPPORTS_MCP: ClassVar[bool] = True """Expose a task's MCP tool servers to the model; set False for harnesses without an MCP client.""" SUPPORTS_USER_SIM: ClassVar[bool] = False """Drive a task's user simulator (multi-turn user injection); opt in per harness.""" From 83d2cdfa6220893a900607c452fcc3a4e6dcec6a Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:32:56 +0000 Subject: [PATCH 19/28] docs: harness example configures via CLI args, drops max_steps Pass endpoint/secret/model as CLI args (as the built-in default harness does) instead of OPENAI_* env vars, which have footguns; note why. Drop the max_steps knob (turn limits are framework-enforced, not a harness concern) and the id pin (harness configs don't pin id), leaving a placeholder config. Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index e111d180cd..366fecc4c1 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -529,9 +529,8 @@ load instead of mis-running: ## Writing one -Define a `HarnessConfig` (its `id` plus any knobs, which surface as `--harness.*`), subclass -`vf.Harness[ConfigT]`, declare the capability flags, and implement `launch`. Export the class via -`__all__`. +Define a `HarnessConfig` (any knobs surface as `--harness.*`), subclass `vf.Harness[ConfigT]`, +declare the capability flags, and implement `launch`. Export the class via `__all__`. ```python import verifiers.v1 as vf @@ -540,8 +539,7 @@ PROGRAM = (Path(__file__).parent / "program.py").read_text() # a uv script, dep class MyHarnessConfig(vf.HarnessConfig): - id: str = "my-harness" - max_steps: int = 50 # a harness-specific knob; surfaces as --harness.max-steps + """Run knobs for this harness (surface as --harness.*).""" class MyHarness(vf.Harness[MyHarnessConfig]): @@ -550,12 +548,15 @@ class MyHarness(vf.Harness[MyHarnessConfig]): async def launch(self, ctx, trace, runtime, endpoint, secret, mcp_urls) -> vf.ProgramResult: system, prompt = self.resolve_prompt(trace.task) - env = {"OPENAI_BASE_URL": endpoint, "OPENAI_API_KEY": secret, - "OPENAI_MODEL": ctx.model, "SYSTEM_PROMPT": system or "", - "MAX_STEPS": str(self.config.max_steps)} + # configure the program with CLI args, not OPENAI_* env vars (less footgun-prone) + args = [f"--base-url={endpoint}", f"--api-key={secret}", f"--model={ctx.model}"] + if system: + args.append(f"--system-prompt={system}") if mcp_urls: # standard mcpServers map the program connects to - env["MCP_CONFIG"] = json.dumps({"mcpServers": {n: {"url": u} for n, u in mcp_urls.items()}}) - return await runtime.run_uv_script(PROGRAM, args=[prompt], env=env) + args.append("--mcp-config=" + json.dumps({"mcpServers": {n: {"url": u} for n, u in mcp_urls.items()}})) + if prompt is not None: + args.append(f"--prompt={prompt}") + return await runtime.run_uv_script(PROGRAM, args=args, env=self.config.env) __all__ = ["MyHarness"] @@ -590,9 +591,10 @@ user message). **Two program styles.** A self-contained chat loop is usually a single-file uv script (`runtime.run_uv_script`, so the harness needs only `uv` in the runtime — its inline deps resolve there, never on the host; identical scripts share one content-addressed uv env). An agent -CLI / binary is installed and launched with `runtime.run(...)`. Either way, harness-owned env vars -(`OPENAI_BASE_URL` / `OPENAI_API_KEY` / `OPENAI_MODEL`, …) are spread *after* `self.config.env`, so -they take precedence over any collision. +CLI / binary is installed and launched with `runtime.run(...)`. Either way, pass `endpoint` / +`secret` / `ctx.model` to the program as **CLI args** (as above) rather than `OPENAI_*` env vars — an +inherited or stray env var can silently redirect the program's model calls; `self.config.env` just +supplies any extra environment. **Harness metrics.** A harness can define its own `@vf.metric` methods (injected `task` / `trace` / `runtime`), run over the finished trace alongside the taskset's — handy to surface what the program From a2d1e761064887ab6c1f20d2d17bb5dd6cdacafb Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:34:59 +0000 Subject: [PATCH 20/28] docs: split the harness "Writing one" section into subsections Break the wall of bold-lead paragraphs into ### subsections (contract, launch, resolve_prompt, program styles, harness metrics), matching the taskset docs' ##/### structure. Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 46 +++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index 366fecc4c1..0a85b575c8 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -562,10 +562,14 @@ class MyHarness(vf.Harness[MyHarnessConfig]): __all__ = ["MyHarness"] ``` -**The contract.** A harness never builds the trace itself: it just points *a program* at -`endpoint` (authorized with `secret`), and the interception server records every model call — -**as long as the program makes its requests in one of the supported dialects** (chat-completions, -Responses, Anthropic Messages). The program can be any executable the runtime can run. +### The contract + +A harness never builds the trace itself: it just points *a program* at `endpoint` (authorized with +`secret`), and the interception server records every model call — **as long as the program makes its +requests in one of the supported dialects** (chat-completions, Responses, Anthropic Messages). The +program can be any executable the runtime can run. + +### The `launch` hook `launch` receives: @@ -582,28 +586,28 @@ becomes `trace.stop("agent_completed")`; a non-zero exit (or an unexpected excep raises `HarnessError` with the tail of stderr — **unless** a `@stop` already fired (the program dying because the interception server cut a turn is expected, not an error). -**`resolve_prompt(trace.task)`** returns `(system_prompt, prompt)` already reconciled with your +### `resolve_prompt` + +`resolve_prompt(trace.task)` returns `(system_prompt, prompt)` already reconciled with your capability flags: the system prompt is handed back only if `APPENDS_SYSTEM_PROMPT` (else folded into `prompt`); `prompt` is a `str`, a `Messages` list (if `SUPPORTS_MESSAGE_PROMPT`), or `None` (no prompt → let the user simulator / interception server open the conversation, so send no opening user message). -**Two program styles.** A self-contained chat loop is usually a single-file uv script -(`runtime.run_uv_script`, so the harness needs only `uv` in the runtime — its inline deps resolve -there, never on the host; identical scripts share one content-addressed uv env). An agent -CLI / binary is installed and launched with `runtime.run(...)`. Either way, pass `endpoint` / -`secret` / `ctx.model` to the program as **CLI args** (as above) rather than `OPENAI_*` env vars — an -inherited or stray env var can silently redirect the program's model calls; `self.config.env` just -supplies any extra environment. - -**Harness metrics.** A harness can define its own `@vf.metric` methods (injected `task` / `trace` / -`runtime`), run over the finished trace alongside the taskset's — handy to surface what the program -left behind in the runtime (e.g. read a `meta.json` the binary wrote). A harness can't define -rewards. - -Copy `environments/compact` (a context-rewrite loop) as a starting point. A harness is resolved by -its `id` the same way a taskset is — built-ins live under `packages/harnesses/harnesses//`; a -custom one is a local package or a Hub id. +### Program styles + +A self-contained chat loop is usually a single-file uv script (`runtime.run_uv_script`, so the +harness needs only `uv` in the runtime — its inline deps resolve there, never on the host; identical +scripts share one content-addressed uv env). An agent CLI / binary is installed and launched with +`runtime.run(...)`. Either way, pass `endpoint` / `secret` / `ctx.model` to the program as **CLI +args** (as above) rather than `OPENAI_*` env vars — an inherited or stray env var can silently +redirect the program's model calls; `self.config.env` just supplies any extra environment. + +### Harness metrics + +A harness can define its own `@vf.metric` methods (injected `task` / `trace` / `runtime`), run over +the finished trace alongside the taskset's — handy to surface what the program left behind in the +runtime (e.g. read a `meta.json` the binary wrote). A harness can't define rewards. --- From edcc52ee09af0135fe4607e0e69e0fb0568841cb Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:38:46 +0000 Subject: [PATCH 21/28] docs: correct the tau2-bench-v1 example description tau2's taskset defines no vf.User/vf.Toolset; it bundles its own harness that runs the whole tau2 simulation in a subprocess and stores the result in trace.info. It's the bundled-harness example, not a tools+user-sim one. Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/GUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index 0a85b575c8..ab2f7d3b71 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -487,7 +487,7 @@ The `*_v1` tasksets under `environments/` are the reference library — each sho | `wiki-search-v1` | a shared, read-only tool server (built once) + an LLM judge | | `scratchpad-v1` | a shared, **writable** tool server — per-rollout state isolated via `self.state` | | `deepwiki-v1` | an existing remote tool server, by URL | -| `tau2-bench-v1` | **both** tools and a user simulator at once, with a bundled harness (the tau2 benchmark) | +| `tau2-bench-v1` | a taskset that **bundles its own harness** (runs the whole tau2 benchmark sim in a subprocess, result into `trace.info`) | | `color-codeword-v1` | a multimodal (image) task | | `scaleswe-v1`, `swelego-v1`, `r2e-gym-v1` | containerized SWE tasks (rlm harness, prime runtime) | | `swebench-verified-v1` | SWE-bench Verified via `harbor-v1` on prime's prebuilt images (no Dockerfile build) | From 19fa3409d764bfd70f0cfdaf785cb2b7ebb0c524 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 19 Jun 2026 21:44:36 +0000 Subject: [PATCH 22/28] chore: drop internal bench/ scripts and COMPARE.md from v1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These are work-in-progress benchmarking scripts and a transient #1559-vs-#1576 comparison doc — not part of the public v1 surface. Co-Authored-By: Claude Opus 4.8 (1M context) --- COMPARE.md | 46 ---------------------------------- bench/.gitignore | 1 - bench/agentic_benchmark.sh | 51 -------------------------------------- bench/bench_aggregate.py | 44 -------------------------------- bench/benchmark.sh | 47 ----------------------------------- 5 files changed, 189 deletions(-) delete mode 100644 COMPARE.md delete mode 100644 bench/.gitignore delete mode 100755 bench/agentic_benchmark.sh delete mode 100644 bench/bench_aggregate.py delete mode 100755 bench/benchmark.sh diff --git a/COMPARE.md b/COMPARE.md deleted file mode 100644 index b1b5a490e6..0000000000 --- a/COMPARE.md +++ /dev/null @@ -1,46 +0,0 @@ -# v1 PRs: #1559 vs #1576 - -Two open v1 refactors of verifiers (the `Taskset × Harness × Runtime` model), both against `main`. - -| | **#1559** — `codex/v1-nano-refactor-draft` | **#1576** — `feat/nano-as-v1` | -|---|---|---| -| Size | +17.6k / −34.1k, 422 files | +11.5k / −50k, 401 files (re-vendors vf-nano) | -| Thesis | Broad v1 surface — many harnesses, in-tree advantages, nested subagents | v0↔v1 bridge + training-readiness — legacy bridge, message graph, multiplexing, benchmarked | -| Rollout record | `State` + flat `Turn` list (serializable, no graph) | delta-native `MessageNode` graph (branches via leaves→root) | -| RL contract | token-level **advantages computed in-lib** (`@advantage`) | trainer (prime-rl) computes advantages; lib exposes trainable `Trace` | - -## Parity — supported by both - -- Core `Taskset × Harness × Runtime` over a typed (pydantic) rollout model -- Runtimes: **subprocess, docker, prime** -- Harnesses: a **default chat harness** + **rlm** -- Taskset authoring: `load_tasks` + `@reward` / `@metric` / `@stop`, **group rewards**, **runtime-based (in-sandbox) scoring**, per-task **image + resources** -- **MCP tool servers** exposed to the model + a first-class **user simulator** (framework-injected user turns) -- **Eval CLI + TOML config** (runtime/harness selected by config) -- **Trainable rollouts** — per-turn token ids + logprobs + mask -- **Interception server** proxying model calls; SIGTERM → graceful teardown -- v1 **unit tests** + live **eval reward** checks - -## Only in #1559 - -- Harness ecosystem: **`CommandHarness`** (agentic-CLI base) + **MiniSWEAgent / OpenCode / Pi / Terminus2 / Replay / NeMoGym** -- **In-tree token-level advantages** — `@advantage` (grpo / rloo / reinforce / sft) writing `Turn.tokens.*_advantages`; `advantage=None` defers to a trainer -- **Nested harnesses / subagents** — `Harness.run(context=parent)` reuses the parent's runtime, clients, toolsets -- **Richer MCP** — placement `dedicated` / `colocated` / `remote` × scope `rollout` / `env` (refcounted, start-once) + **bound-arg tools** (`args`/`sets`/`extends`) that hide state plumbing from the model -- **Replay harness** (SFT) - -## Only in #1576 - -- **Legacy v0 bridge** (`LegacyEnvServer`) — runs classic v0 envs over the **same ZMQ protocol** as native v1, indistinguishable to the trainer; token ids/logprobs carried 1:1; group scoring; eval-split fallback; renderer (train) vs chat-completions (eval) client dispatch. *The headline ("nano bridge").* -- **Delta-native message graph** — each message stored once, branches recovered leaves→root (linear, not quadratic in turns); one training sample per branch -- **Interception multiplexing** (`InterceptionPool`, `multiplex=32`) — N rollouts share servers + tunnels, to beat prime's 512/min tunnel cap -- **ZMQ env server is the v1 training path** (native + bridge both serve over it); prime-rl drives it [#1559's ZMQ is v0-only; its v1 eval runs in-process] -- **Modal runtime functional** (4 working runtimes vs 3 + 2 stubs) -- Framework-enforced limits (`max_turns` / token budgets / `@stop`) applied **harness-agnostically** in the interception server -- **Runtime + multiplex benchmark** (`bench/`) with committed numbers - -> Excluded from #1576's tip via reverts (in separate review — verifiers#1618): multimodal/VLM, user-sim colocation, color-codeword taskset. - ---- - -*Net:* **#1559** is the broader feature surface (harness ecosystem, in-lib advantages, nested subagents, richer MCP). **#1576** is narrower but is the only one that bridges v0→v1. diff --git a/bench/.gitignore b/bench/.gitignore deleted file mode 100644 index e33609d251..0000000000 --- a/bench/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.png diff --git a/bench/agentic_benchmark.sh b/bench/agentic_benchmark.sh deleted file mode 100755 index 39cf974e79..0000000000 --- a/bench/agentic_benchmark.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -# Agentic benchmark: run ONE harbor task at group sizes (-r) across env-server modes and -# write bench/agentic_benchmark.json (per-rollout durations + e2e wall clock), which -# bench/agentic_aggregate.py summarizes. Each rollout is its own sandbox (a coding agent + -# the harbor verifier); with no group reward the rollouts are independent, so the worker -# pool round-robins them across workers — this stresses concurrent agentic execution + -# scoring (where the single-loop server is most likely to stall). -# -# bench/agentic_benchmark.sh -# ROLLOUTS="8 16" WORKERS="0 4" TASK=fix-git bench/agentic_benchmark.sh -# -# Compares WORKERS modes: 0 = single in-process server, N = an N-worker pool. Needs the -# `harbor` CLI (`uv tool install harbor`) and the `terminal-bench-2-v1` example taskset -# (an editable dep), plus a container runtime (prime default; PRIME_API_KEY in ~/.env). -set -uo pipefail - -TASKSET="${TASKSET:-terminal-bench-2-v1}" -TASK="${TASK:-fix-git}" -RUNTIME="${RUNTIME:-prime}" -ROLLOUTS="${ROLLOUTS:-32 64 128}" -WORKERS="${WORKERS:-0 4}" -MODEL="${MODEL:-deepseek/deepseek-v4-flash}" -MAX_TURNS="${MAX_TURNS:-30}" - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT" -set -a; . "$HOME/.env" 2>/dev/null || true; set +a - -OUT="/tmp/vbench/agentic" -rm -rf "$OUT"; mkdir -p "$OUT"; : > "$OUT/e2e.txt" -for w in $WORKERS; do - for r in $ROLLOUTS; do - label="w$w-r$r" - echo "== $label (task=$TASK runtime=$RUNTIME max_turns=$MAX_TURNS) ==" - start=$(date +%s) - uv run eval "$TASKSET" --taskset.tasks "[\"$TASK\"]" \ - --harness.id default --harness.enable_bash true --harness.runtime.type "$RUNTIME" \ - --num_tasks 1 --num_rollouts "$r" --num_workers "$w" \ - --max_concurrent 512 --retry.attempts 1 --max_turns "$MAX_TURNS" \ - --rich false --output_dir "$OUT/$label" \ - > "$OUT/$label.stdout" 2> "$OUT/$label.log" - rc=$? - echo "$w $r $(( $(date +%s) - start ))" >> "$OUT/e2e.txt" - echo "rc=$rc e2e=$(tail -1 "$OUT/e2e.txt" | awk '{print $3}')s" - done -done - -# Aggregate into agentic_benchmark.json: per-(workers, rollouts) e2e + the per-rollout -# generation-duration list (p10/p50/p90), reward, and error count. -uv run python bench/bench_aggregate.py "$OUT" "$TASK ($RUNTIME, max_turns=$MAX_TURNS)" > "$OUT/agentic_benchmark.json" -echo "wrote $OUT/agentic_benchmark.json" diff --git a/bench/bench_aggregate.py b/bench/bench_aggregate.py deleted file mode 100644 index 854be00a74..0000000000 --- a/bench/bench_aggregate.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Aggregate a worker-pool benchmark run (single-turn or agentic) into JSON. - - python bench/bench_aggregate.py