Skip to content
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
c609133
feat(v1): move the runtime config from the harness onto the agent
mikasenghaas Jul 22, 2026
fa82029
feat(v1): stamp the resolved agent config onto the trace
mikasenghaas Jul 22, 2026
14e0a65
fix(v1): keep the provisioned box on debug trace records
mikasenghaas Jul 22, 2026
b33b339
Merge origin/main (pluggable skills, NEEDS_CONTAINER)
mikasenghaas Jul 22, 2026
1cc6917
refactor(v1): keep AgentConfig in agent.py; drop the runtime migratio…
mikasenghaas Jul 22, 2026
f63d1e2
chore(v1): drop redundant TYPE_CHECKING comment
mikasenghaas Jul 22, 2026
af522f6
Merge origin/main (interleaving agents)
mikasenghaas Jul 22, 2026
969be85
refactor(v1): type AgentInfo.config as AgentConfig directly
mikasenghaas Jul 22, 2026
15298ce
refactor(v1): resolve AgentInfo.config at the package root
mikasenghaas Jul 22, 2026
3832dd2
refactor(v1): AgentInfo holds the identity subset, not the agent config
mikasenghaas Jul 22, 2026
c8ee8cf
refactor(v1): separate configs from logic in a configs package
mikasenghaas Jul 23, 2026
9e69dff
feat(v1): stamp the resolved agent config onto the trace
mikasenghaas Jul 23, 2026
20d01f0
feat(v1): read trace records plugin-free via wire agent/harness configs
mikasenghaas Jul 23, 2026
c5c62d0
Merge remote-tracking branch 'origin/main' into feat/agent-runtime-co…
mikasenghaas Jul 23, 2026
9b4cd26
Merge branch 'main' of github.com:PrimeIntellect-ai/verifiers into fe…
mikasenghaas Jul 23, 2026
9c27ec5
refactor(v1): make the strict/wire record read an explicit choice
mikasenghaas Jul 23, 2026
9d158c8
ci: pin ruff-action to the locked ruff version
mikasenghaas Jul 23, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion configs/agentic_judge.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,5 @@ id = "null"
[env.judge]
model = "z-ai/glm-5.2"

[env.judge.harness.runtime]
[env.judge.runtime]
type = "docker"
2 changes: 1 addition & 1 deletion configs/gsm8k_rlm.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ id = "gsm8k-v1"
id = "rlm"
version = "main"

[env.agent.harness.runtime]
[env.agent.runtime]
type = "docker"
4 changes: 3 additions & 1 deletion configs/harbor.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ num_tasks = 10
id = "harbor"
dataset = "terminal-bench/terminal-bench-2"

[env.agent]
runtime = { type = "docker" }

[env.agent.harness]
id = "bash"
runtime = { type = "docker" }
4 changes: 2 additions & 2 deletions docs/v1/evaluation.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ id = "primeintellect/terminal-bench-2"
id = "codex"
version = "0.116.0"

[env.agent.harness.runtime]
[env.agent.runtime]
type = "docker"
```

Expand Down Expand Up @@ -82,7 +82,7 @@ Docker harnesses can keep trusted setup online, then restrict the agent to decla
HTTP(S) destinations:

```toml
[env.agent.harness.runtime]
[env.agent.runtime]
type = "docker"
allow = ["https://*.wikipedia.org"]
block = ["https://upload.wikimedia.org"]
Expand Down
3 changes: 2 additions & 1 deletion docs/v1/harnesses.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ verifiers supports a range of harnesses out of the box, including Claude Code, C

```python
from verifiers.v1.clients import ModelContext
from verifiers.v1.harness import Harness, HarnessConfig
from verifiers.v1.configs.harness import HarnessConfig
from verifiers.v1.harness import Harness
from verifiers.v1.runtimes import ProgramResult, Runtime
from verifiers.v1.task import TaskData
from verifiers.v1.trace import Trace
Expand Down
3 changes: 2 additions & 1 deletion environments/compact/compact/harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@
from pathlib import Path

from verifiers.v1.clients import ModelContext
from verifiers.v1.harness import Harness, HarnessConfig
from verifiers.v1.configs.harness import HarnessConfig
from verifiers.v1.harness import Harness
from verifiers.v1.runtimes import ProgramResult, Runtime
from verifiers.v1.task import TaskData
from verifiers.v1.trace import Trace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
in a real sandbox and keep the solvers on a cheap tool-less chat loop —

uv run eval proposer-solver-v1 -n 4 \
--env.proposer.harness.id codex --env.proposer.harness.runtime.type prime \
--env.proposer.harness.id codex --env.proposer.runtime.type prime \
--env.solver.harness.id null

Train-side, the seats flip independently per run (`--env.train_solver false`
Expand Down
2 changes: 1 addition & 1 deletion examples/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ async def main() -> None:
last = trace.assistant_messages[-1].content if trace.assistant_messages else None
print("answer:", last)
assert trace.agent is not None
print("agent:", trace.agent.name, trace.agent.model)
print("agent:", trace.agent.name, trace.agent.config.model)
print("runtime:", trace.runtime.type if trace.runtime else None)


Expand Down
14 changes: 8 additions & 6 deletions skills/evaluate-environments/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ When the user requests a full run, do not restrict the number of tasks. Ask for
The leading ID is shorthand for `--env.taskset.id`. A harness belongs to an agent — `--env.agent.harness.*` on the single-agent env, `--env.<agent>.harness.*` on a multi-agent one (there is no run-level `--harness.*`):

```bash
prime eval run owner/name --env.agent.harness.id codex --env.agent.harness.runtime.type prime
prime eval run owner/name --env.agent.harness.id codex --env.agent.runtime.type prime
```

The env — the control flow between agents — owns the whole `[env]` block. Empty `--env.id`
Expand All @@ -61,7 +61,7 @@ env); `--env.id` pairs a reusable env with any taskset, its knobs typed under `-
```bash
prime eval run my-task-v1 --env.id best-of-n --env.n 8 # pass@k / rejection sampling
prime eval run my-task-v1 --env.id agentic-judge \
--env.judge.harness.runtime.type docker # a judge agent verifies each attempt in a sandbox
--env.judge.runtime.type docker # a judge agent verifies each attempt in a sandbox
```

When specifying Hub tasksets, always include the owner to resolve them correctly.
Expand Down Expand Up @@ -90,9 +90,9 @@ Harness and runtime settings:
```bash
prime eval run my-task-v1 \
--env.agent.harness.id rlm \
--env.agent.harness.runtime.type docker \
--env.agent.harness.runtime.cpu 4 \
--env.agent.harness.runtime.memory 8
--env.agent.runtime.type docker \
--env.agent.runtime.cpu 4 \
--env.agent.runtime.memory 8
```

Sampling:
Expand Down Expand Up @@ -122,9 +122,11 @@ model = "openai/gpt-5-mini"
id = "my-task-v1"
split = "test"

[env.agent]
runtime = { type = "subprocess" }

[env.agent.harness]
id = "bash"
runtime = { type = "subprocess" }

[sampling]
temperature = 0.7
Expand Down
38 changes: 19 additions & 19 deletions skills/evaluate-environments/references/REFERENCE.md

Large diffs are not rendered by default.

18 changes: 10 additions & 8 deletions tests/v1/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
import pytest

import verifiers.v1 as vf
from verifiers.v1.configs.eval import EvalConfig
from verifiers.v1.configs.cli.eval import EvalConfig
from verifiers.v1.loaders import load_environment
from verifiers.v1.cli.eval.runner import run_eval
from verifiers.v1.trace import Trace
Expand Down Expand Up @@ -123,16 +123,16 @@ def _eval_config(
max_turns: int | None = 4,
rollout_timeout: float = 180,
taskset_overrides: dict | None = None,
harness_overrides: dict | None = None,
runtime: dict | None = None,
env: dict | None = None,
pool: dict | None = None,
model: str | None = None,
reasoning_effort: str | None = None,
) -> EvalConfig:
"""Build the smallest `EvalConfig` that still exercises the path, shared by the in-process
(`run_v1`) and env-server (`run_v1_server`) fixtures. `taskset_overrides` / `harness_overrides`
are merged onto the `{id: ...}` config (placement, runtime, etc.); `model` overrides the default
text model (e.g. a VLM for an image task).
(`run_v1`) and env-server (`run_v1_server`) fixtures. `taskset_overrides` merges onto the
`{id: ...}` config; `runtime` places the `agent` seat's harness (an agent field, not a
harness one); `model` overrides the default text model (e.g. a VLM for an image task).

`temperature=0` (greedy) makes the run reproducible; `max_tokens` is generous headroom,
not a target — these trivial tasks finish in a few hundred tokens, so capping tighter only
Expand All @@ -145,9 +145,11 @@ def _eval_config(
env_cfg = dict(env or {})
_configure_prime_runtimes(taskset_cfg)
if harness:
harness_cfg = {"id": harness, **(harness_overrides or {})}
_configure_prime_runtimes(harness_cfg)
env_cfg.setdefault("agent", {})["harness"] = harness_cfg
env_cfg.setdefault("agent", {})["harness"] = {"id": harness}
if runtime:
runtime_cfg = dict(runtime)
_configure_prime_runtimes(runtime_cfg)
env_cfg.setdefault("agent", {})["runtime"] = runtime_cfg
# Per-run caps live on the seats: resolve the env's declared roles and cap
# each one (a test's own seat dict wins over the shared defaults).
config_cls = vf.env_config_type(taskset, env_cfg.get("id", ""))
Expand Down
2 changes: 1 addition & 1 deletion tests/v1/test_configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

import pytest

from verifiers.v1.configs.eval import EvalConfig
from verifiers.v1.configs.cli.eval import EvalConfig

CONFIGS = sorted(
p
Expand Down
26 changes: 13 additions & 13 deletions tests/v1/test_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
(trace,) = await run_v1(
"echo-v1",
harness=harness,
harness_overrides={"runtime": {"type": harness_runtime}},
runtime={"type": harness_runtime},
output_dir=tmp_path,
max_turns=2,
)
Expand All @@ -109,7 +109,7 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
assert trace.stop_condition == "agent_completed"
assert trace.reward == 1.0
# The seat's resolved identity rides the trace (policy metadata for trainers).
assert trace.agent is not None and trace.agent.sampling.temperature == 0
assert trace.agent is not None and trace.agent.config.sampling.temperature == 0
# Every sampled turn has one per-call record, linked to its assistant node.
sampled = [i for i, n in enumerate(trace.nodes) if n.sampled]
assert [c.node for c in trace.calls if c.error is None] == sampled
Expand All @@ -130,7 +130,7 @@ async def test_user(run_v1, harness_runtime, tmp_path):
(trace,) = await run_v1(
"echo-user-sim-v1",
harness="null",
harness_overrides={"runtime": {"type": harness_runtime}},
runtime={"type": harness_runtime},
output_dir=tmp_path,
max_turns=6,
)
Expand Down Expand Up @@ -189,7 +189,7 @@ async def test_acp_resume_with_tool(run_v1, harness, harness_runtime, tmp_path):
(trace,) = await run_v1(
"echo-acp-resume-v1",
harness=harness,
harness_overrides={"runtime": {"type": harness_runtime}},
runtime={"type": harness_runtime},
output_dir=tmp_path,
max_turns=8,
max_tokens=8192,
Expand Down Expand Up @@ -219,7 +219,7 @@ async def test_tool(run_v1, harness_runtime, tool_runtime, tmp_path):
(trace,) = await run_v1(
"echo-tool-v1",
harness="null",
harness_overrides={"runtime": {"type": harness_runtime}},
runtime={"type": harness_runtime},
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
output_dir=tmp_path,
max_turns=6,
taskset_overrides={"task": {"tools": tool_runtime}},
Expand All @@ -246,7 +246,7 @@ async def test_tool_state(run_v1, harness_runtime, tool_runtime, tmp_path):
(trace,) = await run_v1(
"counter-tool-v1",
harness="null",
harness_overrides={"runtime": {"type": harness_runtime}},
runtime={"type": harness_runtime},
output_dir=tmp_path,
max_turns=8,
taskset_overrides={"task": {"tools": tool_runtime}},
Expand All @@ -267,7 +267,7 @@ async def test_shared_tool_isolation(
traces = await run_v1_server(
"scratchpad-v1",
harness="null",
harness_overrides={"runtime": {"type": harness_runtime}},
runtime={"type": harness_runtime},
output_dir=tmp_path,
num_tasks=2,
n=1,
Expand All @@ -287,7 +287,7 @@ async def test_tool_response_image(run_v1, tmp_path):
(trace,) = await run_v1(
"tool-response-image-v1",
harness="null",
harness_overrides={"runtime": {"type": "subprocess"}},
runtime={"type": "subprocess"},
model="openai/gpt-5.6-luna",
reasoning_effort="none",
output_dir=tmp_path,
Expand All @@ -314,7 +314,7 @@ async def test_rubric_judge(run_v1, tmp_path):
(trace,) = await run_v1(
"echo-v1",
harness="null",
harness_overrides={"runtime": {"type": "subprocess"}},
runtime={"type": "subprocess"},
output_dir=tmp_path,
taskset_overrides={"task": {"judges": [{"id": "rubric", "path": str(rubric)}]}},
max_turns=2,
Expand All @@ -332,7 +332,7 @@ async def test_agentic(run_v1, harness, harness_runtime, tmp_path):
(trace,) = await run_v1(
"echo-agentic-v1",
harness=harness,
harness_overrides={"runtime": {"type": harness_runtime}},
runtime={"type": harness_runtime},
output_dir=tmp_path,
max_turns=10,
)
Expand Down Expand Up @@ -406,7 +406,7 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
# The judge reads the transcript and reasons before it writes the
# verdict file; the shared 2048-token run cap truncates it mid-audit.
"judge": {
"harness": {"runtime": {"type": "docker"}},
"runtime": {"type": "docker"},
"max_output_tokens": 8192,
},
},
Expand Down Expand Up @@ -551,13 +551,13 @@ async def test_replay_round_trip(run_v1, tmp_path):

from verifiers.v1.cli.output import CONFIG_FILE
from verifiers.v1.cli.replay import run_replay
from verifiers.v1.configs.replay import ReplayConfig
from verifiers.v1.configs.cli.replay import ReplayConfig

run_dir = tmp_path / "run"
(source,) = await run_v1(
"reverse-text-v1",
harness="null",
harness_overrides={"runtime": {"type": "subprocess"}},
runtime={"type": "subprocess"},
output_dir=run_dir,
max_turns=2,
)
Expand Down
4 changes: 2 additions & 2 deletions tests/v1/test_legacy.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ async def test_v0_single_turn_matches_v1_shape(run_v0, run_v1, tmp_path):
(v0,) = await run_v0("echo-v0", output_dir=tmp_path / "v0")
(v1,) = await run_v1(
"echo-v1",
harness_overrides={"runtime": {"type": "subprocess"}},
runtime={"type": "subprocess"},
output_dir=tmp_path / "v1",
max_turns=2,
)
Expand All @@ -36,7 +36,7 @@ async def test_v0_multi_turn_matches_v1_shape(run_v0, run_v1, tmp_path):
(v0,) = await run_v0("echo-multi-v0", output_dir=tmp_path / "v0")
(v1,) = await run_v1(
"echo-v1",
harness_overrides={"runtime": {"type": "subprocess"}},
runtime={"type": "subprocess"},
output_dir=tmp_path / "v1",
max_turns=2,
)
Expand Down
47 changes: 14 additions & 33 deletions verifiers/v1/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,25 +13,16 @@
resolve_client,
)
from verifiers.v1.decorators import metric, reward, stop, tool
from verifiers.v1.agent import (
Agent,
AgentConfig,
Agents,
Interaction,
Segment,
make_agent,
)
from verifiers.v1.configs.env import (
from verifiers.v1.configs.agent import AgentConfig
from verifiers.v1.agent import Agent, Agents, Interaction, Segment, make_agent
from verifiers.v1.configs.cli.env import (
ElasticPoolConfig,
EnvServerConfig,
StaticPoolConfig,
pool_serve_kwargs,
)
from verifiers.v1.env import (
EnvConfig,
Env,
default_agent_harness,
)
from verifiers.v1.configs.env import EnvConfig, default_agent_harness
from verifiers.v1.env import Env
from verifiers.v1.envs.single_agent import SingleAgentEnv, SingleAgentEnvConfig
from verifiers.v1.errors import (
EnvError,
Expand All @@ -44,15 +35,10 @@
ToolsetError,
TunnelError,
)
from verifiers.v1.harness import Harness, HarnessConfig
from verifiers.v1.judge import (
Judge,
JudgeConfig,
JudgeResponse,
Judges,
JudgeSamplingConfig,
JudgeView,
)
from verifiers.v1.configs.harness import HarnessConfig
from verifiers.v1.harness import Harness
from verifiers.v1.configs.judge import JudgeConfig, JudgeSamplingConfig, Judges
from verifiers.v1.judge import Judge, JudgeResponse, JudgeView
from verifiers.v1.judges import (
ReferenceJudge,
ReferenceJudgeConfig,
Expand Down Expand Up @@ -86,7 +72,7 @@
read_answer_file_or_last_reply as read_answer_file_or_last_reply,
verify_boxed_math_answer as verify_boxed_math_answer,
)
from verifiers.v1.retries import RetryConfig
from verifiers.v1.configs.retries import RetryConfig
from verifiers.v1.utils.git import (
PATCH_CAP_BYTES as PATCH_CAP_BYTES,
capture_patch as capture_patch,
Expand All @@ -102,15 +88,10 @@
SubprocessConfig,
)
from verifiers.v1.state import State, StateT
from verifiers.v1.task import (
Task,
TaskConfig,
TaskData,
TaskResources,
TaskTimeout,
WireTaskData,
)
from verifiers.v1.taskset import Taskset, TasksetConfig
from verifiers.v1.configs.task import TaskConfig
from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout, WireTaskData
from verifiers.v1.configs.taskset import TasksetConfig
from verifiers.v1.taskset import Taskset
from verifiers.v1.mcp import (
Toolset,
SharedToolsetConfig,
Expand Down
Loading
Loading