feat: move the runtime onto the agent and stamp the agent config onto the trace

.github/workflows/style.yml

@@ -18,11 +18,15 @@ jobs:
         uses: actions/checkout@v5
         with:
           submodules: true
+      # Pin to the ruff uv.lock resolves, so CI and `uv run ruff` agree.
       - name: Lint
         uses: astral-sh/ruff-action@v4.1.0
+        with:
+          version: "0.15.21"
       - name: Format
         uses: astral-sh/ruff-action@v4.1.0
         with:
+          version: "0.15.21"
           args: "format --check"
   ty:
     name: Ty

configs/agentic_judge.toml

@@ -23,5 +23,5 @@ id = "null"
 [env.judge]
 model = "z-ai/glm-5.2"
 
-[env.judge.harness.runtime]
+[env.judge.runtime]
 type = "docker"

configs/gsm8k_rlm.toml

@@ -16,5 +16,5 @@ id = "gsm8k-v1"
 id = "rlm"
 version = "main"
 
-[env.agent.harness.runtime]
+[env.agent.runtime]
 type = "docker"

configs/harbor.toml

@@ -8,6 +8,8 @@ num_tasks = 10
 id = "harbor"
 dataset = "terminal-bench/terminal-bench-2"
 
+[env.agent]
+runtime = { type = "docker" }
+
 [env.agent.harness]
 id = "bash"
-runtime = { type = "docker" }

docs/v1/evaluation.md

@@ -21,7 +21,7 @@ id = "primeintellect/terminal-bench-2"
 id = "codex"
 version = "0.116.0"
 
-[env.agent.harness.runtime]
+[env.agent.runtime]
 type = "docker"
 ```
 
@@ -109,7 +109,7 @@ Docker harnesses can keep trusted setup online, then restrict the agent to decla
 HTTP(S) destinations:
 
 ```toml
-[env.agent.harness.runtime]
+[env.agent.runtime]
 type = "docker"
 allow = ["https://*.wikipedia.org"]
 block = ["https://upload.wikimedia.org"]

docs/v1/harnesses.md

@@ -6,7 +6,8 @@ verifiers supports a range of harnesses out of the box, including Claude Code, C
 
 ```python
 from verifiers.v1.clients import ModelContext
-from verifiers.v1.harness import Harness, HarnessConfig
+from verifiers.v1.configs.harness import HarnessConfig
+from verifiers.v1.harness import Harness
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.task import TaskData
 from verifiers.v1.trace import Trace

environments/compact/compact/harness.py

@@ -12,7 +12,8 @@
 from pathlib import Path
 
 from verifiers.v1.clients import ModelContext
-from verifiers.v1.harness import Harness, HarnessConfig
+from verifiers.v1.configs.harness import HarnessConfig
+from verifiers.v1.harness import Harness
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.task import TaskData
 from verifiers.v1.trace import Trace

environments/proposer_solver_v1/proposer_solver_v1/taskset.py

@@ -13,7 +13,7 @@
 in a real sandbox and keep the solvers on a cheap tool-less chat loop —
 
     uv run eval proposer-solver-v1 -n 4 \
-      --env.proposer.harness.id codex --env.proposer.harness.runtime.type prime \
+      --env.proposer.harness.id codex --env.proposer.runtime.type prime \
       --env.solver.harness.id null
 
 Train-side, the seats flip independently per run (`--env.train_solver false`

examples/agent.py

@@ -26,7 +26,7 @@ async def main() -> None:
     last = trace.assistant_messages[-1].content if trace.assistant_messages else None
     print("answer:", last)
     assert trace.agent is not None
-    print("agent:", trace.agent.name, trace.agent.model)
+    print("agent:", trace.agent.name, trace.agent.config.model)
     print("runtime:", trace.runtime.type if trace.runtime else None)
 
 

skills/evaluate-environments/SKILL.md

@@ -51,7 +51,7 @@ When the user requests a full run, do not restrict the number of tasks. Ask for
 The leading ID is shorthand for `--env.taskset.id`. A harness belongs to an agent — `--env.agent.harness.*` on the single-agent env, `--env.<agent>.harness.*` on a multi-agent one (there is no run-level `--harness.*`):
 
 ```bash
-prime eval run owner/name --env.agent.harness.id codex --env.agent.harness.runtime.type prime
+prime eval run owner/name --env.agent.harness.id codex --env.agent.runtime.type prime
 ```
 
 The env — the control flow between agents — owns the whole `[env]` block. Empty `--env.id`
@@ -61,7 +61,7 @@ env); `--env.id` pairs a reusable env with any taskset, its knobs typed under `-
 ```bash
 prime eval run my-task-v1 --env.id best-of-n --env.n 8      # pass@k / rejection sampling
 prime eval run my-task-v1 --env.id agentic-judge \
-  --env.judge.harness.runtime.type docker                   # a judge agent verifies each attempt in a sandbox
+  --env.judge.runtime.type docker                           # a judge agent verifies each attempt in a sandbox
 ```
 
 When specifying Hub tasksets, always include the owner to resolve them correctly.
@@ -90,9 +90,9 @@ Harness and runtime settings:
 ```bash
 prime eval run my-task-v1 \
   --env.agent.harness.id rlm \
-  --env.agent.harness.runtime.type docker \
-  --env.agent.harness.runtime.cpu 4 \
-  --env.agent.harness.runtime.memory 8
+  --env.agent.runtime.type docker \
+  --env.agent.runtime.cpu 4 \
+  --env.agent.runtime.memory 8
 ```
 
 Sampling:
@@ -122,9 +122,11 @@ model = "openai/gpt-5-mini"
 id = "my-task-v1"
 split = "test"
 
+[env.agent]
+runtime = { type = "subprocess" }
+
 [env.agent.harness]
 id = "bash"
-runtime = { type = "subprocess" }
 
 [sampling]
 temperature = 0.7