feat: move the runtime onto the agent and stamp the agent config onto the trace

configs/agentic_judge.toml

@@ -23,5 +23,5 @@ id = "null"
 [env.judge]
 model = "z-ai/glm-5.2"
 
-[env.judge.harness.runtime]
+[env.judge.runtime]
 type = "docker"

configs/gsm8k_rlm.toml

@@ -16,5 +16,5 @@ id = "gsm8k-v1"
 id = "rlm"
 version = "main"
 
-[env.agent.harness.runtime]
+[env.agent.runtime]
 type = "docker"

configs/harbor.toml

@@ -8,6 +8,8 @@ num_tasks = 10
 id = "harbor"
 dataset = "terminal-bench/terminal-bench-2"
 
+[env.agent]
+runtime = { type = "docker" }
+
 [env.agent.harness]
 id = "bash"
-runtime = { type = "docker" }

docs/v1/evaluation.md

@@ -21,7 +21,7 @@ id = "primeintellect/terminal-bench-2"
 id = "codex"
 version = "0.116.0"
 
-[env.agent.harness.runtime]
+[env.agent.runtime]
 type = "docker"
 ```
 
@@ -82,7 +82,7 @@ Docker harnesses can keep trusted setup online, then restrict the agent to decla
 HTTP(S) destinations:
 
 ```toml
-[env.agent.harness.runtime]
+[env.agent.runtime]
 type = "docker"
 allow = ["https://*.wikipedia.org"]
 block = ["https://upload.wikimedia.org"]

environments/proposer_solver_v1/proposer_solver_v1/taskset.py

@@ -13,7 +13,7 @@
 in a real sandbox and keep the solvers on a cheap tool-less chat loop —
 
     uv run eval proposer-solver-v1 -n 4 \
-      --env.proposer.harness.id codex --env.proposer.harness.runtime.type prime \
+      --env.proposer.harness.id codex --env.proposer.runtime.type prime \
       --env.solver.harness.id null
 
 Train-side, the seats flip independently per run (`--env.train_solver false`

examples/agent.py

@@ -26,7 +26,7 @@ async def main() -> None:
     last = trace.assistant_messages[-1].content if trace.assistant_messages else None
     print("answer:", last)
     assert trace.agent is not None
-    print("agent:", trace.agent.name, trace.agent.model)
+    print("agent:", trace.agent.name, trace.agent.config.model)
     print("runtime:", trace.runtime.type if trace.runtime else None)
 
 

skills/evaluate-environments/SKILL.md

@@ -51,15 +51,15 @@ When the user requests a full run, do not restrict the number of tasks. Ask for
 The leading ID is shorthand for `--env.taskset.id`. A harness belongs to an agent — `--env.agent.harness.*` on the single-agent env, `--env.<agent>.harness.*` on a multi-agent one (there is no run-level `--harness.*`):
 
 ```bash
-prime eval run owner/name --env.agent.harness.id codex --env.agent.harness.runtime.type prime
+prime eval run owner/name --env.agent.harness.id codex --env.agent.runtime.type prime
 ```
 
 The env — the control flow between agents — owns the whole `[env]` block. Empty `--env.id` keeps the taskset's own story (its exported `Environment` subclass, else the single-agent env); `--env.id` pairs a reusable env with any taskset, its knobs typed under `--env.*`:
 
 ```bash
 prime eval run my-task-v1 --env.id best-of-n --env.n 8      # pass@k / rejection sampling
 prime eval run my-task-v1 --env.id agentic-judge \
-  --env.judge.harness.runtime.type docker                   # a judge agent verifies each attempt in a sandbox
+  --env.judge.runtime.type docker                           # a judge agent verifies each attempt in a sandbox
 ```
 
 When specifying Hub tasksets, always include the owner to resolve them correctly.
@@ -88,9 +88,9 @@ Harness and runtime settings:
 ```bash
 prime eval run my-task-v1 \
   --env.agent.harness.id rlm \
-  --env.agent.harness.runtime.type docker \
-  --env.agent.harness.runtime.cpu 4 \
-  --env.agent.harness.runtime.memory 8
+  --env.agent.runtime.type docker \
+  --env.agent.runtime.cpu 4 \
+  --env.agent.runtime.memory 8
 ```
 
 Sampling:
@@ -120,9 +120,11 @@ model = "openai/gpt-5-mini"
 id = "my-task-v1"
 split = "test"
 
+[env.agent]
+runtime = { type = "subprocess" }
+
 [env.agent.harness]
 id = "bash"
-runtime = { type = "subprocess" }
 
 [sampling]
 temperature = 0.7

skills/evaluate-environments/references/REFERENCE.md

@@ -1,6 +1,6 @@
 # REFERENCE.md
 
-A complete reference of every settable config field for **evaluating tasksets in `verifiers.v1`**. The config tree is parsed from CLI flags (dotted, e.g. `--env.agent.harness.runtime.type docker`) and/or `@ file.toml` by `prime-pydantic-config`; every field below is settable either way unless noted.
+A complete reference of every settable config field for **evaluating tasksets in `verifiers.v1`**. The config tree is parsed from CLI flags (dotted, e.g. `--env.agent.runtime.type docker`) and/or `@ file.toml` by `prime-pydantic-config`; every field below is settable either way unless noted.
 
 The root config the eval CLI parses is [`EvalConfig`](#evalconfig--the-run). It composes the environment (`env` — the whole `[env]` block: taskset, agents, limits) with the run knobs (model, sampling, counts) and the worker pool. The tree:
 
@@ -14,7 +14,7 @@ EvalConfig                          (the run)
 │  ├─ <agent>: AgentConfig          (per-agent harness/model/client/sampling +
 │  │  │                              per-run caps; `agent` on the single-agent env)
 │  │  ├─ harness: HarnessConfig     (subclass resolved by --env.<agent>.harness.id)
-│  │  │  └─ runtime: RuntimeConfig  (subprocess | docker | prime | modal)
+│  │  ├─ runtime: RuntimeConfig     (subprocess | docker | prime | modal)
 │  │  ├─ timeout: TimeoutConfig     (per-stage: setup/rollout/finalize/scoring)
 │  │  ├─ retries: RolloutRetryConfig (per-agent whole-run retries)
 │  │  └─ max_turns / max_input_tokens / max_output_tokens / max_total_tokens
@@ -121,7 +121,8 @@ Per-run caps (turns, tokens, stage timeouts, retries) are agent fields, not env
 
 | Field | Type | Default | Notes |
 | --- | --- | --- | --- |
-| `harness` | `HarnessConfig \| None` | `None` | The agent's program + runtime policy; resolved to its concrete subclass by `--env.<agent>.harness.id`. `None` = the taskset's default harness (its bundled one, else `bash`). See [Harness config](#harness-config). |
+| `harness` | `HarnessConfig \| None` | `None` | The agent's program; resolved to its concrete subclass by `--env.<agent>.harness.id`. `None` = the taskset's default harness (its bundled one, else `bash`). See [Harness config](#harness-config). |
+| `runtime` | `RuntimeConfig` | `SubprocessConfig()` | Where the harness runs. Discriminated union — see [Runtime configs](#runtime-configs). Set with `--env.<agent>.runtime.type docker\|prime\|modal`. |
 | `model` | `str \| None` | `None` | Pin a model for this agent (None = the run's `--model`). |
 | `client` | `ClientConfig \| None` | `None` | Pin an endpoint (None = the run's `--client.*`) — route a frozen judge or user sim off the training endpoint. |
 | `sampling` | `SamplingConfig \| None` | `None` | Pin sampling (None = the run's). |
@@ -241,7 +242,6 @@ A taskset implements `load()` and declares exactly one task type through its gen
 | Field | Type | Default | Notes |
 | --- | --- | --- | --- |
 | `id` | `ID` | `"bash"` | The harness id, which selects it. Set via `--env.<agent>.harness.id`. |
-| `runtime` | `RuntimeConfig` | `SubprocessConfig()` | Where the harness runs. Discriminated union — see [Runtime configs](#runtime-configs). Set with `--env.<agent>.harness.runtime.type docker\|prime\|modal`. |
 | `env` | `dict[str, str]` | `{}` | Additional env vars for the harness program. Harness-owned endpoint/auth/model vars take precedence. |
 | `forward_env` | `list[str]` | `[]` | Names of env vars to forward from `os.environ` into the harness program's runtime (for secrets not in checked-in config). Absent names are skipped; explicit `env` wins. |
 | `disabled_tools` | `list[str] \| None` | `None` | Harness-specific tool names to disable. |
@@ -252,7 +252,7 @@ A harness class also declares capability flags (ClassVars, not user-settable): `
 
 ### Built-in harness configs
 
-All inherit the base `HarnessConfig` fields (`id`, `runtime`, `env`, `forward_env`, `disabled_tools`).
+All inherit the base `HarnessConfig` fields (`id`, `env`, `forward_env`, `disabled_tools`); the runtime is the agent's (`--env.<agent>.runtime.*`).
 
 #### `BashHarnessConfig` — `id: "bash"` (the fallback)
 
@@ -314,7 +314,7 @@ Installs the Kimi Code CLI and runs it headlessly.
 
 ## Runtime configs
 
-`verifiers/v1/runtimes/`. Discriminated on `type`; selected with the agent's `--env.<agent>.harness.runtime.type` (or `--runtime.type` for the validate CLI). The same union is reused as `ToolsetConfig.runtime` and `UserConfig.runtime`.
+`verifiers/v1/runtimes/`. Discriminated on `type`; selected with the agent's `--env.<agent>.runtime.type` (or `--runtime.type` for the validate CLI). The same union is reused as `ToolsetConfig.runtime` and `UserConfig.runtime`.
 
 ### `SubprocessConfig` — `type: "subprocess"` (default)
 
@@ -583,12 +583,12 @@ Use `only_gold` or `only_setup` to select one mode; setting both is rejected. Th
 - **Plugin resolution.** The run's `env` field narrows to the selected env's config class
   (`--env.id`, else the taskset's exported env, else `SingleAgentEnvConfig`) *before* validation; inside it, `taskset` narrows by its id and each pinned agent `harness` by its id. Entries in `TaskConfig.judges` are similarly narrowed by each judge `id`. This is why local and Hub plugin fields remain typed and appear in CLI validation instead of living in an untyped arguments dictionary.
 - **Dotted flags.** Every nested field is part of the same CLI tree
-  (`--env.agent.harness.runtime.type docker`, `--env.taskset.split test`, `--pool.max_workers 8`, `--env.agent.retries.max_retries 2`). An `@ file.toml` describes the identical tree; explicit CLI values layer over values loaded from the file.
+  (`--env.agent.runtime.type docker`, `--env.taskset.split test`, `--pool.max_workers 8`, `--env.agent.retries.max_retries 2`). An `@ file.toml` describes the identical tree; explicit CLI values layer over values loaded from the file.
 - **Runtime precedence.** An explicit, non-default CLI/TOML `workdir` or resource field wins over
   `TaskData`; otherwise a non-`None` row value fills it, and otherwise the runtime/provider default remains. `TaskData.image` is the required image for that row and replaces the runtime's base image. Unsupported resource fields are ignored; evaluation warns once per runtime/field.
 - **Timeout precedence.** For eval stages, a non-`None` agent-level `TimeoutConfig` value
   (`--env.<agent>.timeout.*`) wins over the corresponding `TaskData.timeout` value; if both are `None`, there is no framework timeout. The setup value is one deadline shared by task setup and harness provisioning. Validate uses `CheckTimeoutConfig.setup`, then falls back to `TaskData.timeout.setup`, while `CheckTimeoutConfig.total` independently bounds `Task.validate`.
-- **Discriminated unions** are selected by their `type` field: `client.type` (eval|train), `pool.type` (static|elastic), `env.<agent>.harness.runtime.type` / `runtime.type` (subprocess|docker|prime|modal).
+- **Discriminated unions** are selected by their `type` field: `client.type` (eval|train), `pool.type` (static|elastic), `env.<agent>.runtime.type` / `runtime.type` (subprocess|docker|prime|modal).
 - **Frozen models.** `TaskData`, `TaskResources`, and `TaskTimeout` are immutable wire input, not
   mutable runtime state. Put per-rollout coordination on typed `trace.state`. `RolloutLimits` is an immutable framework limit derived from the env's `max_*` fields.
 - **Legacy v0.** Set the run-level `id` (leave `env.taskset` unset) to run a classic `load_environment` env through the bridge. `--resume` is not supported for legacy evals.

tests/v1/conftest.py

@@ -130,16 +130,16 @@ def _eval_config(
     max_turns: int | None = 4,
     rollout_timeout: float = 180,
     taskset_overrides: dict | None = None,
-    harness_overrides: dict | None = None,
+    runtime: dict | None = None,
     env: dict | None = None,
     pool: dict | None = None,
     model: str | None = None,
     reasoning_effort: str | None = None,
 ) -> EvalConfig:
     """Build the smallest `EvalConfig` that still exercises the path, shared by the in-process
-    (`run_v1`) and env-server (`run_v1_server`) fixtures. `taskset_overrides` / `harness_overrides`
-    are merged onto the `{id: ...}` config (placement, runtime, etc.); `model` overrides the default
-    text model (e.g. a VLM for an image task).
+    (`run_v1`) and env-server (`run_v1_server`) fixtures. `taskset_overrides` merges onto the
+    `{id: ...}` config; `runtime` places the `agent` seat's harness (an agent field, not a
+    harness one); `model` overrides the default text model (e.g. a VLM for an image task).
 
     `temperature=0` (greedy) makes the run reproducible; `max_tokens` is generous headroom,
     not a target — these trivial tasks finish in a few hundred tokens, so capping tighter only
@@ -152,9 +152,11 @@ def _eval_config(
     env_cfg = dict(env or {})
     _configure_prime_runtimes(taskset_cfg)
     if harness:
-        harness_cfg = {"id": harness, **(harness_overrides or {})}
-        _configure_prime_runtimes(harness_cfg)
-        env_cfg.setdefault("agent", {})["harness"] = harness_cfg
+        env_cfg.setdefault("agent", {})["harness"] = {"id": harness}
+    if runtime:
+        runtime_cfg = dict(runtime)
+        _configure_prime_runtimes(runtime_cfg)
+        env_cfg.setdefault("agent", {})["runtime"] = runtime_cfg
     # Per-run caps live on the seats: resolve the env's declared roles and cap
     # each one (a test's own seat dict wins over the shared defaults).
     config_cls = vf.env_config_type(taskset, env_cfg.get("id", ""))

tests/v1/test_e2e.py

@@ -93,15 +93,15 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
     (trace,) = await run_v1(
         "echo-v1",
         harness=harness,
-        harness_overrides={"runtime": {"type": harness_runtime}},
+        runtime={"type": harness_runtime},
         output_dir=tmp_path,
         max_turns=2,
     )
     assert trace.ok
     assert trace.num_turns == 1
     assert trace.reward == 1.0
     # The seat's resolved identity rides the trace (policy metadata for trainers).
-    assert trace.agent is not None and trace.agent.sampling.temperature == 0
+    assert trace.agent is not None and trace.agent.config.sampling.temperature == 0
     # Every sampled turn has one per-call record, linked to its assistant node.
     sampled = [i for i, n in enumerate(trace.nodes) if n.sampled]
     assert [c.node for c in trace.calls if c.error is None] == sampled
@@ -120,7 +120,7 @@ async def test_user(run_v1, harness_runtime, user_runtime, tmp_path):
     (trace,) = await run_v1(
         "echo-user-sim-v1",
         harness="null",
-        harness_overrides={"runtime": {"type": harness_runtime}},
+        runtime={"type": harness_runtime},
         output_dir=tmp_path,
         max_turns=6,
         taskset_overrides={"task": {"user": user_runtime}},
@@ -142,7 +142,7 @@ async def test_tool(run_v1, harness_runtime, tool_runtime, tmp_path):
     (trace,) = await run_v1(
         "echo-tool-v1",
         harness="null",
-        harness_overrides={"runtime": {"type": harness_runtime}},
+        runtime={"type": harness_runtime},
         output_dir=tmp_path,
         max_turns=6,
         taskset_overrides={"task": {"tools": tool_runtime}},
@@ -169,7 +169,7 @@ async def test_tool_state(run_v1, harness_runtime, tool_runtime, tmp_path):
     (trace,) = await run_v1(
         "counter-tool-v1",
         harness="null",
-        harness_overrides={"runtime": {"type": harness_runtime}},
+        runtime={"type": harness_runtime},
         output_dir=tmp_path,
         max_turns=8,
         taskset_overrides={"task": {"tools": tool_runtime}},
@@ -190,7 +190,7 @@ async def test_shared_tool_isolation(
     traces = await run_v1_server(
         "scratchpad-v1",
         harness="null",
-        harness_overrides={"runtime": {"type": harness_runtime}},
+        runtime={"type": harness_runtime},
         output_dir=tmp_path,
         num_tasks=2,
         n=1,
@@ -210,7 +210,7 @@ async def test_tool_response_image(run_v1, tmp_path):
     (trace,) = await run_v1(
         "tool-response-image-v1",
         harness="null",
-        harness_overrides={"runtime": {"type": "subprocess"}},
+        runtime={"type": "subprocess"},
         model="openai/gpt-5.6-luna",
         reasoning_effort="none",
         output_dir=tmp_path,
@@ -237,7 +237,7 @@ async def test_rubric_judge(run_v1, tmp_path):
     (trace,) = await run_v1(
         "echo-v1",
         harness="null",
-        harness_overrides={"runtime": {"type": "subprocess"}},
+        runtime={"type": "subprocess"},
         output_dir=tmp_path,
         taskset_overrides={"task": {"judges": [{"id": "rubric", "path": str(rubric)}]}},
         max_turns=2,
@@ -255,7 +255,7 @@ async def test_agentic(run_v1, harness, harness_runtime, tmp_path):
     (trace,) = await run_v1(
         "echo-agentic-v1",
         harness=harness,
-        harness_overrides={"runtime": {"type": harness_runtime}},
+        runtime={"type": harness_runtime},
         output_dir=tmp_path,
         max_turns=10,
     )
@@ -329,7 +329,7 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
             # The judge reads the transcript and reasons before it writes the
             # verdict file; the shared 2048-token run cap truncates it mid-audit.
             "judge": {
-                "harness": {"runtime": {"type": "docker"}},
+                "runtime": {"type": "docker"},
                 "max_output_tokens": 8192,
             },
         },
@@ -384,7 +384,7 @@ async def test_replay_round_trip(run_v1, tmp_path):
     (source,) = await run_v1(
         "reverse-text-v1",
         harness="null",
-        harness_overrides={"runtime": {"type": "subprocess"}},
+        runtime={"type": "subprocess"},
         output_dir=run_dir,
         max_turns=2,
     )

tests/v1/test_legacy.py

@@ -22,7 +22,7 @@ async def test_v0_single_turn_matches_v1_shape(run_v0, run_v1, tmp_path):
     (v0,) = await run_v0("echo-v0", output_dir=tmp_path / "v0")
     (v1,) = await run_v1(
         "echo-v1",
-        harness_overrides={"runtime": {"type": "subprocess"}},
+        runtime={"type": "subprocess"},
         output_dir=tmp_path / "v1",
         max_turns=2,
     )
@@ -36,7 +36,7 @@ async def test_v0_multi_turn_matches_v1_shape(run_v0, run_v1, tmp_path):
     (v0,) = await run_v0("echo-multi-v0", output_dir=tmp_path / "v0")
     (v1,) = await run_v1(
         "echo-v1",
-        harness_overrides={"runtime": {"type": "subprocess"}},
+        runtime={"type": "subprocess"},
         output_dir=tmp_path / "v1",
         max_turns=2,
     )