feat(v1): configurable agentic-judge grading policy, uploads, and reward weights

configs/agentic_judge.toml

@@ -1,14 +1,3 @@
-# Any taskset graded by the bundled agentic judge — the multi-agent TOML shape:
-# `env.id` pairs a reusable env with the taskset, and each agent is its own
-# [env.<agent>] block (a partial override deep-merges into the agent's declared
-# default; there is no run-level harness). The judge pins what makes it a
-# different actor: a container runtime — it verifies the solver's attempt with
-# real execution, never on the host — and, here, its own model off the run's.
-#
-#   uv run eval @ configs/agentic_judge.toml --model <model>
-#
-# The judge writes {"score": 0-10, ...} to /tmp/verdict.json in its box; the env
-# validates it strictly and records it on the solver's trace as the `judge` reward.
 num_tasks = 2
 
 [env]
@@ -18,10 +7,10 @@ id = "agentic-judge"
 id = "gsm8k-v1"
 
 [env.solver.harness]
-id = "null"
+id = "bash"
+
+[env.solver.runtime]
+type = "docker"
 
 [env.judge]
 model = "z-ai/glm-5.2"
-
-[env.judge.runtime]
-type = "docker"

tests/v1/test_e2e.py

@@ -392,23 +392,32 @@ async def test_env_id_best_of_n(run_v1, tmp_path):
 
 @pytest.mark.e2e
 async def test_env_id_agentic_judge(run_v1, tmp_path):
-    """The agentic judge over the echo taskset (needs docker): the judge lands in
-    its own box with the graded transcript uploaded, investigates with real
-    execution, and its parsed verdict lands on the solver's trace under the spec's
-    reward key. Wiring, not taste: the judge followed the verdict-file contract's
-    output contract — the grade itself is the model's call."""
+    """The agentic judge over the echo taskset (needs docker): the box is
+    provisioned once from the solver's runtime policy, the solver plays in it,
+    the judge lands in the SAME box with the graded trace uploaded,
+    investigates with real execution, and its parsed verdict
+    lands on the solver's trace under the spec's reward key. Wiring, not taste:
+    the judge followed the verdict-file contract — the grade itself is the
+    model's call. Exercises the config surface too: a policy-only prompt
+    override (the verdict contract is appended regardless) and
+    reward-composition weights."""
     traces = await run_v1(
         "echo-v1",
         harness=None,  # seats pin their own harness; there is no run-level one
         env={
             "id": "agentic-judge",
-            "solver": {"harness": {"id": "null"}},
-            # The judge reads the transcript and reasons before it writes the
+            # The solver owns the shared box, so the container is pinned here.
+            "solver": {"harness": {"id": "bash"}, "runtime": {"type": "docker"}},
+            # The judge reads the trace and reasons before it writes the
             # verdict file; the shared 2048-token run cap truncates it mid-audit.
             "judge": {
-                "runtime": {"type": "docker"},
+                "harness": {"id": "bash"},
                 "max_output_tokens": 8192,
             },
+            "task": {
+                "prompt": "Check EMPIRICALLY that the agent echoed the word back.",
+            },
+            "score": {"task_weight": 0.5},
         },
         output_dir=tmp_path,
         max_turns=10,
@@ -419,7 +428,7 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
     (judge,) = [t for t in traces if t.agent_name == "judge"]
     assert solver.ok and judge.ok
     assert judge.trainable is False
-    assert solver.rewards["echoed"] == 1.0  # the task's own reward still runs
+    assert solver.rewards["echoed"] == 0.5  # the task's own reward, rescaled
     assert isinstance(judge.info.get("verdict"), dict)  # scraped off the box
     assert 0.0 <= solver.rewards["judge"] <= 1.0
 

verifiers/v1/envs/agentic_judge/__init__.py

@@ -1,3 +1,15 @@
-from verifiers.v1.envs.agentic_judge.env import AgenticJudgeEnv, AgenticJudgeEnvConfig
+from verifiers.v1.envs.agentic_judge.env import (
+    AgenticJudgeEnv,
+    AgenticJudgeEnvConfig,
+    Criterion,
+    JudgeTaskConfig,
+    ScoreConfig,
+)
 
-__all__ = ["AgenticJudgeEnv", "AgenticJudgeEnvConfig"]
+__all__ = [
+    "AgenticJudgeEnv",
+    "AgenticJudgeEnvConfig",
+    "Criterion",
+    "JudgeTaskConfig",
+    "ScoreConfig",
+]