From c3eab90dd8bb65a24f62796ed2a3bf0815a8d974 Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Mon, 27 Jul 2026 02:12:00 +0000 Subject: [PATCH 01/24] feat(v1): grade in an isolated box, with Harbor-native artifacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grading in the box the agent worked in leaves a seam a policy under RL pressure will find: an editable test file, tamperable grading state, an artifact that leaks the answer. This carries only what a task declares into a second box and grades there. Two channels, non-overlapping: - `Trace.info` stays the durable record (`patch`, `verdict`) and never travels. - `/logs/artifacts/` is transport — Harbor's in-sandbox convention, collected with no declaration, restored in the grading box at the original path ("no translation", as in Harbor). `Task.finalize` is the producer hook — it already means "runtime live, agent done, before scoring mutates anything", which is exactly Harbor's collect-hook moment, so `[[verifier.collect]]` maps onto it rather than needing a new stage. The env composes the boxes; no `Task.scoring_runtime`, and no change to the `Runtime` teardown contract. Harbor `task.toml` support: `artifacts = [...]` (string and object form, `exclude` honored) and `[[verifier.collect]]`. Sidecar `service`, `[verifier].user` and an explicit `[verifier.environment]` image are rejected at load; `destination` is inert, being host-trial-directory placement that verifiers has no equivalent for. A failing collect hook fails the rollout, unlike `harbor run` which logs and continues — here the output is a grading input, not observability. `agentic-judge` gains `--env.topology isolated|shared`, defaulting to isolated, and stops overwriting the judge's runtime policy when it has its own box. The judge's workspace note is topology-specific: told it stands in the agent's workspace when it stands in a fresh one, it reads an unmodified tree as failure. One archive per source rather than one combined tar: BusyBox tar (every alpine-based image) has no `-r` to append, and per-source `exclude` patterns cannot share a single create either. Co-Authored-By: Claude Opus 5 (1M context) --- docs/v1/harbor.md | 13 +- docs/v1/tasksets.md | 31 +++ verifiers/v1/__init__.py | 17 ++ verifiers/v1/artifacts.py | 356 ++++++++++++++++++++++++ verifiers/v1/envs/agentic_judge/env.py | 136 ++++++--- verifiers/v1/errors.py | 7 + verifiers/v1/state.py | 4 +- verifiers/v1/task.py | 7 + verifiers/v1/tasksets/harbor/taskset.py | 105 ++++++- verifiers/v1/utils/git.py | 22 +- 10 files changed, 662 insertions(+), 36 deletions(-) create mode 100644 verifiers/v1/artifacts.py diff --git a/docs/v1/harbor.md b/docs/v1/harbor.md index 85717b7e6f..5a7915a5b9 100644 --- a/docs/v1/harbor.md +++ b/docs/v1/harbor.md @@ -87,10 +87,21 @@ added automatically; evaluator-provided `allow` entries add exceptions and `bloc entries can narrow them. Restricted Harbor tasks require Docker or a Prime VM; Prime accepts host-level entries and rejects combinations that need both policy modes. +## Artifacts and collect hooks + +`artifacts = [...]` and `[[verifier.collect]]` are read from `task.toml` ([Harbor Docs](https://www.harborframework.com/docs/run-jobs/results-and-artifacts)). Collect hooks run in the agent's box from the task's `finalize`, which is Harbor's own ordering — after the agent phase, before collection — and declared paths plus the `/logs/artifacts/` convention dir are then carried into the grading box and restored at their original paths ("no translation", as in Harbor). + +Two deliberate differences from `harbor run`: + +- **A failing collect hook fails the rollout.** Harbor logs it and carries on, because there the output is observability; here it is a grading input, and a silently absent file makes the verifier score a stale state. +- **`destination` has no effect.** It positions a file in Harbor's host trial directory; verifiers has no trial directory (the trace is the record), and Harbor never lets `destination` affect verifier-side placement. + +Sidecar `service` entries, `[verifier].user`, and an explicit `[verifier.environment]` image are rejected at load. The grading box is built from the task's own image, so only the agent's delta has to travel; a different verifier image would need the whole working tree copied across. + ## Shortcomings verifiers does not have parity with Harbor yet, so some features are missing and currently being worked on. The most notable missing features right now are: - Switching to a different verifier-phase network policy ([Harbor Docs](https://www.harborframework.com/docs/tasks/network-policy)) -- Shared & separate verifiers ([Harbor Docs](https://www.harborframework.com/docs/tasks#verifier-environment-shared-vs-separate)) +- Sidecar services, and the sidecar artifacts and collect hooks that go with them ([Harbor Docs](https://www.harborframework.com/docs/tasks#sidecar-artifacts-and-collect-hooks)) - Multi-step tasks ([Harbor Docs](https://www.harborframework.com/docs/tasks/multi-step)) diff --git a/docs/v1/tasksets.md b/docs/v1/tasksets.md index 76145f61bd..642ceff145 100644 --- a/docs/v1/tasksets.md +++ b/docs/v1/tasksets.md @@ -224,6 +224,37 @@ class JudgeTraceTaskset(vf.Taskset[JudgedTask, SetConfig]): To override the judge model, set `env.taskset.task.judge.model` in your config (it is a string). +## Grading artifacts + +An environment can grade in a second box rather than the one the agent worked in, so nothing the agent did to its environment can reach the grader. Only what the task declares crosses over. + +Two channels, and they do different jobs: + +- **`trace.info` is the record.** `capture_patch` puts the diff in `trace.info["patch"]`, a judge puts its verdict there, and both ride `traces.jsonl`. It never travels to another box. +- **`/logs/artifacts/` is transport.** Anything written there is collected with no declaration at all, carried to the host, and restored in the grading box at the same path. + +Produce artifacts in `finalize`, while the runtime is live and before scoring mutates anything: + +```python +class MySweTask(vf.Task[MyData]): + async def finalize(self, trace: vf.Trace, runtime: vf.Runtime) -> None: + await vf.capture_patch( + trace, runtime, self.data.base_commit, + publish=f"{vf.CONVENTION_DIR}/patch.diff", # record + transport, one call + ) +``` + +Declare paths outside the convention dir on the task row: + +```python +class MyData(vf.TaskData): + artifacts: list[vf.Artifact] = [vf.Artifact(source="/work/report", exclude=[".git"])] +``` + +A declared path that is missing at collection time fails the rollout: it was declared because grading needs it, and grading a partial state scores the rollout wrong rather than loudly failing it. The convention dir is exempt — it is collected for every task, and most never write to it. + +The grading box boots from the same image as the agent's, so the repo and its dependencies are already present. Only the agent's delta has to travel, which is why the collection cap (`vf.artifacts.MAX_ARTIFACT_BYTES`) is sized for a patch rather than a tree. + ## Beyond one agent One episode doesn't have to be one agent run: agents, the control flow between agents, and cross-agent rewards are the environment's job — see [The Env](env.md). diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 55ab5998b7..6425a9d686 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -25,6 +25,7 @@ from verifiers.v1.env import Env from verifiers.v1.envs.single_agent import SingleAgentEnv, SingleAgentEnvConfig from verifiers.v1.errors import ( + ArtifactError, EnvError, HarnessError, InterceptionError, @@ -89,6 +90,14 @@ ) from verifiers.v1.state import State, StateT from verifiers.v1.configs.task import TaskConfig +from verifiers.v1.artifacts import ( + CONVENTION_DIR, + Artifact, + Collected, + collect, + release, + restore, +) from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout, WireTaskData from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.taskset import Taskset @@ -206,6 +215,7 @@ "ToolsetError", "SandboxError", "TaskError", + "ArtifactError", "InterceptionError", "TunnelError", # clients @@ -279,6 +289,13 @@ "PATCH_CAP_BYTES", "capture_patch", "resolve_head", + # grading artifacts + "CONVENTION_DIR", + "Artifact", + "Collected", + "collect", + "release", + "restore", # scoring "compare_stdout_results", "extract_boxed_answer", diff --git a/verifiers/v1/artifacts.py b/verifiers/v1/artifacts.py new file mode 100644 index 0000000000..81d9c0eda0 --- /dev/null +++ b/verifiers/v1/artifacts.py @@ -0,0 +1,356 @@ +"""Carry a rollout's grading inputs out of the agent's box and into a fresh one. + +A task that is graded in the box the agent worked in has no defense against a policy +under RL pressure: a test file it can edit, grading state it can tamper with, an +artifact that leaks the expected answer. Grading in a *second* box removes the seam — +but only what the task declares crosses over, so the grader sees the agent's output +rather than the agent's environment. + +Two channels, deliberately separate: + +- ``Trace.info`` is the durable record. `capture_patch` puts the diff there, the agentic + judge puts its verdict there, and both ride ``traces.jsonl``. It is never a transport + mechanism. +- ``/logs/artifacts/`` is transport. Harbor's in-sandbox convention: anything written + there is collected with no declaration at all. Content goes box -> host -> box and is + discarded once the grading box has it. + +`collect` runs while the agent's box is still alive (right after `Task.finalize`, which +is where a task snapshots state into files). It is the barrier: once it returns, the +agent's box can be torn down in the background because everything grading needs is on +the host. `restore` then places that content in the grading box at its original paths — +"no translation", matching Harbor, so a verifier script finds its inputs where the task +author put them. + +Strict by design, unlike Harbor's best-effort collection: Harbor collects for +observability, where a dropped file costs a log line. Here a dropped file means grading +against an incomplete state and scoring a rollout wrong, which is worse than failing it. +""" + +from __future__ import annotations + +import asyncio +import io +import logging +import shlex +import tarfile +import uuid +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import TYPE_CHECKING + +from verifiers.v1.errors import ArtifactError +from verifiers.v1.types import StrictBaseModel + +if TYPE_CHECKING: + from verifiers.v1.runtimes import Runtime + +logger = logging.getLogger(__name__) + +CONVENTION_DIR = "/logs/artifacts" +"""Harbor's in-sandbox publish directory, collected implicitly. A task that writes here +needs no `artifacts` declaration at all; an explicit entry for this path replaces the +implicit one (and, being explicit, is then required to exist).""" + +MAX_ARTIFACT_BYTES = 32 * 1024 * 1024 +"""Ceiling on the collected archive. Sized for a delta, not a tree: the grading box boots +from the same image as the agent's, so the repo is already there and only what the agent +produced has to travel.""" + +MAX_ARTIFACT_FILES = 10_000 + +_SYSTEM_ROOTS = frozenset( + { + "/", + "/bin", + "/boot", + "/dev", + "/etc", + "/lib", + "/lib64", + "/proc", + "/root", + "/sbin", + "/sys", + "/usr", + "/var", + } +) +"""Refused as artifact sources: collecting one would sweep up the image rather than the +agent's work, and restoring it would overwrite the grading box's own system files.""" + + +class Artifact(StrictBaseModel): + """One path to carry from the agent's box into the grading box. + + Mirrors the subset of Harbor's `ArtifactConfig` that means something here. Harbor's + `destination` and `service` are deliberately absent: `destination` positions a file + in a host trial directory, which verifiers does not have (the trace is the record), + and `service` addresses a compose sidecar, which no runtime supports yet. + """ + + source: str + """Absolute path in the agent's box. Re-materializes at this same path in the + grading box.""" + exclude: list[str] = [] + """`tar --exclude` patterns, applied when `source` is a directory.""" + + +@dataclass(frozen=True) +class CollectedArtifact: + """One declared source, tarred out of the agent's box.""" + + root: str + """Absolute path this archive covers. Cleared in the grading box before extraction, + so a file baked into the image cannot survive underneath restored content and be + mistaken for the agent's work.""" + archive: bytes + + +@dataclass(frozen=True) +class Collected: + """Artifact content held on the host between the two boxes. Transient — it is not + persisted and not part of the trace; only `Trace.info` is durable. + + One archive per source rather than one combined tar: BusyBox `tar` (every + alpine-based image) implements only `c`/`x`/`t`, with no `-r` to append to an + existing archive, and each source carries its own `exclude` patterns so they cannot + share a single create either. + """ + + entries: list[CollectedArtifact] + + @property + def roots(self) -> list[str]: + return [entry.root for entry in self.entries] + + @property + def is_empty(self) -> bool: + return not self.entries + + @property + def total_bytes(self) -> int: + return sum(len(entry.archive) for entry in self.entries) + + +def _normalize(artifacts: list[Artifact]) -> list[Artifact]: + """Resolve, validate and de-conflict declared sources. + + Overlapping entries raise rather than following Harbor's keep-the-first-and-warn: + there, a skipped entry costs a log line; here it silently narrows what gets graded. + """ + seen: list[Artifact] = [] + for artifact in artifacts: + path = PurePosixPath(artifact.source) + if not path.is_absolute(): + raise ArtifactError( + f"artifact source {artifact.source!r} must be an absolute path in the box" + ) + if ".." in path.parts: + raise ArtifactError( + f"artifact source {artifact.source!r} may not contain '..'" + ) + resolved = path.as_posix().rstrip("/") or "/" + if resolved in _SYSTEM_ROOTS: + raise ArtifactError( + f"artifact source {resolved!r} is a system directory; declare the " + "specific paths the grader needs instead" + ) + for other in seen: + a, b = PurePosixPath(resolved), PurePosixPath(other.source) + if a == b or a.is_relative_to(b) or b.is_relative_to(a): + raise ArtifactError( + f"artifact sources {other.source!r} and {resolved!r} overlap; " + "one would be silently dropped" + ) + seen.append(artifact.model_copy(update={"source": resolved})) + return seen + + +def _vet(archive: bytes) -> None: + """Reject an archive whose members could escape their roots on extraction. + + The agent chose this content, so it is untrusted even though we built the tar: an + absolute or `..`-bearing member would write outside the declared roots when the + grading box extracts at `/`, and a symlink or device node could redirect a later + write. Vetting here means `restore` extracts something already checked. + """ + total = files = 0 + try: + with tarfile.open(fileobj=io.BytesIO(archive), mode="r:") as tar: + for member in tar: + name = member.name + path = PurePosixPath(name) + if not name or path.is_absolute() or ".." in path.parts: + raise ArtifactError( + f"artifact archive contains unsafe path {name!r}" + ) + if not (member.isfile() or member.isdir()): + raise ArtifactError( + f"artifact {name!r} is neither a regular file nor a directory " + "(symlinks and special files are refused)" + ) + files += 1 + total += member.size if member.isfile() else 0 + if files > MAX_ARTIFACT_FILES: + raise ArtifactError( + f"artifacts exceed {MAX_ARTIFACT_FILES} files; narrow the " + "declared sources or add `exclude` patterns" + ) + if total > MAX_ARTIFACT_BYTES: + raise ArtifactError(_over_cap(total)) + except tarfile.TarError as exc: + raise ArtifactError(f"unreadable artifact archive: {exc}") from exc + + +def _over_cap(size: int) -> str: + return ( + f"artifacts total {size} bytes, over the {MAX_ARTIFACT_BYTES} byte limit. The " + "grading box boots from the agent's image, so only the delta needs to travel — " + "declare narrower sources, add `exclude` patterns, or raise MAX_ARTIFACT_BYTES." + ) + + +async def collect( + runtime: Runtime, artifacts: list[Artifact] | None = None +) -> Collected: + """Pull the convention dir and every declared path out of `runtime`, as one archive. + + Call once the agent has finished and `Task.finalize` has run, while the box is still + alive. Returning is the barrier the box's teardown may proceed behind. + + A declared source that is missing raises — it was declared because grading needs it. + The implicit convention dir is exempt: it is injected for every task, and most tasks + never write to it. + """ + declared = _normalize(list(artifacts or [])) + entries = list(declared) + optional: set[str] = set() + # Inject the convention sweep only when nothing declared touches it. A task that + # names a path inside `/logs/artifacts/` has said precisely what it needs, and that + # entry is required; sweeping the parent as well would overlap it and raise. + convention = PurePosixPath(CONVENTION_DIR) + if not any( + (p := PurePosixPath(a.source)) == convention + or p.is_relative_to(convention) + or convention.is_relative_to(p) + for a in declared + ): + entries.insert(0, Artifact(source=CONVENTION_DIR)) + optional = {CONVENTION_DIR} + + collected: list[CollectedArtifact] = [] + budget = MAX_ARTIFACT_BYTES + for artifact in entries: + source = artifact.source + probe = await runtime.run(["test", "-e", source], {}) + if probe.exit_code != 0: + if source in optional: + continue + raise ArtifactError( + f"declared artifact {source!r} does not exist in the box; the task " + "must produce it in finalize() (or a [[verifier.collect]] hook)" + ) + archive = await _tar_out(runtime, artifact, budget) + _vet(archive) + budget -= len(archive) + collected.append(CollectedArtifact(root=source, archive=archive)) + + logger.debug( + "collected %d artifact root(s): %s", len(collected), [c.root for c in collected] + ) + return Collected(entries=collected) + + +async def _tar_out(runtime: Runtime, artifact: Artifact, budget: int) -> bytes: + """Tar one source out of the box, refusing it in-box if it blows the budget.""" + path = f"/tmp/vf-artifact-{uuid.uuid4().hex}.tar" + excludes = " ".join(f"--exclude={shlex.quote(p)}" for p in artifact.exclude) + try: + await _run( + runtime, + f"tar -cf {shlex.quote(path)} -C / {excludes} -- " + f"{shlex.quote(artifact.source.lstrip('/'))}", + f"collect artifact {artifact.source!r}", + ) + # Size it in the box: an oversized collection must be refused before it is + # pulled into host memory, not after. + sized = await runtime.run(["sh", "-c", f"wc -c < {shlex.quote(path)}"], {}) + if sized.exit_code == 0 and (raw := sized.stdout.strip()).isdigit(): + if int(raw) > budget: + raise ArtifactError(_over_cap(MAX_ARTIFACT_BYTES - budget + int(raw))) + return await runtime.read(path) + finally: + # Best-effort: a leftover tar in a box about to be destroyed is harmless, and + # the name is unique per call on a shared-filesystem runtime. + try: + await runtime.run(["rm", "-f", path], {}) + except Exception: # noqa: BLE001 - cleanup must never mask a collection error. + logger.debug("failed to remove %s", path, exc_info=True) + + +async def restore(runtime: Runtime, collected: Collected) -> None: + """Place collected content in `runtime` at its original absolute paths. + + "No translation", matching Harbor: a verifier script finds its inputs where the task + author wrote them. Each root is cleared first so a file baked into the image cannot + survive beneath restored content and be mistaken for the agent's work. + """ + if collected.is_empty: + return + # Restoring clears each root and extracts at `/`. In a container that is the point; + # on the subprocess runtime `/` is the developer's own machine, so refuse rather + # than rm -rf a host path that happens to match an artifact source. + if getattr(runtime.config, "type", None) == "subprocess": + raise ArtifactError( + "refusing to restore artifacts into the subprocess runtime: extraction " + "writes to absolute paths on the host. Grade in a container " + "(--env.judge.runtime.type docker or prime)." + ) + # Clear every root before extracting any of them: doing it per entry would let an + # earlier entry's restored content be deleted by a later, nested root. + roots = " ".join(shlex.quote(root) for root in collected.roots) + await _run(runtime, f"rm -rf -- {roots}", "clear artifact roots") + for entry in collected.entries: + path = f"/tmp/vf-artifact-{uuid.uuid4().hex}.tar" + await runtime.write(path, entry.archive) + await _run( + runtime, + f"tar -xf {shlex.quote(path)} -C / && rm -f {shlex.quote(path)}", + f"restore artifact {entry.root!r}", + ) + + +def release(runtime: Runtime) -> None: + """Begin `runtime`'s teardown and return without waiting for it. + + Once `collect` has returned, nothing downstream needs the agent's box, and on a + remote runtime its teardown is an API round trip the grading box should not wait + behind. The runtime stays in the runtimes module's `_LIVE` weakset, so the atexit + backstop still frees it if the loop dies first. + + Safe to call inside a `provision()` block only if the caller detaches that context + first (`AsyncExitStack.pop_all()`); otherwise the context manager's own `stop()` + awaits a teardown anyway and nothing is gained. + """ + if runtime.stopped: + return + # Mark it synchronously, mirroring `Runtime.stop`'s own "before the await" rule. + # `create_task` only schedules, so without this a second `release()` — or a + # `provision()` exit that beats the loop to it — would start a second teardown. + runtime.stopped = True + task = asyncio.create_task(runtime.stop()) + # asyncio keeps only a weak reference to a running task, so a fire-and-forget + # teardown can be garbage collected mid-flight. Hold it until it finishes. + _PENDING.add(task) + task.add_done_callback(_PENDING.discard) + + +_PENDING: set[asyncio.Task] = set() + + +async def _run(runtime: Runtime, command: str, action: str) -> None: + result = await runtime.run(["sh", "-c", command], {}) + if result.exit_code: + detail = (result.stderr or result.stdout).strip()[-500:] + raise ArtifactError(f"failed to {action}: {detail}") diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index 3ad1a66e70..cc29c90f0c 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -1,25 +1,32 @@ -"""agentic-judge: a solver plays the task, a judge verifies it in the same box. +"""agentic-judge: a solver plays the task, a code-executing judge verifies the work. -A reusable env (`--env.id agentic-judge` over any taskset): the box is -provisioned from the solver's runtime policy, the solver plays the task in it, -and a code-executing judge then inspects the work as the agent left it, with -the solver's full trace record uploaded at `/tmp/trace.json`. The judge grades +A reusable env (`--env.id agentic-judge` over any taskset). The solver plays the +task in a container provisioned from its runtime policy; the judge then grades rubric criteria (`[env.task]`: policy prompt, criteria file) and writes its -verdicts to `/tmp/verdict.json`; `finalize()` validates them strictly onto the -solver's trace — `judge/` metrics plus a weighted-mean `judge` reward, -composed with the taskset's own rewards via `[env.score]` (judge-only by -default). +verdicts to `/tmp/verdict.json`, with the solver's full trace record uploaded at +`/tmp/trace.json`. `finalize()` validates them strictly onto the solver's trace — +`judge/` metrics plus a weighted-mean `judge` reward, composed with the +taskset's own rewards via `[env.score]` (judge-only by default). + +`--env.topology` decides where the judge stands. Under `isolated` (the default) it +gets its own box from the same image, holding only what the task declared as +artifacts, so nothing the agent did to its environment can reach the grader. Under +`shared` it plays in the box the agent worked in, seeing that environment directly +and every seam in it. """ import json import math import re import tomllib +from contextlib import AsyncExitStack from pathlib import Path +from typing import Literal from pydantic import field_validator import verifiers.v1 as vf +from verifiers.v1.artifacts import CONVENTION_DIR from verifiers.v1.types import StrictBaseModel VERDICT_FILE = "/tmp/verdict.json" @@ -97,11 +104,8 @@ def _render(template: str, **fields: str) -> str: return pattern.sub(lambda m: fields[m.group(1)], template) -SANDBOX_NOTE = f"""\ -## Your workspace - -Your sandbox is the SAME box the graded agent worked in, in the state the agent -left it — its edits (and any scoring side effects) are applied. The agent's raw +_RECORD_NOTE = f"""\ +The agent's raw trace record (JSON: messages, tool calls, and its `info` artifacts) is uploaded at `{TRACE_FILE}`. The record can be very large — never dump it whole; peek selectively (list its keys, then slice out specific fields with python or jq) @@ -111,6 +115,25 @@ def _render(template: str, **fields: str) -> str: wrong and references can be narrower than the task; do not over-index on how a reference solves it. Your verdict is what YOU verified by execution.""" +SHARED_SANDBOX_NOTE = f"""\ +## Your workspace + +Your sandbox is the SAME box the graded agent worked in, in the state the agent +left it — its edits (and any scoring side effects) are applied. {_RECORD_NOTE}""" + +ISOLATED_SANDBOX_NOTE = f"""\ +## Your workspace + +Your sandbox is a FRESH box, built from the same image the graded agent started +from — so it holds the task's original state, NOT the state the agent left. The +agent's environment is gone; you cannot inspect it, and nothing it changed is +here except what the task declared as an artifact. Those artifacts have been +restored at their original paths (a patch under `{CONVENTION_DIR}/` is the usual +one for code tasks), so to see the agent's work you generally have to apply or +read them rather than looking at the working tree. If something you need to +check was never declared as an artifact, say so in your reason rather than +assuming its absence means the agent failed. {_RECORD_NOTE}""" + HINT_SECTION = """\ ## Hints @@ -130,8 +153,19 @@ def __init__(self, data: vf.TaskData, files: dict[str, bytes]) -> None: self.files = files @classmethod - def from_trace(cls, solution: vf.Trace, config: "JudgeTaskConfig") -> "JudgeTask": - """Mint the judge's task from the solver's finished trace.""" + def from_trace( + cls, + solution: vf.Trace, + config: "JudgeTaskConfig", + topology: str = "isolated", + ) -> "JudgeTask": + """Mint the judge's task from the solver's finished trace. + + `topology` selects the workspace note. It has to match how the judge is + actually placed: the note is the judge's only account of what its box + contains, and a judge told it is standing in the agent's workspace when it + is standing in a fresh one will read an unmodified tree as a failed attempt. + """ solved = solution.task.data files = {TRACE_FILE: json.dumps(solution.to_record()).encode()} template = config.build_prompt() @@ -139,7 +173,8 @@ def from_trace(cls, solution: vf.Trace, config: "JudgeTaskConfig") -> "JudgeTask if "{prompt}" not in template: # A policy that doesn't place the task statement itself still needs it. body += "\n\n" + _render(TASK_SECTION, prompt=solved.prompt_text) - sections = [body, _verdict_section(config.criteria()), SANDBOX_NOTE] + note = SHARED_SANDBOX_NOTE if topology == "shared" else ISOLATED_SANDBOX_NOTE + sections = [body, _verdict_section(config.criteria()), note] if (hint := config.build_hint()) is not None: sections.insert(1, _render(HINT_SECTION, hint=hint)) prompt = "\n\n".join(sections) @@ -251,23 +286,33 @@ class ScoreConfig(vf.BaseConfig): class AgenticJudgeEnvConfig(vf.EnvConfig): solver: vf.AgentConfig = vf.AgentConfig() - """The solver agent. It owns the shared box, so its runtime must be a - container: `--env.solver.runtime.type docker|prime`.""" + """The solver agent. Its runtime must be a container: + `--env.solver.runtime.type docker|prime`.""" judge: vf.AgentConfig = vf.AgentConfig() - """The judge agent. It plays in the solver's box; its own runtime policy is - ignored (overwritten with the solver's).""" + """The judge agent. Under `isolated` it provisions its own box from this policy; + under `shared` it plays in the solver's box and this policy is ignored.""" + topology: Literal["isolated", "shared"] = "isolated" + """Whether the judge grades in its own box or the solver's. + + `isolated` boots a second box from the same image, carries the task's declared + artifacts across, and grades there — so nothing the agent did to its own + environment can reach the grader. `shared` places the judge in the box the agent + worked in, which lets it inspect that environment directly at the cost of leaving + every seam in it reachable.""" task: JudgeTaskConfig = JudgeTaskConfig() score: ScoreConfig = ScoreConfig() class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]): def __init__(self, config: AgenticJudgeEnvConfig) -> None: - # The judge plays in the solver's box, so its effective runtime IS the - # solver's policy — aligning the config keeps the base env's subprocess - # warning and the runtime stamped on the judge's trace truthful. - config.judge = config.judge.model_copy( - update={"runtime": config.solver.runtime} - ) + if config.topology == "shared": + # Sharing means the judge's effective runtime IS the solver's policy — + # aligning the config keeps the base env's subprocess warning and the + # runtime stamped on the judge's trace truthful. Under `isolated` the + # judge provisions its own box, so its policy is honored as written. + config.judge = config.judge.model_copy( + update={"runtime": config.solver.runtime} + ) super().__init__(config) self._check_agents() # A missing policy file or a malformed rubric fails here, not mid-episode. @@ -288,20 +333,47 @@ def _check_agents(self) -> None: ) if isinstance(self.config.solver.runtime, vf.SubprocessConfig): raise ValueError( - "agentic-judge plays its judge in the solver's box, but the solver " - "(which provisions it) resolves to the subprocess runtime; use " + "agentic-judge runs a code-executing solver in a container, but the " + "solver resolves to the subprocess runtime; use " "--env.solver.runtime.type docker or prime" ) + if self.config.topology == "isolated" and isinstance( + self.config.judge.runtime, vf.SubprocessConfig + ): + raise ValueError( + "agentic-judge grades in the judge's own box under " + "--env.topology isolated, but the judge resolves to the subprocess " + "runtime (which would run the judge's code on the host); use " + "--env.judge.runtime.type docker or prime, or --env.topology shared" + ) async def setup(self, agents: vf.Agents) -> None: # The judge grades the policy; its tokens are never training data. agents.judge.trainable = False async def run(self, task: vf.Task, agents: vf.Agents) -> None: - async with agents.solver.provision(task) as box: + if self.config.topology == "shared": + async with agents.solver.provision(task) as box: + solution = await agents.solver.run(task, runtime=box) + judge_task = JudgeTask.from_trace(solution, self.config.task, "shared") + await agents.judge.run(judge_task, runtime=box) + return + + # The solver's box is detached rather than exited: `provision`'s own teardown + # would be awaited here, and the judge's box has no reason to wait behind it + # once collection has returned. If collection raises, `pop_all` is skipped and + # the context manager tears the box down as usual. + async with AsyncExitStack() as stack: + box = await stack.enter_async_context(agents.solver.provision(task)) solution = await agents.solver.run(task, runtime=box) - judge_task = JudgeTask.from_trace(solution, self.config.task) - await agents.judge.run(judge_task, runtime=box) + collected = await vf.collect(box, task.data.artifacts) + stack.pop_all() + vf.release(box) + + async with agents.judge.provision(task) as judge_box: + await vf.restore(judge_box, collected) + judge_task = JudgeTask.from_trace(solution, self.config.task, "isolated") + await agents.judge.run(judge_task, runtime=judge_box) async def finalize(self, task: vf.Task, episode: vf.Episode) -> None: by_agent = {t.agent_name: t for t in episode.traces} diff --git a/verifiers/v1/errors.py b/verifiers/v1/errors.py index 1ba1706fd1..0becaec2ce 100644 --- a/verifiers/v1/errors.py +++ b/verifiers/v1/errors.py @@ -74,6 +74,13 @@ class TaskError(RolloutError): """Task-authored code raised — `setup`, `finalize`, or a `@reward`/`@metric`.""" +class ArtifactError(RolloutError): + """A grading artifact could not be carried between boxes — a declared source was + missing, the collection exceeded its limits, or the archive was unsafe to extract. + Strict on purpose: an incompletely restored grading box scores the rollout wrong, + which is worse than failing it.""" + + class InterceptionError(RolloutError): """The host interception server (model calls + `/state` + `/task` channels) couldn't be reached.""" diff --git a/verifiers/v1/state.py b/verifiers/v1/state.py index ef2aa12455..19c5268e61 100644 --- a/verifiers/v1/state.py +++ b/verifiers/v1/state.py @@ -1,7 +1,9 @@ """Mutable state shared within one rollout. Tool servers synchronize it through the interception state channel. It is excluded -from serialized traces; persist artifacts in `Trace.info` instead. +from serialized traces; persist artifacts in `Trace.info` instead. `Trace.info` is the +durable record and never leaves the host — files a grader needs in a second box travel +separately, through `verifiers.v1.artifacts`. """ from pydantic import ConfigDict diff --git a/verifiers/v1/task.py b/verifiers/v1/task.py index f3c819741a..8322604d8d 100644 --- a/verifiers/v1/task.py +++ b/verifiers/v1/task.py @@ -33,6 +33,7 @@ from pydantic_config import BaseConfig from typing_extensions import TypeVar +from verifiers.v1.artifacts import Artifact from verifiers.v1.decorators import discover_decorated, invoke_all from verifiers.v1.errors import TaskError, boundary from verifiers.v1.configs.task import TaskConfig @@ -121,6 +122,12 @@ class TaskData(StrictBaseModel): network_block: list[str] = [] """Execution-time destinations denied by this task and combined with runtime blocks. Prime runtimes cannot combine an allowlist and a blocklist.""" + artifacts: list[Artifact] = [] + """Paths carried out of the agent's box and into a grading box, on top of the + implicitly collected `/logs/artifacts/` convention dir. Declare only what a grader + needs: the grading box boots from this task's image, so the repo is already there + and only the agent's output has to travel. A declared path that is missing at + collection time fails the rollout.""" timeout: TaskTimeout = TaskTimeout() resources: TaskResources = TaskResources() diff --git a/verifiers/v1/tasksets/harbor/taskset.py b/verifiers/v1/tasksets/harbor/taskset.py index e295ed8f64..9d5c45e7b4 100644 --- a/verifiers/v1/tasksets/harbor/taskset.py +++ b/verifiers/v1/tasksets/harbor/taskset.py @@ -10,6 +10,7 @@ image unless ``require_image`` is set. """ +import asyncio import hashlib import io import shutil @@ -24,10 +25,12 @@ from pydantic import Field +from verifiers.v1.artifacts import Artifact from verifiers.v1.decorators import reward -from verifiers.v1.errors import SandboxError +from verifiers.v1.errors import SandboxError, TaskError from verifiers.v1.runtimes import Runtime from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout +from verifiers.v1.trace import Trace from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.taskset import Taskset from verifiers.v1.types import StrictBaseModel @@ -76,6 +79,13 @@ class Author(StrictBaseModel): email: str | None = None +class CollectHook(StrictBaseModel): + """One `[[verifier.collect]]` command, run in the agent's box by `finalize`.""" + + command: str + timeout_sec: float = 60.0 + + class HarborData(TaskData): """Parsed ``task.toml`` metadata plus the host-side verifier directory. @@ -94,11 +104,42 @@ class HarborData(TaskData): """Raw [verifier.env] entries (literals or `${VAR}`/`${VAR:-default}` templates). Resolved against the host environment at scoring time, like `harbor run` — so a verifier that needs judge API keys or configuration actually receives them.""" + collect: list[CollectHook] = [] + """`[[verifier.collect]]` blocks: commands that snapshot runtime state into files + after the agent stops, so the files can travel to a grading box as artifacts.""" class HarborTask(Task[HarborData]): """Stage and run Harbor's verifier inside the task's live runtime.""" + async def finalize(self, trace: Trace, runtime: Runtime) -> None: + """Run Harbor's collect hooks while the agent's box is still alive. + + Harbor runs these after the agent phase and before artifact collection, which + is exactly what `finalize` means here, so the hook maps onto the existing + lifecycle rather than needing a stage of its own. + + Strict, unlike `harbor run`, which logs a failed hook and carries on: there the + output is observability, here it is a grading input, and a silently absent file + makes the verifier score a stale state instead of failing loudly. + """ + for hook in self.data.collect: + try: + result = await asyncio.wait_for( + runtime.run(["sh", "-c", hook.command], verifier_env(self.data)), + hook.timeout_sec, + ) + except TimeoutError as exc: + raise TaskError( + f"collect hook timed out after {hook.timeout_sec}s: {hook.command}" + ) from exc + if result.exit_code: + detail = (result.stderr or result.stdout).strip()[-500:] + raise TaskError( + f"collect hook failed (exit {result.exit_code}): " + f"{hook.command}\n{detail}" + ) + @reward(weight=1.0) async def solved(self, runtime: Runtime) -> float: await runtime.write( @@ -279,6 +320,7 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD config = tomllib.loads((task_dir / "task.toml").read_text()) parsed = HarborTaskConfig.model_validate(config) + artifacts, collect = parse_verifier_extras(task_dir, parsed) network = ( parsed.agent.explicit_phase_policy() or parsed.environment.resolve_baseline() ) @@ -326,8 +368,69 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD tags=meta.get("tags", []), task_dir=str(task_dir), verifier_env=config.get("verifier", {}).get("env", {}), + artifacts=artifacts, + collect=collect, + ) + + +def parse_verifier_extras( + task_dir: Path, parsed +) -> tuple[list[Artifact], list[CollectHook]]: + """Harbor's `artifacts` and `[[verifier.collect]]` blocks, narrowed to what a + single-container runtime can honor. + + The convention dir is deliberately not prepended here (Harbor's + `with_convention_entry` would): collection injects it itself, as an optional sweep. + Prepending it would make it an explicitly declared entry, and declared entries are + required — which would fail every task that never writes there. + """ + from harbor.constants import MAIN_SERVICE_NAME + from harbor.models.task.artifacts import ( + effective_artifact_service, + normalize_artifact_entries, ) + verifier = parsed.verifier + if verifier.environment is not None: + raise ValueError( + f"{task_dir.name}: [verifier.environment] declares a separate verifier " + "image. Grading runs in a fresh box built from the task's own image, so " + "only the agent's delta has to travel; a different verifier image needs " + "the full working tree copied over and isn't supported yet." + ) + if verifier.user is not None: + raise ValueError(f"{task_dir.name}: [verifier].user is not supported") + + artifacts: list[Artifact] = [] + for entry in normalize_artifact_entries(parsed.artifacts): + if effective_artifact_service(entry) != MAIN_SERVICE_NAME: + raise ValueError( + f"{task_dir.name}: artifact {entry.source!r} targets service " + f"{entry.service!r}; sidecars need a compose-capable runtime" + ) + # `destination` positions a file in Harbor's host trial directory. Verifiers has + # no such directory (the trace is the record) and Harbor never lets destination + # affect verifier-side placement, so it cannot change any grading outcome. + artifacts.append( + Artifact(source=entry.source, exclude=list(entry.exclude or [])) + ) + + hooks: list[CollectHook] = [] + for hook in verifier.collect: + if hook.service != MAIN_SERVICE_NAME: + raise ValueError( + f"{task_dir.name}: collect hook targets service {hook.service!r}; " + "sidecars need a compose-capable runtime" + ) + if hook.user is not None: + raise ValueError( + f"{task_dir.name}: collect hook `user` is not supported " + "(commands run as the runtime's default user)" + ) + hooks.append(CollectHook(command=hook.command, timeout_sec=hook.timeout_sec)) + + return artifacts, hooks + def verifier_env(task: HarborData) -> dict[str, str]: """Resolve templates at scoring time so host secrets are never serialized.""" diff --git a/verifiers/v1/utils/git.py b/verifiers/v1/utils/git.py index d48c63f574..a1d7cce984 100644 --- a/verifiers/v1/utils/git.py +++ b/verifiers/v1/utils/git.py @@ -15,6 +15,7 @@ from __future__ import annotations import uuid +from pathlib import PurePosixPath from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -75,13 +76,25 @@ async def resolve_head(runtime: Runtime, env: dict | None = None) -> str: async def capture_patch( - trace: Trace, runtime: Runtime, base_commit: str = "", env: dict | None = None + trace: Trace, + runtime: Runtime, + base_commit: str = "", + env: dict | None = None, + publish: str | None = None, ) -> None: """Snapshot the agent's cumulative diff into `trace.info["patch"]`. Best-effort by design: a rollout whose sandbox died or whose repo state is broken records `info["patch_error"]` instead of failing the rollout — scoring still runs and the error stays visible in results. + + `publish` additionally writes the patch to that path inside the box, for tasks + graded in a second box: `trace.info` is the durable record and never travels, so + a grader that needs the diff as a file needs it collected as an artifact. Point it + at `vf.CONVENTION_DIR` (e.g. `/logs/artifacts/patch.diff`) and collection picks it + up with no declaration. Publishing is best-effort like the rest of this helper — a + task that must not grade without the patch should declare that path as an + `Artifact`, which makes collection strict about it. """ nonce = uuid.uuid4().hex full, capped = f"{_FULL}_{nonce}", f"{_CAPPED}_{nonce}" @@ -111,3 +124,10 @@ async def capture_patch( raw = raw[:PATCH_CAP_BYTES] trace.info["patch_truncated"] = True trace.info["patch"] = raw.decode("utf-8", errors="replace") + if publish is not None: + try: + parent = str(PurePosixPath(publish).parent) + await runtime.run(["mkdir", "-p", parent], env or {}) + await runtime.write(publish, raw) + except Exception as exc: # noqa: BLE001 - publishing must never fail the rollout. + trace.info["patch_publish_error"] = f"{type(exc).__name__}: {exc}" From a657bd4999bd004a65b5ee0a5069956a7d3eaee5 Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Mon, 27 Jul 2026 02:44:55 +0000 Subject: [PATCH 02/24] refactor(v1): slim artifacts.py, and narrow the symlink rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review pass on the module. Removed, as dead or redundant: - `Collected`, a wrapper over a list whose `total_bytes` nothing ever called; `collect` now returns `list[CollectedArtifact]` directly. - `_SYSTEM_ROOTS`, thirteen paths guarding against an author typing `/usr` as a source. Sources come from task config, not the agent, and the size cap already refuses anything image-sized. Only `/` needs rejecting, because `lstrip('/')` makes it empty and tar then fails confusingly. - the overlap check in `_normalize`. It was load-bearing when collection built one combined tar; once `restore` began clearing every root before extracting any, overlapping sources became duplicated bytes rather than a correctness bug. - `_over_cap`, a function that only formatted a string, and `MAX_ARTIFACT_FILES`, which counted what the in-box `wc -c` already caps. Narrowed the link rule, which was both unsafe to relax and too blunt as written. Refusing every symlink rejects real content — `/etc` and any repo carrying a `.venv` — while dereferencing instead (`tar -h`) fails the whole collection on a single dangling link. An outward link is only a way into the grading box when another member sits underneath it, since that member is what gets written through the link. So leaf links are kept, in-root and dangling links are kept, and the write-through shape is refused. 224 lines from 356, then 260 with the link rule. Co-Authored-By: Claude Opus 5 (1M context) --- verifiers/v1/__init__.py | 4 +- verifiers/v1/artifacts.py | 412 +++++++++++++++----------------------- 2 files changed, 160 insertions(+), 256 deletions(-) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 6425a9d686..cdd070e67b 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -93,7 +93,7 @@ from verifiers.v1.artifacts import ( CONVENTION_DIR, Artifact, - Collected, + CollectedArtifact, collect, release, restore, @@ -292,7 +292,7 @@ # grading artifacts "CONVENTION_DIR", "Artifact", - "Collected", + "CollectedArtifact", "collect", "release", "restore", diff --git a/verifiers/v1/artifacts.py b/verifiers/v1/artifacts.py index 81d9c0eda0..1522a6fb34 100644 --- a/verifiers/v1/artifacts.py +++ b/verifiers/v1/artifacts.py @@ -1,30 +1,15 @@ -"""Carry a rollout's grading inputs out of the agent's box and into a fresh one. - -A task that is graded in the box the agent worked in has no defense against a policy -under RL pressure: a test file it can edit, grading state it can tamper with, an -artifact that leaks the expected answer. Grading in a *second* box removes the seam — -but only what the task declares crosses over, so the grader sees the agent's output -rather than the agent's environment. - -Two channels, deliberately separate: - -- ``Trace.info`` is the durable record. `capture_patch` puts the diff there, the agentic - judge puts its verdict there, and both ride ``traces.jsonl``. It is never a transport - mechanism. -- ``/logs/artifacts/`` is transport. Harbor's in-sandbox convention: anything written - there is collected with no declaration at all. Content goes box -> host -> box and is - discarded once the grading box has it. - -`collect` runs while the agent's box is still alive (right after `Task.finalize`, which -is where a task snapshots state into files). It is the barrier: once it returns, the -agent's box can be torn down in the background because everything grading needs is on -the host. `restore` then places that content in the grading box at its original paths — -"no translation", matching Harbor, so a verifier script finds its inputs where the task -author put them. - -Strict by design, unlike Harbor's best-effort collection: Harbor collects for -observability, where a dropped file costs a log line. Here a dropped file means grading -against an incomplete state and scoring a rollout wrong, which is worse than failing it. +"""Carry a task's declared files out of the agent's box and into a grading box. + +Grading in the box the agent worked in leaves a seam a policy under RL pressure will +find. Grading in a second box removes it — only what the task declares crosses over. + +Two channels, non-overlapping: `Trace.info` is the durable record (`capture_patch` puts +the diff there, a judge puts its verdict there) and never travels; `/logs/artifacts/` is +transport, Harbor's in-sandbox convention, collected with no declaration and restored at +the same path in the grading box ("no translation", as in Harbor). + +`collect` runs while the agent's box is alive, right after `Task.finalize` produced the +files. It is the barrier: once it returns the box can be torn down in the background. """ from __future__ import annotations @@ -48,270 +33,93 @@ logger = logging.getLogger(__name__) CONVENTION_DIR = "/logs/artifacts" -"""Harbor's in-sandbox publish directory, collected implicitly. A task that writes here -needs no `artifacts` declaration at all; an explicit entry for this path replaces the -implicit one (and, being explicit, is then required to exist).""" +"""Harbor's in-sandbox publish directory, swept implicitly so a task that writes here +needs no declaration.""" MAX_ARTIFACT_BYTES = 32 * 1024 * 1024 -"""Ceiling on the collected archive. Sized for a delta, not a tree: the grading box boots -from the same image as the agent's, so the repo is already there and only what the agent -produced has to travel.""" - -MAX_ARTIFACT_FILES = 10_000 - -_SYSTEM_ROOTS = frozenset( - { - "/", - "/bin", - "/boot", - "/dev", - "/etc", - "/lib", - "/lib64", - "/proc", - "/root", - "/sbin", - "/sys", - "/usr", - "/var", - } -) -"""Refused as artifact sources: collecting one would sweep up the image rather than the -agent's work, and restoring it would overwrite the grading box's own system files.""" +"""Ceiling per collection. Sized for a delta, not a tree: the grading box boots from the +agent's image, so the repo is already there and only its output has to travel.""" class Artifact(StrictBaseModel): - """One path to carry from the agent's box into the grading box. + """One path to carry into the grading box, where it lands at this same path. - Mirrors the subset of Harbor's `ArtifactConfig` that means something here. Harbor's - `destination` and `service` are deliberately absent: `destination` positions a file - in a host trial directory, which verifiers does not have (the trace is the record), - and `service` addresses a compose sidecar, which no runtime supports yet. + Harbor's `ArtifactConfig` minus `destination` (host trial-directory placement, which + verifiers has no equivalent for) and `service` (compose sidecars, unsupported). """ source: str - """Absolute path in the agent's box. Re-materializes at this same path in the - grading box.""" exclude: list[str] = [] """`tar --exclude` patterns, applied when `source` is a directory.""" @dataclass(frozen=True) class CollectedArtifact: - """One declared source, tarred out of the agent's box.""" + """One source, tarred out of the agent's box and held on the host in between. + Transient: only `Trace.info` is durable.""" root: str - """Absolute path this archive covers. Cleared in the grading box before extraction, - so a file baked into the image cannot survive underneath restored content and be - mistaken for the agent's work.""" archive: bytes -@dataclass(frozen=True) -class Collected: - """Artifact content held on the host between the two boxes. Transient — it is not - persisted and not part of the trace; only `Trace.info` is durable. - - One archive per source rather than one combined tar: BusyBox `tar` (every - alpine-based image) implements only `c`/`x`/`t`, with no `-r` to append to an - existing archive, and each source carries its own `exclude` patterns so they cannot - share a single create either. - """ - - entries: list[CollectedArtifact] - - @property - def roots(self) -> list[str]: - return [entry.root for entry in self.entries] - - @property - def is_empty(self) -> bool: - return not self.entries - - @property - def total_bytes(self) -> int: - return sum(len(entry.archive) for entry in self.entries) - - -def _normalize(artifacts: list[Artifact]) -> list[Artifact]: - """Resolve, validate and de-conflict declared sources. - - Overlapping entries raise rather than following Harbor's keep-the-first-and-warn: - there, a skipped entry costs a log line; here it silently narrows what gets graded. - """ - seen: list[Artifact] = [] - for artifact in artifacts: - path = PurePosixPath(artifact.source) - if not path.is_absolute(): - raise ArtifactError( - f"artifact source {artifact.source!r} must be an absolute path in the box" - ) - if ".." in path.parts: - raise ArtifactError( - f"artifact source {artifact.source!r} may not contain '..'" - ) - resolved = path.as_posix().rstrip("/") or "/" - if resolved in _SYSTEM_ROOTS: - raise ArtifactError( - f"artifact source {resolved!r} is a system directory; declare the " - "specific paths the grader needs instead" - ) - for other in seen: - a, b = PurePosixPath(resolved), PurePosixPath(other.source) - if a == b or a.is_relative_to(b) or b.is_relative_to(a): - raise ArtifactError( - f"artifact sources {other.source!r} and {resolved!r} overlap; " - "one would be silently dropped" - ) - seen.append(artifact.model_copy(update={"source": resolved})) - return seen - - -def _vet(archive: bytes) -> None: - """Reject an archive whose members could escape their roots on extraction. - - The agent chose this content, so it is untrusted even though we built the tar: an - absolute or `..`-bearing member would write outside the declared roots when the - grading box extracts at `/`, and a symlink or device node could redirect a later - write. Vetting here means `restore` extracts something already checked. - """ - total = files = 0 - try: - with tarfile.open(fileobj=io.BytesIO(archive), mode="r:") as tar: - for member in tar: - name = member.name - path = PurePosixPath(name) - if not name or path.is_absolute() or ".." in path.parts: - raise ArtifactError( - f"artifact archive contains unsafe path {name!r}" - ) - if not (member.isfile() or member.isdir()): - raise ArtifactError( - f"artifact {name!r} is neither a regular file nor a directory " - "(symlinks and special files are refused)" - ) - files += 1 - total += member.size if member.isfile() else 0 - if files > MAX_ARTIFACT_FILES: - raise ArtifactError( - f"artifacts exceed {MAX_ARTIFACT_FILES} files; narrow the " - "declared sources or add `exclude` patterns" - ) - if total > MAX_ARTIFACT_BYTES: - raise ArtifactError(_over_cap(total)) - except tarfile.TarError as exc: - raise ArtifactError(f"unreadable artifact archive: {exc}") from exc - - -def _over_cap(size: int) -> str: - return ( - f"artifacts total {size} bytes, over the {MAX_ARTIFACT_BYTES} byte limit. The " - "grading box boots from the agent's image, so only the delta needs to travel — " - "declare narrower sources, add `exclude` patterns, or raise MAX_ARTIFACT_BYTES." - ) - - async def collect( runtime: Runtime, artifacts: list[Artifact] | None = None -) -> Collected: - """Pull the convention dir and every declared path out of `runtime`, as one archive. +) -> list[CollectedArtifact]: + """Tar the convention dir and every declared path out of `runtime`. - Call once the agent has finished and `Task.finalize` has run, while the box is still - alive. Returning is the barrier the box's teardown may proceed behind. + A declared source that is missing raises: it was declared because grading needs it, + and grading a partial state scores the rollout wrong rather than failing it. The + implicit convention sweep is exempt — most tasks never write there. - A declared source that is missing raises — it was declared because grading needs it. - The implicit convention dir is exempt: it is injected for every task, and most tasks - never write to it. + One archive per source: BusyBox `tar` (every alpine-based image) implements only + `c`/`x`/`t` with no `-r` to append, and each source carries its own excludes anyway. """ - declared = _normalize(list(artifacts or [])) - entries = list(declared) - optional: set[str] = set() - # Inject the convention sweep only when nothing declared touches it. A task that - # names a path inside `/logs/artifacts/` has said precisely what it needs, and that - # entry is required; sweeping the parent as well would overlap it and raise. + declared = [_checked(a) for a in artifacts or []] convention = PurePosixPath(CONVENTION_DIR) - if not any( + sweep = not any( (p := PurePosixPath(a.source)) == convention or p.is_relative_to(convention) or convention.is_relative_to(p) for a in declared - ): - entries.insert(0, Artifact(source=CONVENTION_DIR)) - optional = {CONVENTION_DIR} + ) + entries = ([Artifact(source=CONVENTION_DIR)] if sweep else []) + declared collected: list[CollectedArtifact] = [] budget = MAX_ARTIFACT_BYTES for artifact in entries: source = artifact.source - probe = await runtime.run(["test", "-e", source], {}) - if probe.exit_code != 0: - if source in optional: + if (await runtime.run(["test", "-e", source], {})).exit_code != 0: + if sweep and source == CONVENTION_DIR: continue raise ArtifactError( - f"declared artifact {source!r} does not exist in the box; the task " - "must produce it in finalize() (or a [[verifier.collect]] hook)" + f"declared artifact {source!r} does not exist in the box; the task must " + "produce it in finalize() (or a [[verifier.collect]] hook)" ) archive = await _tar_out(runtime, artifact, budget) - _vet(archive) budget -= len(archive) collected.append(CollectedArtifact(root=source, archive=archive)) - logger.debug( - "collected %d artifact root(s): %s", len(collected), [c.root for c in collected] - ) - return Collected(entries=collected) + logger.debug("collected artifact roots: %s", [c.root for c in collected]) + return collected -async def _tar_out(runtime: Runtime, artifact: Artifact, budget: int) -> bytes: - """Tar one source out of the box, refusing it in-box if it blows the budget.""" - path = f"/tmp/vf-artifact-{uuid.uuid4().hex}.tar" - excludes = " ".join(f"--exclude={shlex.quote(p)}" for p in artifact.exclude) - try: - await _run( - runtime, - f"tar -cf {shlex.quote(path)} -C / {excludes} -- " - f"{shlex.quote(artifact.source.lstrip('/'))}", - f"collect artifact {artifact.source!r}", - ) - # Size it in the box: an oversized collection must be refused before it is - # pulled into host memory, not after. - sized = await runtime.run(["sh", "-c", f"wc -c < {shlex.quote(path)}"], {}) - if sized.exit_code == 0 and (raw := sized.stdout.strip()).isdigit(): - if int(raw) > budget: - raise ArtifactError(_over_cap(MAX_ARTIFACT_BYTES - budget + int(raw))) - return await runtime.read(path) - finally: - # Best-effort: a leftover tar in a box about to be destroyed is harmless, and - # the name is unique per call on a shared-filesystem runtime. - try: - await runtime.run(["rm", "-f", path], {}) - except Exception: # noqa: BLE001 - cleanup must never mask a collection error. - logger.debug("failed to remove %s", path, exc_info=True) - - -async def restore(runtime: Runtime, collected: Collected) -> None: - """Place collected content in `runtime` at its original absolute paths. - - "No translation", matching Harbor: a verifier script finds its inputs where the task - author wrote them. Each root is cleared first so a file baked into the image cannot - survive beneath restored content and be mistaken for the agent's work. - """ - if collected.is_empty: +async def restore(runtime: Runtime, collected: list[CollectedArtifact]) -> None: + """Extract `collected` in `runtime` at the original absolute paths.""" + if not collected: return - # Restoring clears each root and extracts at `/`. In a container that is the point; - # on the subprocess runtime `/` is the developer's own machine, so refuse rather - # than rm -rf a host path that happens to match an artifact source. + # Extraction writes to absolute paths. In a container that is the point; under the + # subprocess runtime it is the developer's own filesystem. if getattr(runtime.config, "type", None) == "subprocess": raise ArtifactError( "refusing to restore artifacts into the subprocess runtime: extraction " - "writes to absolute paths on the host. Grade in a container " - "(--env.judge.runtime.type docker or prime)." + "writes to absolute paths on the host. Grade in a container." ) - # Clear every root before extracting any of them: doing it per entry would let an - # earlier entry's restored content be deleted by a later, nested root. - roots = " ".join(shlex.quote(root) for root in collected.roots) + # Clear every root up front, not per entry: a later nested root would otherwise + # delete content an earlier one just restored. Clearing also drops any file or + # symlink the image left at the target. + roots = " ".join(shlex.quote(entry.root) for entry in collected) await _run(runtime, f"rm -rf -- {roots}", "clear artifact roots") - for entry in collected.entries: + for entry in collected: path = f"/tmp/vf-artifact-{uuid.uuid4().hex}.tar" await runtime.write(path, entry.archive) await _run( @@ -324,24 +132,19 @@ async def restore(runtime: Runtime, collected: Collected) -> None: def release(runtime: Runtime) -> None: """Begin `runtime`'s teardown and return without waiting for it. - Once `collect` has returned, nothing downstream needs the agent's box, and on a - remote runtime its teardown is an API round trip the grading box should not wait - behind. The runtime stays in the runtimes module's `_LIVE` weakset, so the atexit - backstop still frees it if the loop dies first. - - Safe to call inside a `provision()` block only if the caller detaches that context - first (`AsyncExitStack.pop_all()`); otherwise the context manager's own `stop()` - awaits a teardown anyway and nothing is gained. + Once `collect` has returned nothing needs the agent's box, and on a remote runtime + its teardown is an API round trip the grading box should not wait behind. Safe + inside a `provision()` block only if the caller detached it (`pop_all()`); otherwise + the context manager awaits a teardown anyway and nothing is gained. """ if runtime.stopped: return - # Mark it synchronously, mirroring `Runtime.stop`'s own "before the await" rule. - # `create_task` only schedules, so without this a second `release()` — or a - # `provision()` exit that beats the loop to it — would start a second teardown. + # Mark it synchronously, as `Runtime.stop` does: `create_task` only schedules, so + # otherwise a second call would start a second teardown. runtime.stopped = True task = asyncio.create_task(runtime.stop()) - # asyncio keeps only a weak reference to a running task, so a fire-and-forget - # teardown can be garbage collected mid-flight. Hold it until it finishes. + # asyncio holds only a weak reference to a running task, so a fire-and-forget + # teardown can be collected mid-flight. _PENDING.add(task) task.add_done_callback(_PENDING.discard) @@ -349,6 +152,107 @@ def release(runtime: Runtime) -> None: _PENDING: set[asyncio.Task] = set() +def _checked(artifact: Artifact) -> Artifact: + path = PurePosixPath(artifact.source) + if not path.is_absolute() or ".." in path.parts: + raise ArtifactError( + f"artifact source {artifact.source!r} must be an absolute path with no '..'" + ) + resolved = path.as_posix().rstrip("/") + if not resolved: + raise ArtifactError("artifact source '/' would sweep the whole image") + return artifact.model_copy(update={"source": resolved}) + + +async def _tar_out(runtime: Runtime, artifact: Artifact, budget: int) -> bytes: + path = f"/tmp/vf-artifact-{uuid.uuid4().hex}.tar" + excludes = " ".join(f"--exclude={shlex.quote(p)}" for p in artifact.exclude) + try: + await _run( + runtime, + f"tar -cf {shlex.quote(path)} -C / {excludes} -- " + f"{shlex.quote(artifact.source.lstrip('/'))}", + f"collect artifact {artifact.source!r}", + ) + # Size it in the box: an oversized collection is refused before it reaches host + # memory, not after. + sized = await runtime.run(["sh", "-c", f"wc -c < {shlex.quote(path)}"], {}) + if (raw := sized.stdout.strip()).isdigit() and int(raw) > budget: + raise ArtifactError( + f"artifact {artifact.source!r} takes the collection over the " + f"{MAX_ARTIFACT_BYTES} byte limit. The grading box boots from the " + "agent's image, so only the delta needs to travel — narrow the source " + "or add `exclude` patterns." + ) + archive = await runtime.read(path) + finally: + # Best-effort: the box is about to be destroyed and the name is unique per call. + try: + await runtime.run(["rm", "-f", path], {}) + except Exception: # noqa: BLE001 - cleanup must not mask a collection error. + logger.debug("failed to remove %s", path, exc_info=True) + _vet(archive) + return archive + + +def _vet(archive: bytes) -> None: + """Refuse an archive that could write outside its root when extracted at `/`. + + The agent chose this content. A member named `../x` escapes directly; a link + `a -> /etc` followed by a member `a/passwd` writes through it into the grading box, + which is the tampering this whole path exists to prevent. Links that stay inside + the archive can only redirect writes to content we are restoring anyway, so they + are kept — repos have them, and dereferencing instead (`tar -h`) fails the whole + collection on one dangling link. + """ + names: list[PurePosixPath] = [] + outward: list[tuple[PurePosixPath, str]] = [] + try: + with tarfile.open(fileobj=io.BytesIO(archive), mode="r:") as tar: + for member in tar: + path = PurePosixPath(member.name) + if not member.name or path.is_absolute() or ".." in path.parts: + raise ArtifactError(f"unsafe path in artifact: {member.name!r}") + names.append(path) + if member.isfile() or member.isdir(): + continue + if not (member.issym() or member.islnk()): + raise ArtifactError( + f"artifact {member.name!r} is a special file (device, fifo or " + "socket) and cannot be carried between boxes" + ) + if _escapes(path, member.linkname): + outward.append((path, member.linkname)) + except tarfile.TarError as exc: + raise ArtifactError(f"unreadable artifact archive: {exc}") from exc + + # An outward link is only a way in if something is nested under it — that member is + # what gets written through the link. A leaf link (`.venv/bin/python -> + # /usr/bin/python3`, in half of all Python repos) redirects nothing. + for link, target in outward: + if any(name != link and name.is_relative_to(link) for name in names): + raise ArtifactError( + f"artifact {str(link)!r} links out to {target!r} and other artifacts " + "sit underneath it; extracting would write through the link" + ) + + +def _escapes(member: PurePosixPath, linkname: str) -> bool: + """Whether `linkname`, read from `member`'s directory, lands outside the archive.""" + target = PurePosixPath(linkname) + if target.is_absolute(): + return True + depth = 0 + for part in (*member.parent.parts, *target.parts): + if part == "..": + depth -= 1 + if depth < 0: + return True + elif part != ".": + depth += 1 + return False + + async def _run(runtime: Runtime, command: str, action: str) -> None: result = await runtime.run(["sh", "-c", command], {}) if result.exit_code: From a8c78962c8ccf6f1c391fdcbda945ccc6702e648 Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Mon, 27 Jul 2026 03:08:06 +0000 Subject: [PATCH 03/24] refactor(v1): drop archive vetting, accept the tampered-tar risk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_vet` and `_escapes` defended a shape our own tooling cannot produce. Neither BusyBox nor GNU tar recurses into a symlinked directory, so `tar -c` never emits the write-through pair (`link -> /etc` plus a member at `link/passwd`) that the symlink analysis existed for; and tar strips leading `/` while no real path component is `..`, so the member-name checks were equally unreachable. What remained was a defence against one attacker: an agent that replaces the tar binary in its own box to poison the grading container. That is the only channel between the two boxes, but it is a remote attack, and fifty lines of subtle analysis to close it was a bad trade — the subtlety misfired twice while being written, first refusing every symlink (which would have broken any repo carrying a .venv) and then reading hardlink linknames in the wrong frame. Accepted risk: an agent that tampers with its own tar can place arbitrary files at arbitrary paths in the grading box. Note that extracting into a scratch directory instead would not have avoided the analysis — it closes relative traversal, but an absolute symlink target still redirects the write. 260 -> 198 lines. Co-Authored-By: Claude Opus 5 (1M context) --- verifiers/v1/artifacts.py | 64 +-------------------------------------- 1 file changed, 1 insertion(+), 63 deletions(-) diff --git a/verifiers/v1/artifacts.py b/verifiers/v1/artifacts.py index 1522a6fb34..2d652923ad 100644 --- a/verifiers/v1/artifacts.py +++ b/verifiers/v1/artifacts.py @@ -15,10 +15,8 @@ from __future__ import annotations import asyncio -import io import logging import shlex -import tarfile import uuid from dataclasses import dataclass from pathlib import PurePosixPath @@ -184,73 +182,13 @@ async def _tar_out(runtime: Runtime, artifact: Artifact, budget: int) -> bytes: "agent's image, so only the delta needs to travel — narrow the source " "or add `exclude` patterns." ) - archive = await runtime.read(path) + return await runtime.read(path) finally: # Best-effort: the box is about to be destroyed and the name is unique per call. try: await runtime.run(["rm", "-f", path], {}) except Exception: # noqa: BLE001 - cleanup must not mask a collection error. logger.debug("failed to remove %s", path, exc_info=True) - _vet(archive) - return archive - - -def _vet(archive: bytes) -> None: - """Refuse an archive that could write outside its root when extracted at `/`. - - The agent chose this content. A member named `../x` escapes directly; a link - `a -> /etc` followed by a member `a/passwd` writes through it into the grading box, - which is the tampering this whole path exists to prevent. Links that stay inside - the archive can only redirect writes to content we are restoring anyway, so they - are kept — repos have them, and dereferencing instead (`tar -h`) fails the whole - collection on one dangling link. - """ - names: list[PurePosixPath] = [] - outward: list[tuple[PurePosixPath, str]] = [] - try: - with tarfile.open(fileobj=io.BytesIO(archive), mode="r:") as tar: - for member in tar: - path = PurePosixPath(member.name) - if not member.name or path.is_absolute() or ".." in path.parts: - raise ArtifactError(f"unsafe path in artifact: {member.name!r}") - names.append(path) - if member.isfile() or member.isdir(): - continue - if not (member.issym() or member.islnk()): - raise ArtifactError( - f"artifact {member.name!r} is a special file (device, fifo or " - "socket) and cannot be carried between boxes" - ) - if _escapes(path, member.linkname): - outward.append((path, member.linkname)) - except tarfile.TarError as exc: - raise ArtifactError(f"unreadable artifact archive: {exc}") from exc - - # An outward link is only a way in if something is nested under it — that member is - # what gets written through the link. A leaf link (`.venv/bin/python -> - # /usr/bin/python3`, in half of all Python repos) redirects nothing. - for link, target in outward: - if any(name != link and name.is_relative_to(link) for name in names): - raise ArtifactError( - f"artifact {str(link)!r} links out to {target!r} and other artifacts " - "sit underneath it; extracting would write through the link" - ) - - -def _escapes(member: PurePosixPath, linkname: str) -> bool: - """Whether `linkname`, read from `member`'s directory, lands outside the archive.""" - target = PurePosixPath(linkname) - if target.is_absolute(): - return True - depth = 0 - for part in (*member.parent.parts, *target.parts): - if part == "..": - depth -= 1 - if depth < 0: - return True - elif part != ".": - depth += 1 - return False async def _run(runtime: Runtime, command: str, action: str) -> None: From ab941d88edb887138dafbf5db23dfe086b3564fd Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Mon, 27 Jul 2026 03:13:25 +0000 Subject: [PATCH 04/24] refactor(v1): rename capture_patch's `publish` to `write_path` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `publish` came from Harbor's vocabulary for `/logs/artifacts/` ("the conventional agent publish directory"), but nothing in this codebase names a parameter that way. The dominant convention is `_path` — `output_path`, `session_path`, `script_path`, `config_path` — and `write_path` mirrors `Runtime.write`, which is the call the argument drives. `output_path` was unavailable: it already means the host-side eval output directory. The failure key follows: `patch_publish_error` -> `patch_write_error`. Co-Authored-By: Claude Opus 5 (1M context) --- docs/v1/tasksets.md | 2 +- verifiers/v1/utils/git.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/v1/tasksets.md b/docs/v1/tasksets.md index 642ceff145..92f354180f 100644 --- a/docs/v1/tasksets.md +++ b/docs/v1/tasksets.md @@ -240,7 +240,7 @@ class MySweTask(vf.Task[MyData]): async def finalize(self, trace: vf.Trace, runtime: vf.Runtime) -> None: await vf.capture_patch( trace, runtime, self.data.base_commit, - publish=f"{vf.CONVENTION_DIR}/patch.diff", # record + transport, one call + write_path=f"{vf.CONVENTION_DIR}/patch.diff", # record + transport, one call ) ``` diff --git a/verifiers/v1/utils/git.py b/verifiers/v1/utils/git.py index a1d7cce984..d741a71d80 100644 --- a/verifiers/v1/utils/git.py +++ b/verifiers/v1/utils/git.py @@ -80,7 +80,7 @@ async def capture_patch( runtime: Runtime, base_commit: str = "", env: dict | None = None, - publish: str | None = None, + write_path: str | None = None, ) -> None: """Snapshot the agent's cumulative diff into `trace.info["patch"]`. @@ -88,7 +88,7 @@ async def capture_patch( broken records `info["patch_error"]` instead of failing the rollout — scoring still runs and the error stays visible in results. - `publish` additionally writes the patch to that path inside the box, for tasks + `write_path` additionally writes the patch to that path inside the box, for tasks graded in a second box: `trace.info` is the durable record and never travels, so a grader that needs the diff as a file needs it collected as an artifact. Point it at `vf.CONVENTION_DIR` (e.g. `/logs/artifacts/patch.diff`) and collection picks it @@ -124,10 +124,10 @@ async def capture_patch( raw = raw[:PATCH_CAP_BYTES] trace.info["patch_truncated"] = True trace.info["patch"] = raw.decode("utf-8", errors="replace") - if publish is not None: + if write_path is not None: try: - parent = str(PurePosixPath(publish).parent) + parent = str(PurePosixPath(write_path).parent) await runtime.run(["mkdir", "-p", parent], env or {}) - await runtime.write(publish, raw) - except Exception as exc: # noqa: BLE001 - publishing must never fail the rollout. - trace.info["patch_publish_error"] = f"{type(exc).__name__}: {exc}" + await runtime.write(write_path, raw) + except Exception as exc: # noqa: BLE001 - the write must never fail the rollout. + trace.info["patch_write_error"] = f"{type(exc).__name__}: {exc}" From 6a18ba948b6bc385495a4ed1374f4df41c353ff7 Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Mon, 27 Jul 2026 03:15:00 +0000 Subject: [PATCH 05/24] refactor(v1): drop CollectedArtifact, return a dict of root -> archive The type only ever held `(root, archive)`, so a dict keyed by source path says the same thing with no class: it preserves declaration order, makes a duplicate root impossible by construction, and drops the last dataclass from the module. `collect() -> dict[str, bytes]`, `restore(runtime, collected)` unchanged at its call sites. 191 lines, from 356 at the start of the branch. Co-Authored-By: Claude Opus 5 (1M context) --- verifiers/v1/__init__.py | 2 -- verifiers/v1/artifacts.py | 31 ++++++++++++------------------- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index cdd070e67b..ae0ec61cf6 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -93,7 +93,6 @@ from verifiers.v1.artifacts import ( CONVENTION_DIR, Artifact, - CollectedArtifact, collect, release, restore, @@ -292,7 +291,6 @@ # grading artifacts "CONVENTION_DIR", "Artifact", - "CollectedArtifact", "collect", "release", "restore", diff --git a/verifiers/v1/artifacts.py b/verifiers/v1/artifacts.py index 2d652923ad..f206060d18 100644 --- a/verifiers/v1/artifacts.py +++ b/verifiers/v1/artifacts.py @@ -18,7 +18,6 @@ import logging import shlex import uuid -from dataclasses import dataclass from pathlib import PurePosixPath from typing import TYPE_CHECKING @@ -51,20 +50,14 @@ class Artifact(StrictBaseModel): """`tar --exclude` patterns, applied when `source` is a directory.""" -@dataclass(frozen=True) -class CollectedArtifact: - """One source, tarred out of the agent's box and held on the host in between. - Transient: only `Trace.info` is durable.""" - - root: str - archive: bytes - - async def collect( runtime: Runtime, artifacts: list[Artifact] | None = None -) -> list[CollectedArtifact]: +) -> dict[str, bytes]: """Tar the convention dir and every declared path out of `runtime`. + Keyed by source path; the values are tar archives. Insertion order is the order + they were declared, and a path cannot be collected twice. + A declared source that is missing raises: it was declared because grading needs it, and grading a partial state scores the rollout wrong rather than failing it. The implicit convention sweep is exempt — most tasks never write there. @@ -82,7 +75,7 @@ async def collect( ) entries = ([Artifact(source=CONVENTION_DIR)] if sweep else []) + declared - collected: list[CollectedArtifact] = [] + collected: dict[str, bytes] = {} budget = MAX_ARTIFACT_BYTES for artifact in entries: source = artifact.source @@ -95,13 +88,13 @@ async def collect( ) archive = await _tar_out(runtime, artifact, budget) budget -= len(archive) - collected.append(CollectedArtifact(root=source, archive=archive)) + collected[source] = archive - logger.debug("collected artifact roots: %s", [c.root for c in collected]) + logger.debug("collected artifact roots: %s", list(collected)) return collected -async def restore(runtime: Runtime, collected: list[CollectedArtifact]) -> None: +async def restore(runtime: Runtime, collected: dict[str, bytes]) -> None: """Extract `collected` in `runtime` at the original absolute paths.""" if not collected: return @@ -115,15 +108,15 @@ async def restore(runtime: Runtime, collected: list[CollectedArtifact]) -> None: # Clear every root up front, not per entry: a later nested root would otherwise # delete content an earlier one just restored. Clearing also drops any file or # symlink the image left at the target. - roots = " ".join(shlex.quote(entry.root) for entry in collected) + roots = " ".join(shlex.quote(root) for root in collected) await _run(runtime, f"rm -rf -- {roots}", "clear artifact roots") - for entry in collected: + for root, archive in collected.items(): path = f"/tmp/vf-artifact-{uuid.uuid4().hex}.tar" - await runtime.write(path, entry.archive) + await runtime.write(path, archive) await _run( runtime, f"tar -xf {shlex.quote(path)} -C / && rm -f {shlex.quote(path)}", - f"restore artifact {entry.root!r}", + f"restore artifact {root!r}", ) From af508acbf4f57c58d230600224893cc1c059c167 Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Mon, 27 Jul 2026 03:18:43 +0000 Subject: [PATCH 06/24] refactor(v1): move fire-and-forget teardown onto Runtime.stop_nowait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `release()` was runtime lifecycle wearing an artifacts hat. It read and wrote `runtime.stopped` — another object's lifecycle state — from a module that has nothing to do with lifecycle, and kept its own `_PENDING` set of unawaited teardowns while `runtimes/base.py` already owns exactly that kind of bookkeeping in `_LIVE`. It now sits next to the machinery it belongs to, as `Runtime.stop_nowait`: `stopped` is set on self rather than poked from outside, `_PENDING` sits beside `_LIVE` with the atexit backstop that covers both, and the `_nowait` suffix says what it is — `stop`, without waiting — instead of inventing a verb. `release` was also already taken in v1, by `RolloutSession.release`. This does touch `runtimes/base.py`, having earlier concluded it need not. That still holds for what it was about: adding a re-entry guard to `stop()` would have made a failed teardown unretryable. Adding a sibling method changes no existing semantics. artifacts.py is down to 168 lines. Co-Authored-By: Claude Opus 5 (1M context) --- verifiers/v1/__init__.py | 2 -- verifiers/v1/artifacts.py | 27 ++------------------------ verifiers/v1/envs/agentic_judge/env.py | 2 +- verifiers/v1/runtimes/base.py | 20 +++++++++++++++++++ 4 files changed, 23 insertions(+), 28 deletions(-) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index ae0ec61cf6..19a1fc7af6 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -94,7 +94,6 @@ CONVENTION_DIR, Artifact, collect, - release, restore, ) from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout, WireTaskData @@ -292,7 +291,6 @@ "CONVENTION_DIR", "Artifact", "collect", - "release", "restore", # scoring "compare_stdout_results", diff --git a/verifiers/v1/artifacts.py b/verifiers/v1/artifacts.py index f206060d18..e0ad5abeee 100644 --- a/verifiers/v1/artifacts.py +++ b/verifiers/v1/artifacts.py @@ -9,12 +9,12 @@ the same path in the grading box ("no translation", as in Harbor). `collect` runs while the agent's box is alive, right after `Task.finalize` produced the -files. It is the barrier: once it returns the box can be torn down in the background. +files. It is the barrier: once it returns the box can be torn down (`Runtime.stop_nowait` +puts that teardown behind the grading box rather than in front of it). """ from __future__ import annotations -import asyncio import logging import shlex import uuid @@ -120,29 +120,6 @@ async def restore(runtime: Runtime, collected: dict[str, bytes]) -> None: ) -def release(runtime: Runtime) -> None: - """Begin `runtime`'s teardown and return without waiting for it. - - Once `collect` has returned nothing needs the agent's box, and on a remote runtime - its teardown is an API round trip the grading box should not wait behind. Safe - inside a `provision()` block only if the caller detached it (`pop_all()`); otherwise - the context manager awaits a teardown anyway and nothing is gained. - """ - if runtime.stopped: - return - # Mark it synchronously, as `Runtime.stop` does: `create_task` only schedules, so - # otherwise a second call would start a second teardown. - runtime.stopped = True - task = asyncio.create_task(runtime.stop()) - # asyncio holds only a weak reference to a running task, so a fire-and-forget - # teardown can be collected mid-flight. - _PENDING.add(task) - task.add_done_callback(_PENDING.discard) - - -_PENDING: set[asyncio.Task] = set() - - def _checked(artifact: Artifact) -> Artifact: path = PurePosixPath(artifact.source) if not path.is_absolute() or ".." in path.parts: diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index cc29c90f0c..9bf376ba67 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -368,7 +368,7 @@ async def run(self, task: vf.Task, agents: vf.Agents) -> None: solution = await agents.solver.run(task, runtime=box) collected = await vf.collect(box, task.data.artifacts) stack.pop_all() - vf.release(box) + box.stop_nowait() async with agents.judge.provision(task) as judge_box: await vf.restore(judge_box, collected) diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index e8d80cb3b9..e5b1141c3b 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -74,6 +74,9 @@ def parse_gpu(gpu: str | None) -> tuple[str | None, int]: # none of this. _LIVE: "weakref.WeakSet[Runtime]" = weakref.WeakSet() _atexit_armed = False +_PENDING: "set[asyncio.Task]" = set() +"""Unawaited teardowns from `stop_nowait`, held so asyncio's weak task references cannot +collect one mid-flight.""" def register(runtime: "Runtime") -> None: @@ -164,6 +167,23 @@ async def stop(self) -> None: self.stopped = True # before the await: no new borrows once teardown begins await run_shielded(self.teardown()) + def stop_nowait(self) -> None: + """`stop`, without waiting for it. For an owner with nothing left to do in this + box: on a remote runtime teardown is an API round trip, and whatever comes next + should not sit behind it. + + Only safe once nothing will read from the box again. A context manager that also + calls `stop` on exit gets a no-op second teardown, not a second deletion, but it + will still await one — detach it (`AsyncExitStack.pop_all`) to get the benefit.""" + if self.stopped: + return + self.stopped = True # synchronous, as in `stop`: a task only schedules + task = asyncio.create_task(run_shielded(self.teardown())) + # asyncio holds only a weak reference to a running task, so an unawaited teardown + # can be collected mid-flight. `_LIVE` tracks the runtime, not this task. + _PENDING.add(task) + task.add_done_callback(_PENDING.discard) + async def teardown(self) -> None: """Free the provisioned resource, off the event loop. Override only for teardown that must be async (e.g. a remote API call); `stop` shields it from cancellation. From 85a3af6495f1c76d03303138da482c4523100226 Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Mon, 27 Jul 2026 03:46:15 +0000 Subject: [PATCH 07/24] refactor(v1): drop stop_nowait, tear the solver box down normally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured what the optimisation was buying: an awaited docker teardown is 66ms. For that it cost a public method on the Runtime ABC, a `_PENDING` set of unawaited tasks, and a defused context manager in env code — `pop_all()` to disarm `provision`, then taking ownership by hand — with a window between the two where a cancellation would orphan the box until the atexit backstop ran at process exit. It was also backwards on the resource question. Overlapping the solver's teardown with the judge's startup means an episode can hold two boxes at once; awaiting means it holds one. On a paid runtime that matters more than the latency does, and the latency is not on the throughput path anyway — the concurrency gate wraps the agent run, not teardown. The barrier is unchanged and was never the teardown: `collect()` returning is what gates everything downstream. `runtimes/base.py` is untouched again, byte-identical to main. Co-Authored-By: Claude Opus 5 (1M context) --- verifiers/v1/artifacts.py | 4 ++-- verifiers/v1/envs/agentic_judge/env.py | 13 ++++--------- verifiers/v1/runtimes/base.py | 20 -------------------- 3 files changed, 6 insertions(+), 31 deletions(-) diff --git a/verifiers/v1/artifacts.py b/verifiers/v1/artifacts.py index e0ad5abeee..1b000d194e 100644 --- a/verifiers/v1/artifacts.py +++ b/verifiers/v1/artifacts.py @@ -9,8 +9,8 @@ the same path in the grading box ("no translation", as in Harbor). `collect` runs while the agent's box is alive, right after `Task.finalize` produced the -files. It is the barrier: once it returns the box can be torn down (`Runtime.stop_nowait` -puts that teardown behind the grading box rather than in front of it). +files. It is the barrier: once it returns, nothing downstream needs the agent's box and +it can be torn down. """ from __future__ import annotations diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index 9bf376ba67..49fe6c4caf 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -19,7 +19,6 @@ import math import re import tomllib -from contextlib import AsyncExitStack from pathlib import Path from typing import Literal @@ -359,16 +358,12 @@ async def run(self, task: vf.Task, agents: vf.Agents) -> None: await agents.judge.run(judge_task, runtime=box) return - # The solver's box is detached rather than exited: `provision`'s own teardown - # would be awaited here, and the judge's box has no reason to wait behind it - # once collection has returned. If collection raises, `pop_all` is skipped and - # the context manager tears the box down as usual. - async with AsyncExitStack() as stack: - box = await stack.enter_async_context(agents.solver.provision(task)) + # Collection is the barrier: once it returns, nothing downstream needs the + # solver's box. Letting the context manager tear it down normally keeps an + # episode at one box rather than two, which is what costs on a paid runtime. + async with agents.solver.provision(task) as box: solution = await agents.solver.run(task, runtime=box) collected = await vf.collect(box, task.data.artifacts) - stack.pop_all() - box.stop_nowait() async with agents.judge.provision(task) as judge_box: await vf.restore(judge_box, collected) diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index e5b1141c3b..e8d80cb3b9 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -74,9 +74,6 @@ def parse_gpu(gpu: str | None) -> tuple[str | None, int]: # none of this. _LIVE: "weakref.WeakSet[Runtime]" = weakref.WeakSet() _atexit_armed = False -_PENDING: "set[asyncio.Task]" = set() -"""Unawaited teardowns from `stop_nowait`, held so asyncio's weak task references cannot -collect one mid-flight.""" def register(runtime: "Runtime") -> None: @@ -167,23 +164,6 @@ async def stop(self) -> None: self.stopped = True # before the await: no new borrows once teardown begins await run_shielded(self.teardown()) - def stop_nowait(self) -> None: - """`stop`, without waiting for it. For an owner with nothing left to do in this - box: on a remote runtime teardown is an API round trip, and whatever comes next - should not sit behind it. - - Only safe once nothing will read from the box again. A context manager that also - calls `stop` on exit gets a no-op second teardown, not a second deletion, but it - will still await one — detach it (`AsyncExitStack.pop_all`) to get the benefit.""" - if self.stopped: - return - self.stopped = True # synchronous, as in `stop`: a task only schedules - task = asyncio.create_task(run_shielded(self.teardown())) - # asyncio holds only a weak reference to a running task, so an unawaited teardown - # can be collected mid-flight. `_LIVE` tracks the runtime, not this task. - _PENDING.add(task) - task.add_done_callback(_PENDING.discard) - async def teardown(self) -> None: """Free the provisioned resource, off the event loop. Override only for teardown that must be async (e.g. a remote API call); `stop` shields it from cancellation. From 91fa03b7af0c3fd328cc2441469a1a3d38cc29da Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Mon, 27 Jul 2026 04:06:45 +0000 Subject: [PATCH 08/24] refactor(v1): drop typo guards on artifact sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_checked` rejected a relative source, a `..` component, and bare `/`. Those are author mistakes in task config, not anything an agent controls, and a capable author does not need the framework second-guessing a path they wrote. What remains is the one line that was doing work: stripping a trailing slash, so `/work` and `/work/` cannot key two entries for the same tree — the source doubles as the dict key and as restore's `rm -rf` target. Co-Authored-By: Claude Opus 5 (1M context) --- verifiers/v1/artifacts.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/verifiers/v1/artifacts.py b/verifiers/v1/artifacts.py index 1b000d194e..1e09f26068 100644 --- a/verifiers/v1/artifacts.py +++ b/verifiers/v1/artifacts.py @@ -65,7 +65,11 @@ async def collect( One archive per source: BusyBox `tar` (every alpine-based image) implements only `c`/`x`/`t` with no `-r` to append, and each source carries its own excludes anyway. """ - declared = [_checked(a) for a in artifacts or []] + # Trailing slash stripped only so `/work` and `/work/` cannot key two entries for + # the same tree — the source doubles as the dict key and as `restore`'s rm -rf target. + declared = [ + a.model_copy(update={"source": a.source.rstrip("/")}) for a in artifacts or [] + ] convention = PurePosixPath(CONVENTION_DIR) sweep = not any( (p := PurePosixPath(a.source)) == convention @@ -120,18 +124,6 @@ async def restore(runtime: Runtime, collected: dict[str, bytes]) -> None: ) -def _checked(artifact: Artifact) -> Artifact: - path = PurePosixPath(artifact.source) - if not path.is_absolute() or ".." in path.parts: - raise ArtifactError( - f"artifact source {artifact.source!r} must be an absolute path with no '..'" - ) - resolved = path.as_posix().rstrip("/") - if not resolved: - raise ArtifactError("artifact source '/' would sweep the whole image") - return artifact.model_copy(update={"source": resolved}) - - async def _tar_out(runtime: Runtime, artifact: Artifact, budget: int) -> bytes: path = f"/tmp/vf-artifact-{uuid.uuid4().hex}.tar" excludes = " ".join(f"--exclude={shlex.quote(p)}" for p in artifact.exclude) From e787bc92461363fce38a86ab2d140e115be0e31a Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Mon, 27 Jul 2026 04:25:56 +0000 Subject: [PATCH 09/24] fix(v1): let an unpinned judge runtime inherit the solver's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `AgentConfig.runtime` defaults to `SubprocessConfig` and has no `None` to tell unset from chosen, so with `isolated` as the default topology the subprocess guard rejected every config that pinned only the solver — including the bundled `configs/agentic_judge.toml` and `test_env_id_agentic_judge`, which is how this surfaced. The env raised at construction before any episode ran. The judge now falls back to the solver's runtime policy when it has not pinned a container, which is the same treatment `harness`, `model` and `sampling` already get for unpinned seats in `Env._episode_agents`. It is also the right default on its own terms: the grading box is supposed to boot from the solver's image, which is what makes "only the delta travels" true. A judge that pins its own runtime keeps it under `isolated`; under `shared` the solver's still wins, since there the judge's effective runtime IS the solver's box. The explicit subprocess guard goes with it — unreachable now that the fallback runs first, and the solver-side check already covers both-are-subprocess. Co-Authored-By: Claude Opus 5 (1M context) --- tests/v1/test_e2e.py | 22 ++++++++++++---------- verifiers/v1/envs/agentic_judge/env.py | 25 +++++++++++-------------- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/tests/v1/test_e2e.py b/tests/v1/test_e2e.py index 0c77546ae4..4364553737 100644 --- a/tests/v1/test_e2e.py +++ b/tests/v1/test_e2e.py @@ -392,21 +392,23 @@ async def test_env_id_best_of_n(run_v1, tmp_path): @pytest.mark.e2e async def test_env_id_agentic_judge(run_v1, tmp_path): - """The agentic judge over the echo taskset (needs docker): the box is - provisioned once from the solver's runtime policy, the solver plays in it, - the judge lands in the SAME box with the graded trace uploaded, - investigates with real execution, and its parsed verdict - lands on the solver's trace under the spec's reward key. Wiring, not taste: - the judge followed the verdict-file contract — the grade itself is the - model's call. Exercises the config surface too: a policy-only prompt - override (the verdict contract is appended regardless) and - reward-composition weights.""" + """The agentic judge over the echo taskset (needs docker), on the default + `isolated` topology: the solver plays in a box provisioned from its runtime + policy, that box is torn down, and the judge gets its own from the same + policy with the graded trace uploaded and the task's declared artifacts + restored. Its parsed verdict lands on the solver's trace under the spec's + reward key. Wiring, not taste: the judge followed the verdict-file contract + — the grade itself is the model's call. Exercises the config surface too: an + unpinned judge runtime (inherits the solver's), a policy-only prompt override + (the verdict contract is appended regardless), and reward-composition + weights.""" traces = await run_v1( "echo-v1", harness=None, # seats pin their own harness; there is no run-level one env={ "id": "agentic-judge", - # The solver owns the shared box, so the container is pinned here. + # Pinned here only; the judge's runtime is left unset to exercise the + # inheritance that gives it the same image. "solver": {"harness": {"id": "bash"}, "runtime": {"type": "docker"}}, # The judge reads the trace and reasons before it writes the # verdict file; the shared 2048-token run cap truncates it mid-audit. diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index 49fe6c4caf..2de1944ef5 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -304,11 +304,17 @@ class AgenticJudgeEnvConfig(vf.EnvConfig): class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]): def __init__(self, config: AgenticJudgeEnvConfig) -> None: - if config.topology == "shared": - # Sharing means the judge's effective runtime IS the solver's policy — - # aligning the config keeps the base env's subprocess warning and the - # runtime stamped on the judge's trace truthful. Under `isolated` the - # judge provisions its own box, so its policy is honored as written. + # The judge inherits the solver's runtime policy unless it pins a container of + # its own. Under `shared` that is definitional — its effective runtime IS the + # solver's box, and aligning the config keeps the base env's subprocess warning + # and the runtime stamped on its trace truthful. Under `isolated` it is the + # fallback every other unpinned `AgentConfig` field already gets (harness, + # model, sampling): `runtime` defaults to `SubprocessConfig` with no `None` to + # distinguish unset from chosen, and a code-executing judge can never run on the + # host anyway, so subprocess here always means "not specified". + if config.topology == "shared" or isinstance( + config.judge.runtime, vf.SubprocessConfig + ): config.judge = config.judge.model_copy( update={"runtime": config.solver.runtime} ) @@ -336,15 +342,6 @@ def _check_agents(self) -> None: "solver resolves to the subprocess runtime; use " "--env.solver.runtime.type docker or prime" ) - if self.config.topology == "isolated" and isinstance( - self.config.judge.runtime, vf.SubprocessConfig - ): - raise ValueError( - "agentic-judge grades in the judge's own box under " - "--env.topology isolated, but the judge resolves to the subprocess " - "runtime (which would run the judge's code on the host); use " - "--env.judge.runtime.type docker or prime, or --env.topology shared" - ) async def setup(self, agents: vf.Agents) -> None: # The judge grades the policy; its tokens are never training data. From 4492b14e7d737ae933405574a077d6132363b307 Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:04:46 +0000 Subject: [PATCH 10/24] fix(v1): resolve relative artifact sources against the runtime workdir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Harbor permits a relative `source` for the main service, and the existence probe resolves one against the runtime's workdir — but the tar runs `-C /`, so it did not. `solution.txt` probed `$workdir/solution.txt`, passed, then archived `/solution.txt`. Not an error: a different file, collected and graded silently. Reproduced with a decoy at the root, which is what got collected. Resolution happens at collection rather than at parse: the workdir is a runtime property and `TaskData.workdir` can override it, so it isn't known when task.toml is read. Also applies review comments on the docs — drops a Harbor paragraph that repeated the Shortcomings list two lines below it, and settles on "sandbox" over "box" in the artifacts section. Co-Authored-By: Claude Opus 5 (1M context) --- docs/v1/harbor.md | 2 -- docs/v1/tasksets.md | 6 +++--- verifiers/v1/artifacts.py | 21 ++++++++++++++++++--- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/docs/v1/harbor.md b/docs/v1/harbor.md index a767a28cc2..370e3d1630 100644 --- a/docs/v1/harbor.md +++ b/docs/v1/harbor.md @@ -107,8 +107,6 @@ Two deliberate differences from `harbor run`: - **A failing collect hook fails the rollout.** Harbor logs it and carries on, because there the output is observability; here it is a grading input, and a silently absent file makes the verifier score a stale state. - **`destination` has no effect.** It positions a file in Harbor's host trial directory; verifiers has no trial directory (the trace is the record), and Harbor never lets `destination` affect verifier-side placement. -Sidecar `service` entries, `[verifier].user`, and an explicit `[verifier.environment]` image are rejected at load. The grading box is built from the task's own image, so only the agent's delta has to travel; a different verifier image would need the whole working tree copied across. - ## Shortcomings verifiers does not have parity with Harbor yet, so some features are missing and currently being worked on. The most notable missing features right now are: diff --git a/docs/v1/tasksets.md b/docs/v1/tasksets.md index 7be31cc135..4a0c4bc284 100644 --- a/docs/v1/tasksets.md +++ b/docs/v1/tasksets.md @@ -233,12 +233,12 @@ To override the judge model, set `env.taskset.task.judge.model` in your config ( ## Grading artifacts -An environment can grade in a second box rather than the one the agent worked in, so nothing the agent did to its environment can reach the grader. Only what the task declares crosses over. +An environment can grade in a second sandbox rather than the one the agent worked in, so nothing the agent did to its environment can reach the grader. Only what the task declares crosses over. Two channels, and they do different jobs: - **`trace.info` is the record.** `capture_patch` puts the diff in `trace.info["patch"]`, a judge puts its verdict there, and both ride `traces.jsonl`. It never travels to another box. -- **`/logs/artifacts/` is transport.** Anything written there is collected with no declaration at all, carried to the host, and restored in the grading box at the same path. +- **`/logs/artifacts/` is transport.** Anything written there is collected with no declaration at all, carried to the host, and restored in the grading sandbox at the same path. Produce artifacts in `finalize`, while the runtime is live and before scoring mutates anything: @@ -264,7 +264,7 @@ class MyData(vf.TaskData): A declared path that is missing at collection time fails the rollout: it was declared because grading needs it, and grading a partial state scores the rollout wrong rather than loudly failing it. The convention dir is exempt — it is collected for every task, and most never write to it. -The grading box boots from the same image as the agent's, so the repo and its dependencies are already present. Only the agent's delta has to travel, which is why the collection cap (`vf.artifacts.MAX_ARTIFACT_BYTES`) is sized for a patch rather than a tree. +The grading sandbox boots from the same image as the agent's, so the repo and its dependencies are already present. Only the agent's delta has to travel, which is why the collection cap (`vf.artifacts.MAX_ARTIFACT_BYTES`) is sized for a patch rather than a tree. ## Beyond one agent diff --git a/verifiers/v1/artifacts.py b/verifiers/v1/artifacts.py index ebdb020e71..911ee6ec89 100644 --- a/verifiers/v1/artifacts.py +++ b/verifiers/v1/artifacts.py @@ -67,10 +67,10 @@ async def collect( One archive per source: BusyBox `tar` (every alpine-based image) implements only `c`/`x`/`t` with no `-r` to append, and each source carries its own excludes anyway. """ - # Trailing slash stripped only so `/work` and `/work/` cannot key two entries for - # the same tree — the source doubles as the dict key and as `restore`'s rm -rf target. + workdir = getattr(runtime.config, "workdir", "") or "/" declared = [ - a.model_copy(update={"source": a.source.rstrip("/")}) for a in artifacts or [] + a.model_copy(update={"source": _resolve(a.source, workdir)}) + for a in artifacts or [] ] convention = PurePosixPath(CONVENTION_DIR) sweep = not any( @@ -126,6 +126,21 @@ async def restore(runtime: Runtime, collected: dict[str, bytes]) -> None: ) +def _resolve(source: str, workdir: str) -> str: + """A declared source as an absolute path in the box. + + Harbor permits a relative source for the main service, and the existence probe + resolves one against the runtime's workdir — so the tar, which runs `-C /`, has to + agree. Left unresolved, `solution.txt` probes `$workdir/solution.txt`, passes, then + archives `/solution.txt`: a different file, collected without an error. + + The trailing slash goes so `/work` and `/work/` cannot key two entries for one tree; + the source doubles as the dict key and as `restore`'s `rm -rf` target. + """ + absolute = source if source.startswith("/") else f"{workdir.rstrip('/')}/{source}" + return absolute.rstrip("/") + + async def _tar_out(runtime: Runtime, artifact: Artifact, budget: int) -> bytes: path = f"/tmp/vf-artifact-{uuid.uuid4().hex}.tar" excludes = " ".join(f"--exclude={shlex.quote(p)}" for p in artifact.exclude) From 8355eaf3d62e8735bb787bcbf0586c09c3ffba58 Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:07:09 +0000 Subject: [PATCH 11/24] refactor(v1): inline the workdir resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `PurePosixPath` join already does the whole job — an absolute source discards the workdir, a relative one joins onto it, and a trailing slash normalises away — so the helper and its `rstrip` were both redundant. Co-Authored-By: Claude Opus 5 (1M context) --- verifiers/v1/artifacts.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/verifiers/v1/artifacts.py b/verifiers/v1/artifacts.py index 911ee6ec89..d1674bf2d5 100644 --- a/verifiers/v1/artifacts.py +++ b/verifiers/v1/artifacts.py @@ -67,9 +67,13 @@ async def collect( One archive per source: BusyBox `tar` (every alpine-based image) implements only `c`/`x`/`t` with no `-r` to append, and each source carries its own excludes anyway. """ - workdir = getattr(runtime.config, "workdir", "") or "/" + # Harbor permits a relative source, and the probe below resolves one against the + # runtime's workdir — so the tar, which runs `-C /`, has to agree or it archives a + # different file. Joining also normalises `/work/` to `/work`, so one tree cannot + # key two entries (the source is both the dict key and `restore`'s rm -rf target). + workdir = PurePosixPath(getattr(runtime.config, "workdir", "") or "/") declared = [ - a.model_copy(update={"source": _resolve(a.source, workdir)}) + a.model_copy(update={"source": str(workdir / a.source)}) for a in artifacts or [] ] convention = PurePosixPath(CONVENTION_DIR) @@ -126,21 +130,6 @@ async def restore(runtime: Runtime, collected: dict[str, bytes]) -> None: ) -def _resolve(source: str, workdir: str) -> str: - """A declared source as an absolute path in the box. - - Harbor permits a relative source for the main service, and the existence probe - resolves one against the runtime's workdir — so the tar, which runs `-C /`, has to - agree. Left unresolved, `solution.txt` probes `$workdir/solution.txt`, passes, then - archives `/solution.txt`: a different file, collected without an error. - - The trailing slash goes so `/work` and `/work/` cannot key two entries for one tree; - the source doubles as the dict key and as `restore`'s `rm -rf` target. - """ - absolute = source if source.startswith("/") else f"{workdir.rstrip('/')}/{source}" - return absolute.rstrip("/") - - async def _tar_out(runtime: Runtime, artifact: Artifact, budget: int) -> bytes: path = f"/tmp/vf-artifact-{uuid.uuid4().hex}.tar" excludes = " ".join(f"--exclude={shlex.quote(p)}" for p in artifact.exclude) From 9fe09e068613a9eb3101b9d7010076bcf22f4801 Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:18:04 +0000 Subject: [PATCH 12/24] fix(v1): attribute patch-capture failures, and skip grading on infra failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `capture_patch` treated every failure the same — record `patch_error`, score the rollout anyway. Two different things were being conflated: - the box answered and git refused: a stale `index.lock` from a killed agent command, a deleted `.git`, a `base_commit` the agent rewrote away, a disk it filled. The agent's own environment, so it still records and still scores. A run with no patch grades as a run that changed nothing, which is the right reward. - the box never answered: sandbox gone, exec timed out, transport dropped. Nothing the policy did, so it now raises. Scoring it would feed training a zero that says only that our infrastructure failed. Telling them apart needs more than the exception boundary: `DockerRuntime.run` returns `docker exec`'s own non-zero result rather than raising, so a dead container is indistinguishable from broken git by exit code alone. On the failure path only, one `true` probe asks the box whether it is still there. The judge sandbox is no longer provisioned when the solver rollout errored — its scoring never ran either, so grading it spends a second box to reproduce a failure the trace already records. `finalize` tolerates the missing judge trace. Also corrects the `trace.info` documentation: it is not a transport channel, but an agentic judge does receive the whole serialized trace including `info` at `/tmp/trace.json`, so anything left there is visible to the grader. Co-Authored-By: Claude Opus 5 (1M context) --- docs/v1/tasksets.md | 2 +- verifiers/v1/envs/agentic_judge/env.py | 7 ++++ verifiers/v1/utils/git.py | 48 ++++++++++++++++++-------- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/docs/v1/tasksets.md b/docs/v1/tasksets.md index 4a0c4bc284..cb773bcd57 100644 --- a/docs/v1/tasksets.md +++ b/docs/v1/tasksets.md @@ -237,7 +237,7 @@ An environment can grade in a second sandbox rather than the one the agent worke Two channels, and they do different jobs: -- **`trace.info` is the record.** `capture_patch` puts the diff in `trace.info["patch"]`, a judge puts its verdict there, and both ride `traces.jsonl`. It never travels to another box. +- **`trace.info` is the record.** `capture_patch` puts the diff in `trace.info["patch"]`, a judge puts its verdict there, and both ride `traces.jsonl`. It is not a transport channel — nothing you put there is placed in the grading sandbox as a file. An agentic judge is the exception: it receives the whole serialized trace, `info` included, at `/tmp/trace.json`, so treat anything you leave there as visible to the grader. - **`/logs/artifacts/` is transport.** Anything written there is collected with no declaration at all, carried to the host, and restored in the grading sandbox at the same path. Produce artifacts in `finalize`, while the runtime is live and before scoring mutates anything: diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index 46cb7a2aaa..e8f45d7bd2 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -360,6 +360,11 @@ async def run(self, task: vf.Task, agents: vf.Agents) -> None: # episode at one box rather than two, which is what costs on a paid runtime. async with agents.solver.provision(task) as box: solution = await agents.solver.run(task, runtime=box) + if not solution.ok: + # The rollout errored, so its own scoring never ran either. Provisioning + # a second sandbox to grade it would spend a box to reproduce a failure + # already recorded on the trace. + return collected = await vf.collect(box, task.data.artifacts) async with agents.judge.provision(task) as judge_box: @@ -369,6 +374,8 @@ async def run(self, task: vf.Task, agents: vf.Agents) -> None: async def finalize(self, task: vf.Task, episode: vf.Episode) -> None: by_agent = {t.agent_name: t for t in episode.traces} + if "judge" not in by_agent: + return # solver errored; `run` skipped grading and the trace says why solution, verdict = by_agent["solver"], by_agent["judge"] data = verdict.info.get("verdict") if not isinstance(data, dict) or not isinstance(data.get("verdicts"), list): diff --git a/verifiers/v1/utils/git.py b/verifiers/v1/utils/git.py index 46d67fa003..4f0b18bcfe 100644 --- a/verifiers/v1/utils/git.py +++ b/verifiers/v1/utils/git.py @@ -18,6 +18,8 @@ from pathlib import PurePosixPath from typing import TYPE_CHECKING +from verifiers.v1.errors import SandboxError + if TYPE_CHECKING: from verifiers.v1.runtimes import Runtime from verifiers.v1.trace import Trace @@ -84,17 +86,25 @@ async def capture_patch( ) -> None: """Snapshot the agent's cumulative diff into `trace.info["patch"]`. - Best-effort by design: a rollout whose sandbox died or whose repo state is - broken records `info["patch_error"]` instead of failing the rollout — - scoring still runs and the error stays visible in results. + Two failure modes, attributed differently, because they deserve different outcomes. + + A non-zero exit means the box answered and git refused: a stale `index.lock` from a + killed agent command, a deleted `.git`, a `base_commit` the agent rewrote out of + existence, a disk it filled. That is the agent's own environment, so it records + `info["patch_error"]` and lets the rollout score — a run with no patch grades as a + run that changed nothing, which is the right reward. + + An exception means the box never answered: the sandbox died, the exec timed out, + the transport dropped. Nothing there is the policy's doing, so it raises. Scoring + the rollout anyway would feed a zero to training that says only that our + infrastructure failed. `write_path` additionally writes the patch to that path inside the box, for tasks - graded in a second box: `trace.info` is the durable record and never travels, so - a grader that needs the diff as a file needs it collected as an artifact. Point it - at `vf.CONVENTION_DIR` (e.g. `/logs/artifacts/patch.diff`) and collection picks it - up with no declaration. Publishing is best-effort like the rest of this helper — a - task that must not grade without the patch should declare that path as an - `Artifact`, which makes collection strict about it. + graded in a second sandbox. Point it at `vf.CONVENTION_DIR` (e.g. + `/logs/artifacts/patch.diff`) and collection picks it up with no declaration. Leave + it on the convention sweep rather than declaring it as an `Artifact`: a declared + path is collected strictly, which would turn an agent-broken repo into a rollout + error instead of the low score it should earn. """ nonce = uuid.uuid4().hex full, capped = f"{_FULL}_{nonce}", f"{_CAPPED}_{nonce}" @@ -105,14 +115,21 @@ async def capture_patch( {**(env or {}), "VF_DIFF_BASE": base_commit or "HEAD"}, ) if result.exit_code != 0: + # Not every runtime raises when the box is gone — Docker returns `docker + # exec`'s own non-zero result, which is indistinguishable from git failing. + # One probe on the failure path tells the two apart before we blame anyone. + if (await runtime.run(["true"], {})).exit_code != 0: + raise SandboxError( + f"patch capture failed and the box stopped answering: " + f"{(result.stderr or '').strip()[-300:]}" + ) trace.info["patch_error"] = ( f"exit={result.exit_code} {(result.stderr or '').strip()[-500:]}" ) return raw = await runtime.read(capped) - except Exception as exc: # noqa: BLE001 - capture must never fail the rollout - trace.info["patch_error"] = f"{type(exc).__name__}: {exc}" - return + except Exception as exc: + raise SandboxError(f"patch capture could not reach the box: {exc}") from exc finally: # Unique names don't overwrite each other, so leftovers would accumulate # on shared-filesystem runtimes; removal is best-effort by design. @@ -129,5 +146,8 @@ async def capture_patch( parent = str(PurePosixPath(write_path).parent) await runtime.run(["mkdir", "-p", parent], env or {}) await runtime.write(write_path, raw) - except Exception as exc: # noqa: BLE001 - the write must never fail the rollout. - trace.info["patch_write_error"] = f"{type(exc).__name__}: {exc}" + except Exception as exc: + # Transport again, not the policy: the patch exists, we could not place it. + raise SandboxError( + f"patch capture could not write {write_path!r}: {exc}" + ) from exc From 245f806f272d11742582ffb84a72f59ce0ecf8a8 Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Mon, 27 Jul 2026 22:57:02 +0000 Subject: [PATCH 13/24] docs(v1): say why the failed-solver path returns instead of raising MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comment argued only the cost — a second sandbox spent reproducing a known failure. The correctness argument was the one I got wrong twice while reviewing: `episode.ok` follows the failed trace on its own, and `episode_should_retry` classifies off that trace's errors, so the real exception type is what decides whether to retry. Raising an `EnvError` here would add a second, less specific error and bury it. Co-Authored-By: Claude Opus 5 (1M context) --- verifiers/v1/envs/agentic_judge/env.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index e8f45d7bd2..226f4f8633 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -361,9 +361,11 @@ async def run(self, task: vf.Task, agents: vf.Agents) -> None: async with agents.solver.provision(task) as box: solution = await agents.solver.run(task, runtime=box) if not solution.ok: - # The rollout errored, so its own scoring never ran either. Provisioning - # a second sandbox to grade it would spend a box to reproduce a failure - # already recorded on the trace. + # The rollout errored, so its own scoring never ran either; grading it + # would spend a second sandbox to reproduce a failure the trace already + # records. Return rather than raise: `episode.ok` follows the failed + # trace, and `episode_should_retry` classifies off that trace's own + # error — an exception raised here would bury the real type. return collected = await vf.collect(box, task.data.artifacts) From a6b82e3a35cc459caf2a229488cdf99a3d0df25e Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Mon, 27 Jul 2026 23:21:42 +0000 Subject: [PATCH 14/24] fix(v1): keep `shared` as the agentic-judge default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI caught this: test_env_id_agentic_judge failed on one of three Python jobs and passed on the other two — not a version difference, a flaky judge. Under the isolated default it was put in a fresh box, told "the agent's environment is gone", and asked for a criterion that reads "you verified it with real execution". echo-v1 publishes nothing, so there was nothing to execute; two models talked themselves into a verdict and the third stalled without writing one. The fixture is not the real problem. Nothing in research-environments publishes to the box yet — `git grep -l "write_path\|CONVENTION_DIR\|logs/artifacts"` over its environments/ is empty, because `write_path` is new here and unused. So isolated-by-default would have changed grading for every consumer of that repo's judge setups, and the three plain-Harbor tasksets would have broken outright: their hints open with "Reconstruct the agent's change from the box: `git status`, `git diff`, `git log` in /testbed", which under isolation is a pristine checkout that grades every attempt as untouched. So the default returns to `shared` — the topology that has actually been measured — and `isolated` becomes opt-in per taskset, adopted once that taskset puts its evidence somewhere that travels. The seven capture_patch tasksets are nearly there already; the trace record they rely on does travel. The isolated path itself is unchanged and still verified: two sandboxes, solver's torn down first, artifacts restored at their original paths, trace uploaded, judge told it is in a fresh box. Co-Authored-By: Claude Opus 5 (1M context) --- tests/v1/test_e2e.py | 21 ++++++++--------- verifiers/v1/envs/agentic_judge/env.py | 32 ++++++++++++++++---------- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/tests/v1/test_e2e.py b/tests/v1/test_e2e.py index 4364553737..08c6adcf89 100644 --- a/tests/v1/test_e2e.py +++ b/tests/v1/test_e2e.py @@ -393,22 +393,21 @@ async def test_env_id_best_of_n(run_v1, tmp_path): @pytest.mark.e2e async def test_env_id_agentic_judge(run_v1, tmp_path): """The agentic judge over the echo taskset (needs docker), on the default - `isolated` topology: the solver plays in a box provisioned from its runtime - policy, that box is torn down, and the judge gets its own from the same - policy with the graded trace uploaded and the task's declared artifacts - restored. Its parsed verdict lands on the solver's trace under the spec's - reward key. Wiring, not taste: the judge followed the verdict-file contract - — the grade itself is the model's call. Exercises the config surface too: an - unpinned judge runtime (inherits the solver's), a policy-only prompt override - (the verdict contract is appended regardless), and reward-composition - weights.""" + `shared` topology: the box is provisioned once from the solver's runtime + policy, the solver plays the task in it, the judge lands in the SAME box with + the graded trace uploaded, investigates with real execution, and its parsed + verdict lands on the solver's trace under the spec's reward key. Wiring, not + taste: the judge followed the verdict-file contract — the grade itself is the + model's call. Exercises the config surface too: an unpinned judge runtime + (inherits the solver's), a policy-only prompt override (the verdict contract + is appended regardless), and reward-composition weights.""" traces = await run_v1( "echo-v1", harness=None, # seats pin their own harness; there is no run-level one env={ "id": "agentic-judge", - # Pinned here only; the judge's runtime is left unset to exercise the - # inheritance that gives it the same image. + # The solver owns the shared box, so the container is pinned here; the + # judge's runtime is left unset to exercise the inheritance. "solver": {"harness": {"id": "bash"}, "runtime": {"type": "docker"}}, # The judge reads the trace and reasons before it writes the # verdict file; the shared 2048-token run cap truncates it mid-audit. diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index 226f4f8633..2ebe4d2093 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -8,11 +8,12 @@ `judge/` metrics plus a weighted-mean `judge` reward, composed with the taskset's own rewards via `[env.score]` (judge-only by default). -`--env.topology` decides where the judge stands. Under `isolated` (the default) it -gets its own box from the same image, holding only what the task declared as -artifacts, so nothing the agent did to its environment can reach the grader. Under -`shared` it plays in the box the agent worked in, seeing that environment directly -and every seam in it. +`--env.topology` decides where the judge stands. Under `shared` (the default) it +plays in the box the agent worked in, seeing that environment directly and every +seam in it. Under `isolated` it gets its own box from the same image, holding only +what the task declared as artifacts, so nothing the agent did to its environment +can reach the grader — worth opting into once a taskset publishes its evidence +somewhere that travels. """ import json @@ -156,7 +157,7 @@ def from_trace( cls, solution: vf.Trace, config: "JudgeTaskConfig", - topology: str = "isolated", + topology: str = "shared", ) -> "JudgeTask": """Mint the judge's task from the solver's finished trace. @@ -290,14 +291,21 @@ class AgenticJudgeEnvConfig(vf.EnvConfig): judge: vf.AgentConfig = vf.AgentConfig() """The judge agent. Under `isolated` it provisions its own box from this policy; under `shared` it plays in the solver's box and this policy is ignored.""" - topology: Literal["isolated", "shared"] = "isolated" - """Whether the judge grades in its own box or the solver's. + topology: Literal["shared", "isolated"] = "shared" + """Whether the judge grades in the solver's box or its own. + + `shared` places the judge in the box the agent worked in, so it can inspect that + environment directly — at the cost of leaving every seam in it reachable. It is the + default because it is what has been measured, and because a judge only gains from + isolation once the task publishes what the judge needs. `isolated` boots a second box from the same image, carries the task's declared - artifacts across, and grades there — so nothing the agent did to its own - environment can reach the grader. `shared` places the judge in the box the agent - worked in, which lets it inspect that environment directly at the cost of leaving - every seam in it reachable.""" + artifacts across, and grades there, so nothing the agent did to its environment can + reach the grader. Opt in per taskset, and only once that taskset puts its evidence + somewhere that travels: an artifact under `vf.CONVENTION_DIR` (see + `capture_patch(write_path=...)`), a declared `TaskData.artifacts` path, or the trace + record itself. A task whose judge hint says to run `git diff` in the box will find + a pristine checkout here and grade every attempt as untouched.""" task: JudgeTaskConfig = JudgeTaskConfig() score: ScoreConfig = ScoreConfig() From e7ecfafed5be250dd17cb38cae94e1550ee90427 Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:33:06 +0000 Subject: [PATCH 15/24] fix(v1): capture the agent's edits only, not the image's untracked files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `git add -A` staged everything untracked, so a captured patch carried files the task image shipped and the agent never opened. On R2E-Gym that is three per box (`datasets`, `install.sh`, `run_tests.sh`), and the resulting patch fails `git apply` in a fresh container of that same image — which is exactly what an isolated grading box is. The isolated-judge path was broken for every taskset whose image ships untracked files. Drop untracked files whose mtime predates the agent's first turn. The margin is not subtle: R2E-Gym's are dated 2025-01, the rollouts run now. Pass the cutoff as an age rather than an absolute timestamp — a remote sandbox runs on its own machine, and only a duration survives the clock skew. The listing has to happen before `add -A`, after which nothing is "other" any more. `ignore=` unstages named paths on top, for a taskset that knows its image better than the mtime rule does. Co-Authored-By: Claude Opus 5 (1M context) --- verifiers/v1/utils/git.py | 55 ++++++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/verifiers/v1/utils/git.py b/verifiers/v1/utils/git.py index 4f0b18bcfe..6376b95d15 100644 --- a/verifiers/v1/utils/git.py +++ b/verifiers/v1/utils/git.py @@ -3,7 +3,8 @@ SWE-style tasksets call `capture_patch` from `Task.finalize` — after the harness finishes, while the runtime is live, before scoring mutates the repo (restoring test files, switching commits) — so the diff is exactly what the agent produced, -including edits to test files (intentional: they reveal reward hacking). +including edits to test files (intentional: they reveal reward hacking), and +excluding untracked files the image shipped and the agent never touched. The diff is taken against `base_commit` when the caller has one — a dataset row field, or a SHA recorded with `resolve_head` at setup time and kept in host @@ -14,6 +15,7 @@ from __future__ import annotations +import time import uuid from pathlib import PurePosixPath from typing import TYPE_CHECKING @@ -34,6 +36,8 @@ # rollout forever. _FULL = "/tmp/vf_agent_patch_full" _CAPPED = "/tmp/vf_agent_patch" +_STALE = "/tmp/vf_agent_stale" +_T0 = "/tmp/vf_agent_t0" # `git reset -q` must run even when staging or diffing fails, or the error path # leaves the tree staged and can break scoring's later checkouts. Every step @@ -44,17 +48,32 @@ # tree staged, so reporting success would hide a state later scoring may trip # on; a failed head leaves an empty {capped} (the redirect truncates it before # head runs), which would read back as a silently empty patch. +# +# The two unstage steps run between `add` and `diff` and are deliberately not part of +# that accounting: if either fails the patch is merely as wide as it used to be, which +# is a worse patch, not a broken rollout. `xargs -r` matters — without it an empty +# stale list would run `git reset -q --` with no pathspec and unstage everything. _DIFF = ( - "rm -f {full} {capped}; " + "rm -f {full} {capped} {t0}; : > {stale}; " + # Untracked files older than the agent's first write are the image's, not the + # agent's. Listing has to happen before `add -A`, after which nothing is "other" + # any more. Age rather than an absolute cutoff: a remote sandbox keeps its own + # clock, and only a duration survives the trip across the skew. + 'if [ -n "$VF_AGENT_AGE" ] && touch -d "@$(($(date +%s) - $VF_AGENT_AGE))" {t0}; ' + "then git ls-files --others --exclude-standard -z " + "| xargs -0 -r sh -c 'find \"$@\" -maxdepth 0 ! -newer {t0} -print0' _ > {stale}; " + "fi; " "git add -A; " "add_rc=$?; " + "xargs -0 -r git reset -q -- < {stale}; " + '[ "$#" -gt 0 ] && git reset -q -- "$@"; ' 'git -c core.quotepath=off diff --cached --binary "$VF_DIFF_BASE" > {full}; ' "diff_rc=$?; " "git reset -q; " "reset_rc=$?; " "head -c {cap} {full} > {capped}; " "head_rc=$?; " - "rm -f {full}; " + "rm -f {full} {stale} {t0}; " "rc=$head_rc; " '[ "$diff_rc" -ne 0 ] && rc=$diff_rc; ' '[ "$reset_rc" -ne 0 ] && rc=$reset_rc; ' @@ -83,9 +102,16 @@ async def capture_patch( base_commit: str = "", env: dict | None = None, write_path: str | None = None, + ignore: list[str] | None = None, ) -> None: """Snapshot the agent's cumulative diff into `trace.info["patch"]`. + Untracked files whose mtime predates the agent's first turn are left out: they came + with the image, not the policy. R2E-Gym boxes ship three (`datasets`, `install.sh`, + `run_tests.sh`), and a patch carrying them fails `git apply` in a fresh container of + that very image — which is what an isolated grading box is. `ignore` unstages named + paths on top, for a taskset that knows its image better than the mtime rule does. + Two failure modes, attributed differently, because they deserve different outcomes. A non-zero exit means the box answered and git refused: a stale `index.lock` from a @@ -108,11 +134,26 @@ async def capture_patch( """ nonce = uuid.uuid4().hex full, capped = f"{_FULL}_{nonce}", f"{_CAPPED}_{nonce}" - cmd = _DIFF.format(full=full, capped=capped, cap=PATCH_CAP_BYTES + 1) + stale, t0 = f"{_STALE}_{nonce}", f"{_T0}_{nonce}" + cmd = _DIFF.format( + full=full, capped=capped, stale=stale, t0=t0, cap=PATCH_CAP_BYTES + 1 + ) + # An mtime is a heuristic. The exact answer is the untracked set as it stood before + # the agent ran, and sandbox snapshotting — once runtimes can snapshot and diff a + # filesystem — gives that directly, with no setup-side bookkeeping in each taskset. + # Replace this when it lands. Until then, err a few seconds early: the cutoff wants + # to sit just before the agent's first write, and overshooting backwards only + # readmits files setup wrote moments earlier, while undershooting drops the agent's. + started = trace.timing.generation.start or trace.timing.setup.start + age = str(max(0, int(time.time() - started)) + 5) if started else "" try: result = await runtime.run( - ["sh", "-c", cmd], - {**(env or {}), "VF_DIFF_BASE": base_commit or "HEAD"}, + ["sh", "-c", cmd, "vf-capture-patch", *(ignore or [])], + { + **(env or {}), + "VF_DIFF_BASE": base_commit or "HEAD", + "VF_AGENT_AGE": age, + }, ) if result.exit_code != 0: # Not every runtime raises when the box is gone — Docker returns `docker @@ -134,7 +175,7 @@ async def capture_patch( # Unique names don't overwrite each other, so leftovers would accumulate # on shared-filesystem runtimes; removal is best-effort by design. try: - await runtime.run(["rm", "-f", full, capped], env or {}) + await runtime.run(["rm", "-f", full, capped, stale, t0], env or {}) except Exception: # noqa: BLE001, S110 - cleanup must never fail the rollout pass if len(raw) > PATCH_CAP_BYTES: From 7236d4baaacf8f8f7d9e6e2ed8ed10a3a372cbec Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:42:02 +0000 Subject: [PATCH 16/24] feat(v1): `snapshot_untracked`, the exact form of the same exclusion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mtime cutoff leans on the image's untracked files being old. R2E-Gym's are 556 days old, so it holds today — but it holds for no image we build ourselves, where everything is minutes old at rollout time. Record the untracked set in `setup`, before the agent runs, and hand it to `capture_patch` as `ignore`. Same host-memory pattern as `resolve_head`'s SHA, and no clock in the answer. Tasksets that don't call it keep the mtime rule. Also round the mtime cutoff backwards rather than truncating it: truncation and the round trip into the box both push it later, and a cutoff past the agent's first write drops real work. Co-Authored-By: Claude Opus 5 (1M context) --- verifiers/v1/__init__.py | 4 ++++ verifiers/v1/utils/git.py | 45 +++++++++++++++++++++++++++++++-------- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 847ddb4f3b..90a66fd57e 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -162,6 +162,9 @@ from verifiers.v1.utils.git import ( resolve_head as resolve_head, ) +from verifiers.v1.utils.git import ( + snapshot_untracked as snapshot_untracked, +) __all__ = [ # noqa: RUF022 - grouped by public API area # types @@ -300,6 +303,7 @@ "PATCH_CAP_BYTES", "capture_patch", "resolve_head", + "snapshot_untracked", # grading artifacts "CONVENTION_DIR", "Artifact", diff --git a/verifiers/v1/utils/git.py b/verifiers/v1/utils/git.py index 6376b95d15..2aa7449e7a 100644 --- a/verifiers/v1/utils/git.py +++ b/verifiers/v1/utils/git.py @@ -17,6 +17,7 @@ import time import uuid +from math import ceil from pathlib import PurePosixPath from typing import TYPE_CHECKING @@ -96,6 +97,26 @@ async def resolve_head(runtime: Runtime, env: dict | None = None) -> str: return (result.stdout or "").strip() +async def snapshot_untracked(runtime: Runtime, env: dict | None = None) -> list[str]: + """The repo's untracked files, to hand `capture_patch` as `ignore`. + + Call at the end of `setup`, before the agent runs, and keep the result in host + memory beside `resolve_head`'s SHA. Whatever it lists came with the image, so the + agent cannot be credited with it, and a patch that carries it fails `git apply` in + a fresh container of that same image — which is what an isolated grading box is. + + Exact where `capture_patch`'s own mtime rule is a heuristic, and the only one of the + two that works on an image built minutes before the rollout, where every file looks + new. Prefer it; the mtime rule is what a taskset that never calls this still gets. + """ + result = await runtime.run( + ["sh", "-c", "git ls-files --others --exclude-standard -z"], env or {} + ) + if result.exit_code != 0: + return [] + return [path for path in (result.stdout or "").split("\0") if path] + + async def capture_patch( trace: Trace, runtime: Runtime, @@ -109,8 +130,11 @@ async def capture_patch( Untracked files whose mtime predates the agent's first turn are left out: they came with the image, not the policy. R2E-Gym boxes ship three (`datasets`, `install.sh`, `run_tests.sh`), and a patch carrying them fails `git apply` in a fresh container of - that very image — which is what an isolated grading box is. `ignore` unstages named - paths on top, for a taskset that knows its image better than the mtime rule does. + that very image — which is what an isolated grading box is. + + `ignore` unstages named paths on top of that. Pass `snapshot_untracked`'s list from + setup and the guess becomes a fact; the mtime rule is the fallback for tasksets that + don't. Two failure modes, attributed differently, because they deserve different outcomes. @@ -138,14 +162,17 @@ async def capture_patch( cmd = _DIFF.format( full=full, capped=capped, stale=stale, t0=t0, cap=PATCH_CAP_BYTES + 1 ) - # An mtime is a heuristic. The exact answer is the untracked set as it stood before - # the agent ran, and sandbox snapshotting — once runtimes can snapshot and diff a - # filesystem — gives that directly, with no setup-side bookkeeping in each taskset. - # Replace this when it lands. Until then, err a few seconds early: the cutoff wants - # to sit just before the agent's first write, and overshooting backwards only - # readmits files setup wrote moments earlier, while undershooting drops the agent's. + # An mtime is a heuristic — `snapshot_untracked` is the exact answer, and sandbox + # snapshotting will be a better one still: once runtimes can snapshot and diff a + # filesystem, the pre-agent untracked set comes for free, with no setup-side + # bookkeeping in any taskset. Replace this when that lands. + # + # Round the cutoff backwards, never forwards. Truncation and the round trip that + # carries this age into the box both push it later, and a cutoff that lands after + # the agent's first write drops real work; landing early only readmits whatever + # setup wrote in its last seconds. started = trace.timing.generation.start or trace.timing.setup.start - age = str(max(0, int(time.time() - started)) + 5) if started else "" + age = str(max(0, ceil(time.time() - started)) + 2) if started else "" try: result = await runtime.run( ["sh", "-c", cmd, "vf-capture-patch", *(ignore or [])], From 5564d3fd300a347eb78a138031a6dab296908b17 Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:47:09 +0000 Subject: [PATCH 17/24] refactor(v1): drop the mtime cutoff, keep only the recorded set Two mechanisms for one job, and the heuristic was the weaker half: it needs the image's untracked files to be old, which is true of R2E-Gym's and of no image we build ourselves. `snapshot_untracked` answers the same question from a record taken before the agent ran, with no clock in it. `capture_patch` is back to `git add -A` plus one unstage of `ignore`. Co-Authored-By: Claude Opus 5 (1M context) --- verifiers/v1/utils/git.py | 69 ++++++++++----------------------------- 1 file changed, 17 insertions(+), 52 deletions(-) diff --git a/verifiers/v1/utils/git.py b/verifiers/v1/utils/git.py index 2aa7449e7a..5fc95fbb93 100644 --- a/verifiers/v1/utils/git.py +++ b/verifiers/v1/utils/git.py @@ -4,7 +4,7 @@ finishes, while the runtime is live, before scoring mutates the repo (restoring test files, switching commits) — so the diff is exactly what the agent produced, including edits to test files (intentional: they reveal reward hacking), and -excluding untracked files the image shipped and the agent never touched. +excluding whatever `snapshot_untracked` recorded before the agent started. The diff is taken against `base_commit` when the caller has one — a dataset row field, or a SHA recorded with `resolve_head` at setup time and kept in host @@ -15,9 +15,7 @@ from __future__ import annotations -import time import uuid -from math import ceil from pathlib import PurePosixPath from typing import TYPE_CHECKING @@ -37,8 +35,6 @@ # rollout forever. _FULL = "/tmp/vf_agent_patch_full" _CAPPED = "/tmp/vf_agent_patch" -_STALE = "/tmp/vf_agent_stale" -_T0 = "/tmp/vf_agent_t0" # `git reset -q` must run even when staging or diffing fails, or the error path # leaves the tree staged and can break scoring's later checkouts. Every step @@ -50,23 +46,13 @@ # on; a failed head leaves an empty {capped} (the redirect truncates it before # head runs), which would read back as a silently empty patch. # -# The two unstage steps run between `add` and `diff` and are deliberately not part of -# that accounting: if either fails the patch is merely as wide as it used to be, which -# is a worse patch, not a broken rollout. `xargs -r` matters — without it an empty -# stale list would run `git reset -q --` with no pathspec and unstage everything. +# The unstage step is deliberately outside that accounting: if it fails the patch is +# merely as wide as it used to be, which is a worse patch, not a broken rollout. The +# `$#` test is load-bearing — `git reset -q --` with no pathspec unstages everything. _DIFF = ( - "rm -f {full} {capped} {t0}; : > {stale}; " - # Untracked files older than the agent's first write are the image's, not the - # agent's. Listing has to happen before `add -A`, after which nothing is "other" - # any more. Age rather than an absolute cutoff: a remote sandbox keeps its own - # clock, and only a duration survives the trip across the skew. - 'if [ -n "$VF_AGENT_AGE" ] && touch -d "@$(($(date +%s) - $VF_AGENT_AGE))" {t0}; ' - "then git ls-files --others --exclude-standard -z " - "| xargs -0 -r sh -c 'find \"$@\" -maxdepth 0 ! -newer {t0} -print0' _ > {stale}; " - "fi; " + "rm -f {full} {capped}; " "git add -A; " "add_rc=$?; " - "xargs -0 -r git reset -q -- < {stale}; " '[ "$#" -gt 0 ] && git reset -q -- "$@"; ' 'git -c core.quotepath=off diff --cached --binary "$VF_DIFF_BASE" > {full}; ' "diff_rc=$?; " @@ -74,7 +60,7 @@ "reset_rc=$?; " "head -c {cap} {full} > {capped}; " "head_rc=$?; " - "rm -f {full} {stale} {t0}; " + "rm -f {full}; " "rc=$head_rc; " '[ "$diff_rc" -ne 0 ] && rc=$diff_rc; ' '[ "$reset_rc" -ne 0 ] && rc=$reset_rc; ' @@ -105,9 +91,9 @@ async def snapshot_untracked(runtime: Runtime, env: dict | None = None) -> list[ agent cannot be credited with it, and a patch that carries it fails `git apply` in a fresh container of that same image — which is what an isolated grading box is. - Exact where `capture_patch`'s own mtime rule is a heuristic, and the only one of the - two that works on an image built minutes before the rollout, where every file looks - new. Prefer it; the mtime rule is what a taskset that never calls this still gets. + Sandbox snapshotting will make this free: once runtimes can snapshot and diff a + filesystem, the pre-agent untracked set falls out of the diff with no setup-side + bookkeeping in any taskset. Drop this then. """ result = await runtime.run( ["sh", "-c", "git ls-files --others --exclude-standard -z"], env or {} @@ -127,14 +113,11 @@ async def capture_patch( ) -> None: """Snapshot the agent's cumulative diff into `trace.info["patch"]`. - Untracked files whose mtime predates the agent's first turn are left out: they came - with the image, not the policy. R2E-Gym boxes ship three (`datasets`, `install.sh`, - `run_tests.sh`), and a patch carrying them fails `git apply` in a fresh container of - that very image — which is what an isolated grading box is. - - `ignore` unstages named paths on top of that. Pass `snapshot_untracked`'s list from - setup and the guess becomes a fact; the mtime rule is the fallback for tasksets that - don't. + `ignore` names paths to leave out — pass `snapshot_untracked`'s list from setup, or + `git add -A` credits the agent with untracked files the image shipped. R2E-Gym boxes + ship three (`datasets`, `install.sh`, `run_tests.sh`), and a patch carrying them + fails `git apply` in a fresh container of that very image — which is what an + isolated grading box is. Two failure modes, attributed differently, because they deserve different outcomes. @@ -158,29 +141,11 @@ async def capture_patch( """ nonce = uuid.uuid4().hex full, capped = f"{_FULL}_{nonce}", f"{_CAPPED}_{nonce}" - stale, t0 = f"{_STALE}_{nonce}", f"{_T0}_{nonce}" - cmd = _DIFF.format( - full=full, capped=capped, stale=stale, t0=t0, cap=PATCH_CAP_BYTES + 1 - ) - # An mtime is a heuristic — `snapshot_untracked` is the exact answer, and sandbox - # snapshotting will be a better one still: once runtimes can snapshot and diff a - # filesystem, the pre-agent untracked set comes for free, with no setup-side - # bookkeeping in any taskset. Replace this when that lands. - # - # Round the cutoff backwards, never forwards. Truncation and the round trip that - # carries this age into the box both push it later, and a cutoff that lands after - # the agent's first write drops real work; landing early only readmits whatever - # setup wrote in its last seconds. - started = trace.timing.generation.start or trace.timing.setup.start - age = str(max(0, ceil(time.time() - started)) + 2) if started else "" + cmd = _DIFF.format(full=full, capped=capped, cap=PATCH_CAP_BYTES + 1) try: result = await runtime.run( ["sh", "-c", cmd, "vf-capture-patch", *(ignore or [])], - { - **(env or {}), - "VF_DIFF_BASE": base_commit or "HEAD", - "VF_AGENT_AGE": age, - }, + {**(env or {}), "VF_DIFF_BASE": base_commit or "HEAD"}, ) if result.exit_code != 0: # Not every runtime raises when the box is gone — Docker returns `docker @@ -202,7 +167,7 @@ async def capture_patch( # Unique names don't overwrite each other, so leftovers would accumulate # on shared-filesystem runtimes; removal is best-effort by design. try: - await runtime.run(["rm", "-f", full, capped, stale, t0], env or {}) + await runtime.run(["rm", "-f", full, capped], env or {}) except Exception: # noqa: BLE001, S110 - cleanup must never fail the rollout pass if len(raw) > PATCH_CAP_BYTES: From 9ac6daffd3ca034534bc5eb4e9081a4d413220bd Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Tue, 28 Jul 2026 22:40:44 -0700 Subject: [PATCH 18/24] refactor(v1): address isolated grading review feedback (#2160) * docs(v1): simplify grading artifact guidance Addresses review thread PRRT_kwDONt9L486UYhGk. * docs(v1): keep grading artifacts with environments Addresses review thread PRRT_kwDONt9L486UYhzG. * docs(v1): simplify the agentic judge summary Addresses review thread PRRT_kwDONt9L486UYidF. * refactor(v1): rename the judge sandbox mode Addresses review thread PRRT_kwDONt9L486UYjoC. * refactor(v1): type the judge sandbox mode Addresses review thread PRRT_kwDONt9L486UYkYt. * refactor(v1): reuse the sandbox mode type Addresses review thread PRRT_kwDONt9L486UYnFX. * refactor(v1): remove runtime inheritance commentary Addresses review thread PRRT_kwDONt9L486UYnac. * refactor(v1): remove artifact barrier commentary Addresses review thread PRRT_kwDONt9L486UYnnt. * refactor(v1): trim Harbor artifact parser docs Addresses review thread PRRT_kwDONt9L486UYoq3. * chore(v1): defer runtime liveness cleanup No code change: reviewers marked this concern unrelated and tracked it separately. Addresses review thread PRRT_kwDONt9L486UYp1-. * docs(v1): clarify Harbor artifact destinations Addresses review thread PRRT_kwDONt9L486UYp42. * docs(v1): drop the sidecar shortcomings entry Addresses review thread PRRT_kwDONt9L486UYqM0. * docs(v1): generalize the artifact module summary Addresses review thread PRRT_kwDONt9L486UYqjr. * docs(v1): remove artifact module narration Addresses review thread PRRT_kwDONt9L486UYq2e. * test(v1): remove agentic judge commentary Addresses review thread PRRT_kwDONt9L486UYrLs. * docs(v1): shorten the shared workspace note Addresses review thread PRRT_kwDONt9L486UYrxU. * docs(v1): shorten the isolated workspace note Addresses review thread PRRT_kwDONt9L486UYr3o. * fix(v1): allow long Harbor collect hooks Addresses review thread PRRT_kwDONt9L486UYx9o. * fix(v1): name unsupported Harbor services directly Addresses review thread PRRT_kwDONt9L486UYzut. * docs(v1): remove image-specific tar assumptions Addresses review thread PRRT_kwDONt9L486UY0fH. * refactor(v1): rename the artifact directory constant Addresses review thread PRRT_kwDONt9L486UfNXW. * fix(v1): keep artifact errors framework-neutral Addresses review thread PRRT_kwDONt9L486UfUPM. * refactor(v1): use runtime errors for artifact failures Addresses review thread PRRT_kwDONt9L486UfYvv. * refactor(v1): drop the artifact rollout error type Addresses review thread PRRT_kwDONt9L486UfYhK. * fix(v1): propagate patch capture runtime errors Addresses review thread PRRT_kwDONt9L486UfcBe. * fix(v1): propagate patch publication errors Addresses review thread PRRT_kwDONt9L486UfciI. * refactor(v1): keep untracked snapshots internal Addresses review thread PRRT_kwDONt9L486Ufc8v. * refactor(v1): configure runtime sharing as a boolean Supersedes the earlier literal-mode shape and addresses review thread PRRT_kwDONt9L486UfeU7. * refactor(v1): hand artifacts through task hooks Addresses review thread PRRT_kwDONt9L486UfiZ9. * fix(v1): let task boundaries classify collect failures Addresses review thread PRRT_kwDONt9L486UfkEi. * docs(v1): remove Harbor behavior differences * fix(v1): skip judging failed solvers * docs(v1): remove code-executing phrasing * docs(v1): keep artifact core framework-neutral * fix(v1): clarify unsupported Harbor services --- docs/v1/env.md | 19 +++++ docs/v1/harbor.md | 6 -- docs/v1/tasksets.md | 35 -------- tests/v1/test_e2e.py | 18 +--- verifiers/v1/__init__.py | 10 +-- verifiers/v1/artifacts.py | 54 ++++-------- verifiers/v1/envs/agentic_judge/env.py | 109 ++++++++---------------- verifiers/v1/errors.py | 7 -- verifiers/v1/state.py | 7 +- verifiers/v1/tasksets/harbor/taskset.py | 30 +++---- verifiers/v1/utils/git.py | 16 +--- 11 files changed, 94 insertions(+), 217 deletions(-) diff --git a/docs/v1/env.md b/docs/v1/env.md index 1be0b86874..f1a88c0ac1 100644 --- a/docs/v1/env.md +++ b/docs/v1/env.md @@ -52,3 +52,22 @@ Just like tasksets and harnesses, an `Env` can be user-defined for full expressi | `single-agent` | `agent` | (default) one `agent` plays the taskset | | `best-of-n` | `agent` | `n` independent attempts per episode; its metrics mark the argmax-reward sibling (`best`) and whether any reached `--env.threshold` (`pass_at_n`) — rejection sampling and pass@k. | | `agentic-judge` | `solver`, `judge` | the solver plays the task; a code-executing judge agent verifies the finished attempt with real execution. | + +## Grading artifacts + +Use artifacts to carry files between runtimes. Files written to +`/logs/artifacts/` are collected implicitly; declare other paths on the task data: + +```python +class MyData(vf.TaskData): + artifacts: list[vf.Artifact] = [ + vf.Artifact(source="/work/report", exclude=[".git"]) + ] + + +class MyTask(vf.Task[MyData]): + async def finalize(self, trace: vf.Trace, runtime: vf.Runtime) -> None: + trace.state.artifacts = await vf.collect(runtime, self.data.artifacts) +``` + +Declared paths must exist when collected. The implicit directory is optional. diff --git a/docs/v1/harbor.md b/docs/v1/harbor.md index 370e3d1630..86b43e4a5f 100644 --- a/docs/v1/harbor.md +++ b/docs/v1/harbor.md @@ -102,15 +102,9 @@ accepts host-level entries and rejects combinations that need both policy modes. `artifacts = [...]` and `[[verifier.collect]]` are read from `task.toml` ([Harbor Docs](https://www.harborframework.com/docs/run-jobs/results-and-artifacts)). Collect hooks run in the agent's box from the task's `finalize`, which is Harbor's own ordering — after the agent phase, before collection — and declared paths plus the `/logs/artifacts/` convention dir are then carried into the grading box and restored at their original paths ("no translation", as in Harbor). -Two deliberate differences from `harbor run`: - -- **A failing collect hook fails the rollout.** Harbor logs it and carries on, because there the output is observability; here it is a grading input, and a silently absent file makes the verifier score a stale state. -- **`destination` has no effect.** It positions a file in Harbor's host trial directory; verifiers has no trial directory (the trace is the record), and Harbor never lets `destination` affect verifier-side placement. - ## Shortcomings verifiers does not have parity with Harbor yet, so some features are missing and currently being worked on. The most notable missing features right now are: - Switching to a different verifier-phase network policy ([Harbor Docs](https://www.harborframework.com/docs/tasks/network-policy)) -- Sidecar services, and the sidecar artifacts and collect hooks that go with them ([Harbor Docs](https://www.harborframework.com/docs/tasks#sidecar-artifacts-and-collect-hooks)) - Multi-step tasks ([Harbor Docs](https://www.harborframework.com/docs/tasks/multi-step)) diff --git a/docs/v1/tasksets.md b/docs/v1/tasksets.md index cb773bcd57..01252ffcc6 100644 --- a/docs/v1/tasksets.md +++ b/docs/v1/tasksets.md @@ -231,41 +231,6 @@ class JudgeTraceTaskset(vf.Taskset[JudgedTask, SetConfig]): To override the judge model, set `env.taskset.task.judge.model` in your config (it is a string). -## Grading artifacts - -An environment can grade in a second sandbox rather than the one the agent worked in, so nothing the agent did to its environment can reach the grader. Only what the task declares crosses over. - -Two channels, and they do different jobs: - -- **`trace.info` is the record.** `capture_patch` puts the diff in `trace.info["patch"]`, a judge puts its verdict there, and both ride `traces.jsonl`. It is not a transport channel — nothing you put there is placed in the grading sandbox as a file. An agentic judge is the exception: it receives the whole serialized trace, `info` included, at `/tmp/trace.json`, so treat anything you leave there as visible to the grader. -- **`/logs/artifacts/` is transport.** Anything written there is collected with no declaration at all, carried to the host, and restored in the grading sandbox at the same path. - -Produce artifacts in `finalize`, while the runtime is live and before scoring mutates anything: - -```python -class MySweTask(vf.Task[MyData]): - async def finalize(self, trace: vf.Trace, runtime: vf.Runtime) -> None: - await vf.capture_patch( - trace, - runtime, - self.data.base_commit, - write_path=f"{vf.CONVENTION_DIR}/patch.diff", # record + transport, one call - ) -``` - -Declare paths outside the convention dir on the task row: - -```python -class MyData(vf.TaskData): - artifacts: list[vf.Artifact] = [ - vf.Artifact(source="/work/report", exclude=[".git"]) - ] -``` - -A declared path that is missing at collection time fails the rollout: it was declared because grading needs it, and grading a partial state scores the rollout wrong rather than loudly failing it. The convention dir is exempt — it is collected for every task, and most never write to it. - -The grading sandbox boots from the same image as the agent's, so the repo and its dependencies are already present. Only the agent's delta has to travel, which is why the collection cap (`vf.artifacts.MAX_ARTIFACT_BYTES`) is sized for a patch rather than a tree. - ## Beyond one agent One episode doesn't have to be one agent run: agents, the control flow between agents, and cross-agent rewards are the environment's job — see [The Env](env.md). diff --git a/tests/v1/test_e2e.py b/tests/v1/test_e2e.py index 08c6adcf89..3e39b80c20 100644 --- a/tests/v1/test_e2e.py +++ b/tests/v1/test_e2e.py @@ -392,25 +392,12 @@ async def test_env_id_best_of_n(run_v1, tmp_path): @pytest.mark.e2e async def test_env_id_agentic_judge(run_v1, tmp_path): - """The agentic judge over the echo taskset (needs docker), on the default - `shared` topology: the box is provisioned once from the solver's runtime - policy, the solver plays the task in it, the judge lands in the SAME box with - the graded trace uploaded, investigates with real execution, and its parsed - verdict lands on the solver's trace under the spec's reward key. Wiring, not - taste: the judge followed the verdict-file contract — the grade itself is the - model's call. Exercises the config surface too: an unpinned judge runtime - (inherits the solver's), a policy-only prompt override (the verdict contract - is appended regardless), and reward-composition weights.""" traces = await run_v1( "echo-v1", - harness=None, # seats pin their own harness; there is no run-level one + harness=None, env={ "id": "agentic-judge", - # The solver owns the shared box, so the container is pinned here; the - # judge's runtime is left unset to exercise the inheritance. "solver": {"harness": {"id": "bash"}, "runtime": {"type": "docker"}}, - # The judge reads the trace and reasons before it writes the - # verdict file; the shared 2048-token run cap truncates it mid-audit. "judge": { "harness": {"id": "bash"}, "max_output_tokens": 8192, @@ -429,10 +416,9 @@ async def test_env_id_agentic_judge(run_v1, tmp_path): (judge,) = [t for t in traces if t.agent_name == "judge"] assert solver.ok and judge.ok assert judge.trainable is False - # The task's own reward keeps its raw score; the rescale lands on the weight. assert solver.rewards["echoed"].score == 1.0 assert solver.rewards["echoed"].weight == 0.5 - assert isinstance(judge.info.get("verdict"), dict) # scraped off the box + assert isinstance(judge.info.get("verdict"), dict) assert 0.0 <= solver.rewards["judge"].score <= 1.0 diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 90a66fd57e..299dbe5d4f 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -5,7 +5,7 @@ from verifiers.v1.acp import ACP from verifiers.v1.agent import Agent, Agents, Interaction, Segment, make_agent from verifiers.v1.artifacts import ( - CONVENTION_DIR, + ARTIFACTS_DIR, Artifact, collect, restore, @@ -37,7 +37,6 @@ from verifiers.v1.envs.single_agent import SingleAgentEnv, SingleAgentEnvConfig from verifiers.v1.episode import Episode, WireEpisode from verifiers.v1.errors import ( - ArtifactError, EnvError, HarnessError, InterceptionError, @@ -162,9 +161,6 @@ from verifiers.v1.utils.git import ( resolve_head as resolve_head, ) -from verifiers.v1.utils.git import ( - snapshot_untracked as snapshot_untracked, -) __all__ = [ # noqa: RUF022 - grouped by public API area # types @@ -230,7 +226,6 @@ "ToolsetError", "SandboxError", "TaskError", - "ArtifactError", "InterceptionError", "TunnelError", # clients @@ -303,9 +298,8 @@ "PATCH_CAP_BYTES", "capture_patch", "resolve_head", - "snapshot_untracked", # grading artifacts - "CONVENTION_DIR", + "ARTIFACTS_DIR", "Artifact", "collect", "restore", diff --git a/verifiers/v1/artifacts.py b/verifiers/v1/artifacts.py index d1674bf2d5..0eaaf24400 100644 --- a/verifiers/v1/artifacts.py +++ b/verifiers/v1/artifacts.py @@ -1,17 +1,4 @@ -"""Carry a task's declared files out of the agent's box and into a grading box. - -Grading in the box the agent worked in leaves a seam a policy under RL pressure will -find. Grading in a second box removes it — only what the task declares crosses over. - -Two channels, non-overlapping: `Trace.info` is the durable record (`capture_patch` puts -the diff there, a judge puts its verdict there) and never travels; `/logs/artifacts/` is -transport, Harbor's in-sandbox convention, collected with no declaration and restored at -the same path in the grading box ("no translation", as in Harbor). - -`collect` runs while the agent's box is alive, right after `Task.finalize` produced the -files. It is the barrier: once it returns, nothing downstream needs the agent's box and -it can be torn down. -""" +"""Artifact collection and restoration across runtimes.""" from __future__ import annotations @@ -23,7 +10,6 @@ from pydantic import Field -from verifiers.v1.errors import ArtifactError from verifiers.v1.types import StrictBaseModel if TYPE_CHECKING: @@ -31,9 +17,8 @@ logger = logging.getLogger(__name__) -CONVENTION_DIR = "/logs/artifacts" -"""Harbor's in-sandbox publish directory, swept implicitly so a task that writes here -needs no declaration.""" +ARTIFACTS_DIR = "/logs/artifacts" +"""Implicit artifact directory; tasks that write here need no declaration.""" MAX_ARTIFACT_BYTES = 32 * 1024 * 1024 """Ceiling per collection. Sized for a delta, not a tree: the grading box boots from the @@ -41,11 +26,7 @@ class Artifact(StrictBaseModel): - """One path to carry into the grading box, where it lands at this same path. - - Harbor's `ArtifactConfig` minus `destination` (host trial-directory placement, which - verifiers has no equivalent for) and `service` (compose sidecars, unsupported). - """ + """One path to restore at the same location in another runtime.""" source: str exclude: list[str] = Field(default_factory=list) @@ -64,37 +45,34 @@ async def collect( and grading a partial state scores the rollout wrong rather than failing it. The implicit convention sweep is exempt — most tasks never write there. - One archive per source: BusyBox `tar` (every alpine-based image) implements only - `c`/`x`/`t` with no `-r` to append, and each source carries its own excludes anyway. + Each source is archived separately so its exclude patterns stay local. """ - # Harbor permits a relative source, and the probe below resolves one against the - # runtime's workdir — so the tar, which runs `-C /`, has to agree or it archives a - # different file. Joining also normalises `/work/` to `/work`, so one tree cannot - # key two entries (the source is both the dict key and `restore`'s rm -rf target). + # Resolve relative sources against the runtime workdir. Joining also normalises + # `/work/` to `/work`, so one tree cannot key two entries (the source is both the + # dict key and `restore`'s rm -rf target). workdir = PurePosixPath(getattr(runtime.config, "workdir", "") or "/") declared = [ a.model_copy(update={"source": str(workdir / a.source)}) for a in artifacts or [] ] - convention = PurePosixPath(CONVENTION_DIR) + convention = PurePosixPath(ARTIFACTS_DIR) sweep = not any( (p := PurePosixPath(a.source)) == convention or p.is_relative_to(convention) or convention.is_relative_to(p) for a in declared ) - entries = ([Artifact(source=CONVENTION_DIR)] if sweep else []) + declared + entries = ([Artifact(source=ARTIFACTS_DIR)] if sweep else []) + declared collected: dict[str, bytes] = {} budget = MAX_ARTIFACT_BYTES for artifact in entries: source = artifact.source if (await runtime.run(["test", "-e", source], {})).exit_code != 0: - if sweep and source == CONVENTION_DIR: + if sweep and source == ARTIFACTS_DIR: continue - raise ArtifactError( - f"declared artifact {source!r} does not exist in the box; the task must " - "produce it in finalize() (or a [[verifier.collect]] hook)" + raise RuntimeError( + f"declared artifact {source!r} does not exist in the runtime" ) archive = await _tar_out(runtime, artifact, budget) budget -= len(archive) @@ -111,7 +89,7 @@ async def restore(runtime: Runtime, collected: dict[str, bytes]) -> None: # Extraction writes to absolute paths. In a container that is the point; under the # subprocess runtime it is the developer's own filesystem. if getattr(runtime.config, "type", None) == "subprocess": - raise ArtifactError( + raise RuntimeError( "refusing to restore artifacts into the subprocess runtime: extraction " "writes to absolute paths on the host. Grade in a container." ) @@ -144,7 +122,7 @@ async def _tar_out(runtime: Runtime, artifact: Artifact, budget: int) -> bytes: # memory, not after. sized = await runtime.run(["sh", "-c", f"wc -c < {shlex.quote(path)}"], {}) if (raw := sized.stdout.strip()).isdigit() and int(raw) > budget: - raise ArtifactError( + raise RuntimeError( f"artifact {artifact.source!r} takes the collection over the " f"{MAX_ARTIFACT_BYTES} byte limit. The grading box boots from the " "agent's image, so only the delta needs to travel — narrow the source " @@ -163,4 +141,4 @@ async def _run(runtime: Runtime, command: str, action: str) -> None: result = await runtime.run(["sh", "-c", command], {}) if result.exit_code: detail = (result.stderr or result.stdout).strip()[-500:] - raise ArtifactError(f"failed to {action}: {detail}") + raise RuntimeError(f"failed to {action}: {detail}") diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index 2ebe4d2093..15847afd33 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -1,4 +1,4 @@ -"""agentic-judge: a solver plays the task, a code-executing judge verifies the work. +"""agentic-judge: a solver plays the task, then a judge verifies the work. A reusable env (`--env.id agentic-judge` over any taskset). The solver plays the task in a container provisioned from its runtime policy; the judge then grades @@ -8,12 +8,9 @@ `judge/` metrics plus a weighted-mean `judge` reward, composed with the taskset's own rewards via `[env.score]` (judge-only by default). -`--env.topology` decides where the judge stands. Under `shared` (the default) it -plays in the box the agent worked in, seeing that environment directly and every -seam in it. Under `isolated` it gets its own box from the same image, holding only -what the task declared as artifacts, so nothing the agent did to its environment -can reach the grader — worth opting into once a taskset publishes its evidence -somewhere that travels. +`--env.share-runtime` controls whether the judge uses the solver's runtime. It is +enabled by default. When disabled, the judge gets a fresh runtime containing the +task's collected artifacts. """ import json @@ -21,12 +18,10 @@ import re import tomllib from pathlib import Path -from typing import Literal from pydantic import Field, field_validator import verifiers.v1 as vf -from verifiers.v1.artifacts import CONVENTION_DIR from verifiers.v1.types import StrictBaseModel VERDICT_FILE = "/tmp/verdict.json" @@ -118,21 +113,13 @@ def _render(template: str, **fields: str) -> str: SHARED_SANDBOX_NOTE = f"""\ ## Your workspace -Your sandbox is the SAME box the graded agent worked in, in the state the agent -left it — its edits (and any scoring side effects) are applied. {_RECORD_NOTE}""" +The graded agent worked in this sandbox. {_RECORD_NOTE}""" ISOLATED_SANDBOX_NOTE = f"""\ ## Your workspace -Your sandbox is a FRESH box, built from the same image the graded agent started -from — so it holds the task's original state, NOT the state the agent left. The -agent's environment is gone; you cannot inspect it, and nothing it changed is -here except what the task declared as an artifact. Those artifacts have been -restored at their original paths (a patch under `{CONVENTION_DIR}/` is the usual -one for code tasks), so to see the agent's work you generally have to apply or -read them rather than looking at the working tree. If something you need to -check was never declared as an artifact, say so in your reason rather than -assuming its absence means the agent failed. {_RECORD_NOTE}""" +This is a fresh sandbox. The task's artifacts were restored at their original +paths; other changes made by the graded agent are not present. {_RECORD_NOTE}""" HINT_SECTION = """\ ## Hints @@ -148,20 +135,26 @@ class JudgeTask(vf.Task): NEEDS_CONTAINER = True - def __init__(self, data: vf.TaskData, files: dict[str, bytes]) -> None: + def __init__( + self, + data: vf.TaskData, + files: dict[str, bytes], + artifacts: dict[str, bytes], + ) -> None: super().__init__(data) self.files = files + self.artifacts = artifacts @classmethod def from_trace( cls, solution: vf.Trace, config: "JudgeTaskConfig", - topology: str = "shared", + share_runtime: bool = True, ) -> "JudgeTask": """Mint the judge's task from the solver's finished trace. - `topology` selects the workspace note. It has to match how the judge is + `share_runtime` selects the workspace note. It has to match how the judge is actually placed: the note is the judge's only account of what its box contains, and a judge told it is standing in the agent's workspace when it is standing in a fresh one will read an unmodified tree as a failed attempt. @@ -173,7 +166,7 @@ def from_trace( if "{prompt}" not in template: # A policy that doesn't place the task statement itself still needs it. body += "\n\n" + _render(TASK_SECTION, prompt=solved.prompt_text) - note = SHARED_SANDBOX_NOTE if topology == "shared" else ISOLATED_SANDBOX_NOTE + note = SHARED_SANDBOX_NOTE if share_runtime else ISOLATED_SANDBOX_NOTE sections = [body, _verdict_section(config.criteria()), note] if (hint := config.build_hint()) is not None: sections.insert(1, _render(HINT_SECTION, hint=hint)) @@ -187,9 +180,11 @@ def from_trace( resources=solved.resources, ), files=files, + artifacts={} if share_runtime else solution.state.artifacts, ) async def setup(self, trace: vf.Trace, runtime: vf.Runtime) -> None: + await vf.restore(runtime, self.artifacts) # The solver had this box first: a pre-seeded verdict must never read as # the judge's own, and a file (or planted symlink) at an upload path must # never survive it — a symlinked TRACE_FILE would redirect the write onto @@ -289,38 +284,16 @@ class AgenticJudgeEnvConfig(vf.EnvConfig): """The solver agent. Its runtime must be a container: `--env.solver.runtime.type docker|prime`.""" judge: vf.AgentConfig = vf.AgentConfig() - """The judge agent. Under `isolated` it provisions its own box from this policy; - under `shared` it plays in the solver's box and this policy is ignored.""" - topology: Literal["shared", "isolated"] = "shared" - """Whether the judge grades in the solver's box or its own. - - `shared` places the judge in the box the agent worked in, so it can inspect that - environment directly — at the cost of leaving every seam in it reachable. It is the - default because it is what has been measured, and because a judge only gains from - isolation once the task publishes what the judge needs. - - `isolated` boots a second box from the same image, carries the task's declared - artifacts across, and grades there, so nothing the agent did to its environment can - reach the grader. Opt in per taskset, and only once that taskset puts its evidence - somewhere that travels: an artifact under `vf.CONVENTION_DIR` (see - `capture_patch(write_path=...)`), a declared `TaskData.artifacts` path, or the trace - record itself. A task whose judge hint says to run `git diff` in the box will find - a pristine checkout here and grade every attempt as untouched.""" + """The judge agent. Its runtime is ignored when `share_runtime` is enabled.""" + share_runtime: bool = True + """Whether the judge grades in the solver's runtime.""" task: JudgeTaskConfig = JudgeTaskConfig() score: ScoreConfig = ScoreConfig() class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]): def __init__(self, config: AgenticJudgeEnvConfig) -> None: - # The judge inherits the solver's runtime policy unless it pins a container of - # its own. Under `shared` that is definitional — its effective runtime IS the - # solver's box, and aligning the config keeps the base env's subprocess warning - # and the runtime stamped on its trace truthful. Under `isolated` it is the - # fallback every other unpinned `AgentConfig` field already gets (harness, - # model, sampling): `runtime` defaults to `SubprocessConfig` with no `None` to - # distinguish unset from chosen, and a code-executing judge can never run on the - # host anyway, so subprocess here always means "not specified". - if config.topology == "shared" or isinstance( + if config.share_runtime or isinstance( config.judge.runtime, vf.SubprocessConfig ): config.judge = config.judge.model_copy( @@ -339,15 +312,15 @@ def _check_agents(self) -> None: judge = self._harnesses["judge"] if not judge.EXECUTES_CODE: raise ValueError( - "agentic-judge plays a code-executing judge in its own sandbox, but " + "agentic-judge requires a judge harness that can execute code, but " f"harness {judge.config.id!r} is a tool-less chat loop — a verdict " "that needs no execution is a plugged judge " "(--env.taskset.task.judges), not an agent." ) if isinstance(self.config.solver.runtime, vf.SubprocessConfig): raise TypeError( - "agentic-judge runs a code-executing solver in a container, but the " - "solver resolves to the subprocess runtime; use " + "agentic-judge requires the solver to run in a container, but it " + "resolves to the subprocess runtime; use " "--env.solver.runtime.type docker or prime" ) @@ -356,36 +329,24 @@ async def setup(self, agents: vf.Agents) -> None: agents.judge.trainable = False async def run(self, task: vf.Task, agents: vf.Agents) -> None: - if self.config.topology == "shared": + if self.config.share_runtime: async with agents.solver.provision(task) as box: solution = await agents.solver.run(task, runtime=box) - judge_task = JudgeTask.from_trace(solution, self.config.task, "shared") + judge_task = JudgeTask.from_trace(solution, self.config.task) await agents.judge.run(judge_task, runtime=box) return - # Collection is the barrier: once it returns, nothing downstream needs the - # solver's box. Letting the context manager tear it down normally keeps an - # episode at one box rather than two, which is what costs on a paid runtime. - async with agents.solver.provision(task) as box: - solution = await agents.solver.run(task, runtime=box) - if not solution.ok: - # The rollout errored, so its own scoring never ran either; grading it - # would spend a second sandbox to reproduce a failure the trace already - # records. Return rather than raise: `episode.ok` follows the failed - # trace, and `episode_should_retry` classifies off that trace's own - # error — an exception raised here would bury the real type. - return - collected = await vf.collect(box, task.data.artifacts) - - async with agents.judge.provision(task) as judge_box: - await vf.restore(judge_box, collected) - judge_task = JudgeTask.from_trace(solution, self.config.task, "isolated") - await agents.judge.run(judge_task, runtime=judge_box) + solution = await agents.solver.run(task) + if not solution.ok: + return + await agents.judge.run( + JudgeTask.from_trace(solution, self.config.task, share_runtime=False) + ) async def finalize(self, task: vf.Task, episode: vf.Episode) -> None: by_agent = {t.agent_name: t for t in episode.traces} if "judge" not in by_agent: - return # solver errored; `run` skipped grading and the trace says why + return solution, verdict = by_agent["solver"], by_agent["judge"] data = verdict.info.get("verdict") if not isinstance(data, dict) or not isinstance(data.get("verdicts"), list): diff --git a/verifiers/v1/errors.py b/verifiers/v1/errors.py index 0becaec2ce..1ba1706fd1 100644 --- a/verifiers/v1/errors.py +++ b/verifiers/v1/errors.py @@ -74,13 +74,6 @@ class TaskError(RolloutError): """Task-authored code raised — `setup`, `finalize`, or a `@reward`/`@metric`.""" -class ArtifactError(RolloutError): - """A grading artifact could not be carried between boxes — a declared source was - missing, the collection exceeded its limits, or the archive was unsafe to extract. - Strict on purpose: an incompletely restored grading box scores the rollout wrong, - which is worse than failing it.""" - - class InterceptionError(RolloutError): """The host interception server (model calls + `/state` + `/task` channels) couldn't be reached.""" diff --git a/verifiers/v1/state.py b/verifiers/v1/state.py index 19c5268e61..951661b9c7 100644 --- a/verifiers/v1/state.py +++ b/verifiers/v1/state.py @@ -1,12 +1,10 @@ """Mutable state shared within one rollout. Tool servers synchronize it through the interception state channel. It is excluded -from serialized traces; persist artifacts in `Trace.info` instead. `Trace.info` is the -durable record and never leaves the host — files a grader needs in a second box travel -separately, through `verifiers.v1.artifacts`. +from serialized traces. """ -from pydantic import ConfigDict +from pydantic import ConfigDict, Field from typing_extensions import TypeVar from verifiers.v1.types import StrictBaseModel @@ -15,6 +13,7 @@ class State(StrictBaseModel): model_config = ConfigDict(ser_json_inf_nan="constants") + artifacts: dict[str, bytes] = Field(default_factory=dict) StateT = TypeVar("StateT", bound=State, default=State) diff --git a/verifiers/v1/tasksets/harbor/taskset.py b/verifiers/v1/tasksets/harbor/taskset.py index 4444d1a93b..dfdcdd9f2e 100644 --- a/verifiers/v1/tasksets/harbor/taskset.py +++ b/verifiers/v1/tasksets/harbor/taskset.py @@ -25,10 +25,10 @@ from pydantic import Field -from verifiers.v1.artifacts import Artifact +from verifiers.v1.artifacts import Artifact, collect from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.decorators import reward -from verifiers.v1.errors import SandboxError, TaskError +from verifiers.v1.errors import SandboxError from verifiers.v1.runtimes import Runtime from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout from verifiers.v1.taskset import Taskset @@ -83,7 +83,7 @@ class CollectHook(StrictBaseModel): """One `[[verifier.collect]]` command, run in the agent's box by `finalize`.""" command: str - timeout_sec: float = 60.0 + timeout_sec: float = 600.0 class HarborData(TaskData): @@ -130,15 +130,16 @@ async def finalize(self, trace: Trace, runtime: Runtime) -> None: hook.timeout_sec, ) except TimeoutError as exc: - raise TaskError( + raise RuntimeError( f"collect hook timed out after {hook.timeout_sec}s: {hook.command}" ) from exc if result.exit_code: detail = (result.stderr or result.stdout).strip()[-500:] - raise TaskError( + raise RuntimeError( f"collect hook failed (exit {result.exit_code}): " f"{hook.command}\n{detail}" ) + trace.state.artifacts = await collect(runtime, self.data.artifacts) @reward(weight=1.0) async def solved(self, runtime: Runtime) -> float: @@ -377,14 +378,7 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD def parse_verifier_extras( task_dir: Path, parsed ) -> tuple[list[Artifact], list[CollectHook]]: - """Harbor's `artifacts` and `[[verifier.collect]]` blocks, narrowed to what a - single-container runtime can honor. - - The convention dir is deliberately not prepended here (Harbor's - `with_convention_entry` would): collection injects it itself, as an optional sweep. - Prepending it would make it an explicitly declared entry, and declared entries are - required — which would fail every task that never writes there. - """ + """Parse supported artifact and collect-hook settings.""" from harbor.constants import MAIN_SERVICE_NAME from harbor.models.task.artifacts import ( effective_artifact_service, @@ -406,8 +400,9 @@ def parse_verifier_extras( for entry in normalize_artifact_entries(parsed.artifacts): if effective_artifact_service(entry) != MAIN_SERVICE_NAME: raise ValueError( - f"{task_dir.name}: artifact {entry.source!r} targets service " - f"{entry.service!r}; sidecars need a compose-capable runtime" + f"{task_dir.name}: artifact {entry.source!r} targets additional " + f"service {entry.service!r}; verifiers currently supports artifacts " + "from the main service only" ) # `destination` positions a file in Harbor's host trial directory. Verifiers has # no such directory (the trace is the record) and Harbor never lets destination @@ -420,8 +415,9 @@ def parse_verifier_extras( for hook in verifier.collect: if hook.service != MAIN_SERVICE_NAME: raise ValueError( - f"{task_dir.name}: collect hook targets service {hook.service!r}; " - "sidecars need a compose-capable runtime" + f"{task_dir.name}: collect hook targets additional service " + f"{hook.service!r}; verifiers currently supports collect hooks for " + "the main service only" ) if hook.user is not None: raise ValueError( diff --git a/verifiers/v1/utils/git.py b/verifiers/v1/utils/git.py index 5fc95fbb93..0a29b835b3 100644 --- a/verifiers/v1/utils/git.py +++ b/verifiers/v1/utils/git.py @@ -133,7 +133,7 @@ async def capture_patch( infrastructure failed. `write_path` additionally writes the patch to that path inside the box, for tasks - graded in a second sandbox. Point it at `vf.CONVENTION_DIR` (e.g. + graded in a second sandbox. Point it at `vf.ARTIFACTS_DIR` (e.g. `/logs/artifacts/patch.diff`) and collection picks it up with no declaration. Leave it on the convention sweep rather than declaring it as an `Artifact`: a declared path is collected strictly, which would turn an agent-broken repo into a rollout @@ -161,8 +161,6 @@ async def capture_patch( ) return raw = await runtime.read(capped) - except Exception as exc: - raise SandboxError(f"patch capture could not reach the box: {exc}") from exc finally: # Unique names don't overwrite each other, so leftovers would accumulate # on shared-filesystem runtimes; removal is best-effort by design. @@ -175,12 +173,6 @@ async def capture_patch( trace.info["patch_truncated"] = True trace.info["patch"] = raw.decode("utf-8", errors="replace") if write_path is not None: - try: - parent = str(PurePosixPath(write_path).parent) - await runtime.run(["mkdir", "-p", parent], env or {}) - await runtime.write(write_path, raw) - except Exception as exc: - # Transport again, not the policy: the patch exists, we could not place it. - raise SandboxError( - f"patch capture could not write {write_path!r}: {exc}" - ) from exc + parent = str(PurePosixPath(write_path).parent) + await runtime.run(["mkdir", "-p", parent], env or {}) + await runtime.write(write_path, raw) From 960d45b6464241af8acf80db82d1410ee3be0108 Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Wed, 29 Jul 2026 07:46:22 +0000 Subject: [PATCH 19/24] refactor(v1): one mode-neutral workspace note MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review asked why the judge needs mode-specific sandbox notes at all (#2144 threads on env.py:113/118). Working the question showed the notes were both less necessary and more broken than assumed: - Less necessary: the trace record at /tmp/trace.json is uploaded in both modes, and /logs/artifacts is populated in both modes too — finalize writes it in the solver's box, so the shared judge sees the files where the isolated judge gets them restored. The evidence channels are placement-independent, so the note can be. - More broken: the isolated note's "artifacts were restored at their original paths" reads as "the workspace was reconstructed" — the exact misreading it existed to prevent (a reader deeply familiar with the design made it). And it never named /logs/artifacts, so it asserted restoration without saying where to look. One WORKSPACE_NOTE replaces the pair: the tree may or may not hold the agent's changes, an unmodified tree is not a failed attempt, artifacts sit at the paths they had in the agent's box (naming /logs/artifacts), and the record is harness-written — the judge's one input the graded agent could not author. The prompt is now identical across placements (verified by rendering both), which also deletes the note-must-match-placement invariant from from_trace; share_runtime remains, still selecting whether artifacts travel. 909 non-e2e tests, ruff, ty green. Co-Authored-By: Claude Opus 5 (1M context) --- verifiers/v1/envs/agentic_judge/env.py | 47 ++++++++++++-------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index 15847afd33..c69819eed5 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -99,27 +99,26 @@ def _render(template: str, **fields: str) -> str: return pattern.sub(lambda m: fields[m.group(1)], template) -_RECORD_NOTE = f"""\ -The agent's raw -trace record (JSON: messages, tool calls, and its `info` artifacts) is uploaded -at `{TRACE_FILE}`. The record can be very large — never dump it whole; peek -selectively (list its keys, then slice out specific fields with python or jq) -and pull only what you need. It is complete — it may also carry the task's own -scores/metrics and reference material (a gold answer, a reference solution, -held-out tests). Those are context, not your standard: recorded scores can be -wrong and references can be narrower than the task; do not over-index on how a -reference solves it. Your verdict is what YOU verified by execution.""" - -SHARED_SANDBOX_NOTE = f"""\ +WORKSPACE_NOTE = f"""\ ## Your workspace -The graded agent worked in this sandbox. {_RECORD_NOTE}""" - -ISOLATED_SANDBOX_NOTE = f"""\ -## Your workspace - -This is a fresh sandbox. The task's artifacts were restored at their original -paths; other changes made by the graded agent are not present. {_RECORD_NOTE}""" +Your sandbox is either the box the graded agent worked in or a fresh one built +from the task's image — so the working tree may or may not contain the agent's +changes. An unmodified tree does not mean the agent did nothing: reconstruct +its work from the trace record and the published artifacts. Files the task +published as artifacts are at the same paths they had in the agent's box — the +usual place is `{vf.ARTIFACTS_DIR}/` (for code tasks, typically a patch to read +or apply) plus any task-declared paths. + +The agent's raw trace record (JSON: messages, tool calls, and its `info` +artifacts) is written by the harness — not the agent — at `{TRACE_FILE}`. The +record can be very large — never dump it whole; peek selectively (list its +keys, then slice out specific fields with python or jq) and pull only what you +need. It is complete — it may also carry the task's own scores/metrics and +reference material (a gold answer, a reference solution, held-out tests). Those +are context, not your standard: recorded scores can be wrong and references can +be narrower than the task; do not over-index on how a reference solves it. Your +verdict is what YOU verified by execution.""" HINT_SECTION = """\ ## Hints @@ -154,10 +153,9 @@ def from_trace( ) -> "JudgeTask": """Mint the judge's task from the solver's finished trace. - `share_runtime` selects the workspace note. It has to match how the judge is - actually placed: the note is the judge's only account of what its box - contains, and a judge told it is standing in the agent's workspace when it - is standing in a fresh one will read an unmodified tree as a failed attempt. + `share_runtime` mirrors how the judge is placed: in the solver's box the + published artifacts are already on disk, so none travel; a fresh box gets + the collected set, restored by `setup` at the paths they had. """ solved = solution.task.data files = {TRACE_FILE: json.dumps(solution.to_record()).encode()} @@ -166,8 +164,7 @@ def from_trace( if "{prompt}" not in template: # A policy that doesn't place the task statement itself still needs it. body += "\n\n" + _render(TASK_SECTION, prompt=solved.prompt_text) - note = SHARED_SANDBOX_NOTE if share_runtime else ISOLATED_SANDBOX_NOTE - sections = [body, _verdict_section(config.criteria()), note] + sections = [body, _verdict_section(config.criteria()), WORKSPACE_NOTE] if (hint := config.build_hint()) is not None: sections.insert(1, _render(HINT_SECTION, hint=hint)) prompt = "\n\n".join(sections) From 132d4b33f510d2004468c6e11771822b70bdcf2f Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:31:29 +0200 Subject: [PATCH 20/24] Apply suggestion from @macroscopeapp[bot] Co-authored-by: macroscopeapp[bot] <170038800+macroscopeapp[bot]@users.noreply.github.com> --- verifiers/v1/tasksets/harbor/taskset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verifiers/v1/tasksets/harbor/taskset.py b/verifiers/v1/tasksets/harbor/taskset.py index dfdcdd9f2e..d3d79c509a 100644 --- a/verifiers/v1/tasksets/harbor/taskset.py +++ b/verifiers/v1/tasksets/harbor/taskset.py @@ -126,7 +126,7 @@ async def finalize(self, trace: Trace, runtime: Runtime) -> None: for hook in self.data.collect: try: result = await asyncio.wait_for( - runtime.run(["sh", "-c", hook.command], verifier_env(self.data)), + runtime.run(["sh", "-c", hook.command], {}), hook.timeout_sec, ) except TimeoutError as exc: From 4228c44c72887a33f8edfcd22bdd4d316694944c Mon Sep 17 00:00:00 2001 From: hallerite Date: Thu, 30 Jul 2026 22:18:41 +0000 Subject: [PATCH 21/24] fix(v1): address isolated grading review feedback --- verifiers/v1/artifacts.py | 4 +-- verifiers/v1/envs/agentic_judge/env.py | 46 +++++++++++++++----------- verifiers/v1/task.py | 7 ++-- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/verifiers/v1/artifacts.py b/verifiers/v1/artifacts.py index 0eaaf24400..1d60e6ae1d 100644 --- a/verifiers/v1/artifacts.py +++ b/verifiers/v1/artifacts.py @@ -86,8 +86,8 @@ async def restore(runtime: Runtime, collected: dict[str, bytes]) -> None: """Extract `collected` in `runtime` at the original absolute paths.""" if not collected: return - # Extraction writes to absolute paths. In a container that is the point; under the - # subprocess runtime it is the developer's own filesystem. + # Restoring into the subprocess runtime would extract absolute paths onto the + # developer's filesystem, so refuse it before any archive reaches the host. if getattr(runtime.config, "type", None) == "subprocess": raise RuntimeError( "refusing to restore artifacts into the subprocess runtime: extraction " diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index c69819eed5..de69d1741e 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -99,17 +99,7 @@ def _render(template: str, **fields: str) -> str: return pattern.sub(lambda m: fields[m.group(1)], template) -WORKSPACE_NOTE = f"""\ -## Your workspace - -Your sandbox is either the box the graded agent worked in or a fresh one built -from the task's image — so the working tree may or may not contain the agent's -changes. An unmodified tree does not mean the agent did nothing: reconstruct -its work from the trace record and the published artifacts. Files the task -published as artifacts are at the same paths they had in the agent's box — the -usual place is `{vf.ARTIFACTS_DIR}/` (for code tasks, typically a patch to read -or apply) plus any task-declared paths. - +_RECORD_NOTE = f"""\ The agent's raw trace record (JSON: messages, tool calls, and its `info` artifacts) is written by the harness — not the agent — at `{TRACE_FILE}`. The record can be very large — never dump it whole; peek selectively (list its @@ -120,6 +110,21 @@ def _render(template: str, **fields: str) -> str: be narrower than the task; do not over-index on how a reference solves it. Your verdict is what YOU verified by execution.""" +SHARED_WORKSPACE_NOTE = f"""\ +## Your workspace + +The graded agent worked in this sandbox. Its edits and any scoring side effects +are present. {_RECORD_NOTE}""" + +ISOLATED_WORKSPACE_NOTE = f"""\ +## Your workspace + +This is a fresh sandbox built from the task's image. The task's published +artifacts were restored at their original paths; other changes made by the +graded agent are not present. The usual artifact location is +`{vf.ARTIFACTS_DIR}/` (for code tasks, typically a patch to read or apply), plus +any task-declared paths. {_RECORD_NOTE}""" + HINT_SECTION = """\ ## Hints @@ -153,9 +158,10 @@ def from_trace( ) -> "JudgeTask": """Mint the judge's task from the solver's finished trace. - `share_runtime` mirrors how the judge is placed: in the solver's box the - published artifacts are already on disk, so none travel; a fresh box gets - the collected set, restored by `setup` at the paths they had. + `share_runtime` selects both the workspace note and artifact transport. In + the solver's box the published artifacts are already on disk, so none + travel; a fresh box gets the collected set, restored by `setup` at the + paths they had. """ solved = solution.task.data files = {TRACE_FILE: json.dumps(solution.to_record()).encode()} @@ -164,7 +170,10 @@ def from_trace( if "{prompt}" not in template: # A policy that doesn't place the task statement itself still needs it. body += "\n\n" + _render(TASK_SECTION, prompt=solved.prompt_text) - sections = [body, _verdict_section(config.criteria()), WORKSPACE_NOTE] + workspace_note = ( + SHARED_WORKSPACE_NOTE if share_runtime else ISOLATED_WORKSPACE_NOTE + ) + sections = [body, _verdict_section(config.criteria()), workspace_note] if (hint := config.build_hint()) is not None: sections.insert(1, _render(HINT_SECTION, hint=hint)) prompt = "\n\n".join(sections) @@ -281,7 +290,8 @@ class AgenticJudgeEnvConfig(vf.EnvConfig): """The solver agent. Its runtime must be a container: `--env.solver.runtime.type docker|prime`.""" judge: vf.AgentConfig = vf.AgentConfig() - """The judge agent. Its runtime is ignored when `share_runtime` is enabled.""" + """The judge agent. Its runtime is ignored when `share_runtime` is enabled; + otherwise it must be a container.""" share_runtime: bool = True """Whether the judge grades in the solver's runtime.""" task: JudgeTaskConfig = JudgeTaskConfig() @@ -290,9 +300,7 @@ class AgenticJudgeEnvConfig(vf.EnvConfig): class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]): def __init__(self, config: AgenticJudgeEnvConfig) -> None: - if config.share_runtime or isinstance( - config.judge.runtime, vf.SubprocessConfig - ): + if config.share_runtime: config.judge = config.judge.model_copy( update={"runtime": config.solver.runtime} ) diff --git a/verifiers/v1/task.py b/verifiers/v1/task.py index 28012ed4ed..a7eaa02004 100644 --- a/verifiers/v1/task.py +++ b/verifiers/v1/task.py @@ -124,10 +124,9 @@ class TaskData(StrictBaseModel): blocks. Non-empty concrete allowlists cannot be combined with blocklists. Docker framework routes take precedence; ordinary Prime deny rules pass through unchanged.""" artifacts: list[Artifact] = Field(default_factory=list) - """Paths carried out of the agent's box and into a grading box, on top of the - implicitly collected `/logs/artifacts/` convention dir. Declare only what a grader - needs: the grading box boots from this task's image, so the repo is already there - and only the agent's output has to travel. A declared path that is missing at + """Paths collected from one runtime and restored at the same locations in another, + on top of the implicitly collected `/logs/artifacts/` convention dir. Declare + runtime outputs that must cross that boundary. A declared path that is missing at collection time fails the rollout.""" timeout: TaskTimeout = TaskTimeout() resources: TaskResources = TaskResources() From fef21cd347603bc3a5c8a6adfdbb3d2297635ea9 Mon Sep 17 00:00:00 2001 From: hallerite Date: Fri, 31 Jul 2026 01:03:44 +0200 Subject: [PATCH 22/24] fix(v1): validate isolated judge runtime early --- verifiers/v1/envs/agentic_judge/env.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index 60e4efcd0a..fbf1157550 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -23,6 +23,7 @@ import verifiers.v1 as vf from verifiers.v1.types import StrictBaseModel +from verifiers.v1.utils.compile import validate_pairing VERDICT_FILE = "/tmp/verdict.json" TRACE_FILE = "/tmp/trace.json" @@ -327,6 +328,7 @@ def _check_agents(self) -> None: "resolves to the subprocess runtime; use " "--env.solver.runtime.type docker or prime" ) + validate_pairing(judge, JudgeTask, self.config.judge.runtime) async def setup(self, agents: vf.Agents) -> None: # The judge grades the policy; its tokens are never training data. From 96f2b5937eb6a522d29c9718b548f49868309d60 Mon Sep 17 00:00:00 2001 From: hallerite Date: Fri, 31 Jul 2026 01:22:12 +0200 Subject: [PATCH 23/24] fix(v1): disambiguate judge text sources --- tests/v1/test_e2e.py | 5 ++- verifiers/v1/envs/agentic_judge/__init__.py | 2 ++ verifiers/v1/envs/agentic_judge/env.py | 38 +++++++++++++-------- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/tests/v1/test_e2e.py b/tests/v1/test_e2e.py index bed19a0c75..3de6b7f2eb 100644 --- a/tests/v1/test_e2e.py +++ b/tests/v1/test_e2e.py @@ -413,7 +413,10 @@ async def test_env_id_agentic_judge(run_v1, tmp_path): "harness": {"id": "bash"}, "max_output_tokens": 8192, }, - "task": {"prompt": str(policy)}, + "task": { + "prompt": {"path": str(policy)}, + "hint": "Do not rely on README.md", + }, "score": {"task_weight": 0.5}, }, output_dir=tmp_path, diff --git a/verifiers/v1/envs/agentic_judge/__init__.py b/verifiers/v1/envs/agentic_judge/__init__.py index 0c5a9b86fd..408d318d67 100644 --- a/verifiers/v1/envs/agentic_judge/__init__.py +++ b/verifiers/v1/envs/agentic_judge/__init__.py @@ -4,6 +4,7 @@ Criterion, JudgeTaskConfig, ScoreConfig, + TextFile, ) __all__ = [ @@ -12,4 +13,5 @@ "Criterion", "JudgeTaskConfig", "ScoreConfig", + "TextFile", ] diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index fbf1157550..a4b5472557 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -214,30 +214,40 @@ async def finalize(self, trace: vf.Trace, runtime: vf.Runtime) -> None: trace.info["verdict"] = json.loads(raw) +class TextFile(vf.BaseConfig): + """An explicit file-backed text value for config formats without `Path` values.""" + + path: Path + + +TextSource = str | Path | TextFile + + class JudgeTaskConfig(vf.BaseConfig): """The judge's minted task: the grading policy and what lands in its box.""" - prompt: Path | str | None = None - """Grading policy, either inline or a `.md`/`.txt` file. Replaces only the - policy body — the verdict contract and workspace note are always appended. - May reference `{prompt}` (the solver task's prompt); if it doesn't, the task + prompt: TextSource | None = None + """Grading policy: a string is inline text; a `Path` in Python or + `{ path = "policy.md" }` in config reads a file. Replaces only the policy + body — the verdict contract and workspace note are always appended. May + reference `{prompt}` (the solver task's prompt); if it doesn't, the task statement is appended after.""" - hint: Path | str | None = None - """Optional inline hints or a `.md`/`.txt` hints file, injected as their own - section: task-family pointers into the trace or box — e.g. for math, where - the reference answer lives in the record; for SWE, to diff the repo or read - `info.patch`.""" + hint: TextSource | None = None + """Optional hints, with the same explicit inline/file forms as `prompt`, + injected as their own section: task-family pointers into the trace or box — + e.g. for math, where the reference answer lives in the record; for SWE, to + diff the repo or read `info.patch`.""" rubric: Path | None = None """Criteria the judge grades against: a `.toml`/`.json` file with a `criteria` list — the plugged rubric judge's format, so the same rubric files work for both. None grades the single built-in `solved` criterion.""" @staticmethod - def _resolve(value: Path | str) -> str: - path = Path(value) - if isinstance(value, Path) or path.suffix in (".md", ".txt"): - return path.read_text(encoding="utf-8") - return str(value) + def _resolve(value: TextSource) -> str: + if isinstance(value, str): + return value + path = value if isinstance(value, Path) else value.path + return path.read_text(encoding="utf-8") def build_prompt(self) -> str: if self.prompt is None: From 858ac7292f268d23780924a21410ff8323f4c7e9 Mon Sep 17 00:00:00 2001 From: hallerite Date: Fri, 31 Jul 2026 01:34:40 +0200 Subject: [PATCH 24/24] test(v1): give coding harnesses token headroom --- tests/v1/test_e2e.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/v1/test_e2e.py b/tests/v1/test_e2e.py index 3de6b7f2eb..0aff964ff8 100644 --- a/tests/v1/test_e2e.py +++ b/tests/v1/test_e2e.py @@ -335,6 +335,7 @@ async def test_agentic(run_v1, harness, harness_runtime, tmp_path): runtime={"type": harness_runtime}, output_dir=tmp_path, max_turns=10, + max_tokens=8192, ) assert trace.ok assert trace.num_turns >= 1 # ran a command, then finished