diff --git a/docs/v1/harbor.md b/docs/v1/harbor.md index 1b1941ab58..386b5714ee 100644 --- a/docs/v1/harbor.md +++ b/docs/v1/harbor.md @@ -105,9 +105,26 @@ Prime VM; Prime accepts host-level entries. `artifacts = [...]` and `[[verifier.collect]]` are read from `task.toml` ([Harbor Docs](https://www.harborframework.com/docs/run-jobs/results-and-artifacts)). Collect hooks run in the agent's box from the task's `finalize`, which is Harbor's own ordering — after the agent phase, before collection — and declared paths plus the `/logs/artifacts/` convention dir are then carried into the grading box and restored at their original paths ("no translation", as in Harbor). +Two deliberate differences from `harbor run`: + +- **A failing collect hook fails the rollout.** Harbor logs it and carries on, because there the output is observability; here it is a grading input, and a silently absent file makes the verifier score a stale state. +- **`destination` has no effect.** It positions a file in Harbor's host trial directory; verifiers has no trial directory (the trace is the record), and Harbor never lets `destination` affect verifier-side placement. + +## Separate verifier environments + +`[verifier].environment_mode = "separate"` grades in a second box the agent never touched, instead of the one it worked in ([Harbor Docs](https://www.harborframework.com/docs/tasks/verifier)). The harbor env — this taskset's default — grades such tasks in `finalize`: the solver plays the task as usual, its declared artifacts and the `/logs/artifacts/` convention directory are collected while its box is alive, the box is torn down, and the env then provisions a fresh box, restores those artifacts, stages `tests/` fresh, and grades there, recording the verifier's rewards and metrics onto the solver's trace. The grading box derives from the solver's runtime policy unless `--env.verifier-runtime.*` names its own (a network-restricted verifier on Prime needs `vm true`); infrastructure failures around it retry per `--env.verifier-retries` before the episode fails — a grading box that can't be reached never reads as reward 0. The score is read from `/logs/verifier/reward.json` — a finite number, or an object of finite numbers: with a `reward` key that key is the score and the rest are recorded as metrics; without one every key is recorded as a separate reward. Missing or invalid, it falls back to `reward.txt`. + +Which image the verifier boots from follows Harbor: a declared `[verifier.environment]` if there is one, otherwise a fresh copy of `[environment]`, which is the task's own image. + +A declared `[verifier.environment]` needs a pullable `docker_image`. Without one Harbor would build the verifier image from `tests/Dockerfile`, and verifiers never builds images — so build and push it yourself and name the resulting reference, exactly as for `[environment]`. `ignore_dockerfile` grades in the agent's image instead, which means the verifier runs somewhere the task never declared; it warns when it does. + +Under any other env, a separate-verifier task refuses to grade in the agent's box rather than silently losing its isolation. `ignore_separate_verifier = true` forces every task back into shared grading, trading the isolation for one sandbox per task. + ## Shortcomings verifiers does not have parity with Harbor yet, so some features are missing and currently being worked on. The most notable missing features right now are: -- Switching to a different verifier-phase network policy ([Harbor Docs](https://www.harborframework.com/docs/tasks/network-policy)) +- Switching to a different verifier-phase network policy for a *shared* verifier ([Harbor Docs](https://www.harborframework.com/docs/tasks/network-policy)); a separate verifier's own policy is applied +- Building a verifier image from `tests/Dockerfile`, which Harbor does when a declared `[verifier.environment]` names no `docker_image`. A separate verifier image itself is supported — it just has to be pre-built and pullable (see above), because verifiers never builds images +- Sidecar services, and the sidecar artifacts and collect hooks that go with them ([Harbor Docs](https://www.harborframework.com/docs/tasks#sidecar-artifacts-and-collect-hooks)) - Multi-step tasks ([Harbor Docs](https://www.harborframework.com/docs/tasks/multi-step)) diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index 5151b06a11..8f42bd6fba 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -31,7 +31,7 @@ Runtime, RuntimeConfig, SubprocessConfig, - make_runtime, + provision_runtime, runtime_is_local, ) from verifiers.v1.session import RolloutLimits @@ -561,14 +561,8 @@ async def provision(self, task: Task | None = None) -> AsyncIterator[Runtime]: if task is not None else self.runtime_config ) - runtime = make_runtime(config) - try: - # start() inside the try: a failed start may already hold a remote - # sandbox, so it must reach stop() (safe on a partially-started runtime). - await runtime.start() + async with provision_runtime(config) as runtime: yield runtime - finally: - await runtime.stop() class _EpisodeAgent(Agent): diff --git a/verifiers/v1/runtimes/__init__.py b/verifiers/v1/runtimes/__init__.py index fdfc7303b5..7daa9c5c43 100644 --- a/verifiers/v1/runtimes/__init__.py +++ b/verifiers/v1/runtimes/__init__.py @@ -1,3 +1,5 @@ +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager from typing import Annotated from pydantic import Field @@ -45,6 +47,22 @@ def make_runtime(config: RuntimeConfig, name: str | None = None) -> Runtime: return runtime +@asynccontextmanager +async def provision_runtime( + config: RuntimeConfig, name: str | None = None +) -> AsyncIterator[Runtime]: + """Provision a box from `config` and tear it down on exit. + + `start()` sits inside the `try`: a failed start may already hold a paid sandbox, so + it has to reach `stop()` (which is safe on a partially-started runtime).""" + runtime = make_runtime(config, name) + try: + await runtime.start() + yield runtime + finally: + await runtime.stop() + + def runtime_is_local(config: RuntimeConfig) -> bool: """Whether a runtime of this config exchanges host-local URLs without a public tunnel, read off the runtime class without provisioning one.""" @@ -71,5 +89,6 @@ def runtime_is_local(config: RuntimeConfig) -> bool: "SubprocessRuntime", "SubprocessRuntimeInfo", "make_runtime", + "provision_runtime", "runtime_is_local", ] diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index b29736f5cb..7952b570dc 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -2,6 +2,7 @@ import asyncio import atexit +import base64 import contextlib import hashlib import logging @@ -285,9 +286,42 @@ async def run_uv_script( argv = await self.prepare_uv_script(script, env) return await self.run([*argv, *(args or [])], env or {}) + async def read(self, path: str, max_bytes: int | None = None) -> bytes: + """Read `path` into host memory. `max_bytes` caps the transfer, raising past + the cap — for a file written by something we don't control, whose size we + can't assume. The cap is enforced inside the box rather than after the + transfer, and base64 because `run` returns decoded text. Framework method — + override `_read`, not this.""" + if max_bytes is None: + return await self._read(path) + # Through a temp file, not a pipe: `head | base64` exits with base64's 0 + # even when the path is missing, and a missing file must raise here just + # as it does from `_read`. + result = await self.run( + [ + "sh", + "-c", + ( + "t=$(mktemp) || exit 1; " + 'head -c "$1" -- "$2" > "$t" || { rm -f "$t"; exit 1; }; ' + 'base64 < "$t"; rc=$?; rm -f "$t"; exit $rc' + ), + "sh", + str(max_bytes + 1), + path, + ], + {}, + ) + if result.exit_code: + raise SandboxError(f"read {path!r}: {result.stderr.strip()[-500:]}") + data = base64.b64decode(result.stdout) + if len(data) > max_bytes: + raise SandboxError(f"read {path!r}: over the {max_bytes} byte limit") + return data + @abstractmethod - async def read(self, path: str) -> bytes: - pass + async def _read(self, path: str) -> bytes: + """Read the whole file at `path`; `read` adds the optional transfer cap.""" @abstractmethod async def write(self, path: str, data: bytes) -> None: diff --git a/verifiers/v1/runtimes/docker/__init__.py b/verifiers/v1/runtimes/docker/__init__.py index 12d4cb8c52..9e714ddfea 100644 --- a/verifiers/v1/runtimes/docker/__init__.py +++ b/verifiers/v1/runtimes/docker/__init__.py @@ -334,7 +334,7 @@ async def run_background( if run.exit_code != 0: raise SandboxError(f"docker exec -d failed: {run.stderr.strip()}") - async def read(self, path: str) -> bytes: + async def _read(self, path: str) -> bytes: proc = await asyncio.create_subprocess_exec( "docker", "exec", diff --git a/verifiers/v1/runtimes/modal.py b/verifiers/v1/runtimes/modal.py index 7c65a7331b..41baea5239 100644 --- a/verifiers/v1/runtimes/modal.py +++ b/verifiers/v1/runtimes/modal.py @@ -165,7 +165,7 @@ def _abs(self, path: str) -> str: return path return f"{self.config.workdir.rstrip('/')}/{path}" - async def read(self, path: str) -> bytes: + async def _read(self, path: str) -> bytes: try: return await self._sandbox.filesystem.read_bytes.aio(self._abs(path)) except Exception as e: diff --git a/verifiers/v1/runtimes/prime.py b/verifiers/v1/runtimes/prime.py index 930206184a..f0ae041df4 100644 --- a/verifiers/v1/runtimes/prime.py +++ b/verifiers/v1/runtimes/prime.py @@ -270,7 +270,7 @@ async def run_background( f"prime background launch failed: {result.stderr.strip()}" ) - async def read(self, path: str) -> bytes: + async def _read(self, path: str) -> bytes: # Avoid background-job log limits and base64 overhead by downloading binary data directly. # The temporary file is removed on every exit, and its byte read stays off the event loop. target = ( diff --git a/verifiers/v1/runtimes/subprocess.py b/verifiers/v1/runtimes/subprocess.py index bda055bd9d..a87e3ef71f 100644 --- a/verifiers/v1/runtimes/subprocess.py +++ b/verifiers/v1/runtimes/subprocess.py @@ -96,7 +96,7 @@ async def run_background( proc ) # killed in stop() — a host process won't die on its own - async def read(self, path: str) -> bytes: + async def _read(self, path: str) -> bytes: return await asyncio.to_thread((self.workdir / path).read_bytes) async def write(self, path: str, data: bytes) -> None: diff --git a/verifiers/v1/tasksets/harbor/__init__.py b/verifiers/v1/tasksets/harbor/__init__.py index 2c66acb0de..1d0b31a053 100644 --- a/verifiers/v1/tasksets/harbor/__init__.py +++ b/verifiers/v1/tasksets/harbor/__init__.py @@ -1,3 +1,4 @@ +from verifiers.v1.tasksets.harbor.env import HarborEnv, HarborEnvConfig from verifiers.v1.tasksets.harbor.taskset import ( HarborConfig, HarborData, @@ -5,4 +6,11 @@ HarborTaskset, ) -__all__ = ["HarborConfig", "HarborData", "HarborTask", "HarborTaskset"] +__all__ = [ + "HarborConfig", + "HarborData", + "HarborEnv", + "HarborEnvConfig", + "HarborTask", + "HarborTaskset", +] diff --git a/verifiers/v1/tasksets/harbor/env.py b/verifiers/v1/tasksets/harbor/env.py new file mode 100644 index 0000000000..a5773e860e --- /dev/null +++ b/verifiers/v1/tasksets/harbor/env.py @@ -0,0 +1,124 @@ +"""The harbor taskset's own env: the single solver seat, plus separate-verifier +grading for tasks that declare ``[verifier].environment_mode = "separate"``. + +The default env for harbor runs (the taskset package exports it). A shared-verifier +task runs exactly as under the single-agent env: one `agent` trace, graded in the +box it worked in. A separate-verifier task is graded by `finalize` instead: the +solver's declared artifacts travel (collected by its task `finalize` while its box +is alive), a fresh box is provisioned from the task's verifier declaration, +`tests/` is staged there, and the verifier's rewards land on the solver's trace. +No second agent is involved — the verifier is the task's own `tests/test.sh`. +""" + +import asyncio +import logging +from contextlib import AsyncExitStack + +from pydantic import Field + +import verifiers.v1 as vf +from verifiers.v1.runtimes import RuntimeConfig, provision_runtime +from verifiers.v1.tasksets.harbor.taskset import ( + HarborTask, + verifier_box_data, +) +from verifiers.v1.utils.artifacts import restore +from verifiers.v1.utils.compile import resolve_runtime_config +from verifiers.v1.utils.retries import backoff + +logger = logging.getLogger(__name__) + + +class HarborEnvConfig(vf.EnvConfig): + agent: vf.AgentConfig = vf.AgentConfig() + """The one seat — the policy under evaluation/training; pin + `--env.agent.harness.*` to choose its program or runtime.""" + verifier_runtime: RuntimeConfig | None = None + """Where a separate-verifier task grades. None derives the grading box from + the solver's runtime policy; set it (e.g. `--env.verifier-runtime.type prime + --env.verifier-runtime.vm true`) when the verifier needs different placement + than the agent.""" + verifier_retries: int = Field(2, ge=0) + """Extra attempts at provisioning-and-grading the separate box before the + episode fails. Grading is deterministic; what these retry is the + infrastructure around it (image pulls, provisioning).""" + + +class HarborEnv(vf.Env[HarborEnvConfig]): + async def run(self, task: vf.Task, agents: vf.Agents) -> None: + if not isinstance(task, HarborTask): + raise TypeError( + f"the harbor env runs harbor tasks; got {type(task).__name__}" + ) + if task.data.verifier is None: + await agents.agent.run(task) + return + # Resolve the verifier's box before the solve, so an impossible pairing + # (e.g. a restricted Prime verifier without vm=true) costs nothing + # rather than a full agent run. + self._verifier_config(task) + await agents.agent.run(task.graded_elsewhere()) + + def _verifier_config(self, task: HarborTask) -> RuntimeConfig: + base = ( + self.config.verifier_runtime + if self.config.verifier_runtime is not None + else self.config.agent.runtime + ) + return resolve_runtime_config(base, HarborTask(verifier_box_data(task.data))) + + async def finalize(self, task: vf.Task, episode: vf.Episode) -> None: + """Grade a separate-verifier task in its own box, onto the solver's trace. + + Provision a fresh box from the task's verifier declaration, restore the + solver's collected artifacts, stage `tests/`, run the verifier, and record + its rewards (and any extra reward.json keys as metrics) on the solver's + trace. Infrastructure failures retry per `verifier_retries`; the last one + fails the episode — a grading box that can't be reached must never read + as reward 0.""" + if not isinstance(task, HarborTask) or task.data.verifier is None: + return + solution = episode.traces[0] + if not solution.ok: + return + grader = HarborTask(verifier_box_data(task.data)) + scores = await self._grade(self._verifier_config(task), grader, solution) + items = scores.items() if isinstance(scores, dict) else [("solved", scores)] + for name, value in items: + solution.record_reward(name, value) + + async def _grade( + self, config: RuntimeConfig, grader: HarborTask, solution: vf.Trace + ) -> float | dict[str, float]: + last: Exception | None = None + for attempt in range(self.config.verifier_retries + 1): + if attempt: + delay = backoff(attempt - 1) + logger.warning( + "harbor verifier attempt %d/%d failed (%s); retrying in %.1fs", + attempt, + self.config.verifier_retries + 1, + last, + delay, + ) + await asyncio.sleep(delay) + try: + # The scoring deadline covers provisioning and grading, but not the + # box's teardown: a score already in hand must not be discarded + # because the teardown ran out the clock. + async with AsyncExitStack() as boxes: + async with asyncio.timeout(grader.data.timeout.scoring): + box = await boxes.enter_async_context(provision_runtime(config)) + await box.prepare_setup() + # Artifacts first, tests second: an artifact entry pointing + # into /tests must not survive staging, which wipes and + # rebuilds that directory. + await restore(box, solution.state.artifacts) + await grader._stage_tests(box, wipe=True) + await box.prepare_execution([]) + scores = await grader._graded(box, solution) + return scores + except Exception as e: # noqa: BLE001 - each attempt's failure is retried + last = e + assert last is not None + raise last diff --git a/verifiers/v1/tasksets/harbor/taskset.py b/verifiers/v1/tasksets/harbor/taskset.py index f0a194b682..f6d3cdf13b 100644 --- a/verifiers/v1/tasksets/harbor/taskset.py +++ b/verifiers/v1/tasksets/harbor/taskset.py @@ -1,18 +1,25 @@ """Harbor tasksets backed by Harbor Hub packages. The Harbor CLI downloads and caches each task directory. Its verifier runs in the -same runtime the harness edited, then writes the score to +runtime the harness edited — or, when the task asks for it with +``[verifier].environment_mode = "separate"``, in a second box the agent never +touched, carrying only what the task declared — the harbor env provisions and +grades that box (see ``env.py``). Either way the score lands in ``/logs/verifier/reward.json`` or the legacy ``reward.txt``. A pullable ``[environment].docker_image`` becomes ``TaskData.image``. Verifiers does not build Dockerfile-only environments, so those are rejected unless ``ignore_dockerfile`` deliberately uses the harness runtime image. Tasks without an environment also use that -image unless ``require_image`` is set. +image unless ``require_image`` is set. The same rule applies to a declared +``[verifier.environment]``: it needs a pullable ``docker_image``, since Harbor would +otherwise build the verifier image from ``tests/Dockerfile``. """ import asyncio +import copy import hashlib import io +import logging import shutil import subprocess import sys @@ -26,7 +33,7 @@ from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError from verifiers.v1.configs.taskset import TasksetConfig -from verifiers.v1.errors import SandboxError +from verifiers.v1.errors import SandboxError, TaskError from verifiers.v1.runtimes import Runtime from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout from verifiers.v1.taskset import Taskset @@ -34,9 +41,12 @@ from verifiers.v1.utils.artifacts import Artifact, collect from verifiers.v1.utils.decorators import reward +logger = logging.getLogger(__name__) + CACHE = Path.home() / ".cache" / "harbor" HARBOR_INSTALL_HINT = "uv sync --python 3.12 --extra harbor" REWARD_JSON = "/logs/verifier/reward.json" +MAX_REWARD_BYTES = 1024 * 1024 REWARD_JSON_ADAPTER = TypeAdapter( float | Annotated[dict[str, float], Field(min_length=1)], config=ConfigDict(strict=True, allow_inf_nan=False), @@ -76,6 +86,11 @@ class HarborConfig(TasksetConfig): instead of rejecting it. The Dockerfile is NOT built, so the task scores against the harness image rather than its declared environment — only correct when that image already has what the task needs (e.g. you've pointed the runtime at the right image).""" + ignore_separate_verifier: bool = False + """Grade every task in the agent's own box, even one whose `[verifier]` asks for a + separate one. Trades the isolation for a sandbox per task; useful when provisioning + is the bottleneck. Note what it gives up: the grader becomes reachable by the agent + that just ran.""" class Author(BaseModel): @@ -90,6 +105,27 @@ class CollectHook(BaseModel): timeout_sec: float = 600.0 +class VerifierConfig(BaseModel): + """The box this task's verifier wants, when it wants one of its own. + + `None` on `HarborData` means shared — grade where the agent worked, which is still + Harbor's default and every task that says nothing.""" + + image: str | None = None + """Pullable ref from `[verifier.environment].docker_image`. None keeps the task's + own image, which is what Harbor's fresh copy of `[environment]` resolves to.""" + resources: TaskResources = TaskResources() + workdir: str | None = None + fresh_copy: bool = False + """Whether this came from Harbor's fresh copy of `[environment]` rather than a + declared `[verifier.environment]`. A fresh copy inherits the agent box's resolved + resources; a declared environment states its own, and what it omits falls back to + the run's rather than to the agent's task-derived values.""" + network_allow: list[str] = Field(default_factory=lambda: ["*"]) + """Destinations the verifier may reach, from the verifier's network mode. `["*"]` + is unrestricted; `[]` is Harbor's `no-network` / `allow_internet = false`.""" + + class HarborData(TaskData): """Parsed ``task.toml`` metadata plus the host-side verifier directory. @@ -111,6 +147,9 @@ class HarborData(TaskData): collect: list[CollectHook] = Field(default_factory=list) """`[[verifier.collect]]` blocks: commands that snapshot runtime state into files after the agent stops, so the files can travel to a grading box as artifacts.""" + verifier: VerifierConfig | None = None + """The verifier's own box, when `[verifier].environment_mode` asks for one. None + grades in the agent's box.""" class HarborTask(Task[HarborData]): @@ -145,30 +184,59 @@ async def finalize(self, trace: Trace, runtime: Runtime) -> None: ) trace.state.artifacts = await collect(runtime, self.data.artifacts) - @reward(weight=1.0) - async def solved(self, runtime: Runtime, trace: Trace) -> float | dict[str, float]: + def graded_elsewhere(self) -> "HarborTask": + """A copy whose `solved` records nothing here: the harbor env grades this + task's finished work in a separate box of the task's choosing.""" + clone = copy.copy(self) + clone._graded_elsewhere = True + return clone + + _graded_elsewhere: bool = False + + async def _stage_tests(self, runtime: Runtime, wipe: bool = False) -> None: + """Put the task package's `tests/` in `/tests`, where `test.sh` expects it. + + Raises rather than scoring stale state: a leftover reward file — planted by + the agent or shipped in the image — must be gone before `test.sh` runs, so a + removal that fails must not fall through to reading it. + + `wipe` for a box we did not watch being built: a fresh container of the task's + image can ship its own `/tests`, and a leftover file there would be graded as + though it came from the package. + """ await runtime.write( "/tmp/tests.tgz", make_tar(Path(self.data.task_dir) / "tests") ) - await runtime.run( - [ - "sh", - "-c", - "mkdir -p /logs/verifier /tests && tar -xzf /tmp/tests.tgz -C /tests", - ], - {}, - ) - await runtime.run( - [ - "sh", - "-c", - ( - "rm -f /logs/verifier/reward.json /logs/verifier/reward.txt" - " && cd /tests && bash test.sh" - ), - ], - verifier_env(self.data), + stage = ( + f"{'rm -rf /tests && ' if wipe else ''}" + "rm -f /logs/verifier/reward.json /logs/verifier/reward.txt && " + "mkdir -p /logs/verifier /tests && tar -xzf /tmp/tests.tgz -C /tests" ) + result = await runtime.run(["sh", "-c", stage], {}) + if result.exit_code: + raise TaskError( + f"staging tests failed (exit {result.exit_code}): " + f"{(result.stderr or result.stdout).strip()[-500:]}" + ) + + @reward(weight=1.0) + async def solved(self, runtime: Runtime, trace: Trace) -> float | dict[str, float]: + if self.data.verifier is not None: + if not self._graded_elsewhere: + raise TaskError( + f"task {self.data.name!r} declares a separate verifier " + '([verifier].environment_mode = "separate"); grade it through ' + "the harbor env (this taskset's default), or force shared " + "grading with --taskset.ignore-separate-verifier" + ) + return {} + await self._stage_tests(runtime) + return await self._graded(runtime, trace) + + async def _graded(self, runtime: Runtime, trace: Trace) -> float | dict[str, float]: + # By absolute path, in the runtime's configured workdir: Harbor execs the + # script the same way, and scripts do grade the agent's work at `$PWD`. + await runtime.run(["bash", "/tests/test.sh"], verifier_env(self.data)) scores = await self._reward_json(runtime) if scores is not None: if isinstance(scores, dict) and "reward" in scores: @@ -178,19 +246,75 @@ async def solved(self, runtime: Runtime, trace: Trace) -> float | dict[str, floa return {"reward": scores["reward"]} return scores try: - reward = (await runtime.read("/logs/verifier/reward.txt")).decode().strip() + reward = ( + ( + await runtime.read( + "/logs/verifier/reward.txt", max_bytes=MAX_REWARD_BYTES + ) + ) + .decode() + .strip() + ) return float(reward or 0) except (SandboxError, OSError, ValueError): return 0.0 async def _reward_json(self, runtime: Runtime) -> float | dict[str, float] | None: - """Read Harbor's scalar or keyed JSON reward, if it is valid.""" + """Read Harbor's scalar or keyed JSON reward, if it is valid. + + Bounded: this is a grading input, and nothing guarantees its size. + """ try: - return REWARD_JSON_ADAPTER.validate_json(await runtime.read(REWARD_JSON)) + return REWARD_JSON_ADAPTER.validate_json( + await runtime.read(REWARD_JSON, max_bytes=MAX_REWARD_BYTES) + ) except (SandboxError, OSError, ValidationError): return None +def verifier_box_data(data: HarborData) -> HarborData: + """The verifier's box, declared as task data — the harbor env resolves the + grading runtime from it (image, workdir, resources, network policy), exactly + as the solver's box resolves from the solver task's. + + Which box follows Harbor: a declared `[verifier.environment]` states its own + image, workdir, and resources, and what it omits is the run's default; a + fresh copy of `[environment]` keeps the task's own. The verifier's network + policy applies either way.""" + verifier = data.verifier + if verifier is None: + raise TaskError(f"task {data.name!r} declares no separate verifier") + fresh = verifier.fresh_copy + return data.model_copy( + update={ + "name": f"{data.name} (verifier)", + "image": verifier.image if verifier.image is not None else data.image, + "workdir": data.workdir if fresh else verifier.workdir, + "resources": data.resources if fresh else verifier.resources, + "network_allow": list(verifier.network_allow), + "network_block": [], + } + ) + + +def task_resources(environment, multiplier: float) -> TaskResources: + """Harbor environment resource requests, scaled, as `TaskResources`. + + Harbor declares CPU counts and MB sizes; `TaskResources` wants counts and GB. + GPU requests are never scaled. + """ + return TaskResources( + cpu=environment.cpus * multiplier if environment.cpus else None, + memory=environment.memory_mb / 1024 * multiplier + if environment.memory_mb + else None, + gpu=str(environment.gpus) if environment.gpus else None, + disk=environment.storage_mb / 1024 * multiplier + if environment.storage_mb + else None, + ) + + def harbor_cli() -> str: scripts_dir = Path(sys.executable).parent harbor_bin = shutil.which("harbor", path=str(scripts_dir)) @@ -318,7 +442,7 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD harbor_task = HarborModelTask(task_dir) parsed = harbor_task.config - artifacts, collect = parse_verifier_extras(task_dir, parsed) + artifacts, hooks, verifier = parse_verifier_extras(task_dir, parsed, harbor_config) environment = parsed.environment network = parsed.agent.explicit_phase_policy() or environment.resolve_baseline() task, meta = parsed.task, parsed.metadata @@ -368,18 +492,7 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD if scoring_timeout is not None else None, ), - resources=TaskResources( - cpu=environment.cpus * harbor_config.resource_multiplier - if environment.cpus - else None, - memory=environment.memory_mb / 1024 * harbor_config.resource_multiplier - if environment.memory_mb - else None, - gpu=str(environment.gpus) if environment.gpus else None, - disk=environment.storage_mb / 1024 * harbor_config.resource_multiplier - if environment.storage_mb - else None, - ), + resources=task_resources(environment, harbor_config.resource_multiplier), keywords=task.keywords if task else [], authors=authors, difficulty=meta.get("difficulty"), @@ -388,14 +501,22 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD task_dir=str(task_dir), verifier_env=parsed.verifier.env, artifacts=artifacts, - collect=collect, + collect=hooks, + verifier=verifier, ) def parse_verifier_extras( - task_dir: Path, parsed -) -> tuple[list[Artifact], list[CollectHook]]: - """Parse supported artifact and collect-hook settings.""" + task_dir: Path, parsed, harbor_config: HarborConfig +) -> tuple[list[Artifact], list[CollectHook], VerifierConfig | None]: + """Harbor's `artifacts`, `[[verifier.collect]]` blocks, and verifier environment, + narrowed to what verifiers' verifier-runtime integration can honor. + + The convention dir is deliberately not prepended here (Harbor's + `with_convention_entry` would): collection injects it itself, as an optional sweep. + Prepending it would make it an explicitly declared entry, and declared entries are + required — which would fail every task that never writes there. + """ from harbor.constants import MAIN_SERVICE_NAME from harbor.models.task.artifacts import ( effective_artifact_service, @@ -403,13 +524,6 @@ def parse_verifier_extras( ) verifier = parsed.verifier - if verifier.environment is not None: - raise ValueError( - f"{task_dir.name}: [verifier.environment] declares a separate verifier " - "image. Grading runs in a fresh box built from the task's own image, so " - "only the agent's delta has to travel; a different verifier image needs " - "the full working tree copied over and isn't supported yet." - ) if verifier.user is not None: raise ValueError(f"{task_dir.name}: [verifier].user is not supported") @@ -447,7 +561,89 @@ def parse_verifier_extras( ) hooks.append(CollectHook(command=hook.command, timeout_sec=hook.timeout_sec)) - return artifacts, hooks + return artifacts, hooks, parse_verifier_environment(task_dir, parsed, harbor_config) + + +def parse_verifier_environment( + task_dir: Path, parsed, harbor_config: HarborConfig +) -> VerifierConfig | None: + """The box Harbor wants this task's verifier in, or None to grade in the agent's. + + Harbor resolves `[verifier.environment]` if declared, else a deep copy of + `[environment]` — so a mode-only `separate` lands on the task's own image and needs + nothing but a second box. A declared environment is the case that can name a + different image, and the case that can name none at all: there Harbor builds + `tests/Dockerfile`, which verifiers never does. + """ + from harbor.models.task.config import NetworkMode, TaskOS + from harbor.models.task.verifier_mode import ( + VerifierEnvironmentMode, + resolve_effective_verifier_env_config, + resolve_task_verifier_mode, + ) + + if resolve_task_verifier_mode(parsed) != VerifierEnvironmentMode.SEPARATE: + return None + if harbor_config.ignore_separate_verifier: + logger.warning( + "%s: asks for a separate verifier; grading in the agent's box anyway " + "(--taskset.ignore-separate-verifier)", + task_dir.name, + ) + return None + + environment = resolve_effective_verifier_env_config(parsed, None) + if environment is None: # unreachable while the mode is SEPARATE + raise ValueError(f"{task_dir.name}: separate verifier resolved no environment") + declared = parsed.verifier.environment is not None + + if declared and environment.docker_image is None: + if not harbor_config.ignore_dockerfile: + raise ValueError( + f"{task_dir.name}: [verifier.environment] names no docker_image, so " + "Harbor would build the verifier image from tests/Dockerfile. Verifiers " + "pulls images and never builds them: build and push it yourself (e.g. " + "`prime images push`) and set [verifier.environment].docker_image to the " + "resulting ref, or pass --taskset.ignore-dockerfile to grade in the " + "agent's image instead." + ) + logger.warning( + "%s: [verifier.environment] names no docker_image — grading in the agent's " + "image rather than building tests/Dockerfile, so the verifier runs somewhere " + "the task never declared", + task_dir.name, + ) + unsupported = [ + field + for field in ("healthcheck", "mcp_servers", "skills_dir", "gpu_types", "tpu") + if getattr(environment, field, None) + ] + if environment.os != TaskOS.LINUX or unsupported: + raise ValueError( + f"{task_dir.name}: verifier environment declares " + f"{unsupported or environment.os}, which verifiers' verifier-runtime " + "integration cannot honor" + ) + + network = parsed.verifier.explicit_phase_policy() or environment.resolve_baseline() + return VerifierConfig( + image=environment.docker_image if declared else None, + # A declared environment states its own resources; what it leaves out is the + # run's default, not the agent task's. A fresh copy is the task's environment, + # so it keeps whatever the agent box resolved to. + resources=( + task_resources(environment, harbor_config.resource_multiplier) + if declared + else TaskResources() + ), + workdir=environment.workdir if declared else None, + fresh_copy=not declared, + network_allow=( + ["*"] + if network.network_mode == NetworkMode.PUBLIC + else list(network.allowed_hosts) + ), + ) def verifier_env(task: HarborData) -> dict[str, str]: