From 6fd4691cd6cfda99ce44d2a7673807f0e0c11ad2 Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Fri, 3 Jul 2026 12:23:02 -0300 Subject: [PATCH 1/3] feat(evaluator-sdk): typed workspace seed sources (inline/path/fileset) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the workspace seeding added for the general Codex runtime. A task's inputs["files"] map now accepts three JSON-serializable seed shapes, chosen by a `kind` discriminator (a bare string stays sugar for inline text): - inline — contents in the task; resolvable anywhere. `encoding="base64"` for binary. - path — a file on the authoring host; local-only. - fileset — a workspace/name#glob reference; platform-resolved. The three differ in *where* they resolve, which is what governs a task's portability. inline + path resolve in the pure-SDK runtime; fileset raises a clear "cannot be resolved in local execution" error (same pattern as metric-ref resolution) and is reserved for the service-side path, where a follow-up will resolve filesets and normalize inline/path seeds into a fileset at submit time. Adds `agent_eval/workspace_seeds.py` (the SeedFile union + `seed_workspace` resolver, with workspace-escape protection and binary support). The Codex runtime delegates to it; AgentEvalTask.inputs stays a generic dict — seeding remains a documented convention, not a core task field. Signed-off-by: Sandy Chapman --- .../agent_eval/runtimes/codex/runtime.py | 33 +--- .../agent_eval/workspace_seeds.py | 171 ++++++++++++++++++ .../tests/agent_eval/test_codex_runtime.py | 58 +++++- .../tests/agent_eval/test_workspace_seeds.py | 113 ++++++++++++ .../src/nemo_evaluator/agent_seeds.py | 75 ++++++++ .../src/nemo_evaluator/jobs/agent_evaluate.py | 1 + .../nemo-evaluator/tests/test_agent_seeds.py | 55 ++++++ .../agent_eval/runtimes/codex/runtime.py | 33 +--- .../evaluator/agent_eval/workspace_seeds.py | 171 ++++++++++++++++++ 9 files changed, 653 insertions(+), 57 deletions(-) create mode 100644 packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/workspace_seeds.py create mode 100644 packages/nemo_evaluator_sdk/tests/agent_eval/test_workspace_seeds.py create mode 100644 plugins/nemo-evaluator/src/nemo_evaluator/agent_seeds.py create mode 100644 plugins/nemo-evaluator/tests/test_agent_seeds.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/workspace_seeds.py diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py index 96154dd506..c75d402f9d 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py @@ -20,6 +20,7 @@ from nemo_evaluator_sdk.agent_eval.runtimes.docker_sandbox import DockerSandboxAgentRuntime from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_evaluator_sdk.agent_eval.workspace_seeds import SEED_FILES_INPUT_KEY, seed_workspace from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor DEFAULT_CODEX_TIMEOUT_S = 600 @@ -48,11 +49,6 @@ class EffectiveCodexRuntime(StrEnum): #: (e.g. a benchmark-specific preamble); the default presents the task and invites workspace edits. CodexPromptBuilder = Callable[[AgentEvalTask], str] -#: Optional ``inputs`` key holding a ``{relative_path: contents}`` map of files to seed into the -#: agent's workspace before it runs — how a task hands the agent starter code (a buggy file to fix, -#: a module to test, a project skeleton). Excluded from the default prompt body (listed by name). -SEED_FILES_INPUT_KEY = "files" - class CodexCliAgentRuntime: """AgentTaskRunner that uses the locally installed Codex CLI credentials.""" @@ -113,8 +109,10 @@ async def _run_task(self, index: int, task: AgentEvalTask, config: AgentEvalRunC process: Any | None = None try: # Seed inside the guarded block so a bad seed (e.g. a path escaping the workspace) fails - # just this task rather than aborting the whole run. - seeded_files = _seed_workspace(workspace_dir, task) + # just this task rather than aborting the whole run. Offload to a worker thread: seeding is + # synchronous (a handler may do blocking I/O, e.g. the plugin's fileset download), and this + # runs on the event loop shared by every concurrent task, so a blocking seed would stall them all. + seeded_files = await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY)) process = await self._process_factory( *command, stdin=subprocess.PIPE, @@ -412,27 +410,6 @@ def default_codex_prompt(task: AgentEvalTask) -> str: return "\n".join(lines) + "\n" -def _seed_workspace(workspace_dir: Path, task: AgentEvalTask) -> list[str]: - """Write any ``inputs[SEED_FILES_INPUT_KEY]`` files into the workspace before the agent runs. - - Returns the seeded relative paths (for trial metadata). Paths that escape the workspace (absolute - or ``..`` traversal) are rejected so a task can only stage files inside its own sandbox. - """ - seeds = task.inputs.get(SEED_FILES_INPUT_KEY) - if not isinstance(seeds, Mapping): - return [] - workspace_root = workspace_dir.resolve() - written: list[str] = [] - for rel_path, contents in seeds.items(): - target = (workspace_root / str(rel_path)).resolve() - if target != workspace_root and workspace_root not in target.parents: - raise ValueError(f"seed file path escapes the workspace: {rel_path!r}") - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(contents if isinstance(contents, str) else str(contents), encoding="utf-8") - written.append(str(rel_path)) - return written - - def _failed_codex_trial( task: AgentEvalTask, evidence_dir: Path, diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/workspace_seeds.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/workspace_seeds.py new file mode 100644 index 0000000000..93cc4d1e84 --- /dev/null +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/workspace_seeds.py @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Workspace seed files for agent-eval tasks. + +A task can stage starter files into the agent's workspace before it runs, under +``inputs[SEED_FILES_INPUT_KEY]`` as a ``{relative_path: source}`` map. Each *source* is a +JSON-serializable value whose ``kind`` selects a registered :class:`SeedHandler` (a bare string is +sugar for inline text). + +The SDK ships handlers only for the kinds it can resolve with **no external dependency** — ``inline`` +and ``path``. Other kinds are contributed by consumers via :func:`register_seed_handler`; the SDK has +no knowledge of them (e.g. a platform ``fileset`` handler lives in the plugin and resolves against the +Files service at run time). An unregistered kind raises :class:`WorkspaceSeedError`. +""" + +from __future__ import annotations + +import base64 +from collections.abc import Mapping +from pathlib import Path +from typing import Any, Literal, Protocol, runtime_checkable + +from pydantic import BaseModel, ConfigDict, Field + +#: ``inputs`` key holding the ``{relative_path: seed}`` map of files to stage into the workspace. +SEED_FILES_INPUT_KEY = "files" + + +class WorkspaceSeedError(ValueError): + """A workspace seed could not be parsed, resolved, or written (bad value, unknown kind, ...). + + Subclasses ``ValueError`` so a runner's per-task error handling still catches it. + """ + + +class InlineSeed(BaseModel): + """File contents carried in the task itself. Portable to any runner.""" + + model_config = ConfigDict(extra="forbid") + + kind: Literal["inline"] = "inline" + content: str = Field(description="File contents; UTF-8 text, or base64-encoded bytes when encoding='base64'.") + encoding: Literal["text", "base64"] = "text" + + +class PathSeed(BaseModel): + """A file on the authoring host. Resolvable only where that path exists (local runs).""" + + model_config = ConfigDict(extra="forbid") + + kind: Literal["path"] = "path" + path: str = Field(description="Filesystem path on the authoring host (absolute or relative to the cwd).") + + +@runtime_checkable +class SeedHandler(Protocol): + """Parses + resolves one seed ``kind`` into the bytes to stage. + + Consumers implement this for kinds the SDK doesn't ship (e.g. a platform ``fileset`` handler) and + wire them in with :func:`register_seed_handler`. ``resolve`` runs at seeding time, so a handler + that needs external services (a client, credentials) acquires them there. + """ + + kind: str + + def parse(self, value: Mapping[str, Any]) -> BaseModel: + """Validate a raw seed mapping into this kind's typed model.""" + ... + + def resolve(self, seed: BaseModel) -> bytes: + """Resolve a parsed seed to the bytes to write into the workspace.""" + ... + + +_HANDLERS: dict[str, SeedHandler] = {} + + +def register_seed_handler(handler: SeedHandler) -> None: + """Register a :class:`SeedHandler` under its ``kind`` (replacing any handler already registered).""" + _HANDLERS[handler.kind] = handler + + +def _handler_for(kind: str) -> SeedHandler: + handler = _HANDLERS.get(kind) + if handler is None: + raise WorkspaceSeedError(f"no handler registered for seed kind {kind!r}") + return handler + + +class _InlineSeedHandler: + kind = "inline" + + def parse(self, value: Mapping[str, Any]) -> BaseModel: + return InlineSeed.model_validate(value) + + def resolve(self, seed: BaseModel) -> bytes: + assert isinstance(seed, InlineSeed) + if seed.encoding == "base64": + try: + return base64.b64decode(seed.content, validate=True) + except (ValueError, TypeError) as exc: + raise WorkspaceSeedError(f"inline seed is not valid base64: {exc}") from exc + return seed.content.encode("utf-8") + + +class _PathSeedHandler: + kind = "path" + + def parse(self, value: Mapping[str, Any]) -> BaseModel: + return PathSeed.model_validate(value) + + def resolve(self, seed: BaseModel) -> bytes: + assert isinstance(seed, PathSeed) + source = Path(seed.path).expanduser() + try: + return source.read_bytes() + except OSError as exc: + raise WorkspaceSeedError(f"path seed {seed.path!r} could not be read: {exc}") from exc + + +register_seed_handler(_InlineSeedHandler()) +register_seed_handler(_PathSeedHandler()) + + +def parse_seed(value: str | Mapping[str, Any]) -> BaseModel: + """Coerce a seed map value into its validated model. A bare string is inline UTF-8 text.""" + if isinstance(value, str): + return InlineSeed(content=value) + if not isinstance(value, Mapping): + raise WorkspaceSeedError(f"seed must be a string or mapping, got {type(value).__name__}") + kind = value.get("kind") + if not isinstance(kind, str): + raise WorkspaceSeedError("seed mapping is missing a string 'kind'") + handler = _handler_for(kind) + try: + return handler.parse(value) + except WorkspaceSeedError: + raise + except Exception as exc: # noqa: BLE001 - normalize a handler's validation error into our type + raise WorkspaceSeedError(f"invalid {kind!r} seed: {exc}") from exc + + +def _resolve_seed_bytes(seed: BaseModel) -> bytes: + """Resolve a parsed seed to bytes via its registered handler.""" + kind = getattr(seed, "kind", None) + if not isinstance(kind, str): + raise WorkspaceSeedError("parsed seed has no string 'kind'") + return _handler_for(kind).resolve(seed) + + +def seed_workspace(workspace_dir: str | Path, files: Mapping[str, Any] | None) -> list[str]: + """Write the ``files`` seed map into ``workspace_dir``; return the seeded relative paths. + + Each value is parsed into a seed model and resolved to bytes by its registered handler. Paths that + escape the workspace (absolute, or ``..`` traversal) are rejected so a task can only stage files + inside its own sandbox. ``None``/empty seeds nothing. + """ + if not isinstance(files, Mapping): + return [] + root = Path(workspace_dir).resolve() + written: list[str] = [] + for rel_path, value in files.items(): + target = (root / str(rel_path)).resolve() + if target != root and root not in target.parents: + raise WorkspaceSeedError(f"seed file path escapes the workspace: {rel_path!r}") + data = _resolve_seed_bytes(parse_seed(value)) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + written.append(str(rel_path)) + return written diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime.py index 7699910384..c78f50f7c9 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime.py @@ -2,13 +2,17 @@ # SPDX-License-Identifier: Apache-2.0 import json +import threading +from collections.abc import Mapping from pathlib import Path from typing import Any import pytest +from nemo_evaluator_sdk.agent_eval import workspace_seeds from nemo_evaluator_sdk.agent_eval.runtimes.codex import runtime as codex_runtime from nemo_evaluator_sdk.agent_eval.runtimes.docker_sandbox import DockerSandboxAgentRuntime from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask +from pydantic import BaseModel # The runtime *selection* (local vs docker-cli vs docker-sandbox) is generic and lives here; only the # ProfBench ``score_source`` labels + candidate prompt live in the example (test_profbench_codex_target). @@ -314,6 +318,58 @@ async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: assert trials[0].metadata["seeded_files"] == ["buggy.py"] +@pytest.mark.asyncio +async def test_codex_cli_agent_runtime_seeds_off_the_event_loop_thread( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Seeding is synchronous and a handler may block (e.g. the plugin's fileset download). It runs on + # the event loop shared by every concurrent task, so it must be offloaded to a worker thread — + # otherwise one slow seed stalls the whole run. Register a probe handler that records the thread + # it resolves on and assert it is not the loop thread. + class _ProbeSeed(BaseModel): + kind: str = "thread_probe" + + class _ProbeHandler: + kind = "thread_probe" + resolved_on: int | None = None + + def parse(self, value: Mapping[str, Any]) -> BaseModel: + return _ProbeSeed() + + def resolve(self, seed: BaseModel) -> bytes: + _ProbeHandler.resolved_on = threading.get_ident() + return b"probe" + + monkeypatch.setitem(workspace_seeds._HANDLERS, "thread_probe", _ProbeHandler()) + + class FakeProcess: + returncode = 0 + + def __init__(self, command: tuple[str, ...]) -> None: + self.command = command + + async def communicate(self, input: bytes) -> tuple[bytes, bytes]: + final_output_path = Path(self.command[self.command.index("--output-last-message") + 1]) + final_output_path.write_text("ok", encoding="utf-8") + return b"", b"" + + async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: + return FakeProcess(command) + + monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}") + runtime = codex_runtime.CodexCliAgentRuntime( + work_root=tmp_path / "codex", + process_factory=fake_process_factory, + ) + task = AgentEvalTask(id="probe", intent="probe", inputs={"files": {"p.txt": {"kind": "thread_probe"}}}) + + trials = await runtime.run_tasks([task]) + + assert trials[0].status == "completed" + assert _ProbeHandler.resolved_on is not None + assert _ProbeHandler.resolved_on != threading.get_ident() + + @pytest.mark.asyncio async def test_codex_cli_agent_runtime_rejects_seed_path_escaping_workspace( tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -331,7 +387,7 @@ async def fake_process_factory(*command: str, **kwargs: Any) -> Any: # pragma: # A traversal path is surfaced as a failed trial (the exception is caught per-task). trials = await runtime.run_tasks([task]) assert trials[0].status == "failed" - assert trials[0].metadata["error_type"] == "ValueError" + assert trials[0].metadata["error_type"] == "WorkspaceSeedError" assert trials[0].metadata["agent_ok"] is False diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_workspace_seeds.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_workspace_seeds.py new file mode 100644 index 0000000000..4abcea7046 --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_workspace_seeds.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import base64 +from collections.abc import Mapping +from pathlib import Path +from typing import Any, Literal + +import pytest +from nemo_evaluator_sdk.agent_eval import workspace_seeds +from nemo_evaluator_sdk.agent_eval.workspace_seeds import ( + InlineSeed, + PathSeed, + WorkspaceSeedError, + parse_seed, + register_seed_handler, + seed_workspace, +) +from pydantic import BaseModel + + +def test_parse_bare_string_is_inline_text() -> None: + seed = parse_seed("hello") + assert isinstance(seed, InlineSeed) + assert seed.content == "hello" + assert seed.encoding == "text" + + +def test_parse_dispatches_on_registered_kind() -> None: + assert isinstance(parse_seed({"kind": "inline", "content": "x"}), InlineSeed) + assert isinstance(parse_seed({"kind": "path", "path": "/tmp/x"}), PathSeed) + + +def test_parse_rejects_unregistered_kind() -> None: + # The SDK ships only inline/path; a kind no consumer has registered is a generic unknown-kind error. + with pytest.raises(WorkspaceSeedError, match="no handler registered for seed kind 'url'"): + parse_seed({"kind": "url", "href": "http://x"}) + + +def test_seed_inline_text_and_bare_string(tmp_path: Path) -> None: + written = seed_workspace( + tmp_path, + {"a.txt": "bare", "b.txt": {"kind": "inline", "content": "typed"}}, + ) + assert sorted(written) == ["a.txt", "b.txt"] + assert (tmp_path / "a.txt").read_text() == "bare" + assert (tmp_path / "b.txt").read_text() == "typed" + + +def test_seed_inline_base64_writes_binary(tmp_path: Path) -> None: + payload = b"\x00\x01binary\xff" + seed_workspace( + tmp_path, {"blob.bin": {"kind": "inline", "content": base64.b64encode(payload).decode(), "encoding": "base64"}} + ) + assert (tmp_path / "blob.bin").read_bytes() == payload + + +def test_seed_bad_base64_raises(tmp_path: Path) -> None: + with pytest.raises(WorkspaceSeedError, match="base64"): + seed_workspace(tmp_path, {"x": {"kind": "inline", "content": "not base64!!", "encoding": "base64"}}) + + +def test_seed_path_reads_local_file(tmp_path: Path) -> None: + source = tmp_path / "src.py" + source.write_text("print('hi')\n") + dest_root = tmp_path / "ws" + seed_workspace(dest_root, {"nested/copied.py": {"kind": "path", "path": str(source)}}) + assert (dest_root / "nested" / "copied.py").read_text() == "print('hi')\n" + + +def test_seed_missing_path_raises(tmp_path: Path) -> None: + with pytest.raises(WorkspaceSeedError, match="could not be read"): + seed_workspace(tmp_path, {"x.py": {"kind": "path", "path": str(tmp_path / "nope.py")}}) + + +def test_seed_unregistered_kind_raises(tmp_path: Path) -> None: + # Seeding a kind the SDK doesn't ship (and nobody registered) is a generic unknown-kind error — + # the SDK carries no awareness of platform kinds like 'fileset' or where they resolve. + with pytest.raises(WorkspaceSeedError, match="no handler registered for seed kind 'url'"): + seed_workspace(tmp_path, {"data.csv": {"kind": "url", "href": "http://x/data.csv"}}) + + +def test_register_custom_handler_extends_kinds(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + # A consumer (e.g. the plugin) can register a new kind without the SDK knowing about it. + class _UpperSeed(BaseModel): + kind: Literal["upper"] = "upper" + content: str + + class _UpperHandler: + kind = "upper" + + def parse(self, value: Mapping[str, Any]) -> BaseModel: + return _UpperSeed.model_validate(value) + + def resolve(self, seed: BaseModel) -> bytes: + assert isinstance(seed, _UpperSeed) + return seed.content.upper().encode("utf-8") + + # Isolate the global registry to this test so the extra kind doesn't leak to others. + monkeypatch.setattr(workspace_seeds, "_HANDLERS", dict(workspace_seeds._HANDLERS)) + register_seed_handler(_UpperHandler()) + + seed_workspace(tmp_path, {"shout.txt": {"kind": "upper", "content": "hi"}}) + assert (tmp_path / "shout.txt").read_text() == "HI" + + +def test_seed_rejects_path_escaping_workspace(tmp_path: Path) -> None: + with pytest.raises(WorkspaceSeedError, match="escapes the workspace"): + seed_workspace(tmp_path / "ws", {"../escape.txt": "x"}) + + +def test_seed_none_is_noop(tmp_path: Path) -> None: + assert seed_workspace(tmp_path, None) == [] diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/agent_seeds.py b/plugins/nemo-evaluator/src/nemo_evaluator/agent_seeds.py new file mode 100644 index 0000000000..0798a87e54 --- /dev/null +++ b/plugins/nemo-evaluator/src/nemo_evaluator/agent_seeds.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Platform ``fileset`` workspace-seed handler for agent-eval. + +The SDK's workspace-seed machinery ships only ``inline``/``path`` kinds and knows nothing about +filesets. This module registers a ``fileset`` :class:`~nemo_evaluator_sdk.agent_eval.workspace_seeds.SeedHandler` +as an import side effect, so that when the evaluator plugin runs an agent-eval job, tasks may stage a +file from a stored fileset. Resolution happens at seed time against the Files service, using the +running job's task SDK — the SDK layer never gains a files dependency or awareness of this kind. +""" + +from __future__ import annotations + +import tempfile +from collections.abc import Mapping +from typing import Any, Literal + +from nemo_evaluator.filesets import FilesetRef, download_dataset_sync +from nemo_evaluator_sdk.agent_eval.workspace_seeds import WorkspaceSeedError, register_seed_handler +from nemo_platform_plugin.sdk_provider import get_task_sdk +from pydantic import BaseModel, ConfigDict, Field, field_validator + +#: Service identity used to build the task SDK (matches ``tasks/agent_evaluate.py``). +_EVALUATOR_SERVICE = "evaluator" + + +class FilesetSeed(BaseModel): + """A reference to a stored fileset, resolved via the Files service at seed time.""" + + model_config = ConfigDict(extra="forbid") + + kind: Literal["fileset"] = "fileset" + ref: str = Field( + description="Fileset reference: 'workspace/name', or 'workspace/name#path' for a single file.", + ) + + @field_validator("ref") + @classmethod + def _validate_ref_shape(cls, value: str) -> str: + # Validate the 'workspace/name[#fragment]' shape only — no Files-service call. + base = value.split("#", 1)[0] + parts = [part for part in base.split("/") if part] + if len(parts) != 2: + raise ValueError(f"fileset ref must be 'workspace/name' or 'workspace/name#path', got {value!r}") + return value + + +class FilesetSeedHandler: + """Resolves a :class:`FilesetSeed` to bytes by downloading the referenced file via the Files service.""" + + kind = "fileset" + + def parse(self, value: Mapping[str, Any]) -> BaseModel: + return FilesetSeed.model_validate(value) + + def resolve(self, seed: BaseModel) -> bytes: + assert isinstance(seed, FilesetSeed) + # Acquire the client at resolve time from the running job's ambient identity. + sdk = get_task_sdk(_EVALUATOR_SERVICE) + with tempfile.TemporaryDirectory() as staging: + try: + downloaded = download_dataset_sync(sdk, FilesetRef(root=seed.ref), staging) + except Exception as exc: # noqa: BLE001 - surface any resolution failure as a seed error + raise WorkspaceSeedError(f"fileset seed {seed.ref!r} could not be resolved: {exc}") from exc + files = [downloaded] if downloaded.is_file() else sorted(p for p in downloaded.rglob("*") if p.is_file()) + if len(files) != 1: + raise WorkspaceSeedError( + f"fileset seed {seed.ref!r} resolved to {len(files)} files; a seed maps one path to one " + "file — reference a single file with a '#path' fragment." + ) + return files[0].read_bytes() + + +register_seed_handler(FilesetSeedHandler()) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py index b2c715234d..f332e3327b 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py @@ -23,6 +23,7 @@ from typing import Any, ClassVar from urllib.parse import urlsplit +import nemo_evaluator.agent_seeds # noqa: F401 - registers the platform 'fileset' workspace-seed handler from nemo_evaluator.api.schemas import MetricInline from nemo_evaluator.jobs.agent_compiler import compile_agent_eval_job from nemo_evaluator.jobs.agent_spec import ( diff --git a/plugins/nemo-evaluator/tests/test_agent_seeds.py b/plugins/nemo-evaluator/tests/test_agent_seeds.py new file mode 100644 index 0000000000..1b0891f848 --- /dev/null +++ b/plugins/nemo-evaluator/tests/test_agent_seeds.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the plugin's ``fileset`` workspace-seed handler (registered on import).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from nemo_evaluator import agent_seeds +from nemo_evaluator.agent_seeds import FilesetSeed +from nemo_evaluator_sdk.agent_eval.workspace_seeds import WorkspaceSeedError, parse_seed, seed_workspace +from pydantic import ValidationError + + +def test_fileset_ref_shape_is_validated() -> None: + assert FilesetSeed(ref="ws/name").ref == "ws/name" + assert FilesetSeed(ref="ws/name#data.csv").ref == "ws/name#data.csv" + for bad in ("bad", "a/b/c", "/leading"): + with pytest.raises(ValidationError): + FilesetSeed(ref=bad) + + +def test_importing_plugin_registers_fileset_kind() -> None: + # Importing nemo_evaluator.agent_seeds is enough to teach the SDK registry the 'fileset' kind; + # the SDK itself has no awareness of it. + assert isinstance(parse_seed({"kind": "fileset", "ref": "ws/name"}), FilesetSeed) + + +def test_fileset_seed_resolves_single_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + def fake_download(sdk: object, ref: object, dest: str) -> Path: + target = Path(dest) / "data.csv" + target.write_bytes(b"col\n1\n") + return target + + monkeypatch.setattr(agent_seeds, "get_task_sdk", lambda service: object()) + monkeypatch.setattr(agent_seeds, "download_dataset_sync", fake_download) + + seed_workspace(tmp_path, {"seed/data.csv": {"kind": "fileset", "ref": "ws/name#data.csv"}}) + assert (tmp_path / "seed" / "data.csv").read_bytes() == b"col\n1\n" + + +def test_fileset_seed_rejects_multi_file_ref(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + def fake_download(sdk: object, ref: object, dest: str) -> Path: + root = Path(dest) + (root / "a.txt").write_text("a") + (root / "b.txt").write_text("b") + return root + + monkeypatch.setattr(agent_seeds, "get_task_sdk", lambda service: object()) + monkeypatch.setattr(agent_seeds, "download_dataset_sync", fake_download) + + with pytest.raises(WorkspaceSeedError, match="resolved to 2 files"): + seed_workspace(tmp_path, {"x": {"kind": "fileset", "ref": "ws/name"}}) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py index e7832a9d35..1443f7146a 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py @@ -20,6 +20,7 @@ from nemo_platform.beta.evaluator.agent_eval.runtimes.docker_sandbox import DockerSandboxAgentRuntime from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_platform.beta.evaluator.agent_eval.workspace_seeds import SEED_FILES_INPUT_KEY, seed_workspace from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor DEFAULT_CODEX_TIMEOUT_S = 600 @@ -48,11 +49,6 @@ class EffectiveCodexRuntime(StrEnum): #: (e.g. a benchmark-specific preamble); the default presents the task and invites workspace edits. CodexPromptBuilder = Callable[[AgentEvalTask], str] -#: Optional ``inputs`` key holding a ``{relative_path: contents}`` map of files to seed into the -#: agent's workspace before it runs — how a task hands the agent starter code (a buggy file to fix, -#: a module to test, a project skeleton). Excluded from the default prompt body (listed by name). -SEED_FILES_INPUT_KEY = "files" - class CodexCliAgentRuntime: """AgentTaskRunner that uses the locally installed Codex CLI credentials.""" @@ -113,8 +109,10 @@ async def _run_task(self, index: int, task: AgentEvalTask, config: AgentEvalRunC process: Any | None = None try: # Seed inside the guarded block so a bad seed (e.g. a path escaping the workspace) fails - # just this task rather than aborting the whole run. - seeded_files = _seed_workspace(workspace_dir, task) + # just this task rather than aborting the whole run. Offload to a worker thread: seeding is + # synchronous (a handler may do blocking I/O, e.g. the plugin's fileset download), and this + # runs on the event loop shared by every concurrent task, so a blocking seed would stall them all. + seeded_files = await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY)) process = await self._process_factory( *command, stdin=subprocess.PIPE, @@ -412,27 +410,6 @@ def default_codex_prompt(task: AgentEvalTask) -> str: return "\n".join(lines) + "\n" -def _seed_workspace(workspace_dir: Path, task: AgentEvalTask) -> list[str]: - """Write any ``inputs[SEED_FILES_INPUT_KEY]`` files into the workspace before the agent runs. - - Returns the seeded relative paths (for trial metadata). Paths that escape the workspace (absolute - or ``..`` traversal) are rejected so a task can only stage files inside its own sandbox. - """ - seeds = task.inputs.get(SEED_FILES_INPUT_KEY) - if not isinstance(seeds, Mapping): - return [] - workspace_root = workspace_dir.resolve() - written: list[str] = [] - for rel_path, contents in seeds.items(): - target = (workspace_root / str(rel_path)).resolve() - if target != workspace_root and workspace_root not in target.parents: - raise ValueError(f"seed file path escapes the workspace: {rel_path!r}") - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(contents if isinstance(contents, str) else str(contents), encoding="utf-8") - written.append(str(rel_path)) - return written - - def _failed_codex_trial( task: AgentEvalTask, evidence_dir: Path, diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/workspace_seeds.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/workspace_seeds.py new file mode 100644 index 0000000000..93cc4d1e84 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/workspace_seeds.py @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Workspace seed files for agent-eval tasks. + +A task can stage starter files into the agent's workspace before it runs, under +``inputs[SEED_FILES_INPUT_KEY]`` as a ``{relative_path: source}`` map. Each *source* is a +JSON-serializable value whose ``kind`` selects a registered :class:`SeedHandler` (a bare string is +sugar for inline text). + +The SDK ships handlers only for the kinds it can resolve with **no external dependency** — ``inline`` +and ``path``. Other kinds are contributed by consumers via :func:`register_seed_handler`; the SDK has +no knowledge of them (e.g. a platform ``fileset`` handler lives in the plugin and resolves against the +Files service at run time). An unregistered kind raises :class:`WorkspaceSeedError`. +""" + +from __future__ import annotations + +import base64 +from collections.abc import Mapping +from pathlib import Path +from typing import Any, Literal, Protocol, runtime_checkable + +from pydantic import BaseModel, ConfigDict, Field + +#: ``inputs`` key holding the ``{relative_path: seed}`` map of files to stage into the workspace. +SEED_FILES_INPUT_KEY = "files" + + +class WorkspaceSeedError(ValueError): + """A workspace seed could not be parsed, resolved, or written (bad value, unknown kind, ...). + + Subclasses ``ValueError`` so a runner's per-task error handling still catches it. + """ + + +class InlineSeed(BaseModel): + """File contents carried in the task itself. Portable to any runner.""" + + model_config = ConfigDict(extra="forbid") + + kind: Literal["inline"] = "inline" + content: str = Field(description="File contents; UTF-8 text, or base64-encoded bytes when encoding='base64'.") + encoding: Literal["text", "base64"] = "text" + + +class PathSeed(BaseModel): + """A file on the authoring host. Resolvable only where that path exists (local runs).""" + + model_config = ConfigDict(extra="forbid") + + kind: Literal["path"] = "path" + path: str = Field(description="Filesystem path on the authoring host (absolute or relative to the cwd).") + + +@runtime_checkable +class SeedHandler(Protocol): + """Parses + resolves one seed ``kind`` into the bytes to stage. + + Consumers implement this for kinds the SDK doesn't ship (e.g. a platform ``fileset`` handler) and + wire them in with :func:`register_seed_handler`. ``resolve`` runs at seeding time, so a handler + that needs external services (a client, credentials) acquires them there. + """ + + kind: str + + def parse(self, value: Mapping[str, Any]) -> BaseModel: + """Validate a raw seed mapping into this kind's typed model.""" + ... + + def resolve(self, seed: BaseModel) -> bytes: + """Resolve a parsed seed to the bytes to write into the workspace.""" + ... + + +_HANDLERS: dict[str, SeedHandler] = {} + + +def register_seed_handler(handler: SeedHandler) -> None: + """Register a :class:`SeedHandler` under its ``kind`` (replacing any handler already registered).""" + _HANDLERS[handler.kind] = handler + + +def _handler_for(kind: str) -> SeedHandler: + handler = _HANDLERS.get(kind) + if handler is None: + raise WorkspaceSeedError(f"no handler registered for seed kind {kind!r}") + return handler + + +class _InlineSeedHandler: + kind = "inline" + + def parse(self, value: Mapping[str, Any]) -> BaseModel: + return InlineSeed.model_validate(value) + + def resolve(self, seed: BaseModel) -> bytes: + assert isinstance(seed, InlineSeed) + if seed.encoding == "base64": + try: + return base64.b64decode(seed.content, validate=True) + except (ValueError, TypeError) as exc: + raise WorkspaceSeedError(f"inline seed is not valid base64: {exc}") from exc + return seed.content.encode("utf-8") + + +class _PathSeedHandler: + kind = "path" + + def parse(self, value: Mapping[str, Any]) -> BaseModel: + return PathSeed.model_validate(value) + + def resolve(self, seed: BaseModel) -> bytes: + assert isinstance(seed, PathSeed) + source = Path(seed.path).expanduser() + try: + return source.read_bytes() + except OSError as exc: + raise WorkspaceSeedError(f"path seed {seed.path!r} could not be read: {exc}") from exc + + +register_seed_handler(_InlineSeedHandler()) +register_seed_handler(_PathSeedHandler()) + + +def parse_seed(value: str | Mapping[str, Any]) -> BaseModel: + """Coerce a seed map value into its validated model. A bare string is inline UTF-8 text.""" + if isinstance(value, str): + return InlineSeed(content=value) + if not isinstance(value, Mapping): + raise WorkspaceSeedError(f"seed must be a string or mapping, got {type(value).__name__}") + kind = value.get("kind") + if not isinstance(kind, str): + raise WorkspaceSeedError("seed mapping is missing a string 'kind'") + handler = _handler_for(kind) + try: + return handler.parse(value) + except WorkspaceSeedError: + raise + except Exception as exc: # noqa: BLE001 - normalize a handler's validation error into our type + raise WorkspaceSeedError(f"invalid {kind!r} seed: {exc}") from exc + + +def _resolve_seed_bytes(seed: BaseModel) -> bytes: + """Resolve a parsed seed to bytes via its registered handler.""" + kind = getattr(seed, "kind", None) + if not isinstance(kind, str): + raise WorkspaceSeedError("parsed seed has no string 'kind'") + return _handler_for(kind).resolve(seed) + + +def seed_workspace(workspace_dir: str | Path, files: Mapping[str, Any] | None) -> list[str]: + """Write the ``files`` seed map into ``workspace_dir``; return the seeded relative paths. + + Each value is parsed into a seed model and resolved to bytes by its registered handler. Paths that + escape the workspace (absolute, or ``..`` traversal) are rejected so a task can only stage files + inside its own sandbox. ``None``/empty seeds nothing. + """ + if not isinstance(files, Mapping): + return [] + root = Path(workspace_dir).resolve() + written: list[str] = [] + for rel_path, value in files.items(): + target = (root / str(rel_path)).resolve() + if target != root and root not in target.parents: + raise WorkspaceSeedError(f"seed file path escapes the workspace: {rel_path!r}") + data = _resolve_seed_bytes(parse_seed(value)) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + written.append(str(rel_path)) + return written From c350b0928491078c8d3db3c9c82b37139989dfd3 Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Fri, 3 Jul 2026 14:29:50 -0300 Subject: [PATCH 2/3] feat(evaluator-sdk): add grader-only `reference` field to agent-eval tasks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Held-out ground truth for grading (canonical tests, expected outputs, rubric data) had no home on an agent-eval task, so graders could only read artifacts living in the agent's own writable workspace — which the agent can edit, making metrics like a pytest scorer trivially gameable (edit the test file the metric grades on). Add `AgentEvalTask.reference: dict[str, Any]`, surfaced to metrics via `_metric_row` as `row.data['reference']` but never routed through the agent-visible `_task_row` or seeded into the workspace. The field round-trips through the job wire/canonical DTOs (`_AgentEvalTaskCommon`) and `_to_runtime_task`/`to_spec`, so it survives remote submit. Also stop seeding the serialized task object (`task.json`) into the Docker sandbox workspace. Nothing in the runtime consumes it, and dumping the whole DTO would expose grader-only fields to the agent; the workspace now receives only the agent-facing projection (the prompt plus any declared workspace files). Stored-task persistence of `reference` (Task/TaskInput schemas + entity store) is a follow-up; those surface in the OpenAPI/SDK and are out of scope here. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Sandy Chapman --- .../agent_eval/evaluator.py | 3 +++ .../agent_eval/runtimes/docker_sandbox.py | 5 +++- .../nemo_evaluator_sdk/agent_eval/tasks.py | 6 +++++ .../agent_eval/test_docker_sandbox_runtime.py | 21 +++++++++++++++ .../tests/agent_eval/test_evaluator.py | 26 ++++++++++++++++++- plugins/nemo-evaluator/openapi/openapi.yaml | 16 ++++++++++++ .../src/nemo_evaluator/jobs/agent_evaluate.py | 2 ++ .../src/nemo_evaluator/jobs/agent_spec.py | 6 +++++ .../tests/test_agent_evaluate.py | 23 ++++++++++++++++ .../beta/evaluator/agent_eval/evaluator.py | 3 +++ .../agent_eval/runtimes/docker_sandbox.py | 5 +++- .../beta/evaluator/agent_eval/tasks.py | 6 +++++ 12 files changed, 119 insertions(+), 3 deletions(-) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py index 909f311165..037bdadd37 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py @@ -643,6 +643,9 @@ def _metric_row(task: AgentEvalTask, trial: AgentEvalTrial) -> dict[str, Any]: "metadata": task.metadata, }, "inputs": task.inputs, + # Grader-only ground truth: available to metrics here but never seeded into the agent's + # workspace (see AgentEvalTask.reference), so a metric can grade against held-out artifacts. + "reference": task.reference, "trial": { "id": trial.id, "task_id": trial.task_id, diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py index 29f1b4acae..43354d00b9 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py @@ -158,9 +158,12 @@ async def _run_task( await client.delete(sandbox) def _build_manifest(self, task: AgentEvalTask, sdk: SandboxSDK) -> Any: + # Seed only the agent-facing projection of the task: the prompt (its instruction) plus any + # declared workspace files. We deliberately do NOT serialize the task object into the + # workspace — nothing in the runtime consumes it, and dumping the whole DTO would expose + # grader-only fields (e.g. ``reference`` held-out ground truth) to the agent. entries: dict[str, Any] = { "instruction.md": sdk.File(content=_task_prompt(task).encode("utf-8")), - "task.json": sdk.File(content=task.model_dump_json().encode("utf-8")), "output": sdk.Dir(), } workspace_dir = task.inputs.get("workspace_dir") diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py index c8079bfca3..7d1a23ee7d 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py @@ -71,6 +71,12 @@ class AgentEvalTask(BaseModel): inputs: dict[str, Any] = Field( description="What the agent receives or starts from, e.g. instruction, filesystem seed, or state refs.", ) + reference: dict[str, Any] = Field( + default_factory=dict, + description="Grader-only ground truth (held-out tests, expected outputs, rubric data). Surfaced to " + "metrics as row.data['reference'] but never seeded into the agent's workspace or shown to the " + "agent, so a metric can grade against artifacts the agent cannot influence.", + ) metrics: list[Metric] = Field( default_factory=list, description="Ordered concrete SDK metric instances that score this task; metric types must be unique.", diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py index 182703d908..f94440cf6a 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py @@ -231,6 +231,27 @@ def test_manifest_maps_workspace_dir_to_local_dir(tmp_path: Path) -> None: assert manifest.entries["workspace"].src == workspace.resolve() +def test_manifest_omits_serialized_task_to_avoid_leaking_grader_fields() -> None: + # The workspace is seeded only with the agent-facing projection (prompt + declared files); the + # task object is never serialized in, so grader-only fields like ``reference`` cannot leak. + runtime = DockerSandboxAgentRuntime() + task = AgentEvalTask( + id="task-1", + intent="Intent text.", + inputs={"prompt": "Prompt text."}, + reference={"test_calculator.py": "def test_add(): assert add(2, 3) == 5"}, + ) + + manifest = runtime._build_manifest(task, _fake_sdk()) + + assert "task.json" not in manifest.entries + seeded = b"".join( + entry.content for entry in manifest.entries.values() if isinstance(entry, _FakeFile) + ).decode("utf-8") + assert "reference" not in seeded + assert "test_calculator.py" not in seeded + + def test_manifest_rejects_relative_or_missing_workspace_dir(tmp_path: Path) -> None: runtime = DockerSandboxAgentRuntime() diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py index 4f9b2b9a41..ccafc86d5b 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py @@ -8,7 +8,13 @@ from unittest.mock import AsyncMock, patch import pytest -from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator, _new_run_id, _trial_from_sample +from nemo_evaluator_sdk.agent_eval.evaluator import ( + AgentEvaluator, + _metric_row, + _new_run_id, + _task_row, + _trial_from_sample, +) from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore from nemo_evaluator_sdk.agent_eval.tasks import ( @@ -113,6 +119,24 @@ def test_generated_run_ids_are_unique_within_the_same_second() -> None: assert first.startswith("agent-eval-20260628120000-") +def test_metric_row_exposes_reference_but_task_row_hides_it() -> None: + # ``reference`` is grader-only held-out ground truth: metrics must see it, the agent (via the + # generation ``_task_row``) must not. + task = AgentEvalTask( + id="task-1", + intent="Fix the bug.", + inputs={"instruction": "Fix calculator.py."}, + reference={"test_calculator.py": "def test_add(): assert add(2, 3) == 5"}, + ) + trial = _candidate_trial() + + metric_row = _metric_row(task, trial) + assert metric_row["reference"] == {"test_calculator.py": "def test_add(): assert add(2, 3) == 5"} + + task_row = _task_row(task) + assert "reference" not in task_row + + def _score(summary: AgentEvalSummary, name: str) -> AggregateScore: for aggregate in summary.scores.scores: if aggregate.name == name: diff --git a/plugins/nemo-evaluator/openapi/openapi.yaml b/plugins/nemo-evaluator/openapi/openapi.yaml index e957f3fddd..cfb8c39e82 100644 --- a/plugins/nemo-evaluator/openapi/openapi.yaml +++ b/plugins/nemo-evaluator/openapi/openapi.yaml @@ -1659,6 +1659,14 @@ components: allOf: - $ref: '#/components/schemas/TaskInputs' description: The task's recognized input fields. + reference: + additionalProperties: true + type: object + title: Reference + description: Grader-only ground truth (held-out tests, expected outputs, + rubric data). Surfaced to metrics but never seeded into the agent's workspace + or shown to the agent, so a metric can grade against artifacts the agent + cannot influence. views: additionalProperties: $ref: '#/components/schemas/SemanticView' @@ -1703,6 +1711,14 @@ components: allOf: - $ref: '#/components/schemas/TaskInputs' description: The task's recognized input fields. + reference: + additionalProperties: true + type: object + title: Reference + description: Grader-only ground truth (held-out tests, expected outputs, + rubric data). Surfaced to metrics but never seeded into the agent's workspace + or shown to the agent, so a metric can grade against artifacts the agent + cannot influence. views: additionalProperties: $ref: '#/components/schemas/SemanticView' diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py index f332e3327b..b5c867034b 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py @@ -99,6 +99,7 @@ def _to_runtime_task(task: AgentEvalTaskSpec) -> AgentEvalTask: # The runtime task carries plain dicts; the typed DTOs collapse to them — recognized input # keys only, and the key/value metadata pairs folded into a mapping. inputs=task.inputs.model_dump(exclude_none=True), + reference=task.reference, metrics=[_runtime_metric(metric) for metric in task.metrics], views=task.views, metadata={item.key: item.value for item in task.metadata}, @@ -145,6 +146,7 @@ async def to_spec( id=task.id, intent=task.intent, inputs=task.inputs, + reference=task.reference, metrics=metrics, views=task.views, metadata=task.metadata, diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py index 034860a712..2e4b37f654 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py @@ -98,6 +98,12 @@ class _AgentEvalTaskCommon(BaseModel): id: str = Field(description="Stable task identifier, unique within the task collection.") intent: str = Field(description="Human-readable description of the desired agent behavior.") inputs: TaskInputs = Field(default_factory=TaskInputs, description="The task's recognized input fields.") + reference: dict[str, Any] = Field( + default_factory=dict, + description="Grader-only ground truth (held-out tests, expected outputs, rubric data). Surfaced to " + "metrics but never seeded into the agent's workspace or shown to the agent, so a metric can grade " + "against artifacts the agent cannot influence.", + ) views: dict[str, SemanticView] = Field( default_factory=dict, description="Optional reporting views mapping this task's metric outputs into named semantic scores.", diff --git a/plugins/nemo-evaluator/tests/test_agent_evaluate.py b/plugins/nemo-evaluator/tests/test_agent_evaluate.py index a9d1a09604..225244ecdf 100644 --- a/plugins/nemo-evaluator/tests/test_agent_evaluate.py +++ b/plugins/nemo-evaluator/tests/test_agent_evaluate.py @@ -125,6 +125,29 @@ def test_to_runtime_task_reconstructs_runtime_metric_instances() -> None: assert isinstance(task.metrics[0], ExactMatchMetric) +async def test_reference_round_trips_from_input_spec_to_runtime_task() -> None: + # Grader-only ``reference`` must survive the wire DTO -> canonical spec -> runtime task path so + # metrics can grade against held-out ground truth (never seeded into the agent workspace). + reference = {"test_calculator.py": "def test_add(): assert add(2, 3) == 5"} + input_spec = AgentEvalInputSpec( + target=CodexRunnerTarget(), + tasks=[ + AgentEvalTaskInput( + id="fix-bug", + intent="Fix the bug.", + inputs={"instruction": "Fix calculator.py."}, + reference=reference, + metrics=[_inline_metric()], + ) + ], + ) + + spec = await AgentEvalJob.to_spec(input_spec, workspace="dev", entity_client=None, async_sdk=None, is_local=True) + assert isinstance(spec, AgentEvalSpec) + assert spec.tasks[0].reference == reference + assert _to_runtime_task(spec.tasks[0]).reference == reference + + def test_agent_eval_job_reconstructs_tasks_and_persists_bundle(tmp_path: Path, mocker: MockerFixture) -> None: fake = _FakeEvaluator() mocker.patch.object(AgentEvalJob, "_build_evaluator", return_value=fake) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py index 6ec5d0be0c..adda911e12 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py @@ -643,6 +643,9 @@ def _metric_row(task: AgentEvalTask, trial: AgentEvalTrial) -> dict[str, Any]: "metadata": task.metadata, }, "inputs": task.inputs, + # Grader-only ground truth: available to metrics here but never seeded into the agent's + # workspace (see AgentEvalTask.reference), so a metric can grade against held-out artifacts. + "reference": task.reference, "trial": { "id": trial.id, "task_id": trial.task_id, diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py index f1f6ddd4f3..be39b8e1f8 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py @@ -158,9 +158,12 @@ async def _run_task( await client.delete(sandbox) def _build_manifest(self, task: AgentEvalTask, sdk: SandboxSDK) -> Any: + # Seed only the agent-facing projection of the task: the prompt (its instruction) plus any + # declared workspace files. We deliberately do NOT serialize the task object into the + # workspace — nothing in the runtime consumes it, and dumping the whole DTO would expose + # grader-only fields (e.g. ``reference`` held-out ground truth) to the agent. entries: dict[str, Any] = { "instruction.md": sdk.File(content=_task_prompt(task).encode("utf-8")), - "task.json": sdk.File(content=task.model_dump_json().encode("utf-8")), "output": sdk.Dir(), } workspace_dir = task.inputs.get("workspace_dir") diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py index 8acc03dc3f..445af23e24 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py @@ -71,6 +71,12 @@ class AgentEvalTask(BaseModel): inputs: dict[str, Any] = Field( description="What the agent receives or starts from, e.g. instruction, filesystem seed, or state refs.", ) + reference: dict[str, Any] = Field( + default_factory=dict, + description="Grader-only ground truth (held-out tests, expected outputs, rubric data). Surfaced to " + "metrics as row.data['reference'] but never seeded into the agent's workspace or shown to the " + "agent, so a metric can grade against artifacts the agent cannot influence.", + ) metrics: list[Metric] = Field( default_factory=list, description="Ordered concrete SDK metric instances that score this task; metric types must be unique.", From 88a593e86ee840b2f571ddb352e7e2206007adce Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Mon, 6 Jul 2026 11:48:56 -0300 Subject: [PATCH 3/3] style(evaluator-sdk): format leak-check test to satisfy CI ruff (0.15.7) CI's frozen ruff reformats the b"".join(...) expression in test_manifest_omits_serialized_task_to_avoid_leaking_grader_fields; split it into a list + join so it's stable across ruff versions. Fixes lint-python-style on #566. Signed-off-by: Sandy Chapman --- .../tests/agent_eval/test_docker_sandbox_runtime.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py index f94440cf6a..54d41c3431 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py @@ -245,9 +245,8 @@ def test_manifest_omits_serialized_task_to_avoid_leaking_grader_fields() -> None manifest = runtime._build_manifest(task, _fake_sdk()) assert "task.json" not in manifest.entries - seeded = b"".join( - entry.content for entry in manifest.entries.values() if isinstance(entry, _FakeFile) - ).decode("utf-8") + seeded_files = [entry.content for entry in manifest.entries.values() if isinstance(entry, _FakeFile)] + seeded = b"".join(seeded_files).decode("utf-8") assert "reference" not in seeded assert "test_calculator.py" not in seeded