diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py index 909f311165..037bdadd37 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py @@ -643,6 +643,9 @@ def _metric_row(task: AgentEvalTask, trial: AgentEvalTrial) -> dict[str, Any]: "metadata": task.metadata, }, "inputs": task.inputs, + # Grader-only ground truth: available to metrics here but never seeded into the agent's + # workspace (see AgentEvalTask.reference), so a metric can grade against held-out artifacts. + "reference": task.reference, "trial": { "id": trial.id, "task_id": trial.task_id, diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py index 29f1b4acae..43354d00b9 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py @@ -158,9 +158,12 @@ async def _run_task( await client.delete(sandbox) def _build_manifest(self, task: AgentEvalTask, sdk: SandboxSDK) -> Any: + # Seed only the agent-facing projection of the task: the prompt (its instruction) plus any + # declared workspace files. We deliberately do NOT serialize the task object into the + # workspace — nothing in the runtime consumes it, and dumping the whole DTO would expose + # grader-only fields (e.g. ``reference`` held-out ground truth) to the agent. entries: dict[str, Any] = { "instruction.md": sdk.File(content=_task_prompt(task).encode("utf-8")), - "task.json": sdk.File(content=task.model_dump_json().encode("utf-8")), "output": sdk.Dir(), } workspace_dir = task.inputs.get("workspace_dir") diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py index c8079bfca3..7d1a23ee7d 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py @@ -71,6 +71,12 @@ class AgentEvalTask(BaseModel): inputs: dict[str, Any] = Field( description="What the agent receives or starts from, e.g. instruction, filesystem seed, or state refs.", ) + reference: dict[str, Any] = Field( + default_factory=dict, + description="Grader-only ground truth (held-out tests, expected outputs, rubric data). Surfaced to " + "metrics as row.data['reference'] but never seeded into the agent's workspace or shown to the " + "agent, so a metric can grade against artifacts the agent cannot influence.", + ) metrics: list[Metric] = Field( default_factory=list, description="Ordered concrete SDK metric instances that score this task; metric types must be unique.", diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py index 182703d908..54d41c3431 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py @@ -231,6 +231,26 @@ def test_manifest_maps_workspace_dir_to_local_dir(tmp_path: Path) -> None: assert manifest.entries["workspace"].src == workspace.resolve() +def test_manifest_omits_serialized_task_to_avoid_leaking_grader_fields() -> None: + # The workspace is seeded only with the agent-facing projection (prompt + declared files); the + # task object is never serialized in, so grader-only fields like ``reference`` cannot leak. + runtime = DockerSandboxAgentRuntime() + task = AgentEvalTask( + id="task-1", + intent="Intent text.", + inputs={"prompt": "Prompt text."}, + reference={"test_calculator.py": "def test_add(): assert add(2, 3) == 5"}, + ) + + manifest = runtime._build_manifest(task, _fake_sdk()) + + assert "task.json" not in manifest.entries + seeded_files = [entry.content for entry in manifest.entries.values() if isinstance(entry, _FakeFile)] + seeded = b"".join(seeded_files).decode("utf-8") + assert "reference" not in seeded + assert "test_calculator.py" not in seeded + + def test_manifest_rejects_relative_or_missing_workspace_dir(tmp_path: Path) -> None: runtime = DockerSandboxAgentRuntime() diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py index 4f9b2b9a41..ccafc86d5b 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py @@ -8,7 +8,13 @@ from unittest.mock import AsyncMock, patch import pytest -from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator, _new_run_id, _trial_from_sample +from nemo_evaluator_sdk.agent_eval.evaluator import ( + AgentEvaluator, + _metric_row, + _new_run_id, + _task_row, + _trial_from_sample, +) from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore from nemo_evaluator_sdk.agent_eval.tasks import ( @@ -113,6 +119,24 @@ def test_generated_run_ids_are_unique_within_the_same_second() -> None: assert first.startswith("agent-eval-20260628120000-") +def test_metric_row_exposes_reference_but_task_row_hides_it() -> None: + # ``reference`` is grader-only held-out ground truth: metrics must see it, the agent (via the + # generation ``_task_row``) must not. + task = AgentEvalTask( + id="task-1", + intent="Fix the bug.", + inputs={"instruction": "Fix calculator.py."}, + reference={"test_calculator.py": "def test_add(): assert add(2, 3) == 5"}, + ) + trial = _candidate_trial() + + metric_row = _metric_row(task, trial) + assert metric_row["reference"] == {"test_calculator.py": "def test_add(): assert add(2, 3) == 5"} + + task_row = _task_row(task) + assert "reference" not in task_row + + def _score(summary: AgentEvalSummary, name: str) -> AggregateScore: for aggregate in summary.scores.scores: if aggregate.name == name: diff --git a/plugins/nemo-evaluator/openapi/openapi.yaml b/plugins/nemo-evaluator/openapi/openapi.yaml index e957f3fddd..cfb8c39e82 100644 --- a/plugins/nemo-evaluator/openapi/openapi.yaml +++ b/plugins/nemo-evaluator/openapi/openapi.yaml @@ -1659,6 +1659,14 @@ components: allOf: - $ref: '#/components/schemas/TaskInputs' description: The task's recognized input fields. + reference: + additionalProperties: true + type: object + title: Reference + description: Grader-only ground truth (held-out tests, expected outputs, + rubric data). Surfaced to metrics but never seeded into the agent's workspace + or shown to the agent, so a metric can grade against artifacts the agent + cannot influence. views: additionalProperties: $ref: '#/components/schemas/SemanticView' @@ -1703,6 +1711,14 @@ components: allOf: - $ref: '#/components/schemas/TaskInputs' description: The task's recognized input fields. + reference: + additionalProperties: true + type: object + title: Reference + description: Grader-only ground truth (held-out tests, expected outputs, + rubric data). Surfaced to metrics but never seeded into the agent's workspace + or shown to the agent, so a metric can grade against artifacts the agent + cannot influence. views: additionalProperties: $ref: '#/components/schemas/SemanticView' diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py index f332e3327b..b5c867034b 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py @@ -99,6 +99,7 @@ def _to_runtime_task(task: AgentEvalTaskSpec) -> AgentEvalTask: # The runtime task carries plain dicts; the typed DTOs collapse to them — recognized input # keys only, and the key/value metadata pairs folded into a mapping. inputs=task.inputs.model_dump(exclude_none=True), + reference=task.reference, metrics=[_runtime_metric(metric) for metric in task.metrics], views=task.views, metadata={item.key: item.value for item in task.metadata}, @@ -145,6 +146,7 @@ async def to_spec( id=task.id, intent=task.intent, inputs=task.inputs, + reference=task.reference, metrics=metrics, views=task.views, metadata=task.metadata, diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py index 034860a712..2e4b37f654 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py @@ -98,6 +98,12 @@ class _AgentEvalTaskCommon(BaseModel): id: str = Field(description="Stable task identifier, unique within the task collection.") intent: str = Field(description="Human-readable description of the desired agent behavior.") inputs: TaskInputs = Field(default_factory=TaskInputs, description="The task's recognized input fields.") + reference: dict[str, Any] = Field( + default_factory=dict, + description="Grader-only ground truth (held-out tests, expected outputs, rubric data). Surfaced to " + "metrics but never seeded into the agent's workspace or shown to the agent, so a metric can grade " + "against artifacts the agent cannot influence.", + ) views: dict[str, SemanticView] = Field( default_factory=dict, description="Optional reporting views mapping this task's metric outputs into named semantic scores.", diff --git a/plugins/nemo-evaluator/tests/test_agent_evaluate.py b/plugins/nemo-evaluator/tests/test_agent_evaluate.py index a9d1a09604..225244ecdf 100644 --- a/plugins/nemo-evaluator/tests/test_agent_evaluate.py +++ b/plugins/nemo-evaluator/tests/test_agent_evaluate.py @@ -125,6 +125,29 @@ def test_to_runtime_task_reconstructs_runtime_metric_instances() -> None: assert isinstance(task.metrics[0], ExactMatchMetric) +async def test_reference_round_trips_from_input_spec_to_runtime_task() -> None: + # Grader-only ``reference`` must survive the wire DTO -> canonical spec -> runtime task path so + # metrics can grade against held-out ground truth (never seeded into the agent workspace). + reference = {"test_calculator.py": "def test_add(): assert add(2, 3) == 5"} + input_spec = AgentEvalInputSpec( + target=CodexRunnerTarget(), + tasks=[ + AgentEvalTaskInput( + id="fix-bug", + intent="Fix the bug.", + inputs={"instruction": "Fix calculator.py."}, + reference=reference, + metrics=[_inline_metric()], + ) + ], + ) + + spec = await AgentEvalJob.to_spec(input_spec, workspace="dev", entity_client=None, async_sdk=None, is_local=True) + assert isinstance(spec, AgentEvalSpec) + assert spec.tasks[0].reference == reference + assert _to_runtime_task(spec.tasks[0]).reference == reference + + def test_agent_eval_job_reconstructs_tasks_and_persists_bundle(tmp_path: Path, mocker: MockerFixture) -> None: fake = _FakeEvaluator() mocker.patch.object(AgentEvalJob, "_build_evaluator", return_value=fake) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py index 6ec5d0be0c..adda911e12 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py @@ -643,6 +643,9 @@ def _metric_row(task: AgentEvalTask, trial: AgentEvalTrial) -> dict[str, Any]: "metadata": task.metadata, }, "inputs": task.inputs, + # Grader-only ground truth: available to metrics here but never seeded into the agent's + # workspace (see AgentEvalTask.reference), so a metric can grade against held-out artifacts. + "reference": task.reference, "trial": { "id": trial.id, "task_id": trial.task_id, diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py index f1f6ddd4f3..be39b8e1f8 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py @@ -158,9 +158,12 @@ async def _run_task( await client.delete(sandbox) def _build_manifest(self, task: AgentEvalTask, sdk: SandboxSDK) -> Any: + # Seed only the agent-facing projection of the task: the prompt (its instruction) plus any + # declared workspace files. We deliberately do NOT serialize the task object into the + # workspace — nothing in the runtime consumes it, and dumping the whole DTO would expose + # grader-only fields (e.g. ``reference`` held-out ground truth) to the agent. entries: dict[str, Any] = { "instruction.md": sdk.File(content=_task_prompt(task).encode("utf-8")), - "task.json": sdk.File(content=task.model_dump_json().encode("utf-8")), "output": sdk.Dir(), } workspace_dir = task.inputs.get("workspace_dir") diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py index 8acc03dc3f..445af23e24 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py @@ -71,6 +71,12 @@ class AgentEvalTask(BaseModel): inputs: dict[str, Any] = Field( description="What the agent receives or starts from, e.g. instruction, filesystem seed, or state refs.", ) + reference: dict[str, Any] = Field( + default_factory=dict, + description="Grader-only ground truth (held-out tests, expected outputs, rubric data). Surfaced to " + "metrics as row.data['reference'] but never seeded into the agent's workspace or shown to the " + "agent, so a metric can grade against artifacts the agent cannot influence.", + ) metrics: list[Metric] = Field( default_factory=list, description="Ordered concrete SDK metric instances that score this task; metric types must be unique.",