NVIDIA-NeMo · SandyChapman · Jul 6, 2026 · Jul 3, 2026 · Jul 3, 2026 · Jul 6, 2026
@@ -643,6 +643,9 @@ def _metric_row(task: AgentEvalTask, trial: AgentEvalTrial) -> dict[str, Any]:
             "metadata": task.metadata,
         },
         "inputs": task.inputs,
+        # Grader-only ground truth: available to metrics here but never seeded into the agent's
+        # workspace (see AgentEvalTask.reference), so a metric can grade against held-out artifacts.
+        "reference": task.reference,
         "trial": {
             "id": trial.id,
             "task_id": trial.task_id,

@@ -158,9 +158,12 @@ async def _run_task(
                     await client.delete(sandbox)
 
     def _build_manifest(self, task: AgentEvalTask, sdk: SandboxSDK) -> Any:
+        # Seed only the agent-facing projection of the task: the prompt (its instruction) plus any
+        # declared workspace files. We deliberately do NOT serialize the task object into the
+        # workspace — nothing in the runtime consumes it, and dumping the whole DTO would expose
+        # grader-only fields (e.g. ``reference`` held-out ground truth) to the agent.
         entries: dict[str, Any] = {
             "instruction.md": sdk.File(content=_task_prompt(task).encode("utf-8")),
-            "task.json": sdk.File(content=task.model_dump_json().encode("utf-8")),
             "output": sdk.Dir(),
         }
         workspace_dir = task.inputs.get("workspace_dir")

@@ -71,6 +71,12 @@ class AgentEvalTask(BaseModel):
     inputs: dict[str, Any] = Field(
         description="What the agent receives or starts from, e.g. instruction, filesystem seed, or state refs.",
     )
+    reference: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Grader-only ground truth (held-out tests, expected outputs, rubric data). Surfaced to "
+        "metrics as row.data['reference'] but never seeded into the agent's workspace or shown to the "
+        "agent, so a metric can grade against artifacts the agent cannot influence.",
+    )
     metrics: list[Metric] = Field(
         default_factory=list,
         description="Ordered concrete SDK metric instances that score this task; metric types must be unique.",

@@ -231,6 +231,26 @@ def test_manifest_maps_workspace_dir_to_local_dir(tmp_path: Path) -> None:
     assert manifest.entries["workspace"].src == workspace.resolve()
 
 
+def test_manifest_omits_serialized_task_to_avoid_leaking_grader_fields() -> None:
+    # The workspace is seeded only with the agent-facing projection (prompt + declared files); the
+    # task object is never serialized in, so grader-only fields like ``reference`` cannot leak.
+    runtime = DockerSandboxAgentRuntime()
+    task = AgentEvalTask(
+        id="task-1",
+        intent="Intent text.",
+        inputs={"prompt": "Prompt text."},
+        reference={"test_calculator.py": "def test_add(): assert add(2, 3) == 5"},
+    )
+
+    manifest = runtime._build_manifest(task, _fake_sdk())
+
+    assert "task.json" not in manifest.entries
+    seeded_files = [entry.content for entry in manifest.entries.values() if isinstance(entry, _FakeFile)]
+    seeded = b"".join(seeded_files).decode("utf-8")
+    assert "reference" not in seeded
+    assert "test_calculator.py" not in seeded
+
+
 def test_manifest_rejects_relative_or_missing_workspace_dir(tmp_path: Path) -> None:
     runtime = DockerSandboxAgentRuntime()
 

@@ -8,7 +8,13 @@
 from unittest.mock import AsyncMock, patch
 
 import pytest
-from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator, _new_run_id, _trial_from_sample
+from nemo_evaluator_sdk.agent_eval.evaluator import (
+    AgentEvaluator,
+    _metric_row,
+    _new_run_id,
+    _task_row,
+    _trial_from_sample,
+)
 from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary
 from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore
 from nemo_evaluator_sdk.agent_eval.tasks import (
@@ -113,6 +119,24 @@ def test_generated_run_ids_are_unique_within_the_same_second() -> None:
     assert first.startswith("agent-eval-20260628120000-")
 
 
+def test_metric_row_exposes_reference_but_task_row_hides_it() -> None:
+    # ``reference`` is grader-only held-out ground truth: metrics must see it, the agent (via the
+    # generation ``_task_row``) must not.
+    task = AgentEvalTask(
+        id="task-1",
+        intent="Fix the bug.",
+        inputs={"instruction": "Fix calculator.py."},
+        reference={"test_calculator.py": "def test_add(): assert add(2, 3) == 5"},
+    )
+    trial = _candidate_trial()
+
+    metric_row = _metric_row(task, trial)
+    assert metric_row["reference"] == {"test_calculator.py": "def test_add(): assert add(2, 3) == 5"}
+
+    task_row = _task_row(task)
+    assert "reference" not in task_row
+
+
 def _score(summary: AgentEvalSummary, name: str) -> AggregateScore:
     for aggregate in summary.scores.scores:
         if aggregate.name == name:

@@ -99,6 +99,7 @@ def _to_runtime_task(task: AgentEvalTaskSpec) -> AgentEvalTask:
         # The runtime task carries plain dicts; the typed DTOs collapse to them — recognized input
         # keys only, and the key/value metadata pairs folded into a mapping.
         inputs=task.inputs.model_dump(exclude_none=True),
+        reference=task.reference,
         metrics=[_runtime_metric(metric) for metric in task.metrics],
         views=task.views,
         metadata={item.key: item.value for item in task.metadata},
@@ -145,6 +146,7 @@ async def to_spec(
                     id=task.id,
                     intent=task.intent,
                     inputs=task.inputs,
+                    reference=task.reference,
                     metrics=metrics,
                     views=task.views,
                     metadata=task.metadata,

@@ -98,6 +98,12 @@ class _AgentEvalTaskCommon(BaseModel):
     id: str = Field(description="Stable task identifier, unique within the task collection.")
     intent: str = Field(description="Human-readable description of the desired agent behavior.")
     inputs: TaskInputs = Field(default_factory=TaskInputs, description="The task's recognized input fields.")
+    reference: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Grader-only ground truth (held-out tests, expected outputs, rubric data). Surfaced to "
+        "metrics but never seeded into the agent's workspace or shown to the agent, so a metric can grade "
+        "against artifacts the agent cannot influence.",
+    )
     views: dict[str, SemanticView] = Field(
         default_factory=dict,
         description="Optional reporting views mapping this task's metric outputs into named semantic scores.",

@@ -125,6 +125,29 @@ def test_to_runtime_task_reconstructs_runtime_metric_instances() -> None:
     assert isinstance(task.metrics[0], ExactMatchMetric)
 
 
+async def test_reference_round_trips_from_input_spec_to_runtime_task() -> None:
+    # Grader-only ``reference`` must survive the wire DTO -> canonical spec -> runtime task path so
+    # metrics can grade against held-out ground truth (never seeded into the agent workspace).
+    reference = {"test_calculator.py": "def test_add(): assert add(2, 3) == 5"}
+    input_spec = AgentEvalInputSpec(
+        target=CodexRunnerTarget(),
+        tasks=[
+            AgentEvalTaskInput(
+                id="fix-bug",
+                intent="Fix the bug.",
+                inputs={"instruction": "Fix calculator.py."},
+                reference=reference,
+                metrics=[_inline_metric()],
+            )
+        ],
+    )
+
+    spec = await AgentEvalJob.to_spec(input_spec, workspace="dev", entity_client=None, async_sdk=None, is_local=True)
+    assert isinstance(spec, AgentEvalSpec)
+    assert spec.tasks[0].reference == reference
+    assert _to_runtime_task(spec.tasks[0]).reference == reference
+
+
 def test_agent_eval_job_reconstructs_tasks_and_persists_bundle(tmp_path: Path, mocker: MockerFixture) -> None:
     fake = _FakeEvaluator()
     mocker.patch.object(AgentEvalJob, "_build_evaluator", return_value=fake)