fix(agent-eval): render targets from the task instruction (one canonical input)

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py

@@ -48,7 +48,15 @@
 from nemo_evaluator_sdk.inference import InferenceFn
 from nemo_evaluator_sdk.metrics.protocol import Metric, validate_metric_result
 from nemo_evaluator_sdk.metrics.utils import metric_type_name
-from nemo_evaluator_sdk.values import Agent, AgentBase, Model, RunConfig, RunConfigOnline, RunConfigOnlineModel
+from nemo_evaluator_sdk.values import (
+    Agent,
+    AgentBase,
+    GenericAgent,
+    Model,
+    RunConfig,
+    RunConfigOnline,
+    RunConfigOnlineModel,
+)
 from nemo_evaluator_sdk.values.evidence import (
     EVIDENCE_FORMAT_JSON,
     EVIDENCE_TRACE,
@@ -621,20 +629,25 @@ def _resolve_live_params(
     return RunConfigOnline(**params.model_dump(mode="python"))
 
 
-def _default_prompt_template(target: Model | Agent) -> dict[str, Any]:
+def _default_prompt_template(target: Model | Agent) -> dict[str, Any] | str:
+    # Every default renders against the single canonical task input, ``instruction`` (see
+    # ``AgentEvalTask.agent_prompt``); no other input key is special.
+    if isinstance(target, GenericAgent):
+        # A generic HTTP agent defines its own request entirely through its `body` template, which
+        # renders against the task inputs (e.g. `{{ instruction }}`). Pass the task row through
+        # unchanged so `body` — not a chat/completions assumption — shapes the payload. See
+        # `_resolve_http_agent_invocation`, which renders `body` against this request.
+        return "{{ item }}"
     if isinstance(target, Model) and _is_completions_endpoint(target.url):
-        return {"prompt": "{{item.prompt}}"}
-    return {"messages": [{"role": "user", "content": "{{item.prompt}}"}]}
+        return {"prompt": "{{item.instruction}}"}
+    return {"messages": [{"role": "user", "content": "{{item.instruction}}"}]}
 
 
 def _task_row(task: AgentEvalTask) -> dict[str, Any]:
-    # `prompt` is what the target under evaluation is prompted with (via the `{{item.prompt}}`
-    # template): a dataset `prompt` column or the agent `instruction`.
-    return {
-        **task.inputs,
-        "task_id": task.id,
-        "prompt": task.inputs.get("prompt") or task.inputs.get("instruction"),
-    }
+    # The task inputs verbatim, plus the task id. `instruction` is the single canonical input the
+    # target is prompted with (see `AgentEvalTask.agent_prompt` and `_default_prompt_template`); no
+    # input key is synthesized or aliased here.
+    return {**task.inputs, "task_id": task.id}
 
 
 def _metric_row(task: AgentEvalTask, trial: AgentEvalTrial) -> dict[str, Any]:

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py

@@ -606,14 +606,19 @@ def discover_harbor_tasks(dataset_path: str | Path) -> list[AgentEvalTask]:
         task_name = config.get("task", {}).get("name", task_dir.name)
         instruction_path = task_dir / "instruction.md"
         try:
-            intent = instruction_path.read_text(encoding="utf-8").strip() if instruction_path.is_file() else task_name
+            instruction = (
+                instruction_path.read_text(encoding="utf-8").strip() if instruction_path.is_file() else task_name
+            )
         except (OSError, UnicodeDecodeError) as exc:
             raise ValueError(f"unreadable Harbor instruction at {instruction_path}: {exc}") from exc
         tasks.append(
             AgentEvalTask(
                 id=task_name,
-                intent=intent,
-                inputs={"instruction": intent},
+                # `intent` is human-facing metadata, never shown to the agent; the task name is the
+                # only human label Harbor's task.toml provides. The instruction the agent acts on
+                # lives in `inputs["instruction"]`.
+                intent=task_name,
+                inputs={"instruction": instruction},
                 metrics=[HarborRewardMetric()],
                 metadata={"harbor_dataset_path": str(dataset_path), "harbor_task_dir": str(task_dir)},
             )

packages/nemo_evaluator_sdk/tests/agent_eval/test_callable_runtime.py

@@ -11,7 +11,7 @@
 
 
 def _task(task_id: str) -> AgentEvalTask:
-    return AgentEvalTask(id=task_id, intent="Answer.", inputs={"prompt": f"q-{task_id}"})
+    return AgentEvalTask(id=task_id, intent="Answer.", inputs={"instruction": f"q-{task_id}"})
 
 
 @pytest.mark.asyncio

packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py

@@ -41,7 +41,7 @@
 
 
 def test_trial_from_sample_falls_back_to_reasoning_content() -> None:
-    task = AgentEvalTask(id="task-1", intent="Answer.", inputs={"prompt": "Q?"})
+    task = AgentEvalTask(id="task-1", intent="Answer.", inputs={"instruction": "Q?"})
     target = Model(name="reasoning-model", url="https://example/v1/chat/completions")
     response = {"choices": [{"message": {"content": None, "reasoning_content": "the reasoned answer"}}]}
 
@@ -56,7 +56,7 @@ def test_trial_from_sample_falls_back_to_reasoning_content() -> None:
 
 
 def test_trial_from_sample_fallback_trace_has_trace_kind() -> None:
-    task = AgentEvalTask(id="task-1", intent="Answer.", inputs={"prompt": "Q?"})
+    task = AgentEvalTask(id="task-1", intent="Answer.", inputs={"instruction": "Q?"})
     target = Model(name="target", url="https://example/v1/chat/completions")
 
     trial = _trial_from_sample(task, target, {"output_text": "answer"})
@@ -67,7 +67,7 @@ def test_trial_from_sample_fallback_trace_has_trace_kind() -> None:
 
 
 def test_trial_from_sample_preserves_typed_trace_and_canonical_metadata() -> None:
-    task = AgentEvalTask(id="task-1", intent="Answer.", inputs={"prompt": "Q?"})
+    task = AgentEvalTask(id="task-1", intent="Answer.", inputs={"instruction": "Q?"})
     target = Model(name="canonical-target", url="https://example/v1/chat/completions")
     typed_trace = {
         "schema_version": "ATIF-v1.7",
@@ -200,7 +200,7 @@ def _task(metric: Any | None = None, *, task_id: str = "task-1") -> AgentEvalTas
     return AgentEvalTask(
         id=task_id,
         intent="Answer a professional benchmark prompt.",
-        inputs={"prompt": "What is the answer?", "domain": "Finance MBA"},
+        inputs={"instruction": "What is the answer?", "domain": "Finance MBA"},
         metrics=[metric or _ConstantMetric()],
         metadata={"benchmark": "Example", "domain": "Finance MBA"},
     )
@@ -363,24 +363,27 @@ async def fake_model_inference(
 
 
 @pytest.mark.asyncio
-async def test_live_model_generation_uses_instruction_when_prompt_is_absent() -> None:
+async def test_live_model_generation_completions_endpoint_prompts_with_instruction() -> None:
+    # A bare /v1/completions endpoint uses the `{"prompt": ...}` request shape; the task instruction
+    # is rendered into that wire field (there is no chat `messages` wrapper).
     async def fake_model_inference(
         model: Model,
         request: dict[str, Any],
         max_retries: int | None,
         **kwargs: Any,
     ) -> dict[str, Any]:
         del model, max_retries, kwargs
-        assert request["messages"][0]["content"] == "Use the task instruction."
-        return {"choices": [{"message": {"role": "assistant", "content": "Generated model answer"}}]}
+        assert request["prompt"] == "Use the task instruction."
+        assert "messages" not in request
+        return {"choices": [{"text": "Generated model answer"}]}
 
     task = AgentEvalTask(
         id="task-1",
-        intent="Fallback intent.",
+        intent="Answer the instruction.",
         inputs={"instruction": "Use the task instruction."},
         metrics=[_ConstantMetric()],
     )
-    model = Model(url="https://model.test/v1/chat/completions", name="target-model", format=ModelFormat.OPEN_AI)
+    model = Model(url="https://model.test/v1/completions", name="target-model", format=ModelFormat.OPEN_AI)
 
     await AgentEvaluator(inference_fn=fake_model_inference).run(
         tasks=[task],
@@ -395,7 +398,7 @@ async def test_metric_failure_records_failed_score_and_does_not_stop_other_metri
     other_task = AgentEvalTask(
         id="task-2",
         intent="Answer another prompt.",
-        inputs={"prompt": "Another question?"},
+        inputs={"instruction": "Another question?"},
         metrics=[_OtherMetric()],
     )
     trials = [
@@ -435,7 +438,7 @@ def test_summary_reports_coverage_and_merges_views_into_scores() -> None:
     task = AgentEvalTask(
         id="task-1",
         intent="Answer a prompt.",
-        inputs={"prompt": "Question?"},
+        inputs={"instruction": "Question?"},
         metrics=[_ConstantMetric(), _OtherMetric()],
         views={
             "outcome_correctness": SemanticView(
@@ -471,7 +474,10 @@ async def fake_agent_inference(
         **kwargs: Any,
     ) -> dict[str, Any]:
         del agent, kwargs
-        assert request["messages"][0]["content"] == "What is the answer?"
+        # A generic HTTP agent receives the task row directly — no chat/completions wrapper — so its
+        # `body` template can reference task inputs such as `{{ instruction }}`.
+        assert "messages" not in request
+        assert request["instruction"] == "What is the answer?"
         return {
             "choices": [{"message": {"role": "assistant", "content": "Generated agent answer"}}],
             "trajectory": [{"tool": "search", "line": 3}],
@@ -481,7 +487,7 @@ async def fake_agent_inference(
         url="https://agent.test",
         name="target-agent",
         format=AgentFormat.GENERIC,
-        body={"input": "{{ messages[-1].content }}"},
+        body={"input": "{{ instruction }}"},
         response_path="$.answer",
     )
     result = await AgentEvaluator(inference_fn=fake_agent_inference).run(
@@ -627,7 +633,7 @@ async def test_default_agent_invocation_receives_run_context_and_evidence_dir(tm
     )
     agent = NemoAgentToolkitAgent(url="https://agent.test", name="target-agent", format=AgentFormat.NEMO_AGENT_TOOLKIT)
     prompt_template = {
-        "input_message": "{{ item.prompt }}",
+        "input_message": "{{ item.instruction }}",
         "conversation_id": ("{{ agent_eval.run_id }}-{{ agent_eval.task_id }}-{{ agent_eval.invocation_id }}"),
     }
 
@@ -794,7 +800,7 @@ async def test_run_rejects_tasks_without_trials() -> None:
     other_task = AgentEvalTask(
         id="task-2",
         intent="Answer another prompt.",
-        inputs={"prompt": "Another question?"},
+        inputs={"instruction": "Another question?"},
         metrics=[_OtherMetric()],
     )
 

packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py

@@ -57,9 +57,9 @@ async def test_harbor_runner_scores_through_agent_evaluator_and_adapts_legacy_pa
     _write_trial(job_dir, "noreward-task__ccc", "noreward-task", reward=None)
 
     tasks = [
-        AgentEvalTask(id="pass-task", intent="x", inputs={"prompt": "p"}, metrics=[HarborRewardMetric()]),
-        AgentEvalTask(id="fail-task", intent="y", inputs={"prompt": "q"}, metrics=[HarborRewardMetric()]),
-        AgentEvalTask(id="noreward-task", intent="z", inputs={"prompt": "r"}, metrics=[HarborRewardMetric()]),
+        AgentEvalTask(id="pass-task", intent="x", inputs={"instruction": "p"}, metrics=[HarborRewardMetric()]),
+        AgentEvalTask(id="fail-task", intent="y", inputs={"instruction": "q"}, metrics=[HarborRewardMetric()]),
+        AgentEvalTask(id="noreward-task", intent="z", inputs={"instruction": "r"}, metrics=[HarborRewardMetric()]),
     ]
 
     # Direct adaptation: reward + tokens land on metadata, exception flips status to PARTIAL, evidence present.
@@ -100,7 +100,7 @@ def test_reward_with_no_matching_reward_key_is_partial_and_warns(tmp_path: Path,
     job_dir = tmp_path / "job"
     job_dir.mkdir()
     _write_trial(job_dir, "t__aaa", "t", reward=1.0)  # emitted under "reward"
-    tasks = [AgentEvalTask(id="t", intent="x", inputs={"prompt": "p"}, metrics=[HarborRewardMetric()])]
+    tasks = [AgentEvalTask(id="t", intent="x", inputs={"instruction": "p"}, metrics=[HarborRewardMetric()])]
 
     with caplog.at_level(logging.WARNING):
         trials = build_trials_from_job_dir(job_dir, tasks, reward_key="missing")
@@ -116,7 +116,10 @@ def test_task_discovery_and_taskset_loader_over_bundled_dataset() -> None:
     tasks = discover_harbor_tasks(_HELLO_WORLD_DATASET)
     assert [task.id for task in tasks] == ["harbor/hello-world"]
     task = tasks[0]
-    assert task.intent  # instruction.md content
+    # `intent` is the human-facing task name (metadata), NOT the instruction; the instruction the
+    # agent acts on comes from instruction.md and lives in inputs["instruction"].
+    assert task.intent == "harbor/hello-world"
+    assert task.inputs["instruction"] == 'Create a file called hello.txt with "Hello, world!" as the content.'
     assert [metric_type_name(metric) for metric in task.metrics] == ["harbor_reward"]
     # The dataset dir and task dir are stamped on the task so a native runner can
     # recover them without a separate dataset_path argument.
@@ -195,7 +198,7 @@ def test_multiple_attempts_map_to_one_trial_each(tmp_path: Path) -> None:
     job_dir.mkdir()
     _write_trial(job_dir, "t__aaa", "t", reward=1.0)
     _write_trial(job_dir, "t__bbb", "t", reward=0.0)
-    tasks = [AgentEvalTask(id="t", intent="x", inputs={"prompt": "p"}, metrics=[HarborRewardMetric()])]
+    tasks = [AgentEvalTask(id="t", intent="x", inputs={"instruction": "p"}, metrics=[HarborRewardMetric()])]
 
     trials = build_trials_from_job_dir(job_dir, tasks)
     assert [trial.task_id for trial in trials] == ["t", "t"]
@@ -207,7 +210,7 @@ def test_cache_is_attempt_and_success_aware(tmp_path: Path) -> None:
 
     job_dir = tmp_path / "job"
     job_dir.mkdir()
-    tasks = [AgentEvalTask(id="t", intent="x", inputs={"prompt": "p"}, metrics=[HarborRewardMetric()])]
+    tasks = [AgentEvalTask(id="t", intent="x", inputs={"instruction": "p"}, metrics=[HarborRewardMetric()])]
 
     # One completed attempt: enough for n_attempts=1, not for n_attempts=2.
     _write_trial(job_dir, "t__aaa", "t", reward=1.0)

packages/nemo_evaluator_sdk/tests/agent_eval/test_tasks.py

@@ -51,7 +51,7 @@ def test_task_serializes_metric_instances_as_descriptors() -> None:
     task = AgentEvalTask(
         id="task-1",
         intent="answer the prompt",
-        inputs={"prompt": "Question?"},
+        inputs={"instruction": "Question?"},
         metrics=[_Metric()],
     )
 
@@ -68,7 +68,7 @@ def test_task_rejects_duplicate_metric_types() -> None:
         AgentEvalTask(
             id="task-1",
             intent="answer the prompt",
-            inputs={"prompt": "Question?"},
+            inputs={"instruction": "Question?"},
             metrics=[_Metric(), _Metric()],
         )
 
@@ -78,7 +78,7 @@ def test_task_validates_view_signals_against_metric_outputs() -> None:
         AgentEvalTask(
             id="task-1",
             intent="answer the prompt",
-            inputs={"prompt": "Question?"},
+            inputs={"instruction": "Question?"},
             metrics=[_Metric()],
             views={
                 "outcome_correctness": SemanticView(

packages/nemo_evaluator_sdk/tests/test_agent_inference.py

@@ -22,6 +22,7 @@
     _make_nat_agent_request,
     _parse_sse_frame,
     _persist_stream_evidence,
+    _resolve_http_agent_invocation,
     invoke_agent,
     make_agent_inference_request,
 )
@@ -241,6 +242,49 @@ def test_generic_with_trajectory_path(self):
         assert agent.trajectory_path == "$.steps"
 
 
+# ============================================================================
+# Resolution: _resolve_http_agent_invocation (generic agent payloads)
+# ============================================================================
+
+
+class TestResolveGenericAgentPayload:
+    def test_body_renders_against_task_inputs(self):
+        """A generic agent's payload comes entirely from its `body`, rendered against the task
+        inputs — no chat/completions wrapper is assumed."""
+        agent = GenericAgent(
+            url="http://agent.test/invoke",
+            name="gen",
+            format=AgentFormat.GENERIC,
+            body={"query": "{{ instruction }}", "task": "{{ task_id }}"},
+            response_path="$.answer",
+        )
+
+        invocation = _resolve_http_agent_invocation(
+            agent,
+            {"instruction": "What is the capital of France?", "task_id": "capital-france"},
+        )
+
+        assert invocation.payload == {"query": "What is the capital of France?", "task": "capital-france"}
+        assert invocation.endpoint == "http://agent.test/invoke"
+
+    def test_body_can_reference_arbitrary_input_fields(self):
+        """Any field in the task inputs is available to the body template, not just a fixed set."""
+        agent = GenericAgent(
+            url="http://agent.test/invoke",
+            name="gen",
+            format=AgentFormat.GENERIC,
+            body={"prompt": "{{ instruction }}", "context": "{{ domain }}"},
+            response_path="$.answer",
+        )
+
+        invocation = _resolve_http_agent_invocation(
+            agent,
+            {"instruction": "Summarize the filing.", "domain": "finance"},
+        )
+
+        assert invocation.payload == {"prompt": "Summarize the filing.", "context": "finance"}
+
+
 # ============================================================================
 # Helper: _derive_input_message
 # ============================================================================