diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py index f47e160c24..cf67fd4621 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py @@ -48,7 +48,15 @@ from nemo_evaluator_sdk.inference import InferenceFn from nemo_evaluator_sdk.metrics.protocol import Metric, validate_metric_result from nemo_evaluator_sdk.metrics.utils import metric_type_name -from nemo_evaluator_sdk.values import Agent, AgentBase, Model, RunConfig, RunConfigOnline, RunConfigOnlineModel +from nemo_evaluator_sdk.values import ( + Agent, + AgentBase, + GenericAgent, + Model, + RunConfig, + RunConfigOnline, + RunConfigOnlineModel, +) from nemo_evaluator_sdk.values.evidence import ( EVIDENCE_FORMAT_JSON, EVIDENCE_TRACE, @@ -621,20 +629,25 @@ def _resolve_live_params( return RunConfigOnline(**params.model_dump(mode="python")) -def _default_prompt_template(target: Model | Agent) -> dict[str, Any]: +def _default_prompt_template(target: Model | Agent) -> dict[str, Any] | str: + # Every default renders against the single canonical task input, ``instruction`` (see + # ``AgentEvalTask.agent_prompt``); no other input key is special. + if isinstance(target, GenericAgent): + # A generic HTTP agent defines its own request entirely through its `body` template, which + # renders against the task inputs (e.g. `{{ instruction }}`). Pass the task row through + # unchanged so `body` — not a chat/completions assumption — shapes the payload. See + # `_resolve_http_agent_invocation`, which renders `body` against this request. + return "{{ item }}" if isinstance(target, Model) and _is_completions_endpoint(target.url): - return {"prompt": "{{item.prompt}}"} - return {"messages": [{"role": "user", "content": "{{item.prompt}}"}]} + return {"prompt": "{{item.instruction}}"} + return {"messages": [{"role": "user", "content": "{{item.instruction}}"}]} def _task_row(task: AgentEvalTask) -> dict[str, Any]: - # `prompt` is what the target under evaluation is prompted with (via the `{{item.prompt}}` - # template): a dataset `prompt` column or the agent `instruction`. - return { - **task.inputs, - "task_id": task.id, - "prompt": task.inputs.get("prompt") or task.inputs.get("instruction"), - } + # The task inputs verbatim, plus the task id. `instruction` is the single canonical input the + # target is prompted with (see `AgentEvalTask.agent_prompt` and `_default_prompt_template`); no + # input key is synthesized or aliased here. + return {**task.inputs, "task_id": task.id} def _metric_row(task: AgentEvalTask, trial: AgentEvalTrial) -> dict[str, Any]: diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py index 4f867a5645..cfd3b3da5a 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py @@ -606,14 +606,19 @@ def discover_harbor_tasks(dataset_path: str | Path) -> list[AgentEvalTask]: task_name = config.get("task", {}).get("name", task_dir.name) instruction_path = task_dir / "instruction.md" try: - intent = instruction_path.read_text(encoding="utf-8").strip() if instruction_path.is_file() else task_name + instruction = ( + instruction_path.read_text(encoding="utf-8").strip() if instruction_path.is_file() else task_name + ) except (OSError, UnicodeDecodeError) as exc: raise ValueError(f"unreadable Harbor instruction at {instruction_path}: {exc}") from exc tasks.append( AgentEvalTask( id=task_name, - intent=intent, - inputs={"instruction": intent}, + # `intent` is human-facing metadata, never shown to the agent; the task name is the + # only human label Harbor's task.toml provides. The instruction the agent acts on + # lives in `inputs["instruction"]`. + intent=task_name, + inputs={"instruction": instruction}, metrics=[HarborRewardMetric()], metadata={"harbor_dataset_path": str(dataset_path), "harbor_task_dir": str(task_dir)}, ) diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_callable_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_callable_runtime.py index 8efe6d20e2..5641c0eb4b 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_callable_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_callable_runtime.py @@ -11,7 +11,7 @@ def _task(task_id: str) -> AgentEvalTask: - return AgentEvalTask(id=task_id, intent="Answer.", inputs={"prompt": f"q-{task_id}"}) + return AgentEvalTask(id=task_id, intent="Answer.", inputs={"instruction": f"q-{task_id}"}) @pytest.mark.asyncio diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py index ccafc86d5b..2f7a437634 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py @@ -41,7 +41,7 @@ def test_trial_from_sample_falls_back_to_reasoning_content() -> None: - task = AgentEvalTask(id="task-1", intent="Answer.", inputs={"prompt": "Q?"}) + task = AgentEvalTask(id="task-1", intent="Answer.", inputs={"instruction": "Q?"}) target = Model(name="reasoning-model", url="https://example/v1/chat/completions") response = {"choices": [{"message": {"content": None, "reasoning_content": "the reasoned answer"}}]} @@ -56,7 +56,7 @@ def test_trial_from_sample_falls_back_to_reasoning_content() -> None: def test_trial_from_sample_fallback_trace_has_trace_kind() -> None: - task = AgentEvalTask(id="task-1", intent="Answer.", inputs={"prompt": "Q?"}) + task = AgentEvalTask(id="task-1", intent="Answer.", inputs={"instruction": "Q?"}) target = Model(name="target", url="https://example/v1/chat/completions") trial = _trial_from_sample(task, target, {"output_text": "answer"}) @@ -67,7 +67,7 @@ def test_trial_from_sample_fallback_trace_has_trace_kind() -> None: def test_trial_from_sample_preserves_typed_trace_and_canonical_metadata() -> None: - task = AgentEvalTask(id="task-1", intent="Answer.", inputs={"prompt": "Q?"}) + task = AgentEvalTask(id="task-1", intent="Answer.", inputs={"instruction": "Q?"}) target = Model(name="canonical-target", url="https://example/v1/chat/completions") typed_trace = { "schema_version": "ATIF-v1.7", @@ -200,7 +200,7 @@ def _task(metric: Any | None = None, *, task_id: str = "task-1") -> AgentEvalTas return AgentEvalTask( id=task_id, intent="Answer a professional benchmark prompt.", - inputs={"prompt": "What is the answer?", "domain": "Finance MBA"}, + inputs={"instruction": "What is the answer?", "domain": "Finance MBA"}, metrics=[metric or _ConstantMetric()], metadata={"benchmark": "Example", "domain": "Finance MBA"}, ) @@ -363,7 +363,9 @@ async def fake_model_inference( @pytest.mark.asyncio -async def test_live_model_generation_uses_instruction_when_prompt_is_absent() -> None: +async def test_live_model_generation_completions_endpoint_prompts_with_instruction() -> None: + # A bare /v1/completions endpoint uses the `{"prompt": ...}` request shape; the task instruction + # is rendered into that wire field (there is no chat `messages` wrapper). async def fake_model_inference( model: Model, request: dict[str, Any], @@ -371,16 +373,17 @@ async def fake_model_inference( **kwargs: Any, ) -> dict[str, Any]: del model, max_retries, kwargs - assert request["messages"][0]["content"] == "Use the task instruction." - return {"choices": [{"message": {"role": "assistant", "content": "Generated model answer"}}]} + assert request["prompt"] == "Use the task instruction." + assert "messages" not in request + return {"choices": [{"text": "Generated model answer"}]} task = AgentEvalTask( id="task-1", - intent="Fallback intent.", + intent="Answer the instruction.", inputs={"instruction": "Use the task instruction."}, metrics=[_ConstantMetric()], ) - model = Model(url="https://model.test/v1/chat/completions", name="target-model", format=ModelFormat.OPEN_AI) + model = Model(url="https://model.test/v1/completions", name="target-model", format=ModelFormat.OPEN_AI) await AgentEvaluator(inference_fn=fake_model_inference).run( tasks=[task], @@ -395,7 +398,7 @@ async def test_metric_failure_records_failed_score_and_does_not_stop_other_metri other_task = AgentEvalTask( id="task-2", intent="Answer another prompt.", - inputs={"prompt": "Another question?"}, + inputs={"instruction": "Another question?"}, metrics=[_OtherMetric()], ) trials = [ @@ -435,7 +438,7 @@ def test_summary_reports_coverage_and_merges_views_into_scores() -> None: task = AgentEvalTask( id="task-1", intent="Answer a prompt.", - inputs={"prompt": "Question?"}, + inputs={"instruction": "Question?"}, metrics=[_ConstantMetric(), _OtherMetric()], views={ "outcome_correctness": SemanticView( @@ -471,7 +474,10 @@ async def fake_agent_inference( **kwargs: Any, ) -> dict[str, Any]: del agent, kwargs - assert request["messages"][0]["content"] == "What is the answer?" + # A generic HTTP agent receives the task row directly — no chat/completions wrapper — so its + # `body` template can reference task inputs such as `{{ instruction }}`. + assert "messages" not in request + assert request["instruction"] == "What is the answer?" return { "choices": [{"message": {"role": "assistant", "content": "Generated agent answer"}}], "trajectory": [{"tool": "search", "line": 3}], @@ -481,7 +487,7 @@ async def fake_agent_inference( url="https://agent.test", name="target-agent", format=AgentFormat.GENERIC, - body={"input": "{{ messages[-1].content }}"}, + body={"input": "{{ instruction }}"}, response_path="$.answer", ) result = await AgentEvaluator(inference_fn=fake_agent_inference).run( @@ -627,7 +633,7 @@ async def test_default_agent_invocation_receives_run_context_and_evidence_dir(tm ) agent = NemoAgentToolkitAgent(url="https://agent.test", name="target-agent", format=AgentFormat.NEMO_AGENT_TOOLKIT) prompt_template = { - "input_message": "{{ item.prompt }}", + "input_message": "{{ item.instruction }}", "conversation_id": ("{{ agent_eval.run_id }}-{{ agent_eval.task_id }}-{{ agent_eval.invocation_id }}"), } @@ -794,7 +800,7 @@ async def test_run_rejects_tasks_without_trials() -> None: other_task = AgentEvalTask( id="task-2", intent="Answer another prompt.", - inputs={"prompt": "Another question?"}, + inputs={"instruction": "Another question?"}, metrics=[_OtherMetric()], ) diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py index a7423711f0..b87917b2ef 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py @@ -57,9 +57,9 @@ async def test_harbor_runner_scores_through_agent_evaluator_and_adapts_legacy_pa _write_trial(job_dir, "noreward-task__ccc", "noreward-task", reward=None) tasks = [ - AgentEvalTask(id="pass-task", intent="x", inputs={"prompt": "p"}, metrics=[HarborRewardMetric()]), - AgentEvalTask(id="fail-task", intent="y", inputs={"prompt": "q"}, metrics=[HarborRewardMetric()]), - AgentEvalTask(id="noreward-task", intent="z", inputs={"prompt": "r"}, metrics=[HarborRewardMetric()]), + AgentEvalTask(id="pass-task", intent="x", inputs={"instruction": "p"}, metrics=[HarborRewardMetric()]), + AgentEvalTask(id="fail-task", intent="y", inputs={"instruction": "q"}, metrics=[HarborRewardMetric()]), + AgentEvalTask(id="noreward-task", intent="z", inputs={"instruction": "r"}, metrics=[HarborRewardMetric()]), ] # Direct adaptation: reward + tokens land on metadata, exception flips status to PARTIAL, evidence present. @@ -100,7 +100,7 @@ def test_reward_with_no_matching_reward_key_is_partial_and_warns(tmp_path: Path, job_dir = tmp_path / "job" job_dir.mkdir() _write_trial(job_dir, "t__aaa", "t", reward=1.0) # emitted under "reward" - tasks = [AgentEvalTask(id="t", intent="x", inputs={"prompt": "p"}, metrics=[HarborRewardMetric()])] + tasks = [AgentEvalTask(id="t", intent="x", inputs={"instruction": "p"}, metrics=[HarborRewardMetric()])] with caplog.at_level(logging.WARNING): trials = build_trials_from_job_dir(job_dir, tasks, reward_key="missing") @@ -116,7 +116,10 @@ def test_task_discovery_and_taskset_loader_over_bundled_dataset() -> None: tasks = discover_harbor_tasks(_HELLO_WORLD_DATASET) assert [task.id for task in tasks] == ["harbor/hello-world"] task = tasks[0] - assert task.intent # instruction.md content + # `intent` is the human-facing task name (metadata), NOT the instruction; the instruction the + # agent acts on comes from instruction.md and lives in inputs["instruction"]. + assert task.intent == "harbor/hello-world" + assert task.inputs["instruction"] == 'Create a file called hello.txt with "Hello, world!" as the content.' assert [metric_type_name(metric) for metric in task.metrics] == ["harbor_reward"] # The dataset dir and task dir are stamped on the task so a native runner can # recover them without a separate dataset_path argument. @@ -195,7 +198,7 @@ def test_multiple_attempts_map_to_one_trial_each(tmp_path: Path) -> None: job_dir.mkdir() _write_trial(job_dir, "t__aaa", "t", reward=1.0) _write_trial(job_dir, "t__bbb", "t", reward=0.0) - tasks = [AgentEvalTask(id="t", intent="x", inputs={"prompt": "p"}, metrics=[HarborRewardMetric()])] + tasks = [AgentEvalTask(id="t", intent="x", inputs={"instruction": "p"}, metrics=[HarborRewardMetric()])] trials = build_trials_from_job_dir(job_dir, tasks) assert [trial.task_id for trial in trials] == ["t", "t"] @@ -207,7 +210,7 @@ def test_cache_is_attempt_and_success_aware(tmp_path: Path) -> None: job_dir = tmp_path / "job" job_dir.mkdir() - tasks = [AgentEvalTask(id="t", intent="x", inputs={"prompt": "p"}, metrics=[HarborRewardMetric()])] + tasks = [AgentEvalTask(id="t", intent="x", inputs={"instruction": "p"}, metrics=[HarborRewardMetric()])] # One completed attempt: enough for n_attempts=1, not for n_attempts=2. _write_trial(job_dir, "t__aaa", "t", reward=1.0) diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_tasks.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_tasks.py index e72c4ea582..2dcc31eb5b 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_tasks.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_tasks.py @@ -51,7 +51,7 @@ def test_task_serializes_metric_instances_as_descriptors() -> None: task = AgentEvalTask( id="task-1", intent="answer the prompt", - inputs={"prompt": "Question?"}, + inputs={"instruction": "Question?"}, metrics=[_Metric()], ) @@ -68,7 +68,7 @@ def test_task_rejects_duplicate_metric_types() -> None: AgentEvalTask( id="task-1", intent="answer the prompt", - inputs={"prompt": "Question?"}, + inputs={"instruction": "Question?"}, metrics=[_Metric(), _Metric()], ) @@ -78,7 +78,7 @@ def test_task_validates_view_signals_against_metric_outputs() -> None: AgentEvalTask( id="task-1", intent="answer the prompt", - inputs={"prompt": "Question?"}, + inputs={"instruction": "Question?"}, metrics=[_Metric()], views={ "outcome_correctness": SemanticView( diff --git a/packages/nemo_evaluator_sdk/tests/test_agent_inference.py b/packages/nemo_evaluator_sdk/tests/test_agent_inference.py index 1928dc0306..654f85edf7 100644 --- a/packages/nemo_evaluator_sdk/tests/test_agent_inference.py +++ b/packages/nemo_evaluator_sdk/tests/test_agent_inference.py @@ -22,6 +22,7 @@ _make_nat_agent_request, _parse_sse_frame, _persist_stream_evidence, + _resolve_http_agent_invocation, invoke_agent, make_agent_inference_request, ) @@ -241,6 +242,49 @@ def test_generic_with_trajectory_path(self): assert agent.trajectory_path == "$.steps" +# ============================================================================ +# Resolution: _resolve_http_agent_invocation (generic agent payloads) +# ============================================================================ + + +class TestResolveGenericAgentPayload: + def test_body_renders_against_task_inputs(self): + """A generic agent's payload comes entirely from its `body`, rendered against the task + inputs — no chat/completions wrapper is assumed.""" + agent = GenericAgent( + url="http://agent.test/invoke", + name="gen", + format=AgentFormat.GENERIC, + body={"query": "{{ instruction }}", "task": "{{ task_id }}"}, + response_path="$.answer", + ) + + invocation = _resolve_http_agent_invocation( + agent, + {"instruction": "What is the capital of France?", "task_id": "capital-france"}, + ) + + assert invocation.payload == {"query": "What is the capital of France?", "task": "capital-france"} + assert invocation.endpoint == "http://agent.test/invoke" + + def test_body_can_reference_arbitrary_input_fields(self): + """Any field in the task inputs is available to the body template, not just a fixed set.""" + agent = GenericAgent( + url="http://agent.test/invoke", + name="gen", + format=AgentFormat.GENERIC, + body={"prompt": "{{ instruction }}", "context": "{{ domain }}"}, + response_path="$.answer", + ) + + invocation = _resolve_http_agent_invocation( + agent, + {"instruction": "Summarize the filing.", "domain": "finance"}, + ) + + assert invocation.payload == {"prompt": "Summarize the filing.", "context": "finance"} + + # ============================================================================ # Helper: _derive_input_message # ============================================================================ diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py index c86b06fdb9..ae9e88eeec 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py @@ -48,7 +48,15 @@ from nemo_platform.beta.evaluator.inference import InferenceFn from nemo_platform.beta.evaluator.metrics.protocol import Metric, validate_metric_result from nemo_platform.beta.evaluator.metrics.utils import metric_type_name -from nemo_platform.beta.evaluator.values import Agent, AgentBase, Model, RunConfig, RunConfigOnline, RunConfigOnlineModel +from nemo_platform.beta.evaluator.values import ( + Agent, + AgentBase, + GenericAgent, + Model, + RunConfig, + RunConfigOnline, + RunConfigOnlineModel, +) from nemo_platform.beta.evaluator.values.evidence import ( EVIDENCE_FORMAT_JSON, EVIDENCE_TRACE, @@ -621,20 +629,25 @@ def _resolve_live_params( return RunConfigOnline(**params.model_dump(mode="python")) -def _default_prompt_template(target: Model | Agent) -> dict[str, Any]: +def _default_prompt_template(target: Model | Agent) -> dict[str, Any] | str: + # Every default renders against the single canonical task input, ``instruction`` (see + # ``AgentEvalTask.agent_prompt``); no other input key is special. + if isinstance(target, GenericAgent): + # A generic HTTP agent defines its own request entirely through its `body` template, which + # renders against the task inputs (e.g. `{{ instruction }}`). Pass the task row through + # unchanged so `body` — not a chat/completions assumption — shapes the payload. See + # `_resolve_http_agent_invocation`, which renders `body` against this request. + return "{{ item }}" if isinstance(target, Model) and _is_completions_endpoint(target.url): - return {"prompt": "{{item.prompt}}"} - return {"messages": [{"role": "user", "content": "{{item.prompt}}"}]} + return {"prompt": "{{item.instruction}}"} + return {"messages": [{"role": "user", "content": "{{item.instruction}}"}]} def _task_row(task: AgentEvalTask) -> dict[str, Any]: - # `prompt` is what the target under evaluation is prompted with (via the `{{item.prompt}}` - # template): a dataset `prompt` column or the agent `instruction`. - return { - **task.inputs, - "task_id": task.id, - "prompt": task.inputs.get("prompt") or task.inputs.get("instruction"), - } + # The task inputs verbatim, plus the task id. `instruction` is the single canonical input the + # target is prompted with (see `AgentEvalTask.agent_prompt` and `_default_prompt_template`); no + # input key is synthesized or aliased here. + return {**task.inputs, "task_id": task.id} def _metric_row(task: AgentEvalTask, trial: AgentEvalTrial) -> dict[str, Any]: diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py index c25e031a92..e04dd9b35d 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py @@ -606,14 +606,19 @@ def discover_harbor_tasks(dataset_path: str | Path) -> list[AgentEvalTask]: task_name = config.get("task", {}).get("name", task_dir.name) instruction_path = task_dir / "instruction.md" try: - intent = instruction_path.read_text(encoding="utf-8").strip() if instruction_path.is_file() else task_name + instruction = ( + instruction_path.read_text(encoding="utf-8").strip() if instruction_path.is_file() else task_name + ) except (OSError, UnicodeDecodeError) as exc: raise ValueError(f"unreadable Harbor instruction at {instruction_path}: {exc}") from exc tasks.append( AgentEvalTask( id=task_name, - intent=intent, - inputs={"instruction": intent}, + # `intent` is human-facing metadata, never shown to the agent; the task name is the + # only human label Harbor's task.toml provides. The instruction the agent acts on + # lives in `inputs["instruction"]`. + intent=task_name, + inputs={"instruction": instruction}, metrics=[HarborRewardMetric()], metadata={"harbor_dataset_path": str(dataset_path), "harbor_task_dir": str(task_dir)}, )