feat(evaluator): agent-evaluation plugin job (run_local + submit)

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py

@@ -15,7 +15,7 @@
 from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask
 from nemo_evaluator_sdk.values import Agent, Model
 from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor
-from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator
 
 # Well-known evidence keys produced by ``standard_evidence_descriptors``. Harness
 # code may import these to tag evidence consistently; callers may still add
@@ -44,9 +44,10 @@ class AgentOutput(BaseModel):
         default=None,
         description="User-visible final text produced by the agent, if any.",
     )
-    response: Any | None = Field(
+    response: JsonValue | None = Field(
         default=None,
-        description="Structured final response payload produced by the agent, if any.",
+        description="Final response payload produced by the agent, if any. Any JSON value — a "
+        "structured object, or a raw JSON string/array for agents that don't return an object.",
     )
     metadata: dict[str, Any] = Field(
         default_factory=dict,

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/agents.py

@@ -29,7 +29,21 @@ class Agent(BaseModel):
 
     # TODO: Much of this is duplicated between agent and model. Once we have aligned on model defination.
     # the duplication can be removed by defining EndPoint class and reusing it across both model and agent.
-    model_config = ConfigDict(extra="forbid")
+    #
+    # ``allOf``/``if``/``then`` mirrors the ``_validate_generic_fields`` validator into the OpenAPI
+    # schema: a generic-format agent must carry ``body`` + ``response_path`` (the generic HTTP path
+    # needs them), so the contract rejects a ``url``-only generic agent rather than only failing later.
+    model_config = ConfigDict(
+        extra="forbid",
+        json_schema_extra={
+            "allOf": [
+                {
+                    "if": {"properties": {"format": {"const": "generic"}}},
+                    "then": {"required": ["body", "response_path"]},
+                }
+            ]
+        },
+    )
 
     url: str = Field(description="Base URL of the agent endpoint.")
     name: str = Field(description="Agent name / identifier.")

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/evidence.py

@@ -16,7 +16,7 @@
 from typing import Any, Literal
 from urllib.parse import urlparse
 
-from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, PrivateAttr, model_validator
 
 
 class FilesystemEntry(BaseModel):
@@ -293,7 +293,12 @@ async def _exec(command: list[str], cwd: Path, timeout_s: float | None) -> Comma
 class EvidenceDescriptor(BaseModel):
     """Descriptor for a candidate trace, source, or artifact."""
 
-    model_config = ConfigDict(extra="forbid")
+    # ``anyOf`` mirrors the ``_requires_ref_or_data`` validator into the OpenAPI schema, so a payload
+    # with neither ``ref`` nor ``data`` is rejected by the contract, not just at runtime.
+    model_config = ConfigDict(
+        extra="forbid",
+        json_schema_extra={"anyOf": [{"required": ["ref"]}, {"required": ["data"]}]},
+    )
 
     kind: str = Field(description="Evidence type, e.g. 'filesystem', 'trace', 'log_bundle', or 'review'.")
     ref: str | None = Field(
@@ -304,7 +309,7 @@ class EvidenceDescriptor(BaseModel):
         default=None,
         description="Parser hint for the evidence payload, e.g. 'atif' for normalized traces.",
     )
-    data: Any | None = Field(
+    data: JsonValue | None = Field(
         default=None,
         description="Small inline evidence payload; at least one of ref or data must be set.",
     )

packages/nemo_platform/pyproject.toml

@@ -515,6 +515,7 @@ nemo-switchyard = "nemo_switchyard.middleware:SwitchyardMiddleware"
 "auditor.audit" = "nemo_auditor.jobs.audit:AuditJob"
 "data-designer.create" = "nemo_data_designer_plugin.jobs.create:CreateJob"
 "evaluator.evaluate" = "nemo_evaluator.jobs.evaluate:EvaluateJob"
+"evaluator.agent-evaluate" = "nemo_evaluator.jobs.agent_evaluate:AgentEvalJob"
 
 # Generated from [tool.bundle-package]; do not edit this table by hand.
 [project.entry-points."nemo.sdk"]

packages/nmp_testing/src/nmp/testing/__init__.py

@@ -77,6 +77,7 @@
     assert_exit_0,
     get_repo_root,
     grant_workspace_role,
+    igw_mock_provider_mode,
     run_nemo_local,
     short_unique_name,
     unique_email,
@@ -102,6 +103,7 @@
     "as_user",
     "grant_workspace_role",
     "add_mock_provider",
+    "igw_mock_provider_mode",
     "MockProviderResponse",
     "wait_for_model_entity",
     # Task testing

packages/nmp_testing/src/nmp/testing/e2e/jobs.py

@@ -12,6 +12,7 @@
 from collections.abc import Callable
 
 from nemo_platform import NeMoPlatform
+from nemo_platform.types.jobs.platform_job_response import PlatformJobResponse
 
 logger = logging.getLogger(__name__)
 
@@ -89,7 +90,7 @@ def wait_for_platform_job(
     image_pull_timeout: float = 600.0,
     poll_interval: float = 1.0,
     status_to_check: str = "",
-):
+) -> PlatformJobResponse:
     """Wait for a platform job to reach a terminal state.
 
     Uses the SDK's jobs API to poll for job status until it reaches
@@ -148,6 +149,8 @@ def get_status() -> str:
             error_parts.append(f"Failed to get job status: {detail_err}")
         raise TimeoutError("\n".join(error_parts)) from e
 
+    # poll_until_terminal calls get_status (which sets last_job) at least once before returning.
+    assert last_job is not None
     return last_job
 
 
@@ -218,6 +221,8 @@ def get_status() -> str:
             error_parts.append(f"Full job details: {job_response.json()}")
         raise TimeoutError("\n".join(error_parts)) from e
 
+    # poll_until_terminal calls get_status (which sets last_status) at least once before returning.
+    assert last_status is not None
     return last_status
 
 

packages/nmp_testing/src/nmp/testing/utils.py

@@ -10,7 +10,8 @@
 import tempfile
 import time
 import uuid
-from collections.abc import Callable, Sequence
+from collections.abc import Callable, Iterator, Sequence
+from contextlib import contextmanager
 from dataclasses import dataclass, field
 from datetime import datetime
 from pathlib import Path
@@ -364,6 +365,32 @@ def grant_workspace_role(
     )
 
 
+@contextmanager
+def igw_mock_provider_mode(prefix: str = "igw-mock-") -> Iterator[None]:
+    """Enable IGW mock-provider mode in *this* process for the duration of the ``with`` block.
+
+    Overrides ``InferenceGatewayConfig.mock_provider_prefix`` so :func:`add_mock_provider` can name
+    and resolve mock providers. Use this when driving a *real* platform subprocess (where
+    :func:`create_test_client` isn't in play) but the in-process config still has to recognize the
+    mock prefix.
+
+    Implemented as a ``Configuration`` override, which ``get_service_config`` honors ahead of the
+    env-derived config — so it works regardless of whether the IGW config was already read/cached
+    (an env var would lose that race) — and is scoped to the block, so it can't leak into unrelated
+    suites. The prior ``InferenceGatewayConfig`` override (if any) is restored on exit.
+    """
+    from nmp.common.config import Configuration
+    from nmp.core.inference_gateway.config import InferenceGatewayConfig
+
+    current = Configuration.get_service_config(InferenceGatewayConfig)
+    merged = InferenceGatewayConfig(**{**current.model_dump(), "mock_provider_prefix": prefix})
+    Configuration.set_overrides({InferenceGatewayConfig: merged})
+    try:
+        yield
+    finally:
+        Configuration.clear_override(InferenceGatewayConfig)
+
+
 def add_mock_provider(
     sdk: NeMoPlatform,
     *,