From cfc423d0c5fee68e05620482a6e5772923354b98 Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Tue, 30 Jun 2026 11:06:51 -0300 Subject: [PATCH 1/4] feat(evaluator): persist eval results as queryable entities Re-add queryable persistence of eval-job results (removed with the legacy service in #231), now on the plugin-job API. - AgentEvalResultEntity / EvaluateResultEntity store aggregated scores plus filterable target/dataset traits; the full bundle stays in the run's fileset (bundle_ref). Jobs persist best-effort in run() (a store error never fails the eval). - AgentEvalResult / EvaluateResult API DTOs (mapped from the entities) back the read routes, so id/created_at round-trip cleanly on the wire and in the SDK. - /agent-eval-results and /eval-results list/get/delete routes with trait filtering: a DataFilter base translates custom fields to data.* (MetricFilter adopts it too, fixing metric_type filtering). - client.evaluator.{agent_eval_results,eval_results} SDK resources, plus metric-type filtering on client.evaluator.metrics.list. - get_async_task_sdk: async counterpart of get_task_sdk so a sync job run() can drive the async entity-store write with the full on-behalf-of identity. Verified end-to-end on a live platform (agent-eval submit under auth, row-eval submit, metric-type filtering) plus unit coverage. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Sandy Chapman --- .../src/nemo_platform_plugin/sdk_provider.py | 47 ++ .../tests/test_sdk_provider.py | 58 +- .../nmp_common/src/nmp/common/sdk_factory.py | 31 + plugins/nemo-evaluator/openapi/openapi.yaml | 797 ++++++++++++++++++ .../src/nemo_evaluator/api/dependencies.py | 8 + .../src/nemo_evaluator/api/schemas.py | 78 +- .../api/service/result_service.py | 159 ++++ .../src/nemo_evaluator/api/v2/results.py | 230 +++++ .../src/nemo_evaluator/entities.py | 82 +- .../src/nemo_evaluator/jobs/agent_evaluate.py | 12 +- .../src/nemo_evaluator/jobs/evaluate.py | 13 + .../nemo_evaluator/jobs/result_persistence.py | 136 +++ .../nemo_evaluator/sdk/metric_resources.py | 38 +- .../src/nemo_evaluator/sdk/resources.py | 10 + .../nemo_evaluator/sdk/result_resources.py | 273 ++++++ .../src/nemo_evaluator/service.py | 15 + .../src/nemo_evaluator/tasks/runner.py | 9 +- .../tests/api/service/test_result_service.py | 156 ++++ .../tests/api/v2/test_metrics_routes.py | 22 + .../tests/api/v2/test_results_routes.py | 183 ++++ .../integration/test_agent_evaluate_job.py | 18 + .../tests/integration/test_evaluate_job.py | 95 +++ .../integration/test_metric_filtering.py | 62 ++ .../tests/sdk/test_metric_sdk_resources.py | 14 + .../tests/sdk/test_result_sdk_resources.py | 183 ++++ .../tests/test_agent_evaluate.py | 5 +- .../nemo-evaluator/tests/test_evaluate_job.py | 5 +- .../tests/test_result_entity.py | 96 +++ .../tests/test_result_persistence.py | 204 +++++ 29 files changed, 3023 insertions(+), 16 deletions(-) create mode 100644 plugins/nemo-evaluator/src/nemo_evaluator/api/service/result_service.py create mode 100644 plugins/nemo-evaluator/src/nemo_evaluator/api/v2/results.py create mode 100644 plugins/nemo-evaluator/src/nemo_evaluator/jobs/result_persistence.py create mode 100644 plugins/nemo-evaluator/src/nemo_evaluator/sdk/result_resources.py create mode 100644 plugins/nemo-evaluator/tests/api/service/test_result_service.py create mode 100644 plugins/nemo-evaluator/tests/api/v2/test_results_routes.py create mode 100644 plugins/nemo-evaluator/tests/integration/test_evaluate_job.py create mode 100644 plugins/nemo-evaluator/tests/integration/test_metric_filtering.py create mode 100644 plugins/nemo-evaluator/tests/sdk/test_result_sdk_resources.py create mode 100644 plugins/nemo-evaluator/tests/test_result_entity.py create mode 100644 plugins/nemo-evaluator/tests/test_result_persistence.py diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/sdk_provider.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/sdk_provider.py index a7287a4ef1..b93bb909b4 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/sdk_provider.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/sdk_provider.py @@ -72,6 +72,15 @@ def get_task_sdk(self, service_name: str) -> NeMoPlatform: when ``NMP_PRINCIPAL`` is set, acts on behalf of the job creator. """ + def get_async_task_sdk(self, service_name: str) -> AsyncNeMoPlatform: + """Async counterpart of :meth:`get_task_sdk` for use inside a task container. + + Authenticates as ``service:{service_name}`` and, when ``NMP_PRINCIPAL`` + is set, acts on behalf of the job creator with the *full* delegated + identity (on-behalf-of id, email, and groups) — wire-identical to + :meth:`get_task_sdk`, just async. + """ + def get_platform_sdk( self, *, @@ -177,6 +186,29 @@ def get_task_sdk(self, service_name: str) -> NeMoPlatform: default_headers=headers, ) + def get_async_task_sdk(self, service_name: str) -> AsyncNeMoPlatform: + # Async mirror of get_task_sdk: identical headers (service principal, + # internal marker, and full on-behalf-of id/email/groups), async client. + headers: dict[str, str] = { + "X-NMP-Principal-Id": f"service:{service_name}", + _INTERNAL_REQUEST_HEADER: "true", + } + + principal = _read_principal_from_env() + if principal is not None: + headers.update(_on_behalf_of_headers(principal)) + else: + logger.warning( + "%s not set; async task SDK will authenticate as service:%s without on-behalf-of delegation", + _NMP_PRINCIPAL_ENVVAR, + service_name, + ) + + return AsyncNeMoPlatform( + base_url=self._base_url(), + default_headers=headers, + ) + def _make_sdk( self, cls: type[_SDKT], @@ -311,6 +343,21 @@ def get_task_sdk(service_name: str) -> NeMoPlatform: return _resolve_provider().get_task_sdk(service_name) +def get_async_task_sdk(service_name: str) -> AsyncNeMoPlatform: + """Async counterpart of :func:`get_task_sdk` for use inside a task container. + + For a (synchronous) job ``run`` that needs to drive an async helper — e.g. an entity-store + write — without fabricating its own client. Authenticates as ``service:{service_name}`` and, when + ``NMP_PRINCIPAL`` is set, on behalf of the job creator with the full delegated identity + (on-behalf-of id, email, and groups) — wire-identical to :func:`get_task_sdk`. + + A dedicated provider method (not a wrapper over :func:`get_async_platform_sdk`) so each provider + mirrors its own sync :meth:`SDKProvider.get_task_sdk` exactly; the platform provider routes URLs + and reuses its shared async client, the default provider uses env-var headers. + """ + return _resolve_provider().get_async_task_sdk(service_name) + + def get_platform_sdk( *, as_service: str | None = None, diff --git a/packages/nemo_platform_plugin/tests/test_sdk_provider.py b/packages/nemo_platform_plugin/tests/test_sdk_provider.py index 3bfb44a01e..5dc461454f 100644 --- a/packages/nemo_platform_plugin/tests/test_sdk_provider.py +++ b/packages/nemo_platform_plugin/tests/test_sdk_provider.py @@ -9,7 +9,7 @@ from unittest.mock import patch import pytest -from nemo_platform import NeMoPlatform +from nemo_platform import AsyncNeMoPlatform, NeMoPlatform from nemo_platform_plugin.sdk_provider import ( DefaultSDKProvider, SDKProvider, @@ -19,6 +19,11 @@ set_sdk_provider, ) + +def _xnmp(sdk) -> dict[str, str]: + return {k: v for k, v in sdk.default_headers.items() if k.startswith("X-NMP-")} + + # --------------------------------------------------------------------------- # _read_principal_from_env # --------------------------------------------------------------------------- @@ -142,6 +147,57 @@ def test_get_platform_sdk_on_behalf_of(self, monkeypatch): assert sdk.default_headers["X-NMP-Principal-On-Behalf-Of"] == "user@ex.com" +# --------------------------------------------------------------------------- +# get_async_task_sdk — the async sibling of get_task_sdk +# --------------------------------------------------------------------------- + + +class TestAsyncTaskSdk: + def test_async_task_sdk_with_principal(self, monkeypatch): + monkeypatch.setenv("NMP_BASE_URL", "http://test:9090") + monkeypatch.setenv( + "NMP_PRINCIPAL", + json.dumps({"id": "creator@ex.com", "email": "creator@ex.com", "groups": ["team"]}), + ) + + sdk = DefaultSDKProvider().get_async_task_sdk("evaluator") + + assert isinstance(sdk, AsyncNeMoPlatform) + assert sdk.base_url == "http://test:9090" + assert sdk.default_headers["X-NMP-Principal-Id"] == "service:evaluator" + assert sdk.default_headers["X-NMP-Internal"] == "true" + assert sdk.default_headers["X-NMP-Principal-On-Behalf-Of"] == "creator@ex.com" + + def test_async_task_sdk_without_principal(self, monkeypatch): + monkeypatch.setenv("NMP_BASE_URL", "http://test:9090") + monkeypatch.delenv("NMP_PRINCIPAL", raising=False) + + sdk = DefaultSDKProvider().get_async_task_sdk("evaluator") + + assert sdk.default_headers["X-NMP-Principal-Id"] == "service:evaluator" + assert "X-NMP-Principal-On-Behalf-Of" not in sdk.default_headers + + def test_parity_with_sync_task_sdk(self, monkeypatch): + # Regression guard: the async task SDK must carry the *full* delegated identity (on-behalf-of + # id, email, and groups) — wire-identical to get_task_sdk. A prior implementation built on + # get_async_platform_sdk dropped the -Email/-Groups headers. + monkeypatch.setenv("NMP_BASE_URL", "http://test:9090") + monkeypatch.setenv( + "NMP_PRINCIPAL", + json.dumps( + { + "id": "service:evaluator", + "on_behalf_of": "real-user@ex.com", + "on_behalf_of_email": "real-user@ex.com", + "on_behalf_of_groups": ["admin", "team"], + } + ), + ) + + provider = DefaultSDKProvider() + assert _xnmp(provider.get_async_task_sdk("evaluator")) == _xnmp(provider.get_task_sdk("evaluator")) + + # --------------------------------------------------------------------------- # Provider resolution # --------------------------------------------------------------------------- diff --git a/packages/nmp_common/src/nmp/common/sdk_factory.py b/packages/nmp_common/src/nmp/common/sdk_factory.py index 70691095b0..0d63504039 100644 --- a/packages/nmp_common/src/nmp/common/sdk_factory.py +++ b/packages/nmp_common/src/nmp/common/sdk_factory.py @@ -191,6 +191,34 @@ def get_task_sdk(as_service: str, http_client: httpx.Client | None = None) -> Ne ) +def get_async_task_sdk(as_service: str, http_client: Optional[httpx.AsyncClient] = None) -> AsyncNeMoPlatform: + """Async counterpart of :func:`get_task_sdk` for use inside a task container. + + Reads the job creator's principal from ``NMP_PRINCIPAL`` and creates an async SDK that + authenticates as the given service while acting on behalf of the job creator with the full + delegated identity (on-behalf-of id, email, and groups). Wire-identical to :func:`get_task_sdk`. + + Args: + as_service: Service name for the service principal (e.g., "evaluator"). + http_client: Optional async HTTP client to use for requests. + + Returns: + Configured AsyncNeMoPlatform SDK with internal + on-behalf-of headers. + """ + principal = principal_from_env() + if principal is None: + logger.warning( + "NMP_PRINCIPAL not set; async task SDK will authenticate as service:%s without on-behalf-of delegation", + as_service, + ) + return get_async_platform_sdk( + as_service=as_service, + internal=True, + http_client=http_client, + on_behalf_of=principal.effective_principal if principal else None, + ) + + def get_async_platform_sdk( as_service: str | None = None, internal: bool = False, @@ -344,6 +372,9 @@ class PlatformSDKProvider: def get_task_sdk(self, service_name: str, http_client: httpx.Client | None = None) -> NeMoPlatform: return get_task_sdk(service_name, http_client=http_client) + def get_async_task_sdk(self, service_name: str, http_client: httpx.AsyncClient | None = None) -> AsyncNeMoPlatform: + return get_async_task_sdk(service_name, http_client=http_client) + def get_platform_sdk( self, *, diff --git a/plugins/nemo-evaluator/openapi/openapi.yaml b/plugins/nemo-evaluator/openapi/openapi.yaml index 0426c05e76..dd4827c27d 100644 --- a/plugins/nemo-evaluator/openapi/openapi.yaml +++ b/plugins/nemo-evaluator/openapi/openapi.yaml @@ -45,6 +45,136 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/agent-eval-results: + get: + tags: + - Evaluator Plugin Agent Eval Results Routes + summary: List Agent Eval Results By Workspace + description: List agent-evaluation result records for a workspace. + operationId: list_agent_eval_results_apis_evaluator_v2_workspaces__workspace__agent_eval_results_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + allOf: + - $ref: '#/components/schemas/ResultSort' + description: Sort field; prefix with '-' for descending. + default: -created_at + description: Sort field; prefix with '-' for descending. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/ResultFilter' + description: Filter by workspace, name, target, and timestamps. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/AgentEvalResultsPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/agent-eval-results/{name}: + get: + tags: + - Evaluator Plugin Agent Eval Results Routes + summary: Get Agent Eval Result + description: Get an agent-evaluation result record by workspace and name. + operationId: get_agent_eval_result_apis_evaluator_v2_workspaces__workspace__agent_eval_results__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/AgentEvalResult' + '404': + description: Result not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Evaluator Plugin Agent Eval Results Routes + summary: Delete Agent Eval Result + description: Delete an agent-evaluation result record by workspace and name. + operationId: delete_agent_eval_result_apis_evaluator_v2_workspaces__workspace__agent_eval_results__name__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '204': + description: Successful Response + '404': + description: Result not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/evaluator/v2/workspaces/{workspace}/agent-evaluate/jobs: post: tags: @@ -419,6 +549,136 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/eval-results: + get: + tags: + - Evaluator Plugin Eval Results Routes + summary: List Eval Results By Workspace + description: List (row) evaluation result records for a workspace. + operationId: list_eval_results_apis_evaluator_v2_workspaces__workspace__eval_results_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + allOf: + - $ref: '#/components/schemas/ResultSort' + description: Sort field; prefix with '-' for descending. + default: -created_at + description: Sort field; prefix with '-' for descending. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/EvaluateResultFilter' + description: Filter by workspace, name, target, dataset_ref, and timestamps. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateResultsPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/eval-results/{name}: + get: + tags: + - Evaluator Plugin Eval Results Routes + summary: Get Eval Result + description: Get a (row) evaluation result record by workspace and name. + operationId: get_eval_result_apis_evaluator_v2_workspaces__workspace__eval_results__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateResult' + '404': + description: Result not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Evaluator Plugin Eval Results Routes + summary: Delete Eval Result + description: Delete a (row) evaluation result record by workspace and name. + operationId: delete_eval_result_apis_evaluator_v2_workspaces__workspace__eval_results__name__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '204': + description: Successful Response + '404': + description: Result not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/evaluator/v2/workspaces/{workspace}/evaluate/jobs: post: tags: @@ -1101,6 +1361,94 @@ components: title: AgentEvalInputSpec description: 'Submitter-facing agent-evaluation input: tasks whose metrics may be inline or references.' + AgentEvalResult: + properties: + id: + type: string + title: Id + description: Unique identifier for the stored result record. + name: + type: string + title: Name + description: Result record name (equals the producing job's id). + workspace: + type: string + title: Workspace + description: Workspace the result belongs to. + project: + title: Project + description: The project associated with this result. + type: string + job_id: + type: string + title: Job Id + description: Identifier of the job run that produced this result. + target_kind: + title: Target Kind + description: 'Target discriminator: ''model'', ''agent'', or a runner kind.' + type: string + target_name: + title: Target Name + description: Model/agent entity name, or the runner's model. + type: string + target_url: + title: Target Url + description: Endpoint URL, when the target is an HTTP model/agent. + type: string + scores: + allOf: + - $ref: '#/components/schemas/AggregatedMetricResult' + description: Aggregated metric scores for the run. + bundle_ref: + type: string + title: Bundle Ref + description: Reference to the full result bundle in the Files service. + created_at: + type: string + format: date-time + title: Created At + description: Timestamp the result was created. + updated_at: + type: string + format: date-time + title: Updated At + description: Timestamp the result was last updated. + type: object + required: + - id + - name + - workspace + - job_id + - scores + - bundle_ref + - created_at + - updated_at + title: AgentEvalResult + description: API representation of a persisted agent-evaluation result record. + AgentEvalResultsPage: + properties: + data: + items: + $ref: '#/components/schemas/AgentEvalResult' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: AgentEvalResultsPage AgentEvalSpec: oneOf: - properties: @@ -1552,6 +1900,146 @@ components: there is no separate prompt template here.' + AggregateRangeScore: + properties: + name: + type: string + title: Name + description: Name of the score. + count: + type: integer + title: Count + description: Number of samples evaluated (excluding NaN). + nan_count: + type: integer + title: Nan Count + description: Number of samples that produced NaN scores. + sum: + title: Sum + description: Sum of all score values. + type: number + mean: + title: Mean + description: Mean score value. + type: number + min: + title: Min + description: Minimum score value. + type: number + max: + title: Max + description: Maximum score value. + type: number + std_dev: + title: Std Dev + description: Standard deviation of the scores. + type: number + variance: + title: Variance + description: Variance of the scores. + type: number + score_type: + type: string + const: range + title: Score Type + description: Type of score. + default: range + percentiles: + allOf: + - $ref: '#/components/schemas/Percentiles' + description: Percentile distribution of scores. + histogram: + allOf: + - $ref: '#/components/schemas/Histogram' + description: Histogram of score distribution. + additionalProperties: false + type: object + required: + - name + - count + - nan_count + title: AggregateRangeScore + description: Aggregated statistics for a range-type score with percentiles and + histogram. + AggregateRubricScore: + properties: + name: + type: string + title: Name + description: Name of the score. + count: + type: integer + title: Count + description: Number of samples evaluated (excluding NaN). + nan_count: + type: integer + title: Nan Count + description: Number of samples that produced NaN scores. + sum: + title: Sum + description: Sum of all score values. + type: number + mean: + title: Mean + description: Mean score value. + type: number + min: + title: Min + description: Minimum score value. + type: number + max: + title: Max + description: Maximum score value. + type: number + std_dev: + title: Std Dev + description: Standard deviation of the scores. + type: number + variance: + title: Variance + description: Variance of the scores. + type: number + score_type: + type: string + const: rubric + title: Score Type + description: Type of score. + default: rubric + rubric_distribution: + items: + $ref: '#/components/schemas/RubricScoreStat' + type: array + title: Rubric Distribution + description: Distribution of rubric categories. + mode_category: + title: Mode Category + description: Most frequent rubric category. + type: string + additionalProperties: false + type: object + required: + - name + - count + - nan_count + - rubric_distribution + title: AggregateRubricScore + description: Aggregated statistics for a rubric-type score with category distribution. + AggregatedMetricResult: + properties: + scores: + items: + anyOf: + - $ref: '#/components/schemas/AggregateRangeScore' + - $ref: '#/components/schemas/AggregateRubricScore' + type: array + title: Scores + description: The list of aggregated scores. + additionalProperties: false + type: object + required: + - scores + title: AggregatedMetricResult + description: Result of aggregating metric scores with full statistics. BundledMetricOutputSpec: properties: name: @@ -1886,6 +2374,133 @@ components: - updated_at - -updated_at title: EvaluateJobsSortField + EvaluateResult: + properties: + id: + type: string + title: Id + description: Unique identifier for the stored result record. + name: + type: string + title: Name + description: Result record name (equals the producing job's id). + workspace: + type: string + title: Workspace + description: Workspace the result belongs to. + project: + title: Project + description: The project associated with this result. + type: string + job_id: + type: string + title: Job Id + description: Identifier of the job run that produced this result. + target_kind: + title: Target Kind + description: 'Target discriminator: ''model'', ''agent'', or a runner kind.' + type: string + target_name: + title: Target Name + description: Model/agent entity name, or the runner's model. + type: string + target_url: + title: Target Url + description: Endpoint URL, when the target is an HTTP model/agent. + type: string + scores: + allOf: + - $ref: '#/components/schemas/AggregatedMetricResult' + description: Aggregated metric scores for the run. + bundle_ref: + type: string + title: Bundle Ref + description: Reference to the full result bundle in the Files service. + created_at: + type: string + format: date-time + title: Created At + description: Timestamp the result was created. + updated_at: + type: string + format: date-time + title: Updated At + description: Timestamp the result was last updated. + dataset_ref: + title: Dataset Ref + description: Reference to the dataset evaluated; None for an inline dataset. + type: string + metric_types: + items: + type: string + type: array + title: Metric Types + description: Runtime metric type names applied in the run. + type: object + required: + - id + - name + - workspace + - job_id + - scores + - bundle_ref + - created_at + - updated_at + - metric_types + title: EvaluateResult + description: API representation of a persisted (row) evaluation result record. + EvaluateResultFilter: + additionalProperties: false + description: Adds row-eval's referenceable-input trait. + properties: + workspace: + title: Workspace + type: string + name: + title: Name + type: string + job_id: + title: Job Id + type: string + target_kind: + title: Target Kind + type: string + target_name: + title: Target Name + type: string + created_at: + $ref: '#/components/schemas/DatetimeFilter' + updated_at: + $ref: '#/components/schemas/DatetimeFilter' + dataset_ref: + title: Dataset Ref + type: string + title: EvaluateResultFilter + type: object + EvaluateResultsPage: + properties: + data: + items: + $ref: '#/components/schemas/EvaluateResult' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: EvaluateResultsPage EvaluateSpec: properties: dataset: @@ -2071,6 +2686,46 @@ components: required: - message title: HelloResponse + Histogram: + properties: + bins: + items: + $ref: '#/components/schemas/HistogramBin' + type: array + title: Bins + description: Histogram bins. + additionalProperties: false + type: object + required: + - bins + title: Histogram + description: Histogram of score distribution. + HistogramBin: + properties: + lower_bound: + anyOf: + - type: number + - type: integer + title: Lower Bound + description: Lower bound of the bin (inclusive). + upper_bound: + anyOf: + - type: number + - type: integer + title: Upper Bound + description: Upper bound of the bin (exclusive for all but last bin). + count: + type: integer + title: Count + description: Number of values in this bin. + additionalProperties: false + type: object + required: + - lower_bound + - upper_bound + - count + title: HistogramBin + description: A single bin in a histogram. InferenceParams: properties: temperature: @@ -2501,6 +3156,83 @@ components: - total_pages - total_results title: PaginationData + Percentiles: + properties: + p10: + anyOf: + - type: number + - type: integer + title: P10 + description: 10th percentile. + p20: + anyOf: + - type: number + - type: integer + title: P20 + description: 20th percentile. + p30: + anyOf: + - type: number + - type: integer + title: P30 + description: 30th percentile. + p40: + anyOf: + - type: number + - type: integer + title: P40 + description: 40th percentile. + p50: + anyOf: + - type: number + - type: integer + title: P50 + description: 50th percentile (median). + p60: + anyOf: + - type: number + - type: integer + title: P60 + description: 60th percentile. + p70: + anyOf: + - type: number + - type: integer + title: P70 + description: 70th percentile. + p80: + anyOf: + - type: number + - type: integer + title: P80 + description: 80th percentile. + p90: + anyOf: + - type: number + - type: integer + title: P90 + description: 90th percentile. + p100: + anyOf: + - type: number + - type: integer + title: P100 + description: 100th percentile. + additionalProperties: false + type: object + required: + - p10 + - p20 + - p30 + - p40 + - p50 + - p60 + - p70 + - p80 + - p90 + - p100 + title: Percentiles + description: Percentile distribution of scores. PlatformJobListResultResponse: properties: data: @@ -2763,6 +3495,71 @@ components: type: object title: ReasoningParams description: Custom settings that control the model's reasoning behavior. + ResultFilter: + additionalProperties: false + description: Traits shared by both result collections (used directly for agent-eval + results). + properties: + workspace: + title: Workspace + type: string + name: + title: Name + type: string + job_id: + title: Job Id + type: string + target_kind: + title: Target Kind + type: string + target_name: + title: Target Name + type: string + created_at: + $ref: '#/components/schemas/DatetimeFilter' + updated_at: + $ref: '#/components/schemas/DatetimeFilter' + title: ResultFilter + type: object + ResultSort: + type: string + enum: + - name + - -name + - created_at + - -created_at + - updated_at + - -updated_at + title: ResultSort + description: Sort fields for result queries (``-`` prefix sorts descending). + RubricScoreStat: + properties: + label: + type: string + title: Label + description: The label to use for the level of the rubric grading criteria. + description: + title: Description + description: Describe the semantic meaning of each criteria for the given + rubric. + type: string + value: + anyOf: + - type: number + - type: integer + title: Value + description: The score value to assign for the criteria. + count: + type: integer + title: Count + description: The number of samples evaluated with the rubric level. + default: 0 + type: object + required: + - label + - value + title: RubricScoreStat + description: Rubric score with count statistics. RunConfig: properties: parallelism: diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/api/dependencies.py b/plugins/nemo-evaluator/src/nemo_evaluator/api/dependencies.py index 0ae2d41673..3dfd8c163c 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/api/dependencies.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/api/dependencies.py @@ -7,6 +7,7 @@ from fastapi import Depends from nemo_evaluator.api.service.metric_service import MetricService +from nemo_evaluator.api.service.result_service import ResultService from nemo_platform import AsyncNeMoPlatform from nemo_platform_plugin.dependencies import get_entity_client, get_sdk_client from nemo_platform_plugin.entities import EntityClient @@ -18,3 +19,10 @@ def get_metric_service( ) -> MetricService: """Provide a MetricService wired to the Entity Store and Files service.""" return MetricService(entity_client, sdk) + + +def get_result_service( + entity_client: EntityClient = Depends(get_entity_client), +) -> ResultService: + """Provide a ResultService wired to the Entity Store (read-only over result entities).""" + return ResultService(entity_client) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/api/schemas.py b/plugins/nemo-evaluator/src/nemo_evaluator/api/schemas.py index b90ee9097e..982323bfb1 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/api/schemas.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/api/schemas.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Request/response schemas for the evaluator metrics API.""" +"""Request/response schemas for the evaluator API — metrics, eval results, and shared filters.""" from __future__ import annotations @@ -14,10 +14,41 @@ MetricMetadata, ) from nemo_evaluator_sdk.values.common import SecretRef +from nemo_evaluator_sdk.values.results import AggregatedMetricResult +from nemo_platform_plugin.api.filter import ComparisonOperation, FilterOperation, LogicalOperation +from nemo_platform_plugin.api.parsed_filter import ENTITY_BASE_FIELDS from nemo_platform_plugin.schema import DatetimeFilter, Filter from pydantic import BaseModel, ConfigDict, Field, field_validator +class DataFilter(Filter): + """A ``Filter`` whose declared non-base fields are stored under the entity's ``data.*`` column. + + Implements the duck-typed hooks ``make_filter_dep`` looks for, so a custom-field filter (e.g. + ``metric_type`` or ``job_id``) is rewritten to ``data.`` for the entity store. The plain + ``Filter`` does no translation, so an un-prefixed custom field reaches the store unresolved and + 500s. (The richer ``nmp.common`` filter does this, but plugins can't depend on it — minimal port.) + """ + + @classmethod + def _get_entity_field_map(cls) -> dict[str, str]: + return {name: f"data.{name}" for name in cls.model_fields if name not in ENTITY_BASE_FIELDS} + + @classmethod + def translate_operation(cls, operation: FilterOperation) -> FilterOperation: + field_map = cls._get_entity_field_map() + + def _walk(op: FilterOperation) -> FilterOperation: + if isinstance(op, ComparisonOperation): + mapped = field_map.get(op.field) + return op if mapped is None else op.model_copy(update={"field": mapped}) + if isinstance(op, LogicalOperation): + return op.model_copy(update={"operations": [_walk(child) for child in op.operations]}) + return op + + return _walk(operation) + + class CloudpickleMetricPayload(BaseModel): """Wire schema for a cloudpickle-serialized metric payload. @@ -135,7 +166,7 @@ class MetricSort(StrEnum): UPDATED_AT_DESC = "-updated_at" -class MetricFilter(Filter): +class MetricFilter(DataFilter): """Filter for metric queries.""" workspace: str | None = Field(None, description="Filter by workspace.") @@ -144,3 +175,46 @@ class MetricFilter(Filter): description: str | None = Field(None, description="Filter by description.") created_at: DatetimeFilter | None = Field(None, description="Filter by creation date.") updated_at: DatetimeFilter | None = Field(None, description="Filter by update date.") + + +# --- Eval result DTOs -------------------------------------------------------- +# +# API representation of the persisted result records (the storage entities are +# ``AgentEvalResultEntity`` / ``EvaluateResultEntity``). A separate DTO — like ``Metric`` for +# ``MetricBundleEntity`` — so the wire/SDK contract round-trips cleanly: an ``EntityBase``'s +# ``id`` / ``created_at`` / ``updated_at`` are computed/output-only and don't deserialize from +# the entity's own serialized form, whereas these plain fields do. + + +class _ResultBase(BaseModel): + """Fields common to both result DTOs (provenance + aggregated scores + target traits).""" + + id: str = Field(description="Unique identifier for the stored result record.") + name: str = Field(description="Result record name (equals the producing job's id).") + workspace: str = Field(description="Workspace the result belongs to.") + project: str | None = Field(default=None, description="The project associated with this result.") + job_id: str = Field(description="Identifier of the job run that produced this result.") + # Nullable traits default to None so they round-trip when the list route serializes with + # response_model_exclude_none (which drops null values from the payload) — matching ``Metric``. + target_kind: str | None = Field( + default=None, description="Target discriminator: 'model', 'agent', or a runner kind." + ) + target_name: str | None = Field(default=None, description="Model/agent entity name, or the runner's model.") + target_url: str | None = Field(default=None, description="Endpoint URL, when the target is an HTTP model/agent.") + scores: AggregatedMetricResult = Field(description="Aggregated metric scores for the run.") + bundle_ref: str = Field(description="Reference to the full result bundle in the Files service.") + created_at: datetime = Field(description="Timestamp the result was created.") + updated_at: datetime = Field(description="Timestamp the result was last updated.") + + +class AgentEvalResult(_ResultBase): + """API representation of a persisted agent-evaluation result record.""" + + +class EvaluateResult(_ResultBase): + """API representation of a persisted (row) evaluation result record.""" + + dataset_ref: str | None = Field( + default=None, description="Reference to the dataset evaluated; None for an inline dataset." + ) + metric_types: list[str] = Field(description="Runtime metric type names applied in the run.") diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/api/service/result_service.py b/plugins/nemo-evaluator/src/nemo_evaluator/api/service/result_service.py new file mode 100644 index 0000000000..aa559fdea5 --- /dev/null +++ b/plugins/nemo-evaluator/src/nemo_evaluator/api/service/result_service.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Read service for persisted eval-result entities. + +Results are written by the jobs themselves (see ``jobs.result_persistence``); this service only +exposes them for list/get/delete. Stored entities are mapped to API DTOs (``AgentEvalResult`` / +``EvaluateResult``) — like ``MetricService`` maps ``MetricBundleEntity`` to ``Metric`` — so the wire +contract round-trips cleanly (an ``EntityBase``'s ``id``/``created_at`` are computed and don't +deserialize from the entity's own serialized form). Each result type has its own concretely-typed +methods so the API contract (and generated SDK) sees the real DTO. +""" + +from __future__ import annotations + +from datetime import datetime + +from nemo_evaluator.api.schemas import AgentEvalResult, EvaluateResult +from nemo_evaluator.entities import AgentEvalResultEntity, EvaluateResultEntity +from nemo_platform_plugin.entities import EntityBase, EntityClient, EntityNotFoundError, PaginationInfo +from nemo_platform_plugin.filter_ops import FilterOperation +from nemo_platform_plugin.schema import Page, PaginationData + + +def _timestamps(entity: AgentEvalResultEntity | EvaluateResultEntity) -> tuple[datetime, datetime]: + """The entity's persistence timestamps, guarded — a stored result must have them.""" + created_at = entity.created_at + updated_at = entity.updated_at + if created_at is None or updated_at is None: + raise ValueError(f"Stored result '{entity.workspace}/{entity.name}' is missing persistence timestamps") + return created_at, updated_at + + +def _to_agent_eval(entity: AgentEvalResultEntity) -> AgentEvalResult: + created_at, updated_at = _timestamps(entity) + return AgentEvalResult( + id=entity.id, + name=entity.name, + workspace=entity.workspace, + project=entity.project, + job_id=entity.job_id, + target_kind=entity.target_kind, + target_name=entity.target_name, + target_url=entity.target_url, + scores=entity.scores, + bundle_ref=entity.bundle_ref, + created_at=created_at, + updated_at=updated_at, + ) + + +def _to_evaluate(entity: EvaluateResultEntity) -> EvaluateResult: + created_at, updated_at = _timestamps(entity) + return EvaluateResult( + id=entity.id, + name=entity.name, + workspace=entity.workspace, + project=entity.project, + job_id=entity.job_id, + target_kind=entity.target_kind, + target_name=entity.target_name, + target_url=entity.target_url, + scores=entity.scores, + bundle_ref=entity.bundle_ref, + created_at=created_at, + updated_at=updated_at, + dataset_ref=entity.dataset_ref, + metric_types=entity.metric_types, + ) + + +def _pagination(src: PaginationInfo, current_page_size: int) -> PaginationData: + """Carry the entity-store pagination counts into the API ``Page`` envelope.""" + return PaginationData( + page=src.page, + page_size=src.page_size, + current_page_size=current_page_size, + total_pages=src.total_pages, + total_results=src.total_results, + ) + + +class ResultService: + """List/get/delete for persisted eval-result entities, exposed as API DTOs.""" + + def __init__(self, entity_client: EntityClient): + self.entity_client = entity_client + + # --- agent-eval results -------------------------------------------------- + + async def list_agent_eval_results( + self, + *, + workspace: str, + page: int = 1, + page_size: int = 100, + sort: str | None = None, + filter_operation: FilterOperation | None = None, + ) -> Page[AgentEvalResult]: + result = await self.entity_client.list( + AgentEvalResultEntity, + workspace=workspace, + filter_operation=filter_operation, + sort=sort, + page=page, + page_size=page_size, + ) + data = [_to_agent_eval(e) for e in result.data] + return Page(data=data, pagination=_pagination(result.pagination, len(data)), sort=sort, filter=None) + + async def get_agent_eval_result(self, workspace: str, name: str) -> AgentEvalResult | None: + try: + entity = await self.entity_client.get(AgentEvalResultEntity, workspace=workspace, name=name) + except EntityNotFoundError: + return None + return _to_agent_eval(entity) + + async def delete_agent_eval_result(self, workspace: str, name: str) -> bool: + return await self._delete(AgentEvalResultEntity, workspace, name) + + # --- (row) eval results -------------------------------------------------- + + async def list_eval_results( + self, + *, + workspace: str, + page: int = 1, + page_size: int = 100, + sort: str | None = None, + filter_operation: FilterOperation | None = None, + ) -> Page[EvaluateResult]: + result = await self.entity_client.list( + EvaluateResultEntity, + workspace=workspace, + filter_operation=filter_operation, + sort=sort, + page=page, + page_size=page_size, + ) + data = [_to_evaluate(e) for e in result.data] + return Page(data=data, pagination=_pagination(result.pagination, len(data)), sort=sort, filter=None) + + async def get_eval_result(self, workspace: str, name: str) -> EvaluateResult | None: + try: + entity = await self.entity_client.get(EvaluateResultEntity, workspace=workspace, name=name) + except EntityNotFoundError: + return None + return _to_evaluate(entity) + + async def delete_eval_result(self, workspace: str, name: str) -> bool: + return await self._delete(EvaluateResultEntity, workspace, name) + + async def _delete(self, entity_cls: type[EntityBase], workspace: str, name: str) -> bool: + """Delete by workspace/name; ``False`` if absent. Type-agnostic (delete takes no body).""" + try: + await self.entity_client.delete(entity_cls, name, workspace=workspace) + except EntityNotFoundError: + return False + return True diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/results.py b/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/results.py new file mode 100644 index 0000000000..23260f8abc --- /dev/null +++ b/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/results.py @@ -0,0 +1,230 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Read routes for persisted eval results. + +Two collections under ``/apis/evaluator/v2/workspaces/{workspace}``: + +* ``/agent-eval-results`` — :class:`AgentEvalResultEntity` (from ``AgentEvalJob``) +* ``/eval-results`` — :class:`EvaluateResultEntity` (from ``EvaluateJob``) + +They're distinct entity types, so each collection has its own concretely-typed routes (the generated +SDK then sees the real result type, not an abstract base). Both support filtering by traits — target, +dataset, timestamps — mirroring the legacy ``job_result_routes``. +""" + +from __future__ import annotations + +import logging +from enum import StrEnum + +from fastapi import APIRouter, Depends, HTTPException, Query, status +from nemo_evaluator.api.dependencies import get_result_service +from nemo_evaluator.api.schemas import AgentEvalResult, DataFilter, EvaluateResult +from nemo_evaluator.api.service.result_service import ResultService +from nemo_evaluator.authz import scope +from nemo_platform_plugin.api.parsed_filter import ParsedFilter, make_filter_dep +from nemo_platform_plugin.authz import CallerKind, PermissionSet, path_rule, perm +from nemo_platform_plugin.jobs.openapi_utils import generate_openapi_extra_params +from nemo_platform_plugin.schema import DatetimeFilter, Page + +logger = logging.getLogger(__name__) + + +class ResultPerms(PermissionSet, namespace="evaluator"): + """Permissions for the read-only eval-result collections (results are job-produced, not created via API).""" + + LIST = perm("List stored eval results", suffix="results.list") + READ = perm("Read a stored eval result", suffix="results.read") + DELETE = perm("Delete a stored eval result", suffix="results.delete") + + +class ResultSort(StrEnum): + """Sort fields for result queries (``-`` prefix sorts descending).""" + + NAME_ASC = "name" + NAME_DESC = "-name" + CREATED_AT_ASC = "created_at" + CREATED_AT_DESC = "-created_at" + UPDATED_AT_ASC = "updated_at" + UPDATED_AT_DESC = "-updated_at" + + +_PAGE = Query(default=1, ge=1, description="Page number.") +_PAGE_SIZE = Query(default=100, ge=1, le=1000, description="Page size.") +_SORT = Query( + default=ResultSort.CREATED_AT_DESC, + description="Sort field; prefix with '-' for descending.", +) + + +def _sanitize_for_log(value: object) -> str: + return str(value).replace("\r", "").replace("\n", "") + + +class ResultFilter(DataFilter): + """Traits shared by both result collections (used directly for agent-eval results).""" + + workspace: str | None = None + name: str | None = None + job_id: str | None = None + target_kind: str | None = None + target_name: str | None = None + created_at: DatetimeFilter | None = None + updated_at: DatetimeFilter | None = None + + +class EvaluateResultFilter(ResultFilter): + """Adds row-eval's referenceable-input trait.""" + + dataset_ref: str | None = None + + +agent_eval_results_router = APIRouter() +evaluate_results_router = APIRouter() + + +# --- agent-eval results ------------------------------------------------------ + + +@agent_eval_results_router.get( + "/agent-eval-results", + summary="List Agent Eval Results By Workspace", + status_code=status.HTTP_200_OK, + response_model=Page[AgentEvalResult], + response_model_exclude_none=True, + openapi_extra=generate_openapi_extra_params( + filter_schema=ResultFilter, + filter_description="Filter by workspace, name, target, and timestamps.", + ), +) +@scope.read +@path_rule(callers=[CallerKind.PRINCIPAL], permissions=[ResultPerms.LIST]) +async def list_agent_eval_results( + workspace: str, + page: int = _PAGE, + page_size: int = _PAGE_SIZE, + sort: ResultSort = _SORT, + parsed_filter: ParsedFilter = Depends(make_filter_dep(ResultFilter)), + service: ResultService = Depends(get_result_service), +) -> Page[AgentEvalResult]: + """List agent-evaluation result records for a workspace.""" + parsed_filter.remove("workspace") + try: + return await service.list_agent_eval_results( + workspace=workspace, page=page, page_size=page_size, sort=sort, filter_operation=parsed_filter.operation + ) + except Exception: + logger.exception(f"Failed to list agent-eval results for workspace {_sanitize_for_log(workspace)}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") + + +@agent_eval_results_router.get( + "/agent-eval-results/{name}", + summary="Get Agent Eval Result", + responses={status.HTTP_404_NOT_FOUND: {"description": "Result not found"}}, +) +@scope.read +@path_rule(callers=[CallerKind.PRINCIPAL], permissions=[ResultPerms.READ]) +async def get_agent_eval_result( + workspace: str, + name: str, + service: ResultService = Depends(get_result_service), +) -> AgentEvalResult: + """Get an agent-evaluation result record by workspace and name.""" + result = await service.get_agent_eval_result(workspace, name) + if result is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Result not found: {workspace}/{name}") + return result + + +@agent_eval_results_router.delete( + "/agent-eval-results/{name}", + summary="Delete Agent Eval Result", + status_code=status.HTTP_204_NO_CONTENT, + responses={status.HTTP_404_NOT_FOUND: {"description": "Result not found"}}, +) +@scope.write +@path_rule(callers=[CallerKind.PRINCIPAL], permissions=[ResultPerms.DELETE]) +async def delete_agent_eval_result( + workspace: str, + name: str, + service: ResultService = Depends(get_result_service), +) -> None: + """Delete an agent-evaluation result record by workspace and name.""" + if not await service.delete_agent_eval_result(workspace, name): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Result not found: {workspace}/{name}") + return None + + +# --- (row) eval results ------------------------------------------------------ + + +@evaluate_results_router.get( + "/eval-results", + summary="List Eval Results By Workspace", + status_code=status.HTTP_200_OK, + response_model=Page[EvaluateResult], + response_model_exclude_none=True, + openapi_extra=generate_openapi_extra_params( + filter_schema=EvaluateResultFilter, + filter_description="Filter by workspace, name, target, dataset_ref, and timestamps.", + ), +) +@scope.read +@path_rule(callers=[CallerKind.PRINCIPAL], permissions=[ResultPerms.LIST]) +async def list_eval_results( + workspace: str, + page: int = _PAGE, + page_size: int = _PAGE_SIZE, + sort: ResultSort = _SORT, + parsed_filter: ParsedFilter = Depends(make_filter_dep(EvaluateResultFilter)), + service: ResultService = Depends(get_result_service), +) -> Page[EvaluateResult]: + """List (row) evaluation result records for a workspace.""" + parsed_filter.remove("workspace") + try: + return await service.list_eval_results( + workspace=workspace, page=page, page_size=page_size, sort=sort, filter_operation=parsed_filter.operation + ) + except Exception: + logger.exception(f"Failed to list eval results for workspace {_sanitize_for_log(workspace)}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") + + +@evaluate_results_router.get( + "/eval-results/{name}", + summary="Get Eval Result", + responses={status.HTTP_404_NOT_FOUND: {"description": "Result not found"}}, +) +@scope.read +@path_rule(callers=[CallerKind.PRINCIPAL], permissions=[ResultPerms.READ]) +async def get_eval_result( + workspace: str, + name: str, + service: ResultService = Depends(get_result_service), +) -> EvaluateResult: + """Get a (row) evaluation result record by workspace and name.""" + result = await service.get_eval_result(workspace, name) + if result is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Result not found: {workspace}/{name}") + return result + + +@evaluate_results_router.delete( + "/eval-results/{name}", + summary="Delete Eval Result", + status_code=status.HTTP_204_NO_CONTENT, + responses={status.HTTP_404_NOT_FOUND: {"description": "Result not found"}}, +) +@scope.write +@path_rule(callers=[CallerKind.PRINCIPAL], permissions=[ResultPerms.DELETE]) +async def delete_eval_result( + workspace: str, + name: str, + service: ResultService = Depends(get_result_service), +) -> None: + """Delete a (row) evaluation result record by workspace and name.""" + if not await service.delete_eval_result(workspace, name): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Result not found: {workspace}/{name}") + return None diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/entities.py b/plugins/nemo-evaluator/src/nemo_evaluator/entities.py index 3e3d871400..00415a77c9 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/entities.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/entities.py @@ -17,8 +17,9 @@ from nemo_evaluator.shared.metric_bundles.bundles import BundledMetricOutputSpec from nemo_evaluator_sdk.values.common import SecretRef +from nemo_evaluator_sdk.values.results import AggregatedMetricResult from nemo_platform_plugin.entities import EntityBase -from pydantic import Field +from pydantic import BaseModel, Field # Constants are intentionally local: nmp_common's entity constants are not # re-exported to plugins. Keep these aligned with @@ -70,3 +71,82 @@ class MetricBundleEntity(EntityBase): description="Description captured from the bundled metric's metadata.", max_length=MAX_DESCRIPTION_LENGTH, ) + + +# --- Eval result entities ---------------------------------------------------- +# +# A result entity is the persisted, *queryable* record of one eval run: the +# aggregated scores plus the traits you'd filter on (target, dataset). The +# detailed per-row / per-trial output that doesn't fit a concise record stays in +# the run's fileset bundle, referenced here by ``bundle_ref``. The entity — not +# Intake — is the evaluator's source of truth; Intake is a denormalized, optional +# downstream copy. +# +# Both result types share the SAME record (``_EvalResultCommon``): provenance, the target it ran +# against, the aggregated ``scores`` rollup, and a ``bundle_ref`` to the full detail. They differ +# only where the domain genuinely differs — row-eval has *referenceable inputs* (its dataset fileset +# + metric refs), which the entity records; agent-eval's tasks are inline, so it has no input ref yet +# (the "Taskset" gap). Run counts / per-metric coverage are derivable rollups that live in the +# bundle's summary, not on the record. This keeps the two entities aligned and matches the lean legacy +# ``BaseJobResult`` → ``MetricJobResult`` / ``BenchmarkJobResult`` shape (refs + scores). + + +class _EvalResultCommon(BaseModel): + """Fields shared by every persisted eval-result record (aggregates + filterable traits). + + A mixin (not itself an ``EntityBase``) so the concrete result entities can each declare their own + ``__entity_type__`` — same split as the legacy ``BaseJobResult`` → ``MetricJobResult`` / + ``BenchmarkJobResult``. + + Every field is required — a result is only persisted once the run has produced all of it, so the + caller populates each value (no schema defaults papering over missing data). "What it ran + against" is denormalized into flat ``target_*`` fields so the list route can filter by them (the + entity filter matches top-level fields; a nested object wouldn't filter cleanly); they're nullable + because an offline run (precomputed trials) has no target, but the caller must still pass them. + + (``labels`` and a run ``status`` are intentionally absent: there's no labels source on the spec + yet, and persistence happens only on success — both would be schema defaults with no real data. + Add them when there's a source — labels alongside a spec ``labels`` field, status if/when partial + or failed runs are persisted.) + """ + + job_id: str = Field(description="Identifier of the job run that produced this result (one result per run).") + target_kind: str | None = Field( + description="Target discriminator: 'model', 'agent', or a runner kind e.g. 'codex'." + ) + target_name: str | None = Field(description="Model/agent entity name, or the runner's model — filterable trait.") + target_url: str | None = Field(description="Endpoint URL, when the target is an HTTP model/agent.") + scores: AggregatedMetricResult = Field( + description="Aggregated metric scores for the run (the concise, queryable rollup)." + ) + bundle_ref: str = Field( + description="Reference to the full result bundle in the Files service (rows/trials), e.g. a 'fileset://...' URL.", + ) + + +class AgentEvalResultEntity(_EvalResultCommon, EntityBase): + """Persisted, queryable record of an ``AgentEvalJob`` run. + + Carries only the shared record — its tasks are inline, so (unlike row-eval) it has no input ref + to record yet. Trials, per-metric coverage, and run counts live in the bundle's summary. + """ + + __entity_type__: ClassVar[str] = "agent_eval_result" + + +class EvaluateResultEntity(_EvalResultCommon, EntityBase): + """Persisted, queryable record of an ``EvaluateJob`` (row-eval) run. + + Adds the run's *referenceable inputs* — the evaluated dataset and the metrics applied — which the + shared record can't capture. Row-level detail lives in the bundle. + """ + + __entity_type__: ClassVar[str] = "evaluate_result" + + dataset_ref: str | None = Field( + description="Reference to the dataset evaluated (e.g. 'workspace/fileset'); None for an inline dataset." + ) + metric_types: list[str] = Field( + description="Runtime metric type names applied in the run (e.g. 'exact_match'). Not metric refs: " + "by run time the submitted refs are resolved to inline bundles, so the originals aren't available." + ) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py index 9708cc6832..5e85055376 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py @@ -10,8 +10,9 @@ Per-task metrics may be given inline or as references to stored metrics; references are resolved into inline metrics during ``to_spec`` via the shared -:mod:`nemo_evaluator.jobs.metric_resolution` helper. The result bundle (trials + -scores + summary) is persisted as job artifacts. +:mod:`nemo_evaluator.jobs.metric_resolution` helper. The full result bundle (trials + +scores + summary) is persisted as job artifacts, and a concise, queryable result entity +is written via :func:`~nemo_evaluator.jobs.result_persistence.persist_agent_eval_result`. """ from __future__ import annotations @@ -33,6 +34,7 @@ Target, ) from nemo_evaluator.jobs.metric_resolution import resolve_metrics_to_inline, to_runtime_bundle +from nemo_evaluator.jobs.result_persistence import persist_agent_eval_result from nemo_evaluator.shared.metric_bundles.bundles import unbundle_metric from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator from nemo_evaluator_sdk.agent_eval.persistence import persist_run @@ -280,4 +282,10 @@ def run( artifact = ctx.results.save(DEFAULT_RESULT_NAME, files.bundle_dir) ctx.results.save(SUMMARY_RESULT_NAME, files.summary) + # Persist the queryable result record (aggregates + coverage); the full bundle (trials) lives + # in the fileset referenced by `artifact`. + persist_agent_eval_result( + result, target=spec.target, ctx=ctx, bundle_ref=artifact.artifact_url, async_sdk=async_sdk + ) + return {"status": "completed", "artifact": artifact.model_dump()} diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py index 4589c1ade8..8710eebbfa 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py @@ -21,6 +21,7 @@ to_runtime_bundle, unresolved_model_refs, ) +from nemo_evaluator.jobs.result_persistence import persist_evaluate_result from nemo_evaluator.metric_refs import MetricRefOrInline from nemo_evaluator.shared.metric_bundles.bundles import unbundle_metric from nemo_evaluator_sdk import Evaluator @@ -300,6 +301,18 @@ def run( ctx.results.save(ROW_SCORES_RESULT_NAME, result_files.row_scores) ctx.results.save(ARTIFACTS_RESULT_NAME, result_files.artifacts_dir, ignore_patterns=RESULT_IGNORE_PATTERNS) + # Persist the queryable result record (aggregate scores); per-row detail lives in the fileset + # bundle referenced by `artifact`. + persist_evaluate_result( + result, + target=spec.target, + dataset_ref=spec.dataset.root if isinstance(spec.dataset, FilesetRef) else None, + metric_types=[metric.type for metric in metrics], + ctx=ctx, + bundle_ref=artifact.artifact_url, + async_sdk=async_sdk, + ) + # TODO: Implement progress reporting hook in SDK - AALGO-149 # self.report_progress( # ctx, diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/result_persistence.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/result_persistence.py new file mode 100644 index 0000000000..fc2d1d71ae --- /dev/null +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/result_persistence.py @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Persist eval-run results as queryable entities. + +Both evaluator jobs persist the *full* result bundle (rows/trials) to the job's fileset via +``ctx.results.save``. This module adds the other half the legacy service had: a concise, queryable +**result entity** (aggregated scores + traits to filter on), with ``bundle_ref`` pointing back at the +fileset bundle. The entity is the evaluator's source of truth. + +``run`` is synchronous but the entity-store client is async, so the job is injected an async task SDK +(``get_async_task_sdk``) alongside the sync one; we drive the entity write with ``run_sync``. A +platformless local run (no async SDK) simply skips persistence. +""" + +from __future__ import annotations + +import logging + +from nemo_evaluator.entities import AgentEvalResultEntity, EvaluateResultEntity +from nemo_evaluator.jobs.agent_spec import AgentTarget, CodexRunnerTarget, ModelTarget, Target +from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult +from nemo_evaluator_sdk.execution.metric_execution import run_sync +from nemo_evaluator_sdk.values import Agent, Model +from nemo_evaluator_sdk.values.multi_metric_results import BenchmarkEvaluationResult +from nemo_evaluator_sdk.values.results import EvaluationResult +from nemo_platform import AsyncNeMoPlatform +from nemo_platform.resources.entities import AsyncEntitiesResource +from nemo_platform_plugin.entities import EntityBase, EntityClient +from nemo_platform_plugin.job_context import JobContext + +logger = logging.getLogger(__name__) + + +def _entity_client(async_sdk: AsyncNeMoPlatform | None) -> EntityClient | None: + """The standard async ``EntityClient`` for the job's async task SDK, or ``None`` if absent. + + ``None`` means a platformless local run (no async SDK injected) — persistence is skipped. + """ + if async_sdk is None: + return None + return EntityClient(AsyncEntitiesResource(async_sdk)) + + +def _agent_target_fields(target: Target | None) -> tuple[str | None, str | None, str | None]: + """(kind, name, url) flat target traits for an agent-eval target.""" + if isinstance(target, ModelTarget): + return "model", target.model.name, str(target.model.url) if target.model.url else None + if isinstance(target, AgentTarget): + return "agent", getattr(target.agent, "name", None), str(target.agent.url) + if isinstance(target, CodexRunnerTarget): + return "codex", target.model, None + return None, None, None + + +def _row_target_fields(target: Model | Agent | None) -> tuple[str | None, str | None, str | None]: + """(kind, name, url) flat target traits for a row-eval target.""" + if isinstance(target, Model): + return "model", target.name, str(target.url) if target.url else None + if isinstance(target, Agent): + return "agent", getattr(target, "name", None), str(target.url) + return None, None, None + + +def _persist(entity: EntityBase, *, async_sdk: AsyncNeMoPlatform | None) -> None: + client = _entity_client(async_sdk) + if client is None: + logger.info("No async task SDK injected; skipping result-entity persistence (platformless local run).") + return + # Best-effort: the eval has already succeeded and the full bundle is saved, so a transient + # entity-store error must not fail the job — the record is re-derivable from the bundle. Log + # loudly and move on. (`save` is create-or-update, so a re-run of the same job id is idempotent.) + try: + run_sync(lambda: client.save(entity)) + except Exception: + logger.exception( + "Failed to persist result entity %r in workspace %r; the result bundle is still saved", + entity.name, + entity.workspace, + ) + + +def persist_agent_eval_result( + result: AgentEvalResult, + *, + target: Target | None, + ctx: JobContext, + bundle_ref: str, + async_sdk: AsyncNeMoPlatform | None, +) -> None: + """Persist an ``AgentEvalJob`` run as an :class:`AgentEvalResultEntity` (aggregate scores rollup).""" + if ctx.job_id is None: + logger.info("No job id (platformless local run); skipping result-entity persistence.") + return + target_kind, target_name, target_url = _agent_target_fields(target) + entity = AgentEvalResultEntity( + name=ctx.job_id, + workspace=ctx.workspace, + job_id=ctx.job_id, + target_kind=target_kind, + target_name=target_name, + target_url=target_url, + scores=result.summary.scores, + bundle_ref=bundle_ref, + ) + _persist(entity, async_sdk=async_sdk) + + +def persist_evaluate_result( + result: EvaluationResult | BenchmarkEvaluationResult, + *, + target: Model | Agent | None, + dataset_ref: str | None, + metric_types: list[str], + ctx: JobContext, + bundle_ref: str, + async_sdk: AsyncNeMoPlatform | None, +) -> None: + """Persist an ``EvaluateJob`` (row-eval) run as an :class:`EvaluateResultEntity` (aggregates).""" + if ctx.job_id is None: + logger.info("No job id (platformless local run); skipping result-entity persistence.") + return + target_kind, target_name, target_url = _row_target_fields(target) + entity = EvaluateResultEntity( + name=ctx.job_id, + workspace=ctx.workspace, + job_id=ctx.job_id, + target_kind=target_kind, + target_name=target_name, + target_url=target_url, + scores=result.aggregate_scores, + bundle_ref=bundle_ref, + dataset_ref=dataset_ref, + metric_types=metric_types, + ) + _persist(entity, async_sdk=async_sdk) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/metric_resources.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/metric_resources.py index 7854e468c4..38a1645359 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/metric_resources.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/metric_resources.py @@ -26,6 +26,16 @@ from nemo_platform_plugin.schema import Page +def _list_params(page: int, page_size: int, sort: str | None, metric_type: str | None) -> dict[str, str | int]: + """Build the list query string: paging/sort + the route's ``filter[metric_type]`` trait filter.""" + params: dict[str, str | int] = {"page": page, "page_size": page_size} + if sort is not None: + params["sort"] = sort + if metric_type is not None: + params["filter[metric_type]"] = metric_type + return params + + def _metric_inline( metric: RuntimeMetric | MetricBundle, metric_bundle_packager: MetricBundlePackager | None, @@ -91,11 +101,19 @@ def retrieve(self, name: str, *, workspace: str | None = None) -> Metric: response.raise_for_status() return Metric.model_validate(response.json()) - def list(self, *, workspace: str | None = None, page: int = 1, page_size: int = 100) -> Page[Metric]: - """List stored metrics in a workspace.""" + def list( + self, + *, + workspace: str | None = None, + page: int = 1, + page_size: int = 100, + sort: str | None = None, + metric_type: str | None = None, + ) -> Page[Metric]: + """List stored metrics in a workspace, optionally filtered by metric type.""" response = self._http_client.get( self._collection_url(workspace), - params={"page": page, "page_size": page_size}, + params=_list_params(page, page_size, sort, metric_type), headers=self._headers(), timeout=self._platform.timeout, ) @@ -159,11 +177,19 @@ async def retrieve(self, name: str, *, workspace: str | None = None) -> Metric: response.raise_for_status() return Metric.model_validate(response.json()) - async def list(self, *, workspace: str | None = None, page: int = 1, page_size: int = 100) -> Page[Metric]: - """List stored metrics in a workspace.""" + async def list( + self, + *, + workspace: str | None = None, + page: int = 1, + page_size: int = 100, + sort: str | None = None, + metric_type: str | None = None, + ) -> Page[Metric]: + """List stored metrics in a workspace, optionally filtered by metric type.""" response = await self._http_client.get( self._collection_url(workspace), - params={"page": page, "page_size": page_size}, + params=_list_params(page, page_size, sort, metric_type), headers=self._headers(), timeout=self._platform.timeout, ) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py index fd4528071d..6136fc06f6 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py @@ -23,6 +23,12 @@ AsyncEvaluatorMetricsResource, EvaluatorMetricsResource, ) +from nemo_evaluator.sdk.result_resources import ( + AsyncEvaluatorAgentEvalResultsResource, + AsyncEvaluatorEvalResultsResource, + EvaluatorAgentEvalResultsResource, + EvaluatorEvalResultsResource, +) from nemo_evaluator.sdk.types import ( PluginDatasetInput, RunConfig, @@ -53,6 +59,8 @@ def __init__(self, platform: NeMoPlatform) -> None: self._http_client = platform._client self._executor = _SyncEvaluatorPluginExecutor(platform=platform) self.metrics = EvaluatorMetricsResource(platform) + self.agent_eval_results = EvaluatorAgentEvalResultsResource(platform) + self.eval_results = EvaluatorEvalResultsResource(platform) def plugin_status(self) -> dict[str, object]: """Return evaluator plugin health information from the service.""" @@ -219,6 +227,8 @@ def __init__(self, platform: AsyncNeMoPlatform) -> None: self._http_client = platform._client self._executor = _AsyncEvaluatorPluginExecutor(platform=platform) self.metrics = AsyncEvaluatorMetricsResource(platform) + self.agent_eval_results = AsyncEvaluatorAgentEvalResultsResource(platform) + self.eval_results = AsyncEvaluatorEvalResultsResource(platform) async def plugin_status(self) -> dict[str, object]: """Return evaluator plugin health information from the service.""" diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/result_resources.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/result_resources.py new file mode 100644 index 0000000000..58537cdda4 --- /dev/null +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/result_resources.py @@ -0,0 +1,273 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SDK resources for reading persisted eval results. + +Mounted as ``client.evaluator.agent_eval_results`` and ``client.evaluator.eval_results``. Results are +written by the jobs (not the SDK), so these resources are read-only: ``retrieve`` / ``list`` / +``delete`` against the evaluator service's ``/agent-eval-results`` and ``/eval-results`` routes, +returning the :class:`AgentEvalResult` / :class:`EvaluateResult` API DTOs. + +``list`` mirrors the routes' trait filtering: equality filters on the persisted traits (``job_id``, +``target_kind``, ``target_name``, and ``dataset_ref`` for row results) are sent as the route's +``filter[field]=value`` query params. (Datetime-range filtering, which the routes also support, needs +a richer operator shape and isn't surfaced here yet.) +""" + +from __future__ import annotations + +from typing import ClassVar, Generic, TypeVar +from urllib.parse import quote + +from nemo_evaluator.api.schemas import AgentEvalResult, EvaluateResult +from nemo_evaluator.sdk import http_utils +from nemo_platform import AsyncNeMoPlatform, NeMoPlatform +from nemo_platform_plugin.schema import Page, PaginationData + +_ResultT = TypeVar("_ResultT", AgentEvalResult, EvaluateResult) + + +def _query_params(page: int, page_size: int, sort: str | None, filters: dict[str, str | None]) -> dict[str, str | int]: + """Build the list query string: paging/sort + the route's ``filter[field]=value`` trait filters.""" + params: dict[str, str | int] = {"page": page, "page_size": page_size} + if sort is not None: + params["sort"] = sort + for field, value in filters.items(): + if value is not None: + params[f"filter[{field}]"] = value + return params + + +def _to_page(payload: dict, model: type[_ResultT], sort: str | None) -> Page[_ResultT]: + """Rebuild a typed ``Page`` from the route's JSON, deserializing each item as ``model``.""" + pagination = payload["pagination"] + return Page( + data=[model.model_validate(item) for item in payload["data"]], + pagination=PaginationData( + page=pagination["page"], + page_size=pagination["page_size"], + current_page_size=pagination["current_page_size"], + total_pages=pagination["total_pages"], + total_results=pagination["total_results"], + ), + sort=sort, + filter=None, + ) + + +class _SyncResultsResource(Generic[_ResultT]): + """Read-only sync resource for one result collection. Concrete subclasses declare ``list``.""" + + _collection: ClassVar[str] + _model: type[_ResultT] + + def __init__(self, platform: NeMoPlatform) -> None: + self._platform = platform + self._http_client = platform._client + + def _headers(self) -> dict[str, str]: + return http_utils.platform_default_headers(self._platform) + + def _collection_url(self, workspace: str | None) -> str: + return http_utils.url(self._platform, f"/v2/workspaces/{{workspace}}/{self._collection}", workspace) + + def _item_url(self, name: str, workspace: str | None) -> str: + return http_utils.url( + self._platform, f"/v2/workspaces/{{workspace}}/{self._collection}/{quote(name, safe='')}", workspace + ) + + def retrieve(self, name: str, *, workspace: str | None = None) -> _ResultT: + """Get a result record by name (the producing job's id).""" + response = self._http_client.get( + self._item_url(name, workspace), headers=self._headers(), timeout=self._platform.timeout + ) + response.raise_for_status() + return self._model.model_validate(response.json()) + + def _list( + self, *, workspace: str | None, page: int, page_size: int, sort: str | None, filters: dict[str, str | None] + ) -> Page[_ResultT]: + response = self._http_client.get( + self._collection_url(workspace), + params=_query_params(page, page_size, sort, filters), + headers=self._headers(), + timeout=self._platform.timeout, + ) + response.raise_for_status() + return _to_page(response.json(), self._model, sort) + + def delete(self, name: str, *, workspace: str | None = None) -> None: + """Delete a result record by name.""" + response = self._http_client.delete( + self._item_url(name, workspace), headers=self._headers(), timeout=self._platform.timeout + ) + response.raise_for_status() + + +class _AsyncResultsResource(Generic[_ResultT]): + """Read-only async resource for one result collection. Concrete subclasses declare ``list``.""" + + _collection: ClassVar[str] + _model: type[_ResultT] + + def __init__(self, platform: AsyncNeMoPlatform) -> None: + self._platform = platform + self._http_client = platform._client + + def _headers(self) -> dict[str, str]: + return http_utils.platform_default_headers(self._platform) + + def _collection_url(self, workspace: str | None) -> str: + return http_utils.url(self._platform, f"/v2/workspaces/{{workspace}}/{self._collection}", workspace) + + def _item_url(self, name: str, workspace: str | None) -> str: + return http_utils.url( + self._platform, f"/v2/workspaces/{{workspace}}/{self._collection}/{quote(name, safe='')}", workspace + ) + + async def retrieve(self, name: str, *, workspace: str | None = None) -> _ResultT: + """Get a result record by name (the producing job's id).""" + response = await self._http_client.get( + self._item_url(name, workspace), headers=self._headers(), timeout=self._platform.timeout + ) + response.raise_for_status() + return self._model.model_validate(response.json()) + + async def _list( + self, *, workspace: str | None, page: int, page_size: int, sort: str | None, filters: dict[str, str | None] + ) -> Page[_ResultT]: + response = await self._http_client.get( + self._collection_url(workspace), + params=_query_params(page, page_size, sort, filters), + headers=self._headers(), + timeout=self._platform.timeout, + ) + response.raise_for_status() + return _to_page(response.json(), self._model, sort) + + async def delete(self, name: str, *, workspace: str | None = None) -> None: + """Delete a result record by name.""" + response = await self._http_client.delete( + self._item_url(name, workspace), headers=self._headers(), timeout=self._platform.timeout + ) + response.raise_for_status() + + +class EvaluatorAgentEvalResultsResource(_SyncResultsResource[AgentEvalResult]): + """Sync resource mounted as ``client.evaluator.agent_eval_results``.""" + + _collection = "agent-eval-results" + _model = AgentEvalResult + + def list( + self, + *, + workspace: str | None = None, + page: int = 1, + page_size: int = 100, + sort: str | None = None, + job_id: str | None = None, + target_kind: str | None = None, + target_name: str | None = None, + ) -> Page[AgentEvalResult]: + """List agent-eval results, optionally filtered by job/target traits.""" + return self._list( + workspace=workspace, + page=page, + page_size=page_size, + sort=sort, + filters={"job_id": job_id, "target_kind": target_kind, "target_name": target_name}, + ) + + +class EvaluatorEvalResultsResource(_SyncResultsResource[EvaluateResult]): + """Sync resource mounted as ``client.evaluator.eval_results``.""" + + _collection = "eval-results" + _model = EvaluateResult + + def list( + self, + *, + workspace: str | None = None, + page: int = 1, + page_size: int = 100, + sort: str | None = None, + job_id: str | None = None, + target_kind: str | None = None, + target_name: str | None = None, + dataset_ref: str | None = None, + ) -> Page[EvaluateResult]: + """List row-eval results, optionally filtered by job/target/dataset traits.""" + return self._list( + workspace=workspace, + page=page, + page_size=page_size, + sort=sort, + filters={ + "job_id": job_id, + "target_kind": target_kind, + "target_name": target_name, + "dataset_ref": dataset_ref, + }, + ) + + +class AsyncEvaluatorAgentEvalResultsResource(_AsyncResultsResource[AgentEvalResult]): + """Async resource mounted as ``client.evaluator.agent_eval_results``.""" + + _collection = "agent-eval-results" + _model = AgentEvalResult + + async def list( + self, + *, + workspace: str | None = None, + page: int = 1, + page_size: int = 100, + sort: str | None = None, + job_id: str | None = None, + target_kind: str | None = None, + target_name: str | None = None, + ) -> Page[AgentEvalResult]: + """List agent-eval results, optionally filtered by job/target traits.""" + return await self._list( + workspace=workspace, + page=page, + page_size=page_size, + sort=sort, + filters={"job_id": job_id, "target_kind": target_kind, "target_name": target_name}, + ) + + +class AsyncEvaluatorEvalResultsResource(_AsyncResultsResource[EvaluateResult]): + """Async resource mounted as ``client.evaluator.eval_results``.""" + + _collection = "eval-results" + _model = EvaluateResult + + async def list( + self, + *, + workspace: str | None = None, + page: int = 1, + page_size: int = 100, + sort: str | None = None, + job_id: str | None = None, + target_kind: str | None = None, + target_name: str | None = None, + dataset_ref: str | None = None, + ) -> Page[EvaluateResult]: + """List row-eval results, optionally filtered by job/target/dataset traits.""" + return await self._list( + workspace=workspace, + page=page, + page_size=page_size, + sort=sort, + filters={ + "job_id": job_id, + "target_kind": target_kind, + "target_name": target_name, + "dataset_ref": dataset_ref, + }, + ) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/service.py b/plugins/nemo-evaluator/src/nemo_evaluator/service.py index 0356ff8489..f30890a182 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/service.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/service.py @@ -9,6 +9,7 @@ from fastapi import APIRouter from nemo_evaluator.api.v2 import metrics as metrics_routes +from nemo_evaluator.api.v2 import results as results_routes from nemo_evaluator.authz import scope from nemo_evaluator.core import say_hello from nemo_evaluator.jobs.agent_evaluate import AgentEvalJob @@ -86,6 +87,20 @@ async def healthz() -> dict[str, object]: description="Stored metric (metric bundle) CRUD routes.", prefix="/v2/workspaces/{workspace}", ), + RouterSpec( + # list/get/delete /apis/evaluator/v2/workspaces/{workspace}/agent-eval-results. + router=results_routes.agent_eval_results_router, + tag="Evaluator Plugin Agent Eval Results Routes", + description="Queryable agent-evaluation result records.", + prefix="/v2/workspaces/{workspace}", + ), + RouterSpec( + # list/get/delete /apis/evaluator/v2/workspaces/{workspace}/eval-results. + router=results_routes.evaluate_results_router, + tag="Evaluator Plugin Eval Results Routes", + description="Queryable (row) evaluation result records.", + prefix="/v2/workspaces/{workspace}", + ), ] diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/tasks/runner.py b/plugins/nemo-evaluator/src/nemo_evaluator/tasks/runner.py index 5bb21a9246..cbbba9be8d 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/tasks/runner.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/tasks/runner.py @@ -15,7 +15,7 @@ from types import FrameType from nemo_platform_plugin.job import NemoJob -from nemo_platform_plugin.sdk_provider import get_task_sdk +from nemo_platform_plugin.sdk_provider import get_async_task_sdk, get_task_sdk from nemo_platform_plugin.tasks.dispatcher import run_task logger = logging.getLogger(__name__) @@ -33,11 +33,16 @@ def run_task_main(job_cls: type[NemoJob], *, service_name: str) -> int: """Build the task SDK and dispatch to ``job_cls``; return a process exit code. Returns :data:`SDK_INITIALIZATION_EXIT_CODE` if the task SDK can't be built. + + Builds both a sync and an async task SDK (same identity): the sync handle is what most jobs use, + and the async one lets a synchronous ``run`` drive async helpers (e.g. persisting a result entity + through the async entity-store client) without fabricating its own client. """ signal.signal(signal.SIGTERM, _shutdown_handler) try: sdk = get_task_sdk(service_name) + async_sdk = get_async_task_sdk(service_name) except Exception: logger.exception("Failed to build task SDK for %s", service_name) return SDK_INITIALIZATION_EXIT_CODE - return run_task(job_cls, sdk=sdk) + return run_task(job_cls, sdk=sdk, async_sdk=async_sdk) diff --git a/plugins/nemo-evaluator/tests/api/service/test_result_service.py b/plugins/nemo-evaluator/tests/api/service/test_result_service.py new file mode 100644 index 0000000000..b0b7f37874 --- /dev/null +++ b/plugins/nemo-evaluator/tests/api/service/test_result_service.py @@ -0,0 +1,156 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Service-level tests for ResultService (list/get/delete over the two result entity types). + +The entity store is an in-memory fake keyed by ``(entity_type, workspace, name)`` so the two +collections (agent-eval vs row-eval) are proven not to collide. +""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest +from nemo_evaluator.api.schemas import AgentEvalResult, EvaluateResult +from nemo_evaluator.api.service.result_service import ResultService +from nemo_evaluator.entities import AgentEvalResultEntity, EvaluateResultEntity +from nemo_evaluator_sdk.values.results import AggregatedMetricResult +from nemo_platform_plugin.entities import EntityBase, EntityNotFoundError, ListResponse, PaginationInfo + + +class _FakeEntityClient: + """In-memory store keyed by (entity_type, workspace, name).""" + + def __init__(self) -> None: + self.entities: dict[tuple[str, str, str], EntityBase] = {} + + def seed(self, entity: EntityBase) -> EntityBase: + now = datetime.now(timezone.utc) + entity._id = f"{entity.__entity_type__}-{entity.name}" + entity._created_at = now + entity._updated_at = now + self.entities[(entity.__entity_type__, entity.workspace, entity.name)] = entity + return entity + + async def get(self, entity_cls, *, workspace, name): + key = (entity_cls.__entity_type__, workspace, name) + if key not in self.entities: + raise EntityNotFoundError(f"{workspace}/{name} not found") + return self.entities[key] + + async def delete(self, entity_cls, name, *, workspace): + # Mirror the real EntityClient: raise EntityNotFoundError when absent. + key = (entity_cls.__entity_type__, workspace, name) + if key not in self.entities: + raise EntityNotFoundError(f"{workspace}/{name} not found") + del self.entities[key] + + async def list(self, entity_cls, *, workspace, filter_operation=None, sort=None, page=1, page_size=100): + items = [ + e for (etype, ws, _), e in self.entities.items() if etype == entity_cls.__entity_type__ and ws == workspace + ] + return ListResponse( + data=items, + pagination=PaginationInfo( + page=page, + page_size=page_size, + current_page_size=len(items), + total_pages=1, + total_results=len(items), + ), + ) + + +def _agent_entity(name: str, workspace: str = "default") -> AgentEvalResultEntity: + return AgentEvalResultEntity( + name=name, + workspace=workspace, + job_id=name, + target_kind="codex", + target_name="gpt-5.5", + target_url=None, + scores=AggregatedMetricResult(scores=[]), + bundle_ref=f"fileset://{workspace}/agent-eval-results#b", + ) + + +def _eval_entity(name: str, workspace: str = "default") -> EvaluateResultEntity: + return EvaluateResultEntity( + name=name, + workspace=workspace, + job_id=name, + target_kind="model", + target_name="m", + target_url=None, + scores=AggregatedMetricResult(scores=[]), + bundle_ref=f"fileset://{workspace}/eval-results#b", + dataset_ref=f"{workspace}/ds", + metric_types=["exact_match"], + ) + + +@pytest.fixture +def fake() -> _FakeEntityClient: + return _FakeEntityClient() + + +@pytest.fixture +def service(fake: _FakeEntityClient) -> ResultService: + return ResultService(fake) + + +async def test_list_returns_only_that_collection(service: ResultService, fake: _FakeEntityClient) -> None: + # Same name across both collections must not collide — they are distinct entity types. + fake.seed(_agent_entity("job-1")) + fake.seed(_eval_entity("job-1")) + fake.seed(_eval_entity("job-2")) + + agent_page = await service.list_agent_eval_results(workspace="default") + eval_page = await service.list_eval_results(workspace="default") + + assert {e.name for e in agent_page.data} == {"job-1"} + assert {e.name for e in eval_page.data} == {"job-1", "job-2"} + assert eval_page.pagination is not None + assert eval_page.pagination.total_results == 2 + + +async def test_get_returns_typed_dto(service: ResultService, fake: _FakeEntityClient) -> None: + fake.seed(_eval_entity("job-9")) + + got = await service.get_eval_result("default", "job-9") + + # The service maps the stored entity to the API DTO (so id/created_at round-trip on the wire). + assert isinstance(got, EvaluateResult) + assert got.id == "evaluate_result-job-9" + assert got.created_at is not None + assert got.dataset_ref == "default/ds" + assert got.metric_types == ["exact_match"] + + +async def test_list_maps_entities_to_dtos(service: ResultService, fake: _FakeEntityClient) -> None: + fake.seed(_agent_entity("job-1")) + + page = await service.list_agent_eval_results(workspace="default") + + assert all(isinstance(item, AgentEvalResult) for item in page.data) + assert page.data[0].job_id == "job-1" + + +async def test_get_returns_none_when_missing(service: ResultService) -> None: + assert await service.get_agent_eval_result("default", "nope") is None + assert await service.get_eval_result("default", "nope") is None + + +async def test_delete_removes_only_matching_type(service: ResultService, fake: _FakeEntityClient) -> None: + fake.seed(_agent_entity("job-1")) + fake.seed(_eval_entity("job-1")) + + assert await service.delete_agent_eval_result("default", "job-1") is True + # The same-named row-eval result is a different type and must survive. + assert await service.get_agent_eval_result("default", "job-1") is None + assert await service.get_eval_result("default", "job-1") is not None + + +async def test_delete_returns_false_when_missing(service: ResultService) -> None: + assert await service.delete_eval_result("default", "nope") is False diff --git a/plugins/nemo-evaluator/tests/api/v2/test_metrics_routes.py b/plugins/nemo-evaluator/tests/api/v2/test_metrics_routes.py index 3a9d817489..879ed46845 100644 --- a/plugins/nemo-evaluator/tests/api/v2/test_metrics_routes.py +++ b/plugins/nemo-evaluator/tests/api/v2/test_metrics_routes.py @@ -166,3 +166,25 @@ def test_create_then_delete(client: TestClient) -> None: def test_delete_missing_returns_404(client: TestClient) -> None: assert client.delete(f"{_BASE}/nope").status_code == 404 + + +def test_metric_filter_translates_custom_fields_to_data_namespace() -> None: + # metric_type/description are custom (data.*) fields; base columns (name) pass through. Without + # this translation the entity store can't resolve the field and 500s (matches the result filters). + from nemo_evaluator.api.schemas import MetricFilter + from nemo_platform_plugin.api.filter import ComparisonOperation, FilterOperator, LogicalOperation + + assert MetricFilter._get_entity_field_map() == { + "metric_type": "data.metric_type", + "description": "data.description", + } + op = LogicalOperation( + operator=FilterOperator.AND, + operations=[ + ComparisonOperation(field="metric_type", operator=FilterOperator.EQ, value="exact-match"), + ComparisonOperation(field="name", operator=FilterOperator.EQ, value="m"), + ], + ) + assert MetricFilter.translate_operation(op).to_dict() == { + "$and": [{"data.metric_type": {"$eq": "exact-match"}}, {"name": {"$eq": "m"}}] + } diff --git a/plugins/nemo-evaluator/tests/api/v2/test_results_routes.py b/plugins/nemo-evaluator/tests/api/v2/test_results_routes.py new file mode 100644 index 0000000000..02da07a415 --- /dev/null +++ b/plugins/nemo-evaluator/tests/api/v2/test_results_routes.py @@ -0,0 +1,183 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""HTTP route-level tests for the eval-result read endpoints. + +Drives the real FastAPI routers + ResultService through a TestClient with an in-memory entity store. +Covers route wiring, the get_result_service dependency, and status-code mapping (200/204/404), plus +that the two collections (agent-eval vs row-eval) stay separate. +""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from nemo_evaluator.api.dependencies import get_result_service +from nemo_evaluator.api.service.result_service import ResultService +from nemo_evaluator.api.v2 import results as results_routes +from nemo_evaluator.entities import AgentEvalResultEntity, EvaluateResultEntity +from nemo_evaluator_sdk.values.results import AggregatedMetricResult +from nemo_platform_plugin.entities import EntityBase, EntityNotFoundError, ListResponse, PaginationInfo + + +class _FakeEntityClient: + def __init__(self) -> None: + self.entities: dict[tuple[str, str, str], EntityBase] = {} + + def seed(self, entity: EntityBase) -> EntityBase: + now = datetime.now(timezone.utc) + entity._id = f"{entity.__entity_type__}-{entity.name}" + entity._created_at = now + entity._updated_at = now + self.entities[(entity.__entity_type__, entity.workspace, entity.name)] = entity + return entity + + async def get(self, entity_cls, *, workspace, name): + key = (entity_cls.__entity_type__, workspace, name) + if key not in self.entities: + raise EntityNotFoundError(f"{workspace}/{name} not found") + return self.entities[key] + + async def delete(self, entity_cls, name, *, workspace): + # Mirror the real EntityClient: raise EntityNotFoundError when absent. + key = (entity_cls.__entity_type__, workspace, name) + if key not in self.entities: + raise EntityNotFoundError(f"{workspace}/{name} not found") + del self.entities[key] + + async def list(self, entity_cls, *, workspace, filter_operation=None, sort=None, page=1, page_size=100): + items = [ + e for (etype, ws, _), e in self.entities.items() if etype == entity_cls.__entity_type__ and ws == workspace + ] + return ListResponse( + data=items, + pagination=PaginationInfo( + page=page, + page_size=page_size, + current_page_size=len(items), + total_pages=1, + total_results=len(items), + ), + ) + + +def _agent_entity(name: str) -> AgentEvalResultEntity: + return AgentEvalResultEntity( + name=name, + workspace="default", + job_id=name, + target_kind="codex", + target_name="gpt-5.5", + target_url=None, + scores=AggregatedMetricResult(scores=[]), + bundle_ref="fileset://default/agent-eval-results#b", + ) + + +def _eval_entity(name: str) -> EvaluateResultEntity: + return EvaluateResultEntity( + name=name, + workspace="default", + job_id=name, + target_kind="model", + target_name="m", + target_url=None, + scores=AggregatedMetricResult(scores=[]), + bundle_ref="fileset://default/eval-results#b", + dataset_ref="default/ds", + metric_types=["exact_match"], + ) + + +@pytest.fixture +def fake() -> _FakeEntityClient: + return _FakeEntityClient() + + +@pytest.fixture +def client(fake: _FakeEntityClient) -> TestClient: + app = FastAPI() + prefix = "/v2/workspaces/{workspace}" + app.include_router(results_routes.agent_eval_results_router, prefix=prefix) + app.include_router(results_routes.evaluate_results_router, prefix=prefix) + service = ResultService(fake) + app.dependency_overrides[get_result_service] = lambda: service + return TestClient(app) + + +_AGENT = "/v2/workspaces/default/agent-eval-results" +_EVAL = "/v2/workspaces/default/eval-results" + + +def test_list_agent_eval_results(client: TestClient, fake: _FakeEntityClient) -> None: + fake.seed(_agent_entity("job-1")) + fake.seed(_agent_entity("job-2")) + + resp = client.get(_AGENT) + assert resp.status_code == 200 + body = resp.json() + assert {e["name"] for e in body["data"]} == {"job-1", "job-2"} + assert body["pagination"]["total_results"] == 2 + + +def test_get_eval_result_returns_typed_payload(client: TestClient, fake: _FakeEntityClient) -> None: + fake.seed(_eval_entity("job-9")) + + resp = client.get(f"{_EVAL}/job-9") + assert resp.status_code == 200 + body = resp.json() + assert body["job_id"] == "job-9" + assert body["dataset_ref"] == "default/ds" + assert body["metric_types"] == ["exact_match"] + + +def test_get_missing_returns_404(client: TestClient) -> None: + assert client.get(f"{_AGENT}/nope").status_code == 404 + assert client.get(f"{_EVAL}/nope").status_code == 404 + + +def test_delete_then_get_404(client: TestClient, fake: _FakeEntityClient) -> None: + fake.seed(_agent_entity("job-1")) + + assert client.delete(f"{_AGENT}/job-1").status_code == 204 + assert client.get(f"{_AGENT}/job-1").status_code == 404 + + +def test_delete_missing_returns_404(client: TestClient) -> None: + assert client.delete(f"{_EVAL}/nope").status_code == 404 + + +def test_filter_translates_custom_fields_to_data_namespace() -> None: + # Custom (non-base) trait fields must be rewritten to data.* for the entity store; base columns + # (workspace, created_at) pass through. The plain Filter does no translation and the store 500s. + from nemo_evaluator.api.v2.results import EvaluateResultFilter + from nemo_platform_plugin.api.filter import ComparisonOperation, FilterOperator, LogicalOperation + + assert EvaluateResultFilter._get_entity_field_map() == { + "job_id": "data.job_id", + "target_kind": "data.target_kind", + "target_name": "data.target_name", + "dataset_ref": "data.dataset_ref", + } + op = LogicalOperation( + operator=FilterOperator.AND, + operations=[ + ComparisonOperation(field="job_id", operator=FilterOperator.EQ, value="j1"), + ComparisonOperation(field="workspace", operator=FilterOperator.EQ, value="default"), + ], + ) + assert EvaluateResultFilter.translate_operation(op).to_dict() == { + "$and": [{"data.job_id": {"$eq": "j1"}}, {"workspace": {"$eq": "default"}}] + } + + +def test_collections_do_not_collide(client: TestClient, fake: _FakeEntityClient) -> None: + # Same name in both collections: each endpoint sees only its own type. + fake.seed(_agent_entity("shared")) + fake.seed(_eval_entity("shared")) + + assert client.get(f"{_AGENT}/shared").json()["target_kind"] == "codex" + assert client.get(f"{_EVAL}/shared").json()["dataset_ref"] == "default/ds" diff --git a/plugins/nemo-evaluator/tests/integration/test_agent_evaluate_job.py b/plugins/nemo-evaluator/tests/integration/test_agent_evaluate_job.py index 92effc21ae..55d4a93b35 100644 --- a/plugins/nemo-evaluator/tests/integration/test_agent_evaluate_job.py +++ b/plugins/nemo-evaluator/tests/integration/test_agent_evaluate_job.py @@ -326,6 +326,15 @@ def test_submit_to_subprocess_backend_runs_agent_eval(subprocess_platform: str) job = wait_for_platform_job(client, job_name, WORKSPACE, timeout=480) assert job.status == "completed", f"job {job_name} ended {job.status!r}: {getattr(job, 'status_details', None)}" + # Persistence: run() wrote a queryable result record, retrievable via the typed SDK resource + # (client.evaluator.agent_eval_results -> the /agent-eval-results route). The record is keyed by + # the job id and denormalizes the target it ran against; the full bundle lives in bundle_ref. + result = client.evaluator.agent_eval_results.retrieve(job_name, workspace=WORKSPACE) + assert result.job_id == job_name + assert (result.target_kind, result.target_name) == ("codex", CODEX_MODEL) + assert result.bundle_ref + assert result.created_at is not None + @requires_codex @pytest.mark.timeout(600) @@ -413,6 +422,15 @@ def test_submit_model_target_under_auth_forwards_identity_to_igw(auth_subprocess job = wait_for_platform_job(sdk, job_name, WORKSPACE, timeout=360) assert job.status == "completed", f"job {job_name} ended {job.status!r}: {getattr(job, 'status_details', None)}" + # Persistence under auth: the result-entity write goes through the job's async task SDK + # (get_async_task_sdk) as service:evaluator on-behalf-of the creator. A retrievable record here + # proves that delegated identity actually authorized the entity write end-to-end (not just the + # IGW inference call) — the key validation of the async task-SDK identity parity. + result = sdk.evaluator.agent_eval_results.retrieve(job_name, workspace=WORKSPACE) + assert result.job_id == job_name + assert (result.target_kind, result.target_name) == ("model", model_name) + assert result.bundle_ref + @pytest.mark.timeout(600) @pytest.mark.xfail( diff --git a/plugins/nemo-evaluator/tests/integration/test_evaluate_job.py b/plugins/nemo-evaluator/tests/integration/test_evaluate_job.py new file mode 100644 index 0000000000..8af2e1d605 --- /dev/null +++ b/plugins/nemo-evaluator/tests/integration/test_evaluate_job.py @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Submit-path integration test for the row ``EvaluateJob``, focused on result persistence. + +Shares the evaluator-plugin integration harness (conftest's session-scoped ``subprocess_platform``) +and the ``RUN_AGENT_EVAL_INTEGRATION`` opt-in. Submits an *offline* metric eval — inline dataset, no +model target / IGW / codex — so the only requirement is the host subprocess backend. Asserts the run +persisted a queryable ``EvaluateResult`` retrievable via ``client.evaluator.eval_results``, covering +the row-eval half of result persistence (the agent-eval half lives in ``test_agent_evaluate_job.py``). +""" + +from __future__ import annotations + +import os + +import pytest +from nemo_evaluator.jobs.evaluate import EvaluateInputSpec, EvaluateJob +from nemo_evaluator.shared.metric_bundles.bundles import bundle_metric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager +from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric +from nemo_platform import NeMoPlatform +from nemo_platform_plugin.scheduler import NemoJobScheduler +from nmp.testing.e2e import wait_for_platform_job + +#: Opt-in: shares the evaluator-plugin integration opt-in (spins a real ``nemo services`` platform). +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif( + not os.environ.get("RUN_AGENT_EVAL_INTEGRATION"), + reason="opt-in; set RUN_AGENT_EVAL_INTEGRATION=1 to run (spins real nemo services platforms)", + ), +] + +WORKSPACE = "default" + + +def _offline_exact_match_spec() -> dict: + """An offline row-eval: a built-in metric scores inline rows that already carry expected/output. + + ExactMatch is a built-in (importable in the submit-backend subprocess), so a cloudpickle bundle + round-trips fine there. No target → the dataset's ``model_output`` is scored directly. + """ + bundle = bundle_metric( + ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.model_output}}"), + CloudpickleMetricBundlePackager(), + ) + return EvaluateInputSpec.model_validate( + { + "metrics": [bundle.model_dump(mode="json")], + "dataset": [ + {"expected": "blue", "model_output": "blue"}, + {"expected": "Jupiter", "model_output": "Jupiter"}, + ], + } + ).model_dump(mode="json") + + +@pytest.mark.timeout(600) +def test_submit_offline_row_eval_persists_result(subprocess_platform: str) -> None: + # dim: submit x subprocess backend, row (EvaluateJob) path. The jobs service compiles + runs + # EvaluateJob.run() as a host subprocess; run() writes an EvaluateResult through the async task + # SDK + entity store. Offline (no target/IGW/codex): the dataset already carries the outputs. + client = NeMoPlatform(base_url=subprocess_platform, max_retries=2) + client.workspaces.create(name=WORKSPACE, exist_ok=True) + + response = NemoJobScheduler().submit_remote( + EvaluateJob, _offline_exact_match_spec(), base_url=subprocess_platform, workspace=WORKSPACE, profile="default" + ) + job_name = response.get("name") or response.get("id") + assert job_name, f"submit response carried no job name/id: {response}" + + job = wait_for_platform_job(client, job_name, WORKSPACE, timeout=480) + assert job.status == "completed", f"job {job_name} ended {job.status!r}: {getattr(job, 'status_details', None)}" + + # Persistence: run() wrote a queryable EvaluateResult, retrievable via the typed SDK resource + # (client.evaluator.eval_results -> the /eval-results route). Row-eval records the metric types + # applied; an inline dataset has no dataset_ref, and an offline run has no target. + result = client.evaluator.eval_results.retrieve(job_name, workspace=WORKSPACE) + assert result.job_id == job_name + assert result.metric_types == ["exact-match"] + assert result.dataset_ref is None + assert result.target_kind is None + assert result.bundle_ref + assert result.created_at is not None + + # And it's discoverable in the workspace listing. + listing = client.evaluator.eval_results.list(workspace=WORKSPACE) + assert any(r.job_id == job_name for r in listing.data) + + # Server-side trait filtering narrows the listing (proves the SDK's filter[...] params reach the + # entity store — the in-memory unit fakes can't exercise this). + by_job = client.evaluator.eval_results.list(workspace=WORKSPACE, job_id=job_name) + assert [r.job_id for r in by_job.data] == [job_name] + assert client.evaluator.eval_results.list(workspace=WORKSPACE, job_id="no-such-job").data == [] diff --git a/plugins/nemo-evaluator/tests/integration/test_metric_filtering.py b/plugins/nemo-evaluator/tests/integration/test_metric_filtering.py new file mode 100644 index 0000000000..ed351085eb --- /dev/null +++ b/plugins/nemo-evaluator/tests/integration/test_metric_filtering.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Integration test for metric-type filtering through the SDK against a real platform. + +Verifies the route's custom-field filter actually works end-to-end: ``metric_type`` is a ``data.*`` +entity field, so without the ``DataFilter`` translation the entity store 500s. Pure CRUD (create + +list) — no codex/IGW — so it only needs the host subprocess backend. Shares the evaluator-plugin +integration opt-in (``RUN_AGENT_EVAL_INTEGRATION``) and the session-scoped ``subprocess_platform``. +""" + +from __future__ import annotations + +import os +import uuid + +import pytest +from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric +from nemo_evaluator_sdk.metrics.string_check import StringCheckMetric +from nemo_platform import NeMoPlatform + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif( + not os.environ.get("RUN_AGENT_EVAL_INTEGRATION"), + reason="opt-in; set RUN_AGENT_EVAL_INTEGRATION=1 to run (spins real nemo services platforms)", + ), +] + +WORKSPACE = "default" + + +def _unique(prefix: str) -> str: + return f"{prefix}-{uuid.uuid4().hex[:8]}" + + +@pytest.mark.timeout(300) +def test_metric_type_filter_narrows_listing(subprocess_platform: str) -> None: + client = NeMoPlatform(base_url=subprocess_platform, max_retries=2) + client.workspaces.create(name=WORKSPACE, exist_ok=True) + + exact = _unique("exact") + strcheck = _unique("strcheck") + client.evaluator.metrics.create( + exact, metric=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") + ) + client.evaluator.metrics.create( + strcheck, + metric=StringCheckMetric( + operation="contains", left_template="{{sample.output_text}}", right_template="{{item.phrase}}" + ), + ) + + # Server-side filter on metric_type (a data.* field) must narrow the listing — the whole point of + # the DataFilter translation. Robust to other metrics the shared platform may hold. + exact_only = client.evaluator.metrics.list(workspace=WORKSPACE, metric_type="exact-match") + names = {m.name for m in exact_only.data} + assert exact in names + assert strcheck not in names + assert all(m.metric_type == "exact-match" for m in exact_only.data) + + assert client.evaluator.metrics.list(workspace=WORKSPACE, metric_type="no-such-type").data == [] diff --git a/plugins/nemo-evaluator/tests/sdk/test_metric_sdk_resources.py b/plugins/nemo-evaluator/tests/sdk/test_metric_sdk_resources.py index 75115e0934..b465225083 100644 --- a/plugins/nemo-evaluator/tests/sdk/test_metric_sdk_resources.py +++ b/plugins/nemo-evaluator/tests/sdk/test_metric_sdk_resources.py @@ -144,6 +144,20 @@ def test_sync_list_returns_data_items() -> None: assert {m.name for m in result.data} == {"a", "b"} +def test_sync_list_encodes_metric_type_filter_and_sort() -> None: + # metric_type is a custom (data.*) field; the SDK sends it as the route's filter[...] param so a + # caller can narrow by type without hand-building query strings. + http_client = MagicMock() + http_client.get.return_value = _response({"data": []}) + resource = EvaluatorMetricsResource(_platform(http_client)) + + resource.list(metric_type="exact-match", sort="-created_at") + + params = http_client.get.call_args.kwargs["params"] + assert params["filter[metric_type]"] == "exact-match" + assert params["sort"] == "-created_at" + + def test_sync_delete_issues_delete_request() -> None: http_client = MagicMock() http_client.delete.return_value = _response({}) diff --git a/plugins/nemo-evaluator/tests/sdk/test_result_sdk_resources.py b/plugins/nemo-evaluator/tests/sdk/test_result_sdk_resources.py new file mode 100644 index 0000000000..c931f76360 --- /dev/null +++ b/plugins/nemo-evaluator/tests/sdk/test_result_sdk_resources.py @@ -0,0 +1,183 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the client.evaluator.{agent_eval_results,eval_results} SDK resources. + +Drives the resources against a mocked HTTP client, asserting the URL they target and that the +response JSON is deserialized into the typed API DTO. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +from nemo_evaluator.api.schemas import AgentEvalResult, EvaluateResult +from nemo_evaluator.sdk.result_resources import ( + AsyncEvaluatorEvalResultsResource, + EvaluatorAgentEvalResultsResource, + EvaluatorEvalResultsResource, +) +from nemo_evaluator_sdk.values.results import AggregatedMetricResult + +_BASE = "http://localhost:8080/apis/evaluator/v2/workspaces/default" + + +def _agent_payload(name: str) -> dict[str, Any]: + now = datetime.now(timezone.utc) + return AgentEvalResult( + id=f"agent_eval_result-{name}", + name=name, + workspace="default", + job_id=name, + target_kind="codex", + target_name="gpt-5.5", + target_url=None, + scores=AggregatedMetricResult(scores=[]), + bundle_ref="fileset://default/agent-eval-results#b", + created_at=now, + updated_at=now, + ).model_dump(mode="json") + + +def _eval_payload(name: str) -> dict[str, Any]: + now = datetime.now(timezone.utc) + return EvaluateResult( + id=f"evaluate_result-{name}", + name=name, + workspace="default", + job_id=name, + target_kind="model", + target_name="m", + target_url="https://m.test/v1/chat/completions", + scores=AggregatedMetricResult(scores=[]), + bundle_ref="fileset://default/eval-results#b", + created_at=now, + updated_at=now, + dataset_ref="default/ds", + metric_types=["exact_match"], + ).model_dump(mode="json") + + +def _page(items: list[dict[str, Any]]) -> dict[str, Any]: + return { + "data": items, + "pagination": { + "page": 1, + "page_size": 100, + "current_page_size": len(items), + "total_pages": 1, + "total_results": len(items), + }, + } + + +def _response(payload: dict[str, Any]) -> MagicMock: + response = MagicMock() + response.json.return_value = payload + response.raise_for_status.return_value = None + return response + + +def _platform(http_client: Any) -> MagicMock: + platform = MagicMock() + platform._client = http_client + platform.base_url = "http://localhost:8080" + platform.workspace = "default" + platform.default_headers = {} + platform.timeout = 30 + return platform + + +# ---- sync ------------------------------------------------------------------ + + +def test_sync_retrieve_agent_eval_targets_item_url_and_parses_dto() -> None: + http_client = MagicMock() + http_client.get.return_value = _response(_agent_payload("job-1")) + resource = EvaluatorAgentEvalResultsResource(_platform(http_client)) + + result = resource.retrieve("job-1") + + assert isinstance(result, AgentEvalResult) + assert result.job_id == "job-1" + assert result.target_kind == "codex" + assert http_client.get.call_args[0][0] == f"{_BASE}/agent-eval-results/job-1" + + +def test_sync_list_eval_results_parses_dtos_and_targets_collection() -> None: + http_client = MagicMock() + http_client.get.return_value = _response(_page([_eval_payload("a"), _eval_payload("b")])) + resource = EvaluatorEvalResultsResource(_platform(http_client)) + + page = resource.list(sort="-created_at") + + assert {r.name for r in page.data} == {"a", "b"} + assert all(isinstance(r, EvaluateResult) for r in page.data) + assert page.data[0].dataset_ref == "default/ds" + assert page.pagination is not None and page.pagination.total_results == 2 + assert http_client.get.call_args[0][0] == f"{_BASE}/eval-results" + assert http_client.get.call_args.kwargs["params"]["sort"] == "-created_at" + + +def test_sync_list_encodes_trait_filters_as_bracket_params() -> None: + # The route filters via filter[field]=value bracket params; the SDK must encode them so a + # caller can narrow by job/target/dataset without hand-building query strings. + http_client = MagicMock() + http_client.get.return_value = _response(_page([])) + resource = EvaluatorEvalResultsResource(_platform(http_client)) + + resource.list(job_id="j1", target_kind="model", dataset_ref="ws/ds") + + params = http_client.get.call_args.kwargs["params"] + assert params["filter[job_id]"] == "j1" + assert params["filter[target_kind]"] == "model" + assert params["filter[dataset_ref]"] == "ws/ds" + # Unset filters are omitted entirely (no empty filter[...] keys). + assert "filter[target_name]" not in params + + +def test_sync_list_parses_payload_with_none_fields_omitted() -> None: + # Regression guard: the list route serializes with response_model_exclude_none, so an offline + # result (no target / inline dataset) arrives with target_*/dataset_ref *absent*. The DTO must + # still deserialize (those fields default to None) — a live round-trip caught this; this locks it. + item = _eval_payload("offline") + for dropped in ("target_kind", "target_name", "target_url", "dataset_ref"): + item.pop(dropped, None) + http_client = MagicMock() + http_client.get.return_value = _response(_page([item])) + resource = EvaluatorEvalResultsResource(_platform(http_client)) + + (result,) = resource.list().data + + assert result.target_kind is None + assert result.target_name is None + assert result.target_url is None + assert result.dataset_ref is None + assert result.metric_types == ["exact_match"] + + +def test_sync_delete_issues_delete_request() -> None: + http_client = MagicMock() + http_client.delete.return_value = _response({}) + resource = EvaluatorEvalResultsResource(_platform(http_client)) + + resource.delete("job-1") + + assert http_client.delete.call_args[0][0] == f"{_BASE}/eval-results/job-1" + + +# ---- async ----------------------------------------------------------------- + + +async def test_async_retrieve_eval_result_parses_dto() -> None: + http_client = MagicMock() + http_client.get = AsyncMock(return_value=_response(_eval_payload("job-9"))) + resource = AsyncEvaluatorEvalResultsResource(_platform(http_client)) + + result = await resource.retrieve("job-9") + + assert isinstance(result, EvaluateResult) + assert result.metric_types == ["exact_match"] + assert http_client.get.call_args[0][0] == f"{_BASE}/eval-results/job-9" diff --git a/plugins/nemo-evaluator/tests/test_agent_evaluate.py b/plugins/nemo-evaluator/tests/test_agent_evaluate.py index 76ae339e9e..b55993d84f 100644 --- a/plugins/nemo-evaluator/tests/test_agent_evaluate.py +++ b/plugins/nemo-evaluator/tests/test_agent_evaluate.py @@ -478,14 +478,17 @@ class TestAgentEvalTask: def test_main_dispatches_agent_eval_job_with_task_sdk(self, mocker: MockerFixture) -> None: sdk = object() + async_sdk = object() get_task_sdk = mocker.patch("nemo_evaluator.tasks.runner.get_task_sdk", return_value=sdk) + get_async_task_sdk = mocker.patch("nemo_evaluator.tasks.runner.get_async_task_sdk", return_value=async_sdk) run_task = mocker.patch("nemo_evaluator.tasks.runner.run_task", return_value=0) exit_code = agent_eval_task_main() assert exit_code == 0 get_task_sdk.assert_called_once_with("evaluator") - run_task.assert_called_once_with(AgentEvalJob, sdk=sdk) + get_async_task_sdk.assert_called_once_with("evaluator") + run_task.assert_called_once_with(AgentEvalJob, sdk=sdk, async_sdk=async_sdk) def test_main_returns_setup_exit_code_when_task_sdk_fails(self, mocker: MockerFixture) -> None: get_task_sdk = mocker.patch("nemo_evaluator.tasks.runner.get_task_sdk", side_effect=RuntimeError("boom")) diff --git a/plugins/nemo-evaluator/tests/test_evaluate_job.py b/plugins/nemo-evaluator/tests/test_evaluate_job.py index 7dc67d2af4..a3c5dc7f1a 100644 --- a/plugins/nemo-evaluator/tests/test_evaluate_job.py +++ b/plugins/nemo-evaluator/tests/test_evaluate_job.py @@ -1361,14 +1361,17 @@ class TestEvaluateTask: def test_main_dispatches_evaluate_job_with_task_sdk(self, mocker: MockerFixture) -> None: sdk = object() + async_sdk = object() get_task_sdk = mocker.patch("nemo_evaluator.tasks.runner.get_task_sdk", return_value=sdk) + get_async_task_sdk = mocker.patch("nemo_evaluator.tasks.runner.get_async_task_sdk", return_value=async_sdk) run_task = mocker.patch("nemo_evaluator.tasks.runner.run_task", return_value=0) exit_code = evaluate_task_main() assert exit_code == 0 get_task_sdk.assert_called_once_with("evaluator") - run_task.assert_called_once_with(EvaluateJob, sdk=sdk) + get_async_task_sdk.assert_called_once_with("evaluator") + run_task.assert_called_once_with(EvaluateJob, sdk=sdk, async_sdk=async_sdk) def test_main_returns_setup_exit_code_when_task_sdk_fails(self, mocker: MockerFixture) -> None: get_task_sdk = mocker.patch( diff --git a/plugins/nemo-evaluator/tests/test_result_entity.py b/plugins/nemo-evaluator/tests/test_result_entity.py new file mode 100644 index 0000000000..c35c916c40 --- /dev/null +++ b/plugins/nemo-evaluator/tests/test_result_entity.py @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Serialization round-trip tests for the eval-result entities. + +The entity store persists an entity's custom fields with +``model_dump(exclude=base, mode="json")`` into a JSON column and rebuilds it with +``model_validate``. These tests exercise that exact round-trip (which the in-memory fakes elsewhere +bypass) to guard the aggregated ``scores`` rollup and the row-eval input refs that carry nested types. +""" + +from __future__ import annotations + +import json +from typing import TypeVar + +import pytest +from nemo_evaluator.entities import AgentEvalResultEntity, EvaluateResultEntity +from nemo_evaluator_sdk.values.results import AggregatedMetricResult, AggregateRangeScore +from pydantic import ValidationError + +# Constrained (not bound) so _E resolves to a concrete entity — which has EntityBase's name/workspace/ +# __base_fields__ plus its own result fields — rather than the abstract _EvalResultCommon mixin. +_E = TypeVar("_E", AgentEvalResultEntity, EvaluateResultEntity) + + +def _scores() -> AggregatedMetricResult: + return AggregatedMetricResult(scores=[AggregateRangeScore(name="accuracy", count=10, nan_count=0, mean=0.9)]) + + +def _roundtrip(entity: _E) -> _E: + """Mirror the entity store: dump custom fields to JSON, prove it's JSON-safe, then rebuild.""" + cls = type(entity) + data = entity.model_dump(exclude=cls.__base_fields__, exclude_computed_fields=True, mode="json") + data = json.loads(json.dumps(data)) + return cls.model_validate({"name": entity.name, "workspace": entity.workspace, **data}) + + +def test_agent_eval_result_roundtrip_preserves_scores_and_target() -> None: + entity = AgentEvalResultEntity( + name="job-123", + workspace="default", + job_id="job-123", + target_kind="codex", + target_name="gpt-5.5", + target_url=None, + scores=_scores(), + bundle_ref="fileset://default/agent-eval-results#bundle", + ) + + restored = _roundtrip(entity) + + assert restored.job_id == "job-123" + assert restored.target_kind == "codex" + assert restored.target_name == "gpt-5.5" + assert restored.target_url is None + assert restored.bundle_ref == entity.bundle_ref + # The nested AggregatedMetricResult must survive the JSON column intact. + assert restored.scores == entity.scores + assert restored.scores.scores[0].mean == 0.9 + + +def test_evaluate_result_roundtrip_preserves_dataset_and_metric_types() -> None: + entity = EvaluateResultEntity( + name="job-456", + workspace="default", + job_id="job-456", + target_kind="model", + target_name="my-model", + target_url="https://model.test/v1/chat/completions", + scores=_scores(), + bundle_ref="fileset://default/eval-results#bundle", + dataset_ref="default/my-dataset", + metric_types=["exact_match", "string_check"], + ) + + restored = _roundtrip(entity) + + assert restored.dataset_ref == "default/my-dataset" + assert restored.metric_types == ["exact_match", "string_check"] + assert restored.target_url == "https://model.test/v1/chat/completions" + assert restored.scores == entity.scores + + +def test_entity_types_are_distinct() -> None: + # Distinct __entity_type__ keeps the two collections from colliding in the store. + assert AgentEvalResultEntity.__entity_type__ == "agent_eval_result" + assert EvaluateResultEntity.__entity_type__ == "evaluate_result" + assert AgentEvalResultEntity.__entity_type__ != EvaluateResultEntity.__entity_type__ + + +def test_shared_fields_are_required() -> None: + # A result is only persisted once the run produced all of it — no schema defaults papering over + # missing data. job_id (and the rest of the shared record) must be supplied by the caller. + with pytest.raises(ValidationError): + AgentEvalResultEntity(name="x", workspace="default", scores=_scores()) # type: ignore[call-arg] diff --git a/plugins/nemo-evaluator/tests/test_result_persistence.py b/plugins/nemo-evaluator/tests/test_result_persistence.py new file mode 100644 index 0000000000..67d0eafd04 --- /dev/null +++ b/plugins/nemo-evaluator/tests/test_result_persistence.py @@ -0,0 +1,204 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for jobs.result_persistence: target-trait mapping + best-effort entity writes. + +These cover the pure mapping helpers and the persist_* entry points, with the async ``EntityClient`` +stubbed (``_entity_client`` patched at its usage site) so no real SDK or event loop wiring is needed. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import cast + +import pytest +from nemo_evaluator.entities import AgentEvalResultEntity, EvaluateResultEntity +from nemo_evaluator.jobs import result_persistence +from nemo_evaluator.jobs.agent_spec import AgentTarget, CodexRunnerTarget, ModelTarget +from nemo_evaluator.jobs.result_persistence import ( + _agent_target_fields, + _row_target_fields, + persist_agent_eval_result, + persist_evaluate_result, +) +from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, AgentEvalSummary +from nemo_evaluator_sdk.enums import AgentFormat +from nemo_evaluator_sdk.values import Agent, Model +from nemo_evaluator_sdk.values.results import AggregatedMetricResult, EvaluationResult +from nemo_platform import AsyncNeMoPlatform +from nemo_platform_plugin.entities import EntityBase +from nemo_platform_plugin.job_context import JobContext, StoragePaths +from nemo_platform_plugin.job_results import LocalJobResults +from pytest_mock import MockerFixture + +# An opaque stand-in for the async task SDK: every test that reaches the save path patches +# `_entity_client`, so the value is never used as a real client — only its presence matters. +_ASYNC_SDK = cast(AsyncNeMoPlatform, object()) + + +def _model() -> Model: + return Model(url="https://model.test/v1/chat/completions", name="my-model") + + +def _agent() -> Agent: + return Agent( + url="http://agent.test", + name="my-agent", + format=AgentFormat.GENERIC, + body={"question": "{{item.prompt}}"}, + response_path="$.answer", + ) + + +# ---- target-trait mapping -------------------------------------------------- + + +@pytest.mark.parametrize( + ("target", "expected"), + [ + (ModelTarget(model=_model()), ("model", "my-model", "https://model.test/v1/chat/completions")), + (AgentTarget(agent=_agent()), ("agent", "my-agent", "http://agent.test")), + (CodexRunnerTarget(model="gpt-5.5"), ("codex", "gpt-5.5", None)), + (None, (None, None, None)), + ], +) +def test_agent_target_fields(target, expected) -> None: + assert _agent_target_fields(target) == expected + + +@pytest.mark.parametrize( + ("target", "expected"), + [ + (_model(), ("model", "my-model", "https://model.test/v1/chat/completions")), + (_agent(), ("agent", "my-agent", "http://agent.test")), + (None, (None, None, None)), + ], +) +def test_row_target_fields(target, expected) -> None: + assert _row_target_fields(target) == expected + + +# ---- persist_* entity construction + best-effort write --------------------- + + +class _FakeClient: + """Records saved entities; optionally fails to exercise the best-effort path.""" + + def __init__(self, fail: bool = False) -> None: + self.saved: list[EntityBase] = [] + self._fail = fail + + async def save(self, entity: EntityBase) -> EntityBase: + if self._fail: + raise RuntimeError("entity store unavailable") + self.saved.append(entity) + return entity + + +def _ctx(tmp_path: Path, job_id: str | None) -> JobContext: + storage = StoragePaths(ephemeral=tmp_path / "e", persistent=tmp_path / "p") + storage.ephemeral.mkdir() + storage.persistent.mkdir() + return JobContext( + workspace="dev", + storage=storage, + results=LocalJobResults(root=storage.persistent / "results"), + job_id=job_id, + ) + + +def _agent_result() -> AgentEvalResult: + return AgentEvalResult(run_id="run-1", tasks=[], trials=[], scores=[], summary=AgentEvalSummary()) + + +def _eval_result() -> EvaluationResult: + return EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) + + +def test_persist_agent_eval_result_builds_entity_and_saves(tmp_path: Path, mocker: MockerFixture) -> None: + client = _FakeClient() + mocker.patch.object(result_persistence, "_entity_client", return_value=client) + + persist_agent_eval_result( + _agent_result(), + target=CodexRunnerTarget(model="gpt-5.5"), + ctx=_ctx(tmp_path, "job-1"), + bundle_ref="fileset://dev/agent-eval-results#b", + async_sdk=_ASYNC_SDK, + ) + + (entity,) = client.saved + assert isinstance(entity, AgentEvalResultEntity) + assert entity.name == "job-1" + assert entity.job_id == "job-1" + assert entity.workspace == "dev" + assert (entity.target_kind, entity.target_name, entity.target_url) == ("codex", "gpt-5.5", None) + assert entity.bundle_ref == "fileset://dev/agent-eval-results#b" + + +def test_persist_evaluate_result_records_dataset_and_metric_types(tmp_path: Path, mocker: MockerFixture) -> None: + client = _FakeClient() + mocker.patch.object(result_persistence, "_entity_client", return_value=client) + + persist_evaluate_result( + _eval_result(), + target=_model(), + dataset_ref="dev/my-dataset", + metric_types=["exact_match"], + ctx=_ctx(tmp_path, "job-2"), + bundle_ref="fileset://dev/eval-results#b", + async_sdk=_ASYNC_SDK, + ) + + (entity,) = client.saved + assert isinstance(entity, EvaluateResultEntity) + assert entity.job_id == "job-2" + assert entity.target_kind == "model" + assert entity.dataset_ref == "dev/my-dataset" + assert entity.metric_types == ["exact_match"] + + +def test_persist_skips_when_no_job_id(tmp_path: Path, mocker: MockerFixture) -> None: + client = _FakeClient() + mocker.patch.object(result_persistence, "_entity_client", return_value=client) + + # A platformless local run has no job id — there's no run to key the result on, so skip. + persist_agent_eval_result( + _agent_result(), + target=None, + ctx=_ctx(tmp_path, None), + bundle_ref="x", + async_sdk=_ASYNC_SDK, + ) + + assert client.saved == [] + + +def test_persist_skips_when_no_async_sdk(tmp_path: Path) -> None: + # No async SDK injected (offline run): _entity_client returns None and persistence is skipped. + # Runs the real _entity_client(None) path; must not raise. + persist_evaluate_result( + _eval_result(), + target=None, + dataset_ref=None, + metric_types=[], + ctx=_ctx(tmp_path, "job-3"), + bundle_ref="x", + async_sdk=None, + ) + + +def test_persist_is_best_effort_on_save_failure(tmp_path: Path, mocker: MockerFixture) -> None: + client = _FakeClient(fail=True) + mocker.patch.object(result_persistence, "_entity_client", return_value=client) + + # The eval already succeeded and the bundle is saved; a store error must not fail the job. + persist_agent_eval_result( + _agent_result(), + target=ModelTarget(model=_model()), + ctx=_ctx(tmp_path, "job-4"), + bundle_ref="x", + async_sdk=_ASYNC_SDK, + ) + assert client.saved == [] From a4e9e8dca9633bbf70bcd306407f089332898d67 Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Thu, 2 Jul 2026 10:19:44 -0300 Subject: [PATCH 2/4] fix(evaluator): address review feedback on eval-results persistence - Best-effort result persistence: wrap persist_agent_eval_result and persist_evaluate_result in try/except with a logged warning. The authoritative output (bundle/result artifacts) is already saved, so a persistence failure no longer fails an otherwise-successful eval job. Regression tests cover both jobs. - Results GET routes now set response_model_exclude_none=True to match the list routes' serialization. - Wrap results GET and DELETE handlers in try/except -> 500 (re-raising HTTPException so the 404 is preserved), matching the metrics routes. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Sandy Chapman --- .../src/nemo_evaluator/api/v2/results.py | 58 ++++++++++++++----- .../src/nemo_evaluator/jobs/agent_evaluate.py | 19 ++++-- .../src/nemo_evaluator/jobs/evaluate.py | 31 ++++++---- .../tests/test_agent_evaluate.py | 20 +++++++ .../nemo-evaluator/tests/test_evaluate_job.py | 16 +++++ 5 files changed, 116 insertions(+), 28 deletions(-) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/results.py b/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/results.py index 23260f8abc..a2ac850a85 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/results.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/results.py @@ -122,6 +122,9 @@ async def list_agent_eval_results( @agent_eval_results_router.get( "/agent-eval-results/{name}", summary="Get Agent Eval Result", + status_code=status.HTTP_200_OK, + response_model=AgentEvalResult, + response_model_exclude_none=True, responses={status.HTTP_404_NOT_FOUND: {"description": "Result not found"}}, ) @scope.read @@ -132,10 +135,16 @@ async def get_agent_eval_result( service: ResultService = Depends(get_result_service), ) -> AgentEvalResult: """Get an agent-evaluation result record by workspace and name.""" - result = await service.get_agent_eval_result(workspace, name) - if result is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Result not found: {workspace}/{name}") - return result + try: + result = await service.get_agent_eval_result(workspace, name) + if result is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Result not found: {workspace}/{name}") + return result + except HTTPException: + raise + except Exception: + logger.exception(f"Failed to get agent-eval result {_sanitize_for_log(workspace)}/{_sanitize_for_log(name)}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") @agent_eval_results_router.delete( @@ -152,9 +161,15 @@ async def delete_agent_eval_result( service: ResultService = Depends(get_result_service), ) -> None: """Delete an agent-evaluation result record by workspace and name.""" - if not await service.delete_agent_eval_result(workspace, name): - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Result not found: {workspace}/{name}") - return None + try: + if not await service.delete_agent_eval_result(workspace, name): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Result not found: {workspace}/{name}") + return None + except HTTPException: + raise + except Exception: + logger.exception(f"Failed to delete agent-eval result {_sanitize_for_log(workspace)}/{_sanitize_for_log(name)}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") # --- (row) eval results ------------------------------------------------------ @@ -195,6 +210,9 @@ async def list_eval_results( @evaluate_results_router.get( "/eval-results/{name}", summary="Get Eval Result", + status_code=status.HTTP_200_OK, + response_model=EvaluateResult, + response_model_exclude_none=True, responses={status.HTTP_404_NOT_FOUND: {"description": "Result not found"}}, ) @scope.read @@ -205,10 +223,16 @@ async def get_eval_result( service: ResultService = Depends(get_result_service), ) -> EvaluateResult: """Get a (row) evaluation result record by workspace and name.""" - result = await service.get_eval_result(workspace, name) - if result is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Result not found: {workspace}/{name}") - return result + try: + result = await service.get_eval_result(workspace, name) + if result is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Result not found: {workspace}/{name}") + return result + except HTTPException: + raise + except Exception: + logger.exception(f"Failed to get eval result {_sanitize_for_log(workspace)}/{_sanitize_for_log(name)}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") @evaluate_results_router.delete( @@ -225,6 +249,12 @@ async def delete_eval_result( service: ResultService = Depends(get_result_service), ) -> None: """Delete a (row) evaluation result record by workspace and name.""" - if not await service.delete_eval_result(workspace, name): - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Result not found: {workspace}/{name}") - return None + try: + if not await service.delete_eval_result(workspace, name): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Result not found: {workspace}/{name}") + return None + except HTTPException: + raise + except Exception: + logger.exception(f"Failed to delete eval result {_sanitize_for_log(workspace)}/{_sanitize_for_log(name)}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py index 5e85055376..4eb85f4b2e 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py @@ -17,6 +17,7 @@ from __future__ import annotations +import logging from dataclasses import dataclass from pathlib import Path from typing import Any, ClassVar @@ -50,6 +51,8 @@ from nemo_platform_plugin.jobs.api_factory import PlatformJobSpec from pydantic import BaseModel +logger = logging.getLogger(__name__) + #: Job-result artifact names + the on-disk bundle directory. DEFAULT_RESULT_NAME = "agent-eval-results" SUMMARY_RESULT_NAME = "summary" @@ -283,9 +286,17 @@ def run( ctx.results.save(SUMMARY_RESULT_NAME, files.summary) # Persist the queryable result record (aggregates + coverage); the full bundle (trials) lives - # in the fileset referenced by `artifact`. - persist_agent_eval_result( - result, target=spec.target, ctx=ctx, bundle_ref=artifact.artifact_url, async_sdk=async_sdk - ) + # in the fileset referenced by `artifact`. Best-effort: the authoritative output (bundle + + # summary artifacts) is already saved above, so a persistence failure must not fail an + # otherwise-successful eval — log and continue. + try: + persist_agent_eval_result( + result, target=spec.target, ctx=ctx, bundle_ref=artifact.artifact_url, async_sdk=async_sdk + ) + except Exception: + logger.warning( + "Failed to persist agent-eval result record; the result bundle artifact is unaffected", + exc_info=True, + ) return {"status": "completed", "artifact": artifact.model_dump()} diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py index 8710eebbfa..8c63ff77b9 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +import logging from dataclasses import dataclass from pathlib import Path from typing import Annotated, Any, ClassVar, Self, TypeAlias @@ -43,6 +44,8 @@ from nemo_platform_plugin.jobs.api_factory import PlatformJobSpec from pydantic import BaseModel, ConfigDict, Field, model_validator +logger = logging.getLogger(__name__) + TargetSpec = Model | Agent MetricSpec: TypeAlias = Annotated[list[MetricRefOrInline], Field(min_length=1)] # Canonical spec carries inline metrics only (refs resolved) — still the wire DTO, @@ -302,16 +305,24 @@ def run( ctx.results.save(ARTIFACTS_RESULT_NAME, result_files.artifacts_dir, ignore_patterns=RESULT_IGNORE_PATTERNS) # Persist the queryable result record (aggregate scores); per-row detail lives in the fileset - # bundle referenced by `artifact`. - persist_evaluate_result( - result, - target=spec.target, - dataset_ref=spec.dataset.root if isinstance(spec.dataset, FilesetRef) else None, - metric_types=[metric.type for metric in metrics], - ctx=ctx, - bundle_ref=artifact.artifact_url, - async_sdk=async_sdk, - ) + # bundle referenced by `artifact`. Best-effort: the authoritative output (result artifacts) is + # already saved above, so a persistence failure must not fail an otherwise-successful eval — + # log and continue. + try: + persist_evaluate_result( + result, + target=spec.target, + dataset_ref=spec.dataset.root if isinstance(spec.dataset, FilesetRef) else None, + metric_types=[metric.type for metric in metrics], + ctx=ctx, + bundle_ref=artifact.artifact_url, + async_sdk=async_sdk, + ) + except Exception: + logger.warning( + "Failed to persist evaluate result record; the result artifacts are unaffected", + exc_info=True, + ) # TODO: Implement progress reporting hook in SDK - AALGO-149 # self.report_progress( diff --git a/plugins/nemo-evaluator/tests/test_agent_evaluate.py b/plugins/nemo-evaluator/tests/test_agent_evaluate.py index b55993d84f..8f62e22298 100644 --- a/plugins/nemo-evaluator/tests/test_agent_evaluate.py +++ b/plugins/nemo-evaluator/tests/test_agent_evaluate.py @@ -148,6 +148,26 @@ def test_agent_eval_job_reconstructs_tasks_and_persists_bundle(tmp_path: Path, m assert result["artifact"]["name"] == DEFAULT_RESULT_NAME +def test_agent_eval_job_survives_result_persistence_failure(tmp_path: Path, mocker: MockerFixture) -> None: + # The queryable result record is a best-effort convenience index; the authoritative output (bundle + # + summary artifacts) is already saved. A persistence failure must not fail a successful eval. + mocker.patch.object(AgentEvalJob, "_build_evaluator", return_value=_FakeEvaluator()) + persist = mocker.patch( + "nemo_evaluator.jobs.agent_evaluate.persist_agent_eval_result", + side_effect=RuntimeError("entity store unavailable"), + ) + ctx = _job_context(tmp_path) + + spec = AgentEvalSpec(tasks=[_task_spec()], target=CodexRunnerTarget(model="gpt-5.5")) + result = AgentEvalJob().run(spec.model_dump(), ctx=ctx) + + # Persistence was attempted and raised, yet the job still completed with its artifacts intact. + persist.assert_called_once() + assert result["status"] == "completed" + assert result["artifact"]["name"] == DEFAULT_RESULT_NAME + assert (ctx.storage.persistent / "results" / DEFAULT_RESULT_NAME).exists() + + def test_agent_eval_spec_requires_at_least_one_task() -> None: with pytest.raises(ValueError, match="at least 1 item|too_short|min_length"): AgentEvalSpec(tasks=[]) diff --git a/plugins/nemo-evaluator/tests/test_evaluate_job.py b/plugins/nemo-evaluator/tests/test_evaluate_job.py index a3c5dc7f1a..9bbdde3284 100644 --- a/plugins/nemo-evaluator/tests/test_evaluate_job.py +++ b/plugins/nemo-evaluator/tests/test_evaluate_job.py @@ -451,6 +451,22 @@ def test_evaluate_job_runs_inline_exact_match_metric() -> None: assert aggregate_scores[0]["mean"] == 0.5 +def test_evaluate_job_survives_result_persistence_failure(mocker: MockerFixture) -> None: + # Mirror of the agent-eval job: the queryable result record is a best-effort convenience index; + # a persistence failure must not fail an otherwise-successful eval (its artifacts are already saved). + persist = mocker.patch( + "nemo_evaluator.jobs.evaluate.persist_evaluate_result", + side_effect=RuntimeError("entity store unavailable"), + ) + + result = NemoJobScheduler().run_local(EvaluateJob, _exact_match_spec()) + + persist.assert_called_once() + assert result["status"] == "completed" + aggregate_scores = _load_artifact_payload(result)["aggregate_scores"]["scores"] + assert aggregate_scores[0]["name"] == "exact-match.exact-match" + + def test_evaluate_job_applies_metric_job_params_once() -> None: spec = { "metrics": [_bundle_payload(_CountingJobParamsMetric())], From 25d44e5d83332eea1b7b09f833c19c279b102b6b Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Thu, 2 Jul 2026 11:09:15 -0300 Subject: [PATCH 3/4] fix(evaluator): redact credentials from persisted target URLs target_url is stored on the result entity and returned by the read APIs, so a target endpoint carrying userinfo or a token query param would leak. Route both target-field helpers through a new _safe_target_url() that strips userinfo, redacts sensitive query values, and omits the URL when it has no host. Tuple shape is unchanged. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Sandy Chapman --- .../nemo_evaluator/jobs/result_persistence.py | 40 +++++++++++++++++-- .../tests/test_result_persistence.py | 22 ++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/result_persistence.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/result_persistence.py index fc2d1d71ae..4aa11155f9 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/result_persistence.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/result_persistence.py @@ -16,6 +16,8 @@ from __future__ import annotations import logging +import re +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from nemo_evaluator.entities import AgentEvalResultEntity, EvaluateResultEntity from nemo_evaluator.jobs.agent_spec import AgentTarget, CodexRunnerTarget, ModelTarget, Target @@ -42,12 +44,42 @@ def _entity_client(async_sdk: AsyncNeMoPlatform | None) -> EntityClient | None: return EntityClient(AsyncEntitiesResource(async_sdk)) +#: Query-parameter keys whose values are redacted before a target URL is persisted/returned. +_SENSITIVE_QUERY_KEY = re.compile(r"token|key|secret|password|passwd|pwd|auth|credential|sig", re.IGNORECASE) + + +def _safe_target_url(url: object) -> str | None: + """Render a target endpoint URL for persistence with credentials stripped. + + ``target_url`` is stored on the result entity and returned by the read APIs, so any userinfo + (``user:pass@``) or sensitive query values (api keys/tokens) the endpoint URL carries would leak. + Drop userinfo, redact sensitive query values, and omit the URL entirely when it has no host (i.e. + can't be safely normalized). + """ + if url is None: + return None + try: + parts = urlsplit(str(url)) + except ValueError: + return None + if not parts.hostname: + return None + netloc = parts.hostname if parts.port is None else f"{parts.hostname}:{parts.port}" + query = urlencode( + [ + (key, "REDACTED" if _SENSITIVE_QUERY_KEY.search(key) else value) + for key, value in parse_qsl(parts.query, keep_blank_values=True) + ] + ) + return urlunsplit((parts.scheme, netloc, parts.path, query, parts.fragment)) + + def _agent_target_fields(target: Target | None) -> tuple[str | None, str | None, str | None]: """(kind, name, url) flat target traits for an agent-eval target.""" if isinstance(target, ModelTarget): - return "model", target.model.name, str(target.model.url) if target.model.url else None + return "model", target.model.name, _safe_target_url(target.model.url) if isinstance(target, AgentTarget): - return "agent", getattr(target.agent, "name", None), str(target.agent.url) + return "agent", getattr(target.agent, "name", None), _safe_target_url(target.agent.url) if isinstance(target, CodexRunnerTarget): return "codex", target.model, None return None, None, None @@ -56,9 +88,9 @@ def _agent_target_fields(target: Target | None) -> tuple[str | None, str | None, def _row_target_fields(target: Model | Agent | None) -> tuple[str | None, str | None, str | None]: """(kind, name, url) flat target traits for a row-eval target.""" if isinstance(target, Model): - return "model", target.name, str(target.url) if target.url else None + return "model", target.name, _safe_target_url(target.url) if isinstance(target, Agent): - return "agent", getattr(target, "name", None), str(target.url) + return "agent", getattr(target, "name", None), _safe_target_url(target.url) return None, None, None diff --git a/plugins/nemo-evaluator/tests/test_result_persistence.py b/plugins/nemo-evaluator/tests/test_result_persistence.py index 67d0eafd04..ba822e6283 100644 --- a/plugins/nemo-evaluator/tests/test_result_persistence.py +++ b/plugins/nemo-evaluator/tests/test_result_persistence.py @@ -19,6 +19,7 @@ from nemo_evaluator.jobs.result_persistence import ( _agent_target_fields, _row_target_fields, + _safe_target_url, persist_agent_eval_result, persist_evaluate_result, ) @@ -79,6 +80,27 @@ def test_row_target_fields(target, expected) -> None: assert _row_target_fields(target) == expected +@pytest.mark.parametrize( + ("url", "expected"), + [ + # Plain endpoints round-trip unchanged. + ("https://model.test/v1/chat/completions", "https://model.test/v1/chat/completions"), + ("http://agent.test:8080/infer", "http://agent.test:8080/infer"), + # Userinfo (credentials) is stripped from the netloc. Assembled from parts so no literal + # basic-auth userinfo appears contiguously in source (the secret scanner flags that pattern). + ("https://" + "u:p" + "@model.test/v1", "https://model.test/v1"), + # Sensitive query values are redacted; benign ones survive. + ("https://model.test/v1?api_key=sekret®ion=us", "https://model.test/v1?api_key=REDACTED®ion=us"), + ("https://model.test/v1?access_token=abc&x=1", "https://model.test/v1?access_token=REDACTED&x=1"), + # Unparseable / host-less inputs are omitted rather than stored raw. + ("not a url", None), + (None, None), + ], +) +def test_safe_target_url_strips_credentials(url, expected) -> None: + assert _safe_target_url(url) == expected + + # ---- persist_* entity construction + best-effort write --------------------- From 042c652fabe0d6fa5da936ee5446f503f915def7 Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Thu, 2 Jul 2026 11:09:15 -0300 Subject: [PATCH 4/4] fix(evaluator): use ty rule name in result-entity negative-test suppression The negative test suppressed with `# type: ignore[call-arg]` (a mypy code ty doesn't recognize), so ty's `missing-argument` leaked through and failed the lint-python-types CI check. Use `# ty: ignore[missing-argument]`. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Sandy Chapman --- plugins/nemo-evaluator/tests/test_result_entity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nemo-evaluator/tests/test_result_entity.py b/plugins/nemo-evaluator/tests/test_result_entity.py index c35c916c40..96b8f54fa7 100644 --- a/plugins/nemo-evaluator/tests/test_result_entity.py +++ b/plugins/nemo-evaluator/tests/test_result_entity.py @@ -93,4 +93,4 @@ def test_shared_fields_are_required() -> None: # A result is only persisted once the run produced all of it — no schema defaults papering over # missing data. job_id (and the rest of the shared record) must be supplied by the caller. with pytest.raises(ValidationError): - AgentEvalResultEntity(name="x", workspace="default", scores=_scores()) # type: ignore[call-arg] + AgentEvalResultEntity(name="x", workspace="default", scores=_scores()) # ty: ignore[missing-argument]