From fb31f707195bbd3c7f256ea8c6106c11d845fd77 Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Mon, 20 Jul 2026 11:17:11 -0300 Subject: [PATCH 1/3] docs(evaluator): document Tasks & Tasksets management Add a Manage Tasks & Tasksets page covering the stored task/taskset entities added in the AALGO-307 stack: concepts, create/retrieve/list/ delete via the nemo_platform SDK (client.evaluator.tasks / .tasksets), field references, workspace/project handling, and the underlying REST API. Registers the page in the published Fern nav after SDK Resources. Fills a documented gap: there was no user-facing doc for task/taskset management. Signed-off-by: Sandy Chapman --- docs/evaluator/manage-tasks-tasksets.mdx | 209 +++++++++++++++++++++++ docs/fern/versions/latest.yml | 3 + 2 files changed, 212 insertions(+) create mode 100644 docs/evaluator/manage-tasks-tasksets.mdx diff --git a/docs/evaluator/manage-tasks-tasksets.mdx b/docs/evaluator/manage-tasks-tasksets.mdx new file mode 100644 index 0000000000..fca9a4d35b --- /dev/null +++ b/docs/evaluator/manage-tasks-tasksets.mdx @@ -0,0 +1,209 @@ +--- +title: "Manage Tasks & Tasksets" +description: "" +--- + + +A **task** is a stored, reusable definition of an agent-eval unit of work: an intent (what the agent +should do), the inputs it receives, and the metrics that score it. A **taskset** is a named grouping +of tasks. Both are first-class entities in the Evaluator plugin, addressed by `workspace/name` and +managed through the `nemo_platform` SDK. + +Use stored tasks and tasksets when you want to define an evaluation unit once and reference it across +runs, share it across a team, or assemble suites — rather than re-declaring the intent, inputs, and +metrics inline every time. + +## Concepts + +| Concept | What it is | Members | +|---------|------------|---------| +| **Task** | A reusable agent-eval unit: `intent`, `inputs`, and the `metrics` that score it. | References the metrics that score it. | +| **Taskset** | A flexible grouping of tasks with a description and metadata. | References member tasks by `workspace/name`. Membership is a **set** — order is not significant and duplicate references are rejected. | + +Both are addressed by `workspace/name`. Names are unique within a workspace, limited to 255 +characters, and must match `^[\w\-\.]+$`. + + +Tasks and tasksets support **create, retrieve, list, and delete** — there is no update. To change a +stored task or taskset, delete it and create a new one, or store a new version under a different +name. + + +## Initialize the SDK + +```python +import os + +from nemo_platform import NeMoPlatform + + +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="default", +) + +tasks = client.evaluator.tasks # EvaluatorTasksResource +tasksets = client.evaluator.tasksets # EvaluatorTasksetsResource +``` + +## Manage Tasks + +A task scores its output with metrics. Store the metric first, then reference it from the task by +`workspace/name`. See [Manage Metrics](/documentation/evaluate-models/metrics/manage-metrics) for the +metric classes and options. + +```python +from nemo_evaluator_sdk import ExactMatchMetric + +# Store a metric the task will reference. +client.evaluator.metrics.create( + "answer-exact-match", + metric=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), +) +``` + +Send the task as a `TaskInput` (the authorable subset of a task) and address it by name on create. +Reference the stored metric with a `MetricRef` (`workspace/name`, or a bare `name` resolved against +the task's workspace). The service returns the stored `Task`. + +```python +from nemo_evaluator.api.schemas import MetadataItem, MetricRef, TaskInput, TaskInputs + +task = TaskInput( + intent="Answer the user's geography question with the capital city.", + inputs=TaskInputs(instruction="What is the capital of France?"), + metrics=[MetricRef("default/answer-exact-match")], + metadata=[MetadataItem(key="suite", value="geography")], +) + +stored = tasks.create("capital-of-france", task=task) +print(stored.id, stored.metrics) +``` + +### `TaskInput` fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `intent` | `str` | Yes | Human-readable description of the desired agent behavior. | +| `inputs` | `TaskInputs` | No | The task's recognized input fields. `instruction` is the agent's prompt; it falls back to `intent` when unset. | +| `metrics` | `list[MetricRefOrInline]` | No | The metrics that score the task, as `MetricRef` references (`workspace/name`) to stored metrics. Pre-built inline metric bundles (`MetricInline`) are also accepted and are normalized to stored metrics on create. | +| `views` | `dict[str, SemanticView]` | No | Optional reporting views mapping metric outputs into named semantic scores. | +| `metadata` | `list[MetadataItem]` | No | Key/value annotations. Keys must be unique. | + + +A stored task holds **metric references only**. Any inline metric bundle you pass on create is stored +as a content-addressed *derived* metric, and the task record is normalized to reference it. This is +why `stored.metrics` always comes back as a list of `MetricRef` references. + + +### Retrieve, list, and delete + +```python +# Retrieve one task by name +task = tasks.retrieve("capital-of-france") + +# List tasks in the workspace (paginated) +page = tasks.list(page=1, page_size=100, sort="-created_at") +for item in page.data: + print(item.name, item.intent) + +# Delete a task +tasks.delete("capital-of-france") +``` + +`sort` accepts `name`, `created_at`, or `updated_at`, each optionally prefixed with `-` for +descending order. + +## Manage Tasksets + +A taskset references existing tasks by `workspace/name`. All referenced tasks must already exist when +the taskset is created; a missing or duplicate reference is rejected. + +```python +from nemo_evaluator.api.schemas import TaskRef, TasksetInput + +taskset = TasksetInput( + description="Geography questions for smoke-testing the agent.", + tasks=[ + TaskRef("default/capital-of-france"), + TaskRef("default/capital-of-japan"), + ], +) + +stored = tasksets.create("geography-suite", taskset=taskset) +print(stored.tasks) +``` + +### `TasksetInput` fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `description` | `str` | No | Human-readable description of the grouping. | +| `tasks` | `list[TaskRef]` | No | References to member tasks (`workspace/name`, or bare `name` within the same workspace). Set semantics — duplicates rejected. | +| `metadata` | `list[MetadataItem]` | No | Key/value annotations. Keys must be unique. | + +### Retrieve, list, and delete + +```python +taskset = tasksets.retrieve("geography-suite") + +page = tasksets.list(page=1, page_size=100, sort="name") +for item in page.data: + print(item.name, len(item.tasks)) + +tasksets.delete("geography-suite") +``` + +Deleting a taskset does not delete its member tasks — a taskset only holds references. + +## Async usage + +`AsyncNeMoPlatform` exposes the same surface; await each call. + +```python +import asyncio + +from nemo_platform import AsyncNeMoPlatform + + +async def main() -> None: + client = AsyncNeMoPlatform(base_url="http://localhost:8080", workspace="default") + page = await client.evaluator.tasks.list() + for item in page.data: + print(item.name) + + +asyncio.run(main()) +``` + +## Workspaces and projects + +Every method accepts an optional `workspace` argument that overrides the client's default workspace. +On create, an optional `project` argument associates the task or taskset with a project. When you +omit `workspace`, the client's configured workspace is used. + +## REST API + +The SDK resources are a thin client over the Evaluator plugin REST API, mounted under +`/apis/evaluator/v2/workspaces/{workspace}`: + +| Method | Path | Description | +|--------|------|-------------| +| `GET` | `/tasks` | List tasks (paginated). | +| `POST` | `/tasks/{name}` | Create a task. | +| `GET` | `/tasks/{name}` | Retrieve a task. | +| `DELETE` | `/tasks/{name}` | Delete a task. | +| `GET` | `/tasksets` | List tasksets (paginated). | +| `POST` | `/tasksets/{name}` | Create a taskset. | +| `GET` | `/tasksets/{name}` | Retrieve a taskset. | +| `DELETE` | `/tasksets/{name}` | Delete a taskset. | + +Creating a name that already exists returns `409`. An invalid metric reference (task) or a missing or +duplicate task reference (taskset) returns `422`. Retrieving or deleting a name that does not exist +returns `404`. + +## Related Topics + +- [Manage Metrics](/documentation/evaluate-models/metrics/manage-metrics) - Define and reuse the metrics that score a task +- [SDK Resources](/documentation/evaluate-models/sdk-resources) - Run and submit evaluations through the Evaluator plugin +- [Agent Evaluation](/documentation/evaluate-models/agent-eval) - How agent-eval tasks are executed and scored diff --git a/docs/fern/versions/latest.yml b/docs/fern/versions/latest.yml index 35ea963fc0..520de98b2f 100644 --- a/docs/fern/versions/latest.yml +++ b/docs/fern/versions/latest.yml @@ -398,6 +398,9 @@ navigation: path: ../../evaluator/tutorials/define-run-custom-python-metrics.mdx - page: SDK Resources path: ../../evaluator/sdk-resources.mdx + - page: Manage Tasks & Tasksets + slug: manage-tasks-tasksets + path: ../../evaluator/manage-tasks-tasksets.mdx - section: Metrics path: ../../evaluator/metrics/index.mdx contents: From 666f57e8eaaeb5940b7e7e630df592c1bc470aef Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Tue, 21 Jul 2026 09:23:30 -0300 Subject: [PATCH 2/3] feat(evaluator): resolve taskset references on the agent-eval submit path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AgentEvalInputSpec.tasks becomes a union: an inline list of tasks or a TasksetRef naming a stored taskset. During to_spec a taskset reference is loaded and its member tasks are expanded into inline task DTOs, then the existing metric-ref resolution runs — so a taskset-driven run hydrates to the same canonical AgentEvalSpec as an inline one. - Add TasksetRef (schemas.py) and task_refs.py (resolution helper mirroring metric_refs.py): loads the taskset + member tasks, maps each stored task to an AgentEvalTaskInput, and raises clear errors for a missing taskset, a missing/deleted member, an empty taskset, duplicate expanded ids, or a local run with no entity store. - Stored tasks carry no grader-only reference field, so taskset-driven tasks run with an empty reference (documented). - Document the run-over-a-taskset flow in the Manage Tasks & Tasksets page. - Regenerate the plugin OpenAPI spec for the new union + TasksetRef schema. Tests: unit coverage for resolution + the union field; verified live end to end that a TasksetRef resolves through to_spec into a hydrated AgentEvalSpec with inline metrics. Signed-off-by: Sandy Chapman --- docs/evaluator/manage-tasks-tasksets.mdx | 37 ++++++ plugins/nemo-evaluator/openapi/openapi.yaml | 38 +++++- .../src/nemo_evaluator/api/schemas.py | 13 ++ .../src/nemo_evaluator/jobs/agent_evaluate.py | 13 +- .../src/nemo_evaluator/jobs/agent_spec.py | 21 ++- .../src/nemo_evaluator/jobs/evaluate.py | 4 +- .../nemo_evaluator/jobs/metric_resolution.py | 3 +- .../src/nemo_evaluator/task_refs.py | 108 +++++++++++++++ .../tests/test_agent_evaluate.py | 15 ++- .../nemo-evaluator/tests/test_task_refs.py | 123 ++++++++++++++++++ 10 files changed, 360 insertions(+), 15 deletions(-) create mode 100644 plugins/nemo-evaluator/src/nemo_evaluator/task_refs.py create mode 100644 plugins/nemo-evaluator/tests/test_task_refs.py diff --git a/docs/evaluator/manage-tasks-tasksets.mdx b/docs/evaluator/manage-tasks-tasksets.mdx index fca9a4d35b..0a3a76ec6b 100644 --- a/docs/evaluator/manage-tasks-tasksets.mdx +++ b/docs/evaluator/manage-tasks-tasksets.mdx @@ -156,6 +156,43 @@ tasksets.delete("geography-suite") Deleting a taskset does not delete its member tasks — a taskset only holds references. +## Run an evaluation over a taskset + +An agent evaluation is submitted with an `AgentEvalInputSpec`, whose `tasks` field is either an +inline list of tasks or a **reference to a stored taskset**. Referencing a taskset lets you keep the +task definitions in one place and evaluate the whole set by name, instead of inlining every task on +each run. + +```python +from nemo_evaluator.api.schemas import TasksetRef +from nemo_evaluator.jobs.agent_spec import AgentEvalInputSpec, ModelTarget +from nemo_evaluator_sdk.values import Model +from nemo_evaluator_sdk.enums import ModelFormat + +# Instead of inlining AgentEvalTaskInput objects, point `tasks` at a stored taskset. +input_spec = AgentEvalInputSpec( + tasks=TasksetRef("default/geography-suite"), + target=ModelTarget( + model=Model(url="https://integrate.api.nvidia.com/v1", name="meta/llama-3.3-70b-instruct", format=ModelFormat.OPEN_AI), + ), +) +``` + +When the job runs, the taskset reference is resolved: its member tasks are loaded, and each task's +stored metric references are hydrated into runnable metrics — exactly as if you had inlined them. The +same spec is submitted as the agent-evaluate job input; see +[Agent Evaluation](/documentation/evaluate-models/agent-eval) for the full run, target, and +results flow. + +The inline form remains available for one-off tasks — swap `tasks=TasksetRef(...)` for +`tasks=[AgentEvalTaskInput(...), ...]`. + + +Stored tasks carry no grader-only `reference` (held-out ground truth): that field lives only on inline +`AgentEvalTaskInput`. Taskset-driven tasks therefore run with an empty `reference`, so use a taskset +when your metrics score the agent's output directly rather than against per-task held-out data. + + ## Async usage `AsyncNeMoPlatform` exposes the same surface; await each call. diff --git a/plugins/nemo-evaluator/openapi/openapi.yaml b/plugins/nemo-evaluator/openapi/openapi.yaml index 3800781728..f3286514df 100644 --- a/plugins/nemo-evaluator/openapi/openapi.yaml +++ b/plugins/nemo-evaluator/openapi/openapi.yaml @@ -1664,19 +1664,30 @@ components: title: Benchmark description: Benchmark metadata recorded with the run. tasks: - items: - $ref: '#/components/schemas/AgentEvalTaskInput' - type: array - minItems: 1 + anyOf: + - $ref: '#/components/schemas/TasksetRef' + - items: + $ref: '#/components/schemas/AgentEvalTaskInput' + type: array title: Tasks - description: Tasks to evaluate; at least one is required. + description: 'Tasks to evaluate: an inline list (at least one) or a reference + to a stored taskset.' additionalProperties: false type: object required: - tasks title: AgentEvalInputSpec - description: 'Submitter-facing agent-evaluation input: tasks whose metrics may - be inline or references.' + description: 'Submitter-facing agent-evaluation input. + + + ``tasks`` is either an inline list of tasks (whose metrics may be inline or + references) or a + + :class:`TasksetRef` naming a stored taskset whose member tasks are loaded + and expanded during spec + + resolution. Either way it hydrates to the canonical ``AgentEvalSpec.tasks`` + list.' AgentEvalResult: properties: id: @@ -4586,6 +4597,19 @@ components: description: "Create/replace body for a stored taskset (the name comes from\ \ the path).\n\nThe authorable subset of :class:`Taskset` \u2014 minus server-owned\ \ fields (id, name, workspace,\ntimestamps)." + TasksetRef: + type: string + pattern: ^[\w\-.]+(/[\w\-.]+)?$ + title: TasksetRef + description: 'Reference to a persisted taskset (format: ``workspace/name`` or + ``name``). + + + Same shape and charset as :class:`TaskRef`. Lets an evaluation reference a + stored taskset in place + + of an inline task list; the taskset''s member tasks are loaded and expanded + during spec resolution.' TasksetSort: type: string enum: diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/api/schemas.py b/plugins/nemo-evaluator/src/nemo_evaluator/api/schemas.py index 6e07c9b54c..10c5a62b60 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/api/schemas.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/api/schemas.py @@ -180,6 +180,19 @@ class TaskRef(RootModel[str]): ) +class TasksetRef(RootModel[str]): + """Reference to a persisted taskset (format: ``workspace/name`` or ``name``). + + Same shape and charset as :class:`TaskRef`. Lets an evaluation reference a stored taskset in place + of an inline task list; the taskset's member tasks are loaded and expanded during spec resolution. + """ + + root: str = Field( + pattern=_ENTITY_REF_PATTERN, + description="Reference to a stored taskset (format: workspace/taskset-name, or taskset-name in the job workspace).", + ) + + class Metric(BaseModel): """API representation of a stored metric. diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py index 15db525948..f8ca773cc7 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py @@ -20,7 +20,7 @@ import logging from dataclasses import dataclass from pathlib import Path -from typing import Any, ClassVar +from typing import Any, ClassVar, cast from urllib.parse import urlsplit import nemo_evaluator.agent_seeds # noqa: F401 - registers the platform 'fileset' workspace-seed handler @@ -39,6 +39,7 @@ from nemo_evaluator.jobs.metric_resolution import resolve_metrics_to_inline, to_runtime_bundle from nemo_evaluator.jobs.result_persistence import persist_agent_eval_result from nemo_evaluator.shared.metric_bundles.bundles import unbundle_metric +from nemo_evaluator.task_refs import resolve_agent_eval_tasks from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator from nemo_evaluator_sdk.agent_eval.persistence import persist_run from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult @@ -49,6 +50,7 @@ from nemo_evaluator_sdk.metrics.protocol import Metric from nemo_evaluator_sdk.values import RunConfigOnline, RunConfigOnlineModel from nemo_platform import AsyncNeMoPlatform, NeMoPlatform +from nemo_platform_plugin.entities import EntityClient from nemo_platform_plugin.job import NemoJob from nemo_platform_plugin.job_context import JobContext from nemo_platform_plugin.jobs.api_factory import PlatformJobSpec @@ -135,8 +137,15 @@ async def to_spec( if isinstance(input_spec, AgentEvalInputSpec) else AgentEvalInputSpec.model_validate_json(input_spec.model_dump_json()) ) + entity_client = cast(EntityClient | None, entity_client) + # A `tasks` taskset reference is loaded and expanded into inline task DTOs first, so the + # metric-ref resolution below is identical whether the tasks were submitted inline or via a + # stored taskset. + task_inputs = await resolve_agent_eval_tasks( + submit_spec.tasks, workspace=workspace, entity_client=entity_client + ) resolved_tasks: list[AgentEvalTaskSpec] = [] - for task in submit_spec.tasks: + for task in task_inputs: metrics = await resolve_metrics_to_inline( task.metrics, workspace=workspace, diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py index 58d7d9cc8b..9c59b2b473 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py @@ -18,7 +18,7 @@ # payload kind so MetricBundle payloads round-trip through validation. import nemo_evaluator.shared.metric_bundles.cloudpickle # noqa: F401 import nemo_evaluator.shared.metric_bundles.inline # noqa: F401 -from nemo_evaluator.api.schemas import MetricInline, TaskInputs, TaskMetadataList +from nemo_evaluator.api.schemas import MetricInline, TaskInputs, TaskMetadataList, TasksetRef from nemo_evaluator.jobs.metric_resolution import to_runtime_bundle, unresolved_model_refs from nemo_evaluator.metric_refs import MetricRefOrInline from nemo_evaluator.shared.metric_bundles.bundles import unbundle_metric @@ -212,9 +212,24 @@ def _require_exactly_one_trial_source(self) -> Self: class AgentEvalInputSpec(_AgentEvalSpecCommon): - """Submitter-facing agent-evaluation input: tasks whose metrics may be inline or references.""" + """Submitter-facing agent-evaluation input. - tasks: list[AgentEvalTaskInput] = Field(min_length=1, description="Tasks to evaluate; at least one is required.") + ``tasks`` is either an inline list of tasks (whose metrics may be inline or references) or a + :class:`TasksetRef` naming a stored taskset whose member tasks are loaded and expanded during spec + resolution. Either way it hydrates to the canonical ``AgentEvalSpec.tasks`` list. + """ + + tasks: TasksetRef | list[AgentEvalTaskInput] = Field( + description="Tasks to evaluate: an inline list (at least one) or a reference to a stored taskset.", + ) + + @model_validator(mode="after") + def _reject_empty_inline_tasks(self) -> Self: + # A TasksetRef is validated (and required non-empty) when it is expanded during resolution; an + # inline list must carry at least one task, mirroring the canonical spec's ``min_length=1``. + if isinstance(self.tasks, list) and not self.tasks: + raise ValueError("provide at least one task, or a `tasks` taskset reference") + return self class AgentEvalSpec(_AgentEvalSpecCommon): diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py index bb715b07ba..671d7fce70 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py @@ -9,7 +9,7 @@ import logging from dataclasses import dataclass from pathlib import Path -from typing import Annotated, Any, ClassVar, Self, TypeAlias +from typing import Annotated, Any, ClassVar, Self, TypeAlias, cast # Imported for their registration side effects: each module registers its # payload kind in the bundle registry so MetricBundle payloads validate. @@ -40,6 +40,7 @@ from nemo_evaluator_sdk.values.multi_metric_results import BenchmarkEvaluationResult from nemo_evaluator_sdk.values.results import EvaluationResult from nemo_platform import AsyncNeMoPlatform, NeMoPlatform +from nemo_platform_plugin.entities import EntityClient from nemo_platform_plugin.job import NemoJob from nemo_platform_plugin.job_context import JobContext from nemo_platform_plugin.jobs.api_factory import PlatformJobSpec @@ -229,6 +230,7 @@ async def to_spec( if isinstance(input_spec, EvaluateInputSpec) else EvaluateInputSpec.model_validate_json(input_spec.model_dump_json()) ) + entity_client = cast(EntityClient | None, entity_client) metrics = await resolve_metrics_to_inline( submit_spec.metrics, workspace=workspace, diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/metric_resolution.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/metric_resolution.py index e2beb4fd42..a853747853 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/metric_resolution.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/metric_resolution.py @@ -25,6 +25,7 @@ ) from nemo_evaluator_sdk.metrics.protocol import Metric, MetricWithModels from nemo_platform import AsyncNeMoPlatform, NeMoPlatform +from nemo_platform_plugin.entities import EntityClient def unresolved_model_refs(metrics: list[Metric]) -> list[str]: @@ -58,7 +59,7 @@ async def resolve_metrics_to_inline( metrics: list[MetricRefOrInline], *, workspace: str, - entity_client: object, + entity_client: EntityClient | None, async_sdk: AsyncNeMoPlatform | NeMoPlatform | None, ) -> list[MetricInline]: """Resolve a wire metric list (inline + stored refs) into canonical inline metrics. diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/task_refs.py b/plugins/nemo-evaluator/src/nemo_evaluator/task_refs.py new file mode 100644 index 0000000000..7447d1462e --- /dev/null +++ b/plugins/nemo-evaluator/src/nemo_evaluator/task_refs.py @@ -0,0 +1,108 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""References to persisted tasksets and their resolution into inline tasks. + +An agent-eval submission carries ``tasks`` as either an inline list of +:class:`~nemo_evaluator.jobs.agent_spec.AgentEvalTaskInput` or a +:class:`~nemo_evaluator.api.schemas.TasksetRef` pointing at a stored taskset. During spec resolution +(``AgentEvalJob.to_spec``) a taskset reference is loaded from storage and its member tasks are +expanded into the same inline task DTOs, so the rest of the pipeline (metric-ref resolution, the +canonical :class:`~nemo_evaluator.jobs.agent_spec.AgentEvalSpec`) only ever sees inline tasks. + +This mirrors :mod:`nemo_evaluator.metric_refs`: references are loaded here, next to the entity types, +so the job's ``to_spec`` stays a thin orchestration over ref-resolution helpers. +""" + +from __future__ import annotations + +from nemo_evaluator.api.schemas import TasksetRef, parse_entity_ref +from nemo_evaluator.entities import TaskEntity, TasksetEntity +from nemo_evaluator.jobs.agent_spec import AgentEvalTaskInput +from nemo_platform_plugin.entities import EntityClient, EntityNotFoundError + + +def _entity_to_task_input(entity: TaskEntity) -> AgentEvalTaskInput: + """Project a stored task onto the submitter-facing inline task DTO. + + The task's stable ``id`` is its record ``name``. A stored task holds metric *references* (inline + metrics were normalized to derived stored metrics on create); those resolve to inline bundles in + the shared metric-ref pass that runs after expansion. A stored task carries no grader-only + ``reference`` (the entity has no such field), so taskset-driven tasks run with an empty one. + """ + return AgentEvalTaskInput( + id=entity.name, + intent=entity.intent, + inputs=entity.inputs, + metrics=list(entity.metrics), + views=entity.views, + metadata=entity.metadata, + ) + + +async def resolve_taskset_ref( + ref: TasksetRef, + *, + workspace: str, + entity_client: EntityClient | None, +) -> list[AgentEvalTaskInput]: + """Load a stored taskset and expand its members into inline task DTOs. + + Loading needs only the entity store (metrics stay as refs, resolved downstream), so unlike + metric-ref resolution this does not require an async SDK / file I/O. + """ + if entity_client is None: + raise ValueError( + "A TasksetRef requires a platform connection (entity store) to resolve; it cannot be used " + "in local execution. Pass an inline task list instead." + ) + ref_workspace, name = parse_entity_ref(ref.root, workspace) + try: + taskset = await entity_client.get(TasksetEntity, name=name, workspace=ref_workspace) + except EntityNotFoundError as exc: + raise ValueError( + f"Taskset reference '{ref.root}' not found. " + f"Ensure a stored taskset named '{name}' exists in workspace '{ref_workspace}', " + "or pass an inline task list instead." + ) from exc + + if not taskset.tasks: + raise ValueError(f"Taskset '{ref.root}' has no member tasks; an agent evaluation needs at least one task.") + + tasks: list[AgentEvalTaskInput] = [] + seen_ids: set[str] = set() + for task_ref in taskset.tasks: + task_workspace, task_name = parse_entity_ref(task_ref.root, ref_workspace) + try: + entity = await entity_client.get(TaskEntity, name=task_name, workspace=task_workspace) + except EntityNotFoundError as exc: + raise ValueError( + f"Task '{task_ref.root}' referenced by taskset '{ref.root}' was not found; " + "the stored task may have been deleted after the taskset was created." + ) from exc + # Agent-eval task ids must be unique within a run. Member refs are unique per (workspace, + # name), but refs from different workspaces can share a name — surface that as a clear error + # rather than letting the SDK evaluator reject duplicate ids deeper in the run. + if entity.name in seen_ids: + raise ValueError( + f"Taskset '{ref.root}' expands to more than one task named '{entity.name}'; " + "task ids must be unique within an evaluation." + ) + seen_ids.add(entity.name) + tasks.append(_entity_to_task_input(entity)) + return tasks + + +async def resolve_agent_eval_tasks( + tasks: TasksetRef | list[AgentEvalTaskInput], + *, + workspace: str, + entity_client: EntityClient | None, +) -> list[AgentEvalTaskInput]: + """Normalize an agent-eval ``tasks`` field to an inline task list. + + An inline list passes through unchanged; a :class:`TasksetRef` is loaded and expanded. + """ + if isinstance(tasks, TasksetRef): + return await resolve_taskset_ref(tasks, workspace=workspace, entity_client=entity_client) + return tasks diff --git a/plugins/nemo-evaluator/tests/test_agent_evaluate.py b/plugins/nemo-evaluator/tests/test_agent_evaluate.py index a3413de0eb..57ff9776df 100644 --- a/plugins/nemo-evaluator/tests/test_agent_evaluate.py +++ b/plugins/nemo-evaluator/tests/test_agent_evaluate.py @@ -10,7 +10,7 @@ from typing import Any, cast import pytest -from nemo_evaluator.api.schemas import MetricInline +from nemo_evaluator.api.schemas import MetricInline, TasksetRef from nemo_evaluator.jobs.agent_evaluate import ( AGENT_BUNDLE_DIR, DEFAULT_RESULT_NAME, @@ -334,6 +334,19 @@ def test_input_spec_accepts_stored_metric_reference() -> None: assert isinstance(spec.tasks[0].metrics[0], MetricRef) +def test_input_spec_accepts_a_taskset_reference() -> None: + spec = AgentEvalInputSpec(tasks=TasksetRef("default/geo-suite"), target=CodexRunnerTarget(model="gpt-5.5")) + assert isinstance(spec.tasks, TasksetRef) + assert spec.tasks.root == "default/geo-suite" + # A JSON string round-trips back to the TasksetRef arm of the union, not a list. + assert isinstance(AgentEvalInputSpec.model_validate_json(spec.model_dump_json()).tasks, TasksetRef) + + +def test_input_spec_rejects_empty_inline_task_list() -> None: + with pytest.raises(ValueError, match="at least one task"): + AgentEvalInputSpec(tasks=[], target=CodexRunnerTarget(model="gpt-5.5")) + + async def test_to_spec_resolves_inline_task_metrics_without_a_platform() -> None: # Inline metrics need no entity client/SDK; refs would, but none are used here. input_spec = AgentEvalInputSpec( diff --git a/plugins/nemo-evaluator/tests/test_task_refs.py b/plugins/nemo-evaluator/tests/test_task_refs.py new file mode 100644 index 0000000000..a0b635c3b5 --- /dev/null +++ b/plugins/nemo-evaluator/tests/test_task_refs.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for taskset-reference resolution on the agent-eval submit path.""" + +from __future__ import annotations + +import pytest +from nemo_evaluator.api.schemas import MetadataItem, MetricRef, TaskInputs, TaskRef, TasksetRef +from nemo_evaluator.entities import TaskEntity, TasksetEntity +from nemo_evaluator.jobs.agent_spec import AgentEvalTaskInput +from nemo_evaluator.task_refs import resolve_agent_eval_tasks, resolve_taskset_ref +from nemo_platform_plugin.entities import EntityBase, EntityNotFoundError + + +class _FakeEntityClient: + """Minimal entity store keyed by (type, workspace, name), mirroring EntityClient.get.""" + + def __init__(self) -> None: + self.entities: dict[tuple[str, str, str], EntityBase] = {} + + def add(self, entity: EntityBase) -> None: + self.entities[(entity.__entity_type__, entity.workspace, entity.name)] = entity + + async def get(self, entity_cls, *, workspace, name): + key = (entity_cls.__entity_type__, workspace, name) + if key not in self.entities: + raise EntityNotFoundError(f"{workspace}/{name} not found") + return self.entities[key] + + +def _task(name: str, *, workspace: str = "default", metric: str = "default/m") -> TaskEntity: + return TaskEntity( + name=name, + workspace=workspace, + intent=f"Do {name}.", + inputs=TaskInputs(instruction=f"instruction for {name}"), + metrics=[MetricRef(metric)], + metadata=[MetadataItem(key="suite", value="geo")], + ) + + +def _taskset(name: str, task_refs: list[str], *, workspace: str = "default") -> TasksetEntity: + return TasksetEntity(name=name, workspace=workspace, tasks=[TaskRef(r) for r in task_refs]) + + +def _store(*entities: EntityBase) -> _FakeEntityClient: + client = _FakeEntityClient() + for entity in entities: + client.add(entity) + return client + + +async def test_resolves_taskset_members_to_inline_task_inputs() -> None: + client = _store( + _task("capital-of-france"), + _task("capital-of-japan"), + _taskset("geo", ["default/capital-of-france", "default/capital-of-japan"]), + ) + + tasks = await resolve_taskset_ref(TasksetRef("default/geo"), workspace="default", entity_client=client) + + assert [t.id for t in tasks] == ["capital-of-france", "capital-of-japan"] + assert all(isinstance(t, AgentEvalTaskInput) for t in tasks) + # A stored task's refs pass through untouched (resolved to inline later in the metric pass). + assert tasks[0].metrics == [MetricRef("default/m")] + assert tasks[0].intent == "Do capital-of-france." + assert tasks[0].inputs.instruction == "instruction for capital-of-france" + # A stored task carries no grader-only reference. + assert tasks[0].reference == {} + + +async def test_bare_member_ref_resolves_against_taskset_workspace() -> None: + client = _store(_task("t1", workspace="team"), _taskset("ts", ["t1"], workspace="team")) + + tasks = await resolve_taskset_ref(TasksetRef("team/ts"), workspace="default", entity_client=client) + + assert [t.id for t in tasks] == ["t1"] + + +async def test_unknown_taskset_raises_clear_error() -> None: + with pytest.raises(ValueError, match="Taskset reference 'default/missing' not found"): + await resolve_taskset_ref(TasksetRef("default/missing"), workspace="default", entity_client=_store()) + + +async def test_missing_member_task_raises_clear_error() -> None: + client = _store(_taskset("geo", ["default/gone"])) + with pytest.raises(ValueError, match="Task 'default/gone' referenced by taskset 'default/geo'"): + await resolve_taskset_ref(TasksetRef("default/geo"), workspace="default", entity_client=client) + + +async def test_empty_taskset_raises_clear_error() -> None: + client = _store(_taskset("empty", [])) + with pytest.raises(ValueError, match="has no member tasks"): + await resolve_taskset_ref(TasksetRef("default/empty"), workspace="default", entity_client=client) + + +async def test_duplicate_expanded_task_ids_rejected() -> None: + # Two members from different workspaces share the name 'dup' -> ambiguous task id. + client = _store( + _task("dup", workspace="a"), + _task("dup", workspace="b"), + _taskset("geo", ["a/dup", "b/dup"]), + ) + with pytest.raises(ValueError, match="more than one task named 'dup'"): + await resolve_taskset_ref(TasksetRef("default/geo"), workspace="default", entity_client=client) + + +async def test_taskset_ref_requires_entity_client() -> None: + with pytest.raises(ValueError, match="requires a platform connection"): + await resolve_taskset_ref(TasksetRef("default/geo"), workspace="default", entity_client=None) + + +async def test_resolve_agent_eval_tasks_passes_inline_list_through() -> None: + inline = [AgentEvalTaskInput(id="t", intent="x", metrics=[])] + result = await resolve_agent_eval_tasks(inline, workspace="default", entity_client=None) + assert result is inline + + +async def test_resolve_agent_eval_tasks_expands_a_taskset_ref() -> None: + client = _store(_task("only"), _taskset("geo", ["default/only"])) + result = await resolve_agent_eval_tasks(TasksetRef("default/geo"), workspace="default", entity_client=client) + assert [t.id for t in result] == ["only"] From ea0c60f14e0a814e161202a9877c72c990445b98 Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Tue, 21 Jul 2026 10:28:40 -0300 Subject: [PATCH 3/3] test(evaluator): end-to-end agent eval over a stored taskset ref Integration test (opt-in, RUN_AGENT_EVAL_INTEGRATION=1): stores a numeric metric + two tasks + a taskset via the SDK, then submits an agent-evaluate job whose `tasks` is a TasksetRef (no inline tasks) against a real spun platform with an IGW mock model. Proves the server resolves the taskset -> both member tasks -> each task's stored MetricRef -> inline, runs both, and scores them: asserts nan_count == 0, count == 2, mean == 1.0 (both members ran and scored, no failed samples). Adds a numeric _OutputScoreMetric (continuous_score) so the run aggregate carries a real count/mean; a boolean output would land in nan_count and obscure whether scoring actually succeeded. Signed-off-by: Sandy Chapman --- .../integration/test_agent_evaluate_job.py | 107 +++++++++++++++++- 1 file changed, 106 insertions(+), 1 deletion(-) diff --git a/plugins/nemo-evaluator/tests/integration/test_agent_evaluate_job.py b/plugins/nemo-evaluator/tests/integration/test_agent_evaluate_job.py index c876a29543..2688750901 100644 --- a/plugins/nemo-evaluator/tests/integration/test_agent_evaluate_job.py +++ b/plugins/nemo-evaluator/tests/integration/test_agent_evaluate_job.py @@ -36,7 +36,14 @@ import cloudpickle import httpx import pytest -from nemo_evaluator.api.schemas import MetricInline +from nemo_evaluator.api.schemas import ( + MetricInline, + TaskInput, + TaskInputs, + TaskRef, + TasksetInput, + TasksetRef, +) from nemo_evaluator.jobs.agent_evaluate import AgentEvalJob from nemo_evaluator.jobs.agent_spec import ( AgentEvalInputSpec, @@ -120,6 +127,31 @@ def _output_contains_metric(expected: str) -> MetricInline: return MetricInline.model_validate(bundle.model_dump(mode="json")) +class _OutputScoreMetric: + """Custom *numeric* metric: 1.0 iff the trial's output contains the expected token, else 0.0. + + The continuous-score counterpart to :class:`_OutputContainsMetric`. A numeric output aggregates + into a real ``count``/``mean`` on the run result (a boolean output lands in ``nan_count`` instead), + so a caller can assert on how many samples scored and their mean. + """ + + def __init__(self, expected: str) -> None: + self.expected = expected + + @property + def type(self) -> str: + return "output-score" + + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score("match")] + + async def compute_scores(self, input: MetricInput) -> MetricResult: # noqa: A002 + text = input.candidate.output_text or "" + return MetricResult( + outputs=[MetricOutput(name="match", value=1.0 if self.expected.lower() in text.lower() else 0.0)] + ) + + def _bundle_dir(run_result: dict) -> Path: """The persisted run bundle directory (trials/scores/summary) from a run_local result.""" return Path(run_result["artifact"]["artifact_url"].removeprefix("file://")) @@ -377,6 +409,79 @@ def test_submit_with_stored_metric_ref_resolves_and_scores(subprocess_platform: assert job.status == "completed", f"job {job_name} ended {job.status!r}: {getattr(job, 'status_details', None)}" +@pytest.mark.timeout(420) +def test_submit_over_taskset_ref_resolves_and_scores(subprocess_platform: str) -> None: + # dim 2 (stored taskset ref) x dim 3 (submit): store a metric + two tasks + a taskset, then submit + # an agent eval whose `tasks` is a TasksetRef (no inline tasks). Server-side to_spec must load the + # taskset, expand BOTH member tasks, and resolve each task's stored MetricRef — all against the + # live entity store — before the job runs. A Model target -> IGW mock provider keeps it codex-free. + client = NeMoPlatform(base_url=subprocess_platform, max_retries=2) + client.workspaces.create(name=WORKSPACE, exist_ok=True) + + model_name = _unique("taskset-model") + add_mock_provider(client, workspace=WORKSPACE, name=model_name, mock_response_body=_chat_completion("DONE")) + + # Store the metric that both tasks will reference. Numeric, so the run aggregate carries a real + # count/mean (a boolean output would land in nan_count and obscure whether scoring succeeded). The + # cloudpickle packager is explicit: storing a custom metric to the service requires opting in. + metric_name = _unique("done-score") + client.evaluator.metrics.create( + metric_name, + metric=_OutputScoreMetric("DONE"), + metric_bundle_packager=CloudpickleMetricBundlePackager(), + workspace=WORKSPACE, + ) + + # Store two tasks that reference the metric, then group them in a taskset. + task_names = [_unique("ask-a"), _unique("ask-b")] + for name in task_names: + client.evaluator.tasks.create( + name, + task=TaskInput( + intent="Obtain a one-word reply from the model.", + inputs=TaskInputs(instruction="Reply with the single word DONE and nothing else."), + metrics=[MetricRef(f"{WORKSPACE}/{metric_name}")], + ), + ) + taskset_name = _unique("done-suite") + client.evaluator.tasksets.create( + taskset_name, + taskset=TasksetInput(tasks=[TaskRef(f"{WORKSPACE}/{name}") for name in task_names]), + ) + + # The point of the test: reference the stored taskset instead of inlining the tasks. + spec = AgentEvalInputSpec( + tasks=TasksetRef(f"{WORKSPACE}/{taskset_name}"), + target=ModelTarget( + model=Model( + url=_igw_chat_url(subprocess_platform, model_name), name=model_name, format=ModelFormat.OPEN_AI + ), + prompt_template={"messages": [{"role": "user", "content": "{{item.instruction}}"}]}, + params=RunConfigOnlineModel(), + ), + ).model_dump(mode="json") + + response = NemoJobScheduler().submit_remote( + AgentEvalJob, spec, base_url=subprocess_platform, workspace=WORKSPACE, profile="default" + ) + job_name = response.get("name") or response.get("id") + assert job_name, f"submit response carried no job name/id: {response}" + + job = wait_for_platform_job(client, job_name, WORKSPACE, timeout=360) + assert job.status == "completed", f"job {job_name} ended {job.status!r}: {getattr(job, 'status_details', None)}" + + # The taskset expanded to BOTH members and both were scored: the numeric metric aggregates to + # count == number of members (one sample per task, one trial each), with no NaNs, and mean == 1.0 + # because the mock model returns "DONE" for every task (so every task's output contains "DONE"). + result = client.evaluator.agent_eval_results.retrieve(job_name, workspace=WORKSPACE) + assert (result.target_kind, result.target_name) == ("model", model_name) + assert result.scores.scores, "run produced no aggregated scores" + aggregate = result.scores.scores[0] + assert aggregate.nan_count == 0, f"metric failed to score some samples: nan_count={aggregate.nan_count}" + assert aggregate.count == len(task_names), f"expected one scored sample per member, got count={aggregate.count}" + assert aggregate.mean == 1.0, f"every member's output should score 1.0, got mean={aggregate.mean}" + + @pytest.mark.timeout(420) def test_submit_model_target_under_auth_forwards_identity_to_igw(auth_subprocess_platform: str) -> None: # dim 1 (Model target) x dim 3 (submit) under auth.enabled: the submitted task's get_task_sdk