diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/log_utils.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/log_utils.py new file mode 100644 index 0000000000..56f0a827cb --- /dev/null +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/log_utils.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Logging helpers shared across plugin API routes and services.""" + +from __future__ import annotations + + +def sanitize_for_log(value: object) -> str: + """Strip line-break/control characters from a value before logging (prevents log injection).""" + return str(value).replace("\r", "").replace("\n", "") diff --git a/plugins/nemo-evaluator/openapi/openapi.yaml b/plugins/nemo-evaluator/openapi/openapi.yaml index 8dbd113c9f..3800781728 100644 --- a/plugins/nemo-evaluator/openapi/openapi.yaml +++ b/plugins/nemo-evaluator/openapi/openapi.yaml @@ -1429,6 +1429,188 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/tasksets: + get: + tags: + - Evaluator Plugin Tasksets Routes + summary: List Tasksets By Workspace + description: List stored tasksets for a specific workspace. + operationId: list_tasksets_apis_evaluator_v2_workspaces__workspace__tasksets_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + allOf: + - $ref: '#/components/schemas/TasksetSort' + description: The field to sort by. To sort in decreasing order, use `-` + in front of the field name. + default: created_at + description: The field to sort by. To sort in decreasing order, use `-` in + front of the field name. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/TasksetFilter' + description: Filter tasksets by workspace, name, created_at, and updated_at. + responses: + '200': + description: Return stored tasksets for a workspace + content: + application/json: + schema: + $ref: '#/components/schemas/TasksetsPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/tasksets/{name}: + post: + tags: + - Evaluator Plugin Tasksets Routes + summary: Create Taskset + description: Store a new taskset, addressed by workspace/name. + operationId: create_taskset_apis_evaluator_v2_workspaces__workspace__tasksets__name__post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + maxLength: 255 + pattern: ^[\w\-\.]+$ + title: Name + - name: project + in: query + required: false + schema: + description: Optional project to associate with the taskset. + title: Project + type: string + description: Optional project to associate with the taskset. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/TasksetInput' + responses: + '201': + description: Store a new taskset + content: + application/json: + schema: + $ref: '#/components/schemas/Taskset' + '409': + description: Taskset already exists + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + get: + tags: + - Evaluator Plugin Tasksets Routes + summary: Get Taskset + description: Get a stored taskset by workspace and name. + operationId: get_taskset_apis_evaluator_v2_workspaces__workspace__tasksets__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Return stored taskset details + content: + application/json: + schema: + $ref: '#/components/schemas/Taskset' + '404': + description: Taskset not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Evaluator Plugin Tasksets Routes + summary: Delete Taskset + description: Delete a stored taskset by workspace and name. + operationId: delete_taskset_apis_evaluator_v2_workspaces__workspace__tasksets__name__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '204': + description: Delete a stored taskset + '404': + description: Taskset not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' components: schemas: AgentEvalInputSpec: @@ -4258,6 +4440,14 @@ components: agent''s prompt; the runtime falls back to the task ``intent`` when it is unset.' + TaskRef: + type: string + pattern: ^[\w\-.]+(/[\w\-.]+)?$ + title: TaskRef + description: "Reference to a persisted task (format: ``workspace/name`` or ``name``).\n\ + \nSame shape and charset as :class:`MetricRef` \u2014 a taskset points at\ + \ its member tasks by reference\n(there are no inline tasks), so a stored\ + \ taskset only ever holds refs." TaskSort: type: string enum: @@ -4293,6 +4483,144 @@ components: required: - data title: TasksPage + Taskset: + properties: + id: + type: string + title: Id + description: Unique identifier for the stored taskset record. + name: + type: string + title: Name + description: "Taskset name \u2014 the stable id, unique within its workspace." + workspace: + type: string + title: Workspace + description: Workspace the taskset belongs to. + project: + title: Project + description: The project associated with this taskset. + type: string + description: + title: Description + description: Human-readable description of the grouping. + type: string + tasks: + items: + $ref: '#/components/schemas/TaskRef' + type: array + title: Tasks + description: References to the member tasks (set semantics; duplicates rejected). + metadata: + items: + $ref: '#/components/schemas/MetadataItem' + type: array + title: Metadata + description: Key/value annotations for the taskset. + created_at: + type: string + format: date-time + title: Created At + description: Timestamp the taskset was created. + updated_at: + type: string + format: date-time + title: Updated At + description: Timestamp the taskset was last updated. + type: object + required: + - id + - name + - workspace + - created_at + - updated_at + title: Taskset + description: "API representation of a stored taskset \u2014 a flexible grouping\ + \ of tasks with metadata.\n\nMembers are referenced by ``workspace/name``\ + \ (there are no inline tasks). Membership is a set:\norder is not significant\ + \ and duplicate references are rejected." + TasksetFilter: + additionalProperties: false + description: Filter for taskset queries (top-level entity columns only; custom-field + filtering is a follow-up). + properties: + workspace: + description: Filter by workspace. + title: Workspace + type: string + name: + description: Filter by name. + title: Name + type: string + created_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Filter by creation date. + updated_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Filter by update date. + title: TasksetFilter + type: object + TasksetInput: + properties: + description: + title: Description + description: Human-readable description of the grouping. + type: string + tasks: + items: + $ref: '#/components/schemas/TaskRef' + type: array + title: Tasks + description: References to the member tasks (set semantics; duplicates rejected). + metadata: + items: + $ref: '#/components/schemas/MetadataItem' + type: array + title: Metadata + description: Key/value annotations for the taskset. + additionalProperties: false + type: object + title: TasksetInput + description: "Create/replace body for a stored taskset (the name comes from\ + \ the path).\n\nThe authorable subset of :class:`Taskset` \u2014 minus server-owned\ + \ fields (id, name, workspace,\ntimestamps)." + TasksetSort: + type: string + enum: + - name + - -name + - created_at + - -created_at + - updated_at + - -updated_at + title: TasksetSort + description: Sort fields for taskset queries. + TasksetsPage: + properties: + data: + items: + $ref: '#/components/schemas/Taskset' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: TasksetsPage ValidationError: properties: loc: diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/api/dependencies.py b/plugins/nemo-evaluator/src/nemo_evaluator/api/dependencies.py index ca2c188a17..c039b88c47 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/api/dependencies.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/api/dependencies.py @@ -9,6 +9,7 @@ from nemo_evaluator.api.service.metric_service import MetricService from nemo_evaluator.api.service.result_service import ResultService from nemo_evaluator.api.service.task_service import TaskService +from nemo_evaluator.api.service.taskset_service import TasksetService from nemo_platform import AsyncNeMoPlatform from nemo_platform_plugin.dependencies import get_entity_client, get_sdk_client from nemo_platform_plugin.entities import EntityClient @@ -36,3 +37,12 @@ def get_task_service( """Provide a TaskService. It uses the MetricService to normalize inline task metrics into (derived) stored metrics, so a persisted task holds only references.""" return TaskService(entity_client, metric_service) + + +def get_taskset_service( + entity_client: EntityClient = Depends(get_entity_client), + task_service: TaskService = Depends(get_task_service), +) -> TasksetService: + """Provide a TasksetService. It uses the TaskService to validate that each referenced task + exists when a taskset is created.""" + return TasksetService(entity_client, task_service) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/api/schemas.py b/plugins/nemo-evaluator/src/nemo_evaluator/api/schemas.py index a01e994840..6e07c9b54c 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/api/schemas.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/api/schemas.py @@ -133,16 +133,30 @@ class MetricInline(BaseModel): payload: MetricPayload = Field(description="Format-specific serialized metric.") -# A reference is ``name`` or ``workspace/name``, each segment using the platform name charset. -# Enforced on the field so empty/malformed refs are rejected at validation rather than during parsing. -_METRIC_REF_PATTERN = r"^[\w\-.]+(/[\w\-.]+)?$" +# An entity reference is ``name`` or ``workspace/name``, each segment using the platform name charset. +# Shared by every ``workspace/name`` reference type (metrics, tasks). Enforced on the field so +# empty/malformed refs are rejected at validation rather than during parsing. +_ENTITY_REF_PATTERN = r"^[\w\-.]+(/[\w\-.]+)?$" + + +def parse_entity_ref(root: str, default_workspace: str) -> tuple[str, str]: + """Split a validated ``workspace/name`` (or bare ``name``) reference into ``(workspace, name)``. + + The ``workspace/name`` vs bare-``name`` shape is guaranteed by the field's ``_ENTITY_REF_PATTERN``, + so this only needs to split. Shared by every reference type (metrics, tasks); lives here — next to + the pattern, with no entity dependency — so ref-owning modules can reuse it without cycling. + """ + workspace, separator, name = root.partition("/") + if separator: + return workspace, name + return default_workspace, root class MetricRef(RootModel[str]): """Reference to a persisted metric (format: ``workspace/name`` or ``name``).""" root: str = Field( - pattern=_METRIC_REF_PATTERN, + pattern=_ENTITY_REF_PATTERN, description="Reference to a stored metric (format: workspace/metric-name, or metric-name in the job workspace).", ) @@ -153,6 +167,19 @@ class MetricRef(RootModel[str]): MetricRefOrInline: TypeAlias = MetricInline | MetricRef +class TaskRef(RootModel[str]): + """Reference to a persisted task (format: ``workspace/name`` or ``name``). + + Same shape and charset as :class:`MetricRef` — a taskset points at its member tasks by reference + (there are no inline tasks), so a stored taskset only ever holds refs. + """ + + root: str = Field( + pattern=_ENTITY_REF_PATTERN, + description="Reference to a stored task (format: workspace/task-name, or task-name in the taskset workspace).", + ) + + class Metric(BaseModel): """API representation of a stored metric. @@ -350,3 +377,74 @@ class TaskFilter(Filter): name: str | None = Field(None, description="Filter by name.") created_at: DatetimeFilter | None = Field(None, description="Filter by creation date.") updated_at: DatetimeFilter | None = Field(None, description="Filter by update date.") + + +def _reject_duplicate_task_refs(refs: list[TaskRef]) -> list[TaskRef]: + """A taskset's members are an unordered set expressed as a list; a repeated ref is ambiguous + (it can't mean anything more than membership), so reject duplicates at validation.""" + seen: set[str] = set() + for ref in refs: + if ref.root in seen: + raise ValueError(f"duplicate task reference: {ref.root!r}") + seen.add(ref.root) + return refs + + +#: A list of task references with set semantics (order not significant, duplicates rejected). +TaskRefList: TypeAlias = Annotated[list[TaskRef], AfterValidator(_reject_duplicate_task_refs)] + + +class Taskset(BaseModel): + """API representation of a stored taskset — a flexible grouping of tasks with metadata. + + Members are referenced by ``workspace/name`` (there are no inline tasks). Membership is a set: + order is not significant and duplicate references are rejected. + """ + + id: str = Field(description="Unique identifier for the stored taskset record.") + name: str = Field(description="Taskset name — the stable id, unique within its workspace.") + workspace: str = Field(description="Workspace the taskset belongs to.") + project: str | None = Field(default=None, description="The project associated with this taskset.") + description: str | None = Field(default=None, description="Human-readable description of the grouping.") + tasks: TaskRefList = Field( + default_factory=list, description="References to the member tasks (set semantics; duplicates rejected)." + ) + metadata: TaskMetadataList = Field(default_factory=list, description="Key/value annotations for the taskset.") + created_at: datetime = Field(description="Timestamp the taskset was created.") + updated_at: datetime = Field(description="Timestamp the taskset was last updated.") + + +class TasksetInput(BaseModel): + """Create/replace body for a stored taskset (the name comes from the path). + + The authorable subset of :class:`Taskset` — minus server-owned fields (id, name, workspace, + timestamps). + """ + + model_config = ConfigDict(extra="forbid") + + description: str | None = Field(default=None, description="Human-readable description of the grouping.") + tasks: TaskRefList = Field( + default_factory=list, description="References to the member tasks (set semantics; duplicates rejected)." + ) + metadata: TaskMetadataList = Field(default_factory=list, description="Key/value annotations for the taskset.") + + +class TasksetSort(StrEnum): + """Sort fields for taskset queries.""" + + NAME_ASC = "name" + NAME_DESC = "-name" + CREATED_AT_ASC = "created_at" + CREATED_AT_DESC = "-created_at" + UPDATED_AT_ASC = "updated_at" + UPDATED_AT_DESC = "-updated_at" + + +class TasksetFilter(Filter): + """Filter for taskset queries (top-level entity columns only; custom-field filtering is a follow-up).""" + + workspace: str | None = Field(None, description="Filter by workspace.") + name: str | None = Field(None, description="Filter by name.") + created_at: DatetimeFilter | None = Field(None, description="Filter by creation date.") + updated_at: DatetimeFilter | None = Field(None, description="Filter by update date.") diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/api/service/metric_service.py b/plugins/nemo-evaluator/src/nemo_evaluator/api/service/metric_service.py index 3b85638362..1fb0741651 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/api/service/metric_service.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/api/service/metric_service.py @@ -40,6 +40,7 @@ EntityNotFoundError, ) from nemo_platform_plugin.filter_ops import FilterOperation +from nemo_platform_plugin.log_utils import sanitize_for_log from nemo_platform_plugin.schema import Page, PaginationData #: Reserved name prefix for content-addressed derived metrics (auto-stored from inline task metrics). @@ -54,11 +55,6 @@ logger = logging.getLogger(__name__) -def _sanitize_for_log(value: object) -> str: - """Strip line-break/control characters to prevent log injection.""" - return str(value).replace("\r", "").replace("\n", "") - - def _and_exclude_derived(filter_operation: FilterOperation | None) -> FilterOperation: """Combine an optional filter with "derived is not true", so derived metrics stay hidden. @@ -142,7 +138,7 @@ async def create_metric( ) -> Metric: """Store a new metric (addressed by workspace/name): upload its bundle, then index it.""" logger.debug( - "Creating metric", extra={"workspace": _sanitize_for_log(workspace), "metric_name": _sanitize_for_log(name)} + "Creating metric", extra={"workspace": sanitize_for_log(workspace), "metric_name": sanitize_for_log(name)} ) # Cheap pre-check to avoid uploading a (potentially large) bundle we would @@ -179,7 +175,7 @@ async def create_metric( logger.info( "Metric created", - extra={"workspace": _sanitize_for_log(created.workspace), "metric_name": _sanitize_for_log(created.name)}, + extra={"workspace": sanitize_for_log(created.workspace), "metric_name": sanitize_for_log(created.name)}, ) return _entity_to_schema(created) @@ -283,7 +279,7 @@ async def delete_metric(self, workspace: str, name: str) -> bool: return False await self._discard_bundle(entity.bundle_ref) logger.info( - "Metric deleted", extra={"workspace": _sanitize_for_log(workspace), "metric_name": _sanitize_for_log(name)} + "Metric deleted", extra={"workspace": sanitize_for_log(workspace), "metric_name": sanitize_for_log(name)} ) return True @@ -300,6 +296,6 @@ async def _discard_bundle(self, bundle_ref: str) -> None: except Exception: logger.warning( "Failed to delete unreferenced metric bundle fileset; storage may be leaked", - extra={"bundle_ref": _sanitize_for_log(bundle_ref)}, + extra={"bundle_ref": sanitize_for_log(bundle_ref)}, exc_info=True, ) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/api/service/task_service.py b/plugins/nemo-evaluator/src/nemo_evaluator/api/service/task_service.py index ced3afbe1f..e7e5ad6736 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/api/service/task_service.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/api/service/task_service.py @@ -19,15 +19,12 @@ from nemo_evaluator.entities import TaskEntity from nemo_platform_plugin.entities import EntityClient, EntityConflictError, EntityNotFoundError, PaginationInfo from nemo_platform_plugin.filter_ops import FilterOperation +from nemo_platform_plugin.log_utils import sanitize_for_log from nemo_platform_plugin.schema import Page, PaginationData logger = logging.getLogger(__name__) -def _sanitize_for_log(value: object) -> str: - return str(value).replace("\r", "").replace("\n", "") - - def _entity_to_task(entity: TaskEntity) -> Task: """Map a stored task entity to its API DTO, guarding the persistence timestamps.""" created_at = entity.created_at @@ -97,7 +94,7 @@ async def create_task( except EntityConflictError as exc: raise ValueError(f"Task '{workspace}/{name}' already exists") from exc logger.info( - "Task created", extra={"workspace": _sanitize_for_log(workspace), "task_name": _sanitize_for_log(name)} + "Task created", extra={"workspace": sanitize_for_log(workspace), "task_name": sanitize_for_log(name)} ) return _entity_to_task(created) @@ -135,6 +132,6 @@ async def delete_task(self, workspace: str, name: str) -> bool: except EntityNotFoundError: return False logger.info( - "Task deleted", extra={"workspace": _sanitize_for_log(workspace), "task_name": _sanitize_for_log(name)} + "Task deleted", extra={"workspace": sanitize_for_log(workspace), "task_name": sanitize_for_log(name)} ) return True diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/api/service/taskset_service.py b/plugins/nemo-evaluator/src/nemo_evaluator/api/service/taskset_service.py new file mode 100644 index 0000000000..d1ecfcf84a --- /dev/null +++ b/plugins/nemo-evaluator/src/nemo_evaluator/api/service/taskset_service.py @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CRUD service for persisted taskset entities. + +A taskset is a flexible grouping of stored tasks: it holds references to its members +(``workspace/name``) plus free-form annotations, stored whole in the entity store. Stored +``TasksetEntity`` rows are mapped to the :class:`Taskset` API DTO — the same DTO/entity split +``TaskService`` uses — so the wire contract round-trips cleanly (an ``EntityBase``'s ``id``/ +``created_at`` are computed and don't deserialize from the entity's own serialized form). + +Unlike ``TaskService`` there are no inline members to normalize; instead, each referenced task is +validated to exist at create time (a taskset that points at missing tasks is rejected). +""" + +from __future__ import annotations + +import logging + +from nemo_evaluator.api.schemas import TaskRef, Taskset, TasksetInput, parse_entity_ref +from nemo_evaluator.api.service.task_service import TaskService +from nemo_evaluator.entities import TasksetEntity +from nemo_platform_plugin.entities import EntityClient, EntityConflictError, EntityNotFoundError, PaginationInfo +from nemo_platform_plugin.filter_ops import FilterOperation +from nemo_platform_plugin.log_utils import sanitize_for_log +from nemo_platform_plugin.schema import Page, PaginationData + +logger = logging.getLogger(__name__) + + +class TaskRefNotFoundError(ValueError): + """A taskset references a task that does not exist. + + Subclasses ``ValueError`` so existing callers still catch it, while letting the route distinguish + a missing-member reference (a 422 on the submitted body) from other validation errors. + """ + + +class DuplicateTaskRefError(ValueError): + """A taskset lists two references that resolve to the same task. + + The field validator already rejects byte-identical refs; this catches refs that differ in form + but resolve to the same ``(workspace, name)`` (e.g. ``task-a`` and ``default/task-a`` when the + taskset lives in ``default``). Subclasses ``ValueError`` so the route can map it to a 422. + """ + + +class TasksetExistsError(ValueError): + """A taskset with the given workspace/name already exists. + + Subclasses ``ValueError`` so existing callers still catch it, while letting the route map a + name collision to a 409 without inspecting the message text. + """ + + +def _entity_to_taskset(entity: TasksetEntity) -> Taskset: + """Map a stored taskset entity to its API DTO, guarding the persistence timestamps.""" + created_at = entity.created_at + updated_at = entity.updated_at + if created_at is None or updated_at is None: + raise ValueError(f"Stored taskset '{entity.workspace}/{entity.name}' is missing persistence timestamps") + return Taskset( + id=entity.id, + name=entity.name, + workspace=entity.workspace, + project=entity.project, + description=entity.description, + tasks=entity.tasks, + metadata=entity.metadata, + created_at=created_at, + updated_at=updated_at, + ) + + +def _pagination(src: PaginationInfo, current_page_size: int) -> PaginationData: + """Carry the entity-store pagination counts into the API ``Page`` envelope.""" + return PaginationData( + page=src.page, + page_size=src.page_size, + current_page_size=current_page_size, + total_pages=src.total_pages, + total_results=src.total_results, + ) + + +class TasksetService: + """Create/get/list/delete for persisted taskset entities, exposed as the ``Taskset`` DTO.""" + + def __init__(self, entity_client: EntityClient, task_service: TaskService): + self.entity_client = entity_client + self.task_service = task_service + + async def _validate_tasks_exist(self, tasks: list[TaskRef], *, workspace: str) -> None: + """Validate the member refs: each must resolve to a stored task, and no two may resolve to the + same one. + + A bare ``name`` ref resolves against the taskset's own workspace; a ``workspace/name`` ref + resolves against the named workspace. Raises :class:`DuplicateTaskRefError` if two refs point + at the same ``(workspace, name)`` and :class:`TaskRefNotFoundError` if a referenced task is + missing. + """ + seen: set[tuple[str, str]] = set() + for ref in tasks: + resolved = parse_entity_ref(ref.root, workspace) + if resolved in seen: + raise DuplicateTaskRefError( + f"Task reference '{ref.root}' resolves to '{resolved[0]}/{resolved[1]}', already in this taskset" + ) + seen.add(resolved) + if await self.task_service.get_task(*resolved) is None: + raise TaskRefNotFoundError(f"Task reference '{ref.root}' not found in workspace '{resolved[0]}'") + + async def create_taskset( + self, name: str, taskset_input: TasksetInput, *, workspace: str, project: str | None = None + ) -> Taskset: + """Store a new taskset (addressed by workspace/name). + + Raises ``ValueError`` if it already exists or if any referenced task does not exist. + """ + await self._validate_tasks_exist(taskset_input.tasks, workspace=workspace) + entity = TasksetEntity( + name=name, + workspace=workspace, + project=project, + description=taskset_input.description, + tasks=taskset_input.tasks, + metadata=taskset_input.metadata, + ) + try: + created = await self.entity_client.create(entity) + except EntityConflictError as exc: + raise TasksetExistsError(f"Taskset '{workspace}/{name}' already exists") from exc + logger.info( + "Taskset created", + extra={"workspace": sanitize_for_log(workspace), "taskset_name": sanitize_for_log(name)}, + ) + return _entity_to_taskset(created) + + async def get_taskset(self, workspace: str, name: str) -> Taskset | None: + try: + entity = await self.entity_client.get(TasksetEntity, workspace=workspace, name=name) + except EntityNotFoundError: + return None + return _entity_to_taskset(entity) + + async def list_tasksets( + self, + *, + workspace: str, + page: int = 1, + page_size: int = 100, + sort: str | None = None, + filter_operation: FilterOperation | None = None, + ) -> Page[Taskset]: + result = await self.entity_client.list( + TasksetEntity, + workspace=workspace, + filter_operation=filter_operation, + sort=sort, + page=page, + page_size=page_size, + ) + data = [_entity_to_taskset(entity) for entity in result.data] + return Page(data=data, pagination=_pagination(result.pagination, len(data)), sort=sort, filter=None) + + async def delete_taskset(self, workspace: str, name: str) -> bool: + """Delete a stored taskset; ``False`` if absent.""" + try: + await self.entity_client.delete(TasksetEntity, name, workspace=workspace) + except EntityNotFoundError: + return False + logger.info( + "Taskset deleted", + extra={"workspace": sanitize_for_log(workspace), "taskset_name": sanitize_for_log(name)}, + ) + return True diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/metrics.py b/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/metrics.py index ee4612d85d..135c706de3 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/metrics.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/metrics.py @@ -23,16 +23,12 @@ from nemo_platform_plugin.authz import CallerKind, PermissionSet, path_rule, perm from nemo_platform_plugin.entities import EntityValidationError from nemo_platform_plugin.jobs.openapi_utils import generate_openapi_extra_params +from nemo_platform_plugin.log_utils import sanitize_for_log from nemo_platform_plugin.schema import Page logger = logging.getLogger(__name__) -def _sanitize_for_log(value: object) -> str: - """Prevent log injection by removing line-break/control characters.""" - return str(value).replace("\r", "").replace("\n", "") - - class MetricPerms(PermissionSet, namespace="evaluator.metrics"): """Permissions for the stored-metrics CRUD collection.""" @@ -87,7 +83,7 @@ async def list_metrics( include_derived=include_derived, ) except Exception: - logger.exception(f"Failed to list metrics for workspace {_sanitize_for_log(workspace)}") + logger.exception(f"Failed to list metrics for workspace {sanitize_for_log(workspace)}") raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") @@ -108,8 +104,8 @@ async def create_metric( service: MetricService = Depends(get_metric_service), ) -> Metric: """Store a new metric, addressed by workspace/name.""" - safe_workspace = _sanitize_for_log(workspace) - safe_name = _sanitize_for_log(name) + safe_workspace = sanitize_for_log(workspace) + safe_name = sanitize_for_log(name) logger.info(f"Creating metric: {safe_workspace}/{safe_name}") try: return await service.create_metric(name, metric, workspace=workspace, project=project) @@ -147,7 +143,7 @@ async def get_metric( service: MetricService = Depends(get_metric_service), ) -> Metric: """Get a stored metric by workspace and name.""" - logger.debug(f"Getting metric: {_sanitize_for_log(workspace)}/{_sanitize_for_log(name)}") + logger.debug(f"Getting metric: {sanitize_for_log(workspace)}/{sanitize_for_log(name)}") try: metric = await service.get_metric(workspace, name) if not metric: @@ -159,7 +155,7 @@ async def get_metric( except HTTPException: raise except Exception: - logger.exception(f"Failed to get metric {_sanitize_for_log(workspace)}/{_sanitize_for_log(name)}") + logger.exception(f"Failed to get metric {sanitize_for_log(workspace)}/{sanitize_for_log(name)}") raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") @@ -178,7 +174,7 @@ async def delete_metric( service: MetricService = Depends(get_metric_service), ): """Delete a stored metric by workspace and name.""" - logger.info(f"Deleting metric: {_sanitize_for_log(workspace)}/{_sanitize_for_log(name)}") + logger.info(f"Deleting metric: {sanitize_for_log(workspace)}/{sanitize_for_log(name)}") try: deleted = await service.delete_metric(workspace, name) if not deleted: @@ -190,5 +186,5 @@ async def delete_metric( except HTTPException: raise except Exception: - logger.exception(f"Failed to delete metric {_sanitize_for_log(workspace)}/{_sanitize_for_log(name)}") + logger.exception(f"Failed to delete metric {sanitize_for_log(workspace)}/{sanitize_for_log(name)}") raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/results.py b/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/results.py index a2ac850a85..7bb5810a5d 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/results.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/results.py @@ -26,6 +26,7 @@ from nemo_platform_plugin.api.parsed_filter import ParsedFilter, make_filter_dep from nemo_platform_plugin.authz import CallerKind, PermissionSet, path_rule, perm from nemo_platform_plugin.jobs.openapi_utils import generate_openapi_extra_params +from nemo_platform_plugin.log_utils import sanitize_for_log from nemo_platform_plugin.schema import DatetimeFilter, Page logger = logging.getLogger(__name__) @@ -58,10 +59,6 @@ class ResultSort(StrEnum): ) -def _sanitize_for_log(value: object) -> str: - return str(value).replace("\r", "").replace("\n", "") - - class ResultFilter(DataFilter): """Traits shared by both result collections (used directly for agent-eval results).""" @@ -115,7 +112,7 @@ async def list_agent_eval_results( workspace=workspace, page=page, page_size=page_size, sort=sort, filter_operation=parsed_filter.operation ) except Exception: - logger.exception(f"Failed to list agent-eval results for workspace {_sanitize_for_log(workspace)}") + logger.exception(f"Failed to list agent-eval results for workspace {sanitize_for_log(workspace)}") raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") @@ -143,7 +140,7 @@ async def get_agent_eval_result( except HTTPException: raise except Exception: - logger.exception(f"Failed to get agent-eval result {_sanitize_for_log(workspace)}/{_sanitize_for_log(name)}") + logger.exception(f"Failed to get agent-eval result {sanitize_for_log(workspace)}/{sanitize_for_log(name)}") raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") @@ -168,7 +165,7 @@ async def delete_agent_eval_result( except HTTPException: raise except Exception: - logger.exception(f"Failed to delete agent-eval result {_sanitize_for_log(workspace)}/{_sanitize_for_log(name)}") + logger.exception(f"Failed to delete agent-eval result {sanitize_for_log(workspace)}/{sanitize_for_log(name)}") raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") @@ -203,7 +200,7 @@ async def list_eval_results( workspace=workspace, page=page, page_size=page_size, sort=sort, filter_operation=parsed_filter.operation ) except Exception: - logger.exception(f"Failed to list eval results for workspace {_sanitize_for_log(workspace)}") + logger.exception(f"Failed to list eval results for workspace {sanitize_for_log(workspace)}") raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") @@ -231,7 +228,7 @@ async def get_eval_result( except HTTPException: raise except Exception: - logger.exception(f"Failed to get eval result {_sanitize_for_log(workspace)}/{_sanitize_for_log(name)}") + logger.exception(f"Failed to get eval result {sanitize_for_log(workspace)}/{sanitize_for_log(name)}") raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") @@ -256,5 +253,5 @@ async def delete_eval_result( except HTTPException: raise except Exception: - logger.exception(f"Failed to delete eval result {_sanitize_for_log(workspace)}/{_sanitize_for_log(name)}") + logger.exception(f"Failed to delete eval result {sanitize_for_log(workspace)}/{sanitize_for_log(name)}") raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/tasks.py b/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/tasks.py index fd3119c9de..cbd6d8fef7 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/tasks.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/tasks.py @@ -18,6 +18,7 @@ from nemo_platform_plugin.authz import CallerKind, PermissionSet, path_rule, perm from nemo_platform_plugin.entities import EntityValidationError from nemo_platform_plugin.jobs.openapi_utils import generate_openapi_extra_params +from nemo_platform_plugin.log_utils import sanitize_for_log from nemo_platform_plugin.schema import Page logger = logging.getLogger(__name__) @@ -32,11 +33,6 @@ class TaskPerms(PermissionSet, namespace="evaluator.tasks"): DELETE = perm("Delete a stored task") -def _sanitize_for_log(value: object) -> str: - """Prevent log injection by removing line-break/control characters.""" - return str(value).replace("\r", "").replace("\n", "") - - router = APIRouter() @@ -77,7 +73,7 @@ async def list_tasks( filter_operation=parsed_filter.operation, ) except Exception: - logger.exception(f"Failed to list tasks for workspace {_sanitize_for_log(workspace)}") + logger.exception(f"Failed to list tasks for workspace {sanitize_for_log(workspace)}") raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") @@ -98,8 +94,8 @@ async def create_task( service: TaskService = Depends(get_task_service), ) -> Task: """Store a new task, addressed by workspace/name.""" - safe_workspace = _sanitize_for_log(workspace) - safe_name = _sanitize_for_log(name) + safe_workspace = sanitize_for_log(workspace) + safe_name = sanitize_for_log(name) logger.info(f"Creating task: {safe_workspace}/{safe_name}") try: return await service.create_task(name, task, workspace=workspace, project=project) @@ -137,7 +133,7 @@ async def get_task( service: TaskService = Depends(get_task_service), ) -> Task: """Get a stored task by workspace and name.""" - logger.debug(f"Getting task: {_sanitize_for_log(workspace)}/{_sanitize_for_log(name)}") + logger.debug(f"Getting task: {sanitize_for_log(workspace)}/{sanitize_for_log(name)}") try: task = await service.get_task(workspace, name) if not task: @@ -149,7 +145,7 @@ async def get_task( except HTTPException: raise except Exception: - logger.exception(f"Failed to get task {_sanitize_for_log(workspace)}/{_sanitize_for_log(name)}") + logger.exception(f"Failed to get task {sanitize_for_log(workspace)}/{sanitize_for_log(name)}") raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") @@ -168,7 +164,7 @@ async def delete_task( service: TaskService = Depends(get_task_service), ): """Delete a stored task by workspace and name.""" - logger.info(f"Deleting task: {_sanitize_for_log(workspace)}/{_sanitize_for_log(name)}") + logger.info(f"Deleting task: {sanitize_for_log(workspace)}/{sanitize_for_log(name)}") try: deleted = await service.delete_task(workspace, name) if not deleted: @@ -180,5 +176,5 @@ async def delete_task( except HTTPException: raise except Exception: - logger.exception(f"Failed to delete task {_sanitize_for_log(workspace)}/{_sanitize_for_log(name)}") + logger.exception(f"Failed to delete task {sanitize_for_log(workspace)}/{sanitize_for_log(name)}") raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/tasksets.py b/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/tasksets.py new file mode 100644 index 0000000000..f1cf350527 --- /dev/null +++ b/plugins/nemo-evaluator/src/nemo_evaluator/api/v2/tasksets.py @@ -0,0 +1,182 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CRUD routes for stored tasksets under /apis/evaluator/v2/workspaces/{workspace}/tasksets.""" + +from __future__ import annotations + +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, HTTPException, Path, Query, status +from nemo_evaluator.api.dependencies import get_taskset_service +from nemo_evaluator.api.schemas import Taskset, TasksetFilter, TasksetInput, TasksetSort +from nemo_evaluator.api.service.taskset_service import ( + DuplicateTaskRefError, + TaskRefNotFoundError, + TasksetExistsError, + TasksetService, +) +from nemo_evaluator.authz import scope +from nemo_evaluator.entities import MAX_NAME_LENGTH, NAME_PATTERN +from nemo_platform_plugin.api.parsed_filter import ParsedFilter, make_filter_dep +from nemo_platform_plugin.authz import CallerKind, PermissionSet, path_rule, perm +from nemo_platform_plugin.entities import EntityValidationError +from nemo_platform_plugin.jobs.openapi_utils import generate_openapi_extra_params +from nemo_platform_plugin.log_utils import sanitize_for_log +from nemo_platform_plugin.schema import Page + +logger = logging.getLogger(__name__) + + +class TasksetPerms(PermissionSet, namespace="evaluator.tasksets"): + """Permissions for the stored-taskset CRUD collection.""" + + CREATE = perm("Create a stored taskset") + LIST = perm("List stored tasksets") + READ = perm("Read a stored taskset") + DELETE = perm("Delete a stored taskset") + + +router = APIRouter() + + +@router.get( + "/tasksets", + summary="List Tasksets By Workspace", + response_description="Return stored tasksets for a workspace", + status_code=status.HTTP_200_OK, + response_model=Page[Taskset], + response_model_exclude_none=True, + openapi_extra=generate_openapi_extra_params( + filter_schema=TasksetFilter, + filter_description="Filter tasksets by workspace, name, created_at, and updated_at.", + ), +) +@scope.read +@path_rule(callers=[CallerKind.PRINCIPAL], permissions=[TasksetPerms.LIST]) +async def list_tasksets( + workspace: str, + page: int = Query(default=1, ge=1, description="Page number."), + page_size: int = Query(default=100, ge=1, le=1000, description="Page size."), + sort: TasksetSort = Query( + default=TasksetSort.CREATED_AT_ASC, + description="The field to sort by. To sort in decreasing order, use `-` in front of the field name.", + ), + parsed_filter: ParsedFilter = Depends(make_filter_dep(TasksetFilter)), + service: TasksetService = Depends(get_taskset_service), +) -> Page[Taskset]: + """List stored tasksets for a specific workspace.""" + # Discard any workspace override in the filter — always scope to the path workspace. + parsed_filter.remove("workspace") + try: + return await service.list_tasksets( + workspace=workspace, + page=page, + page_size=page_size, + sort=sort, + filter_operation=parsed_filter.operation, + ) + except Exception: + logger.exception(f"Failed to list tasksets for workspace {sanitize_for_log(workspace)}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") + + +@router.post( + "/tasksets/{name}", + summary="Create Taskset", + response_description="Store a new taskset", + status_code=status.HTTP_201_CREATED, + responses={status.HTTP_409_CONFLICT: {"description": "Taskset already exists"}}, +) +@scope.write +@path_rule(callers=[CallerKind.PRINCIPAL], permissions=[TasksetPerms.CREATE]) +async def create_taskset( + workspace: str, + name: Annotated[str, Path(max_length=MAX_NAME_LENGTH, pattern=NAME_PATTERN)], + taskset: TasksetInput, + project: str | None = Query(default=None, description="Optional project to associate with the taskset."), + service: TasksetService = Depends(get_taskset_service), +) -> Taskset: + """Store a new taskset, addressed by workspace/name.""" + safe_workspace = sanitize_for_log(workspace) + safe_name = sanitize_for_log(name) + logger.info(f"Creating taskset: {safe_workspace}/{safe_name}") + try: + return await service.create_taskset(name, taskset, workspace=workspace, project=project) + except EntityValidationError as e: + logger.warning(f"Entity store validation error during taskset creation: {e}") + raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e)) + except (TaskRefNotFoundError, DuplicateTaskRefError) as e: + # A bad member reference (missing task, or two refs to the same task) — a client error. + logger.warning(f"Taskset has an invalid task reference: {e}") + raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e)) + except TasksetExistsError: + logger.warning(f"Taskset already exists: {safe_workspace}/{safe_name}") + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Taskset with workspace '{workspace}' and name '{name}' already exists", + ) + except ValueError as e: + logger.warning(f"Taskset creation validation error: {e}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid taskset data") + except HTTPException: + raise + except Exception: + logger.exception("Failed to create taskset") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") + + +@router.get( + "/tasksets/{name}", + summary="Get Taskset", + response_description="Return stored taskset details", + status_code=status.HTTP_200_OK, + responses={status.HTTP_404_NOT_FOUND: {"description": "Taskset not found"}}, +) +@scope.read +@path_rule(callers=[CallerKind.PRINCIPAL], permissions=[TasksetPerms.READ]) +async def get_taskset( + workspace: str, + name: str, + service: TasksetService = Depends(get_taskset_service), +) -> Taskset: + """Get a stored taskset by workspace and name.""" + logger.debug(f"Getting taskset: {sanitize_for_log(workspace)}/{sanitize_for_log(name)}") + # Only the service call can fail unexpectedly; wrap just that so the 404 below is raised outside + # the try (no catching HTTPException only to re-raise it). + try: + taskset = await service.get_taskset(workspace, name) + except Exception: + logger.exception(f"Failed to get taskset {sanitize_for_log(workspace)}/{sanitize_for_log(name)}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") + if not taskset: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Taskset not found: {workspace}/{name}") + return taskset + + +@router.delete( + "/tasksets/{name}", + summary="Delete Taskset", + response_description="Delete a stored taskset", + status_code=status.HTTP_204_NO_CONTENT, + responses={status.HTTP_404_NOT_FOUND: {"description": "Taskset not found"}}, +) +@scope.write +@path_rule(callers=[CallerKind.PRINCIPAL], permissions=[TasksetPerms.DELETE]) +async def delete_taskset( + workspace: str, + name: str, + service: TasksetService = Depends(get_taskset_service), +): + """Delete a stored taskset by workspace and name.""" + logger.info(f"Deleting taskset: {sanitize_for_log(workspace)}/{sanitize_for_log(name)}") + # Wrap only the service call so the 404 below is raised outside the try (no catch-and-re-raise). + try: + deleted = await service.delete_taskset(workspace, name) + except Exception: + logger.exception(f"Failed to delete taskset {sanitize_for_log(workspace)}/{sanitize_for_log(name)}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal server error") + if not deleted: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Taskset not found: {workspace}/{name}") + return None diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/entities.py b/plugins/nemo-evaluator/src/nemo_evaluator/entities.py index d2145ebdf5..c91d5d9f48 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/entities.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/entities.py @@ -19,7 +19,7 @@ from typing import ClassVar -from nemo_evaluator.api.schemas import MetricRef, TaskInputs, TaskMetadataList +from nemo_evaluator.api.schemas import MetricRef, TaskInputs, TaskMetadataList, TaskRefList from nemo_evaluator.shared.metric_bundles.bundles import BundledMetricOutputSpec from nemo_evaluator_sdk.agent_eval.tasks import SemanticView from nemo_evaluator_sdk.values.common import SecretRef @@ -187,3 +187,25 @@ class TaskEntity(EntityBase): description="Optional reporting views mapping this task's metric outputs into named semantic scores.", ) metadata: TaskMetadataList = Field(default_factory=list, description="Key/value annotations for the task.") + + +class TasksetEntity(EntityBase): + """Persisted, queryable taskset, addressed by workspace/name. + + A taskset is a flexible grouping of stored tasks: it holds references to its members + (``workspace/name``) plus free-form annotations. Membership is a set — order is not significant + and duplicate references are rejected. Referenced tasks are validated to exist at create time. + """ + + __entity_type__: ClassVar[str] = "taskset" + + description: str | None = Field( + default=None, + description="Human-readable description of the grouping.", + max_length=MAX_DESCRIPTION_LENGTH, + ) + tasks: TaskRefList = Field( + default_factory=list, + description="References to the member tasks (set semantics; duplicates rejected).", + ) + metadata: TaskMetadataList = Field(default_factory=list, description="Key/value annotations for the taskset.") diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/metric_refs.py b/plugins/nemo-evaluator/src/nemo_evaluator/metric_refs.py index 6c2dd61f1f..5c6daeb63a 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/metric_refs.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/metric_refs.py @@ -18,7 +18,7 @@ # entity/DTO modules can reference them without importing this module's entities-dependent resolution # logic (which would create an import cycle). Imported here for use below and re-exported for the # existing ``nemo_evaluator.metric_refs`` import sites. -from nemo_evaluator.api.schemas import MetricRef, MetricRefOrInline +from nemo_evaluator.api.schemas import MetricRef, MetricRefOrInline, parse_entity_ref from nemo_evaluator.entities import MetricBundleEntity from nemo_evaluator.metric_storage import load_bundle from nemo_evaluator.shared.metric_bundles.bundles import MetricBundle @@ -29,13 +29,10 @@ def parse_metric_ref(root: str, default_workspace: str) -> tuple[str, str]: """Split a validated metric reference into ``(workspace, name)``. - The ``workspace/name`` vs bare-``name`` shape is guaranteed by - :class:`MetricRef`'s field pattern, so this only needs to split. + Thin alias over the shared :func:`~nemo_evaluator.api.schemas.parse_entity_ref` (all + ``workspace/name`` refs split identically); kept for the existing ``metric_refs`` call sites. """ - workspace, separator, name = root.partition("/") - if separator: - return workspace, name - return default_workspace, root + return parse_entity_ref(root, default_workspace) async def resolve_metric_ref( diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py index 5c8dac3213..9f6e04bc17 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py @@ -33,6 +33,10 @@ AsyncEvaluatorTasksResource, EvaluatorTasksResource, ) +from nemo_evaluator.sdk.taskset_resources import ( + AsyncEvaluatorTasksetsResource, + EvaluatorTasksetsResource, +) from nemo_evaluator.sdk.types import ( PluginDatasetInput, RunConfig, @@ -66,6 +70,7 @@ def __init__(self, platform: NeMoPlatform) -> None: self.agent_eval_results = EvaluatorAgentEvalResultsResource(platform) self.eval_results = EvaluatorEvalResultsResource(platform) self.tasks = EvaluatorTasksResource(platform) + self.tasksets = EvaluatorTasksetsResource(platform) def plugin_status(self) -> dict[str, object]: """Return evaluator plugin health information from the service.""" @@ -235,6 +240,7 @@ def __init__(self, platform: AsyncNeMoPlatform) -> None: self.agent_eval_results = AsyncEvaluatorAgentEvalResultsResource(platform) self.eval_results = AsyncEvaluatorEvalResultsResource(platform) self.tasks = AsyncEvaluatorTasksResource(platform) + self.tasksets = AsyncEvaluatorTasksetsResource(platform) async def plugin_status(self) -> dict[str, object]: """Return evaluator plugin health information from the service.""" diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/taskset_resources.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/taskset_resources.py new file mode 100644 index 0000000000..8526992fd9 --- /dev/null +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/taskset_resources.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SDK resources for managing stored tasksets (``client.evaluator.tasksets``). + +Thin client over the evaluator service's ``/tasksets`` create/get/list/delete API. A taskset is sent +as a :class:`TasksetInput` (its members as references to stored tasks) and returned as the +:class:`Taskset` DTO; the service owns persistence in the entity store. +""" + +from __future__ import annotations + +from urllib.parse import quote + +from nemo_evaluator.api.schemas import Taskset, TasksetInput +from nemo_evaluator.sdk import http_utils +from nemo_platform import AsyncNeMoPlatform, NeMoPlatform +from nemo_platform_plugin.schema import Page + + +def _list_params(page: int, page_size: int, sort: str | None) -> dict[str, str | int]: + params: dict[str, str | int] = {"page": page, "page_size": page_size} + if sort is not None: + params["sort"] = sort + return params + + +class EvaluatorTasksetsResource: + """Sync resource mounted as ``client.evaluator.tasksets``.""" + + def __init__(self, platform: NeMoPlatform) -> None: + self._platform = platform + self._http_client = platform._client + + def _headers(self) -> dict[str, str]: + return http_utils.platform_default_headers(self._platform) + + def _collection_url(self, workspace: str | None) -> str: + return http_utils.url(self._platform, "/v2/workspaces/{workspace}/tasksets", workspace) + + def _item_url(self, name: str, workspace: str | None) -> str: + return http_utils.url( + self._platform, f"/v2/workspaces/{{workspace}}/tasksets/{quote(name, safe='')}", workspace + ) + + def create( + self, name: str, *, taskset: TasksetInput, project: str | None = None, workspace: str | None = None + ) -> Taskset: + """Store a new taskset (addressed by workspace/name).""" + response = self._http_client.post( + self._item_url(name, workspace), + json=taskset.model_dump(mode="json"), + params={"project": project} if project is not None else None, + headers=self._headers(), + timeout=self._platform.timeout, + ) + response.raise_for_status() + return Taskset.model_validate(response.json()) + + def retrieve(self, name: str, *, workspace: str | None = None) -> Taskset: + """Get a stored taskset by name.""" + response = self._http_client.get( + self._item_url(name, workspace), headers=self._headers(), timeout=self._platform.timeout + ) + response.raise_for_status() + return Taskset.model_validate(response.json()) + + def list( + self, *, workspace: str | None = None, page: int = 1, page_size: int = 100, sort: str | None = None + ) -> Page[Taskset]: + """List stored tasksets in a workspace.""" + response = self._http_client.get( + self._collection_url(workspace), + params=_list_params(page, page_size, sort), + headers=self._headers(), + timeout=self._platform.timeout, + ) + response.raise_for_status() + return Page[Taskset].model_validate(response.json()) + + def delete(self, name: str, *, workspace: str | None = None) -> None: + """Delete a stored taskset by name.""" + response = self._http_client.delete( + self._item_url(name, workspace), headers=self._headers(), timeout=self._platform.timeout + ) + response.raise_for_status() + + +class AsyncEvaluatorTasksetsResource: + """Async resource mounted as ``client.evaluator.tasksets``.""" + + def __init__(self, platform: AsyncNeMoPlatform) -> None: + self._platform = platform + self._http_client = platform._client + + def _headers(self) -> dict[str, str]: + return http_utils.platform_default_headers(self._platform) + + def _collection_url(self, workspace: str | None) -> str: + return http_utils.url(self._platform, "/v2/workspaces/{workspace}/tasksets", workspace) + + def _item_url(self, name: str, workspace: str | None) -> str: + return http_utils.url( + self._platform, f"/v2/workspaces/{{workspace}}/tasksets/{quote(name, safe='')}", workspace + ) + + async def create( + self, name: str, *, taskset: TasksetInput, project: str | None = None, workspace: str | None = None + ) -> Taskset: + """Store a new taskset (addressed by workspace/name).""" + response = await self._http_client.post( + self._item_url(name, workspace), + json=taskset.model_dump(mode="json"), + params={"project": project} if project is not None else None, + headers=self._headers(), + timeout=self._platform.timeout, + ) + response.raise_for_status() + return Taskset.model_validate(response.json()) + + async def retrieve(self, name: str, *, workspace: str | None = None) -> Taskset: + """Get a stored taskset by name.""" + response = await self._http_client.get( + self._item_url(name, workspace), headers=self._headers(), timeout=self._platform.timeout + ) + response.raise_for_status() + return Taskset.model_validate(response.json()) + + async def list( + self, *, workspace: str | None = None, page: int = 1, page_size: int = 100, sort: str | None = None + ) -> Page[Taskset]: + """List stored tasksets in a workspace.""" + response = await self._http_client.get( + self._collection_url(workspace), + params=_list_params(page, page_size, sort), + headers=self._headers(), + timeout=self._platform.timeout, + ) + response.raise_for_status() + return Page[Taskset].model_validate(response.json()) + + async def delete(self, name: str, *, workspace: str | None = None) -> None: + """Delete a stored taskset by name.""" + response = await self._http_client.delete( + self._item_url(name, workspace), headers=self._headers(), timeout=self._platform.timeout + ) + response.raise_for_status() diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/service.py b/plugins/nemo-evaluator/src/nemo_evaluator/service.py index ed4a2083b1..16dd7b22dd 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/service.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/service.py @@ -11,6 +11,7 @@ from nemo_evaluator.api.v2 import metrics as metrics_routes from nemo_evaluator.api.v2 import results as results_routes from nemo_evaluator.api.v2 import tasks as tasks_routes +from nemo_evaluator.api.v2 import tasksets as tasksets_routes from nemo_evaluator.authz import scope from nemo_evaluator.core import say_hello from nemo_evaluator.jobs.agent_evaluate import AgentEvalJob @@ -109,6 +110,13 @@ async def healthz() -> dict[str, object]: description="Stored agent-eval task CRUD routes.", prefix="/v2/workspaces/{workspace}", ), + RouterSpec( + # CRUD /apis/evaluator/v2/workspaces/{workspace}/tasksets. + router=tasksets_routes.router, + tag="Evaluator Plugin Tasksets Routes", + description="Stored taskset CRUD routes.", + prefix="/v2/workspaces/{workspace}", + ), ] diff --git a/plugins/nemo-evaluator/tests/api/service/test_taskset_service.py b/plugins/nemo-evaluator/tests/api/service/test_taskset_service.py new file mode 100644 index 0000000000..8ce7d511f3 --- /dev/null +++ b/plugins/nemo-evaluator/tests/api/service/test_taskset_service.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest +from nemo_evaluator.api.schemas import TaskRef, Taskset, TasksetInput +from nemo_evaluator.api.service.taskset_service import ( + DuplicateTaskRefError, + TaskRefNotFoundError, + TasksetExistsError, + TasksetService, +) +from nemo_platform_plugin.entities import ( + EntityBase, + EntityConflictError, + EntityNotFoundError, + ListResponse, + PaginationInfo, +) + + +class _FakeTaskService: + """Stands in for TaskService's existence check; ``get_task`` resolves only known (workspace, name).""" + + def __init__(self, existing: set[tuple[str, str]]) -> None: + self.existing = existing + + async def get_task(self, workspace: str, name: str) -> object | None: + return object() if (workspace, name) in self.existing else None + + +class _FakeEntityClient: + def __init__(self) -> None: + self.entities: dict[tuple[str, str, str], EntityBase] = {} + + async def create(self, entity): + key = (entity.__entity_type__, entity.workspace, entity.name) + if key in self.entities: + raise EntityConflictError(f"{key} exists") + now = datetime.now(timezone.utc) + entity._id = f"{entity.__entity_type__}-{entity.name}" + entity._created_at = now + entity._updated_at = now + self.entities[key] = entity + return entity + + async def get(self, entity_cls, *, workspace, name): + key = (entity_cls.__entity_type__, workspace, name) + if key not in self.entities: + raise EntityNotFoundError(f"{workspace}/{name} not found") + return self.entities[key] + + async def delete(self, entity_cls, name, *, workspace): + key = (entity_cls.__entity_type__, workspace, name) + if key not in self.entities: + raise EntityNotFoundError(f"{workspace}/{name} not found") + del self.entities[key] + + async def list(self, entity_cls, *, workspace, filter_operation=None, sort=None, page=1, page_size=100): + items = [ + e for (etype, ws, _), e in self.entities.items() if etype == entity_cls.__entity_type__ and ws == workspace + ] + return ListResponse( + data=items, + pagination=PaginationInfo( + page=page, + page_size=page_size, + current_page_size=len(items), + total_pages=1, + total_results=len(items), + ), + ) + + +def _taskset_input() -> TasksetInput: + return TasksetInput( + description="A smoke-test grouping.", + tasks=[TaskRef("task-a"), TaskRef("default/task-b")], + metadata=[{"key": "suite", "value": "smoke"}], + ) + + +@pytest.fixture +def existing_tasks() -> set[tuple[str, str]]: + return {("default", "task-a"), ("default", "task-b")} + + +@pytest.fixture +def service(existing_tasks: set[tuple[str, str]]) -> TasksetService: + return TasksetService(_FakeEntityClient(), _FakeTaskService(existing_tasks)) + + +async def test_create_then_get(service: TasksetService) -> None: + created = await service.create_taskset("ts-1", _taskset_input(), workspace="default") + + assert isinstance(created, Taskset) + assert created.name == "ts-1" + assert created.id == "taskset-ts-1" + assert created.description == "A smoke-test grouping." + assert {t.root for t in created.tasks} == {"task-a", "default/task-b"} + assert created.created_at is not None + + got = await service.get_taskset("default", "ts-1") + assert got is not None and got.name == "ts-1" + + +async def test_create_validates_missing_task_ref(service: TasksetService) -> None: + taskset_input = TasksetInput(tasks=[TaskRef("task-a"), TaskRef("nope")]) + with pytest.raises(TaskRefNotFoundError, match="not found"): + await service.create_taskset("ts-1", taskset_input, workspace="default") + + +async def test_create_resolves_bare_ref_against_taskset_workspace(existing_tasks: set[tuple[str, str]]) -> None: + # A bare "task-a" ref must resolve against the taskset's own workspace ("other"), where it is absent. + service = TasksetService(_FakeEntityClient(), _FakeTaskService(existing_tasks)) + with pytest.raises(ValueError, match="not found in workspace 'other'"): + await service.create_taskset("ts-1", TasksetInput(tasks=[TaskRef("task-a")]), workspace="other") + + +async def test_create_rejects_refs_resolving_to_same_task(service: TasksetService) -> None: + # "task-a" and "default/task-a" resolve to the same (default, task-a) — rejected even though the + # ref strings differ (the field validator only catches byte-identical dupes). + taskset_input = TasksetInput(tasks=[TaskRef("task-a"), TaskRef("default/task-a")]) + with pytest.raises(DuplicateTaskRefError, match="already in this taskset"): + await service.create_taskset("ts-1", taskset_input, workspace="default") + + +async def test_create_rejects_duplicate(service: TasksetService) -> None: + await service.create_taskset("ts-1", _taskset_input(), workspace="default") + with pytest.raises(TasksetExistsError, match="already exists"): + await service.create_taskset("ts-1", _taskset_input(), workspace="default") + + +async def test_create_allows_same_name_in_different_workspaces(service: TasksetService) -> None: + # Taskset names are unique per workspace, not globally: the same name in another workspace is a + # distinct taskset and must not raise TasksetExistsError (409). Empty task lists keep this focused + # on name scoping rather than per-workspace task-ref validation. + first = await service.create_taskset("ts-1", TasksetInput(), workspace="default") + second = await service.create_taskset("ts-1", TasksetInput(), workspace="other") + + assert first.name == second.name == "ts-1" + assert first.workspace == "default" + assert second.workspace == "other" + + +async def test_get_returns_none_when_missing(service: TasksetService) -> None: + assert await service.get_taskset("default", "nope") is None + + +async def test_list_returns_workspace_tasksets(service: TasksetService) -> None: + await service.create_taskset("a", _taskset_input(), workspace="default") + await service.create_taskset("b", _taskset_input(), workspace="default") + + page = await service.list_tasksets(workspace="default") + + assert {t.name for t in page.data} == {"a", "b"} + assert page.pagination is not None and page.pagination.total_results == 2 + + +async def test_delete(service: TasksetService) -> None: + await service.create_taskset("ts-1", _taskset_input(), workspace="default") + assert await service.delete_taskset("default", "ts-1") is True + assert await service.get_taskset("default", "ts-1") is None + + +async def test_delete_returns_false_when_missing(service: TasksetService) -> None: + assert await service.delete_taskset("default", "nope") is False diff --git a/plugins/nemo-evaluator/tests/api/v2/test_tasksets_routes.py b/plugins/nemo-evaluator/tests/api/v2/test_tasksets_routes.py new file mode 100644 index 0000000000..c7e94b2460 --- /dev/null +++ b/plugins/nemo-evaluator/tests/api/v2/test_tasksets_routes.py @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""HTTP route-level tests for the /tasksets CRUD endpoints. + +Drives the real FastAPI router + TasksetService through a TestClient with an in-memory entity store. +Covers route wiring, the get_taskset_service dependency, and status-code mapping (201/204/404/409/422). +""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from nemo_evaluator.api.dependencies import get_taskset_service +from nemo_evaluator.api.schemas import TaskRef, TasksetInput +from nemo_evaluator.api.service.taskset_service import TasksetService +from nemo_evaluator.api.v2 import tasksets as tasksets_routes +from nemo_platform_plugin.entities import ( + EntityBase, + EntityConflictError, + EntityNotFoundError, + ListResponse, + PaginationInfo, +) + + +class _FakeEntityClient: + def __init__(self) -> None: + self.entities: dict[tuple[str, str, str], EntityBase] = {} + + async def create(self, entity): + key = (entity.__entity_type__, entity.workspace, entity.name) + if key in self.entities: + raise EntityConflictError(f"{key} exists") + now = datetime.now(timezone.utc) + entity._id = f"{entity.__entity_type__}-{entity.name}" + entity._created_at = now + entity._updated_at = now + self.entities[key] = entity + return entity + + async def get(self, entity_cls, *, workspace, name): + key = (entity_cls.__entity_type__, workspace, name) + if key not in self.entities: + raise EntityNotFoundError(f"{workspace}/{name} not found") + return self.entities[key] + + async def delete(self, entity_cls, name, *, workspace): + key = (entity_cls.__entity_type__, workspace, name) + if key not in self.entities: + raise EntityNotFoundError(f"{workspace}/{name} not found") + del self.entities[key] + + async def list(self, entity_cls, *, workspace, filter_operation=None, sort=None, page=1, page_size=100): + items = [ + e for (etype, ws, _), e in self.entities.items() if etype == entity_cls.__entity_type__ and ws == workspace + ] + return ListResponse( + data=items, + pagination=PaginationInfo( + page=page, page_size=page_size, current_page_size=len(items), total_pages=1, total_results=len(items) + ), + ) + + +class _FakeTaskService: + """Resolves the member tasks the route tests reference so create-time validation passes.""" + + async def get_task(self, workspace: str, name: str) -> object | None: + return object() if name in {"task-a", "task-b"} else None + + +@pytest.fixture +def client() -> TestClient: + app = FastAPI() + app.include_router(tasksets_routes.router, prefix="/v2/workspaces/{workspace}") + service = TasksetService(_FakeEntityClient(), _FakeTaskService()) + app.dependency_overrides[get_taskset_service] = lambda: service + return TestClient(app) + + +def _body() -> dict: + return TasksetInput( + description="A grouping.", + tasks=[TaskRef("task-a"), TaskRef("default/task-b")], + ).model_dump(mode="json") + + +_BASE = "/v2/workspaces/default/tasksets" + + +def test_create_then_get(client: TestClient) -> None: + resp = client.post(f"{_BASE}/ts-1", json=_body()) + assert resp.status_code == 201 + assert resp.json()["name"] == "ts-1" + + got = client.get(f"{_BASE}/ts-1") + assert got.status_code == 200 + body = got.json() + assert body["description"] == "A grouping." + assert body["tasks"] == ["task-a", "default/task-b"] # TaskRef serializes to a bare string + + +def test_create_rejects_unknown_body_key(client: TestClient) -> None: + # TasksetInput is extra="forbid" — an unknown key is a 422. + body = _body() + body["intent"] = "nope" + assert client.post(f"{_BASE}/ts-1", json=body).status_code == 422 + + +def test_create_rejects_duplicate_task_refs(client: TestClient) -> None: + # Members are a set expressed as a list; a repeated ref is a 422, not a silent collapse. + body = _body() + body["tasks"] = ["task-a", "task-a"] + assert client.post(f"{_BASE}/ts-1", json=body).status_code == 422 + + +def test_create_rejects_refs_resolving_to_same_task(client: TestClient) -> None: + # Distinct ref strings that resolve to the same task ("task-a" vs "default/task-a") are a 422. + body = _body() + body["tasks"] = ["task-a", "default/task-a"] + assert client.post(f"{_BASE}/ts-1", json=body).status_code == 422 + + +def test_create_rejects_missing_task_ref(client: TestClient) -> None: + # A referenced task that does not exist is a 422 (client error in the submitted body). + body = _body() + body["tasks"] = ["task-a", "does-not-exist"] + assert client.post(f"{_BASE}/ts-1", json=body).status_code == 422 + + +def test_create_rejects_duplicate_metadata_keys(client: TestClient) -> None: + body = _body() + body["metadata"] = [{"key": "suite", "value": "smoke"}, {"key": "suite", "value": "regression"}] + assert client.post(f"{_BASE}/ts-1", json=body).status_code == 422 + + +def test_create_duplicate_returns_409(client: TestClient) -> None: + assert client.post(f"{_BASE}/ts-1", json=_body()).status_code == 201 + assert client.post(f"{_BASE}/ts-1", json=_body()).status_code == 409 + + +def test_create_rejects_invalid_name(client: TestClient) -> None: + assert client.post(f"{_BASE}/bad name", json=_body()).status_code == 422 + + +def test_get_missing_returns_404(client: TestClient) -> None: + assert client.get(f"{_BASE}/nope").status_code == 404 + + +def test_list_returns_created_tasksets(client: TestClient) -> None: + client.post(f"{_BASE}/a", json=_body()) + client.post(f"{_BASE}/b", json=_body()) + + resp = client.get(_BASE) + assert resp.status_code == 200 + body = resp.json() + assert {t["name"] for t in body["data"]} == {"a", "b"} + assert body["pagination"]["total_results"] == 2 + + +def test_delete_then_get_404(client: TestClient) -> None: + client.post(f"{_BASE}/ts-1", json=_body()) + assert client.delete(f"{_BASE}/ts-1").status_code == 204 + assert client.get(f"{_BASE}/ts-1").status_code == 404 + + +def test_delete_missing_returns_404(client: TestClient) -> None: + assert client.delete(f"{_BASE}/nope").status_code == 404 diff --git a/plugins/nemo-evaluator/tests/sdk/test_taskset_sdk_resources.py b/plugins/nemo-evaluator/tests/sdk/test_taskset_sdk_resources.py new file mode 100644 index 0000000000..ba98ace269 --- /dev/null +++ b/plugins/nemo-evaluator/tests/sdk/test_taskset_sdk_resources.py @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the client.evaluator.tasksets SDK resources (mocked HTTP).""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +from nemo_evaluator.api.schemas import TaskRef, Taskset, TasksetInput +from nemo_evaluator.sdk.taskset_resources import AsyncEvaluatorTasksetsResource, EvaluatorTasksetsResource + +_BASE = "http://localhost:8080/apis/evaluator/v2/workspaces/default" + + +def _taskset_payload(name: str) -> dict[str, Any]: + now = datetime.now(timezone.utc) + return Taskset( + id=f"taskset-{name}", + name=name, + workspace="default", + description="A grouping.", + tasks=[TaskRef("default/task-a")], + created_at=now, + updated_at=now, + ).model_dump(mode="json") + + +def _taskset_input() -> TasksetInput: + return TasksetInput(description="A grouping.", tasks=[TaskRef("default/task-a")]) + + +def _response(payload: Any) -> MagicMock: + response = MagicMock() + response.json.return_value = payload + response.raise_for_status.return_value = None + return response + + +def _platform(http_client: Any) -> MagicMock: + platform = MagicMock() + platform._client = http_client + platform.base_url = "http://localhost:8080" + platform.workspace = "default" + platform.default_headers = {} + platform.timeout = 30 + return platform + + +def test_sync_create_posts_taskset_input_to_item_url() -> None: + http_client = MagicMock() + http_client.post.return_value = _response(_taskset_payload("ts-1")) + resource = EvaluatorTasksetsResource(_platform(http_client)) + + result = resource.create("ts-1", taskset=_taskset_input()) + + assert isinstance(result, Taskset) + assert result.name == "ts-1" + assert http_client.post.call_args[0][0] == f"{_BASE}/tasksets/ts-1" + assert http_client.post.call_args.kwargs["json"]["tasks"] == ["default/task-a"] + + +def test_sync_retrieve_targets_item_url_and_parses_dto() -> None: + http_client = MagicMock() + http_client.get.return_value = _response(_taskset_payload("ts-1")) + resource = EvaluatorTasksetsResource(_platform(http_client)) + + result = resource.retrieve("ts-1") + + assert isinstance(result, Taskset) + assert isinstance(result.tasks[0], TaskRef) + assert http_client.get.call_args[0][0] == f"{_BASE}/tasksets/ts-1" + + +def test_sync_list_parses_page() -> None: + http_client = MagicMock() + http_client.get.return_value = _response( + { + "data": [_taskset_payload("a"), _taskset_payload("b")], + "pagination": { + "page": 1, + "page_size": 100, + "current_page_size": 2, + "total_pages": 1, + "total_results": 2, + }, + } + ) + resource = EvaluatorTasksetsResource(_platform(http_client)) + + page = resource.list(sort="-created_at") + + assert {t.name for t in page.data} == {"a", "b"} + assert http_client.get.call_args[0][0] == f"{_BASE}/tasksets" + assert http_client.get.call_args.kwargs["params"]["sort"] == "-created_at" + + +def test_sync_delete_issues_delete_request() -> None: + http_client = MagicMock() + http_client.delete.return_value = _response({}) + resource = EvaluatorTasksetsResource(_platform(http_client)) + + resource.delete("ts-1") + + assert http_client.delete.call_args[0][0] == f"{_BASE}/tasksets/ts-1" + + +async def test_async_retrieve_parses_dto() -> None: + http_client = MagicMock() + http_client.get = AsyncMock(return_value=_response(_taskset_payload("ts-9"))) + resource = AsyncEvaluatorTasksetsResource(_platform(http_client)) + + result = await resource.retrieve("ts-9") + + assert isinstance(result, Taskset) + assert result.name == "ts-9" + assert http_client.get.call_args[0][0] == f"{_BASE}/tasksets/ts-9" diff --git a/plugins/nemo-evaluator/tests/test_taskset_entity.py b/plugins/nemo-evaluator/tests/test_taskset_entity.py new file mode 100644 index 0000000000..9f331c816a --- /dev/null +++ b/plugins/nemo-evaluator/tests/test_taskset_entity.py @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Serialization round-trip tests for TasksetEntity. + +The entity store persists custom fields with ``model_dump(exclude=base, mode="json")`` into a JSON +column and rebuilds them with ``model_validate``. This exercises that round-trip for the fields that +carry non-trivial nested types — the task references and metadata. +""" + +from __future__ import annotations + +import json + +from nemo_evaluator.api.schemas import TaskRef +from nemo_evaluator.entities import TasksetEntity + + +def _entity() -> TasksetEntity: + return TasksetEntity( + name="ts-1", + workspace="default", + description="A smoke-test grouping.", + # A workspace-qualified ref and a bare name. + tasks=[TaskRef("default/task-a"), TaskRef("task-b")], + metadata=[{"key": "suite", "value": "smoke"}], + ) + + +def _roundtrip(entity: TasksetEntity) -> TasksetEntity: + data = entity.model_dump(exclude=TasksetEntity.__base_fields__, exclude_computed_fields=True, mode="json") + data = json.loads(json.dumps(data)) # prove JSON-serializable (the store uses a JSON column) + return TasksetEntity.model_validate({"name": entity.name, "workspace": entity.workspace, **data}) + + +def test_roundtrip_preserves_taskset_fields() -> None: + entity = _entity() + + restored = _roundtrip(entity) + + assert restored.description == "A smoke-test grouping." + assert [(m.key, m.value) for m in restored.metadata] == [("suite", "smoke")] + # Task refs survive as RootModel strings. + assert isinstance(restored.tasks[0], TaskRef) + assert restored.tasks[0].root == "default/task-a" + assert restored.tasks[1].root == "task-b" + + +def test_entity_type_is_taskset() -> None: + assert TasksetEntity.__entity_type__ == "taskset"