diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index f91c0ade04..9be005085e 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -3622,6 +3622,55 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + patch: + tags: + - Evaluations + summary: Patch Evaluation + description: "Partially update an evaluation: only fields present in the request\ + \ are changed.\n\nThe common case is curating an evaluation into another ExperimentGroup\ + \ \u2014 PATCH with the merged\n``experiment_ids``. Membership is replaced\ + \ (not appended), so send the full desired set; any new\ngroup must exist\ + \ and the set must be non-empty (an evaluation always belongs to >=1 group).\ + \ Omitted\nfields are left untouched (unlike the full-body PUT, which overwrites\ + \ them)." + operationId: patch_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__patch + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationPatchRequest' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationResponse' + '400': + description: A referenced ExperimentGroup does not exist, or experiment_ids + is empty + '404': + description: Evaluation not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' delete: tags: - Evaluations @@ -10504,17 +10553,75 @@ components: type: object title: EvaluationFilter type: object + EvaluationPatchRequest: + properties: + experiment_ids: + title: Experiment Ids + description: Replace the ExperimentGroups this Evaluation belongs to. Must + be non-empty when provided; each group must already exist. Omit to leave + membership unchanged. + items: + type: string + type: array + source_link: + title: Source Link + description: Optional URL for the source evaluation. + type: string + minLength: 1 + format: uri + metadata: + title: Metadata + description: Free-form producer metadata. + additionalProperties: + type: string + type: object + description: + title: Description + description: Human-readable description. + type: string + parent_evaluation_id: + title: Parent Evaluation Id + description: Entity id of the evaluation this one was derived from (e.g. + a variant of a baseline), if any. + type: string + status: + title: Status + description: Producer-defined lifecycle status of the evaluation. + type: string + root_cause: + title: Root Cause + description: Human- or agent-authored explanation of the evaluation's outcome + (e.g. why it was killed). + type: string + additionalProperties: false + type: object + title: EvaluationPatchRequest + description: "Partial-update body for an Evaluation: only fields present in\ + \ the request are applied.\n\nUnset fields are left unchanged (PATCH semantics\ + \ \u2014 same pattern as the models service's PATCH).\nImmutable fields (name,\ + \ dataset_name, dataset_version) aren't accepted here. ``experiment_ids``,\n\ + when provided, must be non-empty: an evaluation must always belong to at least\ + \ one group." EvaluationRequest: properties: name: type: string title: Name description: Producer-supplied, workspace-unique evaluation id. + experiment_ids: + items: + type: string + type: array + title: Experiment Ids + description: Entity ids of the ExperimentGroups this Evaluation belongs + to (>=1). Preferred; each group must already exist. When omitted, the + deprecated experiment_group_id is used instead. experiment_group_id: - type: string title: Experiment Group Id - description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ - \ group must already exist." + description: Deprecated single-group field; provide experiment_ids instead. + Coalesced into experiment_ids when experiment_ids is omitted. + deprecated: true + type: string dataset_name: type: string title: Dataset Name @@ -10562,7 +10669,6 @@ components: type: object required: - name - - experiment_group_id - dataset_name title: EvaluationRequest description: Request body for creating an Evaluation. @@ -10577,11 +10683,13 @@ components: workspace: type: string title: Workspace - experiment_group_id: - type: string - title: Experiment Group Id - description: Entity id of the owning ExperimentGroup. Required for every - Evaluation. + experiment_ids: + items: + type: string + type: array + title: Experiment Ids + description: Entity ids of the ExperimentGroups this Evaluation belongs + to (>=1). dataset_name: type: string title: Dataset Name @@ -10683,14 +10791,22 @@ components: deprecated: true readOnly: true type: string + experiment_group_id: + type: string + title: Experiment Group Id + description: Deprecated single-group alias; the first of experiment_ids. + Use experiment_ids. + deprecated: true + readOnly: true type: object required: - id - name - workspace - - experiment_group_id + - experiment_ids - dataset_name - parent_experiment_id + - experiment_group_id title: EvaluationResponse description: Evaluation as served by the API, including ClickHouse-hydrated rollups. diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index f91c0ade04..9be005085e 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -3622,6 +3622,55 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + patch: + tags: + - Evaluations + summary: Patch Evaluation + description: "Partially update an evaluation: only fields present in the request\ + \ are changed.\n\nThe common case is curating an evaluation into another ExperimentGroup\ + \ \u2014 PATCH with the merged\n``experiment_ids``. Membership is replaced\ + \ (not appended), so send the full desired set; any new\ngroup must exist\ + \ and the set must be non-empty (an evaluation always belongs to >=1 group).\ + \ Omitted\nfields are left untouched (unlike the full-body PUT, which overwrites\ + \ them)." + operationId: patch_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__patch + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationPatchRequest' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationResponse' + '400': + description: A referenced ExperimentGroup does not exist, or experiment_ids + is empty + '404': + description: Evaluation not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' delete: tags: - Evaluations @@ -10504,17 +10553,75 @@ components: type: object title: EvaluationFilter type: object + EvaluationPatchRequest: + properties: + experiment_ids: + title: Experiment Ids + description: Replace the ExperimentGroups this Evaluation belongs to. Must + be non-empty when provided; each group must already exist. Omit to leave + membership unchanged. + items: + type: string + type: array + source_link: + title: Source Link + description: Optional URL for the source evaluation. + type: string + minLength: 1 + format: uri + metadata: + title: Metadata + description: Free-form producer metadata. + additionalProperties: + type: string + type: object + description: + title: Description + description: Human-readable description. + type: string + parent_evaluation_id: + title: Parent Evaluation Id + description: Entity id of the evaluation this one was derived from (e.g. + a variant of a baseline), if any. + type: string + status: + title: Status + description: Producer-defined lifecycle status of the evaluation. + type: string + root_cause: + title: Root Cause + description: Human- or agent-authored explanation of the evaluation's outcome + (e.g. why it was killed). + type: string + additionalProperties: false + type: object + title: EvaluationPatchRequest + description: "Partial-update body for an Evaluation: only fields present in\ + \ the request are applied.\n\nUnset fields are left unchanged (PATCH semantics\ + \ \u2014 same pattern as the models service's PATCH).\nImmutable fields (name,\ + \ dataset_name, dataset_version) aren't accepted here. ``experiment_ids``,\n\ + when provided, must be non-empty: an evaluation must always belong to at least\ + \ one group." EvaluationRequest: properties: name: type: string title: Name description: Producer-supplied, workspace-unique evaluation id. + experiment_ids: + items: + type: string + type: array + title: Experiment Ids + description: Entity ids of the ExperimentGroups this Evaluation belongs + to (>=1). Preferred; each group must already exist. When omitted, the + deprecated experiment_group_id is used instead. experiment_group_id: - type: string title: Experiment Group Id - description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ - \ group must already exist." + description: Deprecated single-group field; provide experiment_ids instead. + Coalesced into experiment_ids when experiment_ids is omitted. + deprecated: true + type: string dataset_name: type: string title: Dataset Name @@ -10562,7 +10669,6 @@ components: type: object required: - name - - experiment_group_id - dataset_name title: EvaluationRequest description: Request body for creating an Evaluation. @@ -10577,11 +10683,13 @@ components: workspace: type: string title: Workspace - experiment_group_id: - type: string - title: Experiment Group Id - description: Entity id of the owning ExperimentGroup. Required for every - Evaluation. + experiment_ids: + items: + type: string + type: array + title: Experiment Ids + description: Entity ids of the ExperimentGroups this Evaluation belongs + to (>=1). dataset_name: type: string title: Dataset Name @@ -10683,14 +10791,22 @@ components: deprecated: true readOnly: true type: string + experiment_group_id: + type: string + title: Experiment Group Id + description: Deprecated single-group alias; the first of experiment_ids. + Use experiment_ids. + deprecated: true + readOnly: true type: object required: - id - name - workspace - - experiment_group_id + - experiment_ids - dataset_name - parent_experiment_id + - experiment_group_id title: EvaluationResponse description: Evaluation as served by the API, including ClickHouse-hydrated rollups. diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index f91c0ade04..9be005085e 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -3622,6 +3622,55 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + patch: + tags: + - Evaluations + summary: Patch Evaluation + description: "Partially update an evaluation: only fields present in the request\ + \ are changed.\n\nThe common case is curating an evaluation into another ExperimentGroup\ + \ \u2014 PATCH with the merged\n``experiment_ids``. Membership is replaced\ + \ (not appended), so send the full desired set; any new\ngroup must exist\ + \ and the set must be non-empty (an evaluation always belongs to >=1 group).\ + \ Omitted\nfields are left untouched (unlike the full-body PUT, which overwrites\ + \ them)." + operationId: patch_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__patch + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationPatchRequest' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationResponse' + '400': + description: A referenced ExperimentGroup does not exist, or experiment_ids + is empty + '404': + description: Evaluation not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' delete: tags: - Evaluations @@ -10504,17 +10553,75 @@ components: type: object title: EvaluationFilter type: object + EvaluationPatchRequest: + properties: + experiment_ids: + title: Experiment Ids + description: Replace the ExperimentGroups this Evaluation belongs to. Must + be non-empty when provided; each group must already exist. Omit to leave + membership unchanged. + items: + type: string + type: array + source_link: + title: Source Link + description: Optional URL for the source evaluation. + type: string + minLength: 1 + format: uri + metadata: + title: Metadata + description: Free-form producer metadata. + additionalProperties: + type: string + type: object + description: + title: Description + description: Human-readable description. + type: string + parent_evaluation_id: + title: Parent Evaluation Id + description: Entity id of the evaluation this one was derived from (e.g. + a variant of a baseline), if any. + type: string + status: + title: Status + description: Producer-defined lifecycle status of the evaluation. + type: string + root_cause: + title: Root Cause + description: Human- or agent-authored explanation of the evaluation's outcome + (e.g. why it was killed). + type: string + additionalProperties: false + type: object + title: EvaluationPatchRequest + description: "Partial-update body for an Evaluation: only fields present in\ + \ the request are applied.\n\nUnset fields are left unchanged (PATCH semantics\ + \ \u2014 same pattern as the models service's PATCH).\nImmutable fields (name,\ + \ dataset_name, dataset_version) aren't accepted here. ``experiment_ids``,\n\ + when provided, must be non-empty: an evaluation must always belong to at least\ + \ one group." EvaluationRequest: properties: name: type: string title: Name description: Producer-supplied, workspace-unique evaluation id. + experiment_ids: + items: + type: string + type: array + title: Experiment Ids + description: Entity ids of the ExperimentGroups this Evaluation belongs + to (>=1). Preferred; each group must already exist. When omitted, the + deprecated experiment_group_id is used instead. experiment_group_id: - type: string title: Experiment Group Id - description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ - \ group must already exist." + description: Deprecated single-group field; provide experiment_ids instead. + Coalesced into experiment_ids when experiment_ids is omitted. + deprecated: true + type: string dataset_name: type: string title: Dataset Name @@ -10562,7 +10669,6 @@ components: type: object required: - name - - experiment_group_id - dataset_name title: EvaluationRequest description: Request body for creating an Evaluation. @@ -10577,11 +10683,13 @@ components: workspace: type: string title: Workspace - experiment_group_id: - type: string - title: Experiment Group Id - description: Entity id of the owning ExperimentGroup. Required for every - Evaluation. + experiment_ids: + items: + type: string + type: array + title: Experiment Ids + description: Entity ids of the ExperimentGroups this Evaluation belongs + to (>=1). dataset_name: type: string title: Dataset Name @@ -10683,14 +10791,22 @@ components: deprecated: true readOnly: true type: string + experiment_group_id: + type: string + title: Experiment Group Id + description: Deprecated single-group alias; the first of experiment_ids. + Use experiment_ids. + deprecated: true + readOnly: true type: object required: - id - name - workspace - - experiment_group_id + - experiment_ids - dataset_name - parent_experiment_id + - experiment_group_id title: EvaluationResponse description: Evaluation as served by the API, including ClickHouse-hydrated rollups. diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/filter_ops.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/filter_ops.py index 2c46cd9e02..9b9b586dae 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/filter_ops.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/filter_ops.py @@ -26,6 +26,7 @@ class FilterOperator(str, Enum): GTE = "$gte" IN = "$in" NIN = "$nin" + CONTAINS = "$contains" # Logical operators OR = "$or" @@ -71,6 +72,14 @@ def in_op(self, field: str, values: List[Any]) -> Any: def nin(self, field: str, values: List[Any]) -> Any: pass + def contains(self, field: str, value: Any) -> Any: + """Array membership: match rows where the array at ``field`` contains scalar ``value``. + + Optional — additive to the base contract, so repositories that don't support + array-valued fields may leave it unimplemented. + """ + raise NotImplementedError("$contains not supported by this repository") + @abstractmethod def and_op(self, operations: List[Any]) -> Any: pass @@ -133,6 +142,8 @@ def apply(self, repository: FilterRepository) -> Any: return repository.in_op(self.field, self.value) elif self.operator == FilterOperator.NIN: return repository.nin(self.field, self.value) + elif self.operator == FilterOperator.CONTAINS: + return repository.contains(self.field, self.value) elif self.operator == FilterOperator.EXISTS: raise NotImplementedError( "$exists requires a relationship-aware repository (use the entities service parser)" diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/in_memory_filter.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/in_memory_filter.py index 31d4798622..cbaddb1ccb 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/in_memory_filter.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/in_memory_filter.py @@ -117,6 +117,17 @@ def nin(self, field: str, values: List[Any]) -> bool: return False return field_value not in values + def contains(self, field: str, value: Any) -> bool: + """Array membership: true when the list at ``field`` contains ``value``. + + Absent/None/non-list fields match nothing (mirrors the SQL path, where a missing + or scalar element can't contain the token). Elements are compared by native value. + """ + field_value = self._value(field) + if field_value is _MISSING or not isinstance(field_value, (list, tuple)): + return False + return value in field_value + def and_op(self, operations: List[Any]) -> bool: return all(operations) diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index f91c0ade04..9be005085e 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -3622,6 +3622,55 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + patch: + tags: + - Evaluations + summary: Patch Evaluation + description: "Partially update an evaluation: only fields present in the request\ + \ are changed.\n\nThe common case is curating an evaluation into another ExperimentGroup\ + \ \u2014 PATCH with the merged\n``experiment_ids``. Membership is replaced\ + \ (not appended), so send the full desired set; any new\ngroup must exist\ + \ and the set must be non-empty (an evaluation always belongs to >=1 group).\ + \ Omitted\nfields are left untouched (unlike the full-body PUT, which overwrites\ + \ them)." + operationId: patch_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__patch + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationPatchRequest' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationResponse' + '400': + description: A referenced ExperimentGroup does not exist, or experiment_ids + is empty + '404': + description: Evaluation not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' delete: tags: - Evaluations @@ -10504,17 +10553,75 @@ components: type: object title: EvaluationFilter type: object + EvaluationPatchRequest: + properties: + experiment_ids: + title: Experiment Ids + description: Replace the ExperimentGroups this Evaluation belongs to. Must + be non-empty when provided; each group must already exist. Omit to leave + membership unchanged. + items: + type: string + type: array + source_link: + title: Source Link + description: Optional URL for the source evaluation. + type: string + minLength: 1 + format: uri + metadata: + title: Metadata + description: Free-form producer metadata. + additionalProperties: + type: string + type: object + description: + title: Description + description: Human-readable description. + type: string + parent_evaluation_id: + title: Parent Evaluation Id + description: Entity id of the evaluation this one was derived from (e.g. + a variant of a baseline), if any. + type: string + status: + title: Status + description: Producer-defined lifecycle status of the evaluation. + type: string + root_cause: + title: Root Cause + description: Human- or agent-authored explanation of the evaluation's outcome + (e.g. why it was killed). + type: string + additionalProperties: false + type: object + title: EvaluationPatchRequest + description: "Partial-update body for an Evaluation: only fields present in\ + \ the request are applied.\n\nUnset fields are left unchanged (PATCH semantics\ + \ \u2014 same pattern as the models service's PATCH).\nImmutable fields (name,\ + \ dataset_name, dataset_version) aren't accepted here. ``experiment_ids``,\n\ + when provided, must be non-empty: an evaluation must always belong to at least\ + \ one group." EvaluationRequest: properties: name: type: string title: Name description: Producer-supplied, workspace-unique evaluation id. + experiment_ids: + items: + type: string + type: array + title: Experiment Ids + description: Entity ids of the ExperimentGroups this Evaluation belongs + to (>=1). Preferred; each group must already exist. When omitted, the + deprecated experiment_group_id is used instead. experiment_group_id: - type: string title: Experiment Group Id - description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ - \ group must already exist." + description: Deprecated single-group field; provide experiment_ids instead. + Coalesced into experiment_ids when experiment_ids is omitted. + deprecated: true + type: string dataset_name: type: string title: Dataset Name @@ -10562,7 +10669,6 @@ components: type: object required: - name - - experiment_group_id - dataset_name title: EvaluationRequest description: Request body for creating an Evaluation. @@ -10577,11 +10683,13 @@ components: workspace: type: string title: Workspace - experiment_group_id: - type: string - title: Experiment Group Id - description: Entity id of the owning ExperimentGroup. Required for every - Evaluation. + experiment_ids: + items: + type: string + type: array + title: Experiment Ids + description: Entity ids of the ExperimentGroups this Evaluation belongs + to (>=1). dataset_name: type: string title: Dataset Name @@ -10683,14 +10791,22 @@ components: deprecated: true readOnly: true type: string + experiment_group_id: + type: string + title: Experiment Group Id + description: Deprecated single-group alias; the first of experiment_ids. + Use experiment_ids. + deprecated: true + readOnly: true type: object required: - id - name - workspace - - experiment_group_id + - experiment_ids - dataset_name - parent_experiment_id + - experiment_group_id title: EvaluationResponse description: Evaluation as served by the API, including ClickHouse-hydrated rollups. diff --git a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml index f552111d2c..8ebfadbe66 100644 --- a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml @@ -929,6 +929,7 @@ resources: standalone_api: true models: evaluation_filter: EvaluationFilter + evaluation_patch_request: EvaluationPatchRequest evaluation_request: EvaluationRequest evaluation_response: EvaluationResponse evaluation_responses_page: EvaluationResponsesPage @@ -940,6 +941,7 @@ resources: list: get /apis/intake/v2/workspaces/{workspace}/evaluations retrieve: get /apis/intake/v2/workspaces/{workspace}/evaluations/{name} update: put /apis/intake/v2/workspaces/{workspace}/evaluations/{name} + patch: patch /apis/intake/v2/workspaces/{workspace}/evaluations/{name} delete: delete /apis/intake/v2/workspaces/{workspace}/evaluations/{name} pin: post /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin unpin: delete /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/api.md index 485bdd1903..91506a3500 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/api.md +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/api.md @@ -5,6 +5,7 @@ Types: ```python from nemo_platform.types.evaluations import ( EvaluationFilter, + EvaluationPatchRequest, EvaluationRequest, EvaluationResponse, EvaluationResponsesPage, @@ -21,6 +22,7 @@ Methods: - client.evaluations.update(path_name, \*, workspace, \*\*params) -> EvaluationResponse - client.evaluations.list(\*, workspace, \*\*params) -> SyncDefaultPagination[EvaluationResponse] - client.evaluations.delete(name, \*, workspace) -> None +- client.evaluations.patch(name, \*, workspace, \*\*params) -> EvaluationResponse - client.evaluations.pin(name, \*, workspace) -> EvaluationResponse - client.evaluations.unpin(name, \*, workspace) -> EvaluationResponse diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/evaluations.py b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/evaluations.py index f1feaafdce..eb293a081c 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/evaluations.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/evaluations.py @@ -21,7 +21,7 @@ import httpx -from ..._types import Body, Omit, Query, Headers, NoneType, NotGiven, omit, not_given +from ..._types import Body, Omit, Query, Headers, NoneType, NotGiven, SequenceNotStr, omit, not_given from ..._utils import path_template, maybe_transform, async_maybe_transform from .sessions import ( SessionsResource, @@ -43,6 +43,7 @@ from ..._base_client import AsyncPaginator, make_request_options from ...types.evaluations import ( evaluation_list_params, + evaluation_patch_params, evaluation_create_params, evaluation_update_params, ) @@ -82,10 +83,11 @@ def create( *, workspace: str | None = None, dataset_name: str, - experiment_group_id: str, name: str, dataset_version: str | Omit = omit, description: str | Omit = omit, + experiment_group_id: str | Omit = omit, + experiment_ids: SequenceNotStr[str] | Omit = omit, metadata: Dict[str, str] | Omit = omit, parent_evaluation_id: str | Omit = omit, parent_experiment_id: str | Omit = omit, @@ -106,15 +108,19 @@ def create( Args: dataset_name: Producer-supplied dataset name. - experiment_group_id: Entity id of the owning ExperimentGroup. Required — the group must already - exist. - name: Producer-supplied, workspace-unique evaluation id. dataset_version: Producer-supplied dataset version. description: Human-readable description. + experiment_group_id: Deprecated single-group field; provide experiment_ids instead. Coalesced into + experiment_ids when experiment_ids is omitted. + + experiment_ids: Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). Preferred; + each group must already exist. When omitted, the deprecated experiment_group_id + is used instead. + metadata: Free-form producer metadata. parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a @@ -151,10 +157,11 @@ def create( body=maybe_transform( { "dataset_name": dataset_name, - "experiment_group_id": experiment_group_id, "name": name, "dataset_version": dataset_version, "description": description, + "experiment_group_id": experiment_group_id, + "experiment_ids": experiment_ids, "metadata": metadata, "parent_evaluation_id": parent_evaluation_id, "parent_experiment_id": parent_experiment_id, @@ -218,10 +225,11 @@ def update( *, workspace: str | None = None, dataset_name: str, - experiment_group_id: str, body_name: str, dataset_version: str | Omit = omit, description: str | Omit = omit, + experiment_group_id: str | Omit = omit, + experiment_ids: SequenceNotStr[str] | Omit = omit, metadata: Dict[str, str] | Omit = omit, parent_evaluation_id: str | Omit = omit, parent_experiment_id: str | Omit = omit, @@ -241,15 +249,19 @@ def update( Args: dataset_name: Producer-supplied dataset name. - experiment_group_id: Entity id of the owning ExperimentGroup. Required — the group must already - exist. - body_name: Producer-supplied, workspace-unique evaluation id. dataset_version: Producer-supplied dataset version. description: Human-readable description. + experiment_group_id: Deprecated single-group field; provide experiment_ids instead. Coalesced into + experiment_ids when experiment_ids is omitted. + + experiment_ids: Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). Preferred; + each group must already exist. When omitted, the deprecated experiment_group_id + is used instead. + metadata: Free-form producer metadata. parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a @@ -287,10 +299,11 @@ def update( body=maybe_transform( { "dataset_name": dataset_name, - "experiment_group_id": experiment_group_id, "body_name": body_name, "dataset_version": dataset_version, "description": description, + "experiment_group_id": experiment_group_id, + "experiment_ids": experiment_ids, "metadata": metadata, "parent_evaluation_id": parent_evaluation_id, "parent_experiment_id": parent_experiment_id, @@ -418,6 +431,86 @@ def delete( cast_to=NoneType, ) + def patch( + self, + name: str, + *, + workspace: str | None = None, + description: str | Omit = omit, + experiment_ids: SequenceNotStr[str] | Omit = omit, + metadata: Dict[str, str] | Omit = omit, + parent_evaluation_id: str | Omit = omit, + root_cause: str | Omit = omit, + source_link: str | Omit = omit, + status: str | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> EvaluationResponse: + """ + Partially update an evaluation: only fields present in the request are changed. + + The common case is curating an evaluation into another ExperimentGroup — PATCH + with the merged `experiment_ids`. Membership is replaced (not appended), so send + the full desired set; any new group must exist and the set must be non-empty (an + evaluation always belongs to >=1 group). Omitted fields are left untouched + (unlike the full-body PUT, which overwrites them). + + Args: + description: Human-readable description. + + experiment_ids: Replace the ExperimentGroups this Evaluation belongs to. Must be non-empty when + provided; each group must already exist. Omit to leave membership unchanged. + + metadata: Free-form producer metadata. + + parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a + baseline), if any. + + root_cause: Human- or agent-authored explanation of the evaluation's outcome (e.g. why it + was killed). + + source_link: Optional URL for the source evaluation. + + status: Producer-defined lifecycle status of the evaluation. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + return self._patch( + path_template("/apis/intake/v2/workspaces/{workspace}/evaluations/{name}", workspace=workspace, name=name), + body=maybe_transform( + { + "description": description, + "experiment_ids": experiment_ids, + "metadata": metadata, + "parent_evaluation_id": parent_evaluation_id, + "root_cause": root_cause, + "source_link": source_link, + "status": status, + }, + evaluation_patch_params.EvaluationPatchParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=EvaluationResponse, + ) + def pin( self, name: str, @@ -533,10 +626,11 @@ async def create( *, workspace: str | None = None, dataset_name: str, - experiment_group_id: str, name: str, dataset_version: str | Omit = omit, description: str | Omit = omit, + experiment_group_id: str | Omit = omit, + experiment_ids: SequenceNotStr[str] | Omit = omit, metadata: Dict[str, str] | Omit = omit, parent_evaluation_id: str | Omit = omit, parent_experiment_id: str | Omit = omit, @@ -557,15 +651,19 @@ async def create( Args: dataset_name: Producer-supplied dataset name. - experiment_group_id: Entity id of the owning ExperimentGroup. Required — the group must already - exist. - name: Producer-supplied, workspace-unique evaluation id. dataset_version: Producer-supplied dataset version. description: Human-readable description. + experiment_group_id: Deprecated single-group field; provide experiment_ids instead. Coalesced into + experiment_ids when experiment_ids is omitted. + + experiment_ids: Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). Preferred; + each group must already exist. When omitted, the deprecated experiment_group_id + is used instead. + metadata: Free-form producer metadata. parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a @@ -602,10 +700,11 @@ async def create( body=await async_maybe_transform( { "dataset_name": dataset_name, - "experiment_group_id": experiment_group_id, "name": name, "dataset_version": dataset_version, "description": description, + "experiment_group_id": experiment_group_id, + "experiment_ids": experiment_ids, "metadata": metadata, "parent_evaluation_id": parent_evaluation_id, "parent_experiment_id": parent_experiment_id, @@ -669,10 +768,11 @@ async def update( *, workspace: str | None = None, dataset_name: str, - experiment_group_id: str, body_name: str, dataset_version: str | Omit = omit, description: str | Omit = omit, + experiment_group_id: str | Omit = omit, + experiment_ids: SequenceNotStr[str] | Omit = omit, metadata: Dict[str, str] | Omit = omit, parent_evaluation_id: str | Omit = omit, parent_experiment_id: str | Omit = omit, @@ -692,15 +792,19 @@ async def update( Args: dataset_name: Producer-supplied dataset name. - experiment_group_id: Entity id of the owning ExperimentGroup. Required — the group must already - exist. - body_name: Producer-supplied, workspace-unique evaluation id. dataset_version: Producer-supplied dataset version. description: Human-readable description. + experiment_group_id: Deprecated single-group field; provide experiment_ids instead. Coalesced into + experiment_ids when experiment_ids is omitted. + + experiment_ids: Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). Preferred; + each group must already exist. When omitted, the deprecated experiment_group_id + is used instead. + metadata: Free-form producer metadata. parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a @@ -738,10 +842,11 @@ async def update( body=await async_maybe_transform( { "dataset_name": dataset_name, - "experiment_group_id": experiment_group_id, "body_name": body_name, "dataset_version": dataset_version, "description": description, + "experiment_group_id": experiment_group_id, + "experiment_ids": experiment_ids, "metadata": metadata, "parent_evaluation_id": parent_evaluation_id, "parent_experiment_id": parent_experiment_id, @@ -869,6 +974,86 @@ async def delete( cast_to=NoneType, ) + async def patch( + self, + name: str, + *, + workspace: str | None = None, + description: str | Omit = omit, + experiment_ids: SequenceNotStr[str] | Omit = omit, + metadata: Dict[str, str] | Omit = omit, + parent_evaluation_id: str | Omit = omit, + root_cause: str | Omit = omit, + source_link: str | Omit = omit, + status: str | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> EvaluationResponse: + """ + Partially update an evaluation: only fields present in the request are changed. + + The common case is curating an evaluation into another ExperimentGroup — PATCH + with the merged `experiment_ids`. Membership is replaced (not appended), so send + the full desired set; any new group must exist and the set must be non-empty (an + evaluation always belongs to >=1 group). Omitted fields are left untouched + (unlike the full-body PUT, which overwrites them). + + Args: + description: Human-readable description. + + experiment_ids: Replace the ExperimentGroups this Evaluation belongs to. Must be non-empty when + provided; each group must already exist. Omit to leave membership unchanged. + + metadata: Free-form producer metadata. + + parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a + baseline), if any. + + root_cause: Human- or agent-authored explanation of the evaluation's outcome (e.g. why it + was killed). + + source_link: Optional URL for the source evaluation. + + status: Producer-defined lifecycle status of the evaluation. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + return await self._patch( + path_template("/apis/intake/v2/workspaces/{workspace}/evaluations/{name}", workspace=workspace, name=name), + body=await async_maybe_transform( + { + "description": description, + "experiment_ids": experiment_ids, + "metadata": metadata, + "parent_evaluation_id": parent_evaluation_id, + "root_cause": root_cause, + "source_link": source_link, + "status": status, + }, + evaluation_patch_params.EvaluationPatchParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=EvaluationResponse, + ) + async def pin( self, name: str, @@ -974,6 +1159,9 @@ def __init__(self, evaluations: EvaluationsResource) -> None: self.delete = to_raw_response_wrapper( evaluations.delete, ) + self.patch = to_raw_response_wrapper( + evaluations.patch, + ) self.pin = to_raw_response_wrapper( evaluations.pin, ) @@ -1005,6 +1193,9 @@ def __init__(self, evaluations: AsyncEvaluationsResource) -> None: self.delete = async_to_raw_response_wrapper( evaluations.delete, ) + self.patch = async_to_raw_response_wrapper( + evaluations.patch, + ) self.pin = async_to_raw_response_wrapper( evaluations.pin, ) @@ -1036,6 +1227,9 @@ def __init__(self, evaluations: EvaluationsResource) -> None: self.delete = to_streamed_response_wrapper( evaluations.delete, ) + self.patch = to_streamed_response_wrapper( + evaluations.patch, + ) self.pin = to_streamed_response_wrapper( evaluations.pin, ) @@ -1067,6 +1261,9 @@ def __init__(self, evaluations: AsyncEvaluationsResource) -> None: self.delete = async_to_streamed_response_wrapper( evaluations.delete, ) + self.patch = async_to_streamed_response_wrapper( + evaluations.patch, + ) self.pin = async_to_streamed_response_wrapper( evaluations.pin, ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/__init__.py index d83c9d69a2..6ab6fca291 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/__init__.py @@ -23,6 +23,7 @@ from .session_list_params import SessionListParams as SessionListParams from .evaluation_list_params import EvaluationListParams as EvaluationListParams from .evaluation_filter_param import EvaluationFilterParam as EvaluationFilterParam +from .evaluation_patch_params import EvaluationPatchParams as EvaluationPatchParams from .evaluation_create_params import EvaluationCreateParams as EvaluationCreateParams from .evaluation_update_params import EvaluationUpdateParams as EvaluationUpdateParams from .evaluation_responses_page import EvaluationResponsesPage as EvaluationResponsesPage diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_create_params.py index 956ea0db31..3308bbbdcf 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_create_params.py @@ -20,6 +20,8 @@ from typing import Dict from typing_extensions import Required, TypedDict +from ..._types import SequenceNotStr + __all__ = ["EvaluationCreateParams"] @@ -29,12 +31,6 @@ class EvaluationCreateParams(TypedDict, total=False): dataset_name: Required[str] """Producer-supplied dataset name.""" - experiment_group_id: Required[str] - """Entity id of the owning ExperimentGroup. - - Required — the group must already exist. - """ - name: Required[str] """Producer-supplied, workspace-unique evaluation id.""" @@ -44,6 +40,19 @@ class EvaluationCreateParams(TypedDict, total=False): description: str """Human-readable description.""" + experiment_group_id: str + """Deprecated single-group field; provide experiment_ids instead. + + Coalesced into experiment_ids when experiment_ids is omitted. + """ + + experiment_ids: SequenceNotStr[str] + """Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). + + Preferred; each group must already exist. When omitted, the deprecated + experiment_group_id is used instead. + """ + metadata: Dict[str, str] """Free-form producer metadata.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_patch_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_patch_params.py new file mode 100644 index 0000000000..cdd85f3521 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_patch_params.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Dict +from typing_extensions import TypedDict + +from ..._types import SequenceNotStr + +__all__ = ["EvaluationPatchParams"] + + +class EvaluationPatchParams(TypedDict, total=False): + workspace: str + + description: str + """Human-readable description.""" + + experiment_ids: SequenceNotStr[str] + """Replace the ExperimentGroups this Evaluation belongs to. + + Must be non-empty when provided; each group must already exist. Omit to leave + membership unchanged. + """ + + metadata: Dict[str, str] + """Free-form producer metadata.""" + + parent_evaluation_id: str + """Entity id of the evaluation this one was derived from (e.g. + + a variant of a baseline), if any. + """ + + root_cause: str + """Human- or agent-authored explanation of the evaluation's outcome (e.g. + + why it was killed). + """ + + source_link: str + """Optional URL for the source evaluation.""" + + status: str + """Producer-defined lifecycle status of the evaluation.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py index 461be98b32..dcc27fcfc8 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py @@ -33,7 +33,10 @@ class EvaluationResponse(BaseModel): dataset_name: str experiment_group_id: str - """Entity id of the owning ExperimentGroup. Required for every Evaluation.""" + """Deprecated single-group alias; the first of experiment_ids. Use experiment_ids.""" + + experiment_ids: List[str] + """Entity ids of the ExperimentGroups this Evaluation belongs to (>=1).""" name: str diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_update_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_update_params.py index a44d66f57b..75dd13dab5 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_update_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_update_params.py @@ -20,6 +20,7 @@ from typing import Dict from typing_extensions import Required, Annotated, TypedDict +from ..._types import SequenceNotStr from ..._utils import PropertyInfo __all__ = ["EvaluationUpdateParams"] @@ -31,12 +32,6 @@ class EvaluationUpdateParams(TypedDict, total=False): dataset_name: Required[str] """Producer-supplied dataset name.""" - experiment_group_id: Required[str] - """Entity id of the owning ExperimentGroup. - - Required — the group must already exist. - """ - body_name: Required[Annotated[str, PropertyInfo(alias="name")]] """Producer-supplied, workspace-unique evaluation id.""" @@ -46,6 +41,19 @@ class EvaluationUpdateParams(TypedDict, total=False): description: str """Human-readable description.""" + experiment_group_id: str + """Deprecated single-group field; provide experiment_ids instead. + + Coalesced into experiment_ids when experiment_ids is omitted. + """ + + experiment_ids: SequenceNotStr[str] + """Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). + + Preferred; each group must already exist. When omitted, the deprecated + experiment_group_id is used instead. + """ + metadata: Dict[str, str] """Free-form producer metadata.""" diff --git a/sdk/python/nemo-platform/tests/api_resources/test_evaluations.py b/sdk/python/nemo-platform/tests/api_resources/test_evaluations.py index fd8798a6ef..0d8436aa44 100644 --- a/sdk/python/nemo-platform/tests/api_resources/test_evaluations.py +++ b/sdk/python/nemo-platform/tests/api_resources/test_evaluations.py @@ -42,7 +42,6 @@ def test_method_create(self, client: NeMoPlatform) -> None: evaluation = client.evaluations.create( workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", ) assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @@ -53,10 +52,11 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: evaluation = client.evaluations.create( workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", dataset_version="dataset_version", description="description", + experiment_group_id="experiment_group_id", + experiment_ids=["string"], metadata={"foo": "string"}, parent_evaluation_id="parent_evaluation_id", parent_experiment_id="parent_experiment_id", @@ -72,7 +72,6 @@ def test_raw_response_create(self, client: NeMoPlatform) -> None: response = client.evaluations.with_raw_response.create( workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", ) @@ -87,7 +86,6 @@ def test_streaming_response_create(self, client: NeMoPlatform) -> None: with client.evaluations.with_streaming_response.create( workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", ) as response: assert not response.is_closed @@ -105,7 +103,6 @@ def test_path_params_create(self, client: NeMoPlatform) -> None: client.evaluations.with_raw_response.create( workspace="", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", ) @@ -168,7 +165,6 @@ def test_method_update(self, client: NeMoPlatform) -> None: path_name="name", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @@ -180,10 +176,11 @@ def test_method_update_with_all_params(self, client: NeMoPlatform) -> None: path_name="name", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", dataset_version="dataset_version", description="description", + experiment_group_id="experiment_group_id", + experiment_ids=["string"], metadata={"foo": "string"}, parent_evaluation_id="parent_evaluation_id", parent_experiment_id="parent_experiment_id", @@ -200,7 +197,6 @@ def test_raw_response_update(self, client: NeMoPlatform) -> None: path_name="name", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) @@ -216,7 +212,6 @@ def test_streaming_response_update(self, client: NeMoPlatform) -> None: path_name="name", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) as response: assert not response.is_closed @@ -235,7 +230,6 @@ def test_path_params_update(self, client: NeMoPlatform) -> None: path_name="name", workspace="", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) @@ -244,7 +238,6 @@ def test_path_params_update(self, client: NeMoPlatform) -> None: path_name="", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) @@ -533,6 +526,74 @@ def test_path_params_delete(self, client: NeMoPlatform) -> None: workspace="workspace", ) + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_patch(self, client: NeMoPlatform) -> None: + evaluation = client.evaluations.patch( + name="name", + workspace="workspace", + ) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_patch_with_all_params(self, client: NeMoPlatform) -> None: + evaluation = client.evaluations.patch( + name="name", + workspace="workspace", + description="description", + experiment_ids=["string"], + metadata={"foo": "string"}, + parent_evaluation_id="parent_evaluation_id", + root_cause="root_cause", + source_link="https://example.com", + status="status", + ) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_patch(self, client: NeMoPlatform) -> None: + response = client.evaluations.with_raw_response.patch( + name="name", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + evaluation = response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_patch(self, client: NeMoPlatform) -> None: + with client.evaluations.with_streaming_response.patch( + name="name", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + evaluation = response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_patch(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.evaluations.with_raw_response.patch( + name="name", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + client.evaluations.with_raw_response.patch( + name="", + workspace="workspace", + ) + @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_pin(self, client: NeMoPlatform) -> None: @@ -649,7 +710,6 @@ async def test_method_create(self, async_client: AsyncNeMoPlatform) -> None: evaluation = await async_client.evaluations.create( workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", ) assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @@ -660,10 +720,11 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo evaluation = await async_client.evaluations.create( workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", dataset_version="dataset_version", description="description", + experiment_group_id="experiment_group_id", + experiment_ids=["string"], metadata={"foo": "string"}, parent_evaluation_id="parent_evaluation_id", parent_experiment_id="parent_experiment_id", @@ -679,7 +740,6 @@ async def test_raw_response_create(self, async_client: AsyncNeMoPlatform) -> Non response = await async_client.evaluations.with_raw_response.create( workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", ) @@ -694,7 +754,6 @@ async def test_streaming_response_create(self, async_client: AsyncNeMoPlatform) async with async_client.evaluations.with_streaming_response.create( workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", ) as response: assert not response.is_closed @@ -712,7 +771,6 @@ async def test_path_params_create(self, async_client: AsyncNeMoPlatform) -> None await async_client.evaluations.with_raw_response.create( workspace="", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", ) @@ -775,7 +833,6 @@ async def test_method_update(self, async_client: AsyncNeMoPlatform) -> None: path_name="name", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @@ -787,10 +844,11 @@ async def test_method_update_with_all_params(self, async_client: AsyncNeMoPlatfo path_name="name", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", dataset_version="dataset_version", description="description", + experiment_group_id="experiment_group_id", + experiment_ids=["string"], metadata={"foo": "string"}, parent_evaluation_id="parent_evaluation_id", parent_experiment_id="parent_experiment_id", @@ -807,7 +865,6 @@ async def test_raw_response_update(self, async_client: AsyncNeMoPlatform) -> Non path_name="name", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) @@ -823,7 +880,6 @@ async def test_streaming_response_update(self, async_client: AsyncNeMoPlatform) path_name="name", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) as response: assert not response.is_closed @@ -842,7 +898,6 @@ async def test_path_params_update(self, async_client: AsyncNeMoPlatform) -> None path_name="name", workspace="", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) @@ -851,7 +906,6 @@ async def test_path_params_update(self, async_client: AsyncNeMoPlatform) -> None path_name="", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) @@ -1140,6 +1194,74 @@ async def test_path_params_delete(self, async_client: AsyncNeMoPlatform) -> None workspace="workspace", ) + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_patch(self, async_client: AsyncNeMoPlatform) -> None: + evaluation = await async_client.evaluations.patch( + name="name", + workspace="workspace", + ) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_patch_with_all_params(self, async_client: AsyncNeMoPlatform) -> None: + evaluation = await async_client.evaluations.patch( + name="name", + workspace="workspace", + description="description", + experiment_ids=["string"], + metadata={"foo": "string"}, + parent_evaluation_id="parent_evaluation_id", + root_cause="root_cause", + source_link="https://example.com", + status="status", + ) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_patch(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.evaluations.with_raw_response.patch( + name="name", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + evaluation = await response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_patch(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.evaluations.with_streaming_response.patch( + name="name", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + evaluation = await response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_patch(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.evaluations.with_raw_response.patch( + name="name", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + await async_client.evaluations.with_raw_response.patch( + name="", + workspace="workspace", + ) + @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_pin(self, async_client: AsyncNeMoPlatform) -> None: diff --git a/sdk/stainless.yaml b/sdk/stainless.yaml index f552111d2c..8ebfadbe66 100644 --- a/sdk/stainless.yaml +++ b/sdk/stainless.yaml @@ -929,6 +929,7 @@ resources: standalone_api: true models: evaluation_filter: EvaluationFilter + evaluation_patch_request: EvaluationPatchRequest evaluation_request: EvaluationRequest evaluation_response: EvaluationResponse evaluation_responses_page: EvaluationResponsesPage @@ -940,6 +941,7 @@ resources: list: get /apis/intake/v2/workspaces/{workspace}/evaluations retrieve: get /apis/intake/v2/workspaces/{workspace}/evaluations/{name} update: put /apis/intake/v2/workspaces/{workspace}/evaluations/{name} + patch: patch /apis/intake/v2/workspaces/{workspace}/evaluations/{name} delete: delete /apis/intake/v2/workspaces/{workspace}/evaluations/{name} pin: post /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin unpin: delete /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin diff --git a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml index 7df668f042..3ccf30203e 100644 --- a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml +++ b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml @@ -915,6 +915,12 @@ authz: scopes: - intake:read - platform:read + patch: + permissions: + - intake.evaluations.update + scopes: + - intake:write + - platform:write put: permissions: - intake.evaluations.update diff --git a/services/core/entities/src/nmp/core/entities/app/repository/sqlalchemy/filter.py b/services/core/entities/src/nmp/core/entities/app/repository/sqlalchemy/filter.py index 40eb859d82..3c7bc3a914 100644 --- a/services/core/entities/src/nmp/core/entities/app/repository/sqlalchemy/filter.py +++ b/services/core/entities/src/nmp/core/entities/app/repository/sqlalchemy/filter.py @@ -85,16 +85,23 @@ def _coerce_value_for_column(self, column: ColumnElement, value: Any) -> Any: return value + def _cast_json_to_raw_text(self, column: Any) -> Any: + """Cast a JSON column element to its raw serialized text, quotes and all. + + Unlike ``_cast_json_to_text``, this keeps JSON's surrounding double quotes. Use it when the + quotes carry meaning — e.g. matching a quote-delimited array element (``$contains``) or comparing + against the literal ``"null"``/``"true"``/``"false"`` tokens both backends render. + """ + return cast(column, String) + def _cast_json_to_text(self, column: Any) -> Any: - """Cast a JSON column element to text, handling SQLite's quoted output. + """Cast a JSON column element to text, trimming JSON's surrounding double quotes. - SQLite's json_extract returns string values with quotes (e.g., '"value"'). - PostgreSQL's JSONB subscript also returns JSON-formatted strings. - We use TRIM to remove surrounding quotes for consistent comparison. + SQLite's json_extract returns string values with quotes (e.g., '"value"'), and PostgreSQL's + JSONB subscript also returns JSON-formatted strings. Trimming yields a consistent bare value for + equality/substring comparison. Use ``_cast_json_to_raw_text`` when the quotes must be preserved. """ - # Cast to string and trim surrounding double quotes - # This handles both SQLite and PostgreSQL JSON string extraction - return func.trim(cast(column, String), '"') + return func.trim(self._cast_json_to_raw_text(column), '"') def _cast_json_to_numeric(self, column: Any) -> Any: """Cast a JSON column element to a float for numeric comparisons. @@ -121,11 +128,12 @@ def eq(self, field: str, value: Any) -> Any: """Equal comparison.""" column, is_json = self._get_column(field) if is_json: - # Handle None/null: match both missing JSON keys and explicit null values. - # SQLAlchemy's JSON subscript IS NULL doesn't work reliably across backends, - # but cast to String returns "null" for both cases on SQLite and PostgreSQL. + # Handle None/null: match both an explicit JSON null and an absent key. A present-but-null + # value extracts to the JSON text token "null"; a missing key extracts to SQL NULL (notably + # on PostgreSQL, where `data->'key'` on an absent key is SQL NULL, so casting it would never + # equal "null"). Test both so `field == None` catches missing and explicitly-null values. if value is None: - return cast(column, String) == "null" + return or_(self._cast_json_to_raw_text(column) == "null", column.is_(None)) # Handle boolean values specially: # - SQLite stores JSON booleans as integers (0/1), json_extract returns "0" or "1" # - PostgreSQL stores them as "false"/"true" @@ -134,8 +142,8 @@ def eq(self, field: str, value: Any) -> Any: sqlite_value = "1" if value else "0" pg_value = "true" if value else "false" return or_( - cast(column, String) == sqlite_value, - cast(column, String) == pg_value, + self._cast_json_to_raw_text(column) == sqlite_value, + self._cast_json_to_raw_text(column) == pg_value, ) # For string values, use _cast_json_to_text to handle quoted JSON output return self._cast_json_to_text(column) == str(value) @@ -178,6 +186,26 @@ def nin(self, field: str, values: List[Any]) -> Any: return self._cast_json_to_text(column).not_in([str(v) for v in values]) return column.not_in(values) + def contains(self, field: str, value: Any) -> Any: + """Array membership: true when the JSON array at ``field`` contains scalar ``value``. + + Portable across SQLite (JSON) and PostgreSQL (JSONB) without a dialect branch: the + array element serializes as a quote-delimited token (e.g. ``"g1"``) in both backends' + text rendering, so we match that token in the serialized array text. Quoting makes it + collision-safe against prefixes (``"g1"`` does not match ``["g10"]``). ``value`` is + coerced to text and LIKE wildcards are escaped, so only exact elements match. + + Intended for array-valued JSON fields (e.g. ``data.experiment_ids``); values are + assumed to be JSON scalars without embedded double quotes (entity ids qualify). + """ + column, is_json = self._get_column(field) + if not is_json: + raise ValueError(f"$contains requires a JSON array field, got non-JSON field '{field}'") + needle = str(value) + for ch in ("\\", "%", "_"): + needle = needle.replace(ch, f"\\{ch}") + return self._cast_json_to_raw_text(column).like(f'%"{needle}"%', escape="\\") + def and_op(self, operations: List[Any]) -> Any: """Logical AND.""" return and_(*operations) diff --git a/services/core/entities/tests/test_filter_matches_sql_parity.py b/services/core/entities/tests/test_filter_matches_sql_parity.py index c55716413a..efd9a01964 100644 --- a/services/core/entities/tests/test_filter_matches_sql_parity.py +++ b/services/core/entities/tests/test_filter_matches_sql_parity.py @@ -43,11 +43,12 @@ class FakeEntity(Base): # plain-column NULL (name on row 5) and an explicit/absent ``k`` for $eq-None # coverage are the only nullable bits, and $eq agrees with SQL on both. SEED = [ - dict(id=1, name="llama", data={"score": 5, "tier": "free", "flag": True, "k": None}), - dict(id=2, name="Llama-2", data={"score": 9, "tier": "pro", "flag": False}), - dict(id=3, name="zephyr", data={"score": 10, "tier": "pro", "flag": True, "k": "v"}), - dict(id=4, name="mistral", data={"score": 100, "tier": "enterprise", "flag": False}), - dict(id=5, name=None, data={"score": 1, "tier": "free", "flag": False}), + dict(id=1, name="llama", data={"score": 5, "tier": "free", "flag": True, "k": None, "tags": ["red", "blue"]}), + dict(id=2, name="Llama-2", data={"score": 9, "tier": "pro", "flag": False, "tags": ["red"]}), + # "redish" is a deliberate prefix near-miss for "red" — quote-delimited matching must exclude it. + dict(id=3, name="zephyr", data={"score": 10, "tier": "pro", "flag": True, "k": "v", "tags": ["green", "redish"]}), + dict(id=4, name="mistral", data={"score": 100, "tier": "enterprise", "flag": False, "tags": []}), + dict(id=5, name=None, data={"score": 1, "tier": "free", "flag": False, "tags": ["blue"]}), ] @@ -96,6 +97,14 @@ def NOT(op): ("nin_name", C(FilterOperator.NIN, "name", ["llama"])), ("nin_data_tier", C(FilterOperator.NIN, "data.tier", ["pro"])), ("nin_data_score", C(FilterOperator.NIN, "data.score", [5, 9])), + ("contains_tags_red", C(FilterOperator.CONTAINS, "data.tags", "red")), + ("contains_tags_blue", C(FilterOperator.CONTAINS, "data.tags", "blue")), + ("contains_tags_absent", C(FilterOperator.CONTAINS, "data.tags", "nope")), + ("not_contains_tags_red", NOT(C(FilterOperator.CONTAINS, "data.tags", "red"))), + ( + "and_contains_tags", + AND(C(FilterOperator.CONTAINS, "data.tags", "blue"), C(FilterOperator.EQ, "data.tier", "free")), + ), ("gt_data_score", C(FilterOperator.GT, "data.score", 9)), ("gte_data_score", C(FilterOperator.GTE, "data.score", 10)), ("lt_data_score", C(FilterOperator.LT, "data.score", 10)), diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index 8a0e7afe7b..04de88e863 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -27,6 +27,7 @@ from nmp.common.service.dependencies import get_entity_client from nmp.intake.api.v2.experiments.schemas import ( EvaluationFilter, + EvaluationPatchRequest, EvaluationRequest, EvaluationResponse, EvaluationSessionFilter, @@ -296,21 +297,16 @@ async def delete_experiment_group( ) _reject_if_deleted(group, workspace=workspace, name=name, label="Experiment group") - # Cascade is sequential — one update per child. Linear in group size, fine for now. If - # groups routinely hold more than a few hundred evaluations, add a bulk update endpoint on - # the entity store rather than parallelizing here (gather hides partial-failure state - # without removing the per-row API contract). - # - # Each ``_soft_delete`` renames the row and flips ``is_deleted=True``, which drops it out of - # the live filter — so re-fetching page 1 keeps returning the next batch until nothing is - # left. No fixed cap on group size. - # - # ``data.experiment_group_id`` is the entity-store field name; the URL filter dep auto- - # prefixes ``data.`` but manually-constructed ComparisonOperations don't get that translation. + # Reference-counted cascade, sequential — one update per member. Linear in group size, fine for + # now. A member whose *sole* membership was this group is soft-deleted; a member also in another + # group just drops this membership and survives there. Both outcomes remove the member from the + # live-membership filter below (soft-delete renames + flips is_deleted; membership removal drops + # the group id from experiment_ids and leaves no legacy scalar), so re-fetching page 1 keeps + # returning the next unprocessed batch until nothing is left. No fixed cap on group size. live_children_filter = LogicalOperation( operator=FilterOperator.AND, operations=[ - ComparisonOperation(operator=FilterOperator.EQ, field="data.experiment_group_id", value=group.id), + _group_membership_filter(group.id), LogicalOperation( operator=FilterOperator.NOT, operations=[ @@ -330,7 +326,14 @@ async def delete_experiment_group( if not page.data: break for child in page.data: - await _soft_delete(entity_client, child) + remaining = [gid for gid in child.experiment_ids if gid != group.id] + if remaining: + # Shared with another group — drop only this membership; the member lives on there. + child.experiment_ids = remaining + await entity_client.update(child) + else: + # This group was the member's sole membership — cascade the delete. + await _soft_delete(entity_client, child) await _soft_delete(entity_client, group) @@ -346,12 +349,12 @@ async def create_evaluation( body: EvaluationRequest, entity_client: EntityClientDep, ) -> EvaluationResponse: - await _validate_group_exists(entity_client, group_id=body.experiment_group_id) + await _validate_groups_exist(entity_client, group_ids=body.experiment_ids) await _validate_parent_evaluation_exists(entity_client, parent_evaluation_id=body.parent_evaluation_id) entity = Evaluation( workspace=workspace, name=body.name, - experiment_group_id=body.experiment_group_id, + experiment_ids=body.experiment_ids, dataset_name=body.dataset_name, dataset_version=body.dataset_version, source_link=body.source_link, @@ -433,6 +436,9 @@ async def list_evaluations( # the metric ones are applied in memory after hydration. parsed (the full user filter) is left # intact so the response still echoes it. entity_operation, metric_predicates = _extract_metric_predicates(parsed.operation) + # Translate the exposed `experiment_group_id` filter into a membership match over `experiment_ids` + # (plus the legacy scalar), so listing a group returns every evaluation that belongs to it. + entity_operation = _rewrite_group_filter(entity_operation) # Compute-on-read: fetch the whole (entity-filtered) group, hydrate every rollup, then filter, sort, # and paginate in memory so a single request can sort/filter by a ClickHouse metric that lives # outside the entity store. Bounded to hundreds of evaluations per group (see _MAX_GROUP_EVALUATIONS). @@ -546,8 +552,9 @@ async def update_evaluation( label="Evaluation", ) _reject_if_deleted(existing, workspace=workspace, name=name, label="Evaluation") - if body.experiment_group_id != existing.experiment_group_id: - await _validate_group_exists(entity_client, group_id=body.experiment_group_id) + new_group_ids = [gid for gid in body.experiment_ids if gid not in existing.experiment_ids] + if new_group_ids: + await _validate_groups_exist(entity_client, group_ids=new_group_ids) await _validate_parent_evaluation_exists(entity_client, parent_evaluation_id=body.parent_evaluation_id) changed = [f for f in _IMMUTABLE_EVALUATION_FIELDS if getattr(body, f) != getattr(existing, f)] @@ -560,7 +567,7 @@ async def update_evaluation( ), ) - existing.experiment_group_id = body.experiment_group_id + existing.experiment_ids = body.experiment_ids existing.source_link = body.source_link existing.metadata = body.metadata existing.description = body.description @@ -573,6 +580,65 @@ async def update_evaluation( return response +@router.patch( + "/v2/workspaces/{workspace}/evaluations/{name}", + response_model=EvaluationResponse, + tags=[EVALUATIONS_TAG], + responses={ + 400: {"description": "A referenced ExperimentGroup does not exist, or experiment_ids is empty"}, + 404: {"description": "Evaluation not found"}, + }, +) +async def patch_evaluation( + workspace: str, + name: str, + body: EvaluationPatchRequest, + entity_client: EntityClientDep, + rollup_repository: EvaluationRollupRepositoryDep, +) -> EvaluationResponse: + """Partially update an evaluation: only fields present in the request are changed. + + The common case is curating an evaluation into another ExperimentGroup — PATCH with the merged + ``experiment_ids``. Membership is replaced (not appended), so send the full desired set; any new + group must exist and the set must be non-empty (an evaluation always belongs to >=1 group). Omitted + fields are left untouched (unlike the full-body PUT, which overwrites them). + """ + existing = await _get_or_404(entity_client, Evaluation, workspace=workspace, name=name, label="Evaluation") + _reject_if_deleted(existing, workspace=workspace, name=name, label="Evaluation") + + fields_set = body.model_fields_set + if "experiment_ids" in fields_set: + if not body.experiment_ids: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="experiment_ids must be non-empty: an evaluation must belong to at least one group.", + ) + new_group_ids = [gid for gid in body.experiment_ids if gid not in existing.experiment_ids] + if new_group_ids: + await _validate_groups_exist(entity_client, group_ids=new_group_ids) + # Membership is a set — store deduped so a duplicated id can't inflate a group's count. + existing.experiment_ids = list(dict.fromkeys(body.experiment_ids)) + if "parent_evaluation_id" in fields_set: + await _validate_parent_evaluation_exists(entity_client, parent_evaluation_id=body.parent_evaluation_id) + existing.parent_experiment_id = body.parent_evaluation_id + if "source_link" in fields_set: + existing.source_link = body.source_link + if "metadata" in fields_set: + # Entity metadata is a non-null dict; a client clearing it (explicit null) resets to empty. + existing.metadata = body.metadata or {} + if "description" in fields_set: + existing.description = body.description + if "status" in fields_set: + existing.status = body.status + if "root_cause" in fields_set: + existing.root_cause = body.root_cause + + updated = await entity_client.update(existing) + response = EvaluationResponse.from_entity(updated) + await _hydrate_rollups(workspace=workspace, responses=[response], rollup_repository=rollup_repository) + return response + + @router.delete( "/v2/workspaces/{workspace}/evaluations/{name}", status_code=status.HTTP_204_NO_CONTENT, @@ -851,7 +917,7 @@ async def _count_live_evaluations_in_group(entity_client: EntityClient, *, works filter_operation=LogicalOperation( operator=FilterOperator.AND, operations=[ - ComparisonOperation(operator=FilterOperator.EQ, field="data.experiment_group_id", value=group_id), + _group_membership_filter(group_id), LogicalOperation( operator=FilterOperator.NOT, operations=[ @@ -889,7 +955,10 @@ async def _count_live_evaluations_by_group( filter_operation = LogicalOperation( operator=FilterOperator.AND, operations=[ - ComparisonOperation(operator=FilterOperator.IN, field="data.experiment_group_id", value=list(group_ids)), + LogicalOperation( + operator=FilterOperator.OR, + operations=[_group_membership_filter(gid) for gid in dict.fromkeys(group_ids)], + ), LogicalOperation( operator=FilterOperator.NOT, operations=[ @@ -906,8 +975,11 @@ async def _count_live_evaluations_by_group( page=page, page_size=page_size, ) + # An evaluation may belong to several of the requested groups; count it once per membership. for evaluation in result.data: - counts[evaluation.experiment_group_id] = counts.get(evaluation.experiment_group_id, 0) + 1 + for gid in evaluation.experiment_ids: + if gid in counts: + counts[gid] += 1 if page >= result.pagination.total_pages: break page += 1 @@ -943,6 +1015,50 @@ async def _validate_group_exists(entity_client: EntityClient, *, group_id: str) ) +async def _validate_groups_exist(entity_client: EntityClient, *, group_ids: list[str]) -> None: + """Reject with 400 if any referenced ExperimentGroup doesn't exist or is deleted (deduped).""" + for group_id in dict.fromkeys(group_ids): + await _validate_group_exists(entity_client, group_id=group_id) + + +def _group_membership_filter(group_id: str) -> LogicalOperation: + """Match evaluations that belong to ``group_id`` across both membership representations. + + New rows store membership as the ``experiment_ids`` list (matched with ``$contains``); legacy + rows that predate many-to-many still store a scalar ``experiment_group_id`` and are matched by + equality until they are next rewritten. The OR keeps un-migrated rows queryable, no migration. + """ + return LogicalOperation( + operator=FilterOperator.OR, + operations=[ + ComparisonOperation(operator=FilterOperator.CONTAINS, field="data.experiment_ids", value=group_id), + ComparisonOperation(operator=FilterOperator.EQ, field="data.experiment_group_id", value=group_id), + ], + ) + + +def _rewrite_group_filter(operation: FilterOperation | None) -> FilterOperation | None: + """Rewrite an ``experiment_group_id`` equality in a parsed filter into a membership match. + + The API still exposes an ``experiment_group_id`` filter param; with many-to-many membership it + means "belongs to this group", which spans the ``experiment_ids`` list and the legacy scalar. + """ + if operation is None: + return None + if isinstance(operation, ComparisonOperation): + if operation.field == "data.experiment_group_id" and operation.operator == FilterOperator.EQ: + return _group_membership_filter(operation.value) + return operation + if isinstance(operation, LogicalOperation): + return LogicalOperation( + operator=operation.operator, + operations=[ + rewritten for op in operation.operations if (rewritten := _rewrite_group_filter(op)) is not None + ], + ) + return operation + + def _apply_is_deleted_filter(parsed: ParsedFilter) -> None: """Append an ``is_deleted`` clause so list endpoints hide soft-deleted rows by default. diff --git a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py index 3067e8bf2a..2f136c7e5d 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py @@ -56,8 +56,20 @@ class EvaluationRequest(BaseModel): model_config = ConfigDict(extra="forbid") name: str = Field(description="Producer-supplied, workspace-unique evaluation id.") - experiment_group_id: str = Field( - description="Entity id of the owning ExperimentGroup. Required — the group must already exist.", + experiment_ids: list[str] = Field( + default_factory=list, + description=( + "Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). Preferred; each group " + "must already exist. When omitted, the deprecated experiment_group_id is used instead." + ), + ) + experiment_group_id: str | None = Field( + default=None, + deprecated=True, + description=( + "Deprecated single-group field; provide experiment_ids instead. Coalesced into experiment_ids " + "when experiment_ids is omitted." + ), ) dataset_name: str = Field(description="Producer-supplied dataset name.") dataset_version: str | None = Field(default=None, description="Producer-supplied dataset version.") @@ -79,6 +91,25 @@ class EvaluationRequest(BaseModel): description="Human- or agent-authored explanation of the evaluation's outcome (e.g. why it was killed).", ) + @model_validator(mode="after") + def _resolve_group_membership(self) -> Self: + """Require >=1 group. Coalesce the deprecated ``experiment_group_id`` into ``experiment_ids`` + when the latter is omitted; reject a request that supplies neither. + + Read the deprecated value via ``__dict__`` to avoid tripping its deprecation warning per request. + """ + if not self.experiment_ids: + group_id = self.__dict__.get("experiment_group_id") + if group_id: + self.experiment_ids = [group_id] + if not self.experiment_ids: + raise ValueError( + "An evaluation must belong to at least one group: provide experiment_ids or experiment_group_id." + ) + # Membership is a set — drop duplicate group ids (order-preserving) so counts aren't inflated. + self.experiment_ids = list(dict.fromkeys(self.experiment_ids)) + return self + @model_validator(mode="after") def _coalesce_deprecated_parent(self) -> Self: """Accept the deprecated ``parent_experiment_id`` alias; the canonical field wins if both are set. @@ -92,6 +123,37 @@ def _coalesce_deprecated_parent(self) -> Self: return self +class EvaluationPatchRequest(BaseModel): + """Partial-update body for an Evaluation: only fields present in the request are applied. + + Unset fields are left unchanged (PATCH semantics — same pattern as the models service's PATCH). + Immutable fields (name, dataset_name, dataset_version) aren't accepted here. ``experiment_ids``, + when provided, must be non-empty: an evaluation must always belong to at least one group. + """ + + model_config = ConfigDict(extra="forbid") + + experiment_ids: list[str] | None = Field( + default=None, + description=( + "Replace the ExperimentGroups this Evaluation belongs to. Must be non-empty when provided; " + "each group must already exist. Omit to leave membership unchanged." + ), + ) + source_link: AnyUrl | None = Field(default=None, description="Optional URL for the source evaluation.") + metadata: dict[str, str] | None = Field(default=None, description="Free-form producer metadata.") + description: str | None = Field(default=None, description="Human-readable description.") + parent_evaluation_id: str | None = Field( + default=None, + description="Entity id of the evaluation this one was derived from (e.g. a variant of a baseline), if any.", + ) + status: str | None = Field(default=None, description="Producer-defined lifecycle status of the evaluation.") + root_cause: str | None = Field( + default=None, + description="Human- or agent-authored explanation of the evaluation's outcome (e.g. why it was killed).", + ) + + class ExperimentGroupResponse(BaseModel): """ExperimentGroup as served by the API.""" @@ -149,8 +211,8 @@ class EvaluationResponse(BaseModel): id: str name: str workspace: str - experiment_group_id: str = Field( - description="Entity id of the owning ExperimentGroup. Required for every Evaluation.", + experiment_ids: list[str] = Field( + description="Entity ids of the ExperimentGroups this Evaluation belongs to (>=1).", ) dataset_name: str dataset_version: str | None = None @@ -208,13 +270,21 @@ class EvaluationResponse(BaseModel): def parent_experiment_id(self) -> str | None: return self.parent_evaluation_id + @computed_field( # type: ignore[prop-decorator] + deprecated=True, + description="Deprecated single-group alias; the first of experiment_ids. Use experiment_ids.", + ) + @property + def experiment_group_id(self) -> str: + return self.experiment_ids[0] + @classmethod def from_entity(cls, entity: Experiment) -> EvaluationResponse: return cls( id=entity.id, name=entity.name, workspace=entity.workspace, - experiment_group_id=entity.experiment_group_id, + experiment_ids=entity.experiment_ids, dataset_name=entity.dataset_name, dataset_version=entity.dataset_version, source_link=entity.source_link, diff --git a/services/intake/src/nmp/intake/entities/experiments.py b/services/intake/src/nmp/intake/entities/experiments.py index 1d1b8c150b..8710286c0d 100644 --- a/services/intake/src/nmp/intake/entities/experiments.py +++ b/services/intake/src/nmp/intake/entities/experiments.py @@ -22,7 +22,7 @@ from typing import Any, ClassVar from nmp.common.entities.client import EntityBase -from pydantic import AnyUrl, Field, field_validator +from pydantic import AnyUrl, Field, field_validator, model_validator def _stringify_metadata(value: Any) -> Any: @@ -94,13 +94,30 @@ class Experiment(EntityBase): __entity_type__: ClassVar[str] = "experiment" - experiment_group_id: str = Field( + experiment_ids: list[str] = Field( + min_length=1, description=( - "Entity id of the owning ExperimentGroup. Required — every Experiment must belong to a Group. " - "Validated at create/update time; deleting a Group cascades to its Experiments." + "Entity ids of the ExperimentGroups this Experiment belongs to (>=1). An Experiment " + "always belongs to at least one group. Validated at create/update time. Deleting a Group " + "removes it from this set; an Experiment whose sole membership was that Group is deleted " + "with it, while one shared with another Group survives there." ), ) + @model_validator(mode="before") + @classmethod + def _coerce_group_membership(cls, data: Any) -> Any: + """Schema-on-read: coerce a legacy single ``experiment_group_id`` into ``experiment_ids``. + + Rows written before many-to-many membership stored a scalar ``experiment_group_id`` and no + ``experiment_ids``. Synthesize the one-element list so old rows read back unchanged, with no + data migration. When ``experiment_ids`` is already present it wins (a re-saved row drops the + stale scalar). Non-dict input passes through for pydantic to handle. + """ + if isinstance(data, dict) and not data.get("experiment_ids") and data.get("experiment_group_id"): + return {**data, "experiment_ids": [data["experiment_group_id"]]} + return data + dataset_name: str = Field(description="Producer-supplied dataset name.") dataset_version: str | None = Field(default=None, description="Producer-supplied dataset version.") source_link: AnyUrl | None = Field(default=None, description="Optional URL for the source experiment.") diff --git a/services/intake/tests/integration/test_experiments_crud.py b/services/intake/tests/integration/test_experiments_crud.py index a60215513f..432f8fc87d 100644 --- a/services/intake/tests/integration/test_experiments_crud.py +++ b/services/intake/tests/integration/test_experiments_crud.py @@ -517,3 +517,117 @@ def test_sort_by_pinned_at_most_recent_first(client: TestClient) -> None: assert pinned_asc.status_code == 200, pinned_asc.text names_asc = [e["name"] for e in pinned_asc.json()["data"]] assert names_asc.index("exp-old-pin") < names_asc.index("exp-new-pin") + + +def test_evaluation_belongs_to_multiple_groups(client: TestClient) -> None: + """An evaluation created with multiple experiment_ids appears in each group's leaderboard.""" + group_a = client.post(GROUPS, json={"name": "multi-a"}).json() + group_b = client.post(GROUPS, json={"name": "multi-b"}).json() + body = _evaluation_body(name="multi-member", experiment_group_id="placeholder") + body.pop("experiment_group_id") + body["experiment_ids"] = [group_a["id"], group_b["id"]] + + created = client.post(EVALUATIONS, json=body) + assert created.status_code == 201, created.text + assert set(created.json()["experiment_ids"]) == {group_a["id"], group_b["id"]} + # Deprecated single-group alias still resolves (to the first membership). + assert created.json()["experiment_group_id"] == group_a["id"] + + for gid in (group_a["id"], group_b["id"]): + listed = client.get(EVALUATIONS, params={"filter[experiment_group_id]": gid}) + assert listed.status_code == 200, listed.text + assert any(e["name"] == "multi-member" for e in listed.json()["data"]) + + +def test_evaluation_create_rejects_empty_experiment_ids(client: TestClient) -> None: + """The >=1-group invariant: an empty experiment_ids list is rejected.""" + body = _evaluation_body(name="no-groups", experiment_group_id="placeholder") + body.pop("experiment_group_id") + body["experiment_ids"] = [] + resp = client.post(EVALUATIONS, json=body) + assert resp.status_code == 422, resp.text + + +def test_delete_group_reference_counted_cascade(client: TestClient) -> None: + """Deleting a group deletes only its sole-membership members; shared members survive elsewhere.""" + group_a = client.post(GROUPS, json={"name": "rc-a"}).json() + group_b = client.post(GROUPS, json={"name": "rc-b"}).json() + + shared = _evaluation_body(name="rc-shared", experiment_group_id="placeholder") + shared.pop("experiment_group_id") + shared["experiment_ids"] = [group_a["id"], group_b["id"]] + client.post(EVALUATIONS, json=shared) + client.post(EVALUATIONS, json=_evaluation_body(name="rc-sole", experiment_group_id=group_a["id"])) + + assert client.delete(f"{GROUPS}/rc-a").status_code == 204 + + # Sole member gone; shared member survives, now only in group b. + assert client.get(f"{EVALUATIONS}/rc-sole").status_code == 404 + survivor = client.get(f"{EVALUATIONS}/rc-shared") + assert survivor.status_code == 200, survivor.text + assert survivor.json()["experiment_ids"] == [group_b["id"]] + + in_b = client.get(EVALUATIONS, params={"filter[experiment_group_id]": group_b["id"]}) + assert any(e["name"] == "rc-shared" for e in in_b.json()["data"]) + + +def test_patch_evaluation_adds_to_group(client: TestClient) -> None: + """PATCH .../evaluations/{name} with a merged experiment_ids adds a membership; the run shows on the board.""" + group_a = client.post(GROUPS, json={"name": "add-a"}).json() + group_b = client.post(GROUPS, json={"name": "add-b"}).json() + client.post(EVALUATIONS, json=_evaluation_body(name="add-eval", experiment_group_id=group_a["id"])) + + added = client.patch(f"{EVALUATIONS}/add-eval", json={"experiment_ids": [group_a["id"], group_b["id"]]}) + assert added.status_code == 200, added.text + assert set(added.json()["experiment_ids"]) == {group_a["id"], group_b["id"]} + + in_b = client.get(EVALUATIONS, params={"filter[experiment_group_id]": group_b["id"]}) + assert any(e["name"] == "add-eval" for e in in_b.json()["data"]) + + +def test_patch_evaluation_unknown_group_rejected(client: TestClient) -> None: + group = client.post(GROUPS, json={"name": "add-known"}).json() + client.post(EVALUATIONS, json=_evaluation_body(name="add-eval-2", experiment_group_id=group["id"])) + resp = client.patch(f"{EVALUATIONS}/add-eval-2", json={"experiment_ids": [group["id"], "experiment_group-nope"]}) + assert resp.status_code == 400, resp.text + + +def test_patch_evaluation_rejects_empty_experiment_ids(client: TestClient) -> None: + group = client.post(GROUPS, json={"name": "empty-ids"}).json() + client.post(EVALUATIONS, json=_evaluation_body(name="empty-ids-eval", experiment_group_id=group["id"])) + resp = client.patch(f"{EVALUATIONS}/empty-ids-eval", json={"experiment_ids": []}) + assert resp.status_code == 400, resp.text + # The >=1-group invariant holds: membership is unchanged. + assert client.get(f"{EVALUATIONS}/empty-ids-eval").json()["experiment_ids"] == [group["id"]] + + +def test_patch_evaluation_leaves_omitted_fields_unchanged(client: TestClient) -> None: + """Partial semantics: PATCHing experiment_ids alone must not clobber omitted fields (e.g. description).""" + group_a = client.post(GROUPS, json={"name": "patch-a"}).json() + group_b = client.post(GROUPS, json={"name": "patch-b"}).json() + body = _evaluation_body(name="patch-eval", experiment_group_id=group_a["id"]) + body["description"] = "keep me" + body["metadata"] = {"team": "switchyard"} + client.post(EVALUATIONS, json=body) + + patched = client.patch(f"{EVALUATIONS}/patch-eval", json={"experiment_ids": [group_a["id"], group_b["id"]]}) + assert patched.status_code == 200, patched.text + data = patched.json() + assert set(data["experiment_ids"]) == {group_a["id"], group_b["id"]} + assert data["description"] == "keep me" # omitted from the PATCH -> unchanged + assert data["metadata"] == {"team": "switchyard"} + + +def test_patch_evaluation_dedupes_experiment_ids(client: TestClient) -> None: + """Membership is a set: duplicate ids are stored once, so a group can't double-count an evaluation.""" + group = client.post(GROUPS, json={"name": "dedupe-grp"}).json() + client.post(EVALUATIONS, json=_evaluation_body(name="dedupe-eval", experiment_group_id=group["id"])) + + patched = client.patch(f"{EVALUATIONS}/dedupe-eval", json={"experiment_ids": [group["id"], group["id"]]}) + assert patched.status_code == 200, patched.text + assert patched.json()["experiment_ids"] == [group["id"]] + + # The group counts the evaluation once, not twice. + listed = client.get(GROUPS) + grp = next(g for g in listed.json()["data"] if g["id"] == group["id"]) + assert grp["evaluation_count"] == 1 diff --git a/services/intake/tests/test_experiment_default_sort.py b/services/intake/tests/test_experiment_default_sort.py index cc4af97e38..002decbdf5 100644 --- a/services/intake/tests/test_experiment_default_sort.py +++ b/services/intake/tests/test_experiment_default_sort.py @@ -35,7 +35,7 @@ def _exp( id=name, name=name, workspace="default", - experiment_group_id="grp", + experiment_ids=["grp"], dataset_name="ds", cost_usd=EvaluatorAggregate(mean=cost) if cost is not None else None, latency_ms=EvaluatorAggregate(mean=latency) if latency is not None else None, diff --git a/services/intake/tests/test_experiment_metric_filter.py b/services/intake/tests/test_experiment_metric_filter.py index b6c23ed6c5..5b97cfb621 100644 --- a/services/intake/tests/test_experiment_metric_filter.py +++ b/services/intake/tests/test_experiment_metric_filter.py @@ -31,7 +31,7 @@ def _exp(name: str, *, run_count: int = 0, cost_mean: float | None = None) -> Ev id=name, name=name, workspace="default", - experiment_group_id="grp", + experiment_ids=["grp"], dataset_name="ds", run_count=run_count, cost_usd=EvaluatorAggregate(mean=cost_mean) if cost_mean is not None else None, diff --git a/services/intake/tests/test_experiment_sort.py b/services/intake/tests/test_experiment_sort.py index ad7f433ffa..62c9552a35 100644 --- a/services/intake/tests/test_experiment_sort.py +++ b/services/intake/tests/test_experiment_sort.py @@ -22,7 +22,7 @@ def _exp( id=name, name=name, workspace="default", - experiment_group_id="grp", + experiment_ids=["grp"], dataset_name="ds", run_count=run_count, cost_usd=EvaluatorAggregate(mean=cost_mean) if cost_mean is not None else None, diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.test.tsx b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.test.tsx new file mode 100644 index 0000000000..8eb330a505 --- /dev/null +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.test.tsx @@ -0,0 +1,167 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { MockToastProvider } from '@nemo/common/src/tests/MockToastProvider'; +import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import { AddToGroupModal } from '@studio/components/dataViews/ExperimentGroupDataView/AddToGroupModal'; +import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { render, screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +// Hoisted mocks for the SDK hooks the modal calls. +const { mockMutateAsync, mockCreateMutateAsync, mockUseListExperimentGroups } = vi.hoisted(() => ({ + mockMutateAsync: vi.fn<(...args: unknown[]) => Promise>(), + mockCreateMutateAsync: vi.fn<(...args: unknown[]) => Promise>(), + mockUseListExperimentGroups: vi.fn<() => { data: unknown; isLoading: boolean }>(), +})); + +vi.mock('@nemo/sdk/generated/platform/api', async () => { + const actual = await vi.importActual( + '@nemo/sdk/generated/platform/api' + ); + return { + ...actual, + useListExperimentGroups: () => mockUseListExperimentGroups(), + usePatchEvaluation: () => ({ mutateAsync: mockMutateAsync, isPending: false }), + useCreateExperimentGroup: () => ({ mutateAsync: mockCreateMutateAsync, isPending: false }), + }; +}); + +const makeGroup = (id: string, name: string): ExperimentGroupResponse => ({ + id, + name, + workspace: 'default', + default_sort: '-created_at', + experiment_count: 0, +}); + +const GROUPS: ExperimentGroupResponse[] = [ + makeGroup('g1', 'Alpha benchmarks'), + makeGroup('g2', 'Beta benchmarks'), + makeGroup('g3', 'Gamma benchmarks'), +]; + +const makeEvaluation = (name: string, experimentIds: string[]): EvaluationRow => + ({ id: name, name, experiment_ids: experimentIds }) as unknown as EvaluationRow; + +// eval-1 is only in g1; eval-2 is in g1 and g2. So g1 (all members) is excluded, but g2 (only some +// members) is still offered, as is g3 (no members). +const EVALUATIONS = [makeEvaluation('eval-1', ['g1']), makeEvaluation('eval-2', ['g1', 'g2'])]; + +function makeWrapper() { + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return ({ children }: { children: React.ReactNode }) => ( + + {children} + + ); +} + +function renderModal(evaluations: EvaluationRow[] = EVALUATIONS) { + const Wrapper = makeWrapper(); + const onSuccess = vi.fn(); + const utils = render( + + + + ); + return { ...utils, onSuccess }; +} + +describe('AddToGroupModal', () => { + beforeEach(() => { + mockMutateAsync.mockReset(); + mockMutateAsync.mockResolvedValue(undefined); + mockCreateMutateAsync.mockReset(); + mockCreateMutateAsync.mockResolvedValue(makeGroup('g-new', 'Regression suite')); + mockUseListExperimentGroups.mockReset(); + mockUseListExperimentGroups.mockReturnValue({ + data: { data: GROUPS }, + isLoading: false, + }); + }); + + it('offers "Create new group" plus only groups not every selected evaluation already belongs to', async () => { + const user = userEvent.setup(); + renderModal(); + await user.click(screen.getByRole('combobox', { name: /experiment group/i })); + expect(await screen.findByRole('option', { name: /create new group/i })).toBeInTheDocument(); + // g1: both evals are members -> excluded. g2: only eval-2 is a member -> still offered. g3: none. + expect(screen.getByRole('option', { name: 'Beta benchmarks' })).toBeInTheDocument(); + expect(screen.getByRole('option', { name: 'Gamma benchmarks' })).toBeInTheDocument(); + expect(screen.queryByRole('option', { name: 'Alpha benchmarks' })).not.toBeInTheDocument(); + }); + + it('adds every selected evaluation to the chosen group and clears the selection', async () => { + const user = userEvent.setup(); + const { onSuccess } = renderModal(); + + await user.click(screen.getByRole('combobox', { name: /experiment group/i })); + await user.click(await screen.findByRole('option', { name: 'Gamma benchmarks' })); + await user.click(screen.getByRole('button', { name: 'Add' })); + + expect(mockCreateMutateAsync).not.toHaveBeenCalled(); + expect(mockMutateAsync).toHaveBeenCalledTimes(2); + expect(mockMutateAsync).toHaveBeenCalledWith({ + workspace: 'default', + name: 'eval-1', + data: { experiment_ids: ['g1', 'g3'] }, + }); + expect(mockMutateAsync).toHaveBeenCalledWith({ + workspace: 'default', + name: 'eval-2', + data: { experiment_ids: ['g1', 'g2', 'g3'] }, + }); + await waitFor(() => expect(onSuccess).toHaveBeenCalledTimes(1)); + }); + + it('creates a new group then adds every selected evaluation to it', async () => { + const user = userEvent.setup(); + const { onSuccess } = renderModal(); + + await user.click(screen.getByRole('combobox', { name: /experiment group/i })); + await user.click(await screen.findByRole('option', { name: /create new group/i })); + await user.type(screen.getByRole('textbox', { name: /^name$/i }), 'regression-suite'); + await user.click(screen.getByRole('button', { name: 'Create & add' })); + + expect(mockCreateMutateAsync).toHaveBeenCalledWith({ + workspace: 'default', + data: expect.objectContaining({ name: 'regression-suite' }), + }); + // Added to the id returned by the create call. + expect(mockMutateAsync).toHaveBeenCalledTimes(2); + expect(mockMutateAsync).toHaveBeenCalledWith({ + workspace: 'default', + name: 'eval-1', + data: { experiment_ids: ['g1', 'g-new'] }, + }); + expect(mockMutateAsync).toHaveBeenCalledWith({ + workspace: 'default', + name: 'eval-2', + data: { experiment_ids: ['g1', 'g2', 'g-new'] }, + }); + await waitFor(() => expect(onSuccess).toHaveBeenCalledTimes(1)); + }); + + it('still offers "Create new group" when every group already contains all selected evaluations', async () => { + const user = userEvent.setup(); + mockUseListExperimentGroups.mockReturnValue({ + data: { data: [makeGroup('g1', 'Alpha benchmarks')] }, + isLoading: false, + }); + renderModal([makeEvaluation('eval-1', ['g1']), makeEvaluation('eval-2', ['g1'])]); + + const trigger = screen.getByRole('combobox', { name: /experiment group/i }); + expect(trigger).toBeEnabled(); + await user.click(trigger); + expect(await screen.findByRole('option', { name: /create new group/i })).toBeInTheDocument(); + expect(screen.queryByRole('option', { name: 'Alpha benchmarks' })).not.toBeInTheDocument(); + }); +}); diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.tsx b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.tsx new file mode 100644 index 0000000000..fa931472b6 --- /dev/null +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.tsx @@ -0,0 +1,311 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { zodResolver } from '@hookform/resolvers/zod'; +import { FormModal, type FormModalProps } from '@nemo/common/src/components/FormModal'; +import { useToast } from '@nemo/common/src/providers/toast/useToast'; +import { + getListEvaluationsQueryKey, + getListExperimentGroupsQueryKey, + useCreateExperimentGroup, + useListExperimentGroups, + usePatchEvaluation, +} from '@nemo/sdk/generated/platform/api'; +import { + Flex, + FormField, + SelectContent, + SelectItem, + SelectListbox, + SelectRoot, + SelectTrigger, + Stack, + TextArea, + TextInput, +} from '@nvidia/foundations-react-core'; +import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; +import { DefaultSortControl } from '@studio/components/DefaultSortControl'; +import { DEFAULT_SORT } from '@studio/components/DefaultSortControl/util'; +import { + experimentGroupCreateSchema, + type ExperimentGroupCreateFormFields, +} from '@studio/components/ExperimentGroupCreateModal/constants'; +import { DEFAULT_LARGE_PAGE_SIZE } from '@studio/constants/constants'; +import { useQueryClient } from '@tanstack/react-query'; +import { AxiosError } from 'axios'; +import { Plus } from 'lucide-react'; +import { type FC, type FormEvent, useEffect, useMemo, useState } from 'react'; +import { useForm, type SubmitHandler } from 'react-hook-form'; + +// Sentinel Select value for the "create a new group" affordance. Reserved so it can't collide with a +// real group id (group ids are entity ids, never this literal). +const CREATE_NEW = '__create_new__'; + +export interface AddToGroupModalProps extends Pick { + /** Called after the selected evaluations are successfully added (e.g. to clear the row selection). */ + onSuccess?: () => void; + workspace: string; + /** The evaluations being curated into another group (the bulk row selection). */ + evaluations: EvaluationRow[]; +} + +/** + * Adds one or more selected evaluations to another ExperimentGroup. Offers the workspace's groups in a + * dropdown — excluding groups every selected evaluation already belongs to — plus a "Create new group" + * option that reveals a name/description sub-form. On submit it either adds the evaluations to the + * chosen group, or creates the group first and then adds them, then refreshes the board and toasts. + * + * Membership is owned by the evaluation side (there's no atomic "create group with evaluations" + * endpoint), so the create path is two steps: create the group, then add each evaluation to it. A + * created-but-partially-populated group is a valid state, so a failed add is surfaced as a warning + * rather than rolled back. + */ +export const AddToGroupModal: FC = ({ + open, + onClose, + onSuccess, + workspace, + evaluations, +}) => { + const toast = useToast(); + const queryClient = useQueryClient(); + const [selectedGroupId, setSelectedGroupId] = useState(''); + const isCreating = selectedGroupId === CREATE_NEW; + // Default sort is a single `sort`-param string driven by a custom control (not a registered RHF + // input), so it's managed here and merged into the create payload (mirrors ExperimentGroupCreateModal). + const [defaultSort, setDefaultSort] = useState(DEFAULT_SORT); + + const { + register, + handleSubmit, + reset, + setValue, + setError, + formState: { errors, isValid }, + } = useForm({ + resolver: zodResolver(experimentGroupCreateSchema), + mode: 'onChange', + }); + + // Reset the selection and the create sub-form whenever the modal (re)opens so nothing carries over. + useEffect(() => { + if (open) { + setSelectedGroupId(''); + setDefaultSort(DEFAULT_SORT); + reset(); + } + }, [open, reset]); + + // Only fetch groups while the modal is open. A single large page covers any realistic group count. + const { data: groupsPage, isLoading } = useListExperimentGroups( + workspace, + { page_size: DEFAULT_LARGE_PAGE_SIZE }, + { query: { enabled: open && !!workspace } } + ); + + // Exclude a group only when EVERY selected evaluation already belongs to it (nothing to add there). + const groups = useMemo( + () => + (groupsPage?.data ?? []).filter( + (group) => !evaluations.every((evaluation) => evaluation.experiment_ids.includes(group.id)) + ), + [groupsPage, evaluations] + ); + + const groupNameById = useMemo( + () => new Map(groups.map((group) => [group.id, group.name])), + [groups] + ); + + const { mutateAsync: patchEvaluation, isPending: isAdding } = usePatchEvaluation(); + const { mutateAsync: createExperimentGroup, isPending: isCreatingGroup } = + useCreateExperimentGroup(); + + const busy = isAdding || isCreatingGroup; + const count = evaluations.length; + const countLabel = `${count} ${count === 1 ? 'evaluation' : 'evaluations'}`; + + // Add every selected evaluation to `groupId` by PATCHing its membership with the group merged in. + // Membership is a replace, so send the full desired set (existing + new, de-duped); re-adding a group + // the evaluation already belongs to is a no-op. Best-effort: returns how many failed rather than + // throwing on the first error. + const associateEvaluations = async (groupId: string): Promise => { + const results = await Promise.allSettled( + evaluations.map((evaluation) => + patchEvaluation({ + workspace, + name: evaluation.name, + data: { experiment_ids: [...new Set([...evaluation.experiment_ids, groupId])] }, + }) + ) + ); + return results.filter((result) => result.status === 'rejected').length; + }; + + const refreshEvaluationBoards = () => { + queryClient.invalidateQueries({ queryKey: getListEvaluationsQueryKey(workspace) }); + }; + + // Shared finish for both paths once the target group's membership writes have settled. + const finishAdds = (groupName: string, failed: number, createdVerb: string) => { + if (failed === count) { + // Nothing landed — keep the modal open so the user can retry. + toast.error(`Failed to add ${countLabel} to "${groupName}".`); + return; + } + refreshEvaluationBoards(); + if (failed > 0) { + toast.warning( + `${createdVerb} "${groupName}", but ${failed} of ${count} evaluations couldn't be added.` + ); + } else { + toast.success(`${createdVerb} "${groupName}" with ${countLabel}.`); + } + onSuccess?.(); + onClose(); + }; + + const addToExistingGroup = async () => { + const groupName = groupNameById.get(selectedGroupId); + if (!groupName) return; + finishAdds(groupName, await associateEvaluations(selectedGroupId), 'Added to'); + }; + + const createGroupAndAdd: SubmitHandler = async (data) => { + let created; + try { + created = await createExperimentGroup({ + workspace, + data: { + name: data.name, + description: data.description || undefined, + default_sort: defaultSort, + }, + }); + } catch (error) { + // Creation failed (e.g. duplicate name) — surface inline on the name field where possible and + // keep the modal open. No group was created, so there's nothing to add. + const detail = error instanceof AxiosError ? error.response?.data?.detail : undefined; + if (detail === `Experiment group ${data.name} already exists.`) { + setError('name', { message: detail }); + return; + } + const message = + typeof detail === 'string' + ? detail + : error instanceof Error + ? error.message + : 'Unknown error'; + toast.error(`Failed to create experiment group: ${message}`); + return; + } + // Group now exists; adding evaluations is best-effort (a group with fewer evals is still valid). + queryClient.invalidateQueries({ queryKey: getListExperimentGroupsQueryKey(workspace) }); + finishAdds(created.name, await associateEvaluations(created.id), 'Created'); + }; + + const onSubmit = (e: FormEvent) => { + // FormModal renders the
; it doesn't preventDefault. RHF's handleSubmit does it for us on the + // create path; the add path has no RHF wrapper so we preventDefault ourselves. + if (isCreating) { + void handleSubmit(createGroupAndAdd)(e); + } else { + e.preventDefault(); + void addToExistingGroup(); + } + }; + + const submitButtonText = isCreating + ? isCreatingGroup + ? 'Creating…' + : 'Create & add' + : isAdding + ? 'Adding…' + : 'Add'; + + return ( + + + + + { + if (v === CREATE_NEW) return 'Create new group'; + return typeof v === 'string' && v ? (groupNameById.get(v) ?? undefined) : undefined; + }} + /> + + + {groups.map((group) => ( + + {group.name} + + ))} + {/* "Create new" sits at the bottom of the list, matching the workspace dropdown. */} + + + + Create new group + + + + + + + + {isCreating && ( + <> + + + setValue('name', (e.target as HTMLInputElement).value.replace(/[\s-]+/g, '-'), { + shouldValidate: true, + }) + } + /> + + +