diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index 5c62b70686..727439f20d 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -4016,12 +4016,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups: + /apis/intake/v2/workspaces/{workspace}/experiments: post: tags: - - Experiment Groups - summary: Create Experiment Group - operationId: create_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups_post + - Experiments + summary: Create Experiment + operationId: create_experiment_apis_intake_v2_workspaces__workspace__experiments_post parameters: - name: workspace in: path @@ -4034,16 +4034,16 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupRequest' + $ref: '#/components/schemas/ExperimentRequest' responses: '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/ExperimentResponse' '409': - description: Experiment group already exists + description: Experiment already exists '422': description: Validation Error content: @@ -4052,9 +4052,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' get: tags: - - Experiment Groups - summary: List Experiment Groups - operationId: list_experiment_groups_apis_intake_v2_workspaces__workspace__experiment_groups_get + - Experiments + summary: List Experiments + operationId: list_experiments_apis_intake_v2_workspaces__workspace__experiments_get parameters: - name: workspace in: path @@ -4105,29 +4105,29 @@ paths: required: false explode: true schema: - $ref: '#/components/schemas/ExperimentGroupFilter' - description: Filter experiment groups by name, insight_id, is_deleted, or - a metadata key/value (filter[metadata.]=). Pass is_deleted=true - to return only soft-deleted groups; omit to see only live ones. + $ref: '#/components/schemas/ExperimentFilter' + description: Filter experiments by name, insight_id, is_deleted, or a metadata + key/value (filter[metadata.]=). Pass is_deleted=true to return + only soft-deleted experiments; omit to see only live ones. responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponsesPage' + $ref: '#/components/schemas/ExperimentResponsesPage' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: + /apis/intake/v2/workspaces/{workspace}/experiments/{name}: get: tags: - - Experiment Groups - summary: Get Experiment Group - operationId: get_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__get + - Experiments + summary: Get Experiment + operationId: get_experiment_apis_intake_v2_workspaces__workspace__experiments__name__get parameters: - name: workspace in: path @@ -4147,9 +4147,9 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/ExperimentResponse' '404': - description: Experiment group not found + description: Experiment not found '422': description: Validation Error content: @@ -4158,9 +4158,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' put: tags: - - Experiment Groups - summary: Update Experiment Group - operationId: update_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__put + - Experiments + summary: Update Experiment + operationId: update_experiment_apis_intake_v2_workspaces__workspace__experiments__name__put parameters: - name: workspace in: path @@ -4179,18 +4179,18 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupRequest' + $ref: '#/components/schemas/ExperimentRequest' responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/ExperimentResponse' '404': - description: Experiment group not found + description: Experiment not found '409': - description: Attempt to rename the group + description: Attempt to rename the experiment '422': description: Validation Error content: @@ -4199,9 +4199,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' delete: tags: - - Experiment Groups - summary: Delete Experiment Group - operationId: delete_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__delete + - Experiments + summary: Delete Experiment + operationId: delete_experiment_apis_intake_v2_workspaces__workspace__experiments__name__delete parameters: - name: workspace in: path @@ -4219,7 +4219,7 @@ paths: '204': description: Successful Response '404': - description: Experiment group not found + description: Experiment not found '422': description: Validation Error content: @@ -4656,8 +4656,7 @@ paths: schema: $ref: '#/components/schemas/TraceFilter' description: Filter root-span-backed traces by id, session_id, root status, - root span started_at, evaluation_id (or its deprecated alias experiment_id), - and test_case_id. + root span started_at, evaluation_id, and test_case_id. responses: '200': description: Successful Response @@ -8171,12 +8170,6 @@ components: properties: evaluation_context: $ref: '#/components/schemas/EvaluationContext' - experiment_context: - allOf: - - $ref: '#/components/schemas/ExperimentContext' - description: Deprecated. Use evaluation_context; when both are sent, evaluation_context - takes precedence. - deprecated: true schema_version: type: string enum: @@ -8991,12 +8984,6 @@ components: properties: evaluation_context: $ref: '#/components/schemas/EvaluationContext' - experiment_context: - allOf: - - $ref: '#/components/schemas/ExperimentContext' - description: Deprecated. Use evaluation_context; when both are sent, evaluation_context - takes precedence. - deprecated: true request: $ref: '#/components/schemas/CapturedChatCompletionsRequest' response: @@ -10588,8 +10575,8 @@ components: properties: experiment_ids: title: Experiment Ids - description: Replace the ExperimentGroups this Evaluation belongs to. Must - be non-empty when provided; each group must already exist. Omit to leave + description: Replace the Experiments this Evaluation belongs to. Must be + non-empty when provided; each experiment must already exist. Omit to leave membership unchanged. items: type: string @@ -10632,7 +10619,7 @@ components: \ \u2014 same pattern as the models service's PATCH).\nImmutable fields (name,\ \ dataset_name, dataset_version) aren't accepted here. ``experiment_ids``,\n\ when provided, must be non-empty: an evaluation must always belong to at least\ - \ one group." + \ one experiment." EvaluationRequest: properties: name: @@ -10644,13 +10631,13 @@ components: type: string type: array title: Experiment Ids - description: Entity ids of the ExperimentGroups this Evaluation belongs - to (>=1). Preferred; each group must already exist. When omitted, the - deprecated experiment_group_id is used instead. + description: Entity ids of the Experiments this Evaluation belongs to (>=1). + Preferred; each experiment must already exist. When omitted, the deprecated + experiment_group_id is used instead. experiment_group_id: title: Experiment Group Id - description: Deprecated single-group field; provide experiment_ids instead. - Coalesced into experiment_ids when experiment_ids is omitted. + description: Deprecated single-experiment field; provide experiment_ids + instead. Coalesced into experiment_ids when experiment_ids is omitted. deprecated: true type: string dataset_name: @@ -10682,11 +10669,6 @@ components: description: Entity id of the evaluation this one was derived from (e.g. a variant of a baseline), if any. type: string - parent_experiment_id: - title: Parent Experiment Id - description: Deprecated alias for parent_evaluation_id. - deprecated: true - type: string status: title: Status description: Producer-defined lifecycle status of the evaluation. @@ -10719,8 +10701,7 @@ components: type: string type: array title: Experiment Ids - description: Entity ids of the ExperimentGroups this Evaluation belongs - to (>=1). + description: Entity ids of the Experiments this Evaluation belongs to (>=1). dataset_name: type: string title: Dataset Name @@ -10821,16 +10802,10 @@ components: - $ref: '#/components/schemas/EvaluatorAggregate' description: Average total tokens (input + output) per test case, aggregated across the evaluation. - parent_experiment_id: - title: Parent Experiment Id - description: Deprecated alias for parent_evaluation_id. - deprecated: true - readOnly: true - type: string experiment_group_id: type: string title: Experiment Group Id - description: Deprecated single-group alias; the first of experiment_ids. + description: Deprecated single-experiment alias; the first of experiment_ids. Use experiment_ids. deprecated: true readOnly: true @@ -10841,7 +10816,6 @@ components: - workspace - experiment_ids - dataset_name - - parent_experiment_id - experiment_group_id title: EvaluationResponse description: Evaluation as served by the API, including ClickHouse-hydrated @@ -10954,12 +10928,6 @@ components: title: Evaluator Scores description: Per-evaluator session-mean score. Includes NUMERIC and BOOLEAN evaluator results only; text/categorical results are omitted. - experiment_name: - type: string - title: Experiment Name - description: Deprecated alias for evaluation_name. - deprecated: true - readOnly: true type: object required: - workspace @@ -10969,7 +10937,6 @@ components: - root_span_id - started_at - status - - experiment_name title: EvaluationSessionResponse description: "One ingested session of an Evaluation \u2014 a single test case\ \ execution.\n\nHydrated from ClickHouse at read time by reading root/session\ @@ -11233,38 +11200,21 @@ components: - action_name title: ExecutedAction description: Information about an action that was executed. - ExperimentContext: - properties: - experiment_id: - type: string - title: Experiment Id - description: Name of an existing Experiment entity. - test_case_id: - title: Test Case Id - description: Optional producer-supplied test case id. - type: string - additionalProperties: false - type: object - required: - - experiment_id - title: ExperimentContext - description: Deprecated alias for :class:`EvaluationContext`. Producers should - send ``evaluation_context``. - ExperimentGroupFilter: + ExperimentFilter: additionalProperties: false - description: Filter for listing ExperimentGroups. + description: Filter for listing Experiments. properties: name: - description: Filter groups by name. + description: Filter experiments by name. title: Name type: string insight_id: - description: Filter groups by the id of the insight that seeded them. + description: Filter experiments by the id of the insight that seeded them. title: Insight Id type: string is_deleted: - description: When true, returns only soft-deleted groups. Omit (or false) - to see only live groups. + description: When true, returns only soft-deleted experiments. Omit (or + false) to see only live experiments. title: Is Deleted type: boolean metadata: @@ -11273,55 +11223,56 @@ components: additionalProperties: type: string type: object - title: ExperimentGroupFilter + title: ExperimentFilter type: object - ExperimentGroupRequest: + ExperimentRequest: properties: name: type: string title: Name - description: Workspace-unique group name. + description: Workspace-unique experiment name. description: title: Description - description: Human-readable purpose of the group. + description: Human-readable purpose of the experiment. type: string insight_id: title: Insight Id - description: Reference to an external insight that seeded this group, if - any. + description: Reference to an external insight that seeded this experiment, + if any. type: string summary: title: Summary - description: Human- or agent-authored summary of the group's findings. + description: Human- or agent-authored summary of the experiment's findings. type: string metadata: title: Metadata - description: Free-form producer metadata for the group. + description: Free-form producer metadata for the experiment. additionalProperties: type: string type: object default_sort: type: string title: Default Sort - description: 'Default sort for this group''s evaluations list, as a `sort`-param - string: a comma-separated, ordered list of fields where the first is the - primary sort and the rest break ties (leading ''-'' on a field = descending), - e.g. ''-evaluators.reward.mean,cost_usd.mean''. Defaults to ''-created_at''. - Accepts any field the evaluations list `sort` param does; clients apply - it as the list `sort` param.' + description: 'Default sort for this experiment''s evaluations list, as a + `sort`-param string: a comma-separated, ordered list of fields where the + first is the primary sort and the rest break ties (leading ''-'' on a + field = descending), e.g. ''-evaluators.reward.mean,cost_usd.mean''. Defaults + to ''-created_at''. Accepts any field the evaluations list `sort` param + does; clients apply it as the list `sort` param.' default: -created_at pareto: allOf: - $ref: '#/components/schemas/ParetoConfig' - description: Default X/Y metrics for the group's Pareto view. Omit to preserve - the existing value on update; on create, defaults to cost vs. latency. + description: Default X/Y metrics for the experiment's Pareto view. Omit + to preserve the existing value on update; on create, defaults to cost + vs. latency. additionalProperties: false type: object required: - name - title: ExperimentGroupRequest - description: Request body for creating an ExperimentGroup. - ExperimentGroupResponse: + title: ExperimentRequest + description: Request body for creating an Experiment. + ExperimentResponse: properties: id: type: string @@ -11362,28 +11313,21 @@ components: evaluation_count: type: integer title: Evaluation Count - description: Number of live (non-soft-deleted) evaluations in this group. + description: Number of live (non-soft-deleted) evaluations in this experiment. default: 0 - experiment_count: - type: integer - title: Experiment Count - description: Deprecated alias for evaluation_count. - deprecated: true - readOnly: true type: object required: - id - name - workspace - default_sort - - experiment_count - title: ExperimentGroupResponse - description: ExperimentGroup as served by the API. - ExperimentGroupResponsesPage: + title: ExperimentResponse + description: Experiment as served by the API. + ExperimentResponsesPage: properties: data: items: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/ExperimentResponse' type: array title: Data pagination: @@ -11402,7 +11346,7 @@ components: type: object required: - data - title: ExperimentGroupResponsesPage + title: ExperimentResponsesPage FactCheckingRailConfig: properties: parameters: @@ -18572,12 +18516,6 @@ components: type: string evaluation_context: $ref: '#/components/schemas/EvaluationContext' - experiment_context: - allOf: - - $ref: '#/components/schemas/ExperimentContext' - description: Deprecated alias for evaluation_context; will be removed in - a future release. - deprecated: true started_at: type: string format: date-time @@ -18654,12 +18592,6 @@ components: description: Filter by root-span evaluation id. title: Evaluation Id type: string - experiment_id: - deprecated: true - description: Deprecated alias for evaluation_id. Filter by root-span evaluation - id. - title: Experiment Id - type: string test_case_id: description: Filter by root-span evaluation test case id. title: Test Case Id diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index 5c62b70686..727439f20d 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -4016,12 +4016,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups: + /apis/intake/v2/workspaces/{workspace}/experiments: post: tags: - - Experiment Groups - summary: Create Experiment Group - operationId: create_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups_post + - Experiments + summary: Create Experiment + operationId: create_experiment_apis_intake_v2_workspaces__workspace__experiments_post parameters: - name: workspace in: path @@ -4034,16 +4034,16 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupRequest' + $ref: '#/components/schemas/ExperimentRequest' responses: '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/ExperimentResponse' '409': - description: Experiment group already exists + description: Experiment already exists '422': description: Validation Error content: @@ -4052,9 +4052,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' get: tags: - - Experiment Groups - summary: List Experiment Groups - operationId: list_experiment_groups_apis_intake_v2_workspaces__workspace__experiment_groups_get + - Experiments + summary: List Experiments + operationId: list_experiments_apis_intake_v2_workspaces__workspace__experiments_get parameters: - name: workspace in: path @@ -4105,29 +4105,29 @@ paths: required: false explode: true schema: - $ref: '#/components/schemas/ExperimentGroupFilter' - description: Filter experiment groups by name, insight_id, is_deleted, or - a metadata key/value (filter[metadata.]=). Pass is_deleted=true - to return only soft-deleted groups; omit to see only live ones. + $ref: '#/components/schemas/ExperimentFilter' + description: Filter experiments by name, insight_id, is_deleted, or a metadata + key/value (filter[metadata.]=). Pass is_deleted=true to return + only soft-deleted experiments; omit to see only live ones. responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponsesPage' + $ref: '#/components/schemas/ExperimentResponsesPage' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: + /apis/intake/v2/workspaces/{workspace}/experiments/{name}: get: tags: - - Experiment Groups - summary: Get Experiment Group - operationId: get_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__get + - Experiments + summary: Get Experiment + operationId: get_experiment_apis_intake_v2_workspaces__workspace__experiments__name__get parameters: - name: workspace in: path @@ -4147,9 +4147,9 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/ExperimentResponse' '404': - description: Experiment group not found + description: Experiment not found '422': description: Validation Error content: @@ -4158,9 +4158,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' put: tags: - - Experiment Groups - summary: Update Experiment Group - operationId: update_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__put + - Experiments + summary: Update Experiment + operationId: update_experiment_apis_intake_v2_workspaces__workspace__experiments__name__put parameters: - name: workspace in: path @@ -4179,18 +4179,18 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupRequest' + $ref: '#/components/schemas/ExperimentRequest' responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/ExperimentResponse' '404': - description: Experiment group not found + description: Experiment not found '409': - description: Attempt to rename the group + description: Attempt to rename the experiment '422': description: Validation Error content: @@ -4199,9 +4199,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' delete: tags: - - Experiment Groups - summary: Delete Experiment Group - operationId: delete_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__delete + - Experiments + summary: Delete Experiment + operationId: delete_experiment_apis_intake_v2_workspaces__workspace__experiments__name__delete parameters: - name: workspace in: path @@ -4219,7 +4219,7 @@ paths: '204': description: Successful Response '404': - description: Experiment group not found + description: Experiment not found '422': description: Validation Error content: @@ -4656,8 +4656,7 @@ paths: schema: $ref: '#/components/schemas/TraceFilter' description: Filter root-span-backed traces by id, session_id, root status, - root span started_at, evaluation_id (or its deprecated alias experiment_id), - and test_case_id. + root span started_at, evaluation_id, and test_case_id. responses: '200': description: Successful Response @@ -8171,12 +8170,6 @@ components: properties: evaluation_context: $ref: '#/components/schemas/EvaluationContext' - experiment_context: - allOf: - - $ref: '#/components/schemas/ExperimentContext' - description: Deprecated. Use evaluation_context; when both are sent, evaluation_context - takes precedence. - deprecated: true schema_version: type: string enum: @@ -8991,12 +8984,6 @@ components: properties: evaluation_context: $ref: '#/components/schemas/EvaluationContext' - experiment_context: - allOf: - - $ref: '#/components/schemas/ExperimentContext' - description: Deprecated. Use evaluation_context; when both are sent, evaluation_context - takes precedence. - deprecated: true request: $ref: '#/components/schemas/CapturedChatCompletionsRequest' response: @@ -10588,8 +10575,8 @@ components: properties: experiment_ids: title: Experiment Ids - description: Replace the ExperimentGroups this Evaluation belongs to. Must - be non-empty when provided; each group must already exist. Omit to leave + description: Replace the Experiments this Evaluation belongs to. Must be + non-empty when provided; each experiment must already exist. Omit to leave membership unchanged. items: type: string @@ -10632,7 +10619,7 @@ components: \ \u2014 same pattern as the models service's PATCH).\nImmutable fields (name,\ \ dataset_name, dataset_version) aren't accepted here. ``experiment_ids``,\n\ when provided, must be non-empty: an evaluation must always belong to at least\ - \ one group." + \ one experiment." EvaluationRequest: properties: name: @@ -10644,13 +10631,13 @@ components: type: string type: array title: Experiment Ids - description: Entity ids of the ExperimentGroups this Evaluation belongs - to (>=1). Preferred; each group must already exist. When omitted, the - deprecated experiment_group_id is used instead. + description: Entity ids of the Experiments this Evaluation belongs to (>=1). + Preferred; each experiment must already exist. When omitted, the deprecated + experiment_group_id is used instead. experiment_group_id: title: Experiment Group Id - description: Deprecated single-group field; provide experiment_ids instead. - Coalesced into experiment_ids when experiment_ids is omitted. + description: Deprecated single-experiment field; provide experiment_ids + instead. Coalesced into experiment_ids when experiment_ids is omitted. deprecated: true type: string dataset_name: @@ -10682,11 +10669,6 @@ components: description: Entity id of the evaluation this one was derived from (e.g. a variant of a baseline), if any. type: string - parent_experiment_id: - title: Parent Experiment Id - description: Deprecated alias for parent_evaluation_id. - deprecated: true - type: string status: title: Status description: Producer-defined lifecycle status of the evaluation. @@ -10719,8 +10701,7 @@ components: type: string type: array title: Experiment Ids - description: Entity ids of the ExperimentGroups this Evaluation belongs - to (>=1). + description: Entity ids of the Experiments this Evaluation belongs to (>=1). dataset_name: type: string title: Dataset Name @@ -10821,16 +10802,10 @@ components: - $ref: '#/components/schemas/EvaluatorAggregate' description: Average total tokens (input + output) per test case, aggregated across the evaluation. - parent_experiment_id: - title: Parent Experiment Id - description: Deprecated alias for parent_evaluation_id. - deprecated: true - readOnly: true - type: string experiment_group_id: type: string title: Experiment Group Id - description: Deprecated single-group alias; the first of experiment_ids. + description: Deprecated single-experiment alias; the first of experiment_ids. Use experiment_ids. deprecated: true readOnly: true @@ -10841,7 +10816,6 @@ components: - workspace - experiment_ids - dataset_name - - parent_experiment_id - experiment_group_id title: EvaluationResponse description: Evaluation as served by the API, including ClickHouse-hydrated @@ -10954,12 +10928,6 @@ components: title: Evaluator Scores description: Per-evaluator session-mean score. Includes NUMERIC and BOOLEAN evaluator results only; text/categorical results are omitted. - experiment_name: - type: string - title: Experiment Name - description: Deprecated alias for evaluation_name. - deprecated: true - readOnly: true type: object required: - workspace @@ -10969,7 +10937,6 @@ components: - root_span_id - started_at - status - - experiment_name title: EvaluationSessionResponse description: "One ingested session of an Evaluation \u2014 a single test case\ \ execution.\n\nHydrated from ClickHouse at read time by reading root/session\ @@ -11233,38 +11200,21 @@ components: - action_name title: ExecutedAction description: Information about an action that was executed. - ExperimentContext: - properties: - experiment_id: - type: string - title: Experiment Id - description: Name of an existing Experiment entity. - test_case_id: - title: Test Case Id - description: Optional producer-supplied test case id. - type: string - additionalProperties: false - type: object - required: - - experiment_id - title: ExperimentContext - description: Deprecated alias for :class:`EvaluationContext`. Producers should - send ``evaluation_context``. - ExperimentGroupFilter: + ExperimentFilter: additionalProperties: false - description: Filter for listing ExperimentGroups. + description: Filter for listing Experiments. properties: name: - description: Filter groups by name. + description: Filter experiments by name. title: Name type: string insight_id: - description: Filter groups by the id of the insight that seeded them. + description: Filter experiments by the id of the insight that seeded them. title: Insight Id type: string is_deleted: - description: When true, returns only soft-deleted groups. Omit (or false) - to see only live groups. + description: When true, returns only soft-deleted experiments. Omit (or + false) to see only live experiments. title: Is Deleted type: boolean metadata: @@ -11273,55 +11223,56 @@ components: additionalProperties: type: string type: object - title: ExperimentGroupFilter + title: ExperimentFilter type: object - ExperimentGroupRequest: + ExperimentRequest: properties: name: type: string title: Name - description: Workspace-unique group name. + description: Workspace-unique experiment name. description: title: Description - description: Human-readable purpose of the group. + description: Human-readable purpose of the experiment. type: string insight_id: title: Insight Id - description: Reference to an external insight that seeded this group, if - any. + description: Reference to an external insight that seeded this experiment, + if any. type: string summary: title: Summary - description: Human- or agent-authored summary of the group's findings. + description: Human- or agent-authored summary of the experiment's findings. type: string metadata: title: Metadata - description: Free-form producer metadata for the group. + description: Free-form producer metadata for the experiment. additionalProperties: type: string type: object default_sort: type: string title: Default Sort - description: 'Default sort for this group''s evaluations list, as a `sort`-param - string: a comma-separated, ordered list of fields where the first is the - primary sort and the rest break ties (leading ''-'' on a field = descending), - e.g. ''-evaluators.reward.mean,cost_usd.mean''. Defaults to ''-created_at''. - Accepts any field the evaluations list `sort` param does; clients apply - it as the list `sort` param.' + description: 'Default sort for this experiment''s evaluations list, as a + `sort`-param string: a comma-separated, ordered list of fields where the + first is the primary sort and the rest break ties (leading ''-'' on a + field = descending), e.g. ''-evaluators.reward.mean,cost_usd.mean''. Defaults + to ''-created_at''. Accepts any field the evaluations list `sort` param + does; clients apply it as the list `sort` param.' default: -created_at pareto: allOf: - $ref: '#/components/schemas/ParetoConfig' - description: Default X/Y metrics for the group's Pareto view. Omit to preserve - the existing value on update; on create, defaults to cost vs. latency. + description: Default X/Y metrics for the experiment's Pareto view. Omit + to preserve the existing value on update; on create, defaults to cost + vs. latency. additionalProperties: false type: object required: - name - title: ExperimentGroupRequest - description: Request body for creating an ExperimentGroup. - ExperimentGroupResponse: + title: ExperimentRequest + description: Request body for creating an Experiment. + ExperimentResponse: properties: id: type: string @@ -11362,28 +11313,21 @@ components: evaluation_count: type: integer title: Evaluation Count - description: Number of live (non-soft-deleted) evaluations in this group. + description: Number of live (non-soft-deleted) evaluations in this experiment. default: 0 - experiment_count: - type: integer - title: Experiment Count - description: Deprecated alias for evaluation_count. - deprecated: true - readOnly: true type: object required: - id - name - workspace - default_sort - - experiment_count - title: ExperimentGroupResponse - description: ExperimentGroup as served by the API. - ExperimentGroupResponsesPage: + title: ExperimentResponse + description: Experiment as served by the API. + ExperimentResponsesPage: properties: data: items: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/ExperimentResponse' type: array title: Data pagination: @@ -11402,7 +11346,7 @@ components: type: object required: - data - title: ExperimentGroupResponsesPage + title: ExperimentResponsesPage FactCheckingRailConfig: properties: parameters: @@ -18572,12 +18516,6 @@ components: type: string evaluation_context: $ref: '#/components/schemas/EvaluationContext' - experiment_context: - allOf: - - $ref: '#/components/schemas/ExperimentContext' - description: Deprecated alias for evaluation_context; will be removed in - a future release. - deprecated: true started_at: type: string format: date-time @@ -18654,12 +18592,6 @@ components: description: Filter by root-span evaluation id. title: Evaluation Id type: string - experiment_id: - deprecated: true - description: Deprecated alias for evaluation_id. Filter by root-span evaluation - id. - title: Experiment Id - type: string test_case_id: description: Filter by root-span evaluation test case id. title: Test Case Id diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 5c62b70686..727439f20d 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -4016,12 +4016,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups: + /apis/intake/v2/workspaces/{workspace}/experiments: post: tags: - - Experiment Groups - summary: Create Experiment Group - operationId: create_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups_post + - Experiments + summary: Create Experiment + operationId: create_experiment_apis_intake_v2_workspaces__workspace__experiments_post parameters: - name: workspace in: path @@ -4034,16 +4034,16 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupRequest' + $ref: '#/components/schemas/ExperimentRequest' responses: '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/ExperimentResponse' '409': - description: Experiment group already exists + description: Experiment already exists '422': description: Validation Error content: @@ -4052,9 +4052,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' get: tags: - - Experiment Groups - summary: List Experiment Groups - operationId: list_experiment_groups_apis_intake_v2_workspaces__workspace__experiment_groups_get + - Experiments + summary: List Experiments + operationId: list_experiments_apis_intake_v2_workspaces__workspace__experiments_get parameters: - name: workspace in: path @@ -4105,29 +4105,29 @@ paths: required: false explode: true schema: - $ref: '#/components/schemas/ExperimentGroupFilter' - description: Filter experiment groups by name, insight_id, is_deleted, or - a metadata key/value (filter[metadata.]=). Pass is_deleted=true - to return only soft-deleted groups; omit to see only live ones. + $ref: '#/components/schemas/ExperimentFilter' + description: Filter experiments by name, insight_id, is_deleted, or a metadata + key/value (filter[metadata.]=). Pass is_deleted=true to return + only soft-deleted experiments; omit to see only live ones. responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponsesPage' + $ref: '#/components/schemas/ExperimentResponsesPage' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: + /apis/intake/v2/workspaces/{workspace}/experiments/{name}: get: tags: - - Experiment Groups - summary: Get Experiment Group - operationId: get_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__get + - Experiments + summary: Get Experiment + operationId: get_experiment_apis_intake_v2_workspaces__workspace__experiments__name__get parameters: - name: workspace in: path @@ -4147,9 +4147,9 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/ExperimentResponse' '404': - description: Experiment group not found + description: Experiment not found '422': description: Validation Error content: @@ -4158,9 +4158,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' put: tags: - - Experiment Groups - summary: Update Experiment Group - operationId: update_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__put + - Experiments + summary: Update Experiment + operationId: update_experiment_apis_intake_v2_workspaces__workspace__experiments__name__put parameters: - name: workspace in: path @@ -4179,18 +4179,18 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupRequest' + $ref: '#/components/schemas/ExperimentRequest' responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/ExperimentResponse' '404': - description: Experiment group not found + description: Experiment not found '409': - description: Attempt to rename the group + description: Attempt to rename the experiment '422': description: Validation Error content: @@ -4199,9 +4199,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' delete: tags: - - Experiment Groups - summary: Delete Experiment Group - operationId: delete_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__delete + - Experiments + summary: Delete Experiment + operationId: delete_experiment_apis_intake_v2_workspaces__workspace__experiments__name__delete parameters: - name: workspace in: path @@ -4219,7 +4219,7 @@ paths: '204': description: Successful Response '404': - description: Experiment group not found + description: Experiment not found '422': description: Validation Error content: @@ -4656,8 +4656,7 @@ paths: schema: $ref: '#/components/schemas/TraceFilter' description: Filter root-span-backed traces by id, session_id, root status, - root span started_at, evaluation_id (or its deprecated alias experiment_id), - and test_case_id. + root span started_at, evaluation_id, and test_case_id. responses: '200': description: Successful Response @@ -8171,12 +8170,6 @@ components: properties: evaluation_context: $ref: '#/components/schemas/EvaluationContext' - experiment_context: - allOf: - - $ref: '#/components/schemas/ExperimentContext' - description: Deprecated. Use evaluation_context; when both are sent, evaluation_context - takes precedence. - deprecated: true schema_version: type: string enum: @@ -8991,12 +8984,6 @@ components: properties: evaluation_context: $ref: '#/components/schemas/EvaluationContext' - experiment_context: - allOf: - - $ref: '#/components/schemas/ExperimentContext' - description: Deprecated. Use evaluation_context; when both are sent, evaluation_context - takes precedence. - deprecated: true request: $ref: '#/components/schemas/CapturedChatCompletionsRequest' response: @@ -10588,8 +10575,8 @@ components: properties: experiment_ids: title: Experiment Ids - description: Replace the ExperimentGroups this Evaluation belongs to. Must - be non-empty when provided; each group must already exist. Omit to leave + description: Replace the Experiments this Evaluation belongs to. Must be + non-empty when provided; each experiment must already exist. Omit to leave membership unchanged. items: type: string @@ -10632,7 +10619,7 @@ components: \ \u2014 same pattern as the models service's PATCH).\nImmutable fields (name,\ \ dataset_name, dataset_version) aren't accepted here. ``experiment_ids``,\n\ when provided, must be non-empty: an evaluation must always belong to at least\ - \ one group." + \ one experiment." EvaluationRequest: properties: name: @@ -10644,13 +10631,13 @@ components: type: string type: array title: Experiment Ids - description: Entity ids of the ExperimentGroups this Evaluation belongs - to (>=1). Preferred; each group must already exist. When omitted, the - deprecated experiment_group_id is used instead. + description: Entity ids of the Experiments this Evaluation belongs to (>=1). + Preferred; each experiment must already exist. When omitted, the deprecated + experiment_group_id is used instead. experiment_group_id: title: Experiment Group Id - description: Deprecated single-group field; provide experiment_ids instead. - Coalesced into experiment_ids when experiment_ids is omitted. + description: Deprecated single-experiment field; provide experiment_ids + instead. Coalesced into experiment_ids when experiment_ids is omitted. deprecated: true type: string dataset_name: @@ -10682,11 +10669,6 @@ components: description: Entity id of the evaluation this one was derived from (e.g. a variant of a baseline), if any. type: string - parent_experiment_id: - title: Parent Experiment Id - description: Deprecated alias for parent_evaluation_id. - deprecated: true - type: string status: title: Status description: Producer-defined lifecycle status of the evaluation. @@ -10719,8 +10701,7 @@ components: type: string type: array title: Experiment Ids - description: Entity ids of the ExperimentGroups this Evaluation belongs - to (>=1). + description: Entity ids of the Experiments this Evaluation belongs to (>=1). dataset_name: type: string title: Dataset Name @@ -10821,16 +10802,10 @@ components: - $ref: '#/components/schemas/EvaluatorAggregate' description: Average total tokens (input + output) per test case, aggregated across the evaluation. - parent_experiment_id: - title: Parent Experiment Id - description: Deprecated alias for parent_evaluation_id. - deprecated: true - readOnly: true - type: string experiment_group_id: type: string title: Experiment Group Id - description: Deprecated single-group alias; the first of experiment_ids. + description: Deprecated single-experiment alias; the first of experiment_ids. Use experiment_ids. deprecated: true readOnly: true @@ -10841,7 +10816,6 @@ components: - workspace - experiment_ids - dataset_name - - parent_experiment_id - experiment_group_id title: EvaluationResponse description: Evaluation as served by the API, including ClickHouse-hydrated @@ -10954,12 +10928,6 @@ components: title: Evaluator Scores description: Per-evaluator session-mean score. Includes NUMERIC and BOOLEAN evaluator results only; text/categorical results are omitted. - experiment_name: - type: string - title: Experiment Name - description: Deprecated alias for evaluation_name. - deprecated: true - readOnly: true type: object required: - workspace @@ -10969,7 +10937,6 @@ components: - root_span_id - started_at - status - - experiment_name title: EvaluationSessionResponse description: "One ingested session of an Evaluation \u2014 a single test case\ \ execution.\n\nHydrated from ClickHouse at read time by reading root/session\ @@ -11233,38 +11200,21 @@ components: - action_name title: ExecutedAction description: Information about an action that was executed. - ExperimentContext: - properties: - experiment_id: - type: string - title: Experiment Id - description: Name of an existing Experiment entity. - test_case_id: - title: Test Case Id - description: Optional producer-supplied test case id. - type: string - additionalProperties: false - type: object - required: - - experiment_id - title: ExperimentContext - description: Deprecated alias for :class:`EvaluationContext`. Producers should - send ``evaluation_context``. - ExperimentGroupFilter: + ExperimentFilter: additionalProperties: false - description: Filter for listing ExperimentGroups. + description: Filter for listing Experiments. properties: name: - description: Filter groups by name. + description: Filter experiments by name. title: Name type: string insight_id: - description: Filter groups by the id of the insight that seeded them. + description: Filter experiments by the id of the insight that seeded them. title: Insight Id type: string is_deleted: - description: When true, returns only soft-deleted groups. Omit (or false) - to see only live groups. + description: When true, returns only soft-deleted experiments. Omit (or + false) to see only live experiments. title: Is Deleted type: boolean metadata: @@ -11273,55 +11223,56 @@ components: additionalProperties: type: string type: object - title: ExperimentGroupFilter + title: ExperimentFilter type: object - ExperimentGroupRequest: + ExperimentRequest: properties: name: type: string title: Name - description: Workspace-unique group name. + description: Workspace-unique experiment name. description: title: Description - description: Human-readable purpose of the group. + description: Human-readable purpose of the experiment. type: string insight_id: title: Insight Id - description: Reference to an external insight that seeded this group, if - any. + description: Reference to an external insight that seeded this experiment, + if any. type: string summary: title: Summary - description: Human- or agent-authored summary of the group's findings. + description: Human- or agent-authored summary of the experiment's findings. type: string metadata: title: Metadata - description: Free-form producer metadata for the group. + description: Free-form producer metadata for the experiment. additionalProperties: type: string type: object default_sort: type: string title: Default Sort - description: 'Default sort for this group''s evaluations list, as a `sort`-param - string: a comma-separated, ordered list of fields where the first is the - primary sort and the rest break ties (leading ''-'' on a field = descending), - e.g. ''-evaluators.reward.mean,cost_usd.mean''. Defaults to ''-created_at''. - Accepts any field the evaluations list `sort` param does; clients apply - it as the list `sort` param.' + description: 'Default sort for this experiment''s evaluations list, as a + `sort`-param string: a comma-separated, ordered list of fields where the + first is the primary sort and the rest break ties (leading ''-'' on a + field = descending), e.g. ''-evaluators.reward.mean,cost_usd.mean''. Defaults + to ''-created_at''. Accepts any field the evaluations list `sort` param + does; clients apply it as the list `sort` param.' default: -created_at pareto: allOf: - $ref: '#/components/schemas/ParetoConfig' - description: Default X/Y metrics for the group's Pareto view. Omit to preserve - the existing value on update; on create, defaults to cost vs. latency. + description: Default X/Y metrics for the experiment's Pareto view. Omit + to preserve the existing value on update; on create, defaults to cost + vs. latency. additionalProperties: false type: object required: - name - title: ExperimentGroupRequest - description: Request body for creating an ExperimentGroup. - ExperimentGroupResponse: + title: ExperimentRequest + description: Request body for creating an Experiment. + ExperimentResponse: properties: id: type: string @@ -11362,28 +11313,21 @@ components: evaluation_count: type: integer title: Evaluation Count - description: Number of live (non-soft-deleted) evaluations in this group. + description: Number of live (non-soft-deleted) evaluations in this experiment. default: 0 - experiment_count: - type: integer - title: Experiment Count - description: Deprecated alias for evaluation_count. - deprecated: true - readOnly: true type: object required: - id - name - workspace - default_sort - - experiment_count - title: ExperimentGroupResponse - description: ExperimentGroup as served by the API. - ExperimentGroupResponsesPage: + title: ExperimentResponse + description: Experiment as served by the API. + ExperimentResponsesPage: properties: data: items: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/ExperimentResponse' type: array title: Data pagination: @@ -11402,7 +11346,7 @@ components: type: object required: - data - title: ExperimentGroupResponsesPage + title: ExperimentResponsesPage FactCheckingRailConfig: properties: parameters: @@ -18572,12 +18516,6 @@ components: type: string evaluation_context: $ref: '#/components/schemas/EvaluationContext' - experiment_context: - allOf: - - $ref: '#/components/schemas/ExperimentContext' - description: Deprecated alias for evaluation_context; will be removed in - a future release. - deprecated: true started_at: type: string format: date-time @@ -18654,12 +18592,6 @@ components: description: Filter by root-span evaluation id. title: Evaluation Id type: string - experiment_id: - deprecated: true - description: Deprecated alias for evaluation_id. Filter by root-span evaluation - id. - title: Experiment Id - type: string test_case_id: description: Filter by root-span evaluation test case id. title: Test Case Id diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/atif.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/atif.py index 6a96cabe12..6eb1106246 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/atif.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/atif.py @@ -39,13 +39,6 @@ def create_atif( help='Evaluation context accepted by ingest endpoints (the canonical shape).`extra="ignore"` so a producer still sending retired keys (evaluation_sha, evaluation_run_id, metadata) keeps ingesting without error rather than being rejected. (JSON string)', ), ] = None, - experiment_context: Annotated[ - str | None, - typer.Option( - "--experiment-context", - help="Deprecated alias for :class:`EvaluationContext`. Producers should send `evaluation_context`. (JSON string)", - ), - ] = None, extra: Annotated[str | None, typer.Option("--extra", help="JSON string")] = None, final_metrics: Annotated[str | None, typer.Option("--final-metrics", help="JSON string")] = None, notes: Annotated[str | None, typer.Option("--notes")] = None, @@ -90,8 +83,6 @@ def create_atif( input_payload["continued_trajectory_ref"] = continued_trajectory_ref if evaluation_context is not None: input_payload["evaluation_context"] = read_payload("evaluation_context", evaluation_context) - if experiment_context is not None: - input_payload["experiment_context"] = read_payload("experiment_context", experiment_context) if extra is not None: input_payload["extra"] = read_payload("extra", extra) if final_metrics is not None: diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/chat_completions.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/chat_completions.py index c6e03a0fef..95129af9da 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/chat_completions.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/chat_completions.py @@ -57,13 +57,6 @@ def create_chat_completions( help='Evaluation context accepted by ingest endpoints (the canonical shape).`extra="ignore"` so a producer still sending retired keys (evaluation_sha, evaluation_run_id, metadata) keeps ingesting without error rather than being rejected. (JSON string)', ), ] = None, - experiment_context: Annotated[ - str | None, - typer.Option( - "--experiment-context", - help="Deprecated alias for :class:`EvaluationContext`. Producers should send `evaluation_context`. (JSON string)", - ), - ] = None, provider: Annotated[str | None, typer.Option("--provider")] = None, session_id: Annotated[ str | None, @@ -121,8 +114,6 @@ def create_chat_completions( input_payload["cost_usd"] = cost_usd if evaluation_context is not None: input_payload["evaluation_context"] = read_payload("evaluation_context", evaluation_context) - if experiment_context is not None: - input_payload["experiment_context"] = read_payload("experiment_context", experiment_context) if provider is not None: input_payload["provider"] = provider if session_id is not None: diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/traces.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/traces.py index d5f3900eb2..0cdf7b8f81 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/traces.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/traces.py @@ -36,7 +36,7 @@ def list_traces( typer.Option( "--filter", metavar="FILTER_JSON", - help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter root-span-backed traces by id, session_id, root status, root span started_at, evaluation_id (or its deprecated alias experiment_id), and test_case_id.", + help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter root-span-backed traces by id, session_id, root status, root span started_at, evaluation_id, and test_case_id.", rich_help_panel="Filter Options", ), ] = None, @@ -44,9 +44,6 @@ def list_traces( filter_evaluation_id: Annotated[ str | None, typer.Option("--filter.evaluation-id", rich_help_panel="Filter Options") ] = None, - filter_experiment_id: Annotated[ - str | None, typer.Option("--filter.experiment-id", rich_help_panel="Filter Options") - ] = None, filter_session_id: Annotated[ str | None, typer.Option("--filter.session-id", rich_help_panel="Filter Options") ] = None, @@ -89,7 +86,6 @@ def list_traces( filter, id=filter_id, evaluation_id=filter_evaluation_id, - experiment_id=filter_experiment_id, session_id=filter_session_id, status=filter_status, test_case_id=filter_test_case_id, diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/SKILL.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/SKILL.md index c7cd1f31f4..4b4faa537d 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/SKILL.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/SKILL.md @@ -1,6 +1,6 @@ --- name: nemo-experiments-upload -description: End-to-end guide for getting evaluation data into NeMo Platform Intake so it shows up as Experiments. Create an Experiment Group, create an Evaluation, then log traces and evaluator results via the ATIF (Harbor), chat-completions, or OTLP ingest endpoint — and view the rollups in Studio. Use when a user wants to upload, log, ingest, publish, or send evaluation runs, agent traces, or scores to NeMo Experiments / Intake. +description: End-to-end guide for getting evaluation data into NeMo Platform Intake so it shows up as Experiments. Create an Experiment, create an Evaluation, then log traces and evaluator results via the ATIF (Harbor), chat-completions, or OTLP ingest endpoint — and view the rollups in Studio. Use when a user wants to upload, log, ingest, publish, or send evaluation runs, agent traces, or scores to NeMo Experiments / Intake. triggers: - log traces to intake - upload experiment results @@ -24,7 +24,7 @@ allowed-tools: [Bash, Read, Write] # Log evaluation data to NeMo Intake -Get evaluation runs into the platform end-to-end: **create an Experiment Group → create an Evaluation → log traces + scores to an ingest endpoint → see the rollups.** The API says "Evaluation" and "Experiment Group"; the whole feature is called **Experiments**. +Get evaluation runs into the platform end-to-end: **create an Experiment → create an Evaluation → log traces + scores to an ingest endpoint → see the rollups.** The API calls the parent (the leaderboard) an **Experiment** and each row an **Evaluation**; the whole feature is called **Experiments**. Everything below uses `${NMP_BASE_URL}` (default `http://localhost:8080`) and a `${WORKSPACE}` (default `default`). All routes are under `/apis/intake/v2/workspaces/${WORKSPACE}`. @@ -48,33 +48,33 @@ fi Run the steps in order. Steps 1–2 create the entities; step 3 logs the data; steps 4–5 verify. -### 1. Create an Experiment Group +### 1. Create an Experiment -A group is the leaderboard container. You need its `id` for the next step. +An Experiment is the leaderboard container. You need its `id` for the next step. ```bash set -euo pipefail : "${NMP_BASE_URL:=http://localhost:8080}" : "${WORKSPACE:=default}" -groups="${NMP_BASE_URL}/apis/intake/v2/workspaces/${WORKSPACE}/experiment-groups" -# Create the group. 201 = created, 409 = already exists; any other status is a real failure. -code=$(curl -s -o /dev/null -w '%{http_code}' -X POST "${groups}" \ +experiments="${NMP_BASE_URL}/apis/intake/v2/workspaces/${WORKSPACE}/experiments" +# Create the experiment. 201 = created, 409 = already exists; any other status is a real failure. +code=$(curl -s -o /dev/null -w '%{http_code}' -X POST "${experiments}" \ -H 'Content-Type: application/json' \ - -d '{"name": "my-experiment-group", "description": "example run"}') + -d '{"name": "my-experiment", "description": "example run"}') case "${code}" in 201|409) ;; - *) echo "group create failed: HTTP ${code}" >&2; exit 1 ;; + *) echo "experiment create failed: HTTP ${code}" >&2; exit 1 ;; esac # Fetch the id (works whether it was just created or already existed). -GROUP_ID=$(curl -sf "${groups}/my-experiment-group" \ +EXPERIMENT_ID=$(curl -sf "${experiments}/my-experiment" \ | python3 -c 'import sys,json; print(json.load(sys.stdin)["id"])') -[ -n "${GROUP_ID}" ] || { echo "could not resolve experiment group id" >&2; exit 1; } -echo "group id: ${GROUP_ID}" +[ -n "${EXPERIMENT_ID}" ] || { echo "could not resolve experiment id" >&2; exit 1; } +echo "experiment id: ${EXPERIMENT_ID}" ``` The POST accepts only `201` (created) or `409` (already exists) — any other status stops the step instead of masking it. The `id` is then read with a GET, so this works on both first run and re-run; -`set -euo pipefail` + the `[ -n ]` guard keep it from continuing with an empty `GROUP_ID`. +`set -euo pipefail` + the `[ -n ]` guard keep it from continuing with an empty `EXPERIMENT_ID`. ### 2. Create an Evaluation @@ -85,13 +85,13 @@ curl -sf -X POST \ "${NMP_BASE_URL}/apis/intake/v2/workspaces/${WORKSPACE}/evaluations" \ -H 'Content-Type: application/json' \ -d "{\"name\": \"my-eval-baseline\", - \"experiment_group_id\": \"${GROUP_ID}\", + \"experiment_ids\": [\"${EXPERIMENT_ID}\"], \"dataset_name\": \"my-dataset\", \"dataset_version\": \"v1\", \"metadata\": {\"model\": \"provider/model\", \"job_name\": \"baseline\"}}" ``` -- `experiment_group_id` is the group's **`id`** (from step 1). +- `experiment_ids` is a list holding the Experiment's **`id`** (from step 1). - `metadata` values must be **strings** (`dict[str, str]`). - **You must create the Evaluation before you can log to it** — ingesting with an unknown `evaluation_id` returns `400 "…must be created before it can be logged."` @@ -128,7 +128,7 @@ curl -sf "${NMP_BASE_URL}/apis/intake/v2/workspaces/${WORKSPACE}/evaluations/my- ### 5. View in Studio -Open Studio → the **Experiments** area (behind the `VITE_FF_EXPERIMENT` flag) → your group → your evaluation. You'll see the leaderboard row with score/cost/latency rollups and can drill into individual sessions and traces. +Open Studio → the **Experiments** area (behind the `VITE_FF_EXPERIMENT` flag) → your experiment → your evaluation. You'll see the leaderboard row with score/cost/latency rollups and can drill into individual sessions and traces. ## Reference files @@ -159,8 +159,8 @@ If `run_count` is 0 after ingesting, the traces didn't associate — almost alwa ## Gotchas - **Create before you log.** The Evaluation entity must exist before any ingest referencing it — otherwise `400`. -- **`evaluation_id` is the Evaluation's `name`, not its entity id.** But **`experiment_group_id` is the group's `id`.** Different identifiers; easy to swap. +- **`evaluation_id` is the Evaluation's `name`, not its entity id.** But **`experiment_ids` holds the Experiment's `id`.** Different identifiers; easy to swap. - **OTLP uses the attribute key `nemo.experiment.id`** (and `nemo.test_case.id`) — the span-attribute key still says "experiment" even though the JSON body field is `evaluation_context`. Set `nemo.experiment.id` on your root span. -- **Use `/evaluations`, not `/experiments`.** `/experiments` still works as a deprecated hidden alias but you should log to `/evaluations`. +- **The parent lives at `/experiments`; `/experiment-groups` is a deprecated hidden alias.** Prefer `/experiments`. Evaluations are created and logged under `/evaluations`. - **`metadata` is `dict[str, str]`** — stringify non-string values or you'll get a `422`. - **ATIF and chat-completions are `extra="forbid"`** (unknown keys → 422); `evaluation_context` itself is lenient (`extra="ignore"`). diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/harbor-quickstart.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/harbor-quickstart.md index a3c853f2c7..fbc4a70939 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/harbor-quickstart.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/harbor-quickstart.md @@ -8,7 +8,7 @@ each task runs several times (trials). You upload **one ATIF payload per trial** | Harbor concept | NeMo entity | How | |---|---|---| -| A benchmark / sweep | **Experiment Group** | `POST /experiment-groups` once | +| A benchmark / sweep | **Experiment** | `POST /experiments` once | | One agent+config on that benchmark | **Evaluation** | `POST /evaluations` once (its `name` is your `evaluation_id`) | | A task / test case | `test_case_id` | field inside `evaluation_context` | | One trial (attempt) of a task | one ingested **session** | one `POST /ingest/atif` | diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/troubleshooting.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/troubleshooting.md index 5e9e7acf62..736edc1a66 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/troubleshooting.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/troubleshooting.md @@ -3,14 +3,14 @@ Common failures across the create + ingest endpoints, and the fix. Most 4xx bodies include a `detail` string — read it first. -## Create Experiment Group / Evaluation +## Create Experiment / Evaluation | Status | Meaning | Fix | |---|---|---| -| `409` on create group/evaluation | Name already exists in this workspace | Reuse it — `GET .../experiment-groups/{name}` or `.../evaluations/{name}` | -| `400 "…group … does not exist"` on create evaluation | `experiment_group_id` is wrong or the group was deleted | Use the group's **`id`** from the create-group response (not its name) | +| `409` on create experiment/evaluation | Name already exists in this workspace | Reuse it — `GET .../experiments/{name}` or `.../evaluations/{name}` | +| `400 "…experiment … does not exist"` on create evaluation | `experiment_ids` is wrong or the experiment was deleted | Use the Experiment's **`id`** from the create-experiment response (not its name) | | `400 "…parent … does not exist"` | `parent_evaluation_id` doesn't resolve | Omit it, or pass the parent Evaluation's entity **`id`** | -| `422` on create evaluation | Missing required field or non-string metadata value | Required: `name`, `experiment_group_id`, `dataset_name`. `metadata` must be `dict[str, str]` | +| `422` on create evaluation | Missing required field or non-string metadata value | Required: `name`, `experiment_ids`, `dataset_name`. `metadata` must be `dict[str, str]` | ## Ingest (all endpoints) @@ -38,6 +38,6 @@ string — read it first. ## Identifier cheat-sheet (the #1 source of bugs) - `evaluation_context.evaluation_id` → the Evaluation's **`name`**. -- `experiment_group_id` (on create evaluation) → the group's **`id`**. +- `experiment_ids` (on create evaluation) → a list with the Experiment's **`id`**. - OTLP evaluation attribute key → **`nemo.experiment.id`** (test case → `nemo.test_case.id`). -- Log to **`/evaluations`** (the `/experiments` path is a deprecated hidden alias). +- Parent → **`/experiments`** (`/experiment-groups` is a deprecated hidden alias); evaluations → **`/evaluations`**. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/tests.json b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/tests.json index 0130d6cbe8..9b807227bd 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/tests.json +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/tests.json @@ -8,7 +8,7 @@ { "type": "implicit", "prompt": "How do I get my evaluation runs and scores into NeMo?", "expected_skill": "nemo-experiments-upload" }, { "type": "implicit", "prompt": "Send my ATIF trajectory to intake with evaluation context", "expected_skill": "nemo-experiments-upload" }, { "type": "contextual", "prompt": "I have a Harbor run with verifier rewards and I need them to show up as experiment scores", "expected_skill": "nemo-experiments-upload" }, - { "type": "contextual", "prompt": "Create an experiment group and an evaluation, then publish traces to it", "expected_skill": "nemo-experiments-upload" }, + { "type": "contextual", "prompt": "Create an experiment and an evaluation, then publish traces to it", "expected_skill": "nemo-experiments-upload" }, { "type": "negative-control", "prompt": "Help me author an LLM-as-a-judge metric for my benchmark", "expected_skill_not": "nemo-experiments-upload" }, { "type": "negative-control", "prompt": "Is the NeMo platform healthy right now?", "expected_skill_not": "nemo-experiments-upload" }, { "type": "negative-control", "prompt": "What is the status of PR #612 in the platform repo?", "expected_skill_not": "nemo-experiments-upload" } diff --git a/plugins/nemo-customizer/openapi/openapi.yaml b/plugins/nemo-customizer/openapi/openapi.yaml index 2b7724e0aa..d852b91bc8 100644 --- a/plugins/nemo-customizer/openapi/openapi.yaml +++ b/plugins/nemo-customizer/openapi/openapi.yaml @@ -392,12 +392,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs: + /apis/customization/v2/workspaces/{workspace}/rl/jobs: post: tags: - - Unsloth Jobs + - Rl Jobs summary: Create Job - operationId: create_job_apis_customization_v2_workspaces__workspace__unsloth_jobs_post + operationId: create_job_apis_customization_v2_workspaces__workspace__rl_jobs_post parameters: - name: workspace in: path @@ -410,14 +410,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/UnslothJobsJobRequest' + $ref: '#/components/schemas/RlJobsJobRequest' responses: '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/UnslothJobsJob' + $ref: '#/components/schemas/RlJobsJob' '422': description: Validation Error content: @@ -426,9 +426,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' get: tags: - - Unsloth Jobs + - Rl Jobs summary: List Jobs - operationId: list_jobs_apis_customization_v2_workspaces__workspace__unsloth_jobs_get + operationId: list_jobs_apis_customization_v2_workspaces__workspace__rl_jobs_get parameters: - name: workspace in: path @@ -461,7 +461,7 @@ paths: required: false schema: allOf: - - $ref: '#/components/schemas/UnslothJobsJobsSortField' + - $ref: '#/components/schemas/RlJobsJobsSortField' description: The field to sort by. To sort in decreasing order, use `-` in front of the field name. default: -created_at @@ -473,7 +473,7 @@ paths: required: false explode: true schema: - $ref: '#/components/schemas/UnslothJobsJobsListFilter' + $ref: '#/components/schemas/RlJobsJobsListFilter' description: Filter jobs on various criteria. responses: '200': @@ -481,19 +481,19 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/UnslothJobsJobsPage' + $ref: '#/components/schemas/RlJobsJobsPage' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{job}/results/{name}: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{job}/results/{name}: get: tags: - - Unsloth Jobs + - Rl Jobs summary: Get Job Result - operationId: get_job_result_apis_customization_v2_workspaces__workspace__unsloth_jobs__job__results__name__get + operationId: get_job_result_apis_customization_v2_workspaces__workspace__rl_jobs__job__results__name__get parameters: - name: workspace in: path @@ -526,12 +526,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{job}/results/{name}/download: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{job}/results/{name}/download: get: tags: - - Unsloth Jobs + - Rl Jobs summary: Download Job Result - operationId: download_job_result_apis_customization_v2_workspaces__workspace__unsloth_jobs__job__results__name__download_get + operationId: download_job_result_apis_customization_v2_workspaces__workspace__rl_jobs__job__results__name__download_get parameters: - name: workspace in: path @@ -567,12 +567,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{name}: get: tags: - - Unsloth Jobs + - Rl Jobs summary: Get Job - operationId: get_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__get + operationId: get_job_apis_customization_v2_workspaces__workspace__rl_jobs__name__get parameters: - name: workspace in: path @@ -592,7 +592,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/UnslothJobsJob' + $ref: '#/components/schemas/RlJobsJob' '422': description: Validation Error content: @@ -601,9 +601,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' delete: tags: - - Unsloth Jobs + - Rl Jobs summary: Delete Job - operationId: delete_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__delete + operationId: delete_job_apis_customization_v2_workspaces__workspace__rl_jobs__name__delete parameters: - name: workspace in: path @@ -626,12 +626,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/cancel: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{name}/cancel: post: tags: - - Unsloth Jobs + - Rl Jobs summary: Cancel Job - operationId: cancel_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__cancel_post + operationId: cancel_job_apis_customization_v2_workspaces__workspace__rl_jobs__name__cancel_post parameters: - name: workspace in: path @@ -651,19 +651,19 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/UnslothJobsJob' + $ref: '#/components/schemas/RlJobsJob' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/logs: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{name}/logs: get: tags: - - Unsloth Jobs + - Rl Jobs summary: Get Job Logs - operationId: get_job_logs_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__logs_get + operationId: get_job_logs_apis_customization_v2_workspaces__workspace__rl_jobs__name__logs_get parameters: - name: workspace in: path @@ -702,12 +702,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/results: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{name}/results: get: tags: - - Unsloth Jobs + - Rl Jobs summary: List Job Results - operationId: list_job_results_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__results_get + operationId: list_job_results_apis_customization_v2_workspaces__workspace__rl_jobs__name__results_get parameters: - name: workspace in: path @@ -734,12 +734,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/status: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{name}/status: get: tags: - - Unsloth Jobs + - Rl Jobs summary: Get Job Status - operationId: get_job_status_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__status_get + operationId: get_job_status_apis_customization_v2_workspaces__workspace__rl_jobs__name__status_get parameters: - name: workspace in: path @@ -766,89 +766,463 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' -components: - schemas: - AutomodelBatchSpec: - properties: - global_batch_size: - type: integer - exclusiveMinimum: 0.0 - title: Global Batch Size - default: 8 - micro_batch_size: + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs: + post: + tags: + - Unsloth Jobs + summary: Create Job + operationId: create_job_apis_customization_v2_workspaces__workspace__unsloth_jobs_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/UnslothJobsJobRequest' + responses: + '201': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/UnslothJobsJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + get: + tags: + - Unsloth Jobs + summary: List Jobs + operationId: list_jobs_apis_customization_v2_workspaces__workspace__unsloth_jobs_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: type: integer - exclusiveMinimum: 0.0 - title: Micro Batch Size + exclusiveMinimum: 0 + description: Page number. default: 1 - sequence_packing: - type: boolean - title: Sequence Packing - default: false - sequence_packing_max_samples: + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: type: integer - exclusiveMinimum: 0.0 - title: Sequence Packing Max Samples - description: Samples analyzed to estimate the optimal pack size when packing - is enabled. - default: 1000 - additionalProperties: false - type: object - title: AutomodelBatchSpec - AutomodelDatasetSpec: - properties: - training: + exclusiveMinimum: 0 + description: Page size. + default: 10 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + allOf: + - $ref: '#/components/schemas/UnslothJobsJobsSortField' + description: The field to sort by. To sort in decreasing order, use `-` + in front of the field name. + default: -created_at + description: The field to sort by. To sort in decreasing order, use `-` in + front of the field name. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/UnslothJobsJobsListFilter' + description: Filter jobs on various criteria. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/UnslothJobsJobsPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{job}/results/{name}: + get: + tags: + - Unsloth Jobs + summary: Get Job Result + operationId: get_job_result_apis_customization_v2_workspaces__workspace__unsloth_jobs__job__results__name__get + parameters: + - name: workspace + in: path + required: true + schema: type: string - title: Training - description: Training fileset as 'name' or 'workspace/name'. - validation: - title: Validation + title: Workspace + - name: job + in: path + required: true + schema: type: string - prompt_template: - title: Prompt Template + title: Job + - name: name + in: path + required: true + schema: type: string - additionalProperties: false - type: object - required: - - training - title: AutomodelDatasetSpec - AutomodelJobInput: - properties: - name: title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobResultResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{job}/results/{name}/download: + get: + tags: + - Unsloth Jobs + summary: Download Job Result + operationId: download_job_result_apis_customization_v2_workspaces__workspace__unsloth_jobs__job__results__name__download_get + parameters: + - name: workspace + in: path + required: true + schema: type: string - model: + title: Workspace + - name: job + in: path + required: true + schema: type: string - title: Model - dataset: - $ref: '#/components/schemas/AutomodelDatasetSpec' - training: - $ref: '#/components/schemas/AutomodelTrainingSpec' - schedule: - $ref: '#/components/schemas/AutomodelScheduleSpec' - batch: - $ref: '#/components/schemas/AutomodelBatchSpec' - optimizer: - $ref: '#/components/schemas/AutomodelOptimizerSpec' - parallelism: - $ref: '#/components/schemas/AutomodelParallelismSpec' - output: - $ref: '#/components/schemas/AutomodelOutputRequest' - integrations: - $ref: '#/components/schemas/IntegrationsSpecInput' - additionalProperties: false - type: object - required: - - model - - dataset - - training - title: AutomodelJobInput - description: POST body / CLI JSON. - AutomodelJobOutput: - properties: - name: - title: Name + title: Job + - name: name + in: path + required: true + schema: type: string - model: + title: Name + responses: + '200': + description: Successful Response + content: + application/octet-stream: + schema: + type: string + format: binary + '404': + description: Not Found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}: + get: + tags: + - Unsloth Jobs + summary: Get Job + operationId: get_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/UnslothJobsJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Unsloth Jobs + summary: Delete Job + operationId: delete_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '204': + description: Successful Response + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/cancel: + post: + tags: + - Unsloth Jobs + summary: Cancel Job + operationId: cancel_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__cancel_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/UnslothJobsJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/logs: + get: + tags: + - Unsloth Jobs + summary: Get Job Logs + operationId: get_job_logs_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__logs_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + - name: limit + in: query + required: false + schema: + title: Limit + type: integer + - name: page_cursor + in: query + required: false + schema: + title: Page Cursor + type: string + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobLogPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/results: + get: + tags: + - Unsloth Jobs + summary: List Job Results + operationId: list_job_results_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__results_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobListResultResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/status: + get: + tags: + - Unsloth Jobs + summary: Get Job Status + operationId: get_job_status_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__status_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobStatusResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' +components: + schemas: + AutomodelBatchSpec: + properties: + global_batch_size: + type: integer + exclusiveMinimum: 0.0 + title: Global Batch Size + default: 8 + micro_batch_size: + type: integer + exclusiveMinimum: 0.0 + title: Micro Batch Size + default: 1 + sequence_packing: + type: boolean + title: Sequence Packing + default: false + sequence_packing_max_samples: + type: integer + exclusiveMinimum: 0.0 + title: Sequence Packing Max Samples + description: Samples analyzed to estimate the optimal pack size when packing + is enabled. + default: 1000 + additionalProperties: false + type: object + title: AutomodelBatchSpec + AutomodelDatasetSpec: + properties: + training: + type: string + title: Training + description: Training fileset as 'name' or 'workspace/name'. + validation: + title: Validation + type: string + prompt_template: + title: Prompt Template + type: string + additionalProperties: false + type: object + required: + - training + title: AutomodelDatasetSpec + AutomodelJobInput: + properties: + name: + title: Name + type: string + model: + type: string + title: Model + dataset: + $ref: '#/components/schemas/AutomodelDatasetSpec' + training: + $ref: '#/components/schemas/AutomodelTrainingSpec' + schedule: + $ref: '#/components/schemas/AutomodelScheduleSpec' + batch: + $ref: '#/components/schemas/AutomodelBatchSpec' + optimizer: + $ref: '#/components/schemas/AutomodelOptimizerSpec' + parallelism: + $ref: '#/components/schemas/AutomodelParallelismSpec' + output: + $ref: '#/components/schemas/AutomodelOutputRequest' + integrations: + $ref: '#/components/schemas/IntegrationsSpecInput' + additionalProperties: false + type: object + required: + - model + - dataset + - training + title: AutomodelJobInput + description: POST body / CLI JSON. + AutomodelJobOutput: + properties: + name: + title: Name + type: string + model: type: string title: Model dataset: @@ -1401,6 +1775,23 @@ components: To enable MLflow, provide a non-null ``mlflow`` object on :class:`IntegrationsSpec`.' + OptimizerType: + type: string + enum: + - adamw_with_cosine_annealing + - adam_with_cosine_annealing + - adamw_with_flat_lr + - adam_with_flat_lr + title: OptimizerType + description: Optimizer and scheduler combination types. + OutputNameType: + type: string + enum: + - adapter + - model + title: OutputNameType + description: "Output artifact type \u2014 adapter (LoRA only) or model (merged\ + \ / full)." PaginationData: properties: page: @@ -1425,254 +1816,701 @@ components: description: The total number of results. type: object required: - - page - - page_size - - current_page_size - - total_pages - - total_results - title: PaginationData - PlatformJobListResultResponse: + - page + - page_size + - current_page_size + - total_pages + - total_results + title: PaginationData + PlatformJobListResultResponse: + properties: + data: + items: + $ref: '#/components/schemas/PlatformJobResultResponse' + type: array + title: Data + type: object + required: + - data + title: PlatformJobListResultResponse + PlatformJobLog: + properties: + timestamp: + type: string + format: date-time + title: Timestamp + job: + type: string + title: Job + job_step: + type: string + title: Job Step + job_task: + type: string + title: Job Task + message: + type: string + title: Message + type: object + required: + - timestamp + - job + - job_step + - job_task + - message + title: PlatformJobLog + PlatformJobLogPage: + properties: + data: + items: + $ref: '#/components/schemas/PlatformJobLog' + type: array + title: Data + total: + type: integer + title: Total + next_page: + title: Next Page + type: string + prev_page: + title: Prev Page + type: string + type: object + required: + - data + - total + - next_page + - prev_page + title: PlatformJobLogPage + PlatformJobResultResponse: + properties: + name: + type: string + title: Name + job: + type: string + title: Job + workspace: + type: string + title: Workspace + project: + title: Project + type: string + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + artifact_url: + type: string + title: Artifact Url + artifact_storage_type: + $ref: '#/components/schemas/FileStorageType' + download_url: + title: Download Url + type: string + type: object + required: + - name + - job + - workspace + - artifact_url + - artifact_storage_type + title: PlatformJobResultResponse + PlatformJobStatus: + type: string + enum: + - created + - pending + - active + - cancelled + - cancelling + - error + - completed + - paused + - pausing + - resuming + title: PlatformJobStatus + description: 'Enumeration of possible job statuses. + + + This enum represents the various states a job can be in during its lifecycle, + + from creation to a terminal state.' + PlatformJobStatusResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + steps: + items: + $ref: '#/components/schemas/PlatformJobStepStatusResponse' + type: array + title: Steps + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + type: object + required: + - id + - name + - status + - status_details + - error_details + - steps + - created_at + - updated_at + title: PlatformJobStatusResponse + PlatformJobStepStatusResponse: properties: - data: + id: + type: string + title: Id + name: + type: string + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + tasks: items: - $ref: '#/components/schemas/PlatformJobResultResponse' + $ref: '#/components/schemas/PlatformJobTaskStatusResponse' type: array - title: Data + title: Tasks + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At type: object required: - - data - title: PlatformJobListResultResponse - PlatformJobLog: + - id + - name + - status + - status_details + - error_details + - tasks + - created_at + - updated_at + title: PlatformJobStepStatusResponse + PlatformJobTaskStatusResponse: properties: - timestamp: + id: type: string - format: date-time - title: Timestamp - job: + title: Id + name: type: string - title: Job - job_step: + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + error_stack: + title: Error Stack type: string - title: Job Step - job_task: + created_at: type: string - title: Job Task - message: + format: date-time + title: Created At + updated_at: type: string - title: Message + format: date-time + title: Updated At type: object required: - - timestamp - - job - - job_step - - job_task - - message - title: PlatformJobLog - PlatformJobLogPage: + - id + - name + - status + - status_details + - error_details + - error_stack + - created_at + - updated_at + title: PlatformJobTaskStatusResponse + RlDPOTraining: properties: - data: - items: - $ref: '#/components/schemas/PlatformJobLog' - type: array - title: Data - total: + optimizer_type: + allOf: + - $ref: '#/components/schemas/OptimizerType' + description: "Optimizer + LR-scheduler combination (AdamW/Adam \xD7 cosine-annealing/flat-LR).\ + \ Defaults to AdamW with cosine annealing." + learning_rate: + type: number + title: Learning Rate + description: Peak learning rate. + default: 0.0001 + min_learning_rate: + title: Min Learning Rate + description: Minimum LR for cosine decay. + type: number + weight_decay: + type: number + title: Weight Decay + description: Weight decay coefficient. + default: 0.01 + adam_beta1: + type: number + title: Adam Beta1 + description: Adam beta1. + default: 0.9 + adam_beta2: + type: number + title: Adam Beta2 + description: Adam beta2. + default: 0.999 + adam_eps: + type: number + exclusiveMinimum: 0.0 + title: Adam Eps + description: Adam epsilon (numerical stability term). + default: 1.0e-05 + warmup_steps: type: integer - title: Total - next_page: - title: Next Page + minimum: 0.0 + title: Warmup Steps + description: Linear warmup steps. + default: 0 + epochs: + type: integer + exclusiveMinimum: 0.0 + title: Epochs + description: Number of passes through the dataset. + default: 1 + max_steps: + title: Max Steps + description: Max training steps (overrides epochs if set). + type: integer + exclusiveMinimum: 0.0 + val_check_interval: + title: Val Check Interval + description: Validation interval. Float <= 1.0 is fraction of epoch; > 1.0 + is step count. + type: number + val_at_end: + type: boolean + title: Val At End + description: Run a final validation pass after the last training step. Keep + enabled so the final checkpoint carries validation metrics and best-checkpoint + selection works; set False only to skip the extra eval. + default: true + keep_top_k: + type: integer + exclusiveMinimum: 0.0 + title: Keep Top K + description: Number of best checkpoints to retain (ranked by validation + loss). + default: 1 + batch_size: + type: integer + exclusiveMinimum: 0.0 + title: Batch Size + description: Global batch size across all GPUs. + default: 32 + micro_batch_size: + type: integer + exclusiveMinimum: 0.0 + title: Micro Batch Size + description: Per-GPU micro batch size. + default: 1 + activation_checkpointing: + type: boolean + title: Activation Checkpointing + description: Recompute activations during the backward pass to reduce memory + at the cost of compute. Enable to fit larger models or longer sequences. + default: false + max_seq_length: + type: integer + exclusiveMinimum: 0.0 + title: Max Seq Length + description: Maximum token sequence length for training. + default: 2048 + seed: + title: Seed + description: Random seed for reproducibility. + type: integer + parallelism: + $ref: '#/components/schemas/RlParallelismParams' + execution_profile: + title: Execution Profile + description: Execution profile for the GPU training step (operator-configured). + Falls back to the service default when omitted. type: string - prev_page: - title: Prev Page + minLength: 1 + type: type: string + const: dpo + title: Type + default: dpo + ref_policy_kl_penalty: + type: number + minimum: 0.0 + title: Ref Policy Kl Penalty + description: KL penalty coefficient (beta in the DPO paper). + default: 0.05 + preference_average_log_probs: + type: boolean + title: Preference Average Log Probs + description: Average log probabilities for preference loss calculation. + default: false + sft_average_log_probs: + type: boolean + title: Sft Average Log Probs + description: Average log probabilities for SFT regularization loss. + default: false + preference_loss_weight: + type: number + minimum: 0.0 + title: Preference Loss Weight + description: Weight for the preference (DPO) loss term. + default: 1.0 + sft_loss_weight: + type: number + minimum: 0.0 + title: Sft Loss Weight + description: Weight for SFT regularization loss (0 = disabled). + default: 0.0 + max_grad_norm: + type: number + minimum: 0.0 + title: Max Grad Norm + description: Maximum gradient norm for clipping. + default: 1.0 + additionalProperties: false type: object - required: - - data - - total - - next_page - - prev_page - title: PlatformJobLogPage - PlatformJobResultResponse: + title: RlDPOTraining + description: "Direct Preference Optimization (full-weight only \u2014 PEFT unsupported)." + RlJobInput: properties: name: - type: string title: Name - job: - type: string - title: Job - workspace: type: string - title: Workspace - project: - title: Project + model: type: string - created_at: + title: Model + description: Model entity reference ('name' or 'workspace/name'). + dataset: type: string - format: date-time - title: Created At - updated_at: + title: Dataset + description: Preference dataset fileset reference. Must contain training.jsonl + + validation.jsonl. + training: + allOf: + - $ref: '#/components/schemas/RlDPOTraining' + description: DPO training method and hyperparameters. + integrations: + $ref: '#/components/schemas/IntegrationsSpecInput' + output: + $ref: '#/components/schemas/RlOutputRequest' + additionalProperties: false + type: object + required: + - model + - dataset + - training + title: RlJobInput + description: POST body / CLI JSON for ``nemo customization rl submit``. + RlJobOutput: + properties: + name: + title: Name + description: Optional job name; auto-generated when omitted. type: string - format: date-time - title: Updated At - artifact_url: + model: type: string - title: Artifact Url - artifact_storage_type: - $ref: '#/components/schemas/FileStorageType' - download_url: - title: Download Url + title: Model + description: Model entity reference ('name' or 'workspace/name'). + dataset: type: string + title: Dataset + description: Preference dataset fileset reference ('name' or 'workspace/name'). + training: + allOf: + - $ref: '#/components/schemas/RlDPOTraining' + description: Training method and hyperparameters (DPO). + integrations: + allOf: + - $ref: '#/components/schemas/IntegrationsSpecOutput' + description: W&B / MLflow integrations. + output: + allOf: + - $ref: '#/components/schemas/RlOutputResponse' + description: Output artifact created by this job. + additionalProperties: false type: object required: - - name - - job - - workspace - - artifact_url - - artifact_storage_type - title: PlatformJobResultResponse - PlatformJobStatus: - type: string - enum: - - created - - pending - - active - - cancelled - - cancelling - - error - - completed - - paused - - pausing - - resuming - title: PlatformJobStatus - description: 'Enumeration of possible job statuses. + - model + - dataset + - training + - output + title: RlJobOutput + description: 'Canonical NeMo-RL job spec (output of the plugin transform). - This enum represents the various states a job can be in during its lifecycle, + The ``dataset`` fileset must contain ``training.jsonl`` and ``validation.jsonl`` - from creation to a terminal state.' - PlatformJobStatusResponse: + (any of the four supported preference formats); the dataset-preparation step + + splits/normalizes them at runtime.' + RlJobsJob: properties: id: - type: string title: Id + type: string name: type: string title: Name + description: + title: Description + type: string + project: + title: Project + type: string + workspace: + title: Workspace + type: string + created_at: + title: Created At + type: string + format: date-time + updated_at: + title: Updated At + type: string + format: date-time + spec: + $ref: '#/components/schemas/RlJobOutput' status: $ref: '#/components/schemas/PlatformJobStatus' status_details: + title: Status Details additionalProperties: true type: object - title: Status Details error_details: title: Error Details additionalProperties: true type: object - steps: - items: - $ref: '#/components/schemas/PlatformJobStepStatusResponse' - type: array - title: Steps - created_at: - type: string - format: date-time - title: Created At - updated_at: - type: string - format: date-time - title: Updated At + ownership: + title: Ownership + additionalProperties: true + type: object + custom_fields: + title: Custom Fields + additionalProperties: true + type: object type: object required: - - id - name - - status - - status_details - - error_details - - steps - - created_at - - updated_at - title: PlatformJobStatusResponse - PlatformJobStepStatusResponse: + - spec + title: RlJobsJob + RlJobsJobRequest: properties: - id: - type: string - title: Id name: - type: string title: Name - status: - $ref: '#/components/schemas/PlatformJobStatus' - status_details: + type: string + description: + title: Description + type: string + project: + title: Project + type: string + spec: + $ref: '#/components/schemas/RlJobInput' + ownership: + title: Ownership additionalProperties: true type: object - title: Status Details - error_details: - title: Error Details + custom_fields: + title: Custom Fields additionalProperties: true type: object - tasks: - items: - $ref: '#/components/schemas/PlatformJobTaskStatusResponse' - type: array - title: Tasks + type: object + required: + - spec + title: RlJobsJobRequest + RlJobsJobsListFilter: + additionalProperties: false + properties: created_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Jobs created at 'gte' datetime or 'lte' datetime. + name: + anyOf: + - $ref: '#/components/schemas/StringFilter' + - type: string + description: Name of the job. + title: Name + workspace: + description: Workspace of the job. + title: Workspace type: string - format: date-time - title: Created At + project: + description: Project containing the job. + title: Project + type: string + status: + allOf: + - $ref: '#/components/schemas/PlatformJobStatus' + description: The current status. updated_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Jobs updated at 'gte' datetime or 'lte' datetime. + title: RlJobsJobsListFilter + type: object + RlJobsJobsPage: + properties: + data: + items: + $ref: '#/components/schemas/RlJobsJob' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. type: string - format: date-time - title: Updated At + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object type: object required: - - id - - name - - status - - status_details - - error_details - - tasks + - data + title: RlJobsJobsPage + RlJobsJobsSortField: + type: string + enum: - created_at + - -created_at - updated_at - title: PlatformJobStepStatusResponse - PlatformJobTaskStatusResponse: + - -updated_at + title: RlJobsJobsSortField + RlOutputRequest: properties: - id: - type: string - title: Id name: - type: string title: Name - status: - $ref: '#/components/schemas/PlatformJobStatus' - status_details: - additionalProperties: true - type: object - title: Status Details - error_details: - title: Error Details - additionalProperties: true - type: object - error_stack: - title: Error Stack type: string - created_at: + additionalProperties: false + type: object + title: RlOutputRequest + description: Submitter-facing output preferences. ``name`` is auto-derived if + omitted. + RlOutputResponse: + properties: + name: type: string - format: date-time - title: Created At - updated_at: + maxLength: 255 + title: Name + description: Name of the output artifact. Used to identify it during deployment + and inference. + examples: + - my-dpo-llama + type: + allOf: + - $ref: '#/components/schemas/OutputNameType' + description: Output artifact type. DPO is full-weight, so always `model`. + default: model + fileset: type: string - format: date-time - title: Updated At + maxLength: 255 + title: Fileset + description: FileSet name where output artifacts are stored. + additionalProperties: false type: object required: - - id - name - - status - - status_details - - error_details - - error_stack - - created_at - - updated_at - title: PlatformJobTaskStatusResponse + - fileset + title: RlOutputResponse + description: Resolved output artifact details. + RlParallelismParams: + properties: + num_gpus_per_node: + type: integer + exclusiveMinimum: 0.0 + title: Num Gpus Per Node + description: Number of GPUs per node. + default: 1 + num_nodes: + type: integer + exclusiveMinimum: 0.0 + title: Num Nodes + description: "Number of nodes (>1 \u2192 multi-node Ray cluster)." + default: 1 + tensor_parallel_size: + type: integer + exclusiveMinimum: 0.0 + title: Tensor Parallel Size + description: Tensor parallel size. + default: 1 + pipeline_parallel_size: + type: integer + exclusiveMinimum: 0.0 + title: Pipeline Parallel Size + description: Pipeline parallel size. + default: 1 + context_parallel_size: + type: integer + exclusiveMinimum: 0.0 + title: Context Parallel Size + description: Context parallel size. + default: 1 + sequence_parallel: + type: boolean + title: Sequence Parallel + description: Enable sequence parallelism. + default: false + additionalProperties: false + type: object + title: RlParallelismParams + description: 'Distributed training parallelism configuration. + + + Single-node multi-GPU uses ``num_nodes=1`` with ``num_gpus_per_node>1``; + + multi-node sets ``num_nodes>1`` and the compiler emits a distributed-GPU + + executor (see :mod:`nmp.rl.app.jobs.compiler`).' SecretRef: type: string pattern: ^[a-z0-9_-]+(/[a-z0-9_-]+)?$ diff --git a/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py b/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py index 913015a650..f673ab16f2 100644 --- a/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py +++ b/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py @@ -175,13 +175,13 @@ def _result() -> AgentEvalResult: async def test_publish_to_intake_round_trip(platform_base_url: str) -> None: async with AsyncNeMoPlatform(base_url=platform_base_url, max_retries=2) as client: # Precondition: the Experiment must exist before ingest. - group = await client.experiment_groups.create( + group = await client.experiments.create( workspace=WORKSPACE, name=GROUP_NAME, description="Intake IT", exist_ok=True ) await client.evaluations.create( workspace=WORKSPACE, name=EXPERIMENT_NAME, - experiment_group_id=group.id, + experiment_ids=[group.id], dataset_name="intake-it-dataset", dataset_version="v1", exist_ok=True, @@ -274,11 +274,11 @@ async def test_publish_skips_nan_and_failed_scores(platform_base_url: str) -> No # A NaN value is not representable in JSON and a FAILED score is not a real measurement; neither # should reach Intake. Only the finite, completed output should be stored. async with AsyncNeMoPlatform(base_url=platform_base_url, max_retries=2) as client: - group = await client.experiment_groups.create(workspace=WORKSPACE, name=GROUP_NAME, exist_ok=True) + group = await client.experiments.create(workspace=WORKSPACE, name=GROUP_NAME, exist_ok=True) await client.evaluations.create( workspace=WORKSPACE, name=NAN_EXPERIMENT_NAME, - experiment_group_id=group.id, + experiment_ids=[group.id], dataset_name="intake-it-nan-dataset", dataset_version="v1", exist_ok=True, diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experiment_mirror.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experiment_mirror.py index 6e39a6d0ce..75dccfe136 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experiment_mirror.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experiment_mirror.py @@ -4,8 +4,8 @@ """One-way, best-effort projection of the plugin's run/candidate entities onto the platform's native ``ExperimentGroup``/``Experiment``. -This module is the *only* place that talks to ``client.experiment_groups`` / -``client.experiments``; the optimization loop never imports it. Mapping is +This module is the *only* place that talks to ``client.experiments`` / +``client.evaluations``; the optimization loop never imports it. Mapping is ``ExperimentRun → ExperimentGroup`` (1:1) and ``Candidate → Experiment[]`` (one per evaluated split). It mirrors **structure only** (identity/lineage/status/description); eval results (reward/trials) are NOT copied into ``Experiment.metadata`` — those arrive @@ -120,7 +120,7 @@ def __init__(self, client: AsyncNeMoPlatform, workspace: str) -> None: async def ensure_group(self, run: ExperimentRun) -> None: gname = group_name(run.id) try: - grp = await self._client.experiment_groups.create( + grp = await self._client.experiments.create( workspace=self._workspace, name=gname, insight_id=run.insight or omit, @@ -128,14 +128,14 @@ async def ensure_group(self, run: ExperimentRun) -> None: metadata=group_metadata(run), ) except ConflictError: - grp = await self._client.experiment_groups.retrieve(gname, workspace=self._workspace) + grp = await self._client.experiments.retrieve(gname, workspace=self._workspace) self._group_ids[run.id] = grp.id async def update_group(self, run: ExperimentRun) -> None: gname = group_name(run.id) - # experiment_groups.update is a full-replace PUT: omitted fields reset to None + # experiments.update is a full-replace PUT: omitted fields reset to None # server-side, so re-supply the whole body (we have the run). - await self._client.experiment_groups.update( + await self._client.experiments.update( gname, workspace=self._workspace, body_name=gname, @@ -147,7 +147,7 @@ async def update_group(self, run: ExperimentRun) -> None: async def _group_id_for(self, run_id: str) -> str: gid = self._group_ids.get(run_id) if gid is None: # resume path: group exists, look it up by deterministic name - grp = await self._client.experiment_groups.retrieve(group_name(run_id), workspace=self._workspace) + grp = await self._client.experiments.retrieve(group_name(run_id), workspace=self._workspace) gid = self._group_ids[run_id] = grp.id return gid @@ -212,7 +212,7 @@ async def _upsert_experiment( dataset_name=self._dataset_name(split), dataset_version="v1", workspace=self._workspace, - experiment_group_id=group_id, + experiment_ids=[group_id], source_link=link, description=candidate.optimization, parent_evaluation_id=parent_id, @@ -227,7 +227,7 @@ async def _upsert_experiment( dataset_name=self._dataset_name(split), dataset_version="v1", body_name=name, - experiment_group_id=group_id, + experiment_ids=[group_id], source_link=link, description=candidate.optimization, parent_evaluation_id=parent_id, @@ -284,12 +284,12 @@ async def _parent_experiment_id(self, candidate: Candidate, gname: str) -> str | async def finalize(self, *, run_id: str, summary: str, winner: Candidate | None, pr_url: str | None = None) -> None: gname = group_name(run_id) gid = await self._group_id_for(run_id) - # experiment_groups.update is a full-replace PUT: omitted fields reset to None + # experiments.update is a full-replace PUT: omitted fields reset to None # server-side. finalize has no `run`, so read-preserve-write the current # insight_id/metadata before writing the summary (stays within the mirror; no # data flows back into the loop). - grp = await self._client.experiment_groups.retrieve(gname, workspace=self._workspace) - await self._client.experiment_groups.update( + grp = await self._client.experiments.retrieve(gname, workspace=self._workspace) + await self._client.experiments.update( gname, workspace=self._workspace, body_name=gname, diff --git a/plugins/nemo-experimentalist/tests/test_experiment_mirror.py b/plugins/nemo-experimentalist/tests/test_experiment_mirror.py index cd4df4bd5c..f3af6f1fa5 100644 --- a/plugins/nemo-experimentalist/tests/test_experiment_mirror.py +++ b/plugins/nemo-experimentalist/tests/test_experiment_mirror.py @@ -21,11 +21,11 @@ def _client(groups: object, experiments: object) -> AsyncNeMoPlatform: - return cast(AsyncNeMoPlatform, SimpleNamespace(experiment_groups=groups, evaluations=experiments)) + return cast(AsyncNeMoPlatform, SimpleNamespace(experiments=groups, evaluations=experiments)) class _StatefulGroups: - """Models experiment_groups as a full-replace PUT store: create/update set the + """Models experiments as a full-replace PUT store: create/update set the stored body to exactly the fields passed, treating ``omit`` (or an absent kwarg) as "field became None". This is what catches accidental field-wiping on update.""" @@ -209,7 +209,7 @@ async def test_finalize_round0_winner_preserves_existing_source_link(): async def test_group_update_and_finalize_do_not_wipe_insight_or_metadata(): - # experiment_groups.update is a full-replace PUT; ensure_group → update_group → + # experiments.update is a full-replace PUT; ensure_group → update_group → # finalize must all preserve insight_id + metadata rather than reset them to None. groups = _StatefulGroups() mirror = ExperimentMirror(_client(groups, AsyncMock()), workspace="default") diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index 5c62b70686..727439f20d 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -4016,12 +4016,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups: + /apis/intake/v2/workspaces/{workspace}/experiments: post: tags: - - Experiment Groups - summary: Create Experiment Group - operationId: create_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups_post + - Experiments + summary: Create Experiment + operationId: create_experiment_apis_intake_v2_workspaces__workspace__experiments_post parameters: - name: workspace in: path @@ -4034,16 +4034,16 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupRequest' + $ref: '#/components/schemas/ExperimentRequest' responses: '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/ExperimentResponse' '409': - description: Experiment group already exists + description: Experiment already exists '422': description: Validation Error content: @@ -4052,9 +4052,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' get: tags: - - Experiment Groups - summary: List Experiment Groups - operationId: list_experiment_groups_apis_intake_v2_workspaces__workspace__experiment_groups_get + - Experiments + summary: List Experiments + operationId: list_experiments_apis_intake_v2_workspaces__workspace__experiments_get parameters: - name: workspace in: path @@ -4105,29 +4105,29 @@ paths: required: false explode: true schema: - $ref: '#/components/schemas/ExperimentGroupFilter' - description: Filter experiment groups by name, insight_id, is_deleted, or - a metadata key/value (filter[metadata.]=). Pass is_deleted=true - to return only soft-deleted groups; omit to see only live ones. + $ref: '#/components/schemas/ExperimentFilter' + description: Filter experiments by name, insight_id, is_deleted, or a metadata + key/value (filter[metadata.]=). Pass is_deleted=true to return + only soft-deleted experiments; omit to see only live ones. responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponsesPage' + $ref: '#/components/schemas/ExperimentResponsesPage' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: + /apis/intake/v2/workspaces/{workspace}/experiments/{name}: get: tags: - - Experiment Groups - summary: Get Experiment Group - operationId: get_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__get + - Experiments + summary: Get Experiment + operationId: get_experiment_apis_intake_v2_workspaces__workspace__experiments__name__get parameters: - name: workspace in: path @@ -4147,9 +4147,9 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/ExperimentResponse' '404': - description: Experiment group not found + description: Experiment not found '422': description: Validation Error content: @@ -4158,9 +4158,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' put: tags: - - Experiment Groups - summary: Update Experiment Group - operationId: update_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__put + - Experiments + summary: Update Experiment + operationId: update_experiment_apis_intake_v2_workspaces__workspace__experiments__name__put parameters: - name: workspace in: path @@ -4179,18 +4179,18 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupRequest' + $ref: '#/components/schemas/ExperimentRequest' responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/ExperimentResponse' '404': - description: Experiment group not found + description: Experiment not found '409': - description: Attempt to rename the group + description: Attempt to rename the experiment '422': description: Validation Error content: @@ -4199,9 +4199,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' delete: tags: - - Experiment Groups - summary: Delete Experiment Group - operationId: delete_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__delete + - Experiments + summary: Delete Experiment + operationId: delete_experiment_apis_intake_v2_workspaces__workspace__experiments__name__delete parameters: - name: workspace in: path @@ -4219,7 +4219,7 @@ paths: '204': description: Successful Response '404': - description: Experiment group not found + description: Experiment not found '422': description: Validation Error content: @@ -4656,8 +4656,7 @@ paths: schema: $ref: '#/components/schemas/TraceFilter' description: Filter root-span-backed traces by id, session_id, root status, - root span started_at, evaluation_id (or its deprecated alias experiment_id), - and test_case_id. + root span started_at, evaluation_id, and test_case_id. responses: '200': description: Successful Response @@ -8171,12 +8170,6 @@ components: properties: evaluation_context: $ref: '#/components/schemas/EvaluationContext' - experiment_context: - allOf: - - $ref: '#/components/schemas/ExperimentContext' - description: Deprecated. Use evaluation_context; when both are sent, evaluation_context - takes precedence. - deprecated: true schema_version: type: string enum: @@ -8991,12 +8984,6 @@ components: properties: evaluation_context: $ref: '#/components/schemas/EvaluationContext' - experiment_context: - allOf: - - $ref: '#/components/schemas/ExperimentContext' - description: Deprecated. Use evaluation_context; when both are sent, evaluation_context - takes precedence. - deprecated: true request: $ref: '#/components/schemas/CapturedChatCompletionsRequest' response: @@ -10588,8 +10575,8 @@ components: properties: experiment_ids: title: Experiment Ids - description: Replace the ExperimentGroups this Evaluation belongs to. Must - be non-empty when provided; each group must already exist. Omit to leave + description: Replace the Experiments this Evaluation belongs to. Must be + non-empty when provided; each experiment must already exist. Omit to leave membership unchanged. items: type: string @@ -10632,7 +10619,7 @@ components: \ \u2014 same pattern as the models service's PATCH).\nImmutable fields (name,\ \ dataset_name, dataset_version) aren't accepted here. ``experiment_ids``,\n\ when provided, must be non-empty: an evaluation must always belong to at least\ - \ one group." + \ one experiment." EvaluationRequest: properties: name: @@ -10644,13 +10631,13 @@ components: type: string type: array title: Experiment Ids - description: Entity ids of the ExperimentGroups this Evaluation belongs - to (>=1). Preferred; each group must already exist. When omitted, the - deprecated experiment_group_id is used instead. + description: Entity ids of the Experiments this Evaluation belongs to (>=1). + Preferred; each experiment must already exist. When omitted, the deprecated + experiment_group_id is used instead. experiment_group_id: title: Experiment Group Id - description: Deprecated single-group field; provide experiment_ids instead. - Coalesced into experiment_ids when experiment_ids is omitted. + description: Deprecated single-experiment field; provide experiment_ids + instead. Coalesced into experiment_ids when experiment_ids is omitted. deprecated: true type: string dataset_name: @@ -10682,11 +10669,6 @@ components: description: Entity id of the evaluation this one was derived from (e.g. a variant of a baseline), if any. type: string - parent_experiment_id: - title: Parent Experiment Id - description: Deprecated alias for parent_evaluation_id. - deprecated: true - type: string status: title: Status description: Producer-defined lifecycle status of the evaluation. @@ -10719,8 +10701,7 @@ components: type: string type: array title: Experiment Ids - description: Entity ids of the ExperimentGroups this Evaluation belongs - to (>=1). + description: Entity ids of the Experiments this Evaluation belongs to (>=1). dataset_name: type: string title: Dataset Name @@ -10821,16 +10802,10 @@ components: - $ref: '#/components/schemas/EvaluatorAggregate' description: Average total tokens (input + output) per test case, aggregated across the evaluation. - parent_experiment_id: - title: Parent Experiment Id - description: Deprecated alias for parent_evaluation_id. - deprecated: true - readOnly: true - type: string experiment_group_id: type: string title: Experiment Group Id - description: Deprecated single-group alias; the first of experiment_ids. + description: Deprecated single-experiment alias; the first of experiment_ids. Use experiment_ids. deprecated: true readOnly: true @@ -10841,7 +10816,6 @@ components: - workspace - experiment_ids - dataset_name - - parent_experiment_id - experiment_group_id title: EvaluationResponse description: Evaluation as served by the API, including ClickHouse-hydrated @@ -10954,12 +10928,6 @@ components: title: Evaluator Scores description: Per-evaluator session-mean score. Includes NUMERIC and BOOLEAN evaluator results only; text/categorical results are omitted. - experiment_name: - type: string - title: Experiment Name - description: Deprecated alias for evaluation_name. - deprecated: true - readOnly: true type: object required: - workspace @@ -10969,7 +10937,6 @@ components: - root_span_id - started_at - status - - experiment_name title: EvaluationSessionResponse description: "One ingested session of an Evaluation \u2014 a single test case\ \ execution.\n\nHydrated from ClickHouse at read time by reading root/session\ @@ -11233,38 +11200,21 @@ components: - action_name title: ExecutedAction description: Information about an action that was executed. - ExperimentContext: - properties: - experiment_id: - type: string - title: Experiment Id - description: Name of an existing Experiment entity. - test_case_id: - title: Test Case Id - description: Optional producer-supplied test case id. - type: string - additionalProperties: false - type: object - required: - - experiment_id - title: ExperimentContext - description: Deprecated alias for :class:`EvaluationContext`. Producers should - send ``evaluation_context``. - ExperimentGroupFilter: + ExperimentFilter: additionalProperties: false - description: Filter for listing ExperimentGroups. + description: Filter for listing Experiments. properties: name: - description: Filter groups by name. + description: Filter experiments by name. title: Name type: string insight_id: - description: Filter groups by the id of the insight that seeded them. + description: Filter experiments by the id of the insight that seeded them. title: Insight Id type: string is_deleted: - description: When true, returns only soft-deleted groups. Omit (or false) - to see only live groups. + description: When true, returns only soft-deleted experiments. Omit (or + false) to see only live experiments. title: Is Deleted type: boolean metadata: @@ -11273,55 +11223,56 @@ components: additionalProperties: type: string type: object - title: ExperimentGroupFilter + title: ExperimentFilter type: object - ExperimentGroupRequest: + ExperimentRequest: properties: name: type: string title: Name - description: Workspace-unique group name. + description: Workspace-unique experiment name. description: title: Description - description: Human-readable purpose of the group. + description: Human-readable purpose of the experiment. type: string insight_id: title: Insight Id - description: Reference to an external insight that seeded this group, if - any. + description: Reference to an external insight that seeded this experiment, + if any. type: string summary: title: Summary - description: Human- or agent-authored summary of the group's findings. + description: Human- or agent-authored summary of the experiment's findings. type: string metadata: title: Metadata - description: Free-form producer metadata for the group. + description: Free-form producer metadata for the experiment. additionalProperties: type: string type: object default_sort: type: string title: Default Sort - description: 'Default sort for this group''s evaluations list, as a `sort`-param - string: a comma-separated, ordered list of fields where the first is the - primary sort and the rest break ties (leading ''-'' on a field = descending), - e.g. ''-evaluators.reward.mean,cost_usd.mean''. Defaults to ''-created_at''. - Accepts any field the evaluations list `sort` param does; clients apply - it as the list `sort` param.' + description: 'Default sort for this experiment''s evaluations list, as a + `sort`-param string: a comma-separated, ordered list of fields where the + first is the primary sort and the rest break ties (leading ''-'' on a + field = descending), e.g. ''-evaluators.reward.mean,cost_usd.mean''. Defaults + to ''-created_at''. Accepts any field the evaluations list `sort` param + does; clients apply it as the list `sort` param.' default: -created_at pareto: allOf: - $ref: '#/components/schemas/ParetoConfig' - description: Default X/Y metrics for the group's Pareto view. Omit to preserve - the existing value on update; on create, defaults to cost vs. latency. + description: Default X/Y metrics for the experiment's Pareto view. Omit + to preserve the existing value on update; on create, defaults to cost + vs. latency. additionalProperties: false type: object required: - name - title: ExperimentGroupRequest - description: Request body for creating an ExperimentGroup. - ExperimentGroupResponse: + title: ExperimentRequest + description: Request body for creating an Experiment. + ExperimentResponse: properties: id: type: string @@ -11362,28 +11313,21 @@ components: evaluation_count: type: integer title: Evaluation Count - description: Number of live (non-soft-deleted) evaluations in this group. + description: Number of live (non-soft-deleted) evaluations in this experiment. default: 0 - experiment_count: - type: integer - title: Experiment Count - description: Deprecated alias for evaluation_count. - deprecated: true - readOnly: true type: object required: - id - name - workspace - default_sort - - experiment_count - title: ExperimentGroupResponse - description: ExperimentGroup as served by the API. - ExperimentGroupResponsesPage: + title: ExperimentResponse + description: Experiment as served by the API. + ExperimentResponsesPage: properties: data: items: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/ExperimentResponse' type: array title: Data pagination: @@ -11402,7 +11346,7 @@ components: type: object required: - data - title: ExperimentGroupResponsesPage + title: ExperimentResponsesPage FactCheckingRailConfig: properties: parameters: @@ -18572,12 +18516,6 @@ components: type: string evaluation_context: $ref: '#/components/schemas/EvaluationContext' - experiment_context: - allOf: - - $ref: '#/components/schemas/ExperimentContext' - description: Deprecated alias for evaluation_context; will be removed in - a future release. - deprecated: true started_at: type: string format: date-time @@ -18654,12 +18592,6 @@ components: description: Filter by root-span evaluation id. title: Evaluation Id type: string - experiment_id: - deprecated: true - description: Deprecated alias for evaluation_id. Filter by root-span evaluation - id. - title: Experiment Id - type: string test_case_id: description: Filter by root-span evaluation test case id. title: Test Case Id diff --git a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml index 438911b5ef..02a0898d9d 100644 --- a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml @@ -832,7 +832,6 @@ resources: ingest: models: evaluation_context: EvaluationContext - experiment_context: ExperimentContext subresources: atif: models: @@ -912,20 +911,6 @@ resources: methods: list: get /apis/intake/v2/workspaces/{workspace}/traces retrieve: get /apis/intake/v2/workspaces/{workspace}/traces/{id} - experiment_groups: - standalone_api: true - models: - experiment_group_filter: ExperimentGroupFilter - experiment_group_request: ExperimentGroupRequest - experiment_group_response: ExperimentGroupResponse - experiment_group_responses_page: ExperimentGroupResponsesPage - pareto_config: ParetoConfig - methods: - create: post /apis/intake/v2/workspaces/{workspace}/experiment-groups - list: get /apis/intake/v2/workspaces/{workspace}/experiment-groups - retrieve: get /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} - update: put /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} - delete: delete /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} evaluations: standalone_api: true models: @@ -954,3 +939,17 @@ resources: evaluation_session_responses_page: EvaluationSessionResponsesPage methods: list: get /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/sessions + experiments: + standalone_api: true + models: + experiment_filter: ExperimentFilter + experiment_request: ExperimentRequest + experiment_response: ExperimentResponse + experiment_responses_page: ExperimentResponsesPage + pareto_config: ParetoConfig + methods: + create: post /apis/intake/v2/workspaces/{workspace}/experiments + list: get /apis/intake/v2/workspaces/{workspace}/experiments + retrieve: get /apis/intake/v2/workspaces/{workspace}/experiments/{name} + update: put /apis/intake/v2/workspaces/{workspace}/experiments/{name} + delete: delete /apis/intake/v2/workspaces/{workspace}/experiments/{name} diff --git a/sdk/python/nemo-platform/api.md b/sdk/python/nemo-platform/api.md index b235448956..b291b7143b 100644 --- a/sdk/python/nemo-platform/api.md +++ b/sdk/python/nemo-platform/api.md @@ -72,6 +72,6 @@ from nemo_platform.types import ( # [Intake](src/nemo_platform/resources/intake/api.md) -# [ExperimentGroups](src/nemo_platform/resources/experiment_groups/api.md) - # [Evaluations](src/nemo_platform/resources/evaluations/api.md) + +# [Experiments](src/nemo_platform/resources/experiments/api.md) diff --git a/sdk/python/nemo-platform/pyproject.toml b/sdk/python/nemo-platform/pyproject.toml index a1b726ade3..2b9e51278e 100644 --- a/sdk/python/nemo-platform/pyproject.toml +++ b/sdk/python/nemo-platform/pyproject.toml @@ -26,11 +26,10 @@ dependencies = [ "fsspec>=2023.1.0", ] -requires-python = ">= 3.11" +requires-python = ">= 3.12" classifiers = [ "Typing :: Typed", "Intended Audience :: Developers", - "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", @@ -161,7 +160,7 @@ filterwarnings = [ # there are a couple of flags that are still disabled by # default in strict mode as they are experimental and niche. typeCheckingMode = "strict" -pythonVersion = "3.11" +pythonVersion = "3.12" exclude = [ "_dev", @@ -231,7 +230,7 @@ ignore_missing_imports = true [tool.ruff] line-length = 120 output-format = "grouped" -target-version = "py311" +target-version = "py312" [tool.ruff.format] docstring-code-format = true diff --git a/sdk/python/nemo-platform/src/nemo_platform/_client.py b/sdk/python/nemo-platform/src/nemo_platform/_client.py index 893eae284f..bb6f751eb0 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/_client.py +++ b/sdk/python/nemo-platform/src/nemo_platform/_client.py @@ -68,7 +68,7 @@ inference, workspaces, evaluations, - experiment_groups, + experiments, ) from .resources.iam.iam import IamResource, AsyncIamResource from .resources.jobs.jobs import JobsResource, AsyncJobsResource @@ -83,7 +83,7 @@ from .resources.inference.inference import InferenceResource, AsyncInferenceResource from .resources.workspaces.workspaces import WorkspacesResource, AsyncWorkspacesResource from .resources.evaluations.evaluations import EvaluationsResource, AsyncEvaluationsResource - from .resources.experiment_groups.experiment_groups import ExperimentGroupsResource, AsyncExperimentGroupsResource + from .resources.experiments.experiments import ExperimentsResource, AsyncExperimentsResource __all__ = [ "Timeout", @@ -324,18 +324,18 @@ def intake(self) -> IntakeResource: return IntakeResource(self) - @cached_property - def experiment_groups(self) -> ExperimentGroupsResource: - from .resources.experiment_groups import ExperimentGroupsResource - - return ExperimentGroupsResource(self) - @cached_property def evaluations(self) -> EvaluationsResource: from .resources.evaluations import EvaluationsResource return EvaluationsResource(self) + @cached_property + def experiments(self) -> ExperimentsResource: + from .resources.experiments import ExperimentsResource + + return ExperimentsResource(self) + @cached_property def with_raw_response(self) -> NeMoPlatformWithRawResponse: return NeMoPlatformWithRawResponse(self) @@ -688,18 +688,18 @@ def intake(self) -> AsyncIntakeResource: return AsyncIntakeResource(self) - @cached_property - def experiment_groups(self) -> AsyncExperimentGroupsResource: - from .resources.experiment_groups import AsyncExperimentGroupsResource - - return AsyncExperimentGroupsResource(self) - @cached_property def evaluations(self) -> AsyncEvaluationsResource: from .resources.evaluations import AsyncEvaluationsResource return AsyncEvaluationsResource(self) + @cached_property + def experiments(self) -> AsyncExperimentsResource: + from .resources.experiments import AsyncExperimentsResource + + return AsyncExperimentsResource(self) + @cached_property def with_raw_response(self) -> AsyncNeMoPlatformWithRawResponse: return AsyncNeMoPlatformWithRawResponse(self) @@ -909,18 +909,18 @@ def intake(self) -> intake.IntakeResourceWithRawResponse: return IntakeResourceWithRawResponse(self._client.intake) - @cached_property - def experiment_groups(self) -> experiment_groups.ExperimentGroupsResourceWithRawResponse: - from .resources.experiment_groups import ExperimentGroupsResourceWithRawResponse - - return ExperimentGroupsResourceWithRawResponse(self._client.experiment_groups) - @cached_property def evaluations(self) -> evaluations.EvaluationsResourceWithRawResponse: from .resources.evaluations import EvaluationsResourceWithRawResponse return EvaluationsResourceWithRawResponse(self._client.evaluations) + @cached_property + def experiments(self) -> experiments.ExperimentsResourceWithRawResponse: + from .resources.experiments import ExperimentsResourceWithRawResponse + + return ExperimentsResourceWithRawResponse(self._client.experiments) + class AsyncNeMoPlatformWithRawResponse: _client: AsyncNeMoPlatform @@ -1000,18 +1000,18 @@ def intake(self) -> intake.AsyncIntakeResourceWithRawResponse: return AsyncIntakeResourceWithRawResponse(self._client.intake) - @cached_property - def experiment_groups(self) -> experiment_groups.AsyncExperimentGroupsResourceWithRawResponse: - from .resources.experiment_groups import AsyncExperimentGroupsResourceWithRawResponse - - return AsyncExperimentGroupsResourceWithRawResponse(self._client.experiment_groups) - @cached_property def evaluations(self) -> evaluations.AsyncEvaluationsResourceWithRawResponse: from .resources.evaluations import AsyncEvaluationsResourceWithRawResponse return AsyncEvaluationsResourceWithRawResponse(self._client.evaluations) + @cached_property + def experiments(self) -> experiments.AsyncExperimentsResourceWithRawResponse: + from .resources.experiments import AsyncExperimentsResourceWithRawResponse + + return AsyncExperimentsResourceWithRawResponse(self._client.experiments) + class NeMoPlatformWithStreamedResponse: _client: NeMoPlatform @@ -1091,18 +1091,18 @@ def intake(self) -> intake.IntakeResourceWithStreamingResponse: return IntakeResourceWithStreamingResponse(self._client.intake) - @cached_property - def experiment_groups(self) -> experiment_groups.ExperimentGroupsResourceWithStreamingResponse: - from .resources.experiment_groups import ExperimentGroupsResourceWithStreamingResponse - - return ExperimentGroupsResourceWithStreamingResponse(self._client.experiment_groups) - @cached_property def evaluations(self) -> evaluations.EvaluationsResourceWithStreamingResponse: from .resources.evaluations import EvaluationsResourceWithStreamingResponse return EvaluationsResourceWithStreamingResponse(self._client.evaluations) + @cached_property + def experiments(self) -> experiments.ExperimentsResourceWithStreamingResponse: + from .resources.experiments import ExperimentsResourceWithStreamingResponse + + return ExperimentsResourceWithStreamingResponse(self._client.experiments) + class AsyncNeMoPlatformWithStreamedResponse: _client: AsyncNeMoPlatform @@ -1182,18 +1182,18 @@ def intake(self) -> intake.AsyncIntakeResourceWithStreamingResponse: return AsyncIntakeResourceWithStreamingResponse(self._client.intake) - @cached_property - def experiment_groups(self) -> experiment_groups.AsyncExperimentGroupsResourceWithStreamingResponse: - from .resources.experiment_groups import AsyncExperimentGroupsResourceWithStreamingResponse - - return AsyncExperimentGroupsResourceWithStreamingResponse(self._client.experiment_groups) - @cached_property def evaluations(self) -> evaluations.AsyncEvaluationsResourceWithStreamingResponse: from .resources.evaluations import AsyncEvaluationsResourceWithStreamingResponse return AsyncEvaluationsResourceWithStreamingResponse(self._client.evaluations) + @cached_property + def experiments(self) -> experiments.AsyncExperimentsResourceWithStreamingResponse: + from .resources.experiments import AsyncExperimentsResourceWithStreamingResponse + + return AsyncExperimentsResourceWithStreamingResponse(self._client.experiments) + Client = NeMoPlatform diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/atif.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/atif.py index 8f7fd046ac..c14a117efb 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/atif.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/atif.py @@ -39,13 +39,6 @@ def create_atif( help='Evaluation context accepted by ingest endpoints (the canonical shape).`extra="ignore"` so a producer still sending retired keys (evaluation_sha, evaluation_run_id, metadata) keeps ingesting without error rather than being rejected. (JSON string)', ), ] = None, - experiment_context: Annotated[ - str | None, - typer.Option( - "--experiment-context", - help="Deprecated alias for :class:`EvaluationContext`. Producers should send `evaluation_context`. (JSON string)", - ), - ] = None, extra: Annotated[str | None, typer.Option("--extra", help="JSON string")] = None, final_metrics: Annotated[str | None, typer.Option("--final-metrics", help="JSON string")] = None, notes: Annotated[str | None, typer.Option("--notes")] = None, @@ -90,8 +83,6 @@ def create_atif( input_payload["continued_trajectory_ref"] = continued_trajectory_ref if evaluation_context is not None: input_payload["evaluation_context"] = read_payload("evaluation_context", evaluation_context) - if experiment_context is not None: - input_payload["experiment_context"] = read_payload("experiment_context", experiment_context) if extra is not None: input_payload["extra"] = read_payload("extra", extra) if final_metrics is not None: diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/chat_completions.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/chat_completions.py index f450ddfb1a..1dc5f7aaae 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/chat_completions.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/chat_completions.py @@ -57,13 +57,6 @@ def create_chat_completions( help='Evaluation context accepted by ingest endpoints (the canonical shape).`extra="ignore"` so a producer still sending retired keys (evaluation_sha, evaluation_run_id, metadata) keeps ingesting without error rather than being rejected. (JSON string)', ), ] = None, - experiment_context: Annotated[ - str | None, - typer.Option( - "--experiment-context", - help="Deprecated alias for :class:`EvaluationContext`. Producers should send `evaluation_context`. (JSON string)", - ), - ] = None, provider: Annotated[str | None, typer.Option("--provider")] = None, session_id: Annotated[ str | None, @@ -121,8 +114,6 @@ def create_chat_completions( input_payload["cost_usd"] = cost_usd if evaluation_context is not None: input_payload["evaluation_context"] = read_payload("evaluation_context", evaluation_context) - if experiment_context is not None: - input_payload["experiment_context"] = read_payload("experiment_context", experiment_context) if provider is not None: input_payload["provider"] = provider if session_id is not None: diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/traces.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/traces.py index ec9ebf5aa1..961a2b7dae 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/traces.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/traces.py @@ -36,7 +36,7 @@ def list_traces( typer.Option( "--filter", metavar="FILTER_JSON", - help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter root-span-backed traces by id, session_id, root status, root span started_at, evaluation_id (or its deprecated alias experiment_id), and test_case_id.", + help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter root-span-backed traces by id, session_id, root status, root span started_at, evaluation_id, and test_case_id.", rich_help_panel="Filter Options", ), ] = None, @@ -44,9 +44,6 @@ def list_traces( filter_evaluation_id: Annotated[ str | None, typer.Option("--filter.evaluation-id", rich_help_panel="Filter Options") ] = None, - filter_experiment_id: Annotated[ - str | None, typer.Option("--filter.experiment-id", rich_help_panel="Filter Options") - ] = None, filter_session_id: Annotated[ str | None, typer.Option("--filter.session-id", rich_help_panel="Filter Options") ] = None, @@ -89,7 +86,6 @@ def list_traces( filter, id=filter_id, evaluation_id=filter_evaluation_id, - experiment_id=filter_experiment_id, session_id=filter_session_id, status=filter_status, test_case_id=filter_test_case_id, diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/evaluations.py b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/evaluations.py index 5aa001ce64..a8f857177f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/evaluations.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/evaluations.py @@ -90,7 +90,6 @@ def create( experiment_ids: SequenceNotStr[str] | Omit = omit, metadata: Dict[str, str] | Omit = omit, parent_evaluation_id: str | Omit = omit, - parent_experiment_id: str | Omit = omit, root_cause: str | Omit = omit, source_link: str | Omit = omit, status: str | Omit = omit, @@ -114,11 +113,11 @@ def create( description: Human-readable description. - experiment_group_id: Deprecated single-group field; provide experiment_ids instead. Coalesced into - experiment_ids when experiment_ids is omitted. + experiment_group_id: Deprecated single-experiment field; provide experiment_ids instead. Coalesced + into experiment_ids when experiment_ids is omitted. - experiment_ids: Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). Preferred; - each group must already exist. When omitted, the deprecated experiment_group_id + experiment_ids: Entity ids of the Experiments this Evaluation belongs to (>=1). Preferred; each + experiment must already exist. When omitted, the deprecated experiment_group_id is used instead. metadata: Free-form producer metadata. @@ -126,8 +125,6 @@ def create( parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a baseline), if any. - parent_experiment_id: Deprecated alias for parent_evaluation_id. - root_cause: Human- or agent-authored explanation of the evaluation's outcome (e.g. why it was killed). @@ -164,7 +161,6 @@ def create( "experiment_ids": experiment_ids, "metadata": metadata, "parent_evaluation_id": parent_evaluation_id, - "parent_experiment_id": parent_experiment_id, "root_cause": root_cause, "source_link": source_link, "status": status, @@ -232,7 +228,6 @@ def update( experiment_ids: SequenceNotStr[str] | Omit = omit, metadata: Dict[str, str] | Omit = omit, parent_evaluation_id: str | Omit = omit, - parent_experiment_id: str | Omit = omit, root_cause: str | Omit = omit, source_link: str | Omit = omit, status: str | Omit = omit, @@ -255,11 +250,11 @@ def update( description: Human-readable description. - experiment_group_id: Deprecated single-group field; provide experiment_ids instead. Coalesced into - experiment_ids when experiment_ids is omitted. + experiment_group_id: Deprecated single-experiment field; provide experiment_ids instead. Coalesced + into experiment_ids when experiment_ids is omitted. - experiment_ids: Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). Preferred; - each group must already exist. When omitted, the deprecated experiment_group_id + experiment_ids: Entity ids of the Experiments this Evaluation belongs to (>=1). Preferred; each + experiment must already exist. When omitted, the deprecated experiment_group_id is used instead. metadata: Free-form producer metadata. @@ -267,8 +262,6 @@ def update( parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a baseline), if any. - parent_experiment_id: Deprecated alias for parent_evaluation_id. - root_cause: Human- or agent-authored explanation of the evaluation's outcome (e.g. why it was killed). @@ -306,7 +299,6 @@ def update( "experiment_ids": experiment_ids, "metadata": metadata, "parent_evaluation_id": parent_evaluation_id, - "parent_experiment_id": parent_experiment_id, "root_cause": root_cause, "source_link": source_link, "status": status, @@ -463,8 +455,9 @@ def patch( Args: description: Human-readable description. - experiment_ids: Replace the ExperimentGroups this Evaluation belongs to. Must be non-empty when - provided; each group must already exist. Omit to leave membership unchanged. + experiment_ids: Replace the Experiments this Evaluation belongs to. Must be non-empty when + provided; each experiment must already exist. Omit to leave membership + unchanged. metadata: Free-form producer metadata. @@ -634,7 +627,6 @@ async def create( experiment_ids: SequenceNotStr[str] | Omit = omit, metadata: Dict[str, str] | Omit = omit, parent_evaluation_id: str | Omit = omit, - parent_experiment_id: str | Omit = omit, root_cause: str | Omit = omit, source_link: str | Omit = omit, status: str | Omit = omit, @@ -658,11 +650,11 @@ async def create( description: Human-readable description. - experiment_group_id: Deprecated single-group field; provide experiment_ids instead. Coalesced into - experiment_ids when experiment_ids is omitted. + experiment_group_id: Deprecated single-experiment field; provide experiment_ids instead. Coalesced + into experiment_ids when experiment_ids is omitted. - experiment_ids: Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). Preferred; - each group must already exist. When omitted, the deprecated experiment_group_id + experiment_ids: Entity ids of the Experiments this Evaluation belongs to (>=1). Preferred; each + experiment must already exist. When omitted, the deprecated experiment_group_id is used instead. metadata: Free-form producer metadata. @@ -670,8 +662,6 @@ async def create( parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a baseline), if any. - parent_experiment_id: Deprecated alias for parent_evaluation_id. - root_cause: Human- or agent-authored explanation of the evaluation's outcome (e.g. why it was killed). @@ -708,7 +698,6 @@ async def create( "experiment_ids": experiment_ids, "metadata": metadata, "parent_evaluation_id": parent_evaluation_id, - "parent_experiment_id": parent_experiment_id, "root_cause": root_cause, "source_link": source_link, "status": status, @@ -776,7 +765,6 @@ async def update( experiment_ids: SequenceNotStr[str] | Omit = omit, metadata: Dict[str, str] | Omit = omit, parent_evaluation_id: str | Omit = omit, - parent_experiment_id: str | Omit = omit, root_cause: str | Omit = omit, source_link: str | Omit = omit, status: str | Omit = omit, @@ -799,11 +787,11 @@ async def update( description: Human-readable description. - experiment_group_id: Deprecated single-group field; provide experiment_ids instead. Coalesced into - experiment_ids when experiment_ids is omitted. + experiment_group_id: Deprecated single-experiment field; provide experiment_ids instead. Coalesced + into experiment_ids when experiment_ids is omitted. - experiment_ids: Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). Preferred; - each group must already exist. When omitted, the deprecated experiment_group_id + experiment_ids: Entity ids of the Experiments this Evaluation belongs to (>=1). Preferred; each + experiment must already exist. When omitted, the deprecated experiment_group_id is used instead. metadata: Free-form producer metadata. @@ -811,8 +799,6 @@ async def update( parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a baseline), if any. - parent_experiment_id: Deprecated alias for parent_evaluation_id. - root_cause: Human- or agent-authored explanation of the evaluation's outcome (e.g. why it was killed). @@ -850,7 +836,6 @@ async def update( "experiment_ids": experiment_ids, "metadata": metadata, "parent_evaluation_id": parent_evaluation_id, - "parent_experiment_id": parent_experiment_id, "root_cause": root_cause, "source_link": source_link, "status": status, @@ -1007,8 +992,9 @@ async def patch( Args: description: Human-readable description. - experiment_ids: Replace the ExperimentGroups this Evaluation belongs to. Must be non-empty when - provided; each group must already exist. Omit to leave membership unchanged. + experiment_ids: Replace the Experiments this Evaluation belongs to. Must be non-empty when + provided; each experiment must already exist. Omit to leave membership + unchanged. metadata: Free-form producer metadata. diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/__init__.py deleted file mode 100644 index ad45581d2b..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -from .experiment_groups import ( - ExperimentGroupsResource, - AsyncExperimentGroupsResource, - ExperimentGroupsResourceWithRawResponse, - AsyncExperimentGroupsResourceWithRawResponse, - ExperimentGroupsResourceWithStreamingResponse, - AsyncExperimentGroupsResourceWithStreamingResponse, -) - -__all__ = [ - "ExperimentGroupsResource", - "AsyncExperimentGroupsResource", - "ExperimentGroupsResourceWithRawResponse", - "AsyncExperimentGroupsResourceWithRawResponse", - "ExperimentGroupsResourceWithStreamingResponse", - "AsyncExperimentGroupsResourceWithStreamingResponse", -] diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md deleted file mode 100644 index 998ab3da68..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md +++ /dev/null @@ -1,21 +0,0 @@ -# ExperimentGroups - -Types: - -```python -from nemo_platform.types.experiment_groups import ( - ExperimentGroupFilter, - ExperimentGroupRequest, - ExperimentGroupResponse, - ExperimentGroupResponsesPage, - ParetoConfig, -) -``` - -Methods: - -- client.experiment_groups.create(\*, workspace, \*\*params) -> ExperimentGroupResponse -- client.experiment_groups.retrieve(name, \*, workspace) -> ExperimentGroupResponse -- client.experiment_groups.update(path_name, \*, workspace, \*\*params) -> ExperimentGroupResponse -- client.experiment_groups.list(\*, workspace, \*\*params) -> SyncDefaultPagination[ExperimentGroupResponse] -- client.experiment_groups.delete(name, \*, workspace) -> None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/experiment_context_param.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/__init__.py similarity index 59% rename from sdk/python/nemo-platform/src/nemo_platform/types/intake/experiment_context_param.py rename to sdk/python/nemo-platform/src/nemo_platform/resources/experiments/__init__.py index fb37459fef..9dbe086588 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/experiment_context_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/__init__.py @@ -15,21 +15,20 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from __future__ import annotations - -from typing_extensions import Required, TypedDict - -__all__ = ["ExperimentContextParam"] - - -class ExperimentContextParam(TypedDict, total=False): - """Deprecated alias for :class:`EvaluationContext`. - - Producers should send ``evaluation_context``. - """ - - experiment_id: Required[str] - """Name of an existing Experiment entity.""" - - test_case_id: str - """Optional producer-supplied test case id.""" +from .experiments import ( + ExperimentsResource, + AsyncExperimentsResource, + ExperimentsResourceWithRawResponse, + AsyncExperimentsResourceWithRawResponse, + ExperimentsResourceWithStreamingResponse, + AsyncExperimentsResourceWithStreamingResponse, +) + +__all__ = [ + "ExperimentsResource", + "AsyncExperimentsResource", + "ExperimentsResourceWithRawResponse", + "AsyncExperimentsResourceWithRawResponse", + "ExperimentsResourceWithStreamingResponse", + "AsyncExperimentsResourceWithStreamingResponse", +] diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md new file mode 100644 index 0000000000..8c5ef3faa9 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md @@ -0,0 +1,21 @@ +# Experiments + +Types: + +```python +from nemo_platform.types.experiments import ( + ExperimentFilter, + ExperimentRequest, + ExperimentResponse, + ExperimentResponsesPage, + ParetoConfig, +) +``` + +Methods: + +- client.experiments.create(\*, workspace, \*\*params) -> ExperimentResponse +- client.experiments.retrieve(name, \*, workspace) -> ExperimentResponse +- client.experiments.update(path_name, \*, workspace, \*\*params) -> ExperimentResponse +- client.experiments.list(\*, workspace, \*\*params) -> SyncDefaultPagination[ExperimentResponse] +- client.experiments.delete(name, \*, workspace) -> None diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py similarity index 78% rename from sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py rename to sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py index 7d8ea35079..2416de69f3 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py @@ -34,38 +34,38 @@ ) from ...pagination import SyncDefaultPagination, AsyncDefaultPagination from ..._base_client import AsyncPaginator, make_request_options -from ...types.experiment_groups import ( - experiment_group_list_params, - experiment_group_create_params, - experiment_group_update_params, +from ...types.experiments import ( + experiment_list_params, + experiment_create_params, + experiment_update_params, ) -from ...types.experiment_groups.pareto_config_param import ParetoConfigParam -from ...types.experiment_groups.experiment_group_response import ExperimentGroupResponse -from ...types.experiment_groups.experiment_group_filter_param import ExperimentGroupFilterParam +from ...types.experiments.experiment_response import ExperimentResponse +from ...types.experiments.pareto_config_param import ParetoConfigParam +from ...types.experiments.experiment_filter_param import ExperimentFilterParam from ..._exceptions import ConflictError -__all__ = ["ExperimentGroupsResource", "AsyncExperimentGroupsResource"] +__all__ = ["ExperimentsResource", "AsyncExperimentsResource"] -class ExperimentGroupsResource(SyncAPIResource): +class ExperimentsResource(SyncAPIResource): @cached_property - def with_raw_response(self) -> ExperimentGroupsResourceWithRawResponse: + def with_raw_response(self) -> ExperimentsResourceWithRawResponse: """ This property can be used as a prefix for any HTTP method call to return the raw response object instead of the parsed content. For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#accessing-raw-response-data-e-g-headers """ - return ExperimentGroupsResourceWithRawResponse(self) + return ExperimentsResourceWithRawResponse(self) @cached_property - def with_streaming_response(self) -> ExperimentGroupsResourceWithStreamingResponse: + def with_streaming_response(self) -> ExperimentsResourceWithStreamingResponse: """ An alternative to `.with_raw_response` that doesn't eagerly read the response body. For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#with_streaming_response """ - return ExperimentGroupsResourceWithStreamingResponse(self) + return ExperimentsResourceWithStreamingResponse(self) def create( self, @@ -85,25 +85,25 @@ def create( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ExperimentGroupResponse: + ) -> ExperimentResponse: """ - Create Experiment Group + Create Experiment Args: - name: Workspace-unique group name. + name: Workspace-unique experiment name. - default_sort: Default sort for this group's evaluations list, as a `sort`-param string: a + default_sort: Default sort for this experiment's evaluations list, as a `sort`-param string: a comma-separated, ordered list of fields where the first is the primary sort and the rest break ties (leading '-' on a field = descending), e.g. '-evaluators.reward.mean,cost_usd.mean'. Defaults to '-created_at'. Accepts any field the evaluations list `sort` param does; clients apply it as the list `sort` param. - description: Human-readable purpose of the group. + description: Human-readable purpose of the experiment. - insight_id: Reference to an external insight that seeded this group, if any. + insight_id: Reference to an external insight that seeded this experiment, if any. - metadata: Free-form producer metadata for the group. + metadata: Free-form producer metadata for the experiment. pareto: Default X/Y metrics for a group's cost-vs-accuracy Pareto view. @@ -112,7 +112,7 @@ def create( latency (y): both exist for every group, so the chart always has something to render before anyone customizes it. - summary: Human- or agent-authored summary of the group's findings. + summary: Human- or agent-authored summary of the experiment's findings. exist_ok: Do not raise an error if the resource already exists. Returns the existing resource. @@ -132,7 +132,7 @@ def create( if not workspace: raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") return self._post( - path_template("/apis/intake/v2/workspaces/{workspace}/experiment-groups", workspace=workspace), + path_template("/apis/intake/v2/workspaces/{workspace}/experiments", workspace=workspace), body=maybe_transform( { "name": name, @@ -143,12 +143,12 @@ def create( "pareto": pareto, "summary": summary, }, - experiment_group_create_params.ExperimentGroupCreateParams, + experiment_create_params.ExperimentCreateParams, ), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ExperimentGroupResponse, + cast_to=ExperimentResponse, ) except ConflictError: if not exist_ok: @@ -166,9 +166,9 @@ def retrieve( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ExperimentGroupResponse: + ) -> ExperimentResponse: """ - Get Experiment Group + Get Experiment Args: extra_headers: Send extra headers @@ -186,13 +186,11 @@ def retrieve( if not name: raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") return self._get( - path_template( - "/apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}", workspace=workspace, name=name - ), + path_template("/apis/intake/v2/workspaces/{workspace}/experiments/{name}", workspace=workspace, name=name), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ExperimentGroupResponse, + cast_to=ExperimentResponse, ) def update( @@ -213,25 +211,25 @@ def update( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ExperimentGroupResponse: + ) -> ExperimentResponse: """ - Update Experiment Group + Update Experiment Args: - body_name: Workspace-unique group name. + body_name: Workspace-unique experiment name. - default_sort: Default sort for this group's evaluations list, as a `sort`-param string: a + default_sort: Default sort for this experiment's evaluations list, as a `sort`-param string: a comma-separated, ordered list of fields where the first is the primary sort and the rest break ties (leading '-' on a field = descending), e.g. '-evaluators.reward.mean,cost_usd.mean'. Defaults to '-created_at'. Accepts any field the evaluations list `sort` param does; clients apply it as the list `sort` param. - description: Human-readable purpose of the group. + description: Human-readable purpose of the experiment. - insight_id: Reference to an external insight that seeded this group, if any. + insight_id: Reference to an external insight that seeded this experiment, if any. - metadata: Free-form producer metadata for the group. + metadata: Free-form producer metadata for the experiment. pareto: Default X/Y metrics for a group's cost-vs-accuracy Pareto view. @@ -240,7 +238,7 @@ def update( latency (y): both exist for every group, so the chart always has something to render before anyone customizes it. - summary: Human- or agent-authored summary of the group's findings. + summary: Human- or agent-authored summary of the experiment's findings. extra_headers: Send extra headers @@ -258,7 +256,7 @@ def update( raise ValueError(f"Expected a non-empty value for `path_name` but received {path_name!r}") return self._put( path_template( - "/apis/intake/v2/workspaces/{workspace}/experiment-groups/{path_name}", + "/apis/intake/v2/workspaces/{workspace}/experiments/{path_name}", workspace=workspace, path_name=path_name, ), @@ -272,19 +270,19 @@ def update( "pareto": pareto, "summary": summary, }, - experiment_group_update_params.ExperimentGroupUpdateParams, + experiment_update_params.ExperimentUpdateParams, ), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ExperimentGroupResponse, + cast_to=ExperimentResponse, ) def list( self, *, workspace: str | None = None, - filter: ExperimentGroupFilterParam | Omit = omit, + filter: ExperimentFilterParam | Omit = omit, page: int | Omit = omit, page_size: int | Omit = omit, sort: Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"] | Omit = omit, @@ -294,14 +292,14 @@ def list( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> SyncDefaultPagination[ExperimentGroupResponse]: + ) -> SyncDefaultPagination[ExperimentResponse]: """ - List Experiment Groups + List Experiments Args: - filter: Filter experiment groups by name, insight_id, is_deleted, or a metadata - key/value (filter[metadata.]=). Pass is_deleted=true to return only - soft-deleted groups; omit to see only live ones. + filter: Filter experiments by name, insight_id, is_deleted, or a metadata key/value + (filter[metadata.]=). Pass is_deleted=true to return only + soft-deleted experiments; omit to see only live ones. page: Page number. @@ -322,8 +320,8 @@ def list( if not workspace: raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") return self._get_api_list( - path_template("/apis/intake/v2/workspaces/{workspace}/experiment-groups", workspace=workspace), - page=SyncDefaultPagination[ExperimentGroupResponse], + path_template("/apis/intake/v2/workspaces/{workspace}/experiments", workspace=workspace), + page=SyncDefaultPagination[ExperimentResponse], options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, @@ -336,10 +334,10 @@ def list( "page_size": page_size, "sort": sort, }, - experiment_group_list_params.ExperimentGroupListParams, + experiment_list_params.ExperimentListParams, ), ), - model=ExperimentGroupResponse, + model=ExperimentResponse, ) def delete( @@ -355,7 +353,7 @@ def delete( timeout: float | httpx.Timeout | None | NotGiven = not_given, ) -> None: """ - Delete Experiment Group + Delete Experiment Args: extra_headers: Send extra headers @@ -374,9 +372,7 @@ def delete( raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") extra_headers = {"Accept": "*/*", **(extra_headers or {})} return self._delete( - path_template( - "/apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}", workspace=workspace, name=name - ), + path_template("/apis/intake/v2/workspaces/{workspace}/experiments/{name}", workspace=workspace, name=name), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), @@ -384,25 +380,25 @@ def delete( ) -class AsyncExperimentGroupsResource(AsyncAPIResource): +class AsyncExperimentsResource(AsyncAPIResource): @cached_property - def with_raw_response(self) -> AsyncExperimentGroupsResourceWithRawResponse: + def with_raw_response(self) -> AsyncExperimentsResourceWithRawResponse: """ This property can be used as a prefix for any HTTP method call to return the raw response object instead of the parsed content. For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#accessing-raw-response-data-e-g-headers """ - return AsyncExperimentGroupsResourceWithRawResponse(self) + return AsyncExperimentsResourceWithRawResponse(self) @cached_property - def with_streaming_response(self) -> AsyncExperimentGroupsResourceWithStreamingResponse: + def with_streaming_response(self) -> AsyncExperimentsResourceWithStreamingResponse: """ An alternative to `.with_raw_response` that doesn't eagerly read the response body. For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#with_streaming_response """ - return AsyncExperimentGroupsResourceWithStreamingResponse(self) + return AsyncExperimentsResourceWithStreamingResponse(self) async def create( self, @@ -422,25 +418,25 @@ async def create( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ExperimentGroupResponse: + ) -> ExperimentResponse: """ - Create Experiment Group + Create Experiment Args: - name: Workspace-unique group name. + name: Workspace-unique experiment name. - default_sort: Default sort for this group's evaluations list, as a `sort`-param string: a + default_sort: Default sort for this experiment's evaluations list, as a `sort`-param string: a comma-separated, ordered list of fields where the first is the primary sort and the rest break ties (leading '-' on a field = descending), e.g. '-evaluators.reward.mean,cost_usd.mean'. Defaults to '-created_at'. Accepts any field the evaluations list `sort` param does; clients apply it as the list `sort` param. - description: Human-readable purpose of the group. + description: Human-readable purpose of the experiment. - insight_id: Reference to an external insight that seeded this group, if any. + insight_id: Reference to an external insight that seeded this experiment, if any. - metadata: Free-form producer metadata for the group. + metadata: Free-form producer metadata for the experiment. pareto: Default X/Y metrics for a group's cost-vs-accuracy Pareto view. @@ -449,7 +445,7 @@ async def create( latency (y): both exist for every group, so the chart always has something to render before anyone customizes it. - summary: Human- or agent-authored summary of the group's findings. + summary: Human- or agent-authored summary of the experiment's findings. exist_ok: Do not raise an error if the resource already exists. Returns the existing resource. @@ -469,7 +465,7 @@ async def create( if not workspace: raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") return await self._post( - path_template("/apis/intake/v2/workspaces/{workspace}/experiment-groups", workspace=workspace), + path_template("/apis/intake/v2/workspaces/{workspace}/experiments", workspace=workspace), body=await async_maybe_transform( { "name": name, @@ -480,12 +476,12 @@ async def create( "pareto": pareto, "summary": summary, }, - experiment_group_create_params.ExperimentGroupCreateParams, + experiment_create_params.ExperimentCreateParams, ), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ExperimentGroupResponse, + cast_to=ExperimentResponse, ) except ConflictError: if not exist_ok: @@ -503,9 +499,9 @@ async def retrieve( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ExperimentGroupResponse: + ) -> ExperimentResponse: """ - Get Experiment Group + Get Experiment Args: extra_headers: Send extra headers @@ -523,13 +519,11 @@ async def retrieve( if not name: raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") return await self._get( - path_template( - "/apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}", workspace=workspace, name=name - ), + path_template("/apis/intake/v2/workspaces/{workspace}/experiments/{name}", workspace=workspace, name=name), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ExperimentGroupResponse, + cast_to=ExperimentResponse, ) async def update( @@ -550,25 +544,25 @@ async def update( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ExperimentGroupResponse: + ) -> ExperimentResponse: """ - Update Experiment Group + Update Experiment Args: - body_name: Workspace-unique group name. + body_name: Workspace-unique experiment name. - default_sort: Default sort for this group's evaluations list, as a `sort`-param string: a + default_sort: Default sort for this experiment's evaluations list, as a `sort`-param string: a comma-separated, ordered list of fields where the first is the primary sort and the rest break ties (leading '-' on a field = descending), e.g. '-evaluators.reward.mean,cost_usd.mean'. Defaults to '-created_at'. Accepts any field the evaluations list `sort` param does; clients apply it as the list `sort` param. - description: Human-readable purpose of the group. + description: Human-readable purpose of the experiment. - insight_id: Reference to an external insight that seeded this group, if any. + insight_id: Reference to an external insight that seeded this experiment, if any. - metadata: Free-form producer metadata for the group. + metadata: Free-form producer metadata for the experiment. pareto: Default X/Y metrics for a group's cost-vs-accuracy Pareto view. @@ -577,7 +571,7 @@ async def update( latency (y): both exist for every group, so the chart always has something to render before anyone customizes it. - summary: Human- or agent-authored summary of the group's findings. + summary: Human- or agent-authored summary of the experiment's findings. extra_headers: Send extra headers @@ -595,7 +589,7 @@ async def update( raise ValueError(f"Expected a non-empty value for `path_name` but received {path_name!r}") return await self._put( path_template( - "/apis/intake/v2/workspaces/{workspace}/experiment-groups/{path_name}", + "/apis/intake/v2/workspaces/{workspace}/experiments/{path_name}", workspace=workspace, path_name=path_name, ), @@ -609,19 +603,19 @@ async def update( "pareto": pareto, "summary": summary, }, - experiment_group_update_params.ExperimentGroupUpdateParams, + experiment_update_params.ExperimentUpdateParams, ), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ExperimentGroupResponse, + cast_to=ExperimentResponse, ) def list( self, *, workspace: str | None = None, - filter: ExperimentGroupFilterParam | Omit = omit, + filter: ExperimentFilterParam | Omit = omit, page: int | Omit = omit, page_size: int | Omit = omit, sort: Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"] | Omit = omit, @@ -631,14 +625,14 @@ def list( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> AsyncPaginator[ExperimentGroupResponse, AsyncDefaultPagination[ExperimentGroupResponse]]: + ) -> AsyncPaginator[ExperimentResponse, AsyncDefaultPagination[ExperimentResponse]]: """ - List Experiment Groups + List Experiments Args: - filter: Filter experiment groups by name, insight_id, is_deleted, or a metadata - key/value (filter[metadata.]=). Pass is_deleted=true to return only - soft-deleted groups; omit to see only live ones. + filter: Filter experiments by name, insight_id, is_deleted, or a metadata key/value + (filter[metadata.]=). Pass is_deleted=true to return only + soft-deleted experiments; omit to see only live ones. page: Page number. @@ -659,8 +653,8 @@ def list( if not workspace: raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") return self._get_api_list( - path_template("/apis/intake/v2/workspaces/{workspace}/experiment-groups", workspace=workspace), - page=AsyncDefaultPagination[ExperimentGroupResponse], + path_template("/apis/intake/v2/workspaces/{workspace}/experiments", workspace=workspace), + page=AsyncDefaultPagination[ExperimentResponse], options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, @@ -673,10 +667,10 @@ def list( "page_size": page_size, "sort": sort, }, - experiment_group_list_params.ExperimentGroupListParams, + experiment_list_params.ExperimentListParams, ), ), - model=ExperimentGroupResponse, + model=ExperimentResponse, ) async def delete( @@ -692,7 +686,7 @@ async def delete( timeout: float | httpx.Timeout | None | NotGiven = not_given, ) -> None: """ - Delete Experiment Group + Delete Experiment Args: extra_headers: Send extra headers @@ -711,9 +705,7 @@ async def delete( raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") extra_headers = {"Accept": "*/*", **(extra_headers or {})} return await self._delete( - path_template( - "/apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}", workspace=workspace, name=name - ), + path_template("/apis/intake/v2/workspaces/{workspace}/experiments/{name}", workspace=workspace, name=name), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), @@ -721,85 +713,85 @@ async def delete( ) -class ExperimentGroupsResourceWithRawResponse: - def __init__(self, experiment_groups: ExperimentGroupsResource) -> None: - self._experiment_groups = experiment_groups +class ExperimentsResourceWithRawResponse: + def __init__(self, experiments: ExperimentsResource) -> None: + self._experiments = experiments self.create = to_raw_response_wrapper( - experiment_groups.create, + experiments.create, ) self.retrieve = to_raw_response_wrapper( - experiment_groups.retrieve, + experiments.retrieve, ) self.update = to_raw_response_wrapper( - experiment_groups.update, + experiments.update, ) self.list = to_raw_response_wrapper( - experiment_groups.list, + experiments.list, ) self.delete = to_raw_response_wrapper( - experiment_groups.delete, + experiments.delete, ) -class AsyncExperimentGroupsResourceWithRawResponse: - def __init__(self, experiment_groups: AsyncExperimentGroupsResource) -> None: - self._experiment_groups = experiment_groups +class AsyncExperimentsResourceWithRawResponse: + def __init__(self, experiments: AsyncExperimentsResource) -> None: + self._experiments = experiments self.create = async_to_raw_response_wrapper( - experiment_groups.create, + experiments.create, ) self.retrieve = async_to_raw_response_wrapper( - experiment_groups.retrieve, + experiments.retrieve, ) self.update = async_to_raw_response_wrapper( - experiment_groups.update, + experiments.update, ) self.list = async_to_raw_response_wrapper( - experiment_groups.list, + experiments.list, ) self.delete = async_to_raw_response_wrapper( - experiment_groups.delete, + experiments.delete, ) -class ExperimentGroupsResourceWithStreamingResponse: - def __init__(self, experiment_groups: ExperimentGroupsResource) -> None: - self._experiment_groups = experiment_groups +class ExperimentsResourceWithStreamingResponse: + def __init__(self, experiments: ExperimentsResource) -> None: + self._experiments = experiments self.create = to_streamed_response_wrapper( - experiment_groups.create, + experiments.create, ) self.retrieve = to_streamed_response_wrapper( - experiment_groups.retrieve, + experiments.retrieve, ) self.update = to_streamed_response_wrapper( - experiment_groups.update, + experiments.update, ) self.list = to_streamed_response_wrapper( - experiment_groups.list, + experiments.list, ) self.delete = to_streamed_response_wrapper( - experiment_groups.delete, + experiments.delete, ) -class AsyncExperimentGroupsResourceWithStreamingResponse: - def __init__(self, experiment_groups: AsyncExperimentGroupsResource) -> None: - self._experiment_groups = experiment_groups +class AsyncExperimentsResourceWithStreamingResponse: + def __init__(self, experiments: AsyncExperimentsResource) -> None: + self._experiments = experiments self.create = async_to_streamed_response_wrapper( - experiment_groups.create, + experiments.create, ) self.retrieve = async_to_streamed_response_wrapper( - experiment_groups.retrieve, + experiments.retrieve, ) self.update = async_to_streamed_response_wrapper( - experiment_groups.update, + experiments.update, ) self.list = async_to_streamed_response_wrapper( - experiment_groups.list, + experiments.list, ) self.delete = async_to_streamed_response_wrapper( - experiment_groups.delete, + experiments.delete, ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/api.md index a308e47153..692d703b3f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/api.md +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/api.md @@ -58,7 +58,7 @@ Methods: Types: ```python -from nemo_platform.types.intake import EvaluationContext, ExperimentContext +from nemo_platform.types.intake import EvaluationContext ``` ### Atif diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/atif.py b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/atif.py index 01a5302c4b..2ac8104188 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/atif.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/atif.py @@ -37,7 +37,6 @@ from ....types.intake.ingest.atif_step_param import AtifStepParam from ....types.intake.ingest.atif_agent_param import AtifAgentParam from ....types.intake.evaluation_context_param import EvaluationContextParam -from ....types.intake.experiment_context_param import ExperimentContextParam from ....types.intake.ingest.atif_trajectory_param import AtifTrajectoryParam from ....types.intake.ingest.atif_final_metrics_param import AtifFinalMetricsParam @@ -74,7 +73,6 @@ def create( ], continued_trajectory_ref: str | Omit = omit, evaluation_context: EvaluationContextParam | Omit = omit, - experiment_context: ExperimentContextParam | Omit = omit, extra: Dict[str, object] | Omit = omit, final_metrics: AtifFinalMetricsParam | Omit = omit, notes: str | Omit = omit, @@ -99,9 +97,6 @@ def create( evaluation_run_id, metadata) keeps ingesting without error rather than being rejected. - experiment_context: Deprecated alias for :class:`EvaluationContext`. Producers should send - `evaluation_context`. - extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -123,7 +118,6 @@ def create( "schema_version": schema_version, "continued_trajectory_ref": continued_trajectory_ref, "evaluation_context": evaluation_context, - "experiment_context": experiment_context, "extra": extra, "final_metrics": final_metrics, "notes": notes, @@ -171,7 +165,6 @@ async def create( ], continued_trajectory_ref: str | Omit = omit, evaluation_context: EvaluationContextParam | Omit = omit, - experiment_context: ExperimentContextParam | Omit = omit, extra: Dict[str, object] | Omit = omit, final_metrics: AtifFinalMetricsParam | Omit = omit, notes: str | Omit = omit, @@ -196,9 +189,6 @@ async def create( evaluation_run_id, metadata) keeps ingesting without error rather than being rejected. - experiment_context: Deprecated alias for :class:`EvaluationContext`. Producers should send - `evaluation_context`. - extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -220,7 +210,6 @@ async def create( "schema_version": schema_version, "continued_trajectory_ref": continued_trajectory_ref, "evaluation_context": evaluation_context, - "experiment_context": experiment_context, "extra": extra, "final_metrics": final_metrics, "notes": notes, diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/chat_completions.py b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/chat_completions.py index f6b43af70a..4b455117f4 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/chat_completions.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/chat_completions.py @@ -36,7 +36,6 @@ chat_completion_create_params, ) from ....types.intake.evaluation_context_param import EvaluationContextParam -from ....types.intake.experiment_context_param import ExperimentContextParam from ....types.intake.ingest.chat_completions_ingest_response import ChatCompletionsIngestResponse from ....types.intake.ingest.captured_chat_completions_request_param import CapturedChatCompletionsRequestParam from ....types.intake.ingest.captured_chat_completions_response_param import CapturedChatCompletionsResponseParam @@ -75,7 +74,6 @@ def create( cost_output_usd: float | Omit = omit, cost_usd: float | Omit = omit, evaluation_context: EvaluationContextParam | Omit = omit, - experiment_context: ExperimentContextParam | Omit = omit, provider: str | Omit = omit, session_id: str | Omit = omit, trace_id: str | Omit = omit, @@ -109,9 +107,6 @@ def create( evaluation_run_id, metadata) keeps ingesting without error rather than being rejected. - experiment_context: Deprecated alias for :class:`EvaluationContext`. Producers should send - `evaluation_context`. - session_id: Groups related chat-completions calls without forcing them into the same trace. trace_id: Opt into joining an existing trace built via OTel or ATIF. This is not a @@ -141,7 +136,6 @@ def create( "cost_output_usd": cost_output_usd, "cost_usd": cost_usd, "evaluation_context": evaluation_context, - "experiment_context": experiment_context, "provider": provider, "session_id": session_id, "trace_id": trace_id, @@ -186,7 +180,6 @@ async def create( cost_output_usd: float | Omit = omit, cost_usd: float | Omit = omit, evaluation_context: EvaluationContextParam | Omit = omit, - experiment_context: ExperimentContextParam | Omit = omit, provider: str | Omit = omit, session_id: str | Omit = omit, trace_id: str | Omit = omit, @@ -220,9 +213,6 @@ async def create( evaluation_run_id, metadata) keeps ingesting without error rather than being rejected. - experiment_context: Deprecated alias for :class:`EvaluationContext`. Producers should send - `evaluation_context`. - session_id: Groups related chat-completions calls without forcing them into the same trace. trace_id: Opt into joining an existing trace built via OTel or ATIF. This is not a @@ -252,7 +242,6 @@ async def create( "cost_output_usd": cost_output_usd, "cost_usd": cost_usd, "evaluation_context": evaluation_context, - "experiment_context": experiment_context, "provider": provider, "session_id": session_id, "trace_id": trace_id, diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/traces.py b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/traces.py index 745beb7477..45772b94fb 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/traces.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/traces.py @@ -130,8 +130,7 @@ def list( Args: filter: Filter root-span-backed traces by id, session_id, root status, root span - started_at, evaluation_id (or its deprecated alias experiment_id), and - test_case_id. + started_at, evaluation_id, and test_case_id. mode: Response mode. summary returns root-span fields without payloads or rollups; preview adds token, cost, and span-count rollups plus 300-character input/output @@ -265,8 +264,7 @@ def list( Args: filter: Filter root-span-backed traces by id, session_id, root status, root span - started_at, evaluation_id (or its deprecated alias experiment_id), and - test_case_id. + started_at, evaluation_id, and test_case_id. mode: Response mode. summary returns root-span fields without payloads or rollups; preview adds token, cost, and span-count rollups plus 300-character input/output diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/SKILL.md b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/SKILL.md index c7cd1f31f4..4b4faa537d 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/SKILL.md +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/SKILL.md @@ -1,6 +1,6 @@ --- name: nemo-experiments-upload -description: End-to-end guide for getting evaluation data into NeMo Platform Intake so it shows up as Experiments. Create an Experiment Group, create an Evaluation, then log traces and evaluator results via the ATIF (Harbor), chat-completions, or OTLP ingest endpoint — and view the rollups in Studio. Use when a user wants to upload, log, ingest, publish, or send evaluation runs, agent traces, or scores to NeMo Experiments / Intake. +description: End-to-end guide for getting evaluation data into NeMo Platform Intake so it shows up as Experiments. Create an Experiment, create an Evaluation, then log traces and evaluator results via the ATIF (Harbor), chat-completions, or OTLP ingest endpoint — and view the rollups in Studio. Use when a user wants to upload, log, ingest, publish, or send evaluation runs, agent traces, or scores to NeMo Experiments / Intake. triggers: - log traces to intake - upload experiment results @@ -24,7 +24,7 @@ allowed-tools: [Bash, Read, Write] # Log evaluation data to NeMo Intake -Get evaluation runs into the platform end-to-end: **create an Experiment Group → create an Evaluation → log traces + scores to an ingest endpoint → see the rollups.** The API says "Evaluation" and "Experiment Group"; the whole feature is called **Experiments**. +Get evaluation runs into the platform end-to-end: **create an Experiment → create an Evaluation → log traces + scores to an ingest endpoint → see the rollups.** The API calls the parent (the leaderboard) an **Experiment** and each row an **Evaluation**; the whole feature is called **Experiments**. Everything below uses `${NMP_BASE_URL}` (default `http://localhost:8080`) and a `${WORKSPACE}` (default `default`). All routes are under `/apis/intake/v2/workspaces/${WORKSPACE}`. @@ -48,33 +48,33 @@ fi Run the steps in order. Steps 1–2 create the entities; step 3 logs the data; steps 4–5 verify. -### 1. Create an Experiment Group +### 1. Create an Experiment -A group is the leaderboard container. You need its `id` for the next step. +An Experiment is the leaderboard container. You need its `id` for the next step. ```bash set -euo pipefail : "${NMP_BASE_URL:=http://localhost:8080}" : "${WORKSPACE:=default}" -groups="${NMP_BASE_URL}/apis/intake/v2/workspaces/${WORKSPACE}/experiment-groups" -# Create the group. 201 = created, 409 = already exists; any other status is a real failure. -code=$(curl -s -o /dev/null -w '%{http_code}' -X POST "${groups}" \ +experiments="${NMP_BASE_URL}/apis/intake/v2/workspaces/${WORKSPACE}/experiments" +# Create the experiment. 201 = created, 409 = already exists; any other status is a real failure. +code=$(curl -s -o /dev/null -w '%{http_code}' -X POST "${experiments}" \ -H 'Content-Type: application/json' \ - -d '{"name": "my-experiment-group", "description": "example run"}') + -d '{"name": "my-experiment", "description": "example run"}') case "${code}" in 201|409) ;; - *) echo "group create failed: HTTP ${code}" >&2; exit 1 ;; + *) echo "experiment create failed: HTTP ${code}" >&2; exit 1 ;; esac # Fetch the id (works whether it was just created or already existed). -GROUP_ID=$(curl -sf "${groups}/my-experiment-group" \ +EXPERIMENT_ID=$(curl -sf "${experiments}/my-experiment" \ | python3 -c 'import sys,json; print(json.load(sys.stdin)["id"])') -[ -n "${GROUP_ID}" ] || { echo "could not resolve experiment group id" >&2; exit 1; } -echo "group id: ${GROUP_ID}" +[ -n "${EXPERIMENT_ID}" ] || { echo "could not resolve experiment id" >&2; exit 1; } +echo "experiment id: ${EXPERIMENT_ID}" ``` The POST accepts only `201` (created) or `409` (already exists) — any other status stops the step instead of masking it. The `id` is then read with a GET, so this works on both first run and re-run; -`set -euo pipefail` + the `[ -n ]` guard keep it from continuing with an empty `GROUP_ID`. +`set -euo pipefail` + the `[ -n ]` guard keep it from continuing with an empty `EXPERIMENT_ID`. ### 2. Create an Evaluation @@ -85,13 +85,13 @@ curl -sf -X POST \ "${NMP_BASE_URL}/apis/intake/v2/workspaces/${WORKSPACE}/evaluations" \ -H 'Content-Type: application/json' \ -d "{\"name\": \"my-eval-baseline\", - \"experiment_group_id\": \"${GROUP_ID}\", + \"experiment_ids\": [\"${EXPERIMENT_ID}\"], \"dataset_name\": \"my-dataset\", \"dataset_version\": \"v1\", \"metadata\": {\"model\": \"provider/model\", \"job_name\": \"baseline\"}}" ``` -- `experiment_group_id` is the group's **`id`** (from step 1). +- `experiment_ids` is a list holding the Experiment's **`id`** (from step 1). - `metadata` values must be **strings** (`dict[str, str]`). - **You must create the Evaluation before you can log to it** — ingesting with an unknown `evaluation_id` returns `400 "…must be created before it can be logged."` @@ -128,7 +128,7 @@ curl -sf "${NMP_BASE_URL}/apis/intake/v2/workspaces/${WORKSPACE}/evaluations/my- ### 5. View in Studio -Open Studio → the **Experiments** area (behind the `VITE_FF_EXPERIMENT` flag) → your group → your evaluation. You'll see the leaderboard row with score/cost/latency rollups and can drill into individual sessions and traces. +Open Studio → the **Experiments** area (behind the `VITE_FF_EXPERIMENT` flag) → your experiment → your evaluation. You'll see the leaderboard row with score/cost/latency rollups and can drill into individual sessions and traces. ## Reference files @@ -159,8 +159,8 @@ If `run_count` is 0 after ingesting, the traces didn't associate — almost alwa ## Gotchas - **Create before you log.** The Evaluation entity must exist before any ingest referencing it — otherwise `400`. -- **`evaluation_id` is the Evaluation's `name`, not its entity id.** But **`experiment_group_id` is the group's `id`.** Different identifiers; easy to swap. +- **`evaluation_id` is the Evaluation's `name`, not its entity id.** But **`experiment_ids` holds the Experiment's `id`.** Different identifiers; easy to swap. - **OTLP uses the attribute key `nemo.experiment.id`** (and `nemo.test_case.id`) — the span-attribute key still says "experiment" even though the JSON body field is `evaluation_context`. Set `nemo.experiment.id` on your root span. -- **Use `/evaluations`, not `/experiments`.** `/experiments` still works as a deprecated hidden alias but you should log to `/evaluations`. +- **The parent lives at `/experiments`; `/experiment-groups` is a deprecated hidden alias.** Prefer `/experiments`. Evaluations are created and logged under `/evaluations`. - **`metadata` is `dict[str, str]`** — stringify non-string values or you'll get a `422`. - **ATIF and chat-completions are `extra="forbid"`** (unknown keys → 422); `evaluation_context` itself is lenient (`extra="ignore"`). diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/harbor-quickstart.md b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/harbor-quickstart.md index a3c853f2c7..fbc4a70939 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/harbor-quickstart.md +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/harbor-quickstart.md @@ -8,7 +8,7 @@ each task runs several times (trials). You upload **one ATIF payload per trial** | Harbor concept | NeMo entity | How | |---|---|---| -| A benchmark / sweep | **Experiment Group** | `POST /experiment-groups` once | +| A benchmark / sweep | **Experiment** | `POST /experiments` once | | One agent+config on that benchmark | **Evaluation** | `POST /evaluations` once (its `name` is your `evaluation_id`) | | A task / test case | `test_case_id` | field inside `evaluation_context` | | One trial (attempt) of a task | one ingested **session** | one `POST /ingest/atif` | diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/troubleshooting.md b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/troubleshooting.md index 5e9e7acf62..736edc1a66 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/troubleshooting.md +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/references/troubleshooting.md @@ -3,14 +3,14 @@ Common failures across the create + ingest endpoints, and the fix. Most 4xx bodies include a `detail` string — read it first. -## Create Experiment Group / Evaluation +## Create Experiment / Evaluation | Status | Meaning | Fix | |---|---|---| -| `409` on create group/evaluation | Name already exists in this workspace | Reuse it — `GET .../experiment-groups/{name}` or `.../evaluations/{name}` | -| `400 "…group … does not exist"` on create evaluation | `experiment_group_id` is wrong or the group was deleted | Use the group's **`id`** from the create-group response (not its name) | +| `409` on create experiment/evaluation | Name already exists in this workspace | Reuse it — `GET .../experiments/{name}` or `.../evaluations/{name}` | +| `400 "…experiment … does not exist"` on create evaluation | `experiment_ids` is wrong or the experiment was deleted | Use the Experiment's **`id`** from the create-experiment response (not its name) | | `400 "…parent … does not exist"` | `parent_evaluation_id` doesn't resolve | Omit it, or pass the parent Evaluation's entity **`id`** | -| `422` on create evaluation | Missing required field or non-string metadata value | Required: `name`, `experiment_group_id`, `dataset_name`. `metadata` must be `dict[str, str]` | +| `422` on create evaluation | Missing required field or non-string metadata value | Required: `name`, `experiment_ids`, `dataset_name`. `metadata` must be `dict[str, str]` | ## Ingest (all endpoints) @@ -38,6 +38,6 @@ string — read it first. ## Identifier cheat-sheet (the #1 source of bugs) - `evaluation_context.evaluation_id` → the Evaluation's **`name`**. -- `experiment_group_id` (on create evaluation) → the group's **`id`**. +- `experiment_ids` (on create evaluation) → a list with the Experiment's **`id`**. - OTLP evaluation attribute key → **`nemo.experiment.id`** (test case → `nemo.test_case.id`). -- Log to **`/evaluations`** (the `/experiments` path is a deprecated hidden alias). +- Parent → **`/experiments`** (`/experiment-groups` is a deprecated hidden alias); evaluations → **`/evaluations`**. diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_create_params.py index 3308bbbdcf..5d1ec6142d 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_create_params.py @@ -41,15 +41,15 @@ class EvaluationCreateParams(TypedDict, total=False): """Human-readable description.""" experiment_group_id: str - """Deprecated single-group field; provide experiment_ids instead. + """Deprecated single-experiment field; provide experiment_ids instead. Coalesced into experiment_ids when experiment_ids is omitted. """ experiment_ids: SequenceNotStr[str] - """Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). + """Entity ids of the Experiments this Evaluation belongs to (>=1). - Preferred; each group must already exist. When omitted, the deprecated + Preferred; each experiment must already exist. When omitted, the deprecated experiment_group_id is used instead. """ @@ -62,9 +62,6 @@ class EvaluationCreateParams(TypedDict, total=False): a variant of a baseline), if any. """ - parent_experiment_id: str - """Deprecated alias for parent_evaluation_id.""" - root_cause: str """Human- or agent-authored explanation of the evaluation's outcome (e.g. diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_patch_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_patch_params.py index cdd85f3521..48486dba90 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_patch_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_patch_params.py @@ -32,10 +32,10 @@ class EvaluationPatchParams(TypedDict, total=False): """Human-readable description.""" experiment_ids: SequenceNotStr[str] - """Replace the ExperimentGroups this Evaluation belongs to. + """Replace the Experiments this Evaluation belongs to. - Must be non-empty when provided; each group must already exist. Omit to leave - membership unchanged. + Must be non-empty when provided; each experiment must already exist. Omit to + leave membership unchanged. """ metadata: Dict[str, str] diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py index 7b7c109d1e..3764cccfac 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py @@ -33,16 +33,16 @@ class EvaluationResponse(BaseModel): dataset_name: str experiment_group_id: str - """Deprecated single-group alias; the first of experiment_ids. Use experiment_ids.""" + """Deprecated single-experiment alias; the first of experiment_ids. + + Use experiment_ids. + """ experiment_ids: List[str] - """Entity ids of the ExperimentGroups this Evaluation belongs to (>=1).""" + """Entity ids of the Experiments this Evaluation belongs to (>=1).""" name: str - parent_experiment_id: str - """Deprecated alias for parent_evaluation_id.""" - workspace: str agent_names: Optional[List[str]] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_response.py index b573d939e6..fc453c4add 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_response.py @@ -33,9 +33,6 @@ class EvaluationSessionResponse(BaseModel): evaluation_name: str - experiment_name: str - """Deprecated alias for evaluation_name.""" - root_span_id: str session_id: str diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_update_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_update_params.py index 75dd13dab5..ba431c6404 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_update_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_update_params.py @@ -42,15 +42,15 @@ class EvaluationUpdateParams(TypedDict, total=False): """Human-readable description.""" experiment_group_id: str - """Deprecated single-group field; provide experiment_ids instead. + """Deprecated single-experiment field; provide experiment_ids instead. Coalesced into experiment_ids when experiment_ids is omitted. """ experiment_ids: SequenceNotStr[str] - """Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). + """Entity ids of the Experiments this Evaluation belongs to (>=1). - Preferred; each group must already exist. When omitted, the deprecated + Preferred; each experiment must already exist. When omitted, the deprecated experiment_group_id is used instead. """ @@ -63,9 +63,6 @@ class EvaluationUpdateParams(TypedDict, total=False): a variant of a baseline), if any. """ - parent_experiment_id: str - """Deprecated alias for parent_evaluation_id.""" - root_cause: str """Human- or agent-authored explanation of the evaluation's outcome (e.g. diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/__init__.py similarity index 60% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py rename to sdk/python/nemo-platform/src/nemo_platform/types/experiments/__init__.py index 7fb0bced54..567c8cc163 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/__init__.py @@ -18,10 +18,10 @@ from __future__ import annotations from .pareto_config import ParetoConfig as ParetoConfig +from .experiment_response import ExperimentResponse as ExperimentResponse from .pareto_config_param import ParetoConfigParam as ParetoConfigParam -from .experiment_group_response import ExperimentGroupResponse as ExperimentGroupResponse -from .experiment_group_list_params import ExperimentGroupListParams as ExperimentGroupListParams -from .experiment_group_filter_param import ExperimentGroupFilterParam as ExperimentGroupFilterParam -from .experiment_group_create_params import ExperimentGroupCreateParams as ExperimentGroupCreateParams -from .experiment_group_update_params import ExperimentGroupUpdateParams as ExperimentGroupUpdateParams -from .experiment_group_responses_page import ExperimentGroupResponsesPage as ExperimentGroupResponsesPage +from .experiment_list_params import ExperimentListParams as ExperimentListParams +from .experiment_filter_param import ExperimentFilterParam as ExperimentFilterParam +from .experiment_create_params import ExperimentCreateParams as ExperimentCreateParams +from .experiment_update_params import ExperimentUpdateParams as ExperimentUpdateParams +from .experiment_responses_page import ExperimentResponsesPage as ExperimentResponsesPage diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py similarity index 79% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py rename to sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py index ffc2c29860..6f1821d986 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py @@ -22,18 +22,18 @@ from .pareto_config_param import ParetoConfigParam -__all__ = ["ExperimentGroupCreateParams"] +__all__ = ["ExperimentCreateParams"] -class ExperimentGroupCreateParams(TypedDict, total=False): +class ExperimentCreateParams(TypedDict, total=False): workspace: str name: Required[str] - """Workspace-unique group name.""" + """Workspace-unique experiment name.""" default_sort: str """ - Default sort for this group's evaluations list, as a `sort`-param string: a + Default sort for this experiment's evaluations list, as a `sort`-param string: a comma-separated, ordered list of fields where the first is the primary sort and the rest break ties (leading '-' on a field = descending), e.g. '-evaluators.reward.mean,cost_usd.mean'. Defaults to '-created_at'. Accepts any @@ -42,13 +42,13 @@ class ExperimentGroupCreateParams(TypedDict, total=False): """ description: str - """Human-readable purpose of the group.""" + """Human-readable purpose of the experiment.""" insight_id: str - """Reference to an external insight that seeded this group, if any.""" + """Reference to an external insight that seeded this experiment, if any.""" metadata: Dict[str, str] - """Free-form producer metadata for the group.""" + """Free-form producer metadata for the experiment.""" pareto: ParetoConfigParam """Default X/Y metrics for a group's cost-vs-accuracy Pareto view. @@ -60,4 +60,4 @@ class ExperimentGroupCreateParams(TypedDict, total=False): """ summary: str - """Human- or agent-authored summary of the group's findings.""" + """Human- or agent-authored summary of the experiment's findings.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_filter_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_filter_param.py similarity index 75% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_filter_param.py rename to sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_filter_param.py index b7eef3e8a5..f360eba65a 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_filter_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_filter_param.py @@ -20,19 +20,19 @@ from typing import Dict from typing_extensions import TypedDict -__all__ = ["ExperimentGroupFilterParam"] +__all__ = ["ExperimentFilterParam"] -class ExperimentGroupFilterParam(TypedDict, total=False): - """Filter for listing ExperimentGroups.""" +class ExperimentFilterParam(TypedDict, total=False): + """Filter for listing Experiments.""" insight_id: str - """Filter groups by the id of the insight that seeded them.""" + """Filter experiments by the id of the insight that seeded them.""" is_deleted: bool - """When true, returns only soft-deleted groups. + """When true, returns only soft-deleted experiments. - Omit (or false) to see only live groups. + Omit (or false) to see only live experiments. """ metadata: Dict[str, str] @@ -42,4 +42,4 @@ class ExperimentGroupFilterParam(TypedDict, total=False): """ name: str - """Filter groups by name.""" + """Filter experiments by name.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_list_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py similarity index 72% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_list_params.py rename to sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py index 2acfe15417..1b71112ffb 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_list_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py @@ -19,19 +19,19 @@ from typing_extensions import Literal, TypedDict -from .experiment_group_filter_param import ExperimentGroupFilterParam +from .experiment_filter_param import ExperimentFilterParam -__all__ = ["ExperimentGroupListParams"] +__all__ = ["ExperimentListParams"] -class ExperimentGroupListParams(TypedDict, total=False): +class ExperimentListParams(TypedDict, total=False): workspace: str - filter: ExperimentGroupFilterParam + filter: ExperimentFilterParam """ - Filter experiment groups by name, insight_id, is_deleted, or a metadata - key/value (filter[metadata.]=). Pass is_deleted=true to return only - soft-deleted groups; omit to see only live ones. + Filter experiments by name, insight_id, is_deleted, or a metadata key/value + (filter[metadata.]=). Pass is_deleted=true to return only + soft-deleted experiments; omit to see only live ones. """ page: int diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py similarity index 85% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py rename to sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py index ccc20c6731..fef8034a91 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py @@ -21,19 +21,16 @@ from ..._models import BaseModel from .pareto_config import ParetoConfig -__all__ = ["ExperimentGroupResponse"] +__all__ = ["ExperimentResponse"] -class ExperimentGroupResponse(BaseModel): - """ExperimentGroup as served by the API.""" +class ExperimentResponse(BaseModel): + """Experiment as served by the API.""" id: str default_sort: str - experiment_count: int - """Deprecated alias for evaluation_count.""" - name: str workspace: str @@ -43,7 +40,7 @@ class ExperimentGroupResponse(BaseModel): description: Optional[str] = None evaluation_count: Optional[int] = None - """Number of live (non-soft-deleted) evaluations in this group.""" + """Number of live (non-soft-deleted) evaluations in this experiment.""" insight_id: Optional[str] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_responses_page.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_responses_page.py similarity index 85% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_responses_page.py rename to sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_responses_page.py index ad7fa77877..ad2f49105e 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_responses_page.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_responses_page.py @@ -18,14 +18,14 @@ from typing import Dict, List, Optional from ..._models import BaseModel +from .experiment_response import ExperimentResponse from ..shared.pagination_data import PaginationData -from .experiment_group_response import ExperimentGroupResponse -__all__ = ["ExperimentGroupResponsesPage"] +__all__ = ["ExperimentResponsesPage"] -class ExperimentGroupResponsesPage(BaseModel): - data: List[ExperimentGroupResponse] +class ExperimentResponsesPage(BaseModel): + data: List[ExperimentResponse] filter: Optional[Dict[str, object]] = None """Filtering information.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py similarity index 79% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py rename to sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py index f5e7d241e9..986c3db7bf 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py @@ -23,18 +23,18 @@ from ..._utils import PropertyInfo from .pareto_config_param import ParetoConfigParam -__all__ = ["ExperimentGroupUpdateParams"] +__all__ = ["ExperimentUpdateParams"] -class ExperimentGroupUpdateParams(TypedDict, total=False): +class ExperimentUpdateParams(TypedDict, total=False): workspace: str body_name: Required[Annotated[str, PropertyInfo(alias="name")]] - """Workspace-unique group name.""" + """Workspace-unique experiment name.""" default_sort: str """ - Default sort for this group's evaluations list, as a `sort`-param string: a + Default sort for this experiment's evaluations list, as a `sort`-param string: a comma-separated, ordered list of fields where the first is the primary sort and the rest break ties (leading '-' on a field = descending), e.g. '-evaluators.reward.mean,cost_usd.mean'. Defaults to '-created_at'. Accepts any @@ -43,13 +43,13 @@ class ExperimentGroupUpdateParams(TypedDict, total=False): """ description: str - """Human-readable purpose of the group.""" + """Human-readable purpose of the experiment.""" insight_id: str - """Reference to an external insight that seeded this group, if any.""" + """Reference to an external insight that seeded this experiment, if any.""" metadata: Dict[str, str] - """Free-form producer metadata for the group.""" + """Free-form producer metadata for the experiment.""" pareto: ParetoConfigParam """Default X/Y metrics for a group's cost-vs-accuracy Pareto view. @@ -61,4 +61,4 @@ class ExperimentGroupUpdateParams(TypedDict, total=False): """ summary: str - """Human- or agent-authored summary of the group's findings.""" + """Human- or agent-authored summary of the experiment's findings.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/pareto_config.py similarity index 100% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config.py rename to sdk/python/nemo-platform/src/nemo_platform/types/experiments/pareto_config.py diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/pareto_config_param.py similarity index 100% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config_param.py rename to sdk/python/nemo-platform/src/nemo_platform/types/experiments/pareto_config_param.py diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/__init__.py index 45dc3d2f5f..1b2906c26b 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/__init__.py @@ -37,7 +37,6 @@ from .span_filter_param import SpanFilterParam as SpanFilterParam from .trace_list_params import TraceListParams as TraceListParams from .evaluation_context import EvaluationContext as EvaluationContext -from .experiment_context import ExperimentContext as ExperimentContext from .float_filter_param import FloatFilterParam as FloatFilterParam from .trace_filter_param import TraceFilterParam as TraceFilterParam from .feedback_annotation import FeedbackAnnotation as FeedbackAnnotation @@ -53,7 +52,6 @@ from .span_evaluation_context import SpanEvaluationContext as SpanEvaluationContext from .annotation_create_params import AnnotationCreateParams as AnnotationCreateParams from .evaluation_context_param import EvaluationContextParam as EvaluationContextParam -from .experiment_context_param import ExperimentContextParam as ExperimentContextParam from .feedback_annotation_param import FeedbackAnnotationParam as FeedbackAnnotationParam from .metadata_annotation_param import MetadataAnnotationParam as MetadataAnnotationParam from .evaluator_result_data_type import EvaluatorResultDataType as EvaluatorResultDataType diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/experiment_context.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/experiment_context.py deleted file mode 100644 index 3b2280733d..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/experiment_context.py +++ /dev/null @@ -1,35 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -from typing import Optional - -from ..._models import BaseModel - -__all__ = ["ExperimentContext"] - - -class ExperimentContext(BaseModel): - """Deprecated alias for :class:`EvaluationContext`. - - Producers should send ``evaluation_context``. - """ - - experiment_id: str - """Name of an existing Experiment entity.""" - - test_case_id: Optional[str] = None - """Optional producer-supplied test case id.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/atif_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/atif_create_params.py index 0db4bbd0ab..59dd0083a9 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/atif_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/atif_create_params.py @@ -24,7 +24,6 @@ from .atif_agent_param import AtifAgentParam from .atif_final_metrics_param import AtifFinalMetricsParam from ..evaluation_context_param import EvaluationContextParam -from ..experiment_context_param import ExperimentContextParam __all__ = ["AtifCreateParams"] @@ -48,12 +47,6 @@ class AtifCreateParams(TypedDict, total=False): rejected. """ - experiment_context: ExperimentContextParam - """Deprecated alias for :class:`EvaluationContext`. - - Producers should send `evaluation_context`. - """ - extra: Dict[str, object] final_metrics: AtifFinalMetricsParam diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/chat_completion_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/chat_completion_create_params.py index 9a30297c6a..3537849b81 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/chat_completion_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/chat_completion_create_params.py @@ -21,7 +21,6 @@ from typing_extensions import Required, TypedDict from ..evaluation_context_param import EvaluationContextParam -from ..experiment_context_param import ExperimentContextParam from .captured_chat_completions_request_param import CapturedChatCompletionsRequestParam from .captured_chat_completions_response_param import CapturedChatCompletionsResponseParam @@ -61,12 +60,6 @@ class ChatCompletionCreateParams(TypedDict, total=False): rejected. """ - experiment_context: ExperimentContextParam - """Deprecated alias for :class:`EvaluationContext`. - - Producers should send `evaluation_context`. - """ - provider: str session_id: str diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace.py index 284feba451..e5ababe62f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace.py @@ -21,7 +21,6 @@ from ..._models import BaseModel from .span_status import SpanStatus from .evaluation_context import EvaluationContext -from .experiment_context import ExperimentContext __all__ = ["Trace"] @@ -59,12 +58,6 @@ class Trace(BaseModel): rejected. """ - experiment_context: Optional[ExperimentContext] = None - """Deprecated alias for :class:`EvaluationContext`. - - Producers should send `evaluation_context`. - """ - input: Optional[str] = None """Root-span input text. diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_filter_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_filter_param.py index 78e3b2d82c..a18bc2e3b3 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_filter_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_filter_param.py @@ -32,9 +32,6 @@ class TraceFilterParam(TypedDict, total=False): evaluation_id: str """Filter by root-span evaluation id.""" - experiment_id: str - """Deprecated alias for evaluation_id. Filter by root-span evaluation id.""" - session_id: str """Filter by session id.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_list_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_list_params.py index a00009f120..6587d40d33 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_list_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_list_params.py @@ -31,8 +31,7 @@ class TraceListParams(TypedDict, total=False): filter: TraceFilterParam """ Filter root-span-backed traces by id, session_id, root status, root span - started_at, evaluation_id (or its deprecated alias experiment_id), and - test_case_id. + started_at, evaluation_id, and test_case_id. """ mode: Literal["summary", "preview", "detailed"] diff --git a/sdk/python/nemo-platform/tests/api_resources/experiment_groups/__init__.py b/sdk/python/nemo-platform/tests/api_resources/experiments/__init__.py similarity index 100% rename from sdk/python/nemo-platform/tests/api_resources/experiment_groups/__init__.py rename to sdk/python/nemo-platform/tests/api_resources/experiments/__init__.py diff --git a/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_atif.py b/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_atif.py index bee953a576..00c2dee527 100644 --- a/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_atif.py +++ b/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_atif.py @@ -62,10 +62,6 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: "evaluation_id": "evaluation_id", "test_case_id": "test_case_id", }, - experiment_context={ - "experiment_id": "experiment_id", - "test_case_id": "test_case_id", - }, extra={"foo": "bar"}, final_metrics={ "extra": {"foo": "bar"}, @@ -219,10 +215,6 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo "evaluation_id": "evaluation_id", "test_case_id": "test_case_id", }, - experiment_context={ - "experiment_id": "experiment_id", - "test_case_id": "test_case_id", - }, extra={"foo": "bar"}, final_metrics={ "extra": {"foo": "bar"}, diff --git a/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_chat_completions.py b/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_chat_completions.py index b92475aac6..4c651481d6 100644 --- a/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_chat_completions.py +++ b/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_chat_completions.py @@ -68,10 +68,6 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: "evaluation_id": "evaluation_id", "test_case_id": "test_case_id", }, - experiment_context={ - "experiment_id": "experiment_id", - "test_case_id": "test_case_id", - }, provider="provider", session_id="session_id", trace_id="trace_id", @@ -167,10 +163,6 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo "evaluation_id": "evaluation_id", "test_case_id": "test_case_id", }, - experiment_context={ - "experiment_id": "experiment_id", - "test_case_id": "test_case_id", - }, provider="provider", session_id="session_id", trace_id="trace_id", diff --git a/sdk/python/nemo-platform/tests/api_resources/intake/test_traces.py b/sdk/python/nemo-platform/tests/api_resources/intake/test_traces.py index dff3f52980..f77bf6ea7e 100644 --- a/sdk/python/nemo-platform/tests/api_resources/intake/test_traces.py +++ b/sdk/python/nemo-platform/tests/api_resources/intake/test_traces.py @@ -112,7 +112,6 @@ def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: filter={ "id": "id", "evaluation_id": "evaluation_id", - "experiment_id": "experiment_id", "session_id": "session_id", "started_at": { "gte": parse_datetime("2019-12-27T18:11:19.117Z"), @@ -246,7 +245,6 @@ async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform filter={ "id": "id", "evaluation_id": "evaluation_id", - "experiment_id": "experiment_id", "session_id": "session_id", "started_at": { "gte": parse_datetime("2019-12-27T18:11:19.117Z"), diff --git a/sdk/python/nemo-platform/tests/api_resources/test_evaluations.py b/sdk/python/nemo-platform/tests/api_resources/test_evaluations.py index bc6ce70343..1623e172d3 100644 --- a/sdk/python/nemo-platform/tests/api_resources/test_evaluations.py +++ b/sdk/python/nemo-platform/tests/api_resources/test_evaluations.py @@ -59,7 +59,6 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: experiment_ids=["string"], metadata={"foo": "string"}, parent_evaluation_id="parent_evaluation_id", - parent_experiment_id="parent_experiment_id", root_cause="root_cause", source_link="https://example.com", status="status", @@ -183,7 +182,6 @@ def test_method_update_with_all_params(self, client: NeMoPlatform) -> None: experiment_ids=["string"], metadata={"foo": "string"}, parent_evaluation_id="parent_evaluation_id", - parent_experiment_id="parent_experiment_id", root_cause="root_cause", source_link="https://example.com", status="status", @@ -779,7 +777,6 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo experiment_ids=["string"], metadata={"foo": "string"}, parent_evaluation_id="parent_evaluation_id", - parent_experiment_id="parent_experiment_id", root_cause="root_cause", source_link="https://example.com", status="status", @@ -903,7 +900,6 @@ async def test_method_update_with_all_params(self, async_client: AsyncNeMoPlatfo experiment_ids=["string"], metadata={"foo": "string"}, parent_evaluation_id="parent_evaluation_id", - parent_experiment_id="parent_experiment_id", root_cause="root_cause", source_link="https://example.com", status="status", diff --git a/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py b/sdk/python/nemo-platform/tests/api_resources/test_experiments.py similarity index 71% rename from sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py rename to sdk/python/nemo-platform/tests/api_resources/test_experiments.py index 673df6bcc2..0327929115 100644 --- a/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py +++ b/sdk/python/nemo-platform/tests/api_resources/test_experiments.py @@ -25,29 +25,29 @@ from tests.utils import assert_matches_type from nemo_platform import NeMoPlatform, AsyncNeMoPlatform from nemo_platform.pagination import SyncDefaultPagination, AsyncDefaultPagination -from nemo_platform.types.experiment_groups import ( - ExperimentGroupResponse, +from nemo_platform.types.experiments import ( + ExperimentResponse, ) base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") -class TestExperimentGroups: +class TestExperiments: parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_create(self, client: NeMoPlatform) -> None: - experiment_group = client.experiment_groups.create( + experiment = client.experiments.create( workspace="workspace", name="name", ) - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: - experiment_group = client.experiment_groups.create( + experiment = client.experiments.create( workspace="workspace", name="name", default_sort="default_sort", @@ -60,33 +60,33 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: }, summary="summary", ) - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_raw_response_create(self, client: NeMoPlatform) -> None: - response = client.experiment_groups.with_raw_response.create( + response = client.experiments.with_raw_response.create( workspace="workspace", name="name", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = response.parse() - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + experiment = response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_streaming_response_create(self, client: NeMoPlatform) -> None: - with client.experiment_groups.with_streaming_response.create( + with client.experiments.with_streaming_response.create( workspace="workspace", name="name", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = response.parse() - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + experiment = response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) assert cast(Any, response.is_closed) is True @@ -94,7 +94,7 @@ def test_streaming_response_create(self, client: NeMoPlatform) -> None: @parametrize def test_path_params_create(self, client: NeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - client.experiment_groups.with_raw_response.create( + client.experiments.with_raw_response.create( workspace="", name="name", ) @@ -102,37 +102,37 @@ def test_path_params_create(self, client: NeMoPlatform) -> None: @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_retrieve(self, client: NeMoPlatform) -> None: - experiment_group = client.experiment_groups.retrieve( + experiment = client.experiments.retrieve( name="name", workspace="workspace", ) - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_raw_response_retrieve(self, client: NeMoPlatform) -> None: - response = client.experiment_groups.with_raw_response.retrieve( + response = client.experiments.with_raw_response.retrieve( name="name", workspace="workspace", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = response.parse() - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + experiment = response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_streaming_response_retrieve(self, client: NeMoPlatform) -> None: - with client.experiment_groups.with_streaming_response.retrieve( + with client.experiments.with_streaming_response.retrieve( name="name", workspace="workspace", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = response.parse() - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + experiment = response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) assert cast(Any, response.is_closed) is True @@ -140,13 +140,13 @@ def test_streaming_response_retrieve(self, client: NeMoPlatform) -> None: @parametrize def test_path_params_retrieve(self, client: NeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - client.experiment_groups.with_raw_response.retrieve( + client.experiments.with_raw_response.retrieve( name="name", workspace="", ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): - client.experiment_groups.with_raw_response.retrieve( + client.experiments.with_raw_response.retrieve( name="", workspace="workspace", ) @@ -154,17 +154,17 @@ def test_path_params_retrieve(self, client: NeMoPlatform) -> None: @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_update(self, client: NeMoPlatform) -> None: - experiment_group = client.experiment_groups.update( + experiment = client.experiments.update( path_name="name", workspace="workspace", body_name="name", ) - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_update_with_all_params(self, client: NeMoPlatform) -> None: - experiment_group = client.experiment_groups.update( + experiment = client.experiments.update( path_name="name", workspace="workspace", body_name="name", @@ -178,12 +178,12 @@ def test_method_update_with_all_params(self, client: NeMoPlatform) -> None: }, summary="summary", ) - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_raw_response_update(self, client: NeMoPlatform) -> None: - response = client.experiment_groups.with_raw_response.update( + response = client.experiments.with_raw_response.update( path_name="name", workspace="workspace", body_name="name", @@ -191,13 +191,13 @@ def test_raw_response_update(self, client: NeMoPlatform) -> None: assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = response.parse() - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + experiment = response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_streaming_response_update(self, client: NeMoPlatform) -> None: - with client.experiment_groups.with_streaming_response.update( + with client.experiments.with_streaming_response.update( path_name="name", workspace="workspace", body_name="name", @@ -205,8 +205,8 @@ def test_streaming_response_update(self, client: NeMoPlatform) -> None: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = response.parse() - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + experiment = response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) assert cast(Any, response.is_closed) is True @@ -214,14 +214,14 @@ def test_streaming_response_update(self, client: NeMoPlatform) -> None: @parametrize def test_path_params_update(self, client: NeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - client.experiment_groups.with_raw_response.update( + client.experiments.with_raw_response.update( path_name="name", workspace="", body_name="name", ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `path_name` but received ''"): - client.experiment_groups.with_raw_response.update( + client.experiments.with_raw_response.update( path_name="", workspace="workspace", body_name="name", @@ -230,15 +230,15 @@ def test_path_params_update(self, client: NeMoPlatform) -> None: @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_list(self, client: NeMoPlatform) -> None: - experiment_group = client.experiment_groups.list( + experiment = client.experiments.list( workspace="workspace", ) - assert_matches_type(SyncDefaultPagination[ExperimentGroupResponse], experiment_group, path=["response"]) + assert_matches_type(SyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: - experiment_group = client.experiment_groups.list( + experiment = client.experiments.list( workspace="workspace", filter={ "insight_id": "insight_id", @@ -250,31 +250,31 @@ def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: page_size=1, sort="-created_at", ) - assert_matches_type(SyncDefaultPagination[ExperimentGroupResponse], experiment_group, path=["response"]) + assert_matches_type(SyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_raw_response_list(self, client: NeMoPlatform) -> None: - response = client.experiment_groups.with_raw_response.list( + response = client.experiments.with_raw_response.list( workspace="workspace", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = response.parse() - assert_matches_type(SyncDefaultPagination[ExperimentGroupResponse], experiment_group, path=["response"]) + experiment = response.parse() + assert_matches_type(SyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_streaming_response_list(self, client: NeMoPlatform) -> None: - with client.experiment_groups.with_streaming_response.list( + with client.experiments.with_streaming_response.list( workspace="workspace", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = response.parse() - assert_matches_type(SyncDefaultPagination[ExperimentGroupResponse], experiment_group, path=["response"]) + experiment = response.parse() + assert_matches_type(SyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) assert cast(Any, response.is_closed) is True @@ -282,44 +282,44 @@ def test_streaming_response_list(self, client: NeMoPlatform) -> None: @parametrize def test_path_params_list(self, client: NeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - client.experiment_groups.with_raw_response.list( + client.experiments.with_raw_response.list( workspace="", ) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_delete(self, client: NeMoPlatform) -> None: - experiment_group = client.experiment_groups.delete( + experiment = client.experiments.delete( name="name", workspace="workspace", ) - assert experiment_group is None + assert experiment is None @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_raw_response_delete(self, client: NeMoPlatform) -> None: - response = client.experiment_groups.with_raw_response.delete( + response = client.experiments.with_raw_response.delete( name="name", workspace="workspace", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = response.parse() - assert experiment_group is None + experiment = response.parse() + assert experiment is None @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_streaming_response_delete(self, client: NeMoPlatform) -> None: - with client.experiment_groups.with_streaming_response.delete( + with client.experiments.with_streaming_response.delete( name="name", workspace="workspace", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = response.parse() - assert experiment_group is None + experiment = response.parse() + assert experiment is None assert cast(Any, response.is_closed) is True @@ -327,19 +327,19 @@ def test_streaming_response_delete(self, client: NeMoPlatform) -> None: @parametrize def test_path_params_delete(self, client: NeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - client.experiment_groups.with_raw_response.delete( + client.experiments.with_raw_response.delete( name="name", workspace="", ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): - client.experiment_groups.with_raw_response.delete( + client.experiments.with_raw_response.delete( name="", workspace="workspace", ) -class TestAsyncExperimentGroups: +class TestAsyncExperiments: parametrize = pytest.mark.parametrize( "async_client", [False, True, {"http_client": "aiohttp"}], indirect=True, ids=["loose", "strict", "aiohttp"] ) @@ -347,16 +347,16 @@ class TestAsyncExperimentGroups: @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_create(self, async_client: AsyncNeMoPlatform) -> None: - experiment_group = await async_client.experiment_groups.create( + experiment = await async_client.experiments.create( workspace="workspace", name="name", ) - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatform) -> None: - experiment_group = await async_client.experiment_groups.create( + experiment = await async_client.experiments.create( workspace="workspace", name="name", default_sort="default_sort", @@ -369,33 +369,33 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo }, summary="summary", ) - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_raw_response_create(self, async_client: AsyncNeMoPlatform) -> None: - response = await async_client.experiment_groups.with_raw_response.create( + response = await async_client.experiments.with_raw_response.create( workspace="workspace", name="name", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = await response.parse() - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + experiment = await response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_streaming_response_create(self, async_client: AsyncNeMoPlatform) -> None: - async with async_client.experiment_groups.with_streaming_response.create( + async with async_client.experiments.with_streaming_response.create( workspace="workspace", name="name", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = await response.parse() - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + experiment = await response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) assert cast(Any, response.is_closed) is True @@ -403,7 +403,7 @@ async def test_streaming_response_create(self, async_client: AsyncNeMoPlatform) @parametrize async def test_path_params_create(self, async_client: AsyncNeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - await async_client.experiment_groups.with_raw_response.create( + await async_client.experiments.with_raw_response.create( workspace="", name="name", ) @@ -411,37 +411,37 @@ async def test_path_params_create(self, async_client: AsyncNeMoPlatform) -> None @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_retrieve(self, async_client: AsyncNeMoPlatform) -> None: - experiment_group = await async_client.experiment_groups.retrieve( + experiment = await async_client.experiments.retrieve( name="name", workspace="workspace", ) - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_raw_response_retrieve(self, async_client: AsyncNeMoPlatform) -> None: - response = await async_client.experiment_groups.with_raw_response.retrieve( + response = await async_client.experiments.with_raw_response.retrieve( name="name", workspace="workspace", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = await response.parse() - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + experiment = await response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_streaming_response_retrieve(self, async_client: AsyncNeMoPlatform) -> None: - async with async_client.experiment_groups.with_streaming_response.retrieve( + async with async_client.experiments.with_streaming_response.retrieve( name="name", workspace="workspace", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = await response.parse() - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + experiment = await response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) assert cast(Any, response.is_closed) is True @@ -449,13 +449,13 @@ async def test_streaming_response_retrieve(self, async_client: AsyncNeMoPlatform @parametrize async def test_path_params_retrieve(self, async_client: AsyncNeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - await async_client.experiment_groups.with_raw_response.retrieve( + await async_client.experiments.with_raw_response.retrieve( name="name", workspace="", ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): - await async_client.experiment_groups.with_raw_response.retrieve( + await async_client.experiments.with_raw_response.retrieve( name="", workspace="workspace", ) @@ -463,17 +463,17 @@ async def test_path_params_retrieve(self, async_client: AsyncNeMoPlatform) -> No @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_update(self, async_client: AsyncNeMoPlatform) -> None: - experiment_group = await async_client.experiment_groups.update( + experiment = await async_client.experiments.update( path_name="name", workspace="workspace", body_name="name", ) - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_update_with_all_params(self, async_client: AsyncNeMoPlatform) -> None: - experiment_group = await async_client.experiment_groups.update( + experiment = await async_client.experiments.update( path_name="name", workspace="workspace", body_name="name", @@ -487,12 +487,12 @@ async def test_method_update_with_all_params(self, async_client: AsyncNeMoPlatfo }, summary="summary", ) - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_raw_response_update(self, async_client: AsyncNeMoPlatform) -> None: - response = await async_client.experiment_groups.with_raw_response.update( + response = await async_client.experiments.with_raw_response.update( path_name="name", workspace="workspace", body_name="name", @@ -500,13 +500,13 @@ async def test_raw_response_update(self, async_client: AsyncNeMoPlatform) -> Non assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = await response.parse() - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + experiment = await response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_streaming_response_update(self, async_client: AsyncNeMoPlatform) -> None: - async with async_client.experiment_groups.with_streaming_response.update( + async with async_client.experiments.with_streaming_response.update( path_name="name", workspace="workspace", body_name="name", @@ -514,8 +514,8 @@ async def test_streaming_response_update(self, async_client: AsyncNeMoPlatform) assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = await response.parse() - assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + experiment = await response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) assert cast(Any, response.is_closed) is True @@ -523,14 +523,14 @@ async def test_streaming_response_update(self, async_client: AsyncNeMoPlatform) @parametrize async def test_path_params_update(self, async_client: AsyncNeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - await async_client.experiment_groups.with_raw_response.update( + await async_client.experiments.with_raw_response.update( path_name="name", workspace="", body_name="name", ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `path_name` but received ''"): - await async_client.experiment_groups.with_raw_response.update( + await async_client.experiments.with_raw_response.update( path_name="", workspace="workspace", body_name="name", @@ -539,15 +539,15 @@ async def test_path_params_update(self, async_client: AsyncNeMoPlatform) -> None @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_list(self, async_client: AsyncNeMoPlatform) -> None: - experiment_group = await async_client.experiment_groups.list( + experiment = await async_client.experiments.list( workspace="workspace", ) - assert_matches_type(AsyncDefaultPagination[ExperimentGroupResponse], experiment_group, path=["response"]) + assert_matches_type(AsyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform) -> None: - experiment_group = await async_client.experiment_groups.list( + experiment = await async_client.experiments.list( workspace="workspace", filter={ "insight_id": "insight_id", @@ -559,31 +559,31 @@ async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform page_size=1, sort="-created_at", ) - assert_matches_type(AsyncDefaultPagination[ExperimentGroupResponse], experiment_group, path=["response"]) + assert_matches_type(AsyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_raw_response_list(self, async_client: AsyncNeMoPlatform) -> None: - response = await async_client.experiment_groups.with_raw_response.list( + response = await async_client.experiments.with_raw_response.list( workspace="workspace", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = await response.parse() - assert_matches_type(AsyncDefaultPagination[ExperimentGroupResponse], experiment_group, path=["response"]) + experiment = await response.parse() + assert_matches_type(AsyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_streaming_response_list(self, async_client: AsyncNeMoPlatform) -> None: - async with async_client.experiment_groups.with_streaming_response.list( + async with async_client.experiments.with_streaming_response.list( workspace="workspace", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = await response.parse() - assert_matches_type(AsyncDefaultPagination[ExperimentGroupResponse], experiment_group, path=["response"]) + experiment = await response.parse() + assert_matches_type(AsyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) assert cast(Any, response.is_closed) is True @@ -591,44 +591,44 @@ async def test_streaming_response_list(self, async_client: AsyncNeMoPlatform) -> @parametrize async def test_path_params_list(self, async_client: AsyncNeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - await async_client.experiment_groups.with_raw_response.list( + await async_client.experiments.with_raw_response.list( workspace="", ) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_delete(self, async_client: AsyncNeMoPlatform) -> None: - experiment_group = await async_client.experiment_groups.delete( + experiment = await async_client.experiments.delete( name="name", workspace="workspace", ) - assert experiment_group is None + assert experiment is None @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_raw_response_delete(self, async_client: AsyncNeMoPlatform) -> None: - response = await async_client.experiment_groups.with_raw_response.delete( + response = await async_client.experiments.with_raw_response.delete( name="name", workspace="workspace", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = await response.parse() - assert experiment_group is None + experiment = await response.parse() + assert experiment is None @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_streaming_response_delete(self, async_client: AsyncNeMoPlatform) -> None: - async with async_client.experiment_groups.with_streaming_response.delete( + async with async_client.experiments.with_streaming_response.delete( name="name", workspace="workspace", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = await response.parse() - assert experiment_group is None + experiment = await response.parse() + assert experiment is None assert cast(Any, response.is_closed) is True @@ -636,13 +636,13 @@ async def test_streaming_response_delete(self, async_client: AsyncNeMoPlatform) @parametrize async def test_path_params_delete(self, async_client: AsyncNeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - await async_client.experiment_groups.with_raw_response.delete( + await async_client.experiments.with_raw_response.delete( name="name", workspace="", ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): - await async_client.experiment_groups.with_raw_response.delete( + await async_client.experiments.with_raw_response.delete( name="", workspace="workspace", ) diff --git a/sdk/stainless.yaml b/sdk/stainless.yaml index 438911b5ef..02a0898d9d 100644 --- a/sdk/stainless.yaml +++ b/sdk/stainless.yaml @@ -832,7 +832,6 @@ resources: ingest: models: evaluation_context: EvaluationContext - experiment_context: ExperimentContext subresources: atif: models: @@ -912,20 +911,6 @@ resources: methods: list: get /apis/intake/v2/workspaces/{workspace}/traces retrieve: get /apis/intake/v2/workspaces/{workspace}/traces/{id} - experiment_groups: - standalone_api: true - models: - experiment_group_filter: ExperimentGroupFilter - experiment_group_request: ExperimentGroupRequest - experiment_group_response: ExperimentGroupResponse - experiment_group_responses_page: ExperimentGroupResponsesPage - pareto_config: ParetoConfig - methods: - create: post /apis/intake/v2/workspaces/{workspace}/experiment-groups - list: get /apis/intake/v2/workspaces/{workspace}/experiment-groups - retrieve: get /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} - update: put /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} - delete: delete /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} evaluations: standalone_api: true models: @@ -954,3 +939,17 @@ resources: evaluation_session_responses_page: EvaluationSessionResponsesPage methods: list: get /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/sessions + experiments: + standalone_api: true + models: + experiment_filter: ExperimentFilter + experiment_request: ExperimentRequest + experiment_response: ExperimentResponse + experiment_responses_page: ExperimentResponsesPage + pareto_config: ParetoConfig + methods: + create: post /apis/intake/v2/workspaces/{workspace}/experiments + list: get /apis/intake/v2/workspaces/{workspace}/experiments + retrieve: get /apis/intake/v2/workspaces/{workspace}/experiments/{name} + update: put /apis/intake/v2/workspaces/{workspace}/experiments/{name} + delete: delete /apis/intake/v2/workspaces/{workspace}/experiments/{name} diff --git a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml index 13133a13e8..1e867f5e42 100644 --- a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml +++ b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml @@ -128,15 +128,15 @@ authz: description: "Read intake annotations" delete: description: "Delete intake annotations" - experiment-groups: + experiments: create: - description: "Create intake experiment groups" + description: "Create intake experiments" delete: - description: "Delete intake experiment groups" + description: "Delete intake experiments" read: - description: "Read intake experiment groups" + description: "Read intake experiments" update: - description: "Update intake experiment groups" + description: "Update intake experiments" evaluations: create: description: "Create intake evaluations" @@ -308,7 +308,7 @@ authz: - intake.evaluations.read - intake.evaluator-results.list - intake.evaluator-results.read - - intake.experiment-groups.read + - intake.experiments.read - intake.sessions.read - intake.spans.list - intake.spans.read @@ -357,9 +357,9 @@ authz: - intake.evaluations.delete - intake.evaluations.update - intake.evaluator-results.create - - intake.experiment-groups.create - - intake.experiment-groups.delete - - intake.experiment-groups.update + - intake.experiments.create + - intake.experiments.delete + - intake.experiments.update - intake.ingest.create - jobs.cancel - jobs.create @@ -971,35 +971,76 @@ authz: scopes: - intake:read - platform:read + /apis/intake/v2/workspaces/{workspace}/experiments: + get: + permissions: + - intake.experiments.read + scopes: + - intake:read + - platform:read + post: + permissions: + - intake.experiments.create + scopes: + - intake:write + - platform:write + /apis/intake/v2/workspaces/{workspace}/experiments/{name}: + delete: + permissions: + - intake.experiments.delete + scopes: + - intake:write + - platform:write + get: + permissions: + - intake.experiments.read + scopes: + - intake:read + - platform:read + put: + permissions: + - intake.experiments.update + scopes: + - intake:write + - platform:write + # Deprecated /experiment-groups aliases: the parent resource moved to /experiments, but the old + # paths are kept as hidden back-compat routes (include_in_schema=False), so they are absent from + # the OpenAPI-derived bundle. Map them to the intake.experiments.* permissions and mark + # x-not-in-openapi so the orphan check skips them. Remove alongside the router aliases. /apis/intake/v2/workspaces/{workspace}/experiment-groups: get: + x-not-in-openapi: true permissions: - - intake.experiment-groups.read + - intake.experiments.read scopes: - intake:read - platform:read post: + x-not-in-openapi: true permissions: - - intake.experiment-groups.create + - intake.experiments.create scopes: - intake:write - platform:write /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: delete: + x-not-in-openapi: true permissions: - - intake.experiment-groups.delete + - intake.experiments.delete scopes: - intake:write - platform:write get: + x-not-in-openapi: true permissions: - - intake.experiment-groups.read + - intake.experiments.read scopes: - intake:read - platform:read put: + x-not-in-openapi: true permissions: - - intake.experiment-groups.update + - intake.experiments.update scopes: - intake:write - platform:write diff --git a/services/intake/scripts/spans/seed_experiment_rollup_data.py b/services/intake/scripts/spans/seed_experiment_rollup_data.py index 9cacabfe23..9dba4b9cea 100644 --- a/services/intake/scripts/spans/seed_experiment_rollup_data.py +++ b/services/intake/scripts/spans/seed_experiment_rollup_data.py @@ -104,10 +104,10 @@ def _preflight(base_url: str) -> None: def _upsert_group(client: httpx.Client, base_url: str, workspace: str) -> str: body = {"name": DEFAULT_GROUP, "description": "Smoke-test group for the rollup script."} - url = _intake_url(base_url, workspace, "/experiment-groups") + url = _intake_url(base_url, workspace, "/experiments") response = client.post(url, json=body) if response.status_code == 409: - response = client.put(_intake_url(base_url, workspace, f"/experiment-groups/{DEFAULT_GROUP}"), json=body) + response = client.put(_intake_url(base_url, workspace, f"/experiments/{DEFAULT_GROUP}"), json=body) response.raise_for_status() return response.json()["id"] diff --git a/services/intake/scripts/spans/seed_experiments_demo.py b/services/intake/scripts/spans/seed_experiments_demo.py index 978ec48ded..9e8afe92fe 100644 --- a/services/intake/scripts/spans/seed_experiments_demo.py +++ b/services/intake/scripts/spans/seed_experiments_demo.py @@ -452,7 +452,7 @@ def _create_group_if_missing( client: httpx.Client, base_url: str, workspace: str, spec: GroupSpec ) -> tuple[str | None, bool]: """Returns (group_id, created). On 409, returns (None, False) and leaves the existing group alone.""" - url = _intake_url(base_url, workspace, "/experiment-groups") + url = _intake_url(base_url, workspace, "/experiments") body = {"name": spec.name, "description": spec.description} response = client.post(url, json=body) if response.status_code == 409: @@ -651,8 +651,8 @@ def _wipe_workspace(client: httpx.Client, base_url: str, workspace: str) -> None if _delete(client, base_url, workspace, f"/evaluations/{name}"): deleted_evaluations += 1 deleted_groups = 0 - for name in _list_all_names(client, base_url, workspace, "/experiment-groups"): - if _delete(client, base_url, workspace, f"/experiment-groups/{name}"): + for name in _list_all_names(client, base_url, workspace, "/experiments"): + if _delete(client, base_url, workspace, f"/experiments/{name}"): deleted_groups += 1 print(f"deleted {deleted_evaluations} evaluation(s) and {deleted_groups} group(s)\n") diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index 851c0f4e20..4e6ed36fde 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Create, list, get, and delete endpoints for Evaluations and ExperimentGroups. +"""Create, list, get, and delete endpoints for Evaluations and Experiments. Entity-store writes use ``EntityClient`` directly. Evaluation reads use an application service that composes Entities with ClickHouse-backed rollups and @@ -18,6 +18,7 @@ from typing import Annotated, Any, Literal, NamedTuple, TypeVar from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +from fastapi.routing import APIRoute from nmp.common.api.common import Page, PaginationData from nmp.common.api.filter import ComparisonOperation, FilterOperation, FilterOperator, LogicalOperation from nmp.common.api.parsed_filter import ParsedFilter, make_filter_dep @@ -33,9 +34,9 @@ EvaluationSessionMode, EvaluationSessionResponse, EvaluatorAggregate, - ExperimentGroupFilter, - ExperimentGroupRequest, - ExperimentGroupResponse, + ExperimentFilter, + ExperimentRequest, + ExperimentResponse, ) # The API/Studio expose this as an "Evaluation", but it is still stored as the Experiment entity @@ -61,10 +62,10 @@ router = APIRouter(dependencies=[Depends(require_workspace_access)]) -GROUPS_TAG = "Experiment Groups" +EXPERIMENTS_TAG = "Experiments" EVALUATIONS_TAG = "Evaluations" -ExperimentGroupSortField = Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"] +ExperimentSortField = Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"] # The evaluations list is sorted in the application layer (compute-on-read) so a single request can # sort by a ClickHouse rollup metric, not just entity columns. `sort` is therefore a free string, @@ -92,23 +93,23 @@ EntityT = TypeVar("EntityT", Evaluation, ExperimentGroup) -ExperimentGroupFilterDep = Annotated[ParsedFilter, Depends(make_filter_dep(ExperimentGroupFilter))] +ExperimentFilterDep = Annotated[ParsedFilter, Depends(make_filter_dep(ExperimentFilter))] EvaluationFilterDep = Annotated[ParsedFilter, Depends(make_filter_dep(EvaluationFilter))] EvaluationSessionFilterDep = Annotated[ParsedFilter, Depends(make_filter_dep(EvaluationSessionFilter))] @router.post( - "/v2/workspaces/{workspace}/experiment-groups", - response_model=ExperimentGroupResponse, + "/v2/workspaces/{workspace}/experiments", + response_model=ExperimentResponse, status_code=status.HTTP_201_CREATED, - tags=[GROUPS_TAG], - responses={409: {"description": "Experiment group already exists"}}, + tags=[EXPERIMENTS_TAG], + responses={409: {"description": "Experiment already exists"}}, ) -async def create_experiment_group( +async def create_experiment( workspace: str, - body: ExperimentGroupRequest, + body: ExperimentRequest, entity_client: EntityClientDep, -) -> ExperimentGroupResponse: +) -> ExperimentResponse: _validate_default_sort(body.default_sort) entity = ExperimentGroup( workspace=workspace, @@ -125,35 +126,33 @@ async def create_experiment_group( except EntityConflictError as e: raise HTTPException( status_code=status.HTTP_409_CONFLICT, - detail=f"Experiment group '{workspace}/{body.name}' already exists.", + detail=f"Experiment '{workspace}/{body.name}' already exists.", ) from e - return ExperimentGroupResponse.from_entity(created) + return ExperimentResponse.from_entity(created) @router.get( - "/v2/workspaces/{workspace}/experiment-groups", - response_model=Page[ExperimentGroupResponse], - tags=[GROUPS_TAG], + "/v2/workspaces/{workspace}/experiments", + response_model=Page[ExperimentResponse], + tags=[EXPERIMENTS_TAG], openapi_extra=generate_openapi_extra_params( - filter_schema=ExperimentGroupFilter, + filter_schema=ExperimentFilter, filter_description=( - "Filter experiment groups by name, insight_id, is_deleted, or a metadata key/value " + "Filter experiments by name, insight_id, is_deleted, or a metadata key/value " "(filter[metadata.]=). " - "Pass is_deleted=true to return only soft-deleted groups; omit to see only live ones." + "Pass is_deleted=true to return only soft-deleted experiments; omit to see only live ones." ), ), ) -async def list_experiment_groups( +async def list_experiments( workspace: str, request: Request, entity_client: EntityClientDep, - parsed: ExperimentGroupFilterDep, + parsed: ExperimentFilterDep, page: int = Query(default=1, ge=1, description="Page number."), page_size: int = Query(default=100, ge=1, le=1000, description="Page size."), - sort: ExperimentGroupSortField = Query( - default="-created_at", description="Sort field; prefix with '-' for descending." - ), -) -> Page[ExperimentGroupResponse]: + sort: ExperimentSortField = Query(default="-created_at", description="Sort field; prefix with '-' for descending."), +) -> Page[ExperimentResponse]: validate_list_query_params(request) _apply_is_deleted_filter(parsed) result = await entity_client.list( @@ -164,7 +163,7 @@ async def list_experiment_groups( page=page, page_size=page_size, ) - responses = [ExperimentGroupResponse.from_entity(e) for e in result.data] + responses = [ExperimentResponse.from_entity(e) for e in result.data] counts = await _count_live_evaluations_by_group( entity_client, workspace=workspace, group_ids=[g.id for g in result.data] ) @@ -179,25 +178,25 @@ async def list_experiment_groups( @router.get( - "/v2/workspaces/{workspace}/experiment-groups/{name}", - response_model=ExperimentGroupResponse, - tags=[GROUPS_TAG], - responses={404: {"description": "Experiment group not found"}}, + "/v2/workspaces/{workspace}/experiments/{name}", + response_model=ExperimentResponse, + tags=[EXPERIMENTS_TAG], + responses={404: {"description": "Experiment not found"}}, ) -async def get_experiment_group( +async def get_experiment( workspace: str, name: str, entity_client: EntityClientDep, -) -> ExperimentGroupResponse: +) -> ExperimentResponse: entity = await _get_or_404( entity_client, ExperimentGroup, workspace=workspace, name=name, - label="Experiment group", + label="Experiment", ) - _reject_if_deleted(entity, workspace=workspace, name=name, label="Experiment group") - response = ExperimentGroupResponse.from_entity(entity) + _reject_if_deleted(entity, workspace=workspace, name=name, label="Experiment") + response = ExperimentResponse.from_entity(entity) response.evaluation_count = await _count_live_evaluations_in_group( entity_client, workspace=workspace, group_id=entity.id ) @@ -205,32 +204,32 @@ async def get_experiment_group( @router.put( - "/v2/workspaces/{workspace}/experiment-groups/{name}", - response_model=ExperimentGroupResponse, - tags=[GROUPS_TAG], + "/v2/workspaces/{workspace}/experiments/{name}", + response_model=ExperimentResponse, + tags=[EXPERIMENTS_TAG], responses={ - 404: {"description": "Experiment group not found"}, - 409: {"description": "Attempt to rename the group"}, + 404: {"description": "Experiment not found"}, + 409: {"description": "Attempt to rename the experiment"}, }, ) -async def update_experiment_group( +async def update_experiment( workspace: str, name: str, - body: ExperimentGroupRequest, + body: ExperimentRequest, entity_client: EntityClientDep, -) -> ExperimentGroupResponse: +) -> ExperimentResponse: existing = await _get_or_404( entity_client, ExperimentGroup, workspace=workspace, name=name, - label="Experiment group", + label="Experiment", ) - _reject_if_deleted(existing, workspace=workspace, name=name, label="Experiment group") + _reject_if_deleted(existing, workspace=workspace, name=name, label="Experiment") if body.name != name: raise HTTPException( status_code=status.HTTP_409_CONFLICT, - detail="Cannot rename an experiment group; the name is its identity.", + detail="Cannot rename an experiment; the name is its identity.", ) _validate_default_sort(body.default_sort) existing.description = body.description @@ -243,7 +242,7 @@ async def update_experiment_group( if body.pareto is not None: existing.pareto = body.pareto updated = await entity_client.update(existing) - response = ExperimentGroupResponse.from_entity(updated) + response = ExperimentResponse.from_entity(updated) response.evaluation_count = await _count_live_evaluations_in_group( entity_client, workspace=workspace, group_id=updated.id ) @@ -251,12 +250,12 @@ async def update_experiment_group( @router.delete( - "/v2/workspaces/{workspace}/experiment-groups/{name}", + "/v2/workspaces/{workspace}/experiments/{name}", status_code=status.HTTP_204_NO_CONTENT, - tags=[GROUPS_TAG], - responses={404: {"description": "Experiment group not found"}}, + tags=[EXPERIMENTS_TAG], + responses={404: {"description": "Experiment not found"}}, ) -async def delete_experiment_group( +async def delete_experiment( workspace: str, name: str, entity_client: EntityClientDep, @@ -269,9 +268,9 @@ async def delete_experiment_group( ExperimentGroup, workspace=workspace, name=name, - label="Experiment group", + label="Experiment", ) - _reject_if_deleted(group, workspace=workspace, name=name, label="Experiment group") + _reject_if_deleted(group, workspace=workspace, name=name, label="Experiment") # Reference-counted cascade, sequential — one update per member. Linear in group size, fine for # now. A member whose *sole* membership was this group is soft-deleted; a member also in another @@ -1360,3 +1359,23 @@ def _aggregate(rollup: ScoreRollup) -> EvaluatorAggregate: p99=rollup.p99, count=rollup.count, ) + + +# --------------------------------------------------------------------------- +# Backwards-compatible URL aliases (deprecated) +# +# The parent resource moved from `/experiment-groups` to `/experiments`. Register the old +# `/experiment-groups...` paths as hidden aliases (``include_in_schema=False``) pointing at the +# same handlers, so existing callers keep working until they migrate to `/experiments`. +# --------------------------------------------------------------------------- +for _legacy_route in list(router.routes): + if isinstance(_legacy_route, APIRoute) and "/experiments" in _legacy_route.path: + router.add_api_route( + _legacy_route.path.replace("/experiments", "/experiment-groups", 1), + _legacy_route.endpoint, + methods=sorted(_legacy_route.methods), + response_model=_legacy_route.response_model, + status_code=_legacy_route.status_code, + include_in_schema=False, + name=f"{_legacy_route.name}_experiment_groups_alias", + ) diff --git a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py index a94c0365c4..b1e6bd77ff 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py @@ -26,22 +26,24 @@ EvaluationSessionMode = IntakeResponseMode -class ExperimentGroupRequest(BaseModel): - """Request body for creating an ExperimentGroup.""" +class ExperimentRequest(BaseModel): + """Request body for creating an Experiment.""" model_config = ConfigDict(extra="forbid") - name: str = Field(description="Workspace-unique group name.") - description: str | None = Field(default=None, description="Human-readable purpose of the group.") + name: str = Field(description="Workspace-unique experiment name.") + description: str | None = Field(default=None, description="Human-readable purpose of the experiment.") insight_id: str | None = Field( - default=None, description="Reference to an external insight that seeded this group, if any." + default=None, description="Reference to an external insight that seeded this experiment, if any." ) - summary: str | None = Field(default=None, description="Human- or agent-authored summary of the group's findings.") - metadata: dict[str, str] | None = Field(default=None, description="Free-form producer metadata for the group.") + summary: str | None = Field( + default=None, description="Human- or agent-authored summary of the experiment's findings." + ) + metadata: dict[str, str] | None = Field(default=None, description="Free-form producer metadata for the experiment.") default_sort: str = Field( default="-created_at", description=( - "Default sort for this group's evaluations list, as a `sort`-param string: a comma-separated, " + "Default sort for this experiment's evaluations list, as a `sort`-param string: a comma-separated, " "ordered list of fields where the first is the primary sort and the rest break ties (leading " "'-' on a field = descending), e.g. '-evaluators.reward.mean,cost_usd.mean'. Defaults to " "'-created_at'. Accepts any field the evaluations list `sort` param does; clients apply it as " @@ -51,7 +53,7 @@ class ExperimentGroupRequest(BaseModel): pareto: ParetoConfig | None = Field( default=None, description=( - "Default X/Y metrics for the group's Pareto view. Omit to preserve the existing value on " + "Default X/Y metrics for the experiment's Pareto view. Omit to preserve the existing value on " "update; on create, defaults to cost vs. latency." ), ) @@ -66,7 +68,7 @@ class EvaluationRequest(BaseModel): experiment_ids: list[str] = Field( default_factory=list, description=( - "Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). Preferred; each group " + "Entity ids of the Experiments this Evaluation belongs to (>=1). Preferred; each experiment " "must already exist. When omitted, the deprecated experiment_group_id is used instead." ), ) @@ -74,7 +76,7 @@ class EvaluationRequest(BaseModel): default=None, deprecated=True, description=( - "Deprecated single-group field; provide experiment_ids instead. Coalesced into experiment_ids " + "Deprecated single-experiment field; provide experiment_ids instead. Coalesced into experiment_ids " "when experiment_ids is omitted." ), ) @@ -87,11 +89,6 @@ class EvaluationRequest(BaseModel): default=None, description="Entity id of the evaluation this one was derived from (e.g. a variant of a baseline), if any.", ) - parent_experiment_id: str | None = Field( - default=None, - deprecated=True, - description="Deprecated alias for parent_evaluation_id.", - ) status: str | None = Field(default=None, description="Producer-defined lifecycle status of the evaluation.") root_cause: str | None = Field( default=None, @@ -100,7 +97,7 @@ class EvaluationRequest(BaseModel): @model_validator(mode="after") def _resolve_group_membership(self) -> Self: - """Require >=1 group. Coalesce the deprecated ``experiment_group_id`` into ``experiment_ids`` + """Require >=1 experiment. Coalesce the deprecated ``experiment_group_id`` into ``experiment_ids`` when the latter is omitted; reject a request that supplies neither. Read the deprecated value via ``__dict__`` to avoid tripping its deprecation warning per request. @@ -111,31 +108,19 @@ def _resolve_group_membership(self) -> Self: self.experiment_ids = [group_id] if not self.experiment_ids: raise ValueError( - "An evaluation must belong to at least one group: provide experiment_ids or experiment_group_id." + "An evaluation must belong to at least one experiment: provide experiment_ids or experiment_group_id." ) - # Membership is a set — drop duplicate group ids (order-preserving) so counts aren't inflated. + # Membership is a set — drop duplicate experiment ids (order-preserving) so counts aren't inflated. self.experiment_ids = list(dict.fromkeys(self.experiment_ids)) return self - @model_validator(mode="after") - def _coalesce_deprecated_parent(self) -> Self: - """Accept the deprecated ``parent_experiment_id`` alias; the canonical field wins if both are set. - - Read the raw value via ``__dict__`` to avoid tripping the field's deprecation warning on - every request. - """ - deprecated_parent = self.__dict__.get("parent_experiment_id") - if self.parent_evaluation_id is None and deprecated_parent is not None: - self.parent_evaluation_id = deprecated_parent - return self - class EvaluationPatchRequest(BaseModel): """Partial-update body for an Evaluation: only fields present in the request are applied. Unset fields are left unchanged (PATCH semantics — same pattern as the models service's PATCH). Immutable fields (name, dataset_name, dataset_version) aren't accepted here. ``experiment_ids``, - when provided, must be non-empty: an evaluation must always belong to at least one group. + when provided, must be non-empty: an evaluation must always belong to at least one experiment. """ model_config = ConfigDict(extra="forbid") @@ -143,8 +128,8 @@ class EvaluationPatchRequest(BaseModel): experiment_ids: list[str] | None = Field( default=None, description=( - "Replace the ExperimentGroups this Evaluation belongs to. Must be non-empty when provided; " - "each group must already exist. Omit to leave membership unchanged." + "Replace the Experiments this Evaluation belongs to. Must be non-empty when provided; " + "each experiment must already exist. Omit to leave membership unchanged." ), ) source_link: AnyUrl | None = Field(default=None, description="Optional URL for the source evaluation.") @@ -161,8 +146,8 @@ class EvaluationPatchRequest(BaseModel): ) -class ExperimentGroupResponse(BaseModel): - """ExperimentGroup as served by the API.""" +class ExperimentResponse(BaseModel): + """Experiment as served by the API.""" id: str name: str @@ -177,16 +162,11 @@ class ExperimentGroupResponse(BaseModel): updated_at: datetime | None = None evaluation_count: int = Field( default=0, - description="Number of live (non-soft-deleted) evaluations in this group.", + description="Number of live (non-soft-deleted) evaluations in this experiment.", ) - @computed_field(deprecated=True, description="Deprecated alias for evaluation_count.") # type: ignore[prop-decorator] - @property - def experiment_count(self) -> int: - return self.evaluation_count - @classmethod - def from_entity(cls, entity: ExperimentGroup) -> ExperimentGroupResponse: + def from_entity(cls, entity: ExperimentGroup) -> ExperimentResponse: return cls( id=entity.id, name=entity.name, @@ -221,7 +201,7 @@ class EvaluationResponse(BaseModel): name: str workspace: str experiment_ids: list[str] = Field( - description="Entity ids of the ExperimentGroups this Evaluation belongs to (>=1).", + description="Entity ids of the Experiments this Evaluation belongs to (>=1).", ) dataset_name: str dataset_version: str | None = None @@ -278,14 +258,9 @@ class EvaluationResponse(BaseModel): description="Average total tokens (input + output) per test case, aggregated across the evaluation.", ) - @computed_field(deprecated=True, description="Deprecated alias for parent_evaluation_id.") # type: ignore[prop-decorator] - @property - def parent_experiment_id(self) -> str | None: - return self.parent_evaluation_id - @computed_field( # type: ignore[prop-decorator] deprecated=True, - description="Deprecated single-group alias; the first of experiment_ids. Use experiment_ids.", + description="Deprecated single-experiment alias; the first of experiment_ids. Use experiment_ids.", ) @property def experiment_group_id(self) -> str: @@ -332,17 +307,17 @@ class MetricStatFilters(BaseModel): count: NumberFilter | None = None -class ExperimentGroupFilter(Filter): - """Filter for listing ExperimentGroups.""" +class ExperimentFilter(Filter): + """Filter for listing Experiments.""" - name: str | None = Field(default=None, description="Filter groups by name.") + name: str | None = Field(default=None, description="Filter experiments by name.") insight_id: str | None = Field( default=None, - description="Filter groups by the id of the insight that seeded them.", + description="Filter experiments by the id of the insight that seeded them.", ) is_deleted: bool | None = Field( default=None, - description="When true, returns only soft-deleted groups. Omit (or false) to see only live groups.", + description="When true, returns only soft-deleted experiments. Omit (or false) to see only live experiments.", ) metadata: Annotated[dict[str, str] | None, map_entity_field("data.metadata", namespace=True)] = Field( default=None, @@ -433,12 +408,6 @@ class EvaluationSessionResponse(BaseModel): workspace: str evaluation_name: str - - @computed_field(deprecated=True, description="Deprecated alias for evaluation_name.") # type: ignore[prop-decorator] - @property - def experiment_name(self) -> str: - return self.evaluation_name - session_id: str test_case_id: str | None = Field( default=None, diff --git a/services/intake/src/nmp/intake/service.py b/services/intake/src/nmp/intake/service.py index 95f18b7550..3b650d9ac2 100644 --- a/services/intake/src/nmp/intake/service.py +++ b/services/intake/src/nmp/intake/service.py @@ -55,7 +55,7 @@ def get_routers(self) -> List[RouterConfig]: RouterConfig( experiments.router, tag="Experiments", - description="Create, list, get, and delete Evaluations and Experiment Groups", + description="Create, list, get, and delete Evaluations and Experiments", ), RouterConfig(otlp.router, tag="Ingest", description="OTLP/HTTP trace ingest endpoints"), RouterConfig(atif.router, tag="Ingest", description="ATIF trajectory ingest endpoints"), diff --git a/services/intake/src/nmp/intake/spans/api/traces.py b/services/intake/src/nmp/intake/spans/api/traces.py index 8928f79f9c..928d77ee86 100644 --- a/services/intake/src/nmp/intake/spans/api/traces.py +++ b/services/intake/src/nmp/intake/spans/api/traces.py @@ -27,13 +27,11 @@ TRACE_INDEX_FILTER_FIELDS = frozenset( { "evaluation_id", - "experiment_id", # deprecated alias for evaluation_id "test_case_id", } ) TRACE_INDEX_FILTER_ALIASES = { "evaluation_id": "evaluation_id", - "experiment_id": "evaluation_id", # deprecated alias resolves to the evaluation_id filter "test_case_id": "test_case_id", } @@ -47,7 +45,7 @@ filter_schema=TraceFilter, filter_description=( "Filter root-span-backed traces by id, session_id, root status, root span started_at, " - "evaluation_id (or its deprecated alias experiment_id), and test_case_id." + "evaluation_id, and test_case_id." ), ), ) diff --git a/services/intake/src/nmp/intake/spans/api/traces_schemas.py b/services/intake/src/nmp/intake/spans/api/traces_schemas.py index f41d62d6cf..e902b99bb8 100644 --- a/services/intake/src/nmp/intake/spans/api/traces_schemas.py +++ b/services/intake/src/nmp/intake/spans/api/traces_schemas.py @@ -16,7 +16,7 @@ IntakeTrace, SpanStatus, ) -from nmp.intake.spans.ingest.evaluation_context import EvaluationContext, ExperimentContext +from nmp.intake.spans.ingest.evaluation_context import EvaluationContext from nmp.intake.spans.storage import text_for_mode from pydantic import BaseModel, Field @@ -35,11 +35,6 @@ class TraceFilter(BaseModel): status: SpanStatus | None = Field(default=None, description="Filter by root span status.") started_at: DatetimeFilter | None = Field(default=None, description="Filter by root span start timestamp.") evaluation_id: str | None = Field(default=None, description="Filter by root-span evaluation id.") - experiment_id: str | None = Field( - default=None, - deprecated=True, - description="Deprecated alias for evaluation_id. Filter by root-span evaluation id.", - ) test_case_id: str | None = Field(default=None, description="Filter by root-span evaluation test case id.") @@ -64,11 +59,6 @@ class Trace(BaseModel): ), ) evaluation_context: EvaluationContext | None = None - experiment_context: ExperimentContext | None = Field( - default=None, - deprecated=True, - description="Deprecated alias for evaluation_context; will be removed in a future release.", - ) started_at: datetime ended_at: datetime | None = None duration_ms: float | None = None @@ -94,7 +84,6 @@ def from_domain(cls, trace: IntakeTrace, *, mode: TraceMode = "detailed") -> Sel input=text_for_mode(trace.input, mode=mode), output=text_for_mode(trace.output, mode=mode), evaluation_context=_evaluation_context(trace), - experiment_context=_experiment_context(trace), started_at=trace.started_at, ended_at=trace.ended_at, duration_ms=trace.duration_ms, @@ -118,13 +107,3 @@ def _evaluation_context(trace: IntakeTrace) -> EvaluationContext | None: evaluation_id=trace.evaluation_id, test_case_id=trace.test_case_id, ) - - -def _experiment_context(trace: IntakeTrace) -> ExperimentContext | None: - """Deprecated alias for ``_evaluation_context``; populated from the same evaluation id.""" - if trace.evaluation_id is None: - return None - return ExperimentContext( - experiment_id=trace.evaluation_id, - test_case_id=trace.test_case_id, - ) diff --git a/services/intake/src/nmp/intake/spans/ingest/atif.py b/services/intake/src/nmp/intake/spans/ingest/atif.py index 32235fd886..8c306c4ff9 100644 --- a/services/intake/src/nmp/intake/spans/ingest/atif.py +++ b/services/intake/src/nmp/intake/spans/ingest/atif.py @@ -29,10 +29,10 @@ trajectory_to_evaluator_results, trajectory_to_spans, ) -from nmp.intake.spans.ingest.evaluation_context import EvaluationContextIngestModel +from nmp.intake.spans.ingest.evaluation_context import EvaluationContext from nmp.intake.spans.ingest.evaluation_context_validation import validate_evaluation_context from nmp.intake.spans.storage import utc_now -from pydantic import ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, model_validator router = APIRouter(dependencies=[Depends(require_workspace_access)]) API_TAG = "Ingest" @@ -48,7 +48,7 @@ def _atif_max_subagent_depth(request: Request) -> int: return cfg.atif_max_subagent_depth -class AtifIngestRequest(EvaluationContextIngestModel): +class AtifIngestRequest(BaseModel): """Span-based ATIF ingest request. ATIF project scoping is intentionally not accepted here; use the workspace @@ -57,6 +57,7 @@ class AtifIngestRequest(EvaluationContextIngestModel): model_config = ConfigDict(extra="forbid") + evaluation_context: EvaluationContext | None = None schema_version: AtifSchemaVersion session_id: str | None = None trajectory_id: str | None = None @@ -91,7 +92,7 @@ def to_trajectory(self) -> AtifTrajectory: notes=self.notes, extra=self.extra, subagent_trajectories=self.subagent_trajectories, - evaluation_context=self.resolved_evaluation_context(), + evaluation_context=self.evaluation_context, **kwargs, ) @@ -111,7 +112,7 @@ async def ingest_atif( ) -> Response: await validate_evaluation_context( workspace=workspace, - context=body.resolved_evaluation_context(), + context=body.evaluation_context, entity_client=entity_client, ) ingested_at = utc_now() diff --git a/services/intake/src/nmp/intake/spans/ingest/chat_completions.py b/services/intake/src/nmp/intake/spans/ingest/chat_completions.py index 36abe5d562..1929075d90 100644 --- a/services/intake/src/nmp/intake/spans/ingest/chat_completions.py +++ b/services/intake/src/nmp/intake/spans/ingest/chat_completions.py @@ -21,9 +21,7 @@ from nmp.common.service.dependencies import get_entity_client from nmp.intake.spans.api.dependencies import SpansServiceDep, require_workspace_access from nmp.intake.spans.domain import IntakeSpan, SpanKind, SpanStatus, TraceBatch -from nmp.intake.spans.ingest.evaluation_context import ( - EvaluationContextIngestModel, -) +from nmp.intake.spans.ingest.evaluation_context import EvaluationContext from nmp.intake.spans.ingest.evaluation_context_validation import validate_evaluation_context from nmp.intake.spans.span_attribute_bags import SpanAttributeBags from nmp.intake.spans.span_semantic_attributes import SpanSemanticAttributes @@ -97,9 +95,10 @@ def _require_choices_or_error(self) -> Self: return self -class ChatCompletionsIngestRequest(EvaluationContextIngestModel): +class ChatCompletionsIngestRequest(BaseModel): model_config = ConfigDict(extra="forbid") + evaluation_context: EvaluationContext | None = None request: CapturedChatCompletionsRequest response: CapturedChatCompletionsResponse @@ -155,7 +154,7 @@ async def ingest_chat_completion( ) -> ChatCompletionsIngestResponse: await validate_evaluation_context( workspace=workspace, - context=body.resolved_evaluation_context(), + context=body.evaluation_context, entity_client=entity_client, ) ingested_at = utc_now() @@ -228,7 +227,7 @@ def _build_attribute_bags( error_type = _as_str(error.get("type")) or _as_str(error.get("code")) error_message = _as_str(error.get("message")) - evaluation_context = body.resolved_evaluation_context() + evaluation_context = body.evaluation_context semantic = SpanSemanticAttributes( model=_as_str(response.get("model")) or _as_str(request.get("model")), provider=body.provider or _infer_provider(response), diff --git a/services/intake/src/nmp/intake/spans/ingest/evaluation_context.py b/services/intake/src/nmp/intake/spans/ingest/evaluation_context.py index 6e01b4c7fe..c29d08a34a 100644 --- a/services/intake/src/nmp/intake/spans/ingest/evaluation_context.py +++ b/services/intake/src/nmp/intake/spans/ingest/evaluation_context.py @@ -19,38 +19,3 @@ class EvaluationContext(BaseModel): test_case_id: str | None = Field(default=None, description="Optional producer-supplied test case id.") model_config = ConfigDict(extra="ignore") - - -class ExperimentContext(BaseModel): - """Deprecated alias for :class:`EvaluationContext`. Producers should send ``evaluation_context``.""" - - experiment_id: str = Field(description="Name of an existing Experiment entity.") - test_case_id: str | None = Field(default=None, description="Optional producer-supplied test case id.") - - model_config = ConfigDict(extra="forbid") - - def to_evaluation_context(self) -> EvaluationContext: - return EvaluationContext( - evaluation_id=self.experiment_id, - test_case_id=self.test_case_id, - ) - - -class EvaluationContextIngestModel(BaseModel): - """Base model for ingest payloads that carry evaluation context.""" - - evaluation_context: EvaluationContext | None = None - # Deprecated alias; use evaluation_context. Read via __dict__ to avoid a deprecation warning. - experiment_context: ExperimentContext | None = Field( - default=None, - deprecated=True, - description="Deprecated. Use evaluation_context; when both are sent, evaluation_context takes precedence.", - ) - - def resolved_evaluation_context(self) -> EvaluationContext | None: - if self.evaluation_context is not None: - return self.evaluation_context - experiment_context: ExperimentContext | None = self.__dict__.get("experiment_context") - if experiment_context is not None: - return experiment_context.to_evaluation_context() - return None diff --git a/services/intake/tests/integration/spans/test_atif_ingest.py b/services/intake/tests/integration/spans/test_atif_ingest.py index dea013c0a0..51c756834a 100644 --- a/services/intake/tests/integration/spans/test_atif_ingest.py +++ b/services/intake/tests/integration/spans/test_atif_ingest.py @@ -873,10 +873,10 @@ def _create_experiment(client: TestClient, name: str) -> str: def _ensure_group(client: TestClient, name: str = "atif-test-group") -> str: response = client.post( - "/apis/intake/v2/workspaces/default/experiment-groups", + "/apis/intake/v2/workspaces/default/experiments", json={"name": name}, ) if response.status_code == 409: - response = client.get(f"/apis/intake/v2/workspaces/default/experiment-groups/{name}") + response = client.get(f"/apis/intake/v2/workspaces/default/experiments/{name}") response.raise_for_status() return response.json()["id"] diff --git a/services/intake/tests/integration/spans/test_chat_completions_ingest.py b/services/intake/tests/integration/spans/test_chat_completions_ingest.py index b669c36c6f..a6012497f1 100644 --- a/services/intake/tests/integration/spans/test_chat_completions_ingest.py +++ b/services/intake/tests/integration/spans/test_chat_completions_ingest.py @@ -22,11 +22,7 @@ "test_case_id": "chat-case-001", "metadata": {"source": "chat-completions-test"}, } -EXPERIMENT_CONTEXT: dict[str, Any] = { - "experiment_id": EVALUATION_CONTEXT["evaluation_id"], - "test_case_id": EVALUATION_CONTEXT["test_case_id"], -} -EXPECTED_CONTEXT_FROM_EXPERIMENT_CONTEXT: dict[str, Any] = { +EXPECTED_EVALUATION_CONTEXT: dict[str, Any] = { "evaluation_id": EVALUATION_CONTEXT["evaluation_id"], "test_case_id": EVALUATION_CONTEXT["test_case_id"], } @@ -78,12 +74,12 @@ def _openai_response(**overrides: Any) -> dict[str, Any]: def test_chat_completions_ingest_happy_path(client: TestClient): - experiment_id = _create_experiment(client, EXPERIMENT_CONTEXT["experiment_id"]) + evaluation_id = _create_experiment(client, EVALUATION_CONTEXT["evaluation_id"]) body = { "request": _openai_request(), "response": _openai_response(), "session_id": "session-happy", - "experiment_context": {**EXPERIMENT_CONTEXT, "experiment_id": experiment_id}, + "evaluation_context": {**EVALUATION_CONTEXT, "evaluation_id": evaluation_id}, "provider": "openai", } response = client.post(INGEST_URL, json=body) @@ -106,7 +102,7 @@ def test_chat_completions_ingest_happy_path(client: TestClient): assert span["name"] == "gpt-4o-mini-2024-08-06" assert span["model"] == "gpt-4o-mini-2024-08-06" assert span["provider"] == "openai" - assert span["evaluation_context"] == EXPECTED_CONTEXT_FROM_EXPERIMENT_CONTEXT + assert span["evaluation_context"] == EXPECTED_EVALUATION_CONTEXT assert "evaluation_run_id" not in span assert "raw_attributes" not in span assert span["input_tokens"] == 24 @@ -141,19 +137,6 @@ def test_chat_completions_ingest_happy_path(client: TestClient): assert filtered_spans[0]["span_id"] == "chatcmpl-test-abc123" -def test_chat_completions_ingest_rejects_unknown_experiment_context(client: TestClient): - body = { - "request": _openai_request(), - "response": _openai_response(id="chatcmpl-missing-exp"), - "session_id": "session-missing-exp", - "experiment_context": {"experiment_id": "missing-exp", "test_case_id": "case-1"}, - } - response = client.post(INGEST_URL, json=body) - - assert response.status_code == 400, response.text - assert "must be created before it can be logged" in response.json()["detail"] - - def test_chat_completions_ingest_persists_cost_fields(client: TestClient): body = { "request": _openai_request(), @@ -434,27 +417,6 @@ def test_chat_completions_ingest_rejects_legacy_context_fields(client: TestClien assert response.status_code == 422, response.text -def test_chat_completions_ingest_accepts_both_context_shapes_with_evaluation_context_precedence(client: TestClient): - _create_experiment(client, "chat-eval") - # evaluation_context is canonical and wins; experiment_context points at a non-existent entity, so a - # 201 (rather than a 400 for the missing "legacy-exp") proves evaluation_context took precedence. - body = { - "request": _openai_request(), - "response": _openai_response(id="chatcmpl-both-contexts"), - "session_id": "session-both-contexts", - "experiment_context": {"experiment_id": "legacy-exp", "test_case_id": "legacy-case"}, - "evaluation_context": {"evaluation_id": "chat-eval", "test_case_id": "case-1"}, - } - response = client.post(INGEST_URL, json=body) - assert response.status_code == 201, response.text - - listed = client.get(SPANS_URL, params={"filter[session_id]": "session-both-contexts"}) - assert listed.status_code == 200, listed.text - span = listed.json()["data"][0] - assert span["evaluation_context"]["evaluation_id"] == "chat-eval" - assert span["evaluation_context"]["test_case_id"] == "case-1" - - def _create_experiment(client: TestClient, name: str) -> str: group_id = _ensure_group(client) response = client.post( @@ -477,10 +439,10 @@ def _create_experiment(client: TestClient, name: str) -> str: def _ensure_group(client: TestClient, name: str = "chat-completions-test-group") -> str: response = client.post( - "/apis/intake/v2/workspaces/default/experiment-groups", + "/apis/intake/v2/workspaces/default/experiments", json={"name": name}, ) if response.status_code == 409: - response = client.get(f"/apis/intake/v2/workspaces/default/experiment-groups/{name}") + response = client.get(f"/apis/intake/v2/workspaces/default/experiments/{name}") response.raise_for_status() return response.json()["id"] diff --git a/services/intake/tests/integration/spans/test_experiment_metric_sort.py b/services/intake/tests/integration/spans/test_experiment_metric_sort.py index 1669e455ef..c07476726b 100644 --- a/services/intake/tests/integration/spans/test_experiment_metric_sort.py +++ b/services/intake/tests/integration/spans/test_experiment_metric_sort.py @@ -11,13 +11,13 @@ ATIF_INGEST = "/apis/intake/v2/workspaces/default/ingest/atif" EVALUATIONS = "/apis/intake/v2/workspaces/default/evaluations" -GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" +EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" def _ensure_group(client: TestClient, name: str) -> str: - response = client.post(GROUPS, json={"name": name}) + response = client.post(EXPERIMENTS, json={"name": name}) if response.status_code == 409: - response = client.get(f"{GROUPS}/{name}") + response = client.get(f"{EXPERIMENTS}/{name}") response.raise_for_status() return response.json()["id"] diff --git a/services/intake/tests/integration/spans/test_experiment_rollups.py b/services/intake/tests/integration/spans/test_experiment_rollups.py index e3d08cdfa6..8aeec7f918 100644 --- a/services/intake/tests/integration/spans/test_experiment_rollups.py +++ b/services/intake/tests/integration/spans/test_experiment_rollups.py @@ -13,14 +13,14 @@ ATIF_INGEST = "/apis/intake/v2/workspaces/default/ingest/atif" EVALUATIONS = "/apis/intake/v2/workspaces/default/evaluations" -GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" +EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" def _ensure_group(client: TestClient, name: str = "rollup-test-group") -> str: """Create or fetch an ExperimentGroup; returns its id.""" - response = client.post(GROUPS, json={"name": name}) + response = client.post(EXPERIMENTS, json={"name": name}) if response.status_code == 409: - response = client.get(f"{GROUPS}/{name}") + response = client.get(f"{EXPERIMENTS}/{name}") response.raise_for_status() return response.json()["id"] diff --git a/services/intake/tests/integration/spans/test_experiment_sessions.py b/services/intake/tests/integration/spans/test_experiment_sessions.py index b770360227..c824846cd1 100644 --- a/services/intake/tests/integration/spans/test_experiment_sessions.py +++ b/services/intake/tests/integration/spans/test_experiment_sessions.py @@ -13,14 +13,14 @@ ATIF_INGEST = "/apis/intake/v2/workspaces/default/ingest/atif" EVALUATIONS = "/apis/intake/v2/workspaces/default/evaluations" -GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" +EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" def _ensure_group(client: TestClient, name: str = "sessions-test-group") -> str: """Create or fetch an ExperimentGroup; returns its id.""" - response = client.post(GROUPS, json={"name": name}) + response = client.post(EXPERIMENTS, json={"name": name}) if response.status_code == 409: - response = client.get(f"{GROUPS}/{name}") + response = client.get(f"{EXPERIMENTS}/{name}") response.raise_for_status() return response.json()["id"] diff --git a/services/intake/tests/integration/spans/test_traces_read.py b/services/intake/tests/integration/spans/test_traces_read.py index 4ad8cd12a7..dbd8f8d841 100644 --- a/services/intake/tests/integration/spans/test_traces_read.py +++ b/services/intake/tests/integration/spans/test_traces_read.py @@ -94,8 +94,6 @@ def test_traces_read_returns_core_trace_summary(client: TestClient, make_otlp_re assert trace["error_count"] == 0 assert trace["evaluation_context"]["evaluation_id"] == "experiment-a" assert trace["evaluation_context"]["test_case_id"] == "case-a" - assert trace["experiment_context"]["experiment_id"] == "experiment-a" - assert trace["experiment_context"]["test_case_id"] == "case-a" assert "evaluation_id" not in trace assert "experiment_id" not in trace assert "test_case_id" not in trace @@ -119,8 +117,6 @@ def test_traces_read_returns_core_trace_summary(client: TestClient, make_otlp_re assert summary_trace["status"] == "success" assert summary_trace["evaluation_context"]["evaluation_id"] == "experiment-a" assert summary_trace["evaluation_context"]["test_case_id"] == "case-a" - assert summary_trace["experiment_context"]["experiment_id"] == "experiment-a" - assert summary_trace["experiment_context"]["test_case_id"] == "case-a" assert "evaluation_id" not in summary_trace assert "experiment_id" not in summary_trace assert "test_case_id" not in summary_trace diff --git a/services/intake/tests/integration/test_experiments_crud.py b/services/intake/tests/integration/test_experiments_crud.py index 76d5c9903d..e941e98924 100644 --- a/services/intake/tests/integration/test_experiments_crud.py +++ b/services/intake/tests/integration/test_experiments_crud.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""CRUD tests for the Evaluations and ExperimentGroups endpoints.""" +"""CRUD tests for the Evaluations and Experiments endpoints.""" from __future__ import annotations @@ -11,7 +11,9 @@ from fastapi.testclient import TestClient from nmp.intake.api.v2.experiments.dependencies import get_evaluation_rollup_repository -GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" +EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" +# The deprecated pre-rename path; kept as a hidden alias for backwards compatibility. +LEGACY_GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" EVALUATIONS = "/apis/intake/v2/workspaces/default/evaluations" @@ -30,15 +32,29 @@ def _evaluation_body(*, experiment_group_id: str, **overrides: Any) -> dict: def _create_group(client: TestClient, name: str = "default-test-group") -> dict: """Create or fetch an experiment group; returns the JSON body.""" - response = client.post(GROUPS, json={"name": name}) + response = client.post(EXPERIMENTS, json={"name": name}) if response.status_code == 409: - response = client.get(f"{GROUPS}/{name}") + response = client.get(f"{EXPERIMENTS}/{name}") response.raise_for_status() return response.json() +def test_legacy_experiment_groups_alias_is_backwards_compatible(client: TestClient) -> None: + """The pre-rename `/experiment-groups` path still works and shares the `/experiments` handlers.""" + # Create via the legacy path; read it back via the canonical path. + created = client.post(LEGACY_GROUPS, json={"name": "legacy-alias-exp"}) + assert created.status_code == 201, created.text + assert client.get(f"{EXPERIMENTS}/legacy-alias-exp").status_code == 200 + + # Create via the canonical path; read it back via the legacy path. + assert client.post(EXPERIMENTS, json={"name": "canonical-exp"}).status_code == 201 + legacy_get = client.get(f"{LEGACY_GROUPS}/canonical-exp") + assert legacy_get.status_code == 200, legacy_get.text + assert legacy_get.json()["name"] == "canonical-exp" + + def test_experiment_group_crud(client: TestClient) -> None: - created = client.post(GROUPS, json={"name": "tb2-routing-research", "description": "routing sweep"}) + created = client.post(EXPERIMENTS, json={"name": "tb2-routing-research", "description": "routing sweep"}) assert created.status_code == 201, created.text group = created.json() assert group["name"] == "tb2-routing-research" @@ -46,68 +62,68 @@ def test_experiment_group_crud(client: TestClient) -> None: assert group["id"] # Duplicate name conflicts. - duplicate = client.post(GROUPS, json={"name": "tb2-routing-research"}) + duplicate = client.post(EXPERIMENTS, json={"name": "tb2-routing-research"}) assert duplicate.status_code == 409 - fetched = client.get(f"{GROUPS}/tb2-routing-research") + fetched = client.get(f"{EXPERIMENTS}/tb2-routing-research") assert fetched.status_code == 200 assert fetched.json()["id"] == group["id"] - listed = client.get(GROUPS) + listed = client.get(EXPERIMENTS) assert listed.status_code == 200 assert any(g["name"] == "tb2-routing-research" for g in listed.json()["data"]) - deleted = client.delete(f"{GROUPS}/tb2-routing-research") + deleted = client.delete(f"{EXPERIMENTS}/tb2-routing-research") assert deleted.status_code == 204 - missing = client.get(f"{GROUPS}/tb2-routing-research") + missing = client.get(f"{EXPERIMENTS}/tb2-routing-research") assert missing.status_code == 404 def test_filter_groups_by_insight_id(client: TestClient) -> None: """The groups list can be filtered server-side by the seeding insight id.""" - seeded = client.post(GROUPS, json={"name": "seeded-group", "insight_id": "insight-abc"}) + seeded = client.post(EXPERIMENTS, json={"name": "seeded-group", "insight_id": "insight-abc"}) assert seeded.status_code == 201, seeded.text - other = client.post(GROUPS, json={"name": "unseeded-group"}) + other = client.post(EXPERIMENTS, json={"name": "unseeded-group"}) assert other.status_code == 201, other.text - listed = client.get(GROUPS, params={"filter[insight_id]": "insight-abc"}) + listed = client.get(EXPERIMENTS, params={"filter[insight_id]": "insight-abc"}) assert listed.status_code == 200, listed.text names = {g["name"] for g in listed.json()["data"]} assert names == {"seeded-group"} def test_experiment_group_update_description(client: TestClient) -> None: - client.post(GROUPS, json={"name": "grp", "description": "old"}) - updated = client.put(f"{GROUPS}/grp", json={"name": "grp", "description": "new"}) + client.post(EXPERIMENTS, json={"name": "grp", "description": "old"}) + updated = client.put(f"{EXPERIMENTS}/grp", json={"name": "grp", "description": "new"}) assert updated.status_code == 200, updated.text assert updated.json()["description"] == "new" # Renaming via PUT is rejected. - renamed = client.put(f"{GROUPS}/grp", json={"name": "renamed"}) + renamed = client.put(f"{EXPERIMENTS}/grp", json={"name": "renamed"}) assert renamed.status_code == 409 - missing = client.put(f"{GROUPS}/missing", json={"name": "missing"}) + missing = client.put(f"{EXPERIMENTS}/missing", json={"name": "missing"}) assert missing.status_code == 404 def test_experiment_group_pareto_defaults_and_round_trips(client: TestClient) -> None: # Omitting pareto defaults to cost (x) vs latency (y), so the chart always has something to render. - created = client.post(GROUPS, json={"name": "pareto-cfg"}) + created = client.post(EXPERIMENTS, json={"name": "pareto-cfg"}) assert created.status_code == 201, created.text assert created.json()["pareto"] == {"x_metric": "cost_usd", "y_metric": "latency_ms"} - assert client.get(f"{GROUPS}/pareto-cfg").json()["pareto"] == { + assert client.get(f"{EXPERIMENTS}/pareto-cfg").json()["pareto"] == { "x_metric": "cost_usd", "y_metric": "latency_ms", } # A custom selection round-trips through PUT. updated = client.put( - f"{GROUPS}/pareto-cfg", + f"{EXPERIMENTS}/pareto-cfg", json={"name": "pareto-cfg", "pareto": {"x_metric": "cost_usd", "y_metric": "evaluators.reward"}}, ) assert updated.status_code == 200, updated.text assert updated.json()["pareto"] == {"x_metric": "cost_usd", "y_metric": "evaluators.reward"} # An update that omits pareto preserves the saved axes rather than resetting them to the default. - preserved = client.put(f"{GROUPS}/pareto-cfg", json={"name": "pareto-cfg", "summary": "unrelated edit"}) + preserved = client.put(f"{EXPERIMENTS}/pareto-cfg", json={"name": "pareto-cfg", "summary": "unrelated edit"}) assert preserved.status_code == 200, preserved.text assert preserved.json()["pareto"] == {"x_metric": "cost_usd", "y_metric": "evaluators.reward"} @@ -116,19 +132,19 @@ def test_experiment_group_pareto_rejects_unknown_metric(client: TestClient) -> N # Studio can only plot cost_usd, latency_ms, or evaluators.; anything else is rejected so it # can't be persisted as an unusable chart default. rejected = client.post( - GROUPS, json={"name": "bad-pareto", "pareto": {"x_metric": "made_up", "y_metric": "latency_ms"}} + EXPERIMENTS, json={"name": "bad-pareto", "pareto": {"x_metric": "made_up", "y_metric": "latency_ms"}} ) assert rejected.status_code == 422, rejected.text # A dynamic evaluator metric is allowed. ok = client.post( - GROUPS, json={"name": "ok-pareto", "pareto": {"x_metric": "evaluators.safety", "y_metric": "cost_usd"}} + EXPERIMENTS, json={"name": "ok-pareto", "pareto": {"x_metric": "evaluators.safety", "y_metric": "cost_usd"}} ) assert ok.status_code == 201, ok.text def test_evaluation_update_moves_between_groups_and_edits(client: TestClient) -> None: - group_a = client.post(GROUPS, json={"name": "grp-a"}).json() - group_b = client.post(GROUPS, json={"name": "grp-b"}).json() + group_a = client.post(EXPERIMENTS, json={"name": "grp-a"}).json() + group_b = client.post(EXPERIMENTS, json={"name": "grp-b"}).json() client.post(EVALUATIONS, json=_evaluation_body(name="exp-a", experiment_group_id=group_a["id"])) # Move the evaluation from group_a to group_b and edit its description. @@ -157,7 +173,7 @@ def test_evaluation_update_rejects_immutable_change(client: TestClient) -> None: def test_evaluation_crud_and_empty_rollups(client: TestClient) -> None: - group = client.post(GROUPS, json={"name": "grp"}).json() + group = client.post(EXPERIMENTS, json={"name": "grp"}).json() created = client.post(EVALUATIONS, json=_evaluation_body(experiment_group_id=group["id"])) assert created.status_code == 201, created.text @@ -218,11 +234,11 @@ def test_evaluation_create_rejects_unknown_group_id(client: TestClient) -> None: def test_delete_group_cascades_to_evaluations(client: TestClient) -> None: - group = client.post(GROUPS, json={"name": "doomed-group"}).json() + group = client.post(EXPERIMENTS, json={"name": "doomed-group"}).json() client.post(EVALUATIONS, json=_evaluation_body(name="exp-doomed-1", experiment_group_id=group["id"])) client.post(EVALUATIONS, json=_evaluation_body(name="exp-doomed-2", experiment_group_id=group["id"])) - deleted = client.delete(f"{GROUPS}/doomed-group") + deleted = client.delete(f"{EXPERIMENTS}/doomed-group") assert deleted.status_code == 204, deleted.text # Both child evaluations are gone with the group. @@ -231,26 +247,6 @@ def test_delete_group_cascades_to_evaluations(client: TestClient) -> None: assert missing.status_code == 404 -def test_deprecated_field_aliases_are_backwards_compatible(client: TestClient) -> None: - """Renamed fields keep deprecated aliases: requests accept the old name, responses return both.""" - group = _create_group(client, name="alias-compat-group") - # Group response carries both the canonical count and the deprecated experiment_count alias. - assert group["experiment_count"] == group["evaluation_count"] - - parent = client.post(EVALUATIONS, json=_evaluation_body(name="parent-eval", experiment_group_id=group["id"])).json() - - # Create a child referencing the parent via the DEPRECATED parent_experiment_id request field. - body = _evaluation_body(name="child-eval", experiment_group_id=group["id"]) - body["parent_experiment_id"] = parent["id"] - created = client.post(EVALUATIONS, json=body) - assert created.status_code == 201, created.text - - payload = created.json() - # The response echoes both the canonical and deprecated parent field, coalesced from the old input. - assert payload["parent_evaluation_id"] == parent["id"] - assert payload["parent_experiment_id"] == parent["id"] - - def test_evaluation_conflict_and_not_found(client: TestClient) -> None: group = _create_group(client) created = client.post(EVALUATIONS, json=_evaluation_body(experiment_group_id=group["id"])) @@ -264,8 +260,8 @@ def test_evaluation_conflict_and_not_found(client: TestClient) -> None: def test_evaluation_list_and_scope_to_group(client: TestClient) -> None: - group_a = client.post(GROUPS, json={"name": "grp-a"}).json() - group_b = client.post(GROUPS, json={"name": "grp-b"}).json() + group_a = client.post(EXPERIMENTS, json={"name": "grp-a"}).json() + group_b = client.post(EXPERIMENTS, json={"name": "grp-b"}).json() client.post(EVALUATIONS, json=_evaluation_body(name="exp-a", experiment_group_id=group_a["id"])) client.post(EVALUATIONS, json=_evaluation_body(name="exp-b", experiment_group_id=group_b["id"])) @@ -287,8 +283,8 @@ def test_evaluation_list_and_scope_to_group(client: TestClient) -> None: def test_evaluation_filter_experiment_id_matches_deprecated_alias(client: TestClient) -> None: """The canonical `experiment_id` filter returns the same set as the deprecated `experiment_group_id`, both resolving membership over the evaluation's `experiment_ids` list.""" - group = client.post(GROUPS, json={"name": "grp-canon"}).json() - other = client.post(GROUPS, json={"name": "grp-other"}).json() + group = client.post(EXPERIMENTS, json={"name": "grp-canon"}).json() + other = client.post(EXPERIMENTS, json={"name": "grp-other"}).json() # Membership is set via the canonical `experiment_ids` write field (no legacy scalar). client.post(EVALUATIONS, json={"name": "in-group", "experiment_ids": [group["id"]], "dataset_name": "ds"}) client.post(EVALUATIONS, json={"name": "in-other", "experiment_ids": [other["id"]], "dataset_name": "ds"}) @@ -412,20 +408,20 @@ def test_filter_is_deleted_true_returns_only_deleted(client: TestClient) -> None def test_group_soft_delete_cascades_and_frees_names(client: TestClient) -> None: - group = client.post(GROUPS, json={"name": "doomed-group-v2"}).json() + group = client.post(EXPERIMENTS, json={"name": "doomed-group-v2"}).json() client.post(EVALUATIONS, json=_evaluation_body(name="child-1", experiment_group_id=group["id"])) client.post(EVALUATIONS, json=_evaluation_body(name="child-2", experiment_group_id=group["id"])) - deleted = client.delete(f"{GROUPS}/doomed-group-v2") + deleted = client.delete(f"{EXPERIMENTS}/doomed-group-v2") assert deleted.status_code == 204, deleted.text # Group and children all read as 404 (live view). - assert client.get(f"{GROUPS}/doomed-group-v2").status_code == 404 + assert client.get(f"{EXPERIMENTS}/doomed-group-v2").status_code == 404 for child_name in ("child-1", "child-2"): assert client.get(f"{EVALUATIONS}/{child_name}").status_code == 404 # Names are reusable in a fresh group. - fresh_group = client.post(GROUPS, json={"name": "doomed-group-v2"}) + fresh_group = client.post(EXPERIMENTS, json={"name": "doomed-group-v2"}) assert fresh_group.status_code == 201, fresh_group.text revived = client.post( EVALUATIONS, json=_evaluation_body(name="child-1", experiment_group_id=fresh_group.json()["id"]) @@ -433,7 +429,7 @@ def test_group_soft_delete_cascades_and_frees_names(client: TestClient) -> None: assert revived.status_code == 201, revived.text # Trash view still surfaces the cascaded rows. - deleted_groups = client.get(GROUPS, params={"filter[is_deleted]": "true"}) + deleted_groups = client.get(EXPERIMENTS, params={"filter[is_deleted]": "true"}) assert any(g["name"].startswith("doomed-group-v2-deleted-") for g in deleted_groups.json()["data"]) deleted_exps = client.get(EVALUATIONS, params={"filter[is_deleted]": "true"}) deleted_names = {e["name"] for e in deleted_exps.json()["data"]} @@ -442,8 +438,8 @@ def test_group_soft_delete_cascades_and_frees_names(client: TestClient) -> None: def test_create_evaluation_in_deleted_group_rejected(client: TestClient) -> None: - group = client.post(GROUPS, json={"name": "ephemeral-group"}).json() - client.delete(f"{GROUPS}/ephemeral-group") + group = client.post(EXPERIMENTS, json={"name": "ephemeral-group"}).json() + client.delete(f"{EXPERIMENTS}/ephemeral-group") response = client.post( EVALUATIONS, @@ -559,8 +555,8 @@ def test_sort_by_pinned_at_most_recent_first(client: TestClient) -> None: def test_evaluation_belongs_to_multiple_groups(client: TestClient) -> None: """An evaluation created with multiple experiment_ids appears in each group's leaderboard.""" - group_a = client.post(GROUPS, json={"name": "multi-a"}).json() - group_b = client.post(GROUPS, json={"name": "multi-b"}).json() + group_a = client.post(EXPERIMENTS, json={"name": "multi-a"}).json() + group_b = client.post(EXPERIMENTS, json={"name": "multi-b"}).json() body = _evaluation_body(name="multi-member", experiment_group_id="placeholder") body.pop("experiment_group_id") body["experiment_ids"] = [group_a["id"], group_b["id"]] @@ -588,8 +584,8 @@ def test_evaluation_create_rejects_empty_experiment_ids(client: TestClient) -> N def test_delete_group_reference_counted_cascade(client: TestClient) -> None: """Deleting a group deletes only its sole-membership members; shared members survive elsewhere.""" - group_a = client.post(GROUPS, json={"name": "rc-a"}).json() - group_b = client.post(GROUPS, json={"name": "rc-b"}).json() + group_a = client.post(EXPERIMENTS, json={"name": "rc-a"}).json() + group_b = client.post(EXPERIMENTS, json={"name": "rc-b"}).json() shared = _evaluation_body(name="rc-shared", experiment_group_id="placeholder") shared.pop("experiment_group_id") @@ -597,7 +593,7 @@ def test_delete_group_reference_counted_cascade(client: TestClient) -> None: client.post(EVALUATIONS, json=shared) client.post(EVALUATIONS, json=_evaluation_body(name="rc-sole", experiment_group_id=group_a["id"])) - assert client.delete(f"{GROUPS}/rc-a").status_code == 204 + assert client.delete(f"{EXPERIMENTS}/rc-a").status_code == 204 # Sole member gone; shared member survives, now only in group b. assert client.get(f"{EVALUATIONS}/rc-sole").status_code == 404 @@ -611,8 +607,8 @@ def test_delete_group_reference_counted_cascade(client: TestClient) -> None: def test_patch_evaluation_adds_to_group(client: TestClient) -> None: """PATCH .../evaluations/{name} with a merged experiment_ids adds a membership; the run shows on the board.""" - group_a = client.post(GROUPS, json={"name": "add-a"}).json() - group_b = client.post(GROUPS, json={"name": "add-b"}).json() + group_a = client.post(EXPERIMENTS, json={"name": "add-a"}).json() + group_b = client.post(EXPERIMENTS, json={"name": "add-b"}).json() client.post(EVALUATIONS, json=_evaluation_body(name="add-eval", experiment_group_id=group_a["id"])) added = client.patch(f"{EVALUATIONS}/add-eval", json={"experiment_ids": [group_a["id"], group_b["id"]]}) @@ -624,14 +620,14 @@ def test_patch_evaluation_adds_to_group(client: TestClient) -> None: def test_patch_evaluation_unknown_group_rejected(client: TestClient) -> None: - group = client.post(GROUPS, json={"name": "add-known"}).json() + group = client.post(EXPERIMENTS, json={"name": "add-known"}).json() client.post(EVALUATIONS, json=_evaluation_body(name="add-eval-2", experiment_group_id=group["id"])) resp = client.patch(f"{EVALUATIONS}/add-eval-2", json={"experiment_ids": [group["id"], "experiment_group-nope"]}) assert resp.status_code == 400, resp.text def test_patch_evaluation_rejects_empty_experiment_ids(client: TestClient) -> None: - group = client.post(GROUPS, json={"name": "empty-ids"}).json() + group = client.post(EXPERIMENTS, json={"name": "empty-ids"}).json() client.post(EVALUATIONS, json=_evaluation_body(name="empty-ids-eval", experiment_group_id=group["id"])) resp = client.patch(f"{EVALUATIONS}/empty-ids-eval", json={"experiment_ids": []}) assert resp.status_code == 400, resp.text @@ -641,8 +637,8 @@ def test_patch_evaluation_rejects_empty_experiment_ids(client: TestClient) -> No def test_patch_evaluation_leaves_omitted_fields_unchanged(client: TestClient) -> None: """Partial semantics: PATCHing experiment_ids alone must not clobber omitted fields (e.g. description).""" - group_a = client.post(GROUPS, json={"name": "patch-a"}).json() - group_b = client.post(GROUPS, json={"name": "patch-b"}).json() + group_a = client.post(EXPERIMENTS, json={"name": "patch-a"}).json() + group_b = client.post(EXPERIMENTS, json={"name": "patch-b"}).json() body = _evaluation_body(name="patch-eval", experiment_group_id=group_a["id"]) body["description"] = "keep me" body["metadata"] = {"team": "switchyard"} @@ -658,7 +654,7 @@ def test_patch_evaluation_leaves_omitted_fields_unchanged(client: TestClient) -> def test_patch_evaluation_dedupes_experiment_ids(client: TestClient) -> None: """Membership is a set: duplicate ids are stored once, so a group can't double-count an evaluation.""" - group = client.post(GROUPS, json={"name": "dedupe-grp"}).json() + group = client.post(EXPERIMENTS, json={"name": "dedupe-grp"}).json() client.post(EVALUATIONS, json=_evaluation_body(name="dedupe-eval", experiment_group_id=group["id"])) patched = client.patch(f"{EVALUATIONS}/dedupe-eval", json={"experiment_ids": [group["id"], group["id"]]}) @@ -666,6 +662,6 @@ def test_patch_evaluation_dedupes_experiment_ids(client: TestClient) -> None: assert patched.json()["experiment_ids"] == [group["id"]] # The group counts the evaluation once, not twice. - listed = client.get(GROUPS) + listed = client.get(EXPERIMENTS) grp = next(g for g in listed.json()["data"] if g["id"] == group["id"]) assert grp["evaluation_count"] == 1 diff --git a/services/intake/tests/test_atif_v17.py b/services/intake/tests/test_atif_v17.py index 8869fa8b9f..b21de1ab13 100644 --- a/services/intake/tests/test_atif_v17.py +++ b/services/intake/tests/test_atif_v17.py @@ -19,7 +19,7 @@ AtifTrajectory, ) from nmp.intake.spans.ingest.atif_mapping import AtifTrajectoryDepthError, trajectory_to_spans -from nmp.intake.spans.ingest.evaluation_context import EvaluationContext, ExperimentContext +from nmp.intake.spans.ingest.evaluation_context import EvaluationContext from pydantic import ValidationError EVALUATION_CONTEXT: dict[str, Any] = { @@ -456,14 +456,6 @@ def test_atif_v17_subagent_ref_requires_resolution_key() -> None: assert AtifSubagentTrajectoryRef(trajectory_path="subagents/sub-trajectory.json").trajectory_path is not None -def test_experiment_context_maps_to_storage_context() -> None: - context = ExperimentContext(experiment_id="exp-1", test_case_id="case-1") - storage_context = context.to_evaluation_context() - - assert storage_context.evaluation_id == "exp-1" - assert storage_context.test_case_id == "case-1" - - def test_evaluation_context_ignores_retired_fields() -> None: # Retired keys (evaluation_sha, evaluation_run_id, metadata) are accepted and dropped rather # than rejected, so stale producers keep ingesting without ingest erroring on unknown fields. @@ -544,29 +536,6 @@ def test_atif_mapping_writes_evaluation_context_only_on_root_span() -> None: assert "nemo.test_case.id" not in child.attributes_string -def test_atif_mapping_writes_experiment_context_to_experiment_attributes() -> None: - body = AtifIngestRequest.model_validate( - { - "schema_version": "ATIF-v1.7", - "session_id": "trace-session-id", - "experiment_context": {"experiment_id": "exp-1", "test_case_id": "case-1"}, - "agent": {"name": "sample-agent", "version": "1.0.0"}, - "steps": [{"step_id": 1, "source": "user", "message": "solve"}], - } - ) - - spans = trajectory_to_spans( - workspace="default", - trajectory=body.to_trajectory(), - ingested_at=datetime(2026, 5, 18, tzinfo=timezone.utc), - ) - - root = next(span for span in spans if span.name == "sample-agent") - assert root.attributes_string["nemo.experiment.id"] == "exp-1" - assert root.attributes_string["nemo.test_case.id"] == "case-1" - assert "evaluation.id" not in root.attributes_string - - def test_atif_mapping_uses_root_final_metrics_when_steps_have_no_metrics() -> None: trajectory = AtifTrajectory.model_validate( { diff --git a/services/intake/tests/test_experiment_default_sort.py b/services/intake/tests/test_experiment_default_sort.py index 002decbdf5..41f3460592 100644 --- a/services/intake/tests/test_experiment_default_sort.py +++ b/services/intake/tests/test_experiment_default_sort.py @@ -20,7 +20,7 @@ from nmp.intake.api.v2.experiments.schemas import EvaluationResponse, EvaluatorAggregate EVALUATIONS = "/apis/intake/v2/workspaces/default/evaluations" -GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" +EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" def _exp( @@ -149,7 +149,7 @@ def test_entity_coerces_legacy_non_string_metadata_values() -> None: def test_create_group_defaults_sort_to_created_at(client: TestClient) -> None: - resp = client.post(GROUPS, json={"name": "g-default"}) + resp = client.post(EXPERIMENTS, json={"name": "g-default"}) assert resp.status_code == 201, resp.text assert resp.json()["default_sort"] == "-created_at" @@ -158,19 +158,19 @@ def test_create_group_with_default_sort_round_trips(client: TestClient) -> None: for i, value in enumerate( ("-cost_usd.mean", "-created_at", "-evaluators.reward.mean,cost_usd.mean"), ): - resp = client.post(GROUPS, json={"name": f"g-sort-{i}", "default_sort": value}) + resp = client.post(EXPERIMENTS, json={"name": f"g-sort-{i}", "default_sort": value}) assert resp.status_code == 201, resp.text assert resp.json()["default_sort"] == value def test_create_group_rejects_unsortable_field(client: TestClient) -> None: for i, value in enumerate(("bogus", "cost_usd.bogus")): - resp = client.post(GROUPS, json={"name": f"g-bad-{i}", "default_sort": value}) + resp = client.post(EXPERIMENTS, json={"name": f"g-bad-{i}", "default_sort": value}) assert resp.status_code == 400, resp.text def test_default_order_floats_pinned_first(client: TestClient) -> None: - group = client.post(GROUPS, json={"name": "g-pin"}).json() + group = client.post(EXPERIMENTS, json={"name": "g-pin"}).json() for name in ("exp-a", "exp-b"): created = client.post( EVALUATIONS, json={"name": name, "experiment_group_id": group["id"], "dataset_name": "ds"} diff --git a/services/intake/tests/test_experiment_metric_filter.py b/services/intake/tests/test_experiment_metric_filter.py index fee5bc7ebd..b0476562d7 100644 --- a/services/intake/tests/test_experiment_metric_filter.py +++ b/services/intake/tests/test_experiment_metric_filter.py @@ -23,7 +23,7 @@ from nmp.intake.api.v2.experiments.schemas import EvaluationResponse, EvaluatorAggregate, MetricStatFilters EVALUATIONS = "/apis/intake/v2/workspaces/default/evaluations" -GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" +EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" def _exp( @@ -176,7 +176,7 @@ def test_matches_predicates_resolves_tokens_metric() -> None: def _make_evaluation(client: TestClient, name: str = "exp-1", group: str = "grp-1") -> None: - group_resp = client.post(GROUPS, json={"name": group}) + group_resp = client.post(EXPERIMENTS, json={"name": group}) assert group_resp.status_code == 201, group_resp.text exp_resp = client.post( EVALUATIONS, diff --git a/services/intake/tests/test_experiment_optimization_fields.py b/services/intake/tests/test_experiment_optimization_fields.py index cab86e15e1..ac6da62c77 100644 --- a/services/intake/tests/test_experiment_optimization_fields.py +++ b/services/intake/tests/test_experiment_optimization_fields.py @@ -8,11 +8,11 @@ from fastapi.testclient import TestClient EVALUATIONS = "/apis/intake/v2/workspaces/default/evaluations" -GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" +EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" def _group(client: TestClient, name: str = "grp") -> dict: - resp = client.post(GROUPS, json={"name": name}) + resp = client.post(EXPERIMENTS, json={"name": name}) assert resp.status_code == 201, resp.text return resp.json() @@ -25,7 +25,7 @@ def _evaluation(client: TestClient, group_id: str, name: str) -> dict: def test_group_fields_round_trip(client: TestClient) -> None: resp = client.post( - GROUPS, + EXPERIMENTS, json={"name": "g1", "insight_id": "insight-123", "summary": "looks promising", "metadata": {"k": "v"}}, ) assert resp.status_code == 201, resp.text @@ -140,10 +140,10 @@ def test_filter_experiments_by_metadata(client: TestClient) -> None: def test_filter_experiment_groups_by_metadata(client: TestClient) -> None: for name, team in (("grp-sy", "switchyard"), ("grp-opt", "optimizer")): - resp = client.post(GROUPS, json={"name": name, "metadata": {"team": team}}) + resp = client.post(EXPERIMENTS, json={"name": name, "metadata": {"team": team}}) assert resp.status_code == 201, resp.text - listed = client.get(GROUPS, params={"filter[metadata.team]": "switchyard"}) + listed = client.get(EXPERIMENTS, params={"filter[metadata.team]": "switchyard"}) assert listed.status_code == 200, listed.text assert [g["name"] for g in listed.json()["data"]] == ["grp-sy"] diff --git a/services/intake/tests/test_experiment_sort_endpoint.py b/services/intake/tests/test_experiment_sort_endpoint.py index ca9b38aac1..09e1a8c231 100644 --- a/services/intake/tests/test_experiment_sort_endpoint.py +++ b/services/intake/tests/test_experiment_sort_endpoint.py @@ -11,11 +11,11 @@ from fastapi.testclient import TestClient EVALUATIONS = "/apis/intake/v2/workspaces/default/evaluations" -GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" +EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" def _make_evaluation(client: TestClient, name: str = "exp-1", group: str = "grp-1") -> None: - group_resp = client.post(GROUPS, json={"name": group}) + group_resp = client.post(EXPERIMENTS, json={"name": group}) assert group_resp.status_code == 201, group_resp.text exp_resp = client.post( EVALUATIONS, @@ -79,7 +79,7 @@ def test_too_many_evaluations_to_sort_returns_413(client: TestClient, monkeypatc from nmp.intake.api.v2.experiments import endpoints monkeypatch.setattr(endpoints, "_MAX_GROUP_EVALUATIONS", 2) - group_resp = client.post(GROUPS, json={"name": "big-grp"}) + group_resp = client.post(EXPERIMENTS, json={"name": "big-grp"}) group_id = group_resp.json()["id"] for index in range(3): resp = client.post( diff --git a/services/intake/tests/test_spans_schemas.py b/services/intake/tests/test_spans_schemas.py index efcb800db0..916876675f 100644 --- a/services/intake/tests/test_spans_schemas.py +++ b/services/intake/tests/test_spans_schemas.py @@ -169,9 +169,6 @@ def test_trace_response_maps_core_trace_fields(): assert response.evaluation_context is not None assert response.evaluation_context.evaluation_id == "experiment-a" assert response.evaluation_context.test_case_id == "case-a" - assert response.experiment_context is not None - assert response.experiment_context.experiment_id == "experiment-a" - assert response.experiment_context.test_case_id == "case-a" def test_trace_response_applies_payload_mode_at_api_boundary(): diff --git a/services/intake/tests/test_traces_api.py b/services/intake/tests/test_traces_api.py index 9b5ba1870f..b8739204a4 100644 --- a/services/intake/tests/test_traces_api.py +++ b/services/intake/tests/test_traces_api.py @@ -47,21 +47,12 @@ def test_trace_filter_accepts_evaluation_id(): assert filters.evaluation_id == "experiment-a" -def test_trace_filter_accepts_deprecated_experiment_id_alias(): - filters = _trace_filter( - "workspace-a", - _parsed_filter({"experiment_id": "experiment-a"}), - ) - - assert filters.evaluation_id == "experiment-a" - - -def test_trace_filter_schema_exposes_evaluation_id_with_deprecated_experiment_id_alias(): +def test_trace_filter_schema_exposes_evaluation_id(): properties = TraceFilter.model_json_schema()["properties"] assert properties["evaluation_id"]["description"] == "Filter by root-span evaluation id." assert "deprecated" not in properties["evaluation_id"] - assert properties["experiment_id"]["deprecated"] is True + assert "experiment_id" not in properties assert properties["test_case_id"]["description"] == "Filter by root-span evaluation test case id." assert "deprecated" not in properties["test_case_id"] diff --git a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/cli_generator/cli_config.yaml b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/cli_generator/cli_config.yaml index 56e2e9d1f2..3bdb398ed6 100644 --- a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/cli_generator/cli_config.yaml +++ b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/cli_generator/cli_config.yaml @@ -286,7 +286,7 @@ config: - resource: [evaluations] skip: true -- resource: [experiment_groups] +- resource: [experiments] skip: true - resource: [secrets] diff --git a/web/packages/studio/src/components/ExperimentGroupCreateModal/constants.ts b/web/packages/studio/src/components/ExperimentCreateModal/constants.ts similarity index 75% rename from web/packages/studio/src/components/ExperimentGroupCreateModal/constants.ts rename to web/packages/studio/src/components/ExperimentCreateModal/constants.ts index d43640d4c9..93061d3a09 100644 --- a/web/packages/studio/src/components/ExperimentGroupCreateModal/constants.ts +++ b/web/packages/studio/src/components/ExperimentCreateModal/constants.ts @@ -10,14 +10,14 @@ * its affiliates is strictly prohibited. */ -import { CreateExperimentGroupBody } from '@nemo/sdk/generated/platform/zod/experiment-groups/createExperimentGroup'; +import { CreateExperimentBody } from '@nemo/sdk/generated/platform/zod/experiments/createExperiment'; import { workspaceInputSchema } from '@studio/constants/zod'; // Override the SDK-generated `name` validation — the generated zod uses the DTO's loose // string pattern; we validate against the stricter workspace-name rules so the user sees // a useful inline error instead of a 422 toast. -export const experimentGroupCreateSchema = CreateExperimentGroupBody.extend({ +export const experimentCreateSchema = CreateExperimentBody.extend({ name: workspaceInputSchema, }); -export type ExperimentGroupCreateFormFields = typeof experimentGroupCreateSchema._type; +export type ExperimentCreateFormFields = typeof experimentCreateSchema._type; diff --git a/web/packages/studio/src/components/ExperimentGroupCreateModal/index.tsx b/web/packages/studio/src/components/ExperimentCreateModal/index.tsx similarity index 82% rename from web/packages/studio/src/components/ExperimentGroupCreateModal/index.tsx rename to web/packages/studio/src/components/ExperimentCreateModal/index.tsx index 58e3176825..e613f5418a 100644 --- a/web/packages/studio/src/components/ExperimentGroupCreateModal/index.tsx +++ b/web/packages/studio/src/components/ExperimentCreateModal/index.tsx @@ -13,10 +13,7 @@ import { zodResolver } from '@hookform/resolvers/zod'; import { FormModal, type FormModalProps } from '@nemo/common/src/components/FormModal'; import { useToast } from '@nemo/common/src/providers/toast/useToast'; -import { - getListExperimentGroupsQueryKey, - useCreateExperimentGroup, -} from '@nemo/sdk/generated/platform/api'; +import { getListExperimentsQueryKey, useCreateExperiment } from '@nemo/sdk/generated/platform/api'; import { CodeSnippet, FormField, @@ -32,19 +29,19 @@ import { queryClient } from '@studio/api/queryClient'; import { DefaultSortControl } from '@studio/components/DefaultSortControl'; import { DEFAULT_SORT } from '@studio/components/DefaultSortControl/util'; import { - experimentGroupCreateSchema, - type ExperimentGroupCreateFormFields, -} from '@studio/components/ExperimentGroupCreateModal/constants'; + experimentCreateSchema, + type ExperimentCreateFormFields, +} from '@studio/components/ExperimentCreateModal/constants'; import { handleFormErrorsGeneric } from '@studio/util/forms/error'; import { AxiosError } from 'axios'; import { useState, type FC } from 'react'; import { useForm, type SubmitHandler } from 'react-hook-form'; -export interface ExperimentGroupCreateModalProps extends Pick { +export interface ExperimentCreateModalProps extends Pick { workspace: string; } -export const ExperimentGroupCreateModal: FC = ({ +export const ExperimentCreateModal: FC = ({ open, onClose, workspace, @@ -56,8 +53,8 @@ export const ExperimentGroupCreateModal: FC = ( formState: { errors, isSubmitting }, setValue, setError, - } = useForm({ - resolver: zodResolver(experimentGroupCreateSchema), + } = useForm({ + resolver: zodResolver(experimentCreateSchema), mode: 'onChange', }); @@ -68,10 +65,10 @@ export const ExperimentGroupCreateModal: FC = ( const toast = useToast(); - const { mutateAsync: createExperimentGroup } = useCreateExperimentGroup({ + const { mutateAsync: createExperiment } = useCreateExperiment({ mutation: { onSuccess: () => { - queryClient.invalidateQueries({ queryKey: getListExperimentGroupsQueryKey(workspace) }); + queryClient.invalidateQueries({ queryKey: getListExperimentsQueryKey(workspace) }); }, }, }); @@ -82,9 +79,9 @@ export const ExperimentGroupCreateModal: FC = ( onClose(); }; - const onSubmit: SubmitHandler = async (data) => { + const onSubmit: SubmitHandler = async (data) => { try { - await createExperimentGroup({ + await createExperiment({ workspace, data: { name: data.name, @@ -99,7 +96,11 @@ export const ExperimentGroupCreateModal: FC = ( ? error.response.data.detail : undefined; - if (errorDetail === `Experiment group ${data.name} already exists.`) { + if ( + error instanceof AxiosError && + error.response?.status === 409 && + typeof errorDetail === 'string' + ) { setError('name', { message: errorDetail }); } else { let errorMessage: string; @@ -113,21 +114,21 @@ export const ExperimentGroupCreateModal: FC = ( errorMessage = 'Unknown error'; } - toast.error(`Failed to create experiment group: ${errorMessage}`); + toast.error(`Failed to create experiment: ${errorMessage}`); } } }; return ( = ( > - Create group + Create experiment Coding agent CLI command diff --git a/web/packages/studio/src/components/ExperimentGroupEditModal/index.test.tsx b/web/packages/studio/src/components/ExperimentEditModal/index.test.tsx similarity index 71% rename from web/packages/studio/src/components/ExperimentGroupEditModal/index.test.tsx rename to web/packages/studio/src/components/ExperimentEditModal/index.test.tsx index 5deaa229ce..7477b2ead5 100644 --- a/web/packages/studio/src/components/ExperimentGroupEditModal/index.test.tsx +++ b/web/packages/studio/src/components/ExperimentEditModal/index.test.tsx @@ -1,8 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; -import { ExperimentGroupEditModal } from '@studio/components/ExperimentGroupEditModal'; +import type { ExperimentResponse } from '@nemo/sdk/generated/platform/schema'; +import { ExperimentEditModal } from '@studio/components/ExperimentEditModal'; import { server } from '@studio/mocks/node'; import { render, screen } from '@studio/tests/util/render'; import { waitFor } from '@testing-library/react'; @@ -11,7 +11,7 @@ import { http, HttpResponse } from 'msw'; const WORKSPACE = 'test-workspace'; -const group: ExperimentGroupResponse = { +const group: ExperimentResponse = { id: 'group-id', name: 'opt-group', workspace: WORKSPACE, @@ -21,10 +21,9 @@ const group: ExperimentGroupResponse = { metadata: { producer: 'optimizer' }, default_sort: '-created_at', evaluation_count: 0, - experiment_count: 0, }; -describe('ExperimentGroupEditModal', () => { +describe('ExperimentEditModal', () => { it('resends the fields it does not edit, which a full-replace update would otherwise clear', async () => { let body: Record | undefined; server.use( @@ -40,16 +39,13 @@ describe('ExperimentGroupEditModal', () => { }, }) ), - http.put( - '*/apis/intake/v2/workspaces/:workspace/experiment-groups/:name', - async ({ request }) => { - body = (await request.json()) as Record; - return HttpResponse.json(group); - } - ) + http.put('*/apis/intake/v2/workspaces/:workspace/experiments/:name', async ({ request }) => { + body = (await request.json()) as Record; + return HttpResponse.json(group); + }) ); - render(); + render(); await userEvent.click(await screen.findByRole('button', { name: 'Save' })); diff --git a/web/packages/studio/src/components/ExperimentGroupEditModal/index.tsx b/web/packages/studio/src/components/ExperimentEditModal/index.tsx similarity index 82% rename from web/packages/studio/src/components/ExperimentGroupEditModal/index.tsx rename to web/packages/studio/src/components/ExperimentEditModal/index.tsx index 72064544f5..c3ed2ce456 100644 --- a/web/packages/studio/src/components/ExperimentGroupEditModal/index.tsx +++ b/web/packages/studio/src/components/ExperimentEditModal/index.tsx @@ -4,24 +4,24 @@ import { FormModal, type FormModalProps } from '@nemo/common/src/components/FormModal'; import { useToast } from '@nemo/common/src/providers/toast/useToast'; import { - getGetExperimentGroupQueryKey, - getListExperimentGroupsQueryKey, + getGetExperimentQueryKey, + getListExperimentsQueryKey, useListEvaluations, - useUpdateExperimentGroup, + useUpdateExperiment, } from '@nemo/sdk/generated/platform/api'; -import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import type { ExperimentResponse } from '@nemo/sdk/generated/platform/schema'; import { FormField, Stack, TextArea, TextInput } from '@nvidia/foundations-react-core'; import { queryClient } from '@studio/api/queryClient'; import { DefaultSortControl } from '@studio/components/DefaultSortControl'; import { AxiosError } from 'axios'; import { type FC, type FormEvent, useEffect, useMemo, useState } from 'react'; -export interface ExperimentGroupEditModalProps extends Pick { +export interface ExperimentEditModalProps extends Pick { workspace: string; - group: ExperimentGroupResponse; + group: ExperimentResponse; } -export const ExperimentGroupEditModal: FC = ({ +export const ExperimentEditModal: FC = ({ open, onClose, workspace, @@ -55,13 +55,13 @@ export const ExperimentGroupEditModal: FC = ({ [experimentsPage] ); - const { mutateAsync: updateExperimentGroup, isPending } = useUpdateExperimentGroup({ + const { mutateAsync: updateExperiment, isPending } = useUpdateExperiment({ mutation: { onSuccess: () => { queryClient.invalidateQueries({ - queryKey: getGetExperimentGroupQueryKey(workspace, group.name), + queryKey: getGetExperimentQueryKey(workspace, group.name), }); - queryClient.invalidateQueries({ queryKey: getListExperimentGroupsQueryKey(workspace) }); + queryClient.invalidateQueries({ queryKey: getListExperimentsQueryKey(workspace) }); }, }, }); @@ -69,7 +69,7 @@ export const ExperimentGroupEditModal: FC = ({ const onSubmit = async (e: FormEvent) => { e.preventDefault(); // FormModal doesn't preventDefault; without RHF's handler we must. try { - await updateExperimentGroup({ + await updateExperiment({ workspace, name: group.name, data: { @@ -93,14 +93,14 @@ export const ExperimentGroupEditModal: FC = ({ : error instanceof Error ? error.message : 'Unknown error'; - toast.error(`Failed to update experiment group: ${message}`); + toast.error(`Failed to update experiment: ${message}`); } }; return ( * Pareto view for an experiment group: one point per evaluation, frontier highlighted. The two axes * are picked from the group's metrics and persisted on the group (shared across viewers). */ -export const ExperimentGroupParetoChart: FC = ({ +export const ExperimentParetoChart: FC = ({ workspace, group, preloadedEvaluations, @@ -78,12 +75,12 @@ export const ExperimentGroupParetoChart: FC = ( const [xMetricId, setXMetricId] = useState(group.pareto?.x_metric ?? DEFAULT_X_METRIC); const [yMetricId, setYMetricId] = useState(group.pareto?.y_metric ?? DEFAULT_Y_METRIC); - const { mutate: saveGroup, isPending: isSaving } = useUpdateExperimentGroup({ + const { mutate: saveGroup, isPending: isSaving } = useUpdateExperiment({ mutation: { onSuccess: () => { toast.success('Saved the group default Pareto view.'); queryClient.invalidateQueries({ - queryKey: getGetExperimentGroupQueryKey(workspace, group.name), + queryKey: getGetExperimentQueryKey(workspace, group.name), }); }, onError: () => toast.error('Failed to save the Pareto metrics.'), diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts b/web/packages/studio/src/components/charts/ExperimentParetoChart/useParetoEvaluations.ts similarity index 85% rename from web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts rename to web/packages/studio/src/components/charts/ExperimentParetoChart/useParetoEvaluations.ts index c2c5f0428f..479fc90329 100644 --- a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts +++ b/web/packages/studio/src/components/charts/ExperimentParetoChart/useParetoEvaluations.ts @@ -3,7 +3,7 @@ import { useListEvaluations } from '@nemo/sdk/generated/platform/api'; import type { EvaluationFilter } from '@nemo/sdk/generated/platform/schema'; -import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; +import type { EvaluationRow } from '@studio/components/dataViews/ExperimentDataView/useExperimentEvaluations'; import { DEFAULT_LARGE_PAGE_SIZE } from '@studio/constants/constants'; import { useMemo } from 'react'; @@ -17,17 +17,17 @@ export interface ParetoEvaluations { * (each evaluation already carries the rollup means the chart plots). */ export function useParetoEvaluations( workspace: string, - experimentGroupId: string, + experimentId: string, options?: { enabled?: boolean } ): ParetoEvaluations { // Disabled by callers that already hold the whole group, to avoid a redundant all-evaluations fetch. - const enabled = (options?.enabled ?? true) && !!experimentGroupId; + const enabled = (options?.enabled ?? true) && !!experimentId; const { data, isLoading, isError } = useListEvaluations( workspace, { page: 1, page_size: DEFAULT_LARGE_PAGE_SIZE, - filter: { experiment_id: experimentGroupId } as EvaluationFilter, + filter: { experiment_id: experimentId } as EvaluationFilter, }, { query: { enabled } } ); diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/utils.test.ts b/web/packages/studio/src/components/charts/ExperimentParetoChart/utils.test.ts similarity index 96% rename from web/packages/studio/src/components/charts/ExperimentGroupParetoChart/utils.test.ts rename to web/packages/studio/src/components/charts/ExperimentParetoChart/utils.test.ts index 183fb96e1c..8c64e95642 100644 --- a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/utils.test.ts +++ b/web/packages/studio/src/components/charts/ExperimentParetoChart/utils.test.ts @@ -5,8 +5,8 @@ import { buildParetoPoints, deriveParetoMetrics, type ParetoMetric, -} from '@studio/components/charts/ExperimentGroupParetoChart/utils'; -import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; +} from '@studio/components/charts/ExperimentParetoChart/utils'; +import type { EvaluationRow } from '@studio/components/dataViews/ExperimentDataView/useExperimentEvaluations'; // Only the fields the Pareto accessors read (name + cost/latency/evaluator rollup means) are set; // the rest of the rich EvaluationRow shape is irrelevant here, so build a minimal stand-in. diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/utils.ts b/web/packages/studio/src/components/charts/ExperimentParetoChart/utils.ts similarity index 98% rename from web/packages/studio/src/components/charts/ExperimentGroupParetoChart/utils.ts rename to web/packages/studio/src/components/charts/ExperimentParetoChart/utils.ts index f8976e0746..8069b3dbfe 100644 --- a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/utils.ts +++ b/web/packages/studio/src/components/charts/ExperimentParetoChart/utils.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; +import type { EvaluationRow } from '@studio/components/dataViews/ExperimentDataView/useExperimentEvaluations'; /** Which direction on an axis counts as "better": cost/latency minimize, evaluator scores maximize. */ export type MetricDirection = 'min' | 'max'; diff --git a/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/Empty.tsx b/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/Empty.tsx index 596486cb62..f073a63893 100644 --- a/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/Empty.tsx +++ b/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/Empty.tsx @@ -15,14 +15,14 @@ import { LINK_DOCS_EXPERIMENTS_CLI } from '@studio/constants/links'; import { Bot, ChevronRight, File, FlaskConical, Terminal } from 'lucide-react'; interface EmptyProps { - experimentGroupName: string; + experimentName: string; datasetName: string; } -export const Empty = ({ experimentGroupName, datasetName }: EmptyProps) => { +export const Empty = ({ experimentName, datasetName }: EmptyProps) => { const cliCommand = `nemo exp run \\\n` + - ` --group "${experimentGroupName}" \\\n` + + ` --group "${experimentName}" \\\n` + ` --dataset "${datasetName}" \\\n` + ` --evaluators correctness,helpfulness,groundedness,tool-error`; diff --git a/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.test.tsx b/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.test.tsx index c6dad702bc..b64d6e6739 100644 --- a/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.test.tsx +++ b/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.test.tsx @@ -15,7 +15,7 @@ vi.mock('use-debounce', () => ({ })); const WORKSPACE = 'default'; -const EXPERIMENT_GROUP = 'my-group'; +const EXPERIMENT = 'my-group'; const EXPERIMENT_NAME = 'my-experiment'; const TRACE_ID = 'trace-abc-123'; @@ -43,21 +43,21 @@ const mockSessionsPage = { const mockEvaluation = { workspace: WORKSPACE, name: EXPERIMENT_NAME, - experiment_group_name: EXPERIMENT_GROUP, + experiment_group_name: EXPERIMENT, run_count: 1, evaluator_names: [], }; const renderDataView = () => renderRoute(undefined, { - history: getEvaluationDetailRoute(WORKSPACE, EXPERIMENT_GROUP, EXPERIMENT_NAME), + history: getEvaluationDetailRoute(WORKSPACE, EXPERIMENT, EXPERIMENT_NAME), routes: [ { path: ROUTES.workspace.evaluationDetail, element: ( ), }, diff --git a/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.tsx b/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.tsx index 2a166ac8cb..71d0c6e99e 100644 --- a/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.tsx @@ -39,7 +39,7 @@ type SessionRow = EvaluationSessionResponse & { _rowId: string }; interface EvaluationSessionsDataViewProps { evaluationName: string; - experimentGroupName: string; + experimentName: string; } const mapStatusForBadge = (status: EvaluationSessionResponse['status']) => @@ -97,7 +97,7 @@ const getSessionSortParam = (sortingState: { id: string; desc: boolean }[]): str export const EvaluationSessionsDataView: FC = ({ evaluationName, - experimentGroupName, + experimentName, }) => { const workspace = useWorkspaceFromPath(); const navigate = useNavigate(); @@ -303,7 +303,7 @@ export const EvaluationSessionsDataView: FC = ( navigate( getEvaluationSessionTraceDetailRoute( workspace, - experimentGroupName, + experimentName, evaluationName, row.session_id, row.trace_id @@ -355,7 +355,7 @@ export const EvaluationSessionsDataView: FC = ( } return ( '} /> ); diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.test.tsx b/web/packages/studio/src/components/dataViews/ExperimentDataView/AddToGroupModal.test.tsx similarity index 81% rename from web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.test.tsx rename to web/packages/studio/src/components/dataViews/ExperimentDataView/AddToGroupModal.test.tsx index 8eb330a505..4446338452 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.test.tsx +++ b/web/packages/studio/src/components/dataViews/ExperimentDataView/AddToGroupModal.test.tsx @@ -2,18 +2,18 @@ // SPDX-License-Identifier: Apache-2.0 import { MockToastProvider } from '@nemo/common/src/tests/MockToastProvider'; -import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; -import { AddToGroupModal } from '@studio/components/dataViews/ExperimentGroupDataView/AddToGroupModal'; -import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; +import type { ExperimentResponse } from '@nemo/sdk/generated/platform/schema'; +import { AddToGroupModal } from '@studio/components/dataViews/ExperimentDataView/AddToGroupModal'; +import type { EvaluationRow } from '@studio/components/dataViews/ExperimentDataView/useExperimentEvaluations'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { render, screen, waitFor } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; // Hoisted mocks for the SDK hooks the modal calls. -const { mockMutateAsync, mockCreateMutateAsync, mockUseListExperimentGroups } = vi.hoisted(() => ({ +const { mockMutateAsync, mockCreateMutateAsync, mockUseListExperiments } = vi.hoisted(() => ({ mockMutateAsync: vi.fn<(...args: unknown[]) => Promise>(), mockCreateMutateAsync: vi.fn<(...args: unknown[]) => Promise>(), - mockUseListExperimentGroups: vi.fn<() => { data: unknown; isLoading: boolean }>(), + mockUseListExperiments: vi.fn<() => { data: unknown; isLoading: boolean }>(), })); vi.mock('@nemo/sdk/generated/platform/api', async () => { @@ -22,21 +22,20 @@ vi.mock('@nemo/sdk/generated/platform/api', async () => { ); return { ...actual, - useListExperimentGroups: () => mockUseListExperimentGroups(), + useListExperiments: () => mockUseListExperiments(), usePatchEvaluation: () => ({ mutateAsync: mockMutateAsync, isPending: false }), - useCreateExperimentGroup: () => ({ mutateAsync: mockCreateMutateAsync, isPending: false }), + useCreateExperiment: () => ({ mutateAsync: mockCreateMutateAsync, isPending: false }), }; }); -const makeGroup = (id: string, name: string): ExperimentGroupResponse => ({ +const makeGroup = (id: string, name: string): ExperimentResponse => ({ id, name, workspace: 'default', default_sort: '-created_at', - experiment_count: 0, }); -const GROUPS: ExperimentGroupResponse[] = [ +const GROUPS: ExperimentResponse[] = [ makeGroup('g1', 'Alpha benchmarks'), makeGroup('g2', 'Beta benchmarks'), makeGroup('g3', 'Gamma benchmarks'), @@ -81,18 +80,20 @@ describe('AddToGroupModal', () => { mockMutateAsync.mockResolvedValue(undefined); mockCreateMutateAsync.mockReset(); mockCreateMutateAsync.mockResolvedValue(makeGroup('g-new', 'Regression suite')); - mockUseListExperimentGroups.mockReset(); - mockUseListExperimentGroups.mockReturnValue({ + mockUseListExperiments.mockReset(); + mockUseListExperiments.mockReturnValue({ data: { data: GROUPS }, isLoading: false, }); }); - it('offers "Create new group" plus only groups not every selected evaluation already belongs to', async () => { + it('offers "Create new experiment" plus only groups not every selected evaluation already belongs to', async () => { const user = userEvent.setup(); renderModal(); - await user.click(screen.getByRole('combobox', { name: /experiment group/i })); - expect(await screen.findByRole('option', { name: /create new group/i })).toBeInTheDocument(); + await user.click(screen.getByRole('combobox', { name: /experiment/i })); + expect( + await screen.findByRole('option', { name: /create new experiment/i }) + ).toBeInTheDocument(); // g1: both evals are members -> excluded. g2: only eval-2 is a member -> still offered. g3: none. expect(screen.getByRole('option', { name: 'Beta benchmarks' })).toBeInTheDocument(); expect(screen.getByRole('option', { name: 'Gamma benchmarks' })).toBeInTheDocument(); @@ -103,7 +104,7 @@ describe('AddToGroupModal', () => { const user = userEvent.setup(); const { onSuccess } = renderModal(); - await user.click(screen.getByRole('combobox', { name: /experiment group/i })); + await user.click(screen.getByRole('combobox', { name: /experiment/i })); await user.click(await screen.findByRole('option', { name: 'Gamma benchmarks' })); await user.click(screen.getByRole('button', { name: 'Add' })); @@ -126,8 +127,8 @@ describe('AddToGroupModal', () => { const user = userEvent.setup(); const { onSuccess } = renderModal(); - await user.click(screen.getByRole('combobox', { name: /experiment group/i })); - await user.click(await screen.findByRole('option', { name: /create new group/i })); + await user.click(screen.getByRole('combobox', { name: /experiment/i })); + await user.click(await screen.findByRole('option', { name: /create new experiment/i })); await user.type(screen.getByRole('textbox', { name: /^name$/i }), 'regression-suite'); await user.click(screen.getByRole('button', { name: 'Create & add' })); @@ -150,18 +151,20 @@ describe('AddToGroupModal', () => { await waitFor(() => expect(onSuccess).toHaveBeenCalledTimes(1)); }); - it('still offers "Create new group" when every group already contains all selected evaluations', async () => { + it('still offers "Create new experiment" when every group already contains all selected evaluations', async () => { const user = userEvent.setup(); - mockUseListExperimentGroups.mockReturnValue({ + mockUseListExperiments.mockReturnValue({ data: { data: [makeGroup('g1', 'Alpha benchmarks')] }, isLoading: false, }); renderModal([makeEvaluation('eval-1', ['g1']), makeEvaluation('eval-2', ['g1'])]); - const trigger = screen.getByRole('combobox', { name: /experiment group/i }); + const trigger = screen.getByRole('combobox', { name: /experiment/i }); expect(trigger).toBeEnabled(); await user.click(trigger); - expect(await screen.findByRole('option', { name: /create new group/i })).toBeInTheDocument(); + expect( + await screen.findByRole('option', { name: /create new experiment/i }) + ).toBeInTheDocument(); expect(screen.queryByRole('option', { name: 'Alpha benchmarks' })).not.toBeInTheDocument(); }); }); diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.tsx b/web/packages/studio/src/components/dataViews/ExperimentDataView/AddToGroupModal.tsx similarity index 86% rename from web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.tsx rename to web/packages/studio/src/components/dataViews/ExperimentDataView/AddToGroupModal.tsx index fa931472b6..ee664df19c 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.tsx +++ b/web/packages/studio/src/components/dataViews/ExperimentDataView/AddToGroupModal.tsx @@ -6,9 +6,9 @@ import { FormModal, type FormModalProps } from '@nemo/common/src/components/Form import { useToast } from '@nemo/common/src/providers/toast/useToast'; import { getListEvaluationsQueryKey, - getListExperimentGroupsQueryKey, - useCreateExperimentGroup, - useListExperimentGroups, + getListExperimentsQueryKey, + useCreateExperiment, + useListExperiments, usePatchEvaluation, } from '@nemo/sdk/generated/platform/api'; import { @@ -23,13 +23,13 @@ import { TextArea, TextInput, } from '@nvidia/foundations-react-core'; -import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; +import type { EvaluationRow } from '@studio/components/dataViews/ExperimentDataView/useExperimentEvaluations'; import { DefaultSortControl } from '@studio/components/DefaultSortControl'; import { DEFAULT_SORT } from '@studio/components/DefaultSortControl/util'; import { - experimentGroupCreateSchema, - type ExperimentGroupCreateFormFields, -} from '@studio/components/ExperimentGroupCreateModal/constants'; + experimentCreateSchema, + type ExperimentCreateFormFields, +} from '@studio/components/ExperimentCreateModal/constants'; import { DEFAULT_LARGE_PAGE_SIZE } from '@studio/constants/constants'; import { useQueryClient } from '@tanstack/react-query'; import { AxiosError } from 'axios'; @@ -50,7 +50,7 @@ export interface AddToGroupModalProps extends Pick = ({ const [selectedGroupId, setSelectedGroupId] = useState(''); const isCreating = selectedGroupId === CREATE_NEW; // Default sort is a single `sort`-param string driven by a custom control (not a registered RHF - // input), so it's managed here and merged into the create payload (mirrors ExperimentGroupCreateModal). + // input), so it's managed here and merged into the create payload (mirrors ExperimentCreateModal). const [defaultSort, setDefaultSort] = useState(DEFAULT_SORT); const { @@ -82,8 +82,8 @@ export const AddToGroupModal: FC = ({ setValue, setError, formState: { errors, isValid }, - } = useForm({ - resolver: zodResolver(experimentGroupCreateSchema), + } = useForm({ + resolver: zodResolver(experimentCreateSchema), mode: 'onChange', }); @@ -97,7 +97,7 @@ export const AddToGroupModal: FC = ({ }, [open, reset]); // Only fetch groups while the modal is open. A single large page covers any realistic group count. - const { data: groupsPage, isLoading } = useListExperimentGroups( + const { data: groupsPage, isLoading } = useListExperiments( workspace, { page_size: DEFAULT_LARGE_PAGE_SIZE }, { query: { enabled: open && !!workspace } } @@ -118,8 +118,7 @@ export const AddToGroupModal: FC = ({ ); const { mutateAsync: patchEvaluation, isPending: isAdding } = usePatchEvaluation(); - const { mutateAsync: createExperimentGroup, isPending: isCreatingGroup } = - useCreateExperimentGroup(); + const { mutateAsync: createExperiment, isPending: isCreatingGroup } = useCreateExperiment(); const busy = isAdding || isCreatingGroup; const count = evaluations.length; @@ -171,10 +170,10 @@ export const AddToGroupModal: FC = ({ finishAdds(groupName, await associateEvaluations(selectedGroupId), 'Added to'); }; - const createGroupAndAdd: SubmitHandler = async (data) => { + const createGroupAndAdd: SubmitHandler = async (data) => { let created; try { - created = await createExperimentGroup({ + created = await createExperiment({ workspace, data: { name: data.name, @@ -184,9 +183,13 @@ export const AddToGroupModal: FC = ({ }); } catch (error) { // Creation failed (e.g. duplicate name) — surface inline on the name field where possible and - // keep the modal open. No group was created, so there's nothing to add. + // keep the modal open. No experiment was created, so there's nothing to add. const detail = error instanceof AxiosError ? error.response?.data?.detail : undefined; - if (detail === `Experiment group ${data.name} already exists.`) { + if ( + error instanceof AxiosError && + error.response?.status === 409 && + typeof detail === 'string' + ) { setError('name', { message: detail }); return; } @@ -196,11 +199,11 @@ export const AddToGroupModal: FC = ({ : error instanceof Error ? error.message : 'Unknown error'; - toast.error(`Failed to create experiment group: ${message}`); + toast.error(`Failed to create experiment: ${message}`); return; } // Group now exists; adding evaluations is best-effort (a group with fewer evals is still valid). - queryClient.invalidateQueries({ queryKey: getListExperimentGroupsQueryKey(workspace) }); + queryClient.invalidateQueries({ queryKey: getListExperimentsQueryKey(workspace) }); finishAdds(created.name, await associateEvaluations(created.id), 'Created'); }; @@ -225,8 +228,8 @@ export const AddToGroupModal: FC = ({ return ( = ({ className="w-[560px]" > - + = ({ > { - if (v === CREATE_NEW) return 'Create new group'; + if (v === CREATE_NEW) return 'Create new experiment'; return typeof v === 'string' && v ? (groupNameById.get(v) ?? undefined) : undefined; }} /> @@ -264,7 +267,7 @@ export const AddToGroupModal: FC = ({ - Create new group + Create new experiment diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/Empty.tsx b/web/packages/studio/src/components/dataViews/ExperimentDataView/Empty.tsx similarity index 91% rename from web/packages/studio/src/components/dataViews/ExperimentGroupDataView/Empty.tsx rename to web/packages/studio/src/components/dataViews/ExperimentDataView/Empty.tsx index 49e7c05ad9..65b8d99699 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/Empty.tsx +++ b/web/packages/studio/src/components/dataViews/ExperimentDataView/Empty.tsx @@ -15,11 +15,11 @@ import { LINK_DOCS_EXPERIMENTS_CLI } from '@studio/constants/links'; import { Bot, ChevronRight, File, FlaskConical, Terminal } from 'lucide-react'; interface EmptyProps { - experimentGroupName: string; + experimentName: string; } -export const Empty = ({ experimentGroupName }: EmptyProps) => { - const escapedGroupName = experimentGroupName.replace(/'/g, "'\\''"); +export const Empty = ({ experimentName }: EmptyProps) => { + const escapedGroupName = experimentName.replace(/'/g, "'\\''"); const cliCommand = `nemo exp run \\\n` + ` --group '${escapedGroupName}' \\\n` + @@ -30,7 +30,7 @@ export const Empty = ({ experimentGroupName }: EmptyProps) => { } header="No Evaluations" - emptyMessage="Run an experiment to see results for this group." + emptyMessage="Run an evaluation to see results for this experiment." actions={
diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/ExperimentGroupDataView.css b/web/packages/studio/src/components/dataViews/ExperimentDataView/ExperimentDataView.css similarity index 90% rename from web/packages/studio/src/components/dataViews/ExperimentGroupDataView/ExperimentGroupDataView.css rename to web/packages/studio/src/components/dataViews/ExperimentDataView/ExperimentDataView.css index 5380d265cd..d389c3f262 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/ExperimentGroupDataView.css +++ b/web/packages/studio/src/components/dataViews/ExperimentDataView/ExperimentDataView.css @@ -7,13 +7,13 @@ * between those two adjacent action columns. Suppress the divider on a left-pinned cell that is * immediately followed by another left-pinned cell, so the two read as one action area — the * divider at the pinned/scroll boundary (the last left-pinned column) is kept. Scoped to this - * table via .experiment-group-data-view so no other DataView is affected. + * table via .experiment-data-view so no other DataView is affected. * * The shared divider rule is `table.studio-data-view-table.nv-table-root [data-pinned='left']` * (specificity 0,3,1). Match the same `table.studio-data-view-table` base plus our scope class so * this rule out-specifies it (0,4,1) — without it, the shared rule wins and the divider stays. */ -.experiment-group-data-view +.experiment-data-view table.studio-data-view-table [data-pinned='left']:has(+ [data-pinned='left']) { box-shadow: none; diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/MeanValueTooltipCell.tsx b/web/packages/studio/src/components/dataViews/ExperimentDataView/MeanValueTooltipCell.tsx similarity index 100% rename from web/packages/studio/src/components/dataViews/ExperimentGroupDataView/MeanValueTooltipCell.tsx rename to web/packages/studio/src/components/dataViews/ExperimentDataView/MeanValueTooltipCell.tsx diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx b/web/packages/studio/src/components/dataViews/ExperimentDataView/index.tsx similarity index 94% rename from web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx rename to web/packages/studio/src/components/dataViews/ExperimentDataView/index.tsx index c816f8b850..9e43c10c69 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/ExperimentDataView/index.tsx @@ -18,24 +18,21 @@ import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataView import { useToast } from '@nemo/common/src/providers/toast/useToast'; import { formatDurationMs } from '@nemo/common/src/utils/date'; import { formatEvaluatorScore, snakeCaseToTitleCase } from '@nemo/common/src/utils/formatters'; -import type { - EvaluationFilter, - ExperimentGroupResponse, -} from '@nemo/sdk/generated/platform/schema'; +import type { EvaluationFilter, ExperimentResponse } from '@nemo/sdk/generated/platform/schema'; import { Button, Text, Tooltip } from '@nvidia/foundations-react-core'; import { ChangesetBadge } from '@studio/components/ChangesetBadge'; -import { ExperimentGroupParetoChart } from '@studio/components/charts/ExperimentGroupParetoChart'; -import { AddToGroupModal } from '@studio/components/dataViews/ExperimentGroupDataView/AddToGroupModal'; -import '@studio/components/dataViews/ExperimentGroupDataView/ExperimentGroupDataView.css'; -import { Empty } from '@studio/components/dataViews/ExperimentGroupDataView/Empty'; -import { MeanValueTooltipCell } from '@studio/components/dataViews/ExperimentGroupDataView/MeanValueTooltipCell'; +import { ExperimentParetoChart } from '@studio/components/charts/ExperimentParetoChart'; +import { AddToGroupModal } from '@studio/components/dataViews/ExperimentDataView/AddToGroupModal'; +import '@studio/components/dataViews/ExperimentDataView/ExperimentDataView.css'; +import { Empty } from '@studio/components/dataViews/ExperimentDataView/Empty'; +import { MeanValueTooltipCell } from '@studio/components/dataViews/ExperimentDataView/MeanValueTooltipCell'; import { type EvaluationRow, type ListEvaluationsSortParam, - useExperimentGroupEvaluations, -} from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; -import { useSortErrorRecovery } from '@studio/components/dataViews/ExperimentGroupDataView/useSortErrorRecovery'; -import { deriveEvaluatorNames } from '@studio/components/dataViews/ExperimentGroupDataView/util'; + useExperimentEvaluations, +} from '@studio/components/dataViews/ExperimentDataView/useExperimentEvaluations'; +import { useSortErrorRecovery } from '@studio/components/dataViews/ExperimentDataView/useSortErrorRecovery'; +import { deriveEvaluatorNames } from '@studio/components/dataViews/ExperimentDataView/util'; import { QuickActionsMenuRoot } from '@studio/components/QuickActionsMenu/QuickActionsMenuRoot'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { getEvaluationDetailRoute } from '@studio/routes/utils'; @@ -132,25 +129,22 @@ const getEvaluationSortParam = ( return fields.join(',') as ListEvaluationsSortParam; }; -interface ExperimentGroupDataViewProps { +interface ExperimentDataViewProps { /** The loaded group, so the table's initial sort can seed from `default_sort` at first * render — the sorting state is initialized once and not reactive. */ - group: ExperimentGroupResponse; + group: ExperimentResponse; /** Whether the Pareto (cost-vs-accuracy) chart is shown. The toggle lives in the parent * route header (next to the "Evaluations" title); this component only renders the chart. */ paretoVisible: boolean; } /** Lists the experiments that belong to a single experiment group. */ -export const ExperimentGroupDataView: FC = ({ - group, - paretoVisible, -}) => { +export const ExperimentDataView: FC = ({ group, paretoVisible }) => { const workspace = useWorkspaceFromPath(); const navigate = useNavigate(); const toast = useToast(); - const experimentGroupName = group.name; - const experimentGroupId = group.id; + const experimentName = group.name; + const experimentId = group.id; // The bulk "Add to group" modal state: the selected evaluations plus a way to clear the row // selection on success. Null when the modal is closed. @@ -161,7 +155,7 @@ export const ExperimentGroupDataView: FC = ({ // Persist column order to localStorage, keyed by experiment group ID. const [savedColumnOrder, saveColumnOrder] = useLocalStorage( - `nemo-studio:experiment-group-columns:${experimentGroupId}`, + `nemo-studio:experiment-columns:${experimentId}`, [] ); @@ -199,9 +193,9 @@ export const ExperimentGroupDataView: FC = ({ error, isLoading, isSuccess, - } = useExperimentGroupEvaluations({ + } = useExperimentEvaluations({ workspace, - experimentGroupId, + experimentId, filter: dataViewState.apiFilter.filter, search: dataViewState.debouncedSearchBar, page, @@ -495,13 +489,13 @@ export const ExperimentGroupDataView: FC = ({ { // `fill-none` overrides the menu's cloned `fill: 'solid'`, which would otherwise paint // the folder body over the "+"/"-" (they're drawn before the folder shape). - label: 'Add to group', + label: 'Add to experiment', icon: , onSelect: () => setAddToGroupState({ evaluations: [row.original], clearSelection: () => {} }), }, { - label: 'Remove from group', + label: 'Remove from experiment', icon: , onSelect: () => removeFromGroup(row.original), }, @@ -537,7 +531,7 @@ export const ExperimentGroupDataView: FC = ({
{/* Key by group id so the axis selection resets (re-seeds from the new group's saved config) when navigating between groups without a route remount. */} - = ({ makeColumns={makeColumns} searchField="name" onRowClick={(row) => - navigate(getEvaluationDetailRoute(workspace, experimentGroupName, row.name)) + navigate(getEvaluationDetailRoute(workspace, experimentName, row.name)) } renderBulkActions={({ selectedRows, table }) => ( )} toolbarSlotEnd={ @@ -588,13 +582,13 @@ export const ExperimentGroupDataView: FC = ({ data: orderedData, totalCount, requestStatus: isLoading ? 'loading' : undefined, - className: 'experiment-group-data-view', + className: 'experiment-data-view', }, DataViewTableContent: { enableColumnReordering: true, renderEmptyState: ({ hasFiltersApplied, hasSearchApplied }) => hasFiltersApplied || hasSearchApplied ? null : ( - + ), }, }} diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations.test.ts b/web/packages/studio/src/components/dataViews/ExperimentDataView/useExperimentEvaluations.test.ts similarity index 87% rename from web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations.test.ts rename to web/packages/studio/src/components/dataViews/ExperimentDataView/useExperimentEvaluations.test.ts index 1dfa1ad7bc..2688949a02 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations.test.ts +++ b/web/packages/studio/src/components/dataViews/ExperimentDataView/useExperimentEvaluations.test.ts @@ -11,9 +11,9 @@ import { } from '@nemo/sdk/generated/platform/api'; import { type EvaluationRow, - useExperimentGroupEvaluations, - type UseExperimentGroupEvaluationsParams, -} from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; + useExperimentEvaluations, + type UseExperimentEvaluationsParams, +} from '@studio/components/dataViews/ExperimentDataView/useExperimentEvaluations'; import { renderHook } from '@testing-library/react'; vi.mock('@nemo/common/src/providers/toast/useToast'); @@ -62,9 +62,9 @@ const mockLists = ( : queryResult(unpinned.rows, unpinned.total)) as typeof useListEvaluations); }; -const baseParams: UseExperimentGroupEvaluationsParams = { +const baseParams: UseExperimentEvaluationsParams = { workspace: 'ws', - experimentGroupId: 'grp', + experimentId: 'grp', filter: undefined, search: '', page: 1, @@ -72,7 +72,7 @@ const baseParams: UseExperimentGroupEvaluationsParams = { sort: '-created_at', }; -describe('useExperimentGroupEvaluations', () => { +describe('useExperimentEvaluations', () => { beforeEach(() => { vi.clearAllMocks(); mockUseToast.mockReturnValue({ error: vi.fn() } as unknown as ReturnType); @@ -95,7 +95,7 @@ describe('useExperimentGroupEvaluations', () => { { rows: Array.from({ length: 50 }, (_unused, i) => unp(`u${i}`)), total: 50 } ); - const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); + const { result } = renderHook(() => useExperimentEvaluations(baseParams)); expect(result.current.totalCount).toBe(50); }); @@ -103,7 +103,7 @@ describe('useExperimentGroupEvaluations', () => { it('falls back to the pinned count when nothing is unpinned so a fully-pinned group is not empty', () => { mockLists({ rows: [pin('p1'), pin('p2')], total: 2 }, { rows: [], total: 0 }); - const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); + const { result } = renderHook(() => useExperimentEvaluations(baseParams)); expect(result.current.totalCount).toBe(2); }); @@ -111,7 +111,7 @@ describe('useExperimentGroupEvaluations', () => { it('reports a zero count only when both lists are empty', () => { mockLists({ rows: [], total: 0 }, { rows: [], total: 0 }); - const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); + const { result } = renderHook(() => useExperimentEvaluations(baseParams)); expect(result.current.totalCount).toBe(0); }); @@ -120,7 +120,7 @@ describe('useExperimentGroupEvaluations', () => { // 'b' appears in both lists (the brief window where the two queries refetch out of step). mockLists({ rows: [pin('a'), pin('b')], total: 2 }, { rows: [unp('b'), unp('c')], total: 2 }); - const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); + const { result } = renderHook(() => useExperimentEvaluations(baseParams)); expect(result.current.rows.map((row) => row.name)).toEqual(['a', 'b', 'c']); }); @@ -128,7 +128,7 @@ describe('useExperimentGroupEvaluations', () => { it('fetches the full pinned set in one page and paginates the unpinned set by the caller page size', () => { mockLists({ rows: [], total: 0 }, { rows: [], total: 0 }); - renderHook(() => useExperimentGroupEvaluations({ ...baseParams, page: 2, pageSize: 25 })); + renderHook(() => useExperimentEvaluations({ ...baseParams, page: 2, pageSize: 25 })); const calls = mockUseListEvaluations.mock.calls; const pinnedParams = calls.find( @@ -156,7 +156,7 @@ describe('useExperimentGroupEvaluations', () => { error: null, } as unknown as ReturnType)) as typeof useListEvaluations); - const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); + const { result } = renderHook(() => useExperimentEvaluations(baseParams)); expect(result.current.isLoading).toBe(true); }); @@ -164,7 +164,7 @@ describe('useExperimentGroupEvaluations', () => { it('clears loading once both queries have responded', () => { mockLists({ rows: [pin('p')], total: 1 }, { rows: [unp('u')], total: 1 }); - const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); + const { result } = renderHook(() => useExperimentEvaluations(baseParams)); expect(result.current.isLoading).toBe(false); }); @@ -182,7 +182,7 @@ describe('useExperimentGroupEvaluations', () => { error: null, } as unknown as ReturnType)) as typeof useListEvaluations); - const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); + const { result } = renderHook(() => useExperimentEvaluations(baseParams)); expect(result.current.isSuccess).toBe(false); }); @@ -190,7 +190,7 @@ describe('useExperimentGroupEvaluations', () => { it('reports isSuccess once the unpinned query has loaded the current sort', () => { mockLists({ rows: [pin('p')], total: 1 }, { rows: [unp('u')], total: 1 }); - const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); + const { result } = renderHook(() => useExperimentEvaluations(baseParams)); expect(result.current.isSuccess).toBe(true); }); @@ -210,7 +210,7 @@ describe('useExperimentGroupEvaluations', () => { error: null, } as unknown as ReturnType)) as typeof useListEvaluations); - const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); + const { result } = renderHook(() => useExperimentEvaluations(baseParams)); expect(result.current.isSuccess).toBe(false); }); @@ -218,7 +218,7 @@ describe('useExperimentGroupEvaluations', () => { it('scopes pin/unpin invalidation to this group, not the whole workspace', () => { mockLists({ rows: [], total: 0 }, { rows: [], total: 0 }); - renderHook(() => useExperimentGroupEvaluations(baseParams)); + renderHook(() => useExperimentEvaluations(baseParams)); // onSuccess is the group-scoped invalidate the hook wires into both mutations. const onSuccess = mockUsePinEvaluation.mock.calls[0]?.[0]?.mutation?.onSuccess as | (() => void) @@ -240,7 +240,7 @@ describe('useExperimentGroupEvaluations', () => { const error = vi.fn(); mockUseToast.mockReturnValue({ error } as unknown as ReturnType); - const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); + const { result } = renderHook(() => useExperimentEvaluations(baseParams)); result.current.removeFromGroup({ name: 'only-here', experiment_ids: ['grp'], @@ -257,7 +257,7 @@ describe('useExperimentGroupEvaluations', () => { mutate: patch, } as unknown as ReturnType); - const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); + const { result } = renderHook(() => useExperimentEvaluations(baseParams)); result.current.removeFromGroup({ name: 'multi', experiment_ids: ['grp', 'other'], diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations.ts b/web/packages/studio/src/components/dataViews/ExperimentDataView/useExperimentEvaluations.ts similarity index 93% rename from web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations.ts rename to web/packages/studio/src/components/dataViews/ExperimentDataView/useExperimentEvaluations.ts index 34d219f1c3..fb26985725 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations.ts +++ b/web/packages/studio/src/components/dataViews/ExperimentDataView/useExperimentEvaluations.ts @@ -35,9 +35,9 @@ const toRows = (experiments: EvaluationResponse[] | undefined): EvaluationRow[] */ const MAX_PINNED_ROWS = 100; -export interface UseExperimentGroupEvaluationsParams { +export interface UseExperimentEvaluationsParams { workspace: string; - experimentGroupId: string; + experimentId: string; filter: Partial | undefined; search: string; page: number; @@ -46,7 +46,7 @@ export interface UseExperimentGroupEvaluationsParams { sort?: ListEvaluationsSortParam; } -export interface ExperimentGroupEvaluations { +export interface ExperimentEvaluations { /** Pinned rows first (newest-pinned first), then the current page of unpinned rows. */ rows: EvaluationRow[]; /** Pins the row if unpinned, unpins it otherwise, then refetches both lists. */ @@ -80,15 +80,15 @@ export interface ExperimentGroupEvaluations { * once and repeated atop every page rather than paginated. A pin/unpin persists through the API, * then invalidates both lists so the new state is refetched (no optimistic update). */ -export function useExperimentGroupEvaluations({ +export function useExperimentEvaluations({ workspace, - experimentGroupId, + experimentId, filter, search, page, pageSize, sort, -}: UseExperimentGroupEvaluationsParams): ExperimentGroupEvaluations { +}: UseExperimentEvaluationsParams): ExperimentEvaluations { const queryClient = useQueryClient(); const toast = useToast(); @@ -96,12 +96,12 @@ export function useExperimentGroupEvaluations({ ...filter, ...(search && { name: { $like: search } }), // Spread last so the group scope can't be overridden by a user filter. - experiment_id: experimentGroupId, + experiment_id: experimentId, }; const listQueryOptions = { query: { placeholderData: keepPreviousData, - enabled: !!experimentGroupId, + enabled: !!experimentId, }, }; @@ -154,10 +154,10 @@ export function useExperimentGroupEvaluations({ () => queryClient.invalidateQueries({ queryKey: getListEvaluationsQueryKey(workspace, { - filter: { experiment_id: experimentGroupId }, + filter: { experiment_id: experimentId }, }), }), - [queryClient, workspace, experimentGroupId] + [queryClient, workspace, experimentId] ); const { mutate: pinEvaluation } = usePinEvaluation({ @@ -193,10 +193,10 @@ export function useExperimentGroupEvaluations({ const { mutate: patchEvaluation } = usePatchEvaluation({ mutation: { onSuccess: () => { - toast.success('Removed the evaluation from this group.'); + toast.success('Removed the evaluation from this experiment.'); return invalidateList(); }, - onError: () => toast.error('Failed to remove the evaluation from this group.'), + onError: () => toast.error('Failed to remove the evaluation from this experiment.'), onSettled: (_data, _error, { name }) => { pendingRef.current.delete(name); }, @@ -210,17 +210,17 @@ export function useExperimentGroupEvaluations({ // The API rejects an empty `experiment_ids`, so block removing an evaluation's only group and // explain why rather than firing a request that would 400. if (row.experiment_ids.length <= 1) { - toast.error('This evaluation must belong to at least one experiment group.'); + toast.error('This evaluation must belong to at least one experiment.'); return; } pendingRef.current.add(name); patchEvaluation({ workspace, name, - data: { experiment_ids: row.experiment_ids.filter((id) => id !== experimentGroupId) }, + data: { experiment_ids: row.experiment_ids.filter((id) => id !== experimentId) }, }); }, - [workspace, experimentGroupId, patchEvaluation, toast] + [workspace, experimentId, patchEvaluation, toast] ); // Pinned first, then the current page of unpinned. Drop any unpinned row already shown as pinned — diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useSortErrorRecovery.test.ts b/web/packages/studio/src/components/dataViews/ExperimentDataView/useSortErrorRecovery.test.ts similarity index 98% rename from web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useSortErrorRecovery.test.ts rename to web/packages/studio/src/components/dataViews/ExperimentDataView/useSortErrorRecovery.test.ts index 38896bc116..b190d288b6 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useSortErrorRecovery.test.ts +++ b/web/packages/studio/src/components/dataViews/ExperimentDataView/useSortErrorRecovery.test.ts @@ -7,7 +7,7 @@ import { type SortingState, useSortErrorRecovery, type UseSortErrorRecoveryParams, -} from '@studio/components/dataViews/ExperimentGroupDataView/useSortErrorRecovery'; +} from '@studio/components/dataViews/ExperimentDataView/useSortErrorRecovery'; import { renderHook } from '@testing-library/react'; const sort = (id: string, desc = false): SortingState => [{ id, desc }]; diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useSortErrorRecovery.ts b/web/packages/studio/src/components/dataViews/ExperimentDataView/useSortErrorRecovery.ts similarity index 100% rename from web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useSortErrorRecovery.ts rename to web/packages/studio/src/components/dataViews/ExperimentDataView/useSortErrorRecovery.ts diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/util.test.ts b/web/packages/studio/src/components/dataViews/ExperimentDataView/util.test.ts similarity index 98% rename from web/packages/studio/src/components/dataViews/ExperimentGroupDataView/util.test.ts rename to web/packages/studio/src/components/dataViews/ExperimentDataView/util.test.ts index a6939b1067..efcea57645 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/util.test.ts +++ b/web/packages/studio/src/components/dataViews/ExperimentDataView/util.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { deriveEvaluatorNames } from '@studio/components/dataViews/ExperimentGroupDataView/util'; +import { deriveEvaluatorNames } from '@studio/components/dataViews/ExperimentDataView/util'; describe('deriveEvaluatorNames', () => { it('returns the sorted, de-duplicated union of evaluator names across rows', () => { diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/util.ts b/web/packages/studio/src/components/dataViews/ExperimentDataView/util.ts similarity index 100% rename from web/packages/studio/src/components/dataViews/ExperimentGroupDataView/util.ts rename to web/packages/studio/src/components/dataViews/ExperimentDataView/util.ts diff --git a/web/packages/studio/src/constants/routes.ts b/web/packages/studio/src/constants/routes.ts index 8f28e719f1..0966b2a26d 100644 --- a/web/packages/studio/src/constants/routes.ts +++ b/web/packages/studio/src/constants/routes.ts @@ -38,7 +38,7 @@ export const ROUTE_PARAMS = { jobName: 'jobName', /** Benchmark entity name segment under evaluation/benchmarks/:name */ benchmarkName: 'benchmarkName', - experimentGroupName: 'experimentGroupName', + experimentName: 'experimentName', insightId: 'insightId', evaluationName: 'evaluationName', guardrailConfigName: 'guardrailConfigName', @@ -75,9 +75,9 @@ export const ROUTES = { evaluationResultDetails: `/workspaces/:${P.workspace}/evaluation/results/:${P.evaluationJobId}`, /** Empty landing page for the EXPERIMENT feature (gated by VITE_FF_EXPERIMENT). */ experiment: `/workspaces/:${P.workspace}/experiment`, - experimentGroupDetail: `/workspaces/:${P.workspace}/experiment/:${P.experimentGroupName}`, - evaluationDetail: `/workspaces/:${P.workspace}/experiment/:${P.experimentGroupName}/:${P.evaluationName}`, - evaluationSessionDetail: `/workspaces/:${P.workspace}/experiment/:${P.experimentGroupName}/:${P.evaluationName}/sessions/:${P.sessionId}`, + experimentDetail: `/workspaces/:${P.workspace}/experiment/:${P.experimentName}`, + evaluationDetail: `/workspaces/:${P.workspace}/experiment/:${P.experimentName}/:${P.evaluationName}`, + evaluationSessionDetail: `/workspaces/:${P.workspace}/experiment/:${P.experimentName}/:${P.evaluationName}/sessions/:${P.sessionId}`, customizationJobList: `/workspaces/:${P.workspace}/customizations`, customizationJobDetails: `/workspaces/:${P.workspace}/customizations/:${P.customizationJobName}`, filesets: `/workspaces/:${P.workspace}/filesets`, diff --git a/web/packages/studio/src/mocks/handlers.ts b/web/packages/studio/src/mocks/handlers.ts index f3776da6e0..9d3833565d 100644 --- a/web/packages/studio/src/mocks/handlers.ts +++ b/web/packages/studio/src/mocks/handlers.ts @@ -24,7 +24,7 @@ import { workspacesHandlers } from '@studio/mocks/handlers/workspaces'; import { mockEvaluationSessionsPage, mockEvaluationsPage, - mockExperimentGroup, + mockExperiment, } from '@studio/mocks/intake/experiments'; import { createMockAnnotation, @@ -485,8 +485,8 @@ export const handlers = [ const session = mockSessionById(String(params['sessionId'])); return session ? HttpResponse.json(session) : new HttpResponse(null, { status: 404 }); }), - http.get('*/apis/intake/v2/workspaces/:workspace/experiment-groups/:name', ({ params }) => - HttpResponse.json(mockExperimentGroup(String(params['name']))) + http.get('*/apis/intake/v2/workspaces/:workspace/experiments/:name', ({ params }) => + HttpResponse.json(mockExperiment(String(params['name']))) ), http.get('*/apis/intake/v2/workspaces/:workspace/evaluations', () => HttpResponse.json(mockEvaluationsPage()) diff --git a/web/packages/studio/src/mocks/intake/experiments.ts b/web/packages/studio/src/mocks/intake/experiments.ts index 304e21feeb..f688f49e3b 100644 --- a/web/packages/studio/src/mocks/intake/experiments.ts +++ b/web/packages/studio/src/mocks/intake/experiments.ts @@ -6,20 +6,19 @@ import type { EvaluationResponsesPage, EvaluationSessionResponse, EvaluationSessionResponsesPage, - ExperimentGroupResponse, + ExperimentResponse, } from '@nemo/sdk/generated/platform/schema'; const WORKSPACE = 'default'; /** The experiment group the EvaluationSessionDetailRoute tests navigate under. */ -export const mockExperimentGroup = (name: string): ExperimentGroupResponse => ({ +export const mockExperiment = (name: string): ExperimentResponse => ({ id: `grp_${name}`, name, workspace: WORKSPACE, description: '', default_sort: 'started_at', evaluation_count: 2, - experiment_count: 2, }); /** Evaluation names that live in the mock group (each fans out one runs query). */ @@ -31,8 +30,6 @@ const mockEvaluation = (name: string): EvaluationResponse => ({ workspace: WORKSPACE, experiment_ids: ['grp_my-group'], dataset_name: 'sample-dataset', - // Deprecated readonly aliases the generated type still requires. - parent_experiment_id: '', experiment_group_id: 'grp_my-group', }); @@ -47,7 +44,6 @@ const mockRun = ( ): EvaluationSessionResponse => ({ workspace: WORKSPACE, evaluation_name: evaluationName, - experiment_name: evaluationName, session_id: sessionId, test_case_id: testCaseId, trace_id: `trace-${sessionId}`, diff --git a/web/packages/studio/src/mocks/intake/telemetry.ts b/web/packages/studio/src/mocks/intake/telemetry.ts index 77edc39d3e..bc1650656f 100644 --- a/web/packages/studio/src/mocks/intake/telemetry.ts +++ b/web/packages/studio/src/mocks/intake/telemetry.ts @@ -39,10 +39,6 @@ const trace1: Trace = { evaluation_id: 'support-policy-smoke', test_case_id: 'case-0042', }, - experiment_context: { - experiment_id: 'support-policy-smoke', - test_case_id: 'case-0042', - }, }; const trace2: Trace = { diff --git a/web/packages/studio/src/routes/EvaluationDetailRoute/index.test.tsx b/web/packages/studio/src/routes/EvaluationDetailRoute/index.test.tsx index 295f3d58c1..1efb5de4a4 100644 --- a/web/packages/studio/src/routes/EvaluationDetailRoute/index.test.tsx +++ b/web/packages/studio/src/routes/EvaluationDetailRoute/index.test.tsx @@ -1,10 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { - EvaluationResponse, - ExperimentGroupResponse, -} from '@nemo/sdk/generated/platform/schema'; +import type { EvaluationResponse, ExperimentResponse } from '@nemo/sdk/generated/platform/schema'; import { ROUTES } from '@studio/constants/routes'; import { server } from '@studio/mocks/node'; import { EvaluationDetailRoute } from '@studio/routes/EvaluationDetailRoute'; @@ -35,7 +32,7 @@ const group = { insight_id: 'insight-id', default_sort: '-created_at', evaluation_count: 0, -} satisfies Partial; +} satisfies Partial; describe('EvaluationDetailRoute with Optimizer enabled', () => { it('renders the originating insight description instead of relabeling the evaluation description', async () => { @@ -43,7 +40,7 @@ describe('EvaluationDetailRoute with Optimizer enabled', () => { http.get('*/apis/intake/v2/workspaces/:workspace/evaluations/:name', () => HttpResponse.json(evaluation) ), - http.get('*/apis/intake/v2/workspaces/:workspace/experiment-groups/:name', () => + http.get('*/apis/intake/v2/workspaces/:workspace/experiments/:name', () => HttpResponse.json(group) ), http.get('*/apis/intake/v2/workspaces/:workspace/evaluations/:name/sessions', () => diff --git a/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx b/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx index dccc9645f1..db6535d0fe 100644 --- a/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx +++ b/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { useGetEvaluation, useGetExperimentGroup } from '@nemo/sdk/generated/platform/api'; +import { useGetEvaluation, useGetExperiment } from '@nemo/sdk/generated/platform/api'; import { Badge, Card, Flex, PageHeader, Stack, Text } from '@nvidia/foundations-react-core'; import { useOptimizerGetInsight } from '@studio/api/optimizer'; import { AccessibleTitle } from '@studio/components/AccessibleTitle'; @@ -12,20 +12,20 @@ import { ROUTE_PARAMS } from '@studio/constants/routes'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; import { EvaluationDetailMetrics } from '@studio/routes/EvaluationDetailRoute/EvaluationDetailMetrics'; -import { getExperimentGroupDetailRoute, getExperimentRoute } from '@studio/routes/utils'; +import { getExperimentDetailRoute, getExperimentRoute } from '@studio/routes/utils'; import { useRequiredPathParams } from '@studio/util/hooks/useRequiredPathParams'; import { type FC } from 'react'; export const EvaluationDetailRoute: FC = () => { const workspace = useWorkspaceFromPath(); - const { experimentGroupName, evaluationName } = useRequiredPathParams([ - ROUTE_PARAMS.experimentGroupName, + const { experimentName, evaluationName } = useRequiredPathParams([ + ROUTE_PARAMS.experimentName, ROUTE_PARAMS.evaluationName, ]); const { data: evaluation } = useGetEvaluation(workspace, evaluationName); // Evaluations reach their originating insight through the owning group's insight_id. - const { data: experimentGroup } = useGetExperimentGroup(workspace, experimentGroupName); - const insightId = experimentGroup?.insight_id ?? ''; + const { data: experiment } = useGetExperiment(workspace, experimentName); + const insightId = experiment?.insight_id ?? ''; const { data: insight } = useOptimizerGetInsight(workspace, insightId, { query: { enabled: OPTIMIZER_ENABLED && Boolean(insightId) }, }); @@ -33,10 +33,10 @@ export const EvaluationDetailRoute: FC = () => { useBreadcrumbs({ items: [ - { href: getExperimentRoute(workspace), slotLabel: 'Experiment Groups' }, + { href: getExperimentRoute(workspace), slotLabel: 'Experiments' }, { - href: getExperimentGroupDetailRoute(workspace, experimentGroupName), - slotLabel: experimentGroupName, + href: getExperimentDetailRoute(workspace, experimentName), + slotLabel: experimentName, }, { slotLabel: evaluationName }, ], @@ -73,7 +73,7 @@ export const EvaluationDetailRoute: FC = () => {
diff --git a/web/packages/studio/src/routes/EvaluationDetailRoute/optimizerDisabled.test.tsx b/web/packages/studio/src/routes/EvaluationDetailRoute/optimizerDisabled.test.tsx index 6f6125a36a..cb9ed238dd 100644 --- a/web/packages/studio/src/routes/EvaluationDetailRoute/optimizerDisabled.test.tsx +++ b/web/packages/studio/src/routes/EvaluationDetailRoute/optimizerDisabled.test.tsx @@ -1,10 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { - EvaluationResponse, - ExperimentGroupResponse, -} from '@nemo/sdk/generated/platform/schema'; +import type { EvaluationResponse, ExperimentResponse } from '@nemo/sdk/generated/platform/schema'; import { ROUTES } from '@studio/constants/routes'; import { server } from '@studio/mocks/node'; import { EvaluationDetailRoute } from '@studio/routes/EvaluationDetailRoute'; @@ -35,7 +32,7 @@ const group = { insight_id: 'insight-id', default_sort: '-created_at', evaluation_count: 0, -} satisfies Partial; +} satisfies Partial; describe('EvaluationDetailRoute with Optimizer disabled', () => { it('renders the evaluation description without requesting or linking to the insight', async () => { @@ -44,7 +41,7 @@ describe('EvaluationDetailRoute with Optimizer disabled', () => { http.get('*/apis/intake/v2/workspaces/:workspace/evaluations/:name', () => HttpResponse.json(evaluation) ), - http.get('*/apis/intake/v2/workspaces/:workspace/experiment-groups/:name', () => + http.get('*/apis/intake/v2/workspaces/:workspace/experiments/:name', () => HttpResponse.json(group) ), http.get('*/apis/intake/v2/workspaces/:workspace/evaluations/:name/sessions', () => diff --git a/web/packages/studio/src/routes/EvaluationSessionDetailRoute/TestCaseCompare.tsx b/web/packages/studio/src/routes/EvaluationSessionDetailRoute/TestCaseCompare.tsx index 0189fcc5f5..9d5effdf91 100644 --- a/web/packages/studio/src/routes/EvaluationSessionDetailRoute/TestCaseCompare.tsx +++ b/web/packages/studio/src/routes/EvaluationSessionDetailRoute/TestCaseCompare.tsx @@ -10,13 +10,13 @@ import { useBreadcrumbs, } from '@studio/providers/breadcrumbs/useBreadcrumbs'; import { SessionCompareColumn } from '@studio/routes/EvaluationSessionDetailRoute/SessionCompareColumn'; -import { getExperimentGroupDetailRoute, getExperimentRoute } from '@studio/routes/utils'; +import { getExperimentDetailRoute, getExperimentRoute } from '@studio/routes/utils'; import { CircleAlert } from 'lucide-react'; import { type FC, type ReactNode, useEffect } from 'react'; interface TestCaseCompareProps { workspace: string; - experimentGroupName: string; + experimentName: string; testCaseId: string | null | undefined; /** The run shown in the left column (the session the route is on). */ primarySessionId: string; @@ -40,7 +40,7 @@ const CompareEmpty: FC<{ heading: string; message: string }> = ({ heading, messa export const TestCaseCompare: FC = ({ workspace, - experimentGroupName, + experimentName, testCaseId, primarySessionId, primaryRun, @@ -53,15 +53,15 @@ export const TestCaseCompare: FC = ({ useEffect(() => { const breadcrumbs: BreadcrumbsItemProps[] = [ - { slotLabel: 'Experiment Groups', href: getExperimentRoute(workspace) }, + { slotLabel: 'Experiments', href: getExperimentRoute(workspace) }, { - slotLabel: experimentGroupName, - href: getExperimentGroupDetailRoute(workspace, experimentGroupName), + slotLabel: experimentName, + href: getExperimentDetailRoute(workspace, experimentName), }, { slotLabel: 'Test case comparison' }, ]; setBreadcrumbs(breadcrumbs); - }, [setBreadcrumbs, workspace, experimentGroupName]); + }, [setBreadcrumbs, workspace, experimentName]); const heading = testCaseId ? `Test case comparison — Test case ${testCaseId}` diff --git a/web/packages/studio/src/routes/EvaluationSessionDetailRoute/index.test.tsx b/web/packages/studio/src/routes/EvaluationSessionDetailRoute/index.test.tsx index 872ef5f2ec..b377892de5 100644 --- a/web/packages/studio/src/routes/EvaluationSessionDetailRoute/index.test.tsx +++ b/web/packages/studio/src/routes/EvaluationSessionDetailRoute/index.test.tsx @@ -12,7 +12,7 @@ const renderSessionDetail = (search = '') => history: `/workspaces/default/experiment/my-group/my-experiment/sessions/session-agent-run-001${search}`, routes: [ { - path: '/workspaces/:workspace/experiment/:experimentGroupName/:evaluationName/sessions/:sessionId', + path: '/workspaces/:workspace/experiment/:experimentName/:evaluationName/sessions/:sessionId', element: , }, ], diff --git a/web/packages/studio/src/routes/EvaluationSessionDetailRoute/index.tsx b/web/packages/studio/src/routes/EvaluationSessionDetailRoute/index.tsx index c6c1bd0690..3e9070c1e0 100644 --- a/web/packages/studio/src/routes/EvaluationSessionDetailRoute/index.tsx +++ b/web/packages/studio/src/routes/EvaluationSessionDetailRoute/index.tsx @@ -13,7 +13,7 @@ import { SessionDetailContent } from '@studio/routes/IntakeSessionDetailRoute'; import { getEvaluationDetailRoute, getEvaluationSessionDetailRoute, - getExperimentGroupDetailRoute, + getExperimentDetailRoute, getExperimentRoute, } from '@studio/routes/utils'; import { useRequiredPathParams } from '@studio/util/hooks/useRequiredPathParams'; @@ -26,22 +26,15 @@ import { useSearchParams } from 'react-router-dom'; */ const EvaluationSessionCompare: FC<{ workspace: string; - experimentGroupName: string; + experimentName: string; sessionId: string; compareWith: string; onSelectCompare: (sessionId: string) => void; onClearCompare: () => void; -}> = ({ - workspace, - experimentGroupName, - sessionId, - compareWith, - onSelectCompare, - onClearCompare, -}) => { +}> = ({ workspace, experimentName, sessionId, compareWith, onSelectCompare, onClearCompare }) => { const { testCaseId, runs, isRunsLoading } = useSessionCompareRuns( workspace, - experimentGroupName, + experimentName, sessionId ); const primaryRun = runs.find((run) => run.session_id === sessionId); @@ -50,7 +43,7 @@ const EvaluationSessionCompare: FC<{ return ( void; -}> = ({ workspace, experimentGroupName, sessionId, routeContext, onSelectCompare }) => { - const { runs, isRunsLoading } = useSessionCompareRuns(workspace, experimentGroupName, sessionId); +}> = ({ workspace, experimentName, sessionId, routeContext, onSelectCompare }) => { + const { runs, isRunsLoading } = useSessionCompareRuns(workspace, experimentName, sessionId); const contextWithCompare = useMemo( () => @@ -120,9 +113,9 @@ export const EvaluationSessionDetailRoute: FC = () => { // compareWith carries the session_id of the run to show in the right column. const compareWith = searchParams.get(QUERY_PARAMETERS.compareWith); - const { sessionId, experimentGroupName, evaluationName } = useRequiredPathParams([ + const { sessionId, experimentName, evaluationName } = useRequiredPathParams([ ROUTE_PARAMS.sessionId, - ROUTE_PARAMS.experimentGroupName, + ROUTE_PARAMS.experimentName, ROUTE_PARAMS.evaluationName, ]); @@ -149,32 +142,27 @@ export const EvaluationSessionDetailRoute: FC = () => { () => ({ kind: 'evaluation', parentBreadcrumbs: [ - { slotLabel: 'Experiment Groups', href: getExperimentRoute(workspace) }, + { slotLabel: 'Experiments', href: getExperimentRoute(workspace) }, { - slotLabel: experimentGroupName, - href: getExperimentGroupDetailRoute(workspace, experimentGroupName), + slotLabel: experimentName, + href: getExperimentDetailRoute(workspace, experimentName), }, { slotLabel: evaluationName, - href: getEvaluationDetailRoute(workspace, experimentGroupName, evaluationName), + href: getEvaluationDetailRoute(workspace, experimentName, evaluationName), }, ], getSessionHref: (targetSessionId) => - getEvaluationSessionDetailRoute( - workspace, - experimentGroupName, - evaluationName, - targetSessionId - ), + getEvaluationSessionDetailRoute(workspace, experimentName, evaluationName, targetSessionId), }), - [workspace, experimentGroupName, evaluationName] + [workspace, experimentName, evaluationName] ); if (compareWith) { return ( { return ( = ({ - experimentGroupName, -}) => { +export const ExperimentMetrics: FC = ({ experimentName }) => { const workspace = useWorkspaceFromPath(); - const { data: group } = useGetExperimentGroup(workspace, experimentGroupName); + const { data: group } = useGetExperiment(workspace, experimentName); return (
diff --git a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.test.tsx b/web/packages/studio/src/routes/ExperimentDetailRoute/index.test.tsx similarity index 79% rename from web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.test.tsx rename to web/packages/studio/src/routes/ExperimentDetailRoute/index.test.tsx index b786824948..b44bdacc90 100644 --- a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.test.tsx +++ b/web/packages/studio/src/routes/ExperimentDetailRoute/index.test.tsx @@ -1,10 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import type { ExperimentResponse } from '@nemo/sdk/generated/platform/schema'; import { ROUTES } from '@studio/constants/routes'; import { server } from '@studio/mocks/node'; -import { ExperimentGroupDetailRoute } from '@studio/routes/ExperimentGroupDetailRoute'; +import { ExperimentDetailRoute } from '@studio/routes/ExperimentDetailRoute'; import { renderRoute, screen } from '@studio/tests/util/render'; import { http, HttpResponse } from 'msw'; @@ -24,12 +24,12 @@ const group = { insight_id: 'insight-id', default_sort: '-created_at', evaluation_count: 0, -} satisfies Partial; +} satisfies Partial; -const mockGroup = (overrides?: Partial) => { +const mockGroup = (overrides?: Partial) => { const insightRequest = vi.fn(); server.use( - http.get('*/apis/intake/v2/workspaces/:workspace/experiment-groups/:name', () => + http.get('*/apis/intake/v2/workspaces/:workspace/experiments/:name', () => HttpResponse.json({ ...group, ...overrides }) ), http.get('*/apis/intake/v2/workspaces/:workspace/evaluations', () => @@ -50,12 +50,12 @@ const mockGroup = (overrides?: Partial) => { }) ); - renderRoute(, { + renderRoute(, { history: `/workspaces/${WORKSPACE}/experiment/${GROUP_NAME}`, routes: [ { - path: ROUTES.workspace.experimentGroupDetail, - element: , + path: ROUTES.workspace.experimentDetail, + element: , }, ], }); @@ -63,7 +63,7 @@ const mockGroup = (overrides?: Partial) => { return insightRequest; }; -describe('ExperimentGroupDetailRoute', () => { +describe('ExperimentDetailRoute', () => { it('renders the group description and summary without requesting Optimizer when disabled', async () => { const insightRequest = mockGroup(); diff --git a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx b/web/packages/studio/src/routes/ExperimentDetailRoute/index.tsx similarity index 79% rename from web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx rename to web/packages/studio/src/routes/ExperimentDetailRoute/index.tsx index 11c89ccf44..d4ebddde78 100644 --- a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx +++ b/web/packages/studio/src/routes/ExperimentDetailRoute/index.tsx @@ -2,28 +2,28 @@ // SPDX-License-Identifier: Apache-2.0 import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; -import { useGetExperimentGroup } from '@nemo/sdk/generated/platform/api'; +import { useGetExperiment } from '@nemo/sdk/generated/platform/api'; import { Button, Card, Flex, PageHeader, Stack, Text } from '@nvidia/foundations-react-core'; import { useOptimizerGetInsight } from '@studio/api/optimizer'; import { AccessibleTitle } from '@studio/components/AccessibleTitle'; -import { ExperimentGroupDataView } from '@studio/components/dataViews/ExperimentGroupDataView'; -import { ExperimentGroupEditModal } from '@studio/components/ExperimentGroupEditModal'; +import { ExperimentDataView } from '@studio/components/dataViews/ExperimentDataView'; +import { ExperimentEditModal } from '@studio/components/ExperimentEditModal'; import { OriginatingInsightLink } from '@studio/components/OriginatingInsightLink'; import { OPTIMIZER_ENABLED } from '@studio/constants/environment'; import { ROUTE_PARAMS } from '@studio/constants/routes'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; -import { ExperimentGroupMetrics } from '@studio/routes/ExperimentGroupDetailRoute/ExperimentGroupMetrics'; +import { ExperimentMetrics } from '@studio/routes/ExperimentDetailRoute/ExperimentMetrics'; import { getExperimentRoute } from '@studio/routes/utils'; import { useLocalStorage } from '@studio/util/hooks/useLocalStorage'; import { useRequiredPathParams } from '@studio/util/hooks/useRequiredPathParams'; import { ChartScatter, Pencil } from 'lucide-react'; import { type FC, useState } from 'react'; -export const ExperimentGroupDetailRoute: FC = () => { +export const ExperimentDetailRoute: FC = () => { const workspace = useWorkspaceFromPath(); - const { experimentGroupName } = useRequiredPathParams([ROUTE_PARAMS.experimentGroupName]); - const { data: group, error } = useGetExperimentGroup(workspace, experimentGroupName); + const { experimentName } = useRequiredPathParams([ROUTE_PARAMS.experimentName]); + const { data: group, error } = useGetExperiment(workspace, experimentName); // The insight is a group-level concept, reached via the group's insight_id. const { data: insight } = useOptimizerGetInsight(workspace, group?.insight_id ?? '', { query: { enabled: OPTIMIZER_ENABLED && Boolean(group?.insight_id) }, @@ -32,24 +32,24 @@ export const ExperimentGroupDetailRoute: FC = () => { // Pareto (cost-vs-accuracy) view visibility, persisted per group. Hidden by default. const [storedParetoVisible, setParetoVisible] = useLocalStorage( - `nemo-studio:experiment-group-pareto:${group?.id ?? ''}`, + `nemo-studio:experiment-pareto:${group?.id ?? ''}`, false ); const paretoVisible = storedParetoVisible ?? false; useBreadcrumbs({ items: [ - { href: getExperimentRoute(workspace), slotLabel: 'Experiment Groups' }, - { slotLabel: experimentGroupName }, + { href: getExperimentRoute(workspace), slotLabel: 'Experiments' }, + { slotLabel: experimentName }, ], }); return ( - + setEditOpen(true)}> @@ -59,18 +59,18 @@ export const ExperimentGroupDetailRoute: FC = () => { } /> {error ? ( - + ) : ( <> {group && ( - setEditOpen(false)} workspace={workspace} group={group} /> )} - + {(insight?.description || group?.summary) && (
{insight?.description ? ( @@ -108,7 +108,7 @@ export const ExperimentGroupDetailRoute: FC = () => { )}
- {group && } + {group && }
)} diff --git a/web/packages/studio/src/routes/ExperimentRoute/ExperimentGroupCard.tsx b/web/packages/studio/src/routes/ExperimentRoute/ExperimentCard.tsx similarity index 72% rename from web/packages/studio/src/routes/ExperimentRoute/ExperimentGroupCard.tsx rename to web/packages/studio/src/routes/ExperimentRoute/ExperimentCard.tsx index ee1b3051be..b57ea700cd 100644 --- a/web/packages/studio/src/routes/ExperimentRoute/ExperimentGroupCard.tsx +++ b/web/packages/studio/src/routes/ExperimentRoute/ExperimentCard.tsx @@ -1,35 +1,35 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import type { ExperimentResponse } from '@nemo/sdk/generated/platform/schema'; import { Card, Text } from '@nvidia/foundations-react-core'; import { Metric } from '@studio/routes/ExperimentRoute/Metric'; import { UpdatedAt } from '@studio/routes/ExperimentRoute/UpdatedAt'; -import { getExperimentGroupDetailRoute } from '@studio/routes/utils'; +import { getExperimentDetailRoute } from '@studio/routes/utils'; import { type FC } from 'react'; import { useNavigate } from 'react-router-dom'; -interface ExperimentGroupCardProps { - group: ExperimentGroupResponse; +interface ExperimentCardProps { + group: ExperimentResponse; workspace: string; } -export const ExperimentGroupCard: FC = ({ group, workspace }) => { +export const ExperimentCard: FC = ({ group, workspace }) => { const navigate = useNavigate(); return ( navigate(getExperimentGroupDetailRoute(workspace, group.name))} + onClick={() => navigate(getExperimentDetailRoute(workspace, group.name))} role="button" tabIndex={0} onKeyDown={(e) => { if (e.key === 'Enter') { - navigate(getExperimentGroupDetailRoute(workspace, group.name)); + navigate(getExperimentDetailRoute(workspace, group.name)); } else if (e.key === ' ') { e.preventDefault(); - navigate(getExperimentGroupDetailRoute(workspace, group.name)); + navigate(getExperimentDetailRoute(workspace, group.name)); } }} > diff --git a/web/packages/studio/src/routes/ExperimentRoute/index.tsx b/web/packages/studio/src/routes/ExperimentRoute/index.tsx index 891646959b..e43c7820b1 100644 --- a/web/packages/studio/src/routes/ExperimentRoute/index.tsx +++ b/web/packages/studio/src/routes/ExperimentRoute/index.tsx @@ -10,11 +10,8 @@ import * as DataView from '@nemo/common/src/components/DataView/internal'; import { StudioDataView } from '@nemo/common/src/components/DataView/StudioDataView'; import { DEFAULT_PAGE_SIZE_OPTIONS } from '@nemo/common/src/constants/pagination'; import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; -import { useListExperimentGroups } from '@nemo/sdk/generated/platform/api'; -import type { - ExperimentGroupFilter, - ExperimentGroupResponse, -} from '@nemo/sdk/generated/platform/schema'; +import { useListExperiments } from '@nemo/sdk/generated/platform/api'; +import type { ExperimentFilter, ExperimentResponse } from '@nemo/sdk/generated/platform/schema'; import { Block, Button, @@ -33,12 +30,12 @@ import { Text, } from '@nvidia/foundations-react-core'; import { AccessibleTitle } from '@studio/components/AccessibleTitle'; -import { ExperimentGroupCreateModal } from '@studio/components/ExperimentGroupCreateModal'; +import { ExperimentCreateModal } from '@studio/components/ExperimentCreateModal'; import { FeatureFlagBadge } from '@studio/components/FeatureFlagBadge'; import { Loading } from '@studio/components/Layouts/Loading'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; -import { ExperimentGroupCard } from '@studio/routes/ExperimentRoute/ExperimentGroupCard'; +import { ExperimentCard } from '@studio/routes/ExperimentRoute/ExperimentCard'; import { keepPreviousData } from '@tanstack/react-query'; import { CircleAlert } from 'lucide-react'; import { type ComponentProps, type FC, useMemo, useState } from 'react'; @@ -46,25 +43,24 @@ import { type ComponentProps, type FC, useMemo, useState } from 'react'; const DEFAULT_PAGE_SIZE = 10; /** - * Column filters available for experiment groups. `created_at` / `updated_at` are base entity + * Column filters available for experiments. `created_at` / `updated_at` are base entity * fields the list endpoint filters via `$gte` / `$lte` ranges. They are not declared on the - * generated `ExperimentGroupFilter` type, so the API filter is widened here and coerced back at + * generated `ExperimentFilter` type, so the API filter is widened here and coerced back at * the SDK boundary (see `filter` below). */ -interface ExperimentGroupColumnFilters { +interface ExperimentColumnFilters { created_at?: DatetimeFilterValue; updated_at?: DatetimeFilterValue; } -type ExperimentGroupFilterInput = WithFilterOperators & - ExperimentGroupColumnFilters; +type ExperimentFilterInput = WithFilterOperators & ExperimentColumnFilters; /** * Filter-only columns. They are never rendered as a table (the cards come from CustomContent); * they exist solely to feed `meta.filter` into the DataView filter panel and applied-filter tags. */ const makeFilterColumns: ComponentProps< - typeof DataView.Root + typeof DataView.Root >['makeColumns'] = ({ accessor }) => [ accessor('created_at', { id: 'created_at', @@ -81,12 +77,12 @@ const makeFilterColumns: ComponentProps< ]; export const ExperimentRoute: FC = () => { - useBreadcrumbs({ items: [{ slotLabel: 'Experiment Groups' }] }); + useBreadcrumbs({ items: [{ slotLabel: 'Experiments' }] }); const workspace = useWorkspaceFromPath(); const [isCreateModalOpen, setIsCreateModalOpen] = useState(false); - const dataViewState = useStudioDataViewState({ + const dataViewState = useStudioDataViewState({ defaultPageSize: DEFAULT_PAGE_SIZE, }); const page = dataViewState.pagination.state.pageIndex + 1; @@ -94,70 +90,70 @@ export const ExperimentRoute: FC = () => { const searchText = dataViewState.apiFilter.searchText; const columnFilters = dataViewState.apiFilter.filter; - const filter = useMemo(() => { + const filter = useMemo(() => { if (!searchText && !columnFilters) return undefined; - const input: ExperimentGroupFilterInput = { + const input: ExperimentFilterInput = { ...columnFilters, ...(searchText ? { name: { $like: searchText } } : {}), }; - return input as ExperimentGroupFilter; + return input as ExperimentFilter; }, [columnFilters, searchText]); - const { data, isLoading, error } = useListExperimentGroups( + const { data, isLoading, error } = useListExperiments( workspace, { page, page_size: pageSize, filter }, { query: { placeholderData: keepPreviousData } } ); - const groups = data?.data ?? []; + const experiments = data?.data ?? []; const totalResults = data?.pagination?.total_results ?? 0; return ( - + - Experiment groups + Experiments } - slotDescription="Manage groups for online optimization. Review reports down to the frame level." + slotDescription="Manage experiments for online optimization. Review runs down to the frame level." slotActions={ } /> - setIsCreateModalOpen(false)} workspace={workspace} /> - + dataViewState={dataViewState} makeColumns={makeFilterColumns} searchField="name" attributes={{ DataViewRoot: { - data: groups, + data: experiments, totalCount: totalResults, requestStatus: error ? 'error' : isLoading ? 'loading' : undefined, }, - DataViewSearchBar: { placeholder: 'Search experiment groups...' }, + DataViewSearchBar: { placeholder: 'Search experiments...' }, }} > - + renderLoadingState={() => } renderEmptyState={({ hasSearchApplied, hasFiltersApplied }) => ( {hasSearchApplied || hasFiltersApplied - ? 'No experiment groups match your search or filters.' - : 'No experiment groups yet.'} + ? 'No experiments match your search or filters.' + : 'No experiments yet.'} )} @@ -175,7 +171,7 @@ export const ExperimentRoute: FC = () => { {({ rows }) => ( {rows.map((row) => ( - default: module.ExperimentRoute, })) ); -const ExperimentGroupDetailRoute = lazy(() => - import('@studio/routes/ExperimentGroupDetailRoute').then((module) => ({ - default: module.ExperimentGroupDetailRoute, +const ExperimentDetailRoute = lazy(() => + import('@studio/routes/ExperimentDetailRoute').then((module) => ({ + default: module.ExperimentDetailRoute, })) ); const EvaluationDetailRoute = lazy(() => @@ -32,7 +32,7 @@ export const experimentRoutes: RouteObject[] = gateExperimentRoutes([ { path: ROUTES.workspace.experiment, element: , - errorElement: , + errorElement: , }, { path: ROUTES.workspace.evaluationSessionDetail, @@ -40,9 +40,9 @@ export const experimentRoutes: RouteObject[] = gateExperimentRoutes([ errorElement: , }, { - path: ROUTES.workspace.experimentGroupDetail, - element: , - errorElement: , + path: ROUTES.workspace.experimentDetail, + element: , + errorElement: , }, { path: ROUTES.workspace.evaluationDetail, diff --git a/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/InsightExperimentGroups.tsx b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/InsightExperiments.tsx similarity index 88% rename from web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/InsightExperimentGroups.tsx rename to web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/InsightExperiments.tsx index d2b3917b3c..39acd079a5 100644 --- a/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/InsightExperimentGroups.tsx +++ b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/InsightExperiments.tsx @@ -7,16 +7,16 @@ import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; import { DEFAULT_PAGE_SIZE_OPTIONS } from '@nemo/common/src/constants/pagination'; -import { useListExperimentGroups } from '@nemo/sdk/generated/platform/api'; -import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import { useListExperiments } from '@nemo/sdk/generated/platform/api'; +import type { ExperimentResponse } from '@nemo/sdk/generated/platform/schema'; import { Button, Text } from '@nvidia/foundations-react-core'; -import { getExperimentGroupDetailRoute } from '@studio/routes/utils'; +import { getExperimentDetailRoute } from '@studio/routes/utils'; import { keepPreviousData } from '@tanstack/react-query'; import { FlaskConical } from 'lucide-react'; import { type ComponentProps, type FC } from 'react'; import { useNavigate } from 'react-router-dom'; -const makeColumns: ComponentProps>['makeColumns'] = ({ +const makeColumns: ComponentProps>['makeColumns'] = ({ accessor, }) => [ accessor('name', { @@ -44,14 +44,14 @@ const makeColumns: ComponentProps> }), ]; -interface InsightExperimentGroupsProps { +interface InsightExperimentsProps { workspace: string; insightId: string; onRunExperiment: () => void; runExperimentDisabled: boolean; } -export const InsightExperimentGroups: FC = ({ +export const InsightExperiments: FC = ({ workspace, insightId, onRunExperiment, @@ -67,7 +67,7 @@ export const InsightExperimentGroups: FC = ({ isError, isFetching, refetch, - } = useListExperimentGroups( + } = useListExperiments( workspace, { page: pageIndex + 1, @@ -79,8 +79,7 @@ export const InsightExperimentGroups: FC = ({ ); const groups = response?.data ?? []; const { wrapColumns, onClick, className } = useRowClick( - (group: ExperimentGroupResponse) => - navigate(getExperimentGroupDetailRoute(workspace, group.name)), + (group: ExperimentResponse) => navigate(getExperimentDetailRoute(workspace, group.name)), groups ); diff --git a/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.test.tsx b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.test.tsx index fe75bc0565..c52f7a6261 100644 --- a/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.test.tsx +++ b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.test.tsx @@ -3,9 +3,9 @@ import { getListEvaluationsQueryKey, - getListExperimentGroupsQueryKey, + getListExperimentsQueryKey, } from '@nemo/sdk/generated/platform/api'; -import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import type { ExperimentResponse } from '@nemo/sdk/generated/platform/schema'; import type { Insight } from '@studio/api/optimizer'; import { PLATFORM_BASE_URL } from '@studio/constants/environment'; import { ROUTES } from '@studio/constants/routes'; @@ -20,7 +20,7 @@ import { useParams } from 'react-router-dom'; const WORKSPACE = 'workspace-a'; const INSIGHT_ID = 'insight-a'; const INSIGHT_URL = `${PLATFORM_BASE_URL}/apis/insights/v2/workspaces/:workspace/insights/:insightId`; -const GROUPS_URL = `${PLATFORM_BASE_URL}${getListExperimentGroupsQueryKey(':workspace')[0]}`; +const GROUPS_URL = `${PLATFORM_BASE_URL}${getListExperimentsQueryKey(':workspace')[0]}`; const EVALUATIONS_URL = `${PLATFORM_BASE_URL}${getListEvaluationsQueryKey(':workspace')[0]}`; const insight: Insight = { @@ -51,10 +51,7 @@ const pagination = ({ total_results: totalResults, }); -const makeGroup = ( - id: string, - overrides: Partial = {} -): ExperimentGroupResponse => +const makeGroup = (id: string, overrides: Partial = {}): ExperimentResponse => ({ id, name: `${id}-name`, @@ -67,11 +64,11 @@ const makeGroup = ( created_at: '2026-07-19T12:00:00Z', updated_at: '2026-07-20T12:00:00Z', ...overrides, - }) as ExperimentGroupResponse; + }) as ExperimentResponse; -const ExperimentGroupDestination = () => { - const { experimentGroupName } = useParams(); - return
{`Opened ${experimentGroupName}`}
; +const ExperimentDestination = () => { + const { experimentName } = useParams(); + return
{`Opened ${experimentName}`}
; }; const renderInsight = (history = getOptimizerInsightRoute(WORKSPACE, INSIGHT_ID)) => @@ -80,8 +77,8 @@ const renderInsight = (history = getOptimizerInsightRoute(WORKSPACE, INSIGHT_ID) routes: [ { path: ROUTES.workspace.optimizerInsight, element: }, { - path: ROUTES.workspace.experimentGroupDetail, - element: , + path: ROUTES.workspace.experimentDetail, + element: , }, ], }); @@ -133,7 +130,7 @@ describe('OptimizerInsightRoute experiments', () => { expect(screen.queryByText('Failed to load experiments')).not.toBeInTheDocument(); }); - it('requests and displays only the selected ExperimentGroup page', async () => { + it('requests and displays only the selected Experiment page', async () => { const user = userEvent.setup(); const firstPageGroups = Array.from({ length: 10 }, (_unused, index) => makeGroup(`page-one-${index + 1}`) diff --git a/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.tsx b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.tsx index 6bbc162cc7..160af745a0 100644 --- a/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.tsx +++ b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.tsx @@ -33,7 +33,7 @@ import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; import { InsightOpenModal } from '@studio/routes/optimizer/InsightOpenModal'; import { insightActions, insightStatusColor } from '@studio/routes/optimizer/insightStatus'; import { InsightTracesTable } from '@studio/routes/optimizer/InsightTracesTable'; -import { InsightExperimentGroups } from '@studio/routes/optimizer/OptimizerInsightRoute/InsightExperimentGroups'; +import { InsightExperiments } from '@studio/routes/optimizer/OptimizerInsightRoute/InsightExperiments'; import { getOptimizerRoute } from '@studio/routes/utils'; import { useQueryClient } from '@tanstack/react-query'; import { type FC, useState } from 'react'; @@ -191,7 +191,7 @@ export const OptimizerInsightRoute: FC = () => {
- handleAction('open')} diff --git a/web/packages/studio/src/routes/optimizer/OptimizerRoute/index.test.tsx b/web/packages/studio/src/routes/optimizer/OptimizerRoute/index.test.tsx index 565a0ef767..2f22d671dd 100644 --- a/web/packages/studio/src/routes/optimizer/OptimizerRoute/index.test.tsx +++ b/web/packages/studio/src/routes/optimizer/OptimizerRoute/index.test.tsx @@ -12,7 +12,7 @@ import { renderRoute, screen, within } from '@studio/tests/util/render'; import { http, HttpResponse } from 'msw'; const INSIGHTS_URL = `${PLATFORM_BASE_URL}/apis/insights/v2/workspaces/:workspace/insights`; -const EXPERIMENT_GROUPS_URL = '*/apis/intake/v2/workspaces/:workspace/experiment-groups'; +const EXPERIMENTS_URL = '*/apis/intake/v2/workspaces/:workspace/experiments'; const makeInsight = (id: string, title: string): InsightListItem => ({ id, @@ -56,7 +56,7 @@ const findCell = async (insightTitle: string, columnName: string): Promise { it('renders server-provided list metadata without per-row requests', async () => { - const experimentGroupRequest = vi.fn(() => HttpResponse.json({})); + const experimentRequest = vi.fn(() => HttpResponse.json({})); const insights = [ { ...makeInsight('positive', 'Positive count'), @@ -68,7 +68,7 @@ describe('OptimizerRoute', () => { ]; server.use( http.get(INSIGHTS_URL, () => HttpResponse.json(insightsPage(insights))), - http.get(EXPERIMENT_GROUPS_URL, experimentGroupRequest) + http.get(EXPERIMENTS_URL, experimentRequest) ); renderList(); @@ -81,6 +81,6 @@ describe('OptimizerRoute', () => { '2026-07-21T12:00:00Z' ); expect(await findCell('Zero count', 'Last Seen')).toHaveTextContent('—'); - expect(experimentGroupRequest).not.toHaveBeenCalled(); + expect(experimentRequest).not.toHaveBeenCalled(); }); }); diff --git a/web/packages/studio/src/routes/utils.ts b/web/packages/studio/src/routes/utils.ts index 2c84d87d13..67e665e939 100644 --- a/web/packages/studio/src/routes/utils.ts +++ b/web/packages/studio/src/routes/utils.ts @@ -314,34 +314,34 @@ export const getExperimentRoute = (workspace: string) => { return generatePath(ROUTES.workspace.experiment, { workspace }); }; -export const getExperimentGroupDetailRoute = (workspace: string, experimentGroupName: string) => { - return generatePath(ROUTES.workspace.experimentGroupDetail, { +export const getExperimentDetailRoute = (workspace: string, experimentName: string) => { + return generatePath(ROUTES.workspace.experimentDetail, { workspace, - experimentGroupName: encodeURIComponent(experimentGroupName), + experimentName: encodeURIComponent(experimentName), }); }; export const getEvaluationDetailRoute = ( workspace: string, - experimentGroupName: string, + experimentName: string, evaluationName: string ) => { return generatePath(ROUTES.workspace.evaluationDetail, { workspace, - experimentGroupName: encodeURIComponent(experimentGroupName), + experimentName: encodeURIComponent(experimentName), evaluationName: encodeURIComponent(evaluationName), }); }; export const getEvaluationSessionDetailRoute = ( workspace: string, - experimentGroupName: string, + experimentName: string, evaluationName: string, sessionId: string ): string => { return generatePath(ROUTES.workspace.evaluationSessionDetail, { workspace, - experimentGroupName: encodeURIComponent(experimentGroupName), + experimentName: encodeURIComponent(experimentName), evaluationName: encodeURIComponent(evaluationName), sessionId: encodeURIComponent(sessionId), }); @@ -349,7 +349,7 @@ export const getEvaluationSessionDetailRoute = ( export const getEvaluationSessionTraceDetailRoute = ( workspace: string, - experimentGroupName: string, + experimentName: string, evaluationName: string, sessionId: string, traceId: string, @@ -357,7 +357,7 @@ export const getEvaluationSessionTraceDetailRoute = ( ): string => { const path = getEvaluationSessionDetailRoute( workspace, - experimentGroupName, + experimentName, evaluationName, sessionId ); diff --git a/web/packages/studio/src/tests/title-change.test.tsx b/web/packages/studio/src/tests/title-change.test.tsx index 2ebfeaa519..2e23245a9d 100644 --- a/web/packages/studio/src/tests/title-change.test.tsx +++ b/web/packages/studio/src/tests/title-change.test.tsx @@ -34,7 +34,7 @@ const pathParams = { [RP.agentEvalJobName]: 'test-agent-eval-job', [RP.jobName]: 'test-job', [RP.benchmarkName]: 'test-benchmark', - [RP.experimentGroupName]: 'test-experiment-group', + [RP.experimentName]: 'test-experiment-group', [RP.evaluationName]: 'test-experiment', [RP.insightId]: 'test-insight', [RP.guardrailConfigName]: 'test-guardrail-config',