diff --git a/docs/auth/authorization/permissions-reference.md b/docs/auth/authorization/permissions-reference.md index 6cf1529a48..615e7a3966 100644 --- a/docs/auth/authorization/permissions-reference.md +++ b/docs/auth/authorization/permissions-reference.md @@ -73,6 +73,10 @@ For token-level access restrictions, see [API Scopes](api-scopes.md). For the RB | `intake.annotations.(create \| delete)` | Create, delete intake annotations | | ✓ | ✓ | | `intake.evaluator-results.(read \| list)` | Read, list intake evaluator-results | ✓ | ✓ | ✓ | | `intake.evaluator-results.create` | Create intake evaluator results | | ✓ | ✓ | +| `intake.experiment-groups.read` | Read intake experiment groups | ✓ | ✓ | ✓ | +| `intake.experiment-groups.(create \| update \| delete)` | Create, update, delete intake experiment-groups | | ✓ | ✓ | +| `intake.experiments.read` | Read intake experiments | ✓ | ✓ | ✓ | +| `intake.experiments.(create \| update \| delete)` | Create, update, delete intake experiments | | ✓ | ✓ | | `intake.ingest.create` | Ingest traces into intake | | ✓ | ✓ | | `intake.spans.(read \| list)` | Read, list intake spans | ✓ | ✓ | ✓ | | `intake.traces.read` | Read intake traces | ✓ | ✓ | ✓ | diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index bfb19285b1..7384b337b7 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -5881,6 +5881,423 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiment-groups: + post: + tags: + - Experiment Groups + summary: Create Experiment Group + operationId: create_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupRequest' + responses: + '201': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponse' + '409': + description: Experiment group already exists + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + get: + tags: + - Experiment Groups + summary: List Experiment Groups + operationId: list_experiment_groups_apis_intake_v2_workspaces__workspace__experiment_groups_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + enum: + - -created_at + - created_at + - -updated_at + - updated_at + - -name + - name + type: string + description: Sort field; prefix with '-' for descending. + default: -created_at + title: Sort + description: Sort field; prefix with '-' for descending. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/ExperimentGroupFilter' + description: Filter experiment groups by name. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponsesPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: + get: + tags: + - Experiment Groups + summary: Get Experiment Group + operationId: get_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponse' + '404': + description: Experiment group not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + put: + tags: + - Experiment Groups + summary: Update Experiment Group + operationId: update_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__put + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupRequest' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponse' + '404': + description: Experiment group not found + '409': + description: Attempt to rename the group + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Experiment Groups + summary: Delete Experiment Group + operationId: delete_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '204': + description: Successful Response + '404': + description: Experiment group not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiments: + post: + tags: + - Experiments + summary: Create Experiment + operationId: create_experiment_apis_intake_v2_workspaces__workspace__experiments_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentRequest' + responses: + '201': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponse' + '409': + description: Experiment already exists + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + get: + tags: + - Experiments + summary: List Experiments + operationId: list_experiments_apis_intake_v2_workspaces__workspace__experiments_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + enum: + - -created_at + - created_at + - -updated_at + - updated_at + - -name + - name + type: string + description: Sort field; prefix with '-' for descending. + default: -created_at + title: Sort + description: Sort field; prefix with '-' for descending. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/ExperimentFilter' + description: Filter experiments by name, experiment_group_id, agent_name, + and dataset_name. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponsesPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiments/{name}: + get: + tags: + - Experiments + summary: Get Experiment + operationId: get_experiment_apis_intake_v2_workspaces__workspace__experiments__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponse' + '404': + description: Experiment not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + put: + tags: + - Experiments + summary: Update Experiment + operationId: update_experiment_apis_intake_v2_workspaces__workspace__experiments__name__put + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentRequest' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponse' + '404': + description: Experiment not found + '409': + description: Attempt to change an immutable field + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Experiments + summary: Delete Experiment + operationId: delete_experiment_apis_intake_v2_workspaces__workspace__experiments__name__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '204': + description: Successful Response + '404': + description: Experiment not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/intake/v2/workspaces/{workspace}/ingest/atif: post: tags: @@ -14167,6 +14584,30 @@ components: - name title: Evaluator.Model description: Model definition for use without persisting to the Models API. + EvaluatorAggregate: + properties: + sum: + title: Sum + type: number + mean: + title: Mean + type: number + median: + title: Median + type: number + p90: + title: P90 + type: number + p95: + title: P95 + type: number + p99: + title: P99 + type: number + type: object + title: EvaluatorAggregate + description: Cross-run statistics for one evaluator. Populated by the rollup + path (later PR). EvaluatorResult: properties: evaluator_result_id: @@ -14594,6 +15035,267 @@ components: - action_name title: ExecutedAction description: Information about an action that was executed. + ExperimentFilter: + additionalProperties: false + description: Filter for listing Experiments. + properties: + name: + description: Filter experiments by name. + title: Name + type: string + experiment_group_id: + description: Filter experiments by owning group id. + title: Experiment Group Id + type: string + agent_name: + description: Filter experiments by agent name. + title: Agent Name + type: string + dataset_name: + description: Filter experiments by dataset name. + title: Dataset Name + type: string + title: ExperimentFilter + type: object + ExperimentGroupFilter: + additionalProperties: false + description: Filter for listing ExperimentGroups. + properties: + name: + description: Filter groups by name. + title: Name + type: string + title: ExperimentGroupFilter + type: object + ExperimentGroupRequest: + properties: + name: + type: string + title: Name + description: Workspace-unique group name. + description: + title: Description + description: Human-readable purpose of the group. + type: string + additionalProperties: false + type: object + required: + - name + title: ExperimentGroupRequest + description: Request body for creating an ExperimentGroup. + ExperimentGroupResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + workspace: + type: string + title: Workspace + description: + title: Description + type: string + created_at: + title: Created At + type: string + format: date-time + updated_at: + title: Updated At + type: string + format: date-time + type: object + required: + - id + - name + - workspace + title: ExperimentGroupResponse + description: ExperimentGroup as served by the API. + ExperimentGroupResponsesPage: + properties: + data: + items: + $ref: '#/components/schemas/ExperimentGroupResponse' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: ExperimentGroupResponsesPage + ExperimentRequest: + properties: + name: + type: string + title: Name + description: Producer-supplied, workspace-unique experiment id. + experiment_group_id: + title: Experiment Group Id + description: Entity id of the owning ExperimentGroup; optional. Soft reference, + not validated. + type: string + agent_name: + type: string + title: Agent Name + description: Name of the agent under test. + agent_version: + type: string + title: Agent Version + description: Version of the agent under test. + dataset_name: + type: string + title: Dataset Name + description: Producer-supplied dataset name. + dataset_version: + title: Dataset Version + description: Producer-supplied dataset version. + type: string + source_link: + title: Source Link + description: Optional URL for the source experiment. + type: string + minLength: 1 + format: uri + metadata: + additionalProperties: true + type: object + title: Metadata + description: Free-form producer metadata. + description: + title: Description + description: Human-readable description. + type: string + summary: + title: Summary + description: Human-authored summary of results. + type: string + additionalProperties: false + type: object + required: + - name + - agent_name + - agent_version + - dataset_name + title: ExperimentRequest + description: Request body for creating an Experiment. + ExperimentResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + workspace: + type: string + title: Workspace + experiment_group_id: + title: Experiment Group Id + description: Entity id of the owning ExperimentGroup; null when ungrouped. + Soft reference, not validated. + type: string + agent_name: + type: string + title: Agent Name + agent_version: + type: string + title: Agent Version + dataset_name: + type: string + title: Dataset Name + dataset_version: + title: Dataset Version + type: string + source_link: + title: Source Link + type: string + minLength: 1 + format: uri + metadata: + additionalProperties: true + type: object + title: Metadata + description: + title: Description + type: string + summary: + title: Summary + type: string + created_at: + title: Created At + type: string + format: date-time + updated_at: + title: Updated At + type: string + format: date-time + evaluator_names: + items: + type: string + type: array + title: Evaluator Names + model_names: + items: + type: string + type: array + title: Model Names + aggregate_scores: + title: Aggregate Scores + additionalProperties: + $ref: '#/components/schemas/EvaluatorAggregate' + type: object + run_count: + type: integer + title: Run Count + default: 0 + type: object + required: + - id + - name + - workspace + - agent_name + - agent_version + - dataset_name + title: ExperimentResponse + description: Experiment as served by the API, including ClickHouse-hydrated + rollups. + ExperimentResponsesPage: + properties: + data: + items: + $ref: '#/components/schemas/ExperimentResponse' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: ExperimentResponsesPage ExtendedBenchmark: properties: name: diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index bfb19285b1..7384b337b7 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -5881,6 +5881,423 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiment-groups: + post: + tags: + - Experiment Groups + summary: Create Experiment Group + operationId: create_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupRequest' + responses: + '201': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponse' + '409': + description: Experiment group already exists + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + get: + tags: + - Experiment Groups + summary: List Experiment Groups + operationId: list_experiment_groups_apis_intake_v2_workspaces__workspace__experiment_groups_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + enum: + - -created_at + - created_at + - -updated_at + - updated_at + - -name + - name + type: string + description: Sort field; prefix with '-' for descending. + default: -created_at + title: Sort + description: Sort field; prefix with '-' for descending. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/ExperimentGroupFilter' + description: Filter experiment groups by name. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponsesPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: + get: + tags: + - Experiment Groups + summary: Get Experiment Group + operationId: get_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponse' + '404': + description: Experiment group not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + put: + tags: + - Experiment Groups + summary: Update Experiment Group + operationId: update_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__put + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupRequest' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponse' + '404': + description: Experiment group not found + '409': + description: Attempt to rename the group + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Experiment Groups + summary: Delete Experiment Group + operationId: delete_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '204': + description: Successful Response + '404': + description: Experiment group not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiments: + post: + tags: + - Experiments + summary: Create Experiment + operationId: create_experiment_apis_intake_v2_workspaces__workspace__experiments_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentRequest' + responses: + '201': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponse' + '409': + description: Experiment already exists + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + get: + tags: + - Experiments + summary: List Experiments + operationId: list_experiments_apis_intake_v2_workspaces__workspace__experiments_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + enum: + - -created_at + - created_at + - -updated_at + - updated_at + - -name + - name + type: string + description: Sort field; prefix with '-' for descending. + default: -created_at + title: Sort + description: Sort field; prefix with '-' for descending. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/ExperimentFilter' + description: Filter experiments by name, experiment_group_id, agent_name, + and dataset_name. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponsesPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiments/{name}: + get: + tags: + - Experiments + summary: Get Experiment + operationId: get_experiment_apis_intake_v2_workspaces__workspace__experiments__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponse' + '404': + description: Experiment not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + put: + tags: + - Experiments + summary: Update Experiment + operationId: update_experiment_apis_intake_v2_workspaces__workspace__experiments__name__put + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentRequest' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponse' + '404': + description: Experiment not found + '409': + description: Attempt to change an immutable field + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Experiments + summary: Delete Experiment + operationId: delete_experiment_apis_intake_v2_workspaces__workspace__experiments__name__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '204': + description: Successful Response + '404': + description: Experiment not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/intake/v2/workspaces/{workspace}/ingest/atif: post: tags: @@ -14167,6 +14584,30 @@ components: - name title: Evaluator.Model description: Model definition for use without persisting to the Models API. + EvaluatorAggregate: + properties: + sum: + title: Sum + type: number + mean: + title: Mean + type: number + median: + title: Median + type: number + p90: + title: P90 + type: number + p95: + title: P95 + type: number + p99: + title: P99 + type: number + type: object + title: EvaluatorAggregate + description: Cross-run statistics for one evaluator. Populated by the rollup + path (later PR). EvaluatorResult: properties: evaluator_result_id: @@ -14594,6 +15035,267 @@ components: - action_name title: ExecutedAction description: Information about an action that was executed. + ExperimentFilter: + additionalProperties: false + description: Filter for listing Experiments. + properties: + name: + description: Filter experiments by name. + title: Name + type: string + experiment_group_id: + description: Filter experiments by owning group id. + title: Experiment Group Id + type: string + agent_name: + description: Filter experiments by agent name. + title: Agent Name + type: string + dataset_name: + description: Filter experiments by dataset name. + title: Dataset Name + type: string + title: ExperimentFilter + type: object + ExperimentGroupFilter: + additionalProperties: false + description: Filter for listing ExperimentGroups. + properties: + name: + description: Filter groups by name. + title: Name + type: string + title: ExperimentGroupFilter + type: object + ExperimentGroupRequest: + properties: + name: + type: string + title: Name + description: Workspace-unique group name. + description: + title: Description + description: Human-readable purpose of the group. + type: string + additionalProperties: false + type: object + required: + - name + title: ExperimentGroupRequest + description: Request body for creating an ExperimentGroup. + ExperimentGroupResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + workspace: + type: string + title: Workspace + description: + title: Description + type: string + created_at: + title: Created At + type: string + format: date-time + updated_at: + title: Updated At + type: string + format: date-time + type: object + required: + - id + - name + - workspace + title: ExperimentGroupResponse + description: ExperimentGroup as served by the API. + ExperimentGroupResponsesPage: + properties: + data: + items: + $ref: '#/components/schemas/ExperimentGroupResponse' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: ExperimentGroupResponsesPage + ExperimentRequest: + properties: + name: + type: string + title: Name + description: Producer-supplied, workspace-unique experiment id. + experiment_group_id: + title: Experiment Group Id + description: Entity id of the owning ExperimentGroup; optional. Soft reference, + not validated. + type: string + agent_name: + type: string + title: Agent Name + description: Name of the agent under test. + agent_version: + type: string + title: Agent Version + description: Version of the agent under test. + dataset_name: + type: string + title: Dataset Name + description: Producer-supplied dataset name. + dataset_version: + title: Dataset Version + description: Producer-supplied dataset version. + type: string + source_link: + title: Source Link + description: Optional URL for the source experiment. + type: string + minLength: 1 + format: uri + metadata: + additionalProperties: true + type: object + title: Metadata + description: Free-form producer metadata. + description: + title: Description + description: Human-readable description. + type: string + summary: + title: Summary + description: Human-authored summary of results. + type: string + additionalProperties: false + type: object + required: + - name + - agent_name + - agent_version + - dataset_name + title: ExperimentRequest + description: Request body for creating an Experiment. + ExperimentResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + workspace: + type: string + title: Workspace + experiment_group_id: + title: Experiment Group Id + description: Entity id of the owning ExperimentGroup; null when ungrouped. + Soft reference, not validated. + type: string + agent_name: + type: string + title: Agent Name + agent_version: + type: string + title: Agent Version + dataset_name: + type: string + title: Dataset Name + dataset_version: + title: Dataset Version + type: string + source_link: + title: Source Link + type: string + minLength: 1 + format: uri + metadata: + additionalProperties: true + type: object + title: Metadata + description: + title: Description + type: string + summary: + title: Summary + type: string + created_at: + title: Created At + type: string + format: date-time + updated_at: + title: Updated At + type: string + format: date-time + evaluator_names: + items: + type: string + type: array + title: Evaluator Names + model_names: + items: + type: string + type: array + title: Model Names + aggregate_scores: + title: Aggregate Scores + additionalProperties: + $ref: '#/components/schemas/EvaluatorAggregate' + type: object + run_count: + type: integer + title: Run Count + default: 0 + type: object + required: + - id + - name + - workspace + - agent_name + - agent_version + - dataset_name + title: ExperimentResponse + description: Experiment as served by the API, including ClickHouse-hydrated + rollups. + ExperimentResponsesPage: + properties: + data: + items: + $ref: '#/components/schemas/ExperimentResponse' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: ExperimentResponsesPage ExtendedBenchmark: properties: name: diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index bfb19285b1..7384b337b7 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -5881,6 +5881,423 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiment-groups: + post: + tags: + - Experiment Groups + summary: Create Experiment Group + operationId: create_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupRequest' + responses: + '201': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponse' + '409': + description: Experiment group already exists + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + get: + tags: + - Experiment Groups + summary: List Experiment Groups + operationId: list_experiment_groups_apis_intake_v2_workspaces__workspace__experiment_groups_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + enum: + - -created_at + - created_at + - -updated_at + - updated_at + - -name + - name + type: string + description: Sort field; prefix with '-' for descending. + default: -created_at + title: Sort + description: Sort field; prefix with '-' for descending. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/ExperimentGroupFilter' + description: Filter experiment groups by name. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponsesPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: + get: + tags: + - Experiment Groups + summary: Get Experiment Group + operationId: get_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponse' + '404': + description: Experiment group not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + put: + tags: + - Experiment Groups + summary: Update Experiment Group + operationId: update_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__put + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupRequest' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponse' + '404': + description: Experiment group not found + '409': + description: Attempt to rename the group + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Experiment Groups + summary: Delete Experiment Group + operationId: delete_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '204': + description: Successful Response + '404': + description: Experiment group not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiments: + post: + tags: + - Experiments + summary: Create Experiment + operationId: create_experiment_apis_intake_v2_workspaces__workspace__experiments_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentRequest' + responses: + '201': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponse' + '409': + description: Experiment already exists + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + get: + tags: + - Experiments + summary: List Experiments + operationId: list_experiments_apis_intake_v2_workspaces__workspace__experiments_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + enum: + - -created_at + - created_at + - -updated_at + - updated_at + - -name + - name + type: string + description: Sort field; prefix with '-' for descending. + default: -created_at + title: Sort + description: Sort field; prefix with '-' for descending. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/ExperimentFilter' + description: Filter experiments by name, experiment_group_id, agent_name, + and dataset_name. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponsesPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiments/{name}: + get: + tags: + - Experiments + summary: Get Experiment + operationId: get_experiment_apis_intake_v2_workspaces__workspace__experiments__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponse' + '404': + description: Experiment not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + put: + tags: + - Experiments + summary: Update Experiment + operationId: update_experiment_apis_intake_v2_workspaces__workspace__experiments__name__put + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentRequest' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponse' + '404': + description: Experiment not found + '409': + description: Attempt to change an immutable field + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Experiments + summary: Delete Experiment + operationId: delete_experiment_apis_intake_v2_workspaces__workspace__experiments__name__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '204': + description: Successful Response + '404': + description: Experiment not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/intake/v2/workspaces/{workspace}/ingest/atif: post: tags: @@ -14167,6 +14584,30 @@ components: - name title: Evaluator.Model description: Model definition for use without persisting to the Models API. + EvaluatorAggregate: + properties: + sum: + title: Sum + type: number + mean: + title: Mean + type: number + median: + title: Median + type: number + p90: + title: P90 + type: number + p95: + title: P95 + type: number + p99: + title: P99 + type: number + type: object + title: EvaluatorAggregate + description: Cross-run statistics for one evaluator. Populated by the rollup + path (later PR). EvaluatorResult: properties: evaluator_result_id: @@ -14594,6 +15035,267 @@ components: - action_name title: ExecutedAction description: Information about an action that was executed. + ExperimentFilter: + additionalProperties: false + description: Filter for listing Experiments. + properties: + name: + description: Filter experiments by name. + title: Name + type: string + experiment_group_id: + description: Filter experiments by owning group id. + title: Experiment Group Id + type: string + agent_name: + description: Filter experiments by agent name. + title: Agent Name + type: string + dataset_name: + description: Filter experiments by dataset name. + title: Dataset Name + type: string + title: ExperimentFilter + type: object + ExperimentGroupFilter: + additionalProperties: false + description: Filter for listing ExperimentGroups. + properties: + name: + description: Filter groups by name. + title: Name + type: string + title: ExperimentGroupFilter + type: object + ExperimentGroupRequest: + properties: + name: + type: string + title: Name + description: Workspace-unique group name. + description: + title: Description + description: Human-readable purpose of the group. + type: string + additionalProperties: false + type: object + required: + - name + title: ExperimentGroupRequest + description: Request body for creating an ExperimentGroup. + ExperimentGroupResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + workspace: + type: string + title: Workspace + description: + title: Description + type: string + created_at: + title: Created At + type: string + format: date-time + updated_at: + title: Updated At + type: string + format: date-time + type: object + required: + - id + - name + - workspace + title: ExperimentGroupResponse + description: ExperimentGroup as served by the API. + ExperimentGroupResponsesPage: + properties: + data: + items: + $ref: '#/components/schemas/ExperimentGroupResponse' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: ExperimentGroupResponsesPage + ExperimentRequest: + properties: + name: + type: string + title: Name + description: Producer-supplied, workspace-unique experiment id. + experiment_group_id: + title: Experiment Group Id + description: Entity id of the owning ExperimentGroup; optional. Soft reference, + not validated. + type: string + agent_name: + type: string + title: Agent Name + description: Name of the agent under test. + agent_version: + type: string + title: Agent Version + description: Version of the agent under test. + dataset_name: + type: string + title: Dataset Name + description: Producer-supplied dataset name. + dataset_version: + title: Dataset Version + description: Producer-supplied dataset version. + type: string + source_link: + title: Source Link + description: Optional URL for the source experiment. + type: string + minLength: 1 + format: uri + metadata: + additionalProperties: true + type: object + title: Metadata + description: Free-form producer metadata. + description: + title: Description + description: Human-readable description. + type: string + summary: + title: Summary + description: Human-authored summary of results. + type: string + additionalProperties: false + type: object + required: + - name + - agent_name + - agent_version + - dataset_name + title: ExperimentRequest + description: Request body for creating an Experiment. + ExperimentResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + workspace: + type: string + title: Workspace + experiment_group_id: + title: Experiment Group Id + description: Entity id of the owning ExperimentGroup; null when ungrouped. + Soft reference, not validated. + type: string + agent_name: + type: string + title: Agent Name + agent_version: + type: string + title: Agent Version + dataset_name: + type: string + title: Dataset Name + dataset_version: + title: Dataset Version + type: string + source_link: + title: Source Link + type: string + minLength: 1 + format: uri + metadata: + additionalProperties: true + type: object + title: Metadata + description: + title: Description + type: string + summary: + title: Summary + type: string + created_at: + title: Created At + type: string + format: date-time + updated_at: + title: Updated At + type: string + format: date-time + evaluator_names: + items: + type: string + type: array + title: Evaluator Names + model_names: + items: + type: string + type: array + title: Model Names + aggregate_scores: + title: Aggregate Scores + additionalProperties: + $ref: '#/components/schemas/EvaluatorAggregate' + type: object + run_count: + type: integer + title: Run Count + default: 0 + type: object + required: + - id + - name + - workspace + - agent_name + - agent_version + - dataset_name + title: ExperimentResponse + description: Experiment as served by the API, including ClickHouse-hydrated + rollups. + ExperimentResponsesPage: + properties: + data: + items: + $ref: '#/components/schemas/ExperimentResponse' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: ExperimentResponsesPage ExtendedBenchmark: properties: name: diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index bfb19285b1..7384b337b7 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -5881,6 +5881,423 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiment-groups: + post: + tags: + - Experiment Groups + summary: Create Experiment Group + operationId: create_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupRequest' + responses: + '201': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponse' + '409': + description: Experiment group already exists + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + get: + tags: + - Experiment Groups + summary: List Experiment Groups + operationId: list_experiment_groups_apis_intake_v2_workspaces__workspace__experiment_groups_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + enum: + - -created_at + - created_at + - -updated_at + - updated_at + - -name + - name + type: string + description: Sort field; prefix with '-' for descending. + default: -created_at + title: Sort + description: Sort field; prefix with '-' for descending. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/ExperimentGroupFilter' + description: Filter experiment groups by name. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponsesPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: + get: + tags: + - Experiment Groups + summary: Get Experiment Group + operationId: get_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponse' + '404': + description: Experiment group not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + put: + tags: + - Experiment Groups + summary: Update Experiment Group + operationId: update_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__put + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupRequest' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponse' + '404': + description: Experiment group not found + '409': + description: Attempt to rename the group + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Experiment Groups + summary: Delete Experiment Group + operationId: delete_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '204': + description: Successful Response + '404': + description: Experiment group not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiments: + post: + tags: + - Experiments + summary: Create Experiment + operationId: create_experiment_apis_intake_v2_workspaces__workspace__experiments_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentRequest' + responses: + '201': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponse' + '409': + description: Experiment already exists + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + get: + tags: + - Experiments + summary: List Experiments + operationId: list_experiments_apis_intake_v2_workspaces__workspace__experiments_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + enum: + - -created_at + - created_at + - -updated_at + - updated_at + - -name + - name + type: string + description: Sort field; prefix with '-' for descending. + default: -created_at + title: Sort + description: Sort field; prefix with '-' for descending. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/ExperimentFilter' + description: Filter experiments by name, experiment_group_id, agent_name, + and dataset_name. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponsesPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiments/{name}: + get: + tags: + - Experiments + summary: Get Experiment + operationId: get_experiment_apis_intake_v2_workspaces__workspace__experiments__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponse' + '404': + description: Experiment not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + put: + tags: + - Experiments + summary: Update Experiment + operationId: update_experiment_apis_intake_v2_workspaces__workspace__experiments__name__put + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentRequest' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponse' + '404': + description: Experiment not found + '409': + description: Attempt to change an immutable field + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Experiments + summary: Delete Experiment + operationId: delete_experiment_apis_intake_v2_workspaces__workspace__experiments__name__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '204': + description: Successful Response + '404': + description: Experiment not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/intake/v2/workspaces/{workspace}/ingest/atif: post: tags: @@ -14167,6 +14584,30 @@ components: - name title: Evaluator.Model description: Model definition for use without persisting to the Models API. + EvaluatorAggregate: + properties: + sum: + title: Sum + type: number + mean: + title: Mean + type: number + median: + title: Median + type: number + p90: + title: P90 + type: number + p95: + title: P95 + type: number + p99: + title: P99 + type: number + type: object + title: EvaluatorAggregate + description: Cross-run statistics for one evaluator. Populated by the rollup + path (later PR). EvaluatorResult: properties: evaluator_result_id: @@ -14594,6 +15035,267 @@ components: - action_name title: ExecutedAction description: Information about an action that was executed. + ExperimentFilter: + additionalProperties: false + description: Filter for listing Experiments. + properties: + name: + description: Filter experiments by name. + title: Name + type: string + experiment_group_id: + description: Filter experiments by owning group id. + title: Experiment Group Id + type: string + agent_name: + description: Filter experiments by agent name. + title: Agent Name + type: string + dataset_name: + description: Filter experiments by dataset name. + title: Dataset Name + type: string + title: ExperimentFilter + type: object + ExperimentGroupFilter: + additionalProperties: false + description: Filter for listing ExperimentGroups. + properties: + name: + description: Filter groups by name. + title: Name + type: string + title: ExperimentGroupFilter + type: object + ExperimentGroupRequest: + properties: + name: + type: string + title: Name + description: Workspace-unique group name. + description: + title: Description + description: Human-readable purpose of the group. + type: string + additionalProperties: false + type: object + required: + - name + title: ExperimentGroupRequest + description: Request body for creating an ExperimentGroup. + ExperimentGroupResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + workspace: + type: string + title: Workspace + description: + title: Description + type: string + created_at: + title: Created At + type: string + format: date-time + updated_at: + title: Updated At + type: string + format: date-time + type: object + required: + - id + - name + - workspace + title: ExperimentGroupResponse + description: ExperimentGroup as served by the API. + ExperimentGroupResponsesPage: + properties: + data: + items: + $ref: '#/components/schemas/ExperimentGroupResponse' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: ExperimentGroupResponsesPage + ExperimentRequest: + properties: + name: + type: string + title: Name + description: Producer-supplied, workspace-unique experiment id. + experiment_group_id: + title: Experiment Group Id + description: Entity id of the owning ExperimentGroup; optional. Soft reference, + not validated. + type: string + agent_name: + type: string + title: Agent Name + description: Name of the agent under test. + agent_version: + type: string + title: Agent Version + description: Version of the agent under test. + dataset_name: + type: string + title: Dataset Name + description: Producer-supplied dataset name. + dataset_version: + title: Dataset Version + description: Producer-supplied dataset version. + type: string + source_link: + title: Source Link + description: Optional URL for the source experiment. + type: string + minLength: 1 + format: uri + metadata: + additionalProperties: true + type: object + title: Metadata + description: Free-form producer metadata. + description: + title: Description + description: Human-readable description. + type: string + summary: + title: Summary + description: Human-authored summary of results. + type: string + additionalProperties: false + type: object + required: + - name + - agent_name + - agent_version + - dataset_name + title: ExperimentRequest + description: Request body for creating an Experiment. + ExperimentResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + workspace: + type: string + title: Workspace + experiment_group_id: + title: Experiment Group Id + description: Entity id of the owning ExperimentGroup; null when ungrouped. + Soft reference, not validated. + type: string + agent_name: + type: string + title: Agent Name + agent_version: + type: string + title: Agent Version + dataset_name: + type: string + title: Dataset Name + dataset_version: + title: Dataset Version + type: string + source_link: + title: Source Link + type: string + minLength: 1 + format: uri + metadata: + additionalProperties: true + type: object + title: Metadata + description: + title: Description + type: string + summary: + title: Summary + type: string + created_at: + title: Created At + type: string + format: date-time + updated_at: + title: Updated At + type: string + format: date-time + evaluator_names: + items: + type: string + type: array + title: Evaluator Names + model_names: + items: + type: string + type: array + title: Model Names + aggregate_scores: + title: Aggregate Scores + additionalProperties: + $ref: '#/components/schemas/EvaluatorAggregate' + type: object + run_count: + type: integer + title: Run Count + default: 0 + type: object + required: + - id + - name + - workspace + - agent_name + - agent_version + - dataset_name + title: ExperimentResponse + description: Experiment as served by the API, including ClickHouse-hydrated + rollups. + ExperimentResponsesPage: + properties: + data: + items: + $ref: '#/components/schemas/ExperimentResponse' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: ExperimentResponsesPage ExtendedBenchmark: properties: name: diff --git a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml index 99b16fcf2a..074e76fc68 100644 --- a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml @@ -1082,3 +1082,30 @@ resources: methods: list: get /apis/intake/v2/workspaces/{workspace}/traces retrieve: get /apis/intake/v2/workspaces/{workspace}/traces/{id} + experiment_groups: + standalone_api: true + models: + experiment_group_filter: ExperimentGroupFilter + experiment_group_request: ExperimentGroupRequest + experiment_group_response: ExperimentGroupResponse + experiment_group_responses_page: ExperimentGroupResponsesPage + methods: + create: post /apis/intake/v2/workspaces/{workspace}/experiment-groups + list: get /apis/intake/v2/workspaces/{workspace}/experiment-groups + retrieve: get /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} + update: put /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} + delete: delete /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} + experiments: + standalone_api: true + models: + evaluator_aggregate: EvaluatorAggregate + experiment_filter: ExperimentFilter + experiment_request: ExperimentRequest + experiment_response: ExperimentResponse + experiment_responses_page: ExperimentResponsesPage + methods: + create: post /apis/intake/v2/workspaces/{workspace}/experiments + list: get /apis/intake/v2/workspaces/{workspace}/experiments + retrieve: get /apis/intake/v2/workspaces/{workspace}/experiments/{name} + update: put /apis/intake/v2/workspaces/{workspace}/experiments/{name} + delete: delete /apis/intake/v2/workspaces/{workspace}/experiments/{name} diff --git a/sdk/python/nemo-platform/api.md b/sdk/python/nemo-platform/api.md index dfc32bd301..adb244f02a 100644 --- a/sdk/python/nemo-platform/api.md +++ b/sdk/python/nemo-platform/api.md @@ -69,3 +69,7 @@ from nemo_platform.types import ( # [Adapters](src/nemo_platform/resources/adapters/api.md) # [Intake](src/nemo_platform/resources/intake/api.md) + +# [ExperimentGroups](src/nemo_platform/resources/experiment_groups/api.md) + +# [Experiments](src/nemo_platform/resources/experiments/api.md) diff --git a/sdk/python/nemo-platform/src/nemo_platform/_client.py b/sdk/python/nemo-platform/src/nemo_platform/_client.py index d13d2f4dfc..79ecaf1d71 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/_client.py +++ b/sdk/python/nemo-platform/src/nemo_platform/_client.py @@ -66,6 +66,8 @@ inference, evaluation, workspaces, + experiments, + experiment_groups, ) from .resources.iam.iam import IamResource, AsyncIamResource from .resources.jobs.jobs import JobsResource, AsyncJobsResource @@ -81,6 +83,8 @@ from .resources.inference.inference import InferenceResource, AsyncInferenceResource from .resources.evaluation.evaluation import EvaluationResource, AsyncEvaluationResource from .resources.workspaces.workspaces import WorkspacesResource, AsyncWorkspacesResource + from .resources.experiments.experiments import ExperimentsResource, AsyncExperimentsResource + from .resources.experiment_groups.experiment_groups import ExperimentGroupsResource, AsyncExperimentGroupsResource __all__ = [ "Timeout", @@ -304,6 +308,18 @@ def intake(self) -> IntakeResource: return IntakeResource(self) + @cached_property + def experiment_groups(self) -> ExperimentGroupsResource: + from .resources.experiment_groups import ExperimentGroupsResource + + return ExperimentGroupsResource(self) + + @cached_property + def experiments(self) -> ExperimentsResource: + from .resources.experiments import ExperimentsResource + + return ExperimentsResource(self) + @cached_property def with_raw_response(self) -> NeMoPlatformWithRawResponse: return NeMoPlatformWithRawResponse(self) @@ -662,6 +678,18 @@ def intake(self) -> AsyncIntakeResource: return AsyncIntakeResource(self) + @cached_property + def experiment_groups(self) -> AsyncExperimentGroupsResource: + from .resources.experiment_groups import AsyncExperimentGroupsResource + + return AsyncExperimentGroupsResource(self) + + @cached_property + def experiments(self) -> AsyncExperimentsResource: + from .resources.experiments import AsyncExperimentsResource + + return AsyncExperimentsResource(self) + @cached_property def with_raw_response(self) -> AsyncNeMoPlatformWithRawResponse: return AsyncNeMoPlatformWithRawResponse(self) @@ -883,6 +911,18 @@ def intake(self) -> intake.IntakeResourceWithRawResponse: return IntakeResourceWithRawResponse(self._client.intake) + @cached_property + def experiment_groups(self) -> experiment_groups.ExperimentGroupsResourceWithRawResponse: + from .resources.experiment_groups import ExperimentGroupsResourceWithRawResponse + + return ExperimentGroupsResourceWithRawResponse(self._client.experiment_groups) + + @cached_property + def experiments(self) -> experiments.ExperimentsResourceWithRawResponse: + from .resources.experiments import ExperimentsResourceWithRawResponse + + return ExperimentsResourceWithRawResponse(self._client.experiments) + class AsyncNeMoPlatformWithRawResponse: _client: AsyncNeMoPlatform @@ -974,6 +1014,18 @@ def intake(self) -> intake.AsyncIntakeResourceWithRawResponse: return AsyncIntakeResourceWithRawResponse(self._client.intake) + @cached_property + def experiment_groups(self) -> experiment_groups.AsyncExperimentGroupsResourceWithRawResponse: + from .resources.experiment_groups import AsyncExperimentGroupsResourceWithRawResponse + + return AsyncExperimentGroupsResourceWithRawResponse(self._client.experiment_groups) + + @cached_property + def experiments(self) -> experiments.AsyncExperimentsResourceWithRawResponse: + from .resources.experiments import AsyncExperimentsResourceWithRawResponse + + return AsyncExperimentsResourceWithRawResponse(self._client.experiments) + class NeMoPlatformWithStreamedResponse: _client: NeMoPlatform @@ -1065,6 +1117,18 @@ def intake(self) -> intake.IntakeResourceWithStreamingResponse: return IntakeResourceWithStreamingResponse(self._client.intake) + @cached_property + def experiment_groups(self) -> experiment_groups.ExperimentGroupsResourceWithStreamingResponse: + from .resources.experiment_groups import ExperimentGroupsResourceWithStreamingResponse + + return ExperimentGroupsResourceWithStreamingResponse(self._client.experiment_groups) + + @cached_property + def experiments(self) -> experiments.ExperimentsResourceWithStreamingResponse: + from .resources.experiments import ExperimentsResourceWithStreamingResponse + + return ExperimentsResourceWithStreamingResponse(self._client.experiments) + class AsyncNeMoPlatformWithStreamedResponse: _client: AsyncNeMoPlatform @@ -1156,6 +1220,18 @@ def intake(self) -> intake.AsyncIntakeResourceWithStreamingResponse: return AsyncIntakeResourceWithStreamingResponse(self._client.intake) + @cached_property + def experiment_groups(self) -> experiment_groups.AsyncExperimentGroupsResourceWithStreamingResponse: + from .resources.experiment_groups import AsyncExperimentGroupsResourceWithStreamingResponse + + return AsyncExperimentGroupsResourceWithStreamingResponse(self._client.experiment_groups) + + @cached_property + def experiments(self) -> experiments.AsyncExperimentsResourceWithStreamingResponse: + from .resources.experiments import AsyncExperimentsResourceWithStreamingResponse + + return AsyncExperimentsResourceWithStreamingResponse(self._client.experiments) + Client = NeMoPlatform diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/__init__.py new file mode 100644 index 0000000000..ad45581d2b --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/__init__.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from .experiment_groups import ( + ExperimentGroupsResource, + AsyncExperimentGroupsResource, + ExperimentGroupsResourceWithRawResponse, + AsyncExperimentGroupsResourceWithRawResponse, + ExperimentGroupsResourceWithStreamingResponse, + AsyncExperimentGroupsResourceWithStreamingResponse, +) + +__all__ = [ + "ExperimentGroupsResource", + "AsyncExperimentGroupsResource", + "ExperimentGroupsResourceWithRawResponse", + "AsyncExperimentGroupsResourceWithRawResponse", + "ExperimentGroupsResourceWithStreamingResponse", + "AsyncExperimentGroupsResourceWithStreamingResponse", +] diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md new file mode 100644 index 0000000000..877402928d --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md @@ -0,0 +1,20 @@ +# ExperimentGroups + +Types: + +```python +from nemo_platform.types.experiment_groups import ( + ExperimentGroupFilter, + ExperimentGroupRequest, + ExperimentGroupResponse, + ExperimentGroupResponsesPage, +) +``` + +Methods: + +- client.experiment_groups.create(\*, workspace, \*\*params) -> ExperimentGroupResponse +- client.experiment_groups.retrieve(name, \*, workspace) -> ExperimentGroupResponse +- client.experiment_groups.update(path_name, \*, workspace, \*\*params) -> ExperimentGroupResponse +- client.experiment_groups.list(\*, workspace, \*\*params) -> SyncDefaultPagination[ExperimentGroupResponse] +- client.experiment_groups.delete(name, \*, workspace) -> None diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py new file mode 100644 index 0000000000..92c8f8e0a2 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py @@ -0,0 +1,679 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal + +import httpx + +from ..._types import Body, Omit, Query, Headers, NoneType, NotGiven, omit, not_given +from ..._utils import path_template, maybe_transform, async_maybe_transform +from ..._compat import cached_property +from ..._resource import SyncAPIResource, AsyncAPIResource +from ..._response import ( + to_raw_response_wrapper, + to_streamed_response_wrapper, + async_to_raw_response_wrapper, + async_to_streamed_response_wrapper, +) +from ...pagination import SyncDefaultPagination, AsyncDefaultPagination +from ..._base_client import AsyncPaginator, make_request_options +from ...types.experiment_groups import ( + experiment_group_list_params, + experiment_group_create_params, + experiment_group_update_params, +) +from ...types.experiment_groups.experiment_group_response import ExperimentGroupResponse +from ...types.experiment_groups.experiment_group_filter_param import ExperimentGroupFilterParam +from ..._exceptions import ConflictError + +__all__ = ["ExperimentGroupsResource", "AsyncExperimentGroupsResource"] + + +class ExperimentGroupsResource(SyncAPIResource): + @cached_property + def with_raw_response(self) -> ExperimentGroupsResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#accessing-raw-response-data-e-g-headers + """ + return ExperimentGroupsResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> ExperimentGroupsResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#with_streaming_response + """ + return ExperimentGroupsResourceWithStreamingResponse(self) + + def create( + self, + *, + workspace: str | None = None, + name: str, + description: str | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + exist_ok: bool = False, + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ExperimentGroupResponse: + """ + Create Experiment Group + + Args: + name: Workspace-unique group name. + + description: Human-readable purpose of the group. + + + exist_ok: Do not raise an error if the resource already exists. Returns the existing resource. + + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + try: + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + return self._post( + path_template("/apis/intake/v2/workspaces/{workspace}/experiment-groups", workspace=workspace), + body=maybe_transform( + { + "name": name, + "description": description, + }, + experiment_group_create_params.ExperimentGroupCreateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ExperimentGroupResponse, + ) + except ConflictError: + if not exist_ok: + raise + return self.retrieve(name = name, workspace = workspace) + + def retrieve( + self, + name: str, + *, + workspace: str | None = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ExperimentGroupResponse: + """ + Get Experiment Group + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + return self._get( + path_template( + "/apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}", workspace=workspace, name=name + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ExperimentGroupResponse, + ) + + def update( + self, + path_name: str, + *, + workspace: str | None = None, + body_name: str, + description: str | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ExperimentGroupResponse: + """ + Update Experiment Group + + Args: + body_name: Workspace-unique group name. + + description: Human-readable purpose of the group. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not path_name: + raise ValueError(f"Expected a non-empty value for `path_name` but received {path_name!r}") + return self._put( + path_template( + "/apis/intake/v2/workspaces/{workspace}/experiment-groups/{path_name}", + workspace=workspace, + path_name=path_name, + ), + body=maybe_transform( + { + "body_name": body_name, + "description": description, + }, + experiment_group_update_params.ExperimentGroupUpdateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ExperimentGroupResponse, + ) + + def list( + self, + *, + workspace: str | None = None, + filter: ExperimentGroupFilterParam | Omit = omit, + page: int | Omit = omit, + page_size: int | Omit = omit, + sort: Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"] | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> SyncDefaultPagination[ExperimentGroupResponse]: + """ + List Experiment Groups + + Args: + filter: Filter experiment groups by name. + + page: Page number. + + page_size: Page size. + + sort: Sort field; prefix with '-' for descending. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + return self._get_api_list( + path_template("/apis/intake/v2/workspaces/{workspace}/experiment-groups", workspace=workspace), + page=SyncDefaultPagination[ExperimentGroupResponse], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "filter": filter, + "page": page, + "page_size": page_size, + "sort": sort, + }, + experiment_group_list_params.ExperimentGroupListParams, + ), + ), + model=ExperimentGroupResponse, + ) + + def delete( + self, + name: str, + *, + workspace: str | None = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> None: + """ + Delete Experiment Group + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + extra_headers = {"Accept": "*/*", **(extra_headers or {})} + return self._delete( + path_template( + "/apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}", workspace=workspace, name=name + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=NoneType, + ) + + +class AsyncExperimentGroupsResource(AsyncAPIResource): + @cached_property + def with_raw_response(self) -> AsyncExperimentGroupsResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#accessing-raw-response-data-e-g-headers + """ + return AsyncExperimentGroupsResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncExperimentGroupsResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#with_streaming_response + """ + return AsyncExperimentGroupsResourceWithStreamingResponse(self) + + async def create( + self, + *, + workspace: str | None = None, + name: str, + description: str | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + exist_ok: bool = False, + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ExperimentGroupResponse: + """ + Create Experiment Group + + Args: + name: Workspace-unique group name. + + description: Human-readable purpose of the group. + + + exist_ok: Do not raise an error if the resource already exists. Returns the existing resource. + + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + try: + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + return await self._post( + path_template("/apis/intake/v2/workspaces/{workspace}/experiment-groups", workspace=workspace), + body=await async_maybe_transform( + { + "name": name, + "description": description, + }, + experiment_group_create_params.ExperimentGroupCreateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ExperimentGroupResponse, + ) + except ConflictError: + if not exist_ok: + raise + return await self.retrieve(name = name, workspace = workspace) + + async def retrieve( + self, + name: str, + *, + workspace: str | None = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ExperimentGroupResponse: + """ + Get Experiment Group + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + return await self._get( + path_template( + "/apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}", workspace=workspace, name=name + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ExperimentGroupResponse, + ) + + async def update( + self, + path_name: str, + *, + workspace: str | None = None, + body_name: str, + description: str | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ExperimentGroupResponse: + """ + Update Experiment Group + + Args: + body_name: Workspace-unique group name. + + description: Human-readable purpose of the group. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not path_name: + raise ValueError(f"Expected a non-empty value for `path_name` but received {path_name!r}") + return await self._put( + path_template( + "/apis/intake/v2/workspaces/{workspace}/experiment-groups/{path_name}", + workspace=workspace, + path_name=path_name, + ), + body=await async_maybe_transform( + { + "body_name": body_name, + "description": description, + }, + experiment_group_update_params.ExperimentGroupUpdateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ExperimentGroupResponse, + ) + + def list( + self, + *, + workspace: str | None = None, + filter: ExperimentGroupFilterParam | Omit = omit, + page: int | Omit = omit, + page_size: int | Omit = omit, + sort: Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"] | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> AsyncPaginator[ExperimentGroupResponse, AsyncDefaultPagination[ExperimentGroupResponse]]: + """ + List Experiment Groups + + Args: + filter: Filter experiment groups by name. + + page: Page number. + + page_size: Page size. + + sort: Sort field; prefix with '-' for descending. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + return self._get_api_list( + path_template("/apis/intake/v2/workspaces/{workspace}/experiment-groups", workspace=workspace), + page=AsyncDefaultPagination[ExperimentGroupResponse], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "filter": filter, + "page": page, + "page_size": page_size, + "sort": sort, + }, + experiment_group_list_params.ExperimentGroupListParams, + ), + ), + model=ExperimentGroupResponse, + ) + + async def delete( + self, + name: str, + *, + workspace: str | None = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> None: + """ + Delete Experiment Group + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + extra_headers = {"Accept": "*/*", **(extra_headers or {})} + return await self._delete( + path_template( + "/apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}", workspace=workspace, name=name + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=NoneType, + ) + + +class ExperimentGroupsResourceWithRawResponse: + def __init__(self, experiment_groups: ExperimentGroupsResource) -> None: + self._experiment_groups = experiment_groups + + self.create = to_raw_response_wrapper( + experiment_groups.create, + ) + self.retrieve = to_raw_response_wrapper( + experiment_groups.retrieve, + ) + self.update = to_raw_response_wrapper( + experiment_groups.update, + ) + self.list = to_raw_response_wrapper( + experiment_groups.list, + ) + self.delete = to_raw_response_wrapper( + experiment_groups.delete, + ) + + +class AsyncExperimentGroupsResourceWithRawResponse: + def __init__(self, experiment_groups: AsyncExperimentGroupsResource) -> None: + self._experiment_groups = experiment_groups + + self.create = async_to_raw_response_wrapper( + experiment_groups.create, + ) + self.retrieve = async_to_raw_response_wrapper( + experiment_groups.retrieve, + ) + self.update = async_to_raw_response_wrapper( + experiment_groups.update, + ) + self.list = async_to_raw_response_wrapper( + experiment_groups.list, + ) + self.delete = async_to_raw_response_wrapper( + experiment_groups.delete, + ) + + +class ExperimentGroupsResourceWithStreamingResponse: + def __init__(self, experiment_groups: ExperimentGroupsResource) -> None: + self._experiment_groups = experiment_groups + + self.create = to_streamed_response_wrapper( + experiment_groups.create, + ) + self.retrieve = to_streamed_response_wrapper( + experiment_groups.retrieve, + ) + self.update = to_streamed_response_wrapper( + experiment_groups.update, + ) + self.list = to_streamed_response_wrapper( + experiment_groups.list, + ) + self.delete = to_streamed_response_wrapper( + experiment_groups.delete, + ) + + +class AsyncExperimentGroupsResourceWithStreamingResponse: + def __init__(self, experiment_groups: AsyncExperimentGroupsResource) -> None: + self._experiment_groups = experiment_groups + + self.create = async_to_streamed_response_wrapper( + experiment_groups.create, + ) + self.retrieve = async_to_streamed_response_wrapper( + experiment_groups.retrieve, + ) + self.update = async_to_streamed_response_wrapper( + experiment_groups.update, + ) + self.list = async_to_streamed_response_wrapper( + experiment_groups.list, + ) + self.delete = async_to_streamed_response_wrapper( + experiment_groups.delete, + ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/__init__.py new file mode 100644 index 0000000000..9dbe086588 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/__init__.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from .experiments import ( + ExperimentsResource, + AsyncExperimentsResource, + ExperimentsResourceWithRawResponse, + AsyncExperimentsResourceWithRawResponse, + ExperimentsResourceWithStreamingResponse, + AsyncExperimentsResourceWithStreamingResponse, +) + +__all__ = [ + "ExperimentsResource", + "AsyncExperimentsResource", + "ExperimentsResourceWithRawResponse", + "AsyncExperimentsResourceWithRawResponse", + "ExperimentsResourceWithStreamingResponse", + "AsyncExperimentsResourceWithStreamingResponse", +] diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md new file mode 100644 index 0000000000..0c72350ab0 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md @@ -0,0 +1,21 @@ +# Experiments + +Types: + +```python +from nemo_platform.types.experiments import ( + EvaluatorAggregate, + ExperimentFilter, + ExperimentRequest, + ExperimentResponse, + ExperimentResponsesPage, +) +``` + +Methods: + +- client.experiments.create(\*, workspace, \*\*params) -> ExperimentResponse +- client.experiments.retrieve(name, \*, workspace) -> ExperimentResponse +- client.experiments.update(path_name, \*, workspace, \*\*params) -> ExperimentResponse +- client.experiments.list(\*, workspace, \*\*params) -> SyncDefaultPagination[ExperimentResponse] +- client.experiments.delete(name, \*, workspace) -> None diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py new file mode 100644 index 0000000000..48bd162d47 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py @@ -0,0 +1,804 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Dict +from typing_extensions import Literal + +import httpx + +from ..._types import Body, Omit, Query, Headers, NoneType, NotGiven, omit, not_given +from ..._utils import path_template, maybe_transform, async_maybe_transform +from ..._compat import cached_property +from ..._resource import SyncAPIResource, AsyncAPIResource +from ..._response import ( + to_raw_response_wrapper, + to_streamed_response_wrapper, + async_to_raw_response_wrapper, + async_to_streamed_response_wrapper, +) +from ...pagination import SyncDefaultPagination, AsyncDefaultPagination +from ..._exceptions import ConflictError +from ..._base_client import AsyncPaginator, make_request_options +from ...types.experiments import ( + experiment_list_params, + experiment_create_params, + experiment_update_params, +) +from ...types.experiments.experiment_response import ExperimentResponse +from ...types.experiments.experiment_filter_param import ExperimentFilterParam + +__all__ = ["ExperimentsResource", "AsyncExperimentsResource"] + + +class ExperimentsResource(SyncAPIResource): + @cached_property + def with_raw_response(self) -> ExperimentsResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#accessing-raw-response-data-e-g-headers + """ + return ExperimentsResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> ExperimentsResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#with_streaming_response + """ + return ExperimentsResourceWithStreamingResponse(self) + + def create( + self, + *, + workspace: str | None = None, + agent_name: str, + agent_version: str, + dataset_name: str, + name: str, + dataset_version: str | Omit = omit, + description: str | Omit = omit, + experiment_group_id: str | Omit = omit, + metadata: Dict[str, object] | Omit = omit, + source_link: str | Omit = omit, + summary: str | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + exist_ok: bool = False, + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ExperimentResponse: + """ + Create Experiment + + Args: + agent_name: Name of the agent under test. + + agent_version: Version of the agent under test. + + dataset_name: Producer-supplied dataset name. + + name: Producer-supplied, workspace-unique experiment id. + + dataset_version: Producer-supplied dataset version. + + description: Human-readable description. + + experiment_group_id: Entity id of the owning ExperimentGroup; optional. Soft reference, not + validated. + + metadata: Free-form producer metadata. + + source_link: Optional URL for the source experiment. + + summary: Human-authored summary of results. + + + exist_ok: Do not raise an error if the resource already exists. Returns the existing resource. + + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + try: + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + return self._post( + path_template("/apis/intake/v2/workspaces/{workspace}/experiments", workspace=workspace), + body=maybe_transform( + { + "agent_name": agent_name, + "agent_version": agent_version, + "dataset_name": dataset_name, + "name": name, + "dataset_version": dataset_version, + "description": description, + "experiment_group_id": experiment_group_id, + "metadata": metadata, + "source_link": source_link, + "summary": summary, + }, + experiment_create_params.ExperimentCreateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ExperimentResponse, + ) + except ConflictError: + if not exist_ok: + raise + return self.retrieve(name=name, workspace=workspace) + + def retrieve( + self, + name: str, + *, + workspace: str | None = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ExperimentResponse: + """ + Get Experiment + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + return self._get( + path_template("/apis/intake/v2/workspaces/{workspace}/experiments/{name}", workspace=workspace, name=name), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ExperimentResponse, + ) + + def update( + self, + path_name: str, + *, + workspace: str | None = None, + agent_name: str, + agent_version: str, + dataset_name: str, + body_name: str, + dataset_version: str | Omit = omit, + description: str | Omit = omit, + experiment_group_id: str | Omit = omit, + metadata: Dict[str, object] | Omit = omit, + source_link: str | Omit = omit, + summary: str | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ExperimentResponse: + """ + Update Experiment + + Args: + agent_name: Name of the agent under test. + + agent_version: Version of the agent under test. + + dataset_name: Producer-supplied dataset name. + + body_name: Producer-supplied, workspace-unique experiment id. + + dataset_version: Producer-supplied dataset version. + + description: Human-readable description. + + experiment_group_id: Entity id of the owning ExperimentGroup; optional. Soft reference, not + validated. + + metadata: Free-form producer metadata. + + source_link: Optional URL for the source experiment. + + summary: Human-authored summary of results. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not path_name: + raise ValueError(f"Expected a non-empty value for `path_name` but received {path_name!r}") + return self._put( + path_template( + "/apis/intake/v2/workspaces/{workspace}/experiments/{path_name}", + workspace=workspace, + path_name=path_name, + ), + body=maybe_transform( + { + "agent_name": agent_name, + "agent_version": agent_version, + "dataset_name": dataset_name, + "body_name": body_name, + "dataset_version": dataset_version, + "description": description, + "experiment_group_id": experiment_group_id, + "metadata": metadata, + "source_link": source_link, + "summary": summary, + }, + experiment_update_params.ExperimentUpdateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ExperimentResponse, + ) + + def list( + self, + *, + workspace: str | None = None, + filter: ExperimentFilterParam | Omit = omit, + page: int | Omit = omit, + page_size: int | Omit = omit, + sort: Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"] | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> SyncDefaultPagination[ExperimentResponse]: + """ + List Experiments + + Args: + filter: Filter experiments by name, experiment_group_id, agent_name, and dataset_name. + + page: Page number. + + page_size: Page size. + + sort: Sort field; prefix with '-' for descending. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + return self._get_api_list( + path_template("/apis/intake/v2/workspaces/{workspace}/experiments", workspace=workspace), + page=SyncDefaultPagination[ExperimentResponse], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "filter": filter, + "page": page, + "page_size": page_size, + "sort": sort, + }, + experiment_list_params.ExperimentListParams, + ), + ), + model=ExperimentResponse, + ) + + def delete( + self, + name: str, + *, + workspace: str | None = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> None: + """ + Delete Experiment + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + extra_headers = {"Accept": "*/*", **(extra_headers or {})} + return self._delete( + path_template("/apis/intake/v2/workspaces/{workspace}/experiments/{name}", workspace=workspace, name=name), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=NoneType, + ) + + +class AsyncExperimentsResource(AsyncAPIResource): + @cached_property + def with_raw_response(self) -> AsyncExperimentsResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#accessing-raw-response-data-e-g-headers + """ + return AsyncExperimentsResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncExperimentsResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#with_streaming_response + """ + return AsyncExperimentsResourceWithStreamingResponse(self) + + async def create( + self, + *, + workspace: str | None = None, + agent_name: str, + agent_version: str, + dataset_name: str, + name: str, + dataset_version: str | Omit = omit, + description: str | Omit = omit, + experiment_group_id: str | Omit = omit, + metadata: Dict[str, object] | Omit = omit, + source_link: str | Omit = omit, + summary: str | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + exist_ok: bool = False, + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ExperimentResponse: + """ + Create Experiment + + Args: + agent_name: Name of the agent under test. + + agent_version: Version of the agent under test. + + dataset_name: Producer-supplied dataset name. + + name: Producer-supplied, workspace-unique experiment id. + + dataset_version: Producer-supplied dataset version. + + description: Human-readable description. + + experiment_group_id: Entity id of the owning ExperimentGroup; optional. Soft reference, not + validated. + + metadata: Free-form producer metadata. + + source_link: Optional URL for the source experiment. + + summary: Human-authored summary of results. + + + exist_ok: Do not raise an error if the resource already exists. Returns the existing resource. + + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + try: + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + return await self._post( + path_template("/apis/intake/v2/workspaces/{workspace}/experiments", workspace=workspace), + body=await async_maybe_transform( + { + "agent_name": agent_name, + "agent_version": agent_version, + "dataset_name": dataset_name, + "name": name, + "dataset_version": dataset_version, + "description": description, + "experiment_group_id": experiment_group_id, + "metadata": metadata, + "source_link": source_link, + "summary": summary, + }, + experiment_create_params.ExperimentCreateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ExperimentResponse, + ) + except ConflictError: + if not exist_ok: + raise + return await self.retrieve(name=name, workspace=workspace) + + async def retrieve( + self, + name: str, + *, + workspace: str | None = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ExperimentResponse: + """ + Get Experiment + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + return await self._get( + path_template("/apis/intake/v2/workspaces/{workspace}/experiments/{name}", workspace=workspace, name=name), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ExperimentResponse, + ) + + async def update( + self, + path_name: str, + *, + workspace: str | None = None, + agent_name: str, + agent_version: str, + dataset_name: str, + body_name: str, + dataset_version: str | Omit = omit, + description: str | Omit = omit, + experiment_group_id: str | Omit = omit, + metadata: Dict[str, object] | Omit = omit, + source_link: str | Omit = omit, + summary: str | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ExperimentResponse: + """ + Update Experiment + + Args: + agent_name: Name of the agent under test. + + agent_version: Version of the agent under test. + + dataset_name: Producer-supplied dataset name. + + body_name: Producer-supplied, workspace-unique experiment id. + + dataset_version: Producer-supplied dataset version. + + description: Human-readable description. + + experiment_group_id: Entity id of the owning ExperimentGroup; optional. Soft reference, not + validated. + + metadata: Free-form producer metadata. + + source_link: Optional URL for the source experiment. + + summary: Human-authored summary of results. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not path_name: + raise ValueError(f"Expected a non-empty value for `path_name` but received {path_name!r}") + return await self._put( + path_template( + "/apis/intake/v2/workspaces/{workspace}/experiments/{path_name}", + workspace=workspace, + path_name=path_name, + ), + body=await async_maybe_transform( + { + "agent_name": agent_name, + "agent_version": agent_version, + "dataset_name": dataset_name, + "body_name": body_name, + "dataset_version": dataset_version, + "description": description, + "experiment_group_id": experiment_group_id, + "metadata": metadata, + "source_link": source_link, + "summary": summary, + }, + experiment_update_params.ExperimentUpdateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ExperimentResponse, + ) + + def list( + self, + *, + workspace: str | None = None, + filter: ExperimentFilterParam | Omit = omit, + page: int | Omit = omit, + page_size: int | Omit = omit, + sort: Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"] | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> AsyncPaginator[ExperimentResponse, AsyncDefaultPagination[ExperimentResponse]]: + """ + List Experiments + + Args: + filter: Filter experiments by name, experiment_group_id, agent_name, and dataset_name. + + page: Page number. + + page_size: Page size. + + sort: Sort field; prefix with '-' for descending. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + return self._get_api_list( + path_template("/apis/intake/v2/workspaces/{workspace}/experiments", workspace=workspace), + page=AsyncDefaultPagination[ExperimentResponse], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "filter": filter, + "page": page, + "page_size": page_size, + "sort": sort, + }, + experiment_list_params.ExperimentListParams, + ), + ), + model=ExperimentResponse, + ) + + async def delete( + self, + name: str, + *, + workspace: str | None = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> None: + """ + Delete Experiment + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + extra_headers = {"Accept": "*/*", **(extra_headers or {})} + return await self._delete( + path_template("/apis/intake/v2/workspaces/{workspace}/experiments/{name}", workspace=workspace, name=name), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=NoneType, + ) + + +class ExperimentsResourceWithRawResponse: + def __init__(self, experiments: ExperimentsResource) -> None: + self._experiments = experiments + + self.create = to_raw_response_wrapper( + experiments.create, + ) + self.retrieve = to_raw_response_wrapper( + experiments.retrieve, + ) + self.update = to_raw_response_wrapper( + experiments.update, + ) + self.list = to_raw_response_wrapper( + experiments.list, + ) + self.delete = to_raw_response_wrapper( + experiments.delete, + ) + + +class AsyncExperimentsResourceWithRawResponse: + def __init__(self, experiments: AsyncExperimentsResource) -> None: + self._experiments = experiments + + self.create = async_to_raw_response_wrapper( + experiments.create, + ) + self.retrieve = async_to_raw_response_wrapper( + experiments.retrieve, + ) + self.update = async_to_raw_response_wrapper( + experiments.update, + ) + self.list = async_to_raw_response_wrapper( + experiments.list, + ) + self.delete = async_to_raw_response_wrapper( + experiments.delete, + ) + + +class ExperimentsResourceWithStreamingResponse: + def __init__(self, experiments: ExperimentsResource) -> None: + self._experiments = experiments + + self.create = to_streamed_response_wrapper( + experiments.create, + ) + self.retrieve = to_streamed_response_wrapper( + experiments.retrieve, + ) + self.update = to_streamed_response_wrapper( + experiments.update, + ) + self.list = to_streamed_response_wrapper( + experiments.list, + ) + self.delete = to_streamed_response_wrapper( + experiments.delete, + ) + + +class AsyncExperimentsResourceWithStreamingResponse: + def __init__(self, experiments: AsyncExperimentsResource) -> None: + self._experiments = experiments + + self.create = async_to_streamed_response_wrapper( + experiments.create, + ) + self.retrieve = async_to_streamed_response_wrapper( + experiments.retrieve, + ) + self.update = async_to_streamed_response_wrapper( + experiments.update, + ) + self.list = async_to_streamed_response_wrapper( + experiments.list, + ) + self.delete = async_to_streamed_response_wrapper( + experiments.delete, + ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py new file mode 100644 index 0000000000..3506053fad --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from .experiment_group_response import ExperimentGroupResponse as ExperimentGroupResponse +from .experiment_group_list_params import ExperimentGroupListParams as ExperimentGroupListParams +from .experiment_group_filter_param import ExperimentGroupFilterParam as ExperimentGroupFilterParam +from .experiment_group_create_params import ExperimentGroupCreateParams as ExperimentGroupCreateParams +from .experiment_group_update_params import ExperimentGroupUpdateParams as ExperimentGroupUpdateParams +from .experiment_group_responses_page import ExperimentGroupResponsesPage as ExperimentGroupResponsesPage diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py new file mode 100644 index 0000000000..5feef3cccd --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Required, TypedDict + +__all__ = ["ExperimentGroupCreateParams"] + + +class ExperimentGroupCreateParams(TypedDict, total=False): + workspace: str + + name: Required[str] + """Workspace-unique group name.""" + + description: str + """Human-readable purpose of the group.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_filter_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_filter_param.py new file mode 100644 index 0000000000..4124d2eb8f --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_filter_param.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import TypedDict + +__all__ = ["ExperimentGroupFilterParam"] + + +class ExperimentGroupFilterParam(TypedDict, total=False): + """Filter for listing ExperimentGroups.""" + + name: str + """Filter groups by name.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_list_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_list_params.py new file mode 100644 index 0000000000..6e287963f9 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_list_params.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal, TypedDict + +from .experiment_group_filter_param import ExperimentGroupFilterParam + +__all__ = ["ExperimentGroupListParams"] + + +class ExperimentGroupListParams(TypedDict, total=False): + workspace: str + + filter: ExperimentGroupFilterParam + """Filter experiment groups by name.""" + + page: int + """Page number.""" + + page_size: int + """Page size.""" + + sort: Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"] + """Sort field; prefix with '-' for descending.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py new file mode 100644 index 0000000000..2a8aa6ed06 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from datetime import datetime + +from ..._models import BaseModel + +__all__ = ["ExperimentGroupResponse"] + + +class ExperimentGroupResponse(BaseModel): + """ExperimentGroup as served by the API.""" + + id: str + + name: str + + workspace: str + + created_at: Optional[datetime] = None + + description: Optional[str] = None + + updated_at: Optional[datetime] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_responses_page.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_responses_page.py new file mode 100644 index 0000000000..ad7fa77877 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_responses_page.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Dict, List, Optional + +from ..._models import BaseModel +from ..shared.pagination_data import PaginationData +from .experiment_group_response import ExperimentGroupResponse + +__all__ = ["ExperimentGroupResponsesPage"] + + +class ExperimentGroupResponsesPage(BaseModel): + data: List[ExperimentGroupResponse] + + filter: Optional[Dict[str, object]] = None + """Filtering information.""" + + pagination: Optional[PaginationData] = None + """Pagination information.""" + + sort: Optional[str] = None + """The field on which the results are sorted.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py new file mode 100644 index 0000000000..2483e1b2de --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Required, Annotated, TypedDict + +from ..._utils import PropertyInfo + +__all__ = ["ExperimentGroupUpdateParams"] + + +class ExperimentGroupUpdateParams(TypedDict, total=False): + workspace: str + + body_name: Required[Annotated[str, PropertyInfo(alias="name")]] + """Workspace-unique group name.""" + + description: str + """Human-readable purpose of the group.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/__init__.py new file mode 100644 index 0000000000..9663ee8371 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/__init__.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from .evaluator_aggregate import EvaluatorAggregate as EvaluatorAggregate +from .experiment_response import ExperimentResponse as ExperimentResponse +from .experiment_list_params import ExperimentListParams as ExperimentListParams +from .experiment_filter_param import ExperimentFilterParam as ExperimentFilterParam +from .experiment_create_params import ExperimentCreateParams as ExperimentCreateParams +from .experiment_update_params import ExperimentUpdateParams as ExperimentUpdateParams +from .experiment_responses_page import ExperimentResponsesPage as ExperimentResponsesPage diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/evaluator_aggregate.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/evaluator_aggregate.py new file mode 100644 index 0000000000..8551b4ff43 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/evaluator_aggregate.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional + +from ..._models import BaseModel + +__all__ = ["EvaluatorAggregate"] + + +class EvaluatorAggregate(BaseModel): + """Cross-run statistics for one evaluator. + + Populated by the rollup path (later PR). + """ + + sum: Optional[float] = None + + mean: Optional[float] = None + + median: Optional[float] = None + + p90: Optional[float] = None + + p95: Optional[float] = None + + p99: Optional[float] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py new file mode 100644 index 0000000000..ad2eee9890 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Dict +from typing_extensions import Required, TypedDict + +__all__ = ["ExperimentCreateParams"] + + +class ExperimentCreateParams(TypedDict, total=False): + workspace: str + + agent_name: Required[str] + """Name of the agent under test.""" + + agent_version: Required[str] + """Version of the agent under test.""" + + dataset_name: Required[str] + """Producer-supplied dataset name.""" + + name: Required[str] + """Producer-supplied, workspace-unique experiment id.""" + + dataset_version: str + """Producer-supplied dataset version.""" + + description: str + """Human-readable description.""" + + experiment_group_id: str + """Entity id of the owning ExperimentGroup; optional. + + Soft reference, not validated. + """ + + metadata: Dict[str, object] + """Free-form producer metadata.""" + + source_link: str + """Optional URL for the source experiment.""" + + summary: str + """Human-authored summary of results.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_filter_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_filter_param.py new file mode 100644 index 0000000000..5cccf67e14 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_filter_param.py @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import TypedDict + +__all__ = ["ExperimentFilterParam"] + + +class ExperimentFilterParam(TypedDict, total=False): + """Filter for listing Experiments.""" + + agent_name: str + """Filter experiments by agent name.""" + + dataset_name: str + """Filter experiments by dataset name.""" + + experiment_group_id: str + """Filter experiments by owning group id.""" + + name: str + """Filter experiments by name.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py new file mode 100644 index 0000000000..15d7211780 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal, TypedDict + +from .experiment_filter_param import ExperimentFilterParam + +__all__ = ["ExperimentListParams"] + + +class ExperimentListParams(TypedDict, total=False): + workspace: str + + filter: ExperimentFilterParam + """Filter experiments by name, experiment_group_id, agent_name, and dataset_name.""" + + page: int + """Page number.""" + + page_size: int + """Page size.""" + + sort: Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"] + """Sort field; prefix with '-' for descending.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py new file mode 100644 index 0000000000..33c7ec3500 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Dict, List, Optional +from datetime import datetime + +from ..._models import BaseModel +from .evaluator_aggregate import EvaluatorAggregate + +__all__ = ["ExperimentResponse"] + + +class ExperimentResponse(BaseModel): + """Experiment as served by the API, including ClickHouse-hydrated rollups.""" + + id: str + + agent_name: str + + agent_version: str + + dataset_name: str + + name: str + + workspace: str + + aggregate_scores: Optional[Dict[str, EvaluatorAggregate]] = None + + created_at: Optional[datetime] = None + + dataset_version: Optional[str] = None + + description: Optional[str] = None + + evaluator_names: Optional[List[str]] = None + + experiment_group_id: Optional[str] = None + """Entity id of the owning ExperimentGroup; null when ungrouped. + + Soft reference, not validated. + """ + + metadata: Optional[Dict[str, object]] = None + + model_names: Optional[List[str]] = None + + run_count: Optional[int] = None + + source_link: Optional[str] = None + + summary: Optional[str] = None + + updated_at: Optional[datetime] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_responses_page.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_responses_page.py new file mode 100644 index 0000000000..ad2f49105e --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_responses_page.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Dict, List, Optional + +from ..._models import BaseModel +from .experiment_response import ExperimentResponse +from ..shared.pagination_data import PaginationData + +__all__ = ["ExperimentResponsesPage"] + + +class ExperimentResponsesPage(BaseModel): + data: List[ExperimentResponse] + + filter: Optional[Dict[str, object]] = None + """Filtering information.""" + + pagination: Optional[PaginationData] = None + """Pagination information.""" + + sort: Optional[str] = None + """The field on which the results are sorted.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py new file mode 100644 index 0000000000..e895ebfa20 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Dict +from typing_extensions import Required, Annotated, TypedDict + +from ..._utils import PropertyInfo + +__all__ = ["ExperimentUpdateParams"] + + +class ExperimentUpdateParams(TypedDict, total=False): + workspace: str + + agent_name: Required[str] + """Name of the agent under test.""" + + agent_version: Required[str] + """Version of the agent under test.""" + + dataset_name: Required[str] + """Producer-supplied dataset name.""" + + body_name: Required[Annotated[str, PropertyInfo(alias="name")]] + """Producer-supplied, workspace-unique experiment id.""" + + dataset_version: str + """Producer-supplied dataset version.""" + + description: str + """Human-readable description.""" + + experiment_group_id: str + """Entity id of the owning ExperimentGroup; optional. + + Soft reference, not validated. + """ + + metadata: Dict[str, object] + """Free-form producer metadata.""" + + source_link: str + """Optional URL for the source experiment.""" + + summary: str + """Human-authored summary of results.""" diff --git a/sdk/python/nemo-platform/tests/api_resources/experiment_groups/__init__.py b/sdk/python/nemo-platform/tests/api_resources/experiment_groups/__init__.py new file mode 100644 index 0000000000..92808494e3 --- /dev/null +++ b/sdk/python/nemo-platform/tests/api_resources/experiment_groups/__init__.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. diff --git a/sdk/python/nemo-platform/tests/api_resources/experiments/__init__.py b/sdk/python/nemo-platform/tests/api_resources/experiments/__init__.py new file mode 100644 index 0000000000..92808494e3 --- /dev/null +++ b/sdk/python/nemo-platform/tests/api_resources/experiments/__init__.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. diff --git a/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py b/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py new file mode 100644 index 0000000000..90f6e4ce35 --- /dev/null +++ b/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py @@ -0,0 +1,606 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import os +from typing import Any, cast + +import pytest + +from tests.utils import assert_matches_type +from nemo_platform import NeMoPlatform, AsyncNeMoPlatform +from nemo_platform.pagination import SyncDefaultPagination, AsyncDefaultPagination +from nemo_platform.types.experiment_groups import ( + ExperimentGroupResponse, +) + +base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") + + +class TestExperimentGroups: + parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_create(self, client: NeMoPlatform) -> None: + experiment_group = client.experiment_groups.create( + workspace="workspace", + name="name", + ) + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: + experiment_group = client.experiment_groups.create( + workspace="workspace", + name="name", + description="description", + ) + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_create(self, client: NeMoPlatform) -> None: + response = client.experiment_groups.with_raw_response.create( + workspace="workspace", + name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment_group = response.parse() + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_create(self, client: NeMoPlatform) -> None: + with client.experiment_groups.with_streaming_response.create( + workspace="workspace", + name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment_group = response.parse() + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_create(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.experiment_groups.with_raw_response.create( + workspace="", + name="name", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_retrieve(self, client: NeMoPlatform) -> None: + experiment_group = client.experiment_groups.retrieve( + name="name", + workspace="workspace", + ) + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_retrieve(self, client: NeMoPlatform) -> None: + response = client.experiment_groups.with_raw_response.retrieve( + name="name", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment_group = response.parse() + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_retrieve(self, client: NeMoPlatform) -> None: + with client.experiment_groups.with_streaming_response.retrieve( + name="name", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment_group = response.parse() + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_retrieve(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.experiment_groups.with_raw_response.retrieve( + name="name", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + client.experiment_groups.with_raw_response.retrieve( + name="", + workspace="workspace", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_update(self, client: NeMoPlatform) -> None: + experiment_group = client.experiment_groups.update( + path_name="name", + workspace="workspace", + body_name="name", + ) + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_update_with_all_params(self, client: NeMoPlatform) -> None: + experiment_group = client.experiment_groups.update( + path_name="name", + workspace="workspace", + body_name="name", + description="description", + ) + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_update(self, client: NeMoPlatform) -> None: + response = client.experiment_groups.with_raw_response.update( + path_name="name", + workspace="workspace", + body_name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment_group = response.parse() + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_update(self, client: NeMoPlatform) -> None: + with client.experiment_groups.with_streaming_response.update( + path_name="name", + workspace="workspace", + body_name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment_group = response.parse() + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_update(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.experiment_groups.with_raw_response.update( + path_name="name", + workspace="", + body_name="name", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `path_name` but received ''"): + client.experiment_groups.with_raw_response.update( + path_name="", + workspace="workspace", + body_name="name", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_list(self, client: NeMoPlatform) -> None: + experiment_group = client.experiment_groups.list( + workspace="workspace", + ) + assert_matches_type(SyncDefaultPagination[ExperimentGroupResponse], experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: + experiment_group = client.experiment_groups.list( + workspace="workspace", + filter={"name": "name"}, + page=1, + page_size=1, + sort="-created_at", + ) + assert_matches_type(SyncDefaultPagination[ExperimentGroupResponse], experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_list(self, client: NeMoPlatform) -> None: + response = client.experiment_groups.with_raw_response.list( + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment_group = response.parse() + assert_matches_type(SyncDefaultPagination[ExperimentGroupResponse], experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_list(self, client: NeMoPlatform) -> None: + with client.experiment_groups.with_streaming_response.list( + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment_group = response.parse() + assert_matches_type(SyncDefaultPagination[ExperimentGroupResponse], experiment_group, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_list(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.experiment_groups.with_raw_response.list( + workspace="", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_delete(self, client: NeMoPlatform) -> None: + experiment_group = client.experiment_groups.delete( + name="name", + workspace="workspace", + ) + assert experiment_group is None + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_delete(self, client: NeMoPlatform) -> None: + response = client.experiment_groups.with_raw_response.delete( + name="name", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment_group = response.parse() + assert experiment_group is None + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_delete(self, client: NeMoPlatform) -> None: + with client.experiment_groups.with_streaming_response.delete( + name="name", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment_group = response.parse() + assert experiment_group is None + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_delete(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.experiment_groups.with_raw_response.delete( + name="name", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + client.experiment_groups.with_raw_response.delete( + name="", + workspace="workspace", + ) + + +class TestAsyncExperimentGroups: + parametrize = pytest.mark.parametrize( + "async_client", [False, True, {"http_client": "aiohttp"}], indirect=True, ids=["loose", "strict", "aiohttp"] + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_create(self, async_client: AsyncNeMoPlatform) -> None: + experiment_group = await async_client.experiment_groups.create( + workspace="workspace", + name="name", + ) + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatform) -> None: + experiment_group = await async_client.experiment_groups.create( + workspace="workspace", + name="name", + description="description", + ) + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_create(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.experiment_groups.with_raw_response.create( + workspace="workspace", + name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment_group = await response.parse() + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_create(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.experiment_groups.with_streaming_response.create( + workspace="workspace", + name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment_group = await response.parse() + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_create(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.experiment_groups.with_raw_response.create( + workspace="", + name="name", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_retrieve(self, async_client: AsyncNeMoPlatform) -> None: + experiment_group = await async_client.experiment_groups.retrieve( + name="name", + workspace="workspace", + ) + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_retrieve(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.experiment_groups.with_raw_response.retrieve( + name="name", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment_group = await response.parse() + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_retrieve(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.experiment_groups.with_streaming_response.retrieve( + name="name", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment_group = await response.parse() + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_retrieve(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.experiment_groups.with_raw_response.retrieve( + name="name", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + await async_client.experiment_groups.with_raw_response.retrieve( + name="", + workspace="workspace", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_update(self, async_client: AsyncNeMoPlatform) -> None: + experiment_group = await async_client.experiment_groups.update( + path_name="name", + workspace="workspace", + body_name="name", + ) + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_update_with_all_params(self, async_client: AsyncNeMoPlatform) -> None: + experiment_group = await async_client.experiment_groups.update( + path_name="name", + workspace="workspace", + body_name="name", + description="description", + ) + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_update(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.experiment_groups.with_raw_response.update( + path_name="name", + workspace="workspace", + body_name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment_group = await response.parse() + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_update(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.experiment_groups.with_streaming_response.update( + path_name="name", + workspace="workspace", + body_name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment_group = await response.parse() + assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_update(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.experiment_groups.with_raw_response.update( + path_name="name", + workspace="", + body_name="name", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `path_name` but received ''"): + await async_client.experiment_groups.with_raw_response.update( + path_name="", + workspace="workspace", + body_name="name", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_list(self, async_client: AsyncNeMoPlatform) -> None: + experiment_group = await async_client.experiment_groups.list( + workspace="workspace", + ) + assert_matches_type(AsyncDefaultPagination[ExperimentGroupResponse], experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform) -> None: + experiment_group = await async_client.experiment_groups.list( + workspace="workspace", + filter={"name": "name"}, + page=1, + page_size=1, + sort="-created_at", + ) + assert_matches_type(AsyncDefaultPagination[ExperimentGroupResponse], experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_list(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.experiment_groups.with_raw_response.list( + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment_group = await response.parse() + assert_matches_type(AsyncDefaultPagination[ExperimentGroupResponse], experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_list(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.experiment_groups.with_streaming_response.list( + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment_group = await response.parse() + assert_matches_type(AsyncDefaultPagination[ExperimentGroupResponse], experiment_group, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_list(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.experiment_groups.with_raw_response.list( + workspace="", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_delete(self, async_client: AsyncNeMoPlatform) -> None: + experiment_group = await async_client.experiment_groups.delete( + name="name", + workspace="workspace", + ) + assert experiment_group is None + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_delete(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.experiment_groups.with_raw_response.delete( + name="name", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment_group = await response.parse() + assert experiment_group is None + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_delete(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.experiment_groups.with_streaming_response.delete( + name="name", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment_group = await response.parse() + assert experiment_group is None + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_delete(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.experiment_groups.with_raw_response.delete( + name="name", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + await async_client.experiment_groups.with_raw_response.delete( + name="", + workspace="workspace", + ) diff --git a/sdk/python/nemo-platform/tests/api_resources/test_experiments.py b/sdk/python/nemo-platform/tests/api_resources/test_experiments.py new file mode 100644 index 0000000000..b1dd5efc73 --- /dev/null +++ b/sdk/python/nemo-platform/tests/api_resources/test_experiments.py @@ -0,0 +1,702 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import os +from typing import Any, cast + +import pytest + +from tests.utils import assert_matches_type +from nemo_platform import NeMoPlatform, AsyncNeMoPlatform +from nemo_platform.pagination import SyncDefaultPagination, AsyncDefaultPagination +from nemo_platform.types.experiments import ( + ExperimentResponse, +) + +base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") + + +class TestExperiments: + parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_create(self, client: NeMoPlatform) -> None: + experiment = client.experiments.create( + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + name="name", + ) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: + experiment = client.experiments.create( + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + name="name", + dataset_version="dataset_version", + description="description", + experiment_group_id="experiment_group_id", + metadata={"foo": "bar"}, + source_link="https://example.com/experiments/source", + summary="summary", + ) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_create(self, client: NeMoPlatform) -> None: + response = client.experiments.with_raw_response.create( + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_create(self, client: NeMoPlatform) -> None: + with client.experiments.with_streaming_response.create( + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_create(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.experiments.with_raw_response.create( + workspace="", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + name="name", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_retrieve(self, client: NeMoPlatform) -> None: + experiment = client.experiments.retrieve( + name="name", + workspace="workspace", + ) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_retrieve(self, client: NeMoPlatform) -> None: + response = client.experiments.with_raw_response.retrieve( + name="name", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_retrieve(self, client: NeMoPlatform) -> None: + with client.experiments.with_streaming_response.retrieve( + name="name", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_retrieve(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.experiments.with_raw_response.retrieve( + name="name", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + client.experiments.with_raw_response.retrieve( + name="", + workspace="workspace", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_update(self, client: NeMoPlatform) -> None: + experiment = client.experiments.update( + path_name="name", + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + body_name="name", + ) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_update_with_all_params(self, client: NeMoPlatform) -> None: + experiment = client.experiments.update( + path_name="name", + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + body_name="name", + dataset_version="dataset_version", + description="description", + experiment_group_id="experiment_group_id", + metadata={"foo": "bar"}, + source_link="https://example.com/experiments/source", + summary="summary", + ) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_update(self, client: NeMoPlatform) -> None: + response = client.experiments.with_raw_response.update( + path_name="name", + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + body_name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_update(self, client: NeMoPlatform) -> None: + with client.experiments.with_streaming_response.update( + path_name="name", + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + body_name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_update(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.experiments.with_raw_response.update( + path_name="name", + workspace="", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + body_name="name", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `path_name` but received ''"): + client.experiments.with_raw_response.update( + path_name="", + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + body_name="name", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_list(self, client: NeMoPlatform) -> None: + experiment = client.experiments.list( + workspace="workspace", + ) + assert_matches_type(SyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: + experiment = client.experiments.list( + workspace="workspace", + filter={ + "agent_name": "agent_name", + "dataset_name": "dataset_name", + "experiment_group_id": "experiment_group_id", + "name": "name", + }, + page=1, + page_size=1, + sort="-created_at", + ) + assert_matches_type(SyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_list(self, client: NeMoPlatform) -> None: + response = client.experiments.with_raw_response.list( + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = response.parse() + assert_matches_type(SyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_list(self, client: NeMoPlatform) -> None: + with client.experiments.with_streaming_response.list( + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = response.parse() + assert_matches_type(SyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_list(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.experiments.with_raw_response.list( + workspace="", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_delete(self, client: NeMoPlatform) -> None: + experiment = client.experiments.delete( + name="name", + workspace="workspace", + ) + assert experiment is None + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_delete(self, client: NeMoPlatform) -> None: + response = client.experiments.with_raw_response.delete( + name="name", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = response.parse() + assert experiment is None + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_delete(self, client: NeMoPlatform) -> None: + with client.experiments.with_streaming_response.delete( + name="name", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = response.parse() + assert experiment is None + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_delete(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.experiments.with_raw_response.delete( + name="name", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + client.experiments.with_raw_response.delete( + name="", + workspace="workspace", + ) + + +class TestAsyncExperiments: + parametrize = pytest.mark.parametrize( + "async_client", [False, True, {"http_client": "aiohttp"}], indirect=True, ids=["loose", "strict", "aiohttp"] + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_create(self, async_client: AsyncNeMoPlatform) -> None: + experiment = await async_client.experiments.create( + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + name="name", + ) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatform) -> None: + experiment = await async_client.experiments.create( + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + name="name", + dataset_version="dataset_version", + description="description", + experiment_group_id="experiment_group_id", + metadata={"foo": "bar"}, + source_link="https://example.com/experiments/source", + summary="summary", + ) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_create(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.experiments.with_raw_response.create( + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = await response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_create(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.experiments.with_streaming_response.create( + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = await response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_create(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.experiments.with_raw_response.create( + workspace="", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + name="name", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_retrieve(self, async_client: AsyncNeMoPlatform) -> None: + experiment = await async_client.experiments.retrieve( + name="name", + workspace="workspace", + ) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_retrieve(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.experiments.with_raw_response.retrieve( + name="name", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = await response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_retrieve(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.experiments.with_streaming_response.retrieve( + name="name", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = await response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_retrieve(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.experiments.with_raw_response.retrieve( + name="name", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + await async_client.experiments.with_raw_response.retrieve( + name="", + workspace="workspace", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_update(self, async_client: AsyncNeMoPlatform) -> None: + experiment = await async_client.experiments.update( + path_name="name", + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + body_name="name", + ) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_update_with_all_params(self, async_client: AsyncNeMoPlatform) -> None: + experiment = await async_client.experiments.update( + path_name="name", + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + body_name="name", + dataset_version="dataset_version", + description="description", + experiment_group_id="experiment_group_id", + metadata={"foo": "bar"}, + source_link="https://example.com/experiments/source", + summary="summary", + ) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_update(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.experiments.with_raw_response.update( + path_name="name", + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + body_name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = await response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_update(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.experiments.with_streaming_response.update( + path_name="name", + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + body_name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = await response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_update(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.experiments.with_raw_response.update( + path_name="name", + workspace="", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + body_name="name", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `path_name` but received ''"): + await async_client.experiments.with_raw_response.update( + path_name="", + workspace="workspace", + agent_name="agent_name", + agent_version="agent_version", + dataset_name="dataset_name", + body_name="name", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_list(self, async_client: AsyncNeMoPlatform) -> None: + experiment = await async_client.experiments.list( + workspace="workspace", + ) + assert_matches_type(AsyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform) -> None: + experiment = await async_client.experiments.list( + workspace="workspace", + filter={ + "agent_name": "agent_name", + "dataset_name": "dataset_name", + "experiment_group_id": "experiment_group_id", + "name": "name", + }, + page=1, + page_size=1, + sort="-created_at", + ) + assert_matches_type(AsyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_list(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.experiments.with_raw_response.list( + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = await response.parse() + assert_matches_type(AsyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_list(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.experiments.with_streaming_response.list( + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = await response.parse() + assert_matches_type(AsyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_list(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.experiments.with_raw_response.list( + workspace="", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_delete(self, async_client: AsyncNeMoPlatform) -> None: + experiment = await async_client.experiments.delete( + name="name", + workspace="workspace", + ) + assert experiment is None + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_delete(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.experiments.with_raw_response.delete( + name="name", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = await response.parse() + assert experiment is None + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_delete(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.experiments.with_streaming_response.delete( + name="name", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = await response.parse() + assert experiment is None + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_delete(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.experiments.with_raw_response.delete( + name="name", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + await async_client.experiments.with_raw_response.delete( + name="", + workspace="workspace", + ) diff --git a/sdk/stainless.yaml b/sdk/stainless.yaml index 99b16fcf2a..074e76fc68 100644 --- a/sdk/stainless.yaml +++ b/sdk/stainless.yaml @@ -1082,3 +1082,30 @@ resources: methods: list: get /apis/intake/v2/workspaces/{workspace}/traces retrieve: get /apis/intake/v2/workspaces/{workspace}/traces/{id} + experiment_groups: + standalone_api: true + models: + experiment_group_filter: ExperimentGroupFilter + experiment_group_request: ExperimentGroupRequest + experiment_group_response: ExperimentGroupResponse + experiment_group_responses_page: ExperimentGroupResponsesPage + methods: + create: post /apis/intake/v2/workspaces/{workspace}/experiment-groups + list: get /apis/intake/v2/workspaces/{workspace}/experiment-groups + retrieve: get /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} + update: put /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} + delete: delete /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} + experiments: + standalone_api: true + models: + evaluator_aggregate: EvaluatorAggregate + experiment_filter: ExperimentFilter + experiment_request: ExperimentRequest + experiment_response: ExperimentResponse + experiment_responses_page: ExperimentResponsesPage + methods: + create: post /apis/intake/v2/workspaces/{workspace}/experiments + list: get /apis/intake/v2/workspaces/{workspace}/experiments + retrieve: get /apis/intake/v2/workspaces/{workspace}/experiments/{name} + update: put /apis/intake/v2/workspaces/{workspace}/experiments/{name} + delete: delete /apis/intake/v2/workspaces/{workspace}/experiments/{name} diff --git a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml index 4475dadec1..1117d2b97d 100644 --- a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml +++ b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml @@ -161,6 +161,24 @@ authz: description: "Read intake annotations" delete: description: "Delete intake annotations" + experiment-groups: + create: + description: "Create intake experiment groups" + delete: + description: "Delete intake experiment groups" + read: + description: "Read intake experiment groups" + update: + description: "Update intake experiment groups" + experiments: + create: + description: "Create intake experiments" + delete: + description: "Delete intake experiments" + read: + description: "Read intake experiments" + update: + description: "Update intake experiments" ingest: create: description: "Ingest traces into intake" @@ -309,6 +327,8 @@ authz: - intake.annotations.read - intake.evaluator-results.list - intake.evaluator-results.read + - intake.experiment-groups.read + - intake.experiments.read - intake.spans.list - intake.spans.read - intake.traces.read @@ -360,6 +380,12 @@ authz: - intake.annotations.create - intake.annotations.delete - intake.evaluator-results.create + - intake.experiment-groups.create + - intake.experiment-groups.delete + - intake.experiment-groups.update + - intake.experiments.create + - intake.experiments.delete + - intake.experiments.update - intake.ingest.create - jobs.cancel - jobs.create @@ -1170,6 +1196,70 @@ authz: scopes: - intake:read - platform:read + /apis/intake/v2/workspaces/{workspace}/experiment-groups: + get: + permissions: + - intake.experiment-groups.read + scopes: + - intake:read + - platform:read + post: + permissions: + - intake.experiment-groups.create + scopes: + - intake:write + - platform:write + /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: + delete: + permissions: + - intake.experiment-groups.delete + scopes: + - intake:write + - platform:write + get: + permissions: + - intake.experiment-groups.read + scopes: + - intake:read + - platform:read + put: + permissions: + - intake.experiment-groups.update + scopes: + - intake:write + - platform:write + /apis/intake/v2/workspaces/{workspace}/experiments: + get: + permissions: + - intake.experiments.read + scopes: + - intake:read + - platform:read + post: + permissions: + - intake.experiments.create + scopes: + - intake:write + - platform:write + /apis/intake/v2/workspaces/{workspace}/experiments/{name}: + delete: + permissions: + - intake.experiments.delete + scopes: + - intake:write + - platform:write + get: + permissions: + - intake.experiments.read + scopes: + - intake:read + - platform:read + put: + permissions: + - intake.experiments.update + scopes: + - intake:write + - platform:write /apis/intake/v2/workspaces/{workspace}/ingest/atif: post: permissions: diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py new file mode 100644 index 0000000000..052220b523 --- /dev/null +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -0,0 +1,343 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Create, list, get, and delete endpoints for Experiments and ExperimentGroups. + +Entity-store (Postgres) operations wired directly onto ``EntityClient``, following +the inline pattern used by the core services. PUT updates only the mutable fields +(group membership, summary, description, metadata); an Experiment's identity and the +dataset/agent it ran against are fixed and changing them is rejected. Rollup fields on +the read models are hydrated from ClickHouse in a later PR; for now they return defaults. +""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +from nmp.common.api.common import Page, PaginationData +from nmp.common.api.parsed_filter import ParsedFilter, make_filter_dep +from nmp.common.api.utils import generate_openapi_extra_params +from nmp.common.entities.client import EntityClient, EntityConflictError, EntityNotFoundError +from nmp.common.service.dependencies import get_entity_client +from nmp.intake.api.v2.experiments.schemas import ( + ExperimentFilter, + ExperimentGroupFilter, + ExperimentGroupRequest, + ExperimentGroupResponse, + ExperimentRequest, + ExperimentResponse, +) +from nmp.intake.entities.experiments import Experiment, ExperimentGroup +from nmp.intake.spans.api.dependencies import require_workspace_access, validate_list_query_params + +router = APIRouter(dependencies=[Depends(require_workspace_access)]) + +GROUPS_TAG = "Experiment Groups" +EXPERIMENTS_TAG = "Experiments" + +SortField = Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"] + +EntityClientDep = Annotated[EntityClient, Depends(get_entity_client)] +ExperimentGroupFilterDep = Annotated[ParsedFilter, Depends(make_filter_dep(ExperimentGroupFilter))] +ExperimentFilterDep = Annotated[ParsedFilter, Depends(make_filter_dep(ExperimentFilter))] + + +# ============================================================================= +# Experiment Groups +# ============================================================================= + + +@router.post( + "/v2/workspaces/{workspace}/experiment-groups", + response_model=ExperimentGroupResponse, + status_code=status.HTTP_201_CREATED, + tags=[GROUPS_TAG], + responses={409: {"description": "Experiment group already exists"}}, +) +async def create_experiment_group( + workspace: str, + body: ExperimentGroupRequest, + entity_client: EntityClientDep, +) -> ExperimentGroupResponse: + entity = ExperimentGroup(workspace=workspace, name=body.name, description=body.description) + try: + created = await entity_client.create(entity) + except EntityConflictError as e: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Experiment group '{workspace}/{body.name}' already exists.", + ) from e + return ExperimentGroupResponse.from_entity(created) + + +@router.get( + "/v2/workspaces/{workspace}/experiment-groups", + response_model=Page[ExperimentGroupResponse], + tags=[GROUPS_TAG], + openapi_extra=generate_openapi_extra_params( + filter_schema=ExperimentGroupFilter, + filter_description="Filter experiment groups by name.", + ), +) +async def list_experiment_groups( + workspace: str, + request: Request, + entity_client: EntityClientDep, + parsed: ExperimentGroupFilterDep, + page: int = Query(default=1, ge=1, description="Page number."), + page_size: int = Query(default=100, ge=1, le=1000, description="Page size."), + sort: SortField = Query(default="-created_at", description="Sort field; prefix with '-' for descending."), +) -> Page[ExperimentGroupResponse]: + validate_list_query_params(request) + result = await entity_client.list( + ExperimentGroup, + workspace=workspace, + filter_operation=parsed.operation, + sort=sort, + page=page, + page_size=page_size, + ) + return Page( + data=[ExperimentGroupResponse.from_entity(e) for e in result.data], + pagination=PaginationData(**result.pagination.model_dump()), + sort=sort, + filter=parsed.to_response(), + ) + + +@router.get( + "/v2/workspaces/{workspace}/experiment-groups/{name}", + response_model=ExperimentGroupResponse, + tags=[GROUPS_TAG], + responses={404: {"description": "Experiment group not found"}}, +) +async def get_experiment_group( + workspace: str, + name: str, + entity_client: EntityClientDep, +) -> ExperimentGroupResponse: + try: + entity = await entity_client.get(ExperimentGroup, name=name, workspace=workspace) + except EntityNotFoundError as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Experiment group '{workspace}/{name}' not found.", + ) from e + return ExperimentGroupResponse.from_entity(entity) + + +@router.put( + "/v2/workspaces/{workspace}/experiment-groups/{name}", + response_model=ExperimentGroupResponse, + tags=[GROUPS_TAG], + responses={ + 404: {"description": "Experiment group not found"}, + 409: {"description": "Attempt to rename the group"}, + }, +) +async def update_experiment_group( + workspace: str, + name: str, + body: ExperimentGroupRequest, + entity_client: EntityClientDep, +) -> ExperimentGroupResponse: + try: + existing = await entity_client.get(ExperimentGroup, name=name, workspace=workspace) + except EntityNotFoundError as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Experiment group '{workspace}/{name}' not found.", + ) from e + if body.name != name: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Cannot rename an experiment group; the name is its identity.", + ) + existing.description = body.description + updated = await entity_client.update(existing) + return ExperimentGroupResponse.from_entity(updated) + + +@router.delete( + "/v2/workspaces/{workspace}/experiment-groups/{name}", + status_code=status.HTTP_204_NO_CONTENT, + tags=[GROUPS_TAG], + responses={404: {"description": "Experiment group not found"}}, +) +async def delete_experiment_group( + workspace: str, + name: str, + entity_client: EntityClientDep, +) -> None: + try: + await entity_client.delete(ExperimentGroup, name=name, workspace=workspace) + except EntityNotFoundError as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Experiment group '{workspace}/{name}' not found.", + ) from e + + +# ============================================================================= +# Experiments +# ============================================================================= + + +@router.post( + "/v2/workspaces/{workspace}/experiments", + response_model=ExperimentResponse, + status_code=status.HTTP_201_CREATED, + tags=[EXPERIMENTS_TAG], + responses={409: {"description": "Experiment already exists"}}, +) +async def create_experiment( + workspace: str, + body: ExperimentRequest, + entity_client: EntityClientDep, +) -> ExperimentResponse: + entity = Experiment( + workspace=workspace, + name=body.name, + experiment_group_id=body.experiment_group_id, + agent_name=body.agent_name, + agent_version=body.agent_version, + dataset_name=body.dataset_name, + dataset_version=body.dataset_version, + source_link=body.source_link, + metadata=body.metadata, + description=body.description, + summary=body.summary, + ) + try: + created = await entity_client.create(entity) + except EntityConflictError as e: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Experiment '{workspace}/{body.name}' already exists.", + ) from e + return ExperimentResponse.from_entity(created) + + +@router.get( + "/v2/workspaces/{workspace}/experiments", + response_model=Page[ExperimentResponse], + tags=[EXPERIMENTS_TAG], + openapi_extra=generate_openapi_extra_params( + filter_schema=ExperimentFilter, + filter_description="Filter experiments by name, experiment_group_id, agent_name, and dataset_name.", + ), +) +async def list_experiments( + workspace: str, + request: Request, + entity_client: EntityClientDep, + parsed: ExperimentFilterDep, + page: int = Query(default=1, ge=1, description="Page number."), + page_size: int = Query(default=100, ge=1, le=1000, description="Page size."), + sort: SortField = Query(default="-created_at", description="Sort field; prefix with '-' for descending."), +) -> Page[ExperimentResponse]: + validate_list_query_params(request) + result = await entity_client.list( + Experiment, + workspace=workspace, + filter_operation=parsed.operation, + sort=sort, + page=page, + page_size=page_size, + ) + return Page( + data=[ExperimentResponse.from_entity(e) for e in result.data], + pagination=PaginationData(**result.pagination.model_dump()), + sort=sort, + filter=parsed.to_response(), + ) + + +@router.get( + "/v2/workspaces/{workspace}/experiments/{name}", + response_model=ExperimentResponse, + tags=[EXPERIMENTS_TAG], + responses={404: {"description": "Experiment not found"}}, +) +async def get_experiment( + workspace: str, + name: str, + entity_client: EntityClientDep, +) -> ExperimentResponse: + try: + entity = await entity_client.get(Experiment, name=name, workspace=workspace) + except EntityNotFoundError as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Experiment '{workspace}/{name}' not found.", + ) from e + return ExperimentResponse.from_entity(entity) + + +# Identity and the dataset/agent it was run against are fixed for the life of an +# Experiment (see the ingest invariants); changing them means it's a different +# Experiment. PUT may only edit group membership, source link, summary, description, metadata. +_IMMUTABLE_EXPERIMENT_FIELDS = ("name", "agent_name", "agent_version", "dataset_name", "dataset_version") + + +@router.put( + "/v2/workspaces/{workspace}/experiments/{name}", + response_model=ExperimentResponse, + tags=[EXPERIMENTS_TAG], + responses={ + 404: {"description": "Experiment not found"}, + 409: {"description": "Attempt to change an immutable field"}, + }, +) +async def update_experiment( + workspace: str, + name: str, + body: ExperimentRequest, + entity_client: EntityClientDep, +) -> ExperimentResponse: + try: + existing = await entity_client.get(Experiment, name=name, workspace=workspace) + except EntityNotFoundError as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Experiment '{workspace}/{name}' not found.", + ) from e + + changed = [f for f in _IMMUTABLE_EXPERIMENT_FIELDS if getattr(body, f) != getattr(existing, f)] + if changed: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Cannot change immutable field(s) {changed} on an existing experiment; " + "create a new experiment instead." + ), + ) + + existing.experiment_group_id = body.experiment_group_id + existing.source_link = body.source_link + existing.metadata = body.metadata + existing.description = body.description + existing.summary = body.summary + updated = await entity_client.update(existing) + return ExperimentResponse.from_entity(updated) + + +@router.delete( + "/v2/workspaces/{workspace}/experiments/{name}", + status_code=status.HTTP_204_NO_CONTENT, + tags=[EXPERIMENTS_TAG], + responses={404: {"description": "Experiment not found"}}, +) +async def delete_experiment( + workspace: str, + name: str, + entity_client: EntityClientDep, +) -> None: + try: + await entity_client.delete(Experiment, name=name, workspace=workspace) + except EntityNotFoundError as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Experiment '{workspace}/{name}' not found.", + ) from e diff --git a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py new file mode 100644 index 0000000000..d342594ceb --- /dev/null +++ b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Request, response, and filter schemas for the Experiments API. + +Response models are standalone (not entity subclasses): they translate from the +stored entity via ``from_entity`` and carry rollup fields that are hydrated from +ClickHouse at read time. In this PR the rollups are always defaults; the +hydration path lands in a later PR. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from nmp.common.entities.values import Filter +from nmp.intake.entities.experiments import Experiment, ExperimentGroup +from pydantic import AnyUrl, BaseModel, ConfigDict, Field + +# ============================================================================= +# Requests (workspace comes from the route parameter) +# ============================================================================= + + +class ExperimentGroupRequest(BaseModel): + """Request body for creating an ExperimentGroup.""" + + model_config = ConfigDict(extra="forbid") + + name: str = Field(description="Workspace-unique group name.") + description: str | None = Field(default=None, description="Human-readable purpose of the group.") + + +class ExperimentRequest(BaseModel): + """Request body for creating an Experiment.""" + + model_config = ConfigDict(extra="forbid") + + name: str = Field(description="Producer-supplied, workspace-unique experiment id.") + experiment_group_id: str | None = Field( + default=None, + description="Entity id of the owning ExperimentGroup; optional. Soft reference, not validated.", + ) + agent_name: str = Field(description="Name of the agent under test.") + agent_version: str = Field(description="Version of the agent under test.") + dataset_name: str = Field(description="Producer-supplied dataset name.") + dataset_version: str | None = Field(default=None, description="Producer-supplied dataset version.") + source_link: AnyUrl | None = Field(default=None, description="Optional URL for the source experiment.") + metadata: dict[str, Any] = Field(default_factory=dict, description="Free-form producer metadata.") + description: str | None = Field(default=None, description="Human-readable description.") + summary: str | None = Field(default=None, description="Human-authored summary of results.") + + +# ============================================================================= +# Responses +# ============================================================================= + + +class ExperimentGroupResponse(BaseModel): + """ExperimentGroup as served by the API.""" + + id: str + name: str + workspace: str + description: str | None = None + created_at: datetime | None = None + updated_at: datetime | None = None + + @classmethod + def from_entity(cls, entity: ExperimentGroup) -> ExperimentGroupResponse: + return cls( + id=entity.id, + name=entity.name, + workspace=entity.workspace, + description=entity.description, + created_at=entity.created_at, + updated_at=entity.updated_at, + ) + + +class EvaluatorAggregate(BaseModel): + """Cross-run statistics for one evaluator. Populated by the rollup path (later PR).""" + + sum: float | None = None + mean: float | None = None + median: float | None = None + p90: float | None = None + p95: float | None = None + p99: float | None = None + + +class ExperimentResponse(BaseModel): + """Experiment as served by the API, including ClickHouse-hydrated rollups.""" + + id: str + name: str + workspace: str + experiment_group_id: str | None = Field( + default=None, + description="Entity id of the owning ExperimentGroup; null when ungrouped. Soft reference, not validated.", + ) + agent_name: str + agent_version: str + dataset_name: str + dataset_version: str | None = None + source_link: AnyUrl | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + description: str | None = None + summary: str | None = None + created_at: datetime | None = None + updated_at: datetime | None = None + + # Hydrated from ClickHouse at read time in a later PR; defaults until then. + evaluator_names: list[str] = Field(default_factory=list) + model_names: list[str] = Field(default_factory=list) + aggregate_scores: dict[str, EvaluatorAggregate] | None = None + run_count: int = 0 + + @classmethod + def from_entity(cls, entity: Experiment) -> ExperimentResponse: + return cls( + id=entity.id, + name=entity.name, + workspace=entity.workspace, + experiment_group_id=entity.experiment_group_id, + agent_name=entity.agent_name, + agent_version=entity.agent_version, + dataset_name=entity.dataset_name, + dataset_version=entity.dataset_version, + source_link=entity.source_link, + metadata=entity.metadata, + description=entity.description, + summary=entity.summary, + created_at=entity.created_at, + updated_at=entity.updated_at, + ) + + +# ============================================================================= +# List filters (declarative; the entity store applies them) +# ============================================================================= + + +class ExperimentGroupFilter(Filter): + """Filter for listing ExperimentGroups.""" + + name: str | None = Field(default=None, description="Filter groups by name.") + + +class ExperimentFilter(Filter): + """Filter for listing Experiments.""" + + name: str | None = Field(default=None, description="Filter experiments by name.") + experiment_group_id: str | None = Field(default=None, description="Filter experiments by owning group id.") + agent_name: str | None = Field(default=None, description="Filter experiments by agent name.") + dataset_name: str | None = Field(default=None, description="Filter experiments by dataset name.") diff --git a/services/intake/src/nmp/intake/entities/experiments.py b/services/intake/src/nmp/intake/entities/experiments.py new file mode 100644 index 0000000000..3725030c4d --- /dev/null +++ b/services/intake/src/nmp/intake/entities/experiments.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Experiment and ExperimentGroup entity definitions for the Intake service. + +These are entity-store (Postgres) entities, distinct from the ClickHouse-backed +telemetry (spans, evaluator_results). They hold the durable, producer-supplied +metadata that organizes telemetry into leaderboard-shaped views. + +Cross-run rollups (per-evaluator aggregate scores, run count, and the unions of +evaluator/model names) are intentionally *not* stored here. They are derived from +ClickHouse and hydrated onto the read model at query time; see +``nmp.intake.api.v2.experiments.schemas.ExperimentResponse``. +""" + +from __future__ import annotations + +from typing import Any, ClassVar + +from nmp.common.entities.client import EntityBase +from pydantic import AnyUrl, Field + + +class ExperimentGroup(EntityBase): + """A named container of Experiments pursuing a single optimization goal. + + A group does not constrain dataset or agent identity across its Experiments. + """ + + __entity_type__: ClassVar[str] = "experiment_group" + + description: str | None = Field(default=None, description="Human-readable purpose of the group.") + + +class Experiment(EntityBase): + """A single agent/config run against a dataset: one row on a leaderboard. + + ``name`` is the producer-supplied, workspace-unique experiment id (e.g. + ``"terminal-bench-2_claude-code_opus_baseline"``); create is keyed on it. + """ + + __entity_type__: ClassVar[str] = "experiment" + + experiment_group_id: str | None = Field( + default=None, + description=( + "Entity id of the owning ExperimentGroup; null when ungrouped. A soft reference: " + "it is not validated on write, and deleting a group does not cascade to its Experiments." + ), + ) + + agent_name: str = Field(description="Name of the agent under test.") + agent_version: str = Field(description="Version of the agent under test.") + + dataset_name: str = Field(description="Producer-supplied dataset name.") + dataset_version: str | None = Field(default=None, description="Producer-supplied dataset version.") + source_link: AnyUrl | None = Field(default=None, description="Optional URL for the source experiment.") + + metadata: dict[str, Any] = Field( + default_factory=dict, + description="Free-form producer metadata (config snapshot, domain-specific attributes, etc.).", + ) + + description: str | None = Field(default=None, description="Human-readable description of the experiment.") + summary: str | None = Field(default=None, description="Human-authored summary of results.") diff --git a/services/intake/src/nmp/intake/service.py b/services/intake/src/nmp/intake/service.py index 7c487fa5a5..26b5ee4c67 100644 --- a/services/intake/src/nmp/intake/service.py +++ b/services/intake/src/nmp/intake/service.py @@ -7,6 +7,7 @@ from typing import ClassVar, List from nmp.common.service import RouterConfig, Service +from nmp.intake.api.v2.experiments import endpoints as experiments from nmp.intake.config import IntakeConfig from nmp.intake.spans.api import annotations, evaluator_results, spans, traces from nmp.intake.spans.clickhouse_client import ClickHouseSettings, ClickHouseSpanClient @@ -57,6 +58,11 @@ def get_routers(self) -> List[RouterConfig]: tag="Ingest", description="OpenAI-compatible chat-completion ingest endpoint", ), + RouterConfig( + experiments.router, + tag="Experiments", + description="Create, list, get, and delete Experiments and Experiment Groups", + ), ] async def on_startup(self) -> None: diff --git a/services/intake/tests/integration/test_experiments_crud.py b/services/intake/tests/integration/test_experiments_crud.py new file mode 100644 index 0000000000..494a78ebcb --- /dev/null +++ b/services/intake/tests/integration/test_experiments_crud.py @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CRUD tests for the Experiments and ExperimentGroups endpoints.""" + +from __future__ import annotations + +from typing import Any + +from fastapi.testclient import TestClient + +GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" +EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" + + +def _experiment_body(**overrides: Any) -> dict: + body = { + "name": "terminal-bench-2_claude-code_opus_baseline", + "agent_name": "claude-code", + "agent_version": "0.125.0", + "dataset_name": "terminal-bench-2", + "dataset_version": "v1", + "source_link": "https://example.com/experiments/tb2-baseline", + "metadata": {"job_name": "tb2-baseline"}, + } + body.update(overrides) + return body + + +def test_experiment_group_crud(client: TestClient) -> None: + created = client.post(GROUPS, json={"name": "tb2-routing-research", "description": "routing sweep"}) + assert created.status_code == 201, created.text + group = created.json() + assert group["name"] == "tb2-routing-research" + assert group["description"] == "routing sweep" + assert group["id"] + + # Duplicate name conflicts. + duplicate = client.post(GROUPS, json={"name": "tb2-routing-research"}) + assert duplicate.status_code == 409 + + fetched = client.get(f"{GROUPS}/tb2-routing-research") + assert fetched.status_code == 200 + assert fetched.json()["id"] == group["id"] + + listed = client.get(GROUPS) + assert listed.status_code == 200 + assert any(g["name"] == "tb2-routing-research" for g in listed.json()["data"]) + + deleted = client.delete(f"{GROUPS}/tb2-routing-research") + assert deleted.status_code == 204 + missing = client.get(f"{GROUPS}/tb2-routing-research") + assert missing.status_code == 404 + + +def test_experiment_group_update_description(client: TestClient) -> None: + client.post(GROUPS, json={"name": "grp", "description": "old"}) + updated = client.put(f"{GROUPS}/grp", json={"name": "grp", "description": "new"}) + assert updated.status_code == 200, updated.text + assert updated.json()["description"] == "new" + # Renaming via PUT is rejected. + renamed = client.put(f"{GROUPS}/grp", json={"name": "renamed"}) + assert renamed.status_code == 409 + missing = client.put(f"{GROUPS}/missing", json={"name": "missing"}) + assert missing.status_code == 404 + + +def test_experiment_update_adds_to_group_and_edits(client: TestClient) -> None: + group = client.post(GROUPS, json={"name": "grp"}).json() + client.post(EXPERIMENTS, json=_experiment_body(name="exp-a")) + + # Add the existing (ungrouped) experiment to the group and edit its summary. + body = _experiment_body(name="exp-a", experiment_group_id=group["id"], summary="looks good") + updated = client.put(f"{EXPERIMENTS}/exp-a", json=body) + assert updated.status_code == 200, updated.text + assert updated.json()["experiment_group_id"] == group["id"] + assert updated.json()["summary"] == "looks good" + + +def test_experiment_update_rejects_immutable_change(client: TestClient) -> None: + client.post(EXPERIMENTS, json=_experiment_body(name="exp-a", agent_name="claude-code")) + changed = _experiment_body(name="exp-a", agent_name="cursor") + resp = client.put(f"{EXPERIMENTS}/exp-a", json=changed) + assert resp.status_code == 409, resp.text + assert "agent_name" in resp.json()["detail"] + missing = client.put(f"{EXPERIMENTS}/missing", json=_experiment_body(name="missing")) + assert missing.status_code == 404 + + +def test_experiment_crud_and_empty_rollups(client: TestClient) -> None: + group = client.post(GROUPS, json={"name": "grp"}).json() + + created = client.post(EXPERIMENTS, json=_experiment_body(experiment_group_id=group["id"])) + assert created.status_code == 201, created.text + exp = created.json() + assert exp["name"] == "terminal-bench-2_claude-code_opus_baseline" + assert exp["experiment_group_id"] == group["id"] + assert exp["agent_name"] == "claude-code" + assert exp["dataset_name"] == "terminal-bench-2" + assert exp["source_link"] == "https://example.com/experiments/tb2-baseline" + assert exp["metadata"] == {"job_name": "tb2-baseline"} + + # Rollups exist on the read model but are empty until ClickHouse hydration lands. + assert exp["evaluator_names"] == [] + assert exp["model_names"] == [] + assert exp["aggregate_scores"] is None + assert exp["run_count"] == 0 + + +def test_experiment_group_ref_is_soft(client: TestClient) -> None: + # experiment_group_id is a soft reference: a non-existent group id is accepted. + created = client.post(EXPERIMENTS, json=_experiment_body(experiment_group_id="grp-does-not-exist")) + assert created.status_code == 201, created.text + assert created.json()["experiment_group_id"] == "grp-does-not-exist" + + +def test_experiment_conflict_and_not_found(client: TestClient) -> None: + created = client.post(EXPERIMENTS, json=_experiment_body()) + assert created.status_code == 201 + duplicate = client.post(EXPERIMENTS, json=_experiment_body()) + assert duplicate.status_code == 409 + missing = client.get(f"{EXPERIMENTS}/does-not-exist") + assert missing.status_code == 404 + missing_delete = client.delete(f"{EXPERIMENTS}/does-not-exist") + assert missing_delete.status_code == 404 + + +def test_experiment_list_and_scope_to_group(client: TestClient) -> None: + group = client.post(GROUPS, json={"name": "grp"}).json() + client.post(EXPERIMENTS, json=_experiment_body(name="exp-a", experiment_group_id=group["id"])) + client.post(EXPERIMENTS, json=_experiment_body(name="exp-b")) + + all_resp = client.get(EXPERIMENTS) + assert all_resp.status_code == 200 + names = {e["name"] for e in all_resp.json()["data"]} + assert {"exp-a", "exp-b"} <= names + + in_group = client.get(EXPERIMENTS, params={"filter[experiment_group_id]": group["id"]}) + assert in_group.status_code == 200 + assert {e["name"] for e in in_group.json()["data"]} == {"exp-a"} + + deleted = client.delete(f"{EXPERIMENTS}/exp-a") + assert deleted.status_code == 204 + missing = client.get(f"{EXPERIMENTS}/exp-a") + assert missing.status_code == 404 diff --git a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/cli_generator/cli_config.yaml b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/cli_generator/cli_config.yaml index 8cdcec73d5..2a7086c5f0 100644 --- a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/cli_generator/cli_config.yaml +++ b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/cli_generator/cli_config.yaml @@ -303,6 +303,14 @@ config: live_global: skip: true +# Experiments are in the OpenAPI spec and SDK (for the frontend) but not yet exposed +# as CLI commands; drop the skip once the feature is production-ready. +- resource: [experiments] + skip: true + +- resource: [experiment_groups] + skip: true + - resource: [secrets] methods: create: