diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index bc0343b03d..fd1e06dbb6 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -3327,12 +3327,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/evaluator-results: + /apis/intake/v2/workspaces/{workspace}/evaluations: post: tags: - - Evaluator Results - summary: Create Evaluator Result - operationId: create_evaluator_result_apis_intake_v2_workspaces__workspace__evaluator_results_post + - Evaluations + summary: Create Evaluation + operationId: create_evaluation_apis_intake_v2_workspaces__workspace__evaluations_post parameters: - name: workspace in: path @@ -3345,14 +3345,16 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/EvaluatorResultInput' + $ref: '#/components/schemas/EvaluationRequest' responses: '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/EvaluatorResult' + $ref: '#/components/schemas/EvaluationResponse' + '409': + description: Evaluation already exists '422': description: Validation Error content: @@ -3361,9 +3363,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' get: tags: - - Evaluator Results - summary: List Evaluator Results - operationId: list_evaluator_results_apis_intake_v2_workspaces__workspace__evaluator_results_get + - Evaluations + summary: List Evaluations + operationId: list_evaluations_apis_intake_v2_workspaces__workspace__evaluations_get parameters: - name: workspace in: path @@ -3389,44 +3391,65 @@ paths: maximum: 1000 minimum: 1 description: Page size. - default: 10 + default: 100 title: Page Size description: Page size. - name: sort in: query required: false schema: - allOf: - - $ref: '#/components/schemas/EvaluatorResultSortField' - default: -created_at + description: 'Field to sort by; prefix with ''-'' for descending. Sort by + an evaluation attribute (name, created_at, updated_at, pinned_at) or by + an aggregate metric: run_count, cost_usd., latency_ms., or + evaluators.., where is one of mean, median, p90, p95, + p99, sum, count. When omitted, defaults to -created_at with pinned evaluations + first.' + title: Sort + type: string + description: 'Field to sort by; prefix with ''-'' for descending. Sort by + an evaluation attribute (name, created_at, updated_at, pinned_at) or by + an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., + where is one of mean, median, p90, p95, p99, sum, count. When omitted, + defaults to -created_at with pinned evaluations first.' - in: query name: filter style: deepObject required: false explode: true schema: - $ref: '#/components/schemas/EvaluatorResultFilter' - description: Filter evaluator results by span_id, session_id, name, data_type, - created_by, value range, and created_at range. + $ref: '#/components/schemas/EvaluationFilter' + description: 'Filter evaluations by name, experiment_group_id, dataset_name, + dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true + to return only soft-deleted evaluations; omit to see only live ones. Pass + is_pinned=true (or false) to filter by pinned state; omit to return both. + Filter by a metadata key/value: filter[metadata.]=. Filter by + a rollup metric with numeric range operators ($gte/$lte/$gt/$lt/$eq): filter[run_count][$gte]=5, + filter[cost_usd.mean][$lte]=0.5, filter[latency_ms.p95][$lte]=1000, or filter[evaluators..mean][$gte]=0.8.' responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/EvaluatorResultsPage' + $ref: '#/components/schemas/EvaluationResponsesPage' + '400': + description: Unsupported sort or filter field + '413': + description: Too many evaluations selected to sort in one request + '503': + description: Telemetry store unavailable for a metric-based sort or filter '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/evaluator-results/{evaluator_result_id}: + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}: get: tags: - - Evaluator Results - summary: Get Evaluator Result - operationId: get_evaluator_result_apis_intake_v2_workspaces__workspace__evaluator_results__evaluator_result_id__get + - Evaluations + summary: Get Evaluation + operationId: get_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__get parameters: - name: workspace in: path @@ -3434,31 +3457,32 @@ paths: schema: type: string title: Workspace - - name: evaluator_result_id + - name: name in: path required: true schema: type: string - title: Evaluator Result Id + title: Name responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/EvaluatorResult' + $ref: '#/components/schemas/EvaluationResponse' + '404': + description: Evaluation not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups: - post: + put: tags: - - Experiment Groups - summary: Create Experiment Group - operationId: create_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups_post + - Evaluations + summary: Update Evaluation + operationId: update_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__put parameters: - name: workspace in: path @@ -3466,32 +3490,40 @@ paths: schema: type: string title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name requestBody: required: true content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupRequest' + $ref: '#/components/schemas/EvaluationRequest' responses: - '201': + '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/EvaluationResponse' + '404': + description: Evaluation not found '409': - description: Experiment group already exists + description: Attempt to change an immutable field '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - get: + delete: tags: - - Experiment Groups - summary: List Experiment Groups - operationId: list_experiment_groups_apis_intake_v2_workspaces__workspace__experiment_groups_get + - Evaluations + summary: Delete Evaluation + operationId: delete_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__delete parameters: - name: workspace in: path @@ -3499,71 +3531,36 @@ paths: schema: type: string title: Workspace - - name: page - in: query - required: false - schema: - type: integer - minimum: 1 - description: Page number. - default: 1 - title: Page - description: Page number. - - name: page_size - in: query - required: false - schema: - type: integer - maximum: 1000 - minimum: 1 - description: Page size. - default: 100 - title: Page Size - description: Page size. - - name: sort - in: query - required: false + - name: name + in: path + required: true schema: - enum: - - -created_at - - created_at - - -updated_at - - updated_at - - -name - - name type: string - description: Sort field; prefix with '-' for descending. - default: -created_at - title: Sort - description: Sort field; prefix with '-' for descending. - - in: query - name: filter - style: deepObject - required: false - explode: true - schema: - $ref: '#/components/schemas/ExperimentGroupFilter' - description: 'Filter experiment groups by name, or by a metadata key/value: - filter[metadata.]=.' + title: Name responses: - '200': + '204': description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/ExperimentGroupResponsesPage' + '404': + description: Evaluation not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: - get: + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin: + post: tags: - - Experiment Groups - summary: Get Experiment Group - operationId: get_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__get + - Evaluations + summary: Pin Evaluation + description: 'Pin an evaluation to the top of the list (workspace-shared). + + + Re-pinning an already-pinned evaluation refreshes ``pinned_at`` to the current + timestamp, + + which is intentional (most-recently-pinned sorts first).' + operationId: pin_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__pin_post parameters: - name: workspace in: path @@ -3583,20 +3580,22 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/EvaluationResponse' '404': - description: Experiment group not found + description: Evaluation not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - put: + delete: tags: - - Experiment Groups - summary: Update Experiment Group - operationId: update_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__put + - Evaluations + summary: Unpin Evaluation + description: 'Unpin an evaluation. Idempotent: unpinning an already-unpinned + evaluation is a no-op.' + operationId: unpin_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__pin_delete parameters: - name: workspace in: path @@ -3610,34 +3609,27 @@ paths: schema: type: string title: Name - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/ExperimentGroupRequest' responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/EvaluationResponse' '404': - description: Experiment group not found - '409': - description: Attempt to rename the group + description: Evaluation not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - delete: + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/sessions: + get: tags: - - Experiment Groups - summary: Delete Experiment Group - operationId: delete_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__delete + - Evaluations + summary: List Evaluation Sessions + operationId: list_evaluation_sessions_apis_intake_v2_workspaces__workspace__evaluations__name__sessions_get parameters: - name: workspace in: path @@ -3651,23 +3643,76 @@ paths: schema: type: string title: Name + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: mode + in: query + required: false + schema: + enum: + - summary + - detailed + type: string + description: Response payload mode. summary keeps the same session row fields + but truncates root-span input to 1000 characters; detailed returns the + full root-span input. + default: detailed + title: Mode + description: Response payload mode. summary keeps the same session row fields + but truncates root-span input to 1000 characters; detailed returns the full + root-span input. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/EvaluationSessionFilter' + description: Filter sessions by test_case_id and status. responses: - '204': + '200': description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationSessionResponsesPage' + '400': + description: Invalid filter value '404': - description: Experiment group not found + description: Evaluation not found + '503': + description: ClickHouse unavailable '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiments: + /apis/intake/v2/workspaces/{workspace}/evaluator-results: post: tags: - - Experiments - summary: Create Experiment - operationId: create_experiment_apis_intake_v2_workspaces__workspace__experiments_post + - Evaluator Results + summary: Create Evaluator Result + operationId: create_evaluator_result_apis_intake_v2_workspaces__workspace__evaluator_results_post parameters: - name: workspace in: path @@ -3680,16 +3725,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentRequest' + $ref: '#/components/schemas/EvaluatorResultInput' responses: '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' - '409': - description: Experiment already exists + $ref: '#/components/schemas/EvaluatorResult' '422': description: Validation Error content: @@ -3698,9 +3741,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' get: tags: - - Experiments - summary: List Experiments - operationId: list_experiments_apis_intake_v2_workspaces__workspace__experiments_get + - Evaluator Results + summary: List Evaluator Results + operationId: list_evaluator_results_apis_intake_v2_workspaces__workspace__evaluator_results_get parameters: - name: workspace in: path @@ -3726,65 +3769,44 @@ paths: maximum: 1000 minimum: 1 description: Page size. - default: 100 + default: 10 title: Page Size description: Page size. - name: sort in: query required: false schema: - description: 'Field to sort by; prefix with ''-'' for descending. Sort by - an experiment attribute (name, created_at, updated_at, pinned_at) or by - an aggregate metric: run_count, cost_usd., latency_ms., or - evaluators.., where is one of mean, median, p90, p95, - p99, sum, count. When omitted, defaults to -created_at with pinned experiments - first.' - title: Sort - type: string - description: 'Field to sort by; prefix with ''-'' for descending. Sort by - an experiment attribute (name, created_at, updated_at, pinned_at) or by - an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., - where is one of mean, median, p90, p95, p99, sum, count. When omitted, - defaults to -created_at with pinned experiments first.' + allOf: + - $ref: '#/components/schemas/EvaluatorResultSortField' + default: -created_at - in: query name: filter style: deepObject required: false explode: true schema: - $ref: '#/components/schemas/ExperimentFilter' - description: 'Filter experiments by name, experiment_group_id, dataset_name, - dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true - to return only soft-deleted experiments; omit to see only live ones. Pass - is_pinned=true (or false) to filter by pinned state; omit to return both. - Filter by a metadata key/value: filter[metadata.]=. Filter by - a rollup metric with numeric range operators ($gte/$lte/$gt/$lt/$eq): filter[run_count][$gte]=5, - filter[cost_usd.mean][$lte]=0.5, filter[latency_ms.p95][$lte]=1000, or filter[evaluators..mean][$gte]=0.8.' + $ref: '#/components/schemas/EvaluatorResultFilter' + description: Filter evaluator results by span_id, session_id, name, data_type, + created_by, value range, and created_at range. responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponsesPage' - '400': - description: Unsupported sort or filter field - '413': - description: Too many experiments selected to sort in one request - '503': - description: Telemetry store unavailable for a metric-based sort or filter + $ref: '#/components/schemas/EvaluatorResultsPage' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiments/{name}: + /apis/intake/v2/workspaces/{workspace}/evaluator-results/{evaluator_result_id}: get: tags: - - Experiments - summary: Get Experiment - operationId: get_experiment_apis_intake_v2_workspaces__workspace__experiments__name__get + - Evaluator Results + summary: Get Evaluator Result + operationId: get_evaluator_result_apis_intake_v2_workspaces__workspace__evaluator_results__evaluator_result_id__get parameters: - name: workspace in: path @@ -3792,32 +3814,31 @@ paths: schema: type: string title: Workspace - - name: name + - name: evaluator_result_id in: path required: true schema: type: string - title: Name + title: Evaluator Result Id responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' - '404': - description: Experiment not found + $ref: '#/components/schemas/EvaluatorResult' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - put: + /apis/intake/v2/workspaces/{workspace}/experiment-groups: + post: tags: - - Experiments - summary: Update Experiment - operationId: update_experiment_apis_intake_v2_workspaces__workspace__experiments__name__put + - Experiment Groups + summary: Create Experiment Group + operationId: create_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups_post parameters: - name: workspace in: path @@ -3825,40 +3846,32 @@ paths: schema: type: string title: Workspace - - name: name - in: path - required: true - schema: - type: string - title: Name requestBody: required: true content: application/json: schema: - $ref: '#/components/schemas/ExperimentRequest' + $ref: '#/components/schemas/ExperimentGroupRequest' responses: - '200': + '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' - '404': - description: Experiment not found + $ref: '#/components/schemas/ExperimentGroupResponse' '409': - description: Attempt to change an immutable field + description: Experiment group already exists '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - delete: + get: tags: - - Experiments - summary: Delete Experiment - operationId: delete_experiment_apis_intake_v2_workspaces__workspace__experiments__name__delete + - Experiment Groups + summary: List Experiment Groups + operationId: list_experiment_groups_apis_intake_v2_workspaces__workspace__experiment_groups_get parameters: - name: workspace in: path @@ -3866,36 +3879,71 @@ paths: schema: type: string title: Workspace - - name: name - in: path - required: true + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: sort + in: query + required: false schema: + enum: + - -created_at + - created_at + - -updated_at + - updated_at + - -name + - name type: string - title: Name + description: Sort field; prefix with '-' for descending. + default: -created_at + title: Sort + description: Sort field; prefix with '-' for descending. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/ExperimentGroupFilter' + description: 'Filter experiment groups by name, or by a metadata key/value: + filter[metadata.]=.' responses: - '204': + '200': description: Successful Response - '404': - description: Experiment not found + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponsesPage' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin: - post: + /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: + get: tags: - - Experiments - summary: Pin Experiment - description: 'Pin an experiment to the top of the list (workspace-shared). - - - Re-pinning an already-pinned experiment refreshes ``pinned_at`` to the current - timestamp, - - which is intentional (most-recently-pinned sorts first).' - operationId: pin_experiment_apis_intake_v2_workspaces__workspace__experiments__name__pin_post + - Experiment Groups + summary: Get Experiment Group + operationId: get_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__get parameters: - name: workspace in: path @@ -3915,22 +3963,20 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' + $ref: '#/components/schemas/ExperimentGroupResponse' '404': - description: Experiment not found + description: Experiment group not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - delete: + put: tags: - - Experiments - summary: Unpin Experiment - description: 'Unpin an experiment. Idempotent: unpinning an already-unpinned - experiment is a no-op.' - operationId: unpin_experiment_apis_intake_v2_workspaces__workspace__experiments__name__pin_delete + - Experiment Groups + summary: Update Experiment Group + operationId: update_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__put parameters: - name: workspace in: path @@ -3944,27 +3990,34 @@ paths: schema: type: string title: Name + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupRequest' responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' + $ref: '#/components/schemas/ExperimentGroupResponse' '404': - description: Experiment not found + description: Experiment group not found + '409': + description: Attempt to rename the group '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiments/{name}/sessions: - get: + delete: tags: - - Experiments - summary: List Experiment Sessions - operationId: list_experiment_sessions_apis_intake_v2_workspaces__workspace__experiments__name__sessions_get + - Experiment Groups + summary: Delete Experiment Group + operationId: delete_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__delete parameters: - name: workspace in: path @@ -3978,62 +4031,11 @@ paths: schema: type: string title: Name - - name: page - in: query - required: false - schema: - type: integer - minimum: 1 - description: Page number. - default: 1 - title: Page - description: Page number. - - name: page_size - in: query - required: false - schema: - type: integer - maximum: 1000 - minimum: 1 - description: Page size. - default: 100 - title: Page Size - description: Page size. - - name: mode - in: query - required: false - schema: - enum: - - summary - - detailed - type: string - description: Response payload mode. summary keeps the same session row fields - but truncates root-span input to 1000 characters; detailed returns the - full root-span input. - default: detailed - title: Mode - description: Response payload mode. summary keeps the same session row fields - but truncates root-span input to 1000 characters; detailed returns the full - root-span input. - - in: query - name: filter - style: deepObject - required: false - explode: true - schema: - $ref: '#/components/schemas/ExperimentSessionFilter' - description: Filter sessions by test_case_id and status. responses: - '200': + '204': description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/ExperimentSessionResponsesPage' '404': - description: Experiment not found - '503': - description: ClickHouse unavailable + description: Experiment group not found '422': description: Validation Error content: @@ -4426,7 +4428,8 @@ paths: schema: $ref: '#/components/schemas/TraceFilter' description: Filter root-span-backed traces by id, session_id, root status, - root span started_at, experiment_id, and test_case_id. + root span started_at, evaluation_id (or its deprecated alias experiment_id), + and test_case_id. responses: '200': description: Successful Response @@ -10188,7 +10191,7 @@ components: properties: evaluation_id: title: Evaluation Id - description: Name of an existing Experiment entity. + description: Name of an existing Evaluation. type: string test_case_id: title: Test Case Id @@ -10204,299 +10207,48 @@ components: evaluation_run_id, metadata) keeps ingesting without error rather than being rejected.' - EvaluatorAggregate: - properties: - sum: - title: Sum - type: number - mean: - title: Mean - type: number - median: - title: Median - type: number - p90: - title: P90 - type: number - p95: - title: P95 - type: number - p99: - title: P99 - type: number - count: - type: integer - title: Count - default: 0 - type: object - title: EvaluatorAggregate - description: Aggregate statistics over evaluator scores or session-level metric - values. - EvaluatorResult: + EvaluationFilter: + additionalProperties: false + description: Filter for listing Evaluations. properties: - evaluator_result_id: - type: string - title: Evaluator Result Id - span_id: - type: string - title: Span Id - session_id: - type: string - title: Session Id - workspace: - type: string - title: Workspace name: - type: string + description: Filter evaluations by name. title: Name - value: - title: Value - type: number - string_value: - title: String Value type: string - data_type: - $ref: '#/components/schemas/EvaluatorResultDataType' - comment: - title: Comment + experiment_group_id: + description: Filter evaluations by owning group id. + title: Experiment Group Id + type: string + dataset_name: + description: Filter evaluations by dataset name. + title: Dataset Name + type: string + dataset_version: + description: Filter evaluations by dataset version. + title: Dataset Version type: string created_by: - title: Created By - type: string - created_at: - type: string - format: date-time - title: Created At - ingested_at: - type: string - format: date-time - title: Ingested At - type: object - required: - - evaluator_result_id - - span_id - - session_id - - workspace - - name - - data_type - - created_at - - ingested_at - title: EvaluatorResult - description: Response model for evaluator_results read endpoints. - EvaluatorResultDataType: - type: string - enum: - - NUMERIC - - CATEGORICAL - - BOOLEAN - - TEXT - title: EvaluatorResultDataType - EvaluatorResultFilter: - properties: - span_id: - description: Filter by target span id. - title: Span Id - type: string - session_id: - description: Filter by target session id. - title: Session Id - type: string - name: - description: Filter by evaluator/metric name. - title: Name - type: string - data_type: - allOf: - - $ref: '#/components/schemas/EvaluatorResultDataType' - description: Filter by data_type. - created_by: - description: Filter by principal/system that wrote the row. - title: Created By - type: string - value: - allOf: - - $ref: '#/components/schemas/FloatFilter' - description: Filter by numeric value (range supported). - created_at: - allOf: - - $ref: '#/components/schemas/DatetimeFilter' - description: Filter by row creation time (range supported). - title: EvaluatorResultFilter - type: object - EvaluatorResultInput: - properties: - span_id: - type: string - title: Span Id - description: Target span id. Not validated against existing spans (loose - target policy). - session_id: - type: string - title: Session Id - description: Session id the target span belongs to. Denormalized so session-scoped - reads stay fast. - name: - type: string - title: Name - description: Evaluator / metric identity (e.g. 'faithfulness/v1'). - value: - title: Value - description: Numeric value. Required when data_type is NUMERIC or BOOLEAN - (0|1). - type: number - string_value: - title: String Value - description: String value. Required when data_type is CATEGORICAL or TEXT. - type: string - data_type: - allOf: - - $ref: '#/components/schemas/EvaluatorResultDataType' - description: Discriminator for which of value / string_value carries the - payload. - comment: - title: Comment - description: Free-text rationale or explanation. - type: string - additionalProperties: false - type: object - required: - - span_id - - session_id - - name - - data_type - title: EvaluatorResultInput - description: "Request body for POST /evaluator-results.\n\nServer fills in `evaluator_result_id`,\ - \ `created_at`, `ingested_at`, and\n`created_by`. Producer supplies the target\ - \ span (loose target \u2014 not\nvalidated against the spans table), the score,\ - \ and provenance." - EvaluatorResultSortField: - type: string - enum: - - created_at - - -created_at - - value - - -value - title: EvaluatorResultSortField - EvaluatorResultsPage: - properties: - data: - items: - $ref: '#/components/schemas/EvaluatorResult' - type: array - title: Data - pagination: - allOf: - - $ref: '#/components/schemas/PaginationData' - description: Pagination information. - sort: - title: Sort - description: The field on which the results are sorted. - type: string - filter: - title: Filter - description: Filtering information. - additionalProperties: true - type: object - type: object - required: - - data - title: EvaluatorResultsPage - ExecutedAction: - properties: - action_name: - type: string - title: Action Name - description: The name of the action that was executed. - action_params: - additionalProperties: true - type: object - title: Action Params - description: The parameters for the action. - return_value: - title: Return Value - description: The value returned by the action. - llm_calls: - items: - $ref: '#/components/schemas/LLMCallInfo' - type: array - title: Llm Calls - description: Information about the LLM calls made by the action. - started_at: - title: Started At - description: Timestamp for when the action started. - type: number - finished_at: - title: Finished At - description: Timestamp for when the action finished. - type: number - duration: - title: Duration - description: How long the action took to execute, in seconds. - type: number - type: object - required: - - action_name - title: ExecutedAction - description: Information about an action that was executed. - ExperimentContext: - properties: - experiment_id: - type: string - title: Experiment Id - description: Name of an existing Experiment entity. - test_case_id: - title: Test Case Id - description: Optional producer-supplied test case id. - type: string - additionalProperties: false - type: object - required: - - experiment_id - title: ExperimentContext - description: Deprecated alias for :class:`EvaluationContext`. Producers should - send ``evaluation_context``. - ExperimentFilter: - additionalProperties: false - description: Filter for listing Experiments. - properties: - name: - description: Filter experiments by name. - title: Name - type: string - experiment_group_id: - description: Filter experiments by owning group id. - title: Experiment Group Id - type: string - dataset_name: - description: Filter experiments by dataset name. - title: Dataset Name - type: string - dataset_version: - description: Filter experiments by dataset version. - title: Dataset Version - type: string - created_by: - description: Filter experiments by the principal that created them. + description: Filter evaluations by the principal that created them. title: Created By type: string created_at: allOf: - $ref: '#/components/schemas/DatetimeFilter' - description: Filter experiments by creation timestamp; supports `$gte` and + description: Filter evaluations by creation timestamp; supports `$gte` and `$lte` for ranges. updated_at: allOf: - $ref: '#/components/schemas/DatetimeFilter' - description: Filter experiments by last-updated timestamp; supports `$gte` + description: Filter evaluations by last-updated timestamp; supports `$gte` and `$lte` for ranges. is_deleted: - description: When true, returns only soft-deleted experiments. Omit (or - false) to see only live experiments. + description: When true, returns only soft-deleted evaluations. Omit (or + false) to see only live evaluations. title: Is Deleted type: boolean is_pinned: - description: When true, returns only pinned experiments. When false, returns - only unpinned experiments. Omit to return both. + description: When true, returns only pinned evaluations. When false, returns + only unpinned evaluations. Omit to return both. title: Is Pinned type: boolean metadata: @@ -10523,189 +10275,61 @@ components: additionalProperties: $ref: '#/components/schemas/MetricStatFilters' type: object - title: ExperimentFilter + title: EvaluationFilter type: object - ExperimentGroupFilter: - additionalProperties: false - description: Filter for listing ExperimentGroups. + EvaluationRequest: properties: name: - description: Filter groups by name. + type: string title: Name + description: Producer-supplied, workspace-unique evaluation id. + experiment_group_id: type: string - is_deleted: - description: When true, returns only soft-deleted groups. Omit (or false) - to see only live groups. - title: Is Deleted - type: boolean + title: Experiment Group Id + description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ + \ group must already exist." + dataset_name: + type: string + title: Dataset Name + description: Producer-supplied dataset name. + dataset_version: + title: Dataset Version + description: Producer-supplied dataset version. + type: string + source_link: + title: Source Link + description: Optional URL for the source evaluation. + type: string + minLength: 1 + format: uri metadata: - description: Filter by a metadata key/value pair, e.g. filter[metadata.model]=claude-opus-4-8. - title: Metadata additionalProperties: type: string type: object - title: ExperimentGroupFilter - type: object - ExperimentGroupRequest: - properties: - name: - type: string - title: Name - description: Workspace-unique group name. + title: Metadata + description: Free-form producer metadata. description: title: Description - description: Human-readable purpose of the group. + description: Human-readable description. type: string - insight_id: - title: Insight Id - description: Reference to an external insight that seeded this group, if - any. + parent_evaluation_id: + title: Parent Evaluation Id + description: Entity id of the evaluation this one was derived from (e.g. + a variant of a baseline), if any. type: string - summary: - title: Summary - description: Human- or agent-authored summary of the group's findings. + parent_experiment_id: + title: Parent Experiment Id + description: Deprecated alias for parent_evaluation_id. + deprecated: true type: string - metadata: - title: Metadata - description: Free-form producer metadata for the group. - additionalProperties: - type: string - type: object - default_sort: - type: string - title: Default Sort - description: Default sort for this group's experiments list, as a `sort`-param - string (leading '-' = descending); defaults to '-created_at'. Accepts - any field the experiments list `sort` param does; clients apply it as - the list `sort` param. - default: -created_at - additionalProperties: false - type: object - required: - - name - title: ExperimentGroupRequest - description: Request body for creating an ExperimentGroup. - ExperimentGroupResponse: - properties: - id: - type: string - title: Id - name: - type: string - title: Name - workspace: - type: string - title: Workspace - description: - title: Description - type: string - insight_id: - title: Insight Id - type: string - summary: - title: Summary - type: string - metadata: - title: Metadata - additionalProperties: - type: string - type: object - default_sort: - type: string - title: Default Sort - created_at: - title: Created At - type: string - format: date-time - updated_at: - title: Updated At - type: string - format: date-time - experiment_count: - type: integer - title: Experiment Count - description: Number of live (non-soft-deleted) experiments in this group. - default: 0 - type: object - required: - - id - - name - - workspace - - default_sort - title: ExperimentGroupResponse - description: ExperimentGroup as served by the API. - ExperimentGroupResponsesPage: - properties: - data: - items: - $ref: '#/components/schemas/ExperimentGroupResponse' - type: array - title: Data - pagination: - allOf: - - $ref: '#/components/schemas/PaginationData' - description: Pagination information. - sort: - title: Sort - description: The field on which the results are sorted. - type: string - filter: - title: Filter - description: Filtering information. - additionalProperties: true - type: object - type: object - required: - - data - title: ExperimentGroupResponsesPage - ExperimentRequest: - properties: - name: - type: string - title: Name - description: Producer-supplied, workspace-unique experiment id. - experiment_group_id: - type: string - title: Experiment Group Id - description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ - \ group must already exist." - dataset_name: - type: string - title: Dataset Name - description: Producer-supplied dataset name. - dataset_version: - title: Dataset Version - description: Producer-supplied dataset version. - type: string - source_link: - title: Source Link - description: Optional URL for the source experiment. - type: string - minLength: 1 - format: uri - metadata: - additionalProperties: - type: string - type: object - title: Metadata - description: Free-form producer metadata. - description: - title: Description - description: Human-readable description. - type: string - parent_experiment_id: - title: Parent Experiment Id - description: Entity id of the experiment this one was derived from (e.g. - a variant of a baseline), if any. - type: string - status: - title: Status - description: Producer-defined lifecycle status of the experiment. - type: string - root_cause: - title: Root Cause - description: Human- or agent-authored explanation of the experiment's outcome - (e.g. why it was killed). + status: + title: Status + description: Producer-defined lifecycle status of the evaluation. + type: string + root_cause: + title: Root Cause + description: Human- or agent-authored explanation of the evaluation's outcome + (e.g. why it was killed). type: string additionalProperties: false type: object @@ -10713,9 +10337,9 @@ components: - name - experiment_group_id - dataset_name - title: ExperimentRequest - description: Request body for creating an Experiment. - ExperimentResponse: + title: EvaluationRequest + description: Request body for creating an Evaluation. + EvaluationResponse: properties: id: type: string @@ -10730,7 +10354,7 @@ components: type: string title: Experiment Group Id description: Entity id of the owning ExperimentGroup. Required for every - Experiment. + Evaluation. dataset_name: type: string title: Dataset Name @@ -10750,8 +10374,8 @@ components: description: title: Description type: string - parent_experiment_id: - title: Parent Experiment Id + parent_evaluation_id: + title: Parent Evaluation Id type: string status: title: Status @@ -10769,8 +10393,8 @@ components: format: date-time pinned_at: title: Pinned At - description: Timestamp at which the experiment was pinned, or null if unpinned. - Managed via POST/DELETE /experiments/{name}/pin. + description: Timestamp at which the evaluation was pinned, or null if unpinned. + Managed via POST/DELETE /evaluations/{name}/pin. nullable: true type: string format: date-time @@ -10786,7 +10410,7 @@ components: uniqueItems: true title: Model Names description: Distinct model names observed across ingested sessions for - this experiment. + this evaluation. agent_names: items: type: string @@ -10794,7 +10418,7 @@ components: uniqueItems: true title: Agent Names description: Distinct agent names observed across ingested sessions for - this experiment. + this evaluation. agent_versions: items: type: string @@ -10802,7 +10426,7 @@ components: uniqueItems: true title: Agent Versions description: Distinct agent versions observed across ingested sessions for - this experiment. + this evaluation. aggregate_scores: title: Aggregate Scores additionalProperties: @@ -10811,28 +10435,356 @@ components: run_count: type: integer title: Run Count - description: Number of distinct ingested experiment sessions; one session + description: Number of distinct ingested evaluation sessions; one session is treated as one run. default: 0 cost_usd: $ref: '#/components/schemas/EvaluatorAggregate' latency_ms: $ref: '#/components/schemas/EvaluatorAggregate' + parent_experiment_id: + title: Parent Experiment Id + description: Deprecated alias for parent_evaluation_id. + deprecated: true + readOnly: true + type: string + type: object + required: + - id + - name + - workspace + - experiment_group_id + - dataset_name + - parent_experiment_id + title: EvaluationResponse + description: Evaluation as served by the API, including ClickHouse-hydrated + rollups. + EvaluationResponsesPage: + properties: + data: + items: + $ref: '#/components/schemas/EvaluationResponse' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: EvaluationResponsesPage + EvaluationSessionFilter: + additionalProperties: false + description: Filter for listing EvaluationSessions. + properties: + test_case_id: + description: Filter by producer-supplied test case id. + title: Test Case Id + type: string + status: + description: Filter by root-span status (success, error, cancelled, unknown). + title: Status + type: string + title: EvaluationSessionFilter + type: object + EvaluationSessionResponse: + properties: + workspace: + type: string + title: Workspace + evaluation_name: + type: string + title: Evaluation Name + session_id: + type: string + title: Session Id + test_case_id: + title: Test Case Id + description: Producer-supplied test case identifier; null when the producer + did not set one. + type: string + trace_id: + type: string + title: Trace Id + root_span_id: + type: string + title: Root Span Id + started_at: + type: string + format: date-time + title: Started At + ended_at: + title: Ended At + type: string + format: date-time + latency_ms: + title: Latency Ms + type: number + status: + allOf: + - $ref: '#/components/schemas/SpanStatus' + description: 'Root-span status: success, error, cancelled, or unknown.' + input: + title: Input + description: Root-span input text. In summary mode this is truncated to + 1000 characters. + type: string + input_tokens: + title: Input Tokens + description: Sum of input tokens across this session's spans. + type: integer + output_tokens: + title: Output Tokens + description: Sum of output tokens across this session's spans. + type: integer + cached_tokens: + title: Cached Tokens + description: Sum of cached tokens across this session's spans. + type: integer + cost_total_usd: + title: Cost Total Usd + description: Sum of cost across this session's spans. + type: number + evaluator_scores: + additionalProperties: + type: number + type: object + title: Evaluator Scores + description: Per-evaluator session-mean score. Includes NUMERIC and BOOLEAN + evaluator results only; text/categorical results are omitted. + experiment_name: + type: string + title: Experiment Name + description: Deprecated alias for evaluation_name. + deprecated: true + readOnly: true + type: object + required: + - workspace + - evaluation_name + - session_id + - trace_id + - root_span_id + - started_at + - status + - experiment_name + title: EvaluationSessionResponse + description: "One ingested session of an Evaluation \u2014 a single test case\ + \ execution.\n\nHydrated from ClickHouse at read time by reading root/session\ + \ membership from\n``trace_index`` and joining page-bounded span/evaluator\ + \ rollups." + EvaluationSessionResponsesPage: + properties: + data: + items: + $ref: '#/components/schemas/EvaluationSessionResponse' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: EvaluationSessionResponsesPage + EvaluatorAggregate: + properties: + sum: + title: Sum + type: number + mean: + title: Mean + type: number + median: + title: Median + type: number + p90: + title: P90 + type: number + p95: + title: P95 + type: number + p99: + title: P99 + type: number + count: + type: integer + title: Count + default: 0 + type: object + title: EvaluatorAggregate + description: Aggregate statistics over evaluator scores or session-level metric + values. + EvaluatorResult: + properties: + evaluator_result_id: + type: string + title: Evaluator Result Id + span_id: + type: string + title: Span Id + session_id: + type: string + title: Session Id + workspace: + type: string + title: Workspace + name: + type: string + title: Name + value: + title: Value + type: number + string_value: + title: String Value + type: string + data_type: + $ref: '#/components/schemas/EvaluatorResultDataType' + comment: + title: Comment + type: string + created_by: + title: Created By + type: string + created_at: + type: string + format: date-time + title: Created At + ingested_at: + type: string + format: date-time + title: Ingested At + type: object + required: + - evaluator_result_id + - span_id + - session_id + - workspace + - name + - data_type + - created_at + - ingested_at + title: EvaluatorResult + description: Response model for evaluator_results read endpoints. + EvaluatorResultDataType: + type: string + enum: + - NUMERIC + - CATEGORICAL + - BOOLEAN + - TEXT + title: EvaluatorResultDataType + EvaluatorResultFilter: + properties: + span_id: + description: Filter by target span id. + title: Span Id + type: string + session_id: + description: Filter by target session id. + title: Session Id + type: string + name: + description: Filter by evaluator/metric name. + title: Name + type: string + data_type: + allOf: + - $ref: '#/components/schemas/EvaluatorResultDataType' + description: Filter by data_type. + created_by: + description: Filter by principal/system that wrote the row. + title: Created By + type: string + value: + allOf: + - $ref: '#/components/schemas/FloatFilter' + description: Filter by numeric value (range supported). + created_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Filter by row creation time (range supported). + title: EvaluatorResultFilter + type: object + EvaluatorResultInput: + properties: + span_id: + type: string + title: Span Id + description: Target span id. Not validated against existing spans (loose + target policy). + session_id: + type: string + title: Session Id + description: Session id the target span belongs to. Denormalized so session-scoped + reads stay fast. + name: + type: string + title: Name + description: Evaluator / metric identity (e.g. 'faithfulness/v1'). + value: + title: Value + description: Numeric value. Required when data_type is NUMERIC or BOOLEAN + (0|1). + type: number + string_value: + title: String Value + description: String value. Required when data_type is CATEGORICAL or TEXT. + type: string + data_type: + allOf: + - $ref: '#/components/schemas/EvaluatorResultDataType' + description: Discriminator for which of value / string_value carries the + payload. + comment: + title: Comment + description: Free-text rationale or explanation. + type: string + additionalProperties: false type: object required: - - id + - span_id + - session_id - name - - workspace - - experiment_group_id - - dataset_name - title: ExperimentResponse - description: Experiment as served by the API, including ClickHouse-hydrated - rollups. - ExperimentResponsesPage: + - data_type + title: EvaluatorResultInput + description: "Request body for POST /evaluator-results.\n\nServer fills in `evaluator_result_id`,\ + \ `created_at`, `ingested_at`, and\n`created_by`. Producer supplies the target\ + \ span (loose target \u2014 not\nvalidated against the spans table), the score,\ + \ and provenance." + EvaluatorResultSortField: + type: string + enum: + - created_at + - -created_at + - value + - -value + title: EvaluatorResultSortField + EvaluatorResultsPage: properties: data: items: - $ref: '#/components/schemas/ExperimentResponse' + $ref: '#/components/schemas/EvaluatorResult' type: array title: Data pagination: @@ -10851,105 +10803,182 @@ components: type: object required: - data - title: ExperimentResponsesPage - ExperimentSessionFilter: - additionalProperties: false - description: Filter for listing ExperimentSessions. + title: EvaluatorResultsPage + ExecutedAction: + properties: + action_name: + type: string + title: Action Name + description: The name of the action that was executed. + action_params: + additionalProperties: true + type: object + title: Action Params + description: The parameters for the action. + return_value: + title: Return Value + description: The value returned by the action. + llm_calls: + items: + $ref: '#/components/schemas/LLMCallInfo' + type: array + title: Llm Calls + description: Information about the LLM calls made by the action. + started_at: + title: Started At + description: Timestamp for when the action started. + type: number + finished_at: + title: Finished At + description: Timestamp for when the action finished. + type: number + duration: + title: Duration + description: How long the action took to execute, in seconds. + type: number + type: object + required: + - action_name + title: ExecutedAction + description: Information about an action that was executed. + ExperimentContext: properties: + experiment_id: + type: string + title: Experiment Id + description: Name of an existing Experiment entity. test_case_id: - description: Filter by producer-supplied test case id. title: Test Case Id + description: Optional producer-supplied test case id. type: string - status: - description: Filter by root-span status (success, error, cancelled, unknown). - title: Status + additionalProperties: false + type: object + required: + - experiment_id + title: ExperimentContext + description: Deprecated alias for :class:`EvaluationContext`. Producers should + send ``evaluation_context``. + ExperimentGroupFilter: + additionalProperties: false + description: Filter for listing ExperimentGroups. + properties: + name: + description: Filter groups by name. + title: Name + type: string + is_deleted: + description: When true, returns only soft-deleted groups. Omit (or false) + to see only live groups. + title: Is Deleted + type: boolean + metadata: + description: Filter by a metadata key/value pair, e.g. filter[metadata.model]=claude-opus-4-8. + title: Metadata + additionalProperties: + type: string + type: object + title: ExperimentGroupFilter + type: object + ExperimentGroupRequest: + properties: + name: + type: string + title: Name + description: Workspace-unique group name. + description: + title: Description + description: Human-readable purpose of the group. + type: string + insight_id: + title: Insight Id + description: Reference to an external insight that seeded this group, if + any. + type: string + summary: + title: Summary + description: Human- or agent-authored summary of the group's findings. + type: string + metadata: + title: Metadata + description: Free-form producer metadata for the group. + additionalProperties: + type: string + type: object + default_sort: type: string - title: ExperimentSessionFilter + title: Default Sort + description: Default sort for this group's evaluations list, as a `sort`-param + string (leading '-' = descending); defaults to '-created_at'. Accepts + any field the evaluations list `sort` param does; clients apply it as + the list `sort` param. + default: -created_at + additionalProperties: false type: object - ExperimentSessionResponse: + required: + - name + title: ExperimentGroupRequest + description: Request body for creating an ExperimentGroup. + ExperimentGroupResponse: properties: + id: + type: string + title: Id + name: + type: string + title: Name workspace: type: string title: Workspace - experiment_name: - type: string - title: Experiment Name - session_id: + description: + title: Description type: string - title: Session Id - test_case_id: - title: Test Case Id - description: Producer-supplied test case identifier; null when the producer - did not set one. + insight_id: + title: Insight Id type: string - trace_id: + summary: + title: Summary type: string - title: Trace Id - root_span_id: + metadata: + title: Metadata + additionalProperties: + type: string + type: object + default_sort: type: string - title: Root Span Id - started_at: + title: Default Sort + created_at: + title: Created At type: string format: date-time - title: Started At - ended_at: - title: Ended At + updated_at: + title: Updated At type: string format: date-time - latency_ms: - title: Latency Ms - type: number - status: - allOf: - - $ref: '#/components/schemas/SpanStatus' - description: 'Root-span status: success, error, cancelled, or unknown.' - input: - title: Input - description: Root-span input text. In summary mode this is truncated to - 1000 characters. - type: string - input_tokens: - title: Input Tokens - description: Sum of input tokens across this session's spans. - type: integer - output_tokens: - title: Output Tokens - description: Sum of output tokens across this session's spans. + evaluation_count: type: integer - cached_tokens: - title: Cached Tokens - description: Sum of cached tokens across this session's spans. + title: Evaluation Count + description: Number of live (non-soft-deleted) evaluations in this group. + default: 0 + experiment_count: type: integer - cost_total_usd: - title: Cost Total Usd - description: Sum of cost across this session's spans. - type: number - evaluator_scores: - additionalProperties: - type: number - type: object - title: Evaluator Scores - description: Per-evaluator session-mean score. Includes NUMERIC and BOOLEAN - evaluator results only; text/categorical results are omitted. + title: Experiment Count + description: Deprecated alias for evaluation_count. + deprecated: true + readOnly: true type: object required: + - id + - name - workspace - - experiment_name - - session_id - - trace_id - - root_span_id - - started_at - - status - title: ExperimentSessionResponse - description: "One ingested session of an Experiment \u2014 a single test case\ - \ execution.\n\nHydrated from ClickHouse at read time by reading root/session\ - \ membership from\n``trace_index`` and joining page-bounded span/evaluator\ - \ rollups." - ExperimentSessionResponsesPage: + - default_sort + - experiment_count + title: ExperimentGroupResponse + description: ExperimentGroup as served by the API. + ExperimentGroupResponsesPage: properties: data: items: - $ref: '#/components/schemas/ExperimentSessionResponse' + $ref: '#/components/schemas/ExperimentGroupResponse' type: array title: Data pagination: @@ -10968,7 +10997,7 @@ components: type: object required: - data - title: ExperimentSessionResponsesPage + title: ExperimentGroupResponsesPage FactCheckingRailConfig: properties: parameters: @@ -13248,7 +13277,7 @@ components: These stats must stay in sync with the runtime sort/filter grammar (``_METRIC_STATS`` in the - experiments + evaluations endpoints); a unit test guards the parity.' properties: @@ -17875,8 +17904,14 @@ components: name: title: Name type: string + evaluation_context: + $ref: '#/components/schemas/EvaluationContext' experiment_context: - $ref: '#/components/schemas/ExperimentContext' + allOf: + - $ref: '#/components/schemas/ExperimentContext' + description: Deprecated alias for evaluation_context; will be removed in + a future release. + deprecated: true started_at: type: string format: date-time @@ -17949,12 +17984,18 @@ components: allOf: - $ref: '#/components/schemas/DatetimeFilter' description: Filter by root span start timestamp. + evaluation_id: + description: Filter by root-span evaluation id. + title: Evaluation Id + type: string experiment_id: - description: Filter by root-span experiment id. + deprecated: true + description: Deprecated alias for evaluation_id. Filter by root-span evaluation + id. title: Experiment Id type: string test_case_id: - description: Filter by root-span experiment test case id. + description: Filter by root-span evaluation test case id. title: Test Case Id type: string title: TraceFilter diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index bc0343b03d..fd1e06dbb6 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -3327,12 +3327,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/evaluator-results: + /apis/intake/v2/workspaces/{workspace}/evaluations: post: tags: - - Evaluator Results - summary: Create Evaluator Result - operationId: create_evaluator_result_apis_intake_v2_workspaces__workspace__evaluator_results_post + - Evaluations + summary: Create Evaluation + operationId: create_evaluation_apis_intake_v2_workspaces__workspace__evaluations_post parameters: - name: workspace in: path @@ -3345,14 +3345,16 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/EvaluatorResultInput' + $ref: '#/components/schemas/EvaluationRequest' responses: '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/EvaluatorResult' + $ref: '#/components/schemas/EvaluationResponse' + '409': + description: Evaluation already exists '422': description: Validation Error content: @@ -3361,9 +3363,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' get: tags: - - Evaluator Results - summary: List Evaluator Results - operationId: list_evaluator_results_apis_intake_v2_workspaces__workspace__evaluator_results_get + - Evaluations + summary: List Evaluations + operationId: list_evaluations_apis_intake_v2_workspaces__workspace__evaluations_get parameters: - name: workspace in: path @@ -3389,44 +3391,65 @@ paths: maximum: 1000 minimum: 1 description: Page size. - default: 10 + default: 100 title: Page Size description: Page size. - name: sort in: query required: false schema: - allOf: - - $ref: '#/components/schemas/EvaluatorResultSortField' - default: -created_at + description: 'Field to sort by; prefix with ''-'' for descending. Sort by + an evaluation attribute (name, created_at, updated_at, pinned_at) or by + an aggregate metric: run_count, cost_usd., latency_ms., or + evaluators.., where is one of mean, median, p90, p95, + p99, sum, count. When omitted, defaults to -created_at with pinned evaluations + first.' + title: Sort + type: string + description: 'Field to sort by; prefix with ''-'' for descending. Sort by + an evaluation attribute (name, created_at, updated_at, pinned_at) or by + an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., + where is one of mean, median, p90, p95, p99, sum, count. When omitted, + defaults to -created_at with pinned evaluations first.' - in: query name: filter style: deepObject required: false explode: true schema: - $ref: '#/components/schemas/EvaluatorResultFilter' - description: Filter evaluator results by span_id, session_id, name, data_type, - created_by, value range, and created_at range. + $ref: '#/components/schemas/EvaluationFilter' + description: 'Filter evaluations by name, experiment_group_id, dataset_name, + dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true + to return only soft-deleted evaluations; omit to see only live ones. Pass + is_pinned=true (or false) to filter by pinned state; omit to return both. + Filter by a metadata key/value: filter[metadata.]=. Filter by + a rollup metric with numeric range operators ($gte/$lte/$gt/$lt/$eq): filter[run_count][$gte]=5, + filter[cost_usd.mean][$lte]=0.5, filter[latency_ms.p95][$lte]=1000, or filter[evaluators..mean][$gte]=0.8.' responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/EvaluatorResultsPage' + $ref: '#/components/schemas/EvaluationResponsesPage' + '400': + description: Unsupported sort or filter field + '413': + description: Too many evaluations selected to sort in one request + '503': + description: Telemetry store unavailable for a metric-based sort or filter '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/evaluator-results/{evaluator_result_id}: + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}: get: tags: - - Evaluator Results - summary: Get Evaluator Result - operationId: get_evaluator_result_apis_intake_v2_workspaces__workspace__evaluator_results__evaluator_result_id__get + - Evaluations + summary: Get Evaluation + operationId: get_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__get parameters: - name: workspace in: path @@ -3434,31 +3457,32 @@ paths: schema: type: string title: Workspace - - name: evaluator_result_id + - name: name in: path required: true schema: type: string - title: Evaluator Result Id + title: Name responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/EvaluatorResult' + $ref: '#/components/schemas/EvaluationResponse' + '404': + description: Evaluation not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups: - post: + put: tags: - - Experiment Groups - summary: Create Experiment Group - operationId: create_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups_post + - Evaluations + summary: Update Evaluation + operationId: update_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__put parameters: - name: workspace in: path @@ -3466,32 +3490,40 @@ paths: schema: type: string title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name requestBody: required: true content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupRequest' + $ref: '#/components/schemas/EvaluationRequest' responses: - '201': + '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/EvaluationResponse' + '404': + description: Evaluation not found '409': - description: Experiment group already exists + description: Attempt to change an immutable field '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - get: + delete: tags: - - Experiment Groups - summary: List Experiment Groups - operationId: list_experiment_groups_apis_intake_v2_workspaces__workspace__experiment_groups_get + - Evaluations + summary: Delete Evaluation + operationId: delete_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__delete parameters: - name: workspace in: path @@ -3499,71 +3531,36 @@ paths: schema: type: string title: Workspace - - name: page - in: query - required: false - schema: - type: integer - minimum: 1 - description: Page number. - default: 1 - title: Page - description: Page number. - - name: page_size - in: query - required: false - schema: - type: integer - maximum: 1000 - minimum: 1 - description: Page size. - default: 100 - title: Page Size - description: Page size. - - name: sort - in: query - required: false + - name: name + in: path + required: true schema: - enum: - - -created_at - - created_at - - -updated_at - - updated_at - - -name - - name type: string - description: Sort field; prefix with '-' for descending. - default: -created_at - title: Sort - description: Sort field; prefix with '-' for descending. - - in: query - name: filter - style: deepObject - required: false - explode: true - schema: - $ref: '#/components/schemas/ExperimentGroupFilter' - description: 'Filter experiment groups by name, or by a metadata key/value: - filter[metadata.]=.' + title: Name responses: - '200': + '204': description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/ExperimentGroupResponsesPage' + '404': + description: Evaluation not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: - get: + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin: + post: tags: - - Experiment Groups - summary: Get Experiment Group - operationId: get_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__get + - Evaluations + summary: Pin Evaluation + description: 'Pin an evaluation to the top of the list (workspace-shared). + + + Re-pinning an already-pinned evaluation refreshes ``pinned_at`` to the current + timestamp, + + which is intentional (most-recently-pinned sorts first).' + operationId: pin_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__pin_post parameters: - name: workspace in: path @@ -3583,20 +3580,22 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/EvaluationResponse' '404': - description: Experiment group not found + description: Evaluation not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - put: + delete: tags: - - Experiment Groups - summary: Update Experiment Group - operationId: update_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__put + - Evaluations + summary: Unpin Evaluation + description: 'Unpin an evaluation. Idempotent: unpinning an already-unpinned + evaluation is a no-op.' + operationId: unpin_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__pin_delete parameters: - name: workspace in: path @@ -3610,34 +3609,27 @@ paths: schema: type: string title: Name - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/ExperimentGroupRequest' responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/EvaluationResponse' '404': - description: Experiment group not found - '409': - description: Attempt to rename the group + description: Evaluation not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - delete: + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/sessions: + get: tags: - - Experiment Groups - summary: Delete Experiment Group - operationId: delete_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__delete + - Evaluations + summary: List Evaluation Sessions + operationId: list_evaluation_sessions_apis_intake_v2_workspaces__workspace__evaluations__name__sessions_get parameters: - name: workspace in: path @@ -3651,23 +3643,76 @@ paths: schema: type: string title: Name + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: mode + in: query + required: false + schema: + enum: + - summary + - detailed + type: string + description: Response payload mode. summary keeps the same session row fields + but truncates root-span input to 1000 characters; detailed returns the + full root-span input. + default: detailed + title: Mode + description: Response payload mode. summary keeps the same session row fields + but truncates root-span input to 1000 characters; detailed returns the full + root-span input. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/EvaluationSessionFilter' + description: Filter sessions by test_case_id and status. responses: - '204': + '200': description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationSessionResponsesPage' + '400': + description: Invalid filter value '404': - description: Experiment group not found + description: Evaluation not found + '503': + description: ClickHouse unavailable '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiments: + /apis/intake/v2/workspaces/{workspace}/evaluator-results: post: tags: - - Experiments - summary: Create Experiment - operationId: create_experiment_apis_intake_v2_workspaces__workspace__experiments_post + - Evaluator Results + summary: Create Evaluator Result + operationId: create_evaluator_result_apis_intake_v2_workspaces__workspace__evaluator_results_post parameters: - name: workspace in: path @@ -3680,16 +3725,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentRequest' + $ref: '#/components/schemas/EvaluatorResultInput' responses: '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' - '409': - description: Experiment already exists + $ref: '#/components/schemas/EvaluatorResult' '422': description: Validation Error content: @@ -3698,9 +3741,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' get: tags: - - Experiments - summary: List Experiments - operationId: list_experiments_apis_intake_v2_workspaces__workspace__experiments_get + - Evaluator Results + summary: List Evaluator Results + operationId: list_evaluator_results_apis_intake_v2_workspaces__workspace__evaluator_results_get parameters: - name: workspace in: path @@ -3726,65 +3769,44 @@ paths: maximum: 1000 minimum: 1 description: Page size. - default: 100 + default: 10 title: Page Size description: Page size. - name: sort in: query required: false schema: - description: 'Field to sort by; prefix with ''-'' for descending. Sort by - an experiment attribute (name, created_at, updated_at, pinned_at) or by - an aggregate metric: run_count, cost_usd., latency_ms., or - evaluators.., where is one of mean, median, p90, p95, - p99, sum, count. When omitted, defaults to -created_at with pinned experiments - first.' - title: Sort - type: string - description: 'Field to sort by; prefix with ''-'' for descending. Sort by - an experiment attribute (name, created_at, updated_at, pinned_at) or by - an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., - where is one of mean, median, p90, p95, p99, sum, count. When omitted, - defaults to -created_at with pinned experiments first.' + allOf: + - $ref: '#/components/schemas/EvaluatorResultSortField' + default: -created_at - in: query name: filter style: deepObject required: false explode: true schema: - $ref: '#/components/schemas/ExperimentFilter' - description: 'Filter experiments by name, experiment_group_id, dataset_name, - dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true - to return only soft-deleted experiments; omit to see only live ones. Pass - is_pinned=true (or false) to filter by pinned state; omit to return both. - Filter by a metadata key/value: filter[metadata.]=. Filter by - a rollup metric with numeric range operators ($gte/$lte/$gt/$lt/$eq): filter[run_count][$gte]=5, - filter[cost_usd.mean][$lte]=0.5, filter[latency_ms.p95][$lte]=1000, or filter[evaluators..mean][$gte]=0.8.' + $ref: '#/components/schemas/EvaluatorResultFilter' + description: Filter evaluator results by span_id, session_id, name, data_type, + created_by, value range, and created_at range. responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponsesPage' - '400': - description: Unsupported sort or filter field - '413': - description: Too many experiments selected to sort in one request - '503': - description: Telemetry store unavailable for a metric-based sort or filter + $ref: '#/components/schemas/EvaluatorResultsPage' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiments/{name}: + /apis/intake/v2/workspaces/{workspace}/evaluator-results/{evaluator_result_id}: get: tags: - - Experiments - summary: Get Experiment - operationId: get_experiment_apis_intake_v2_workspaces__workspace__experiments__name__get + - Evaluator Results + summary: Get Evaluator Result + operationId: get_evaluator_result_apis_intake_v2_workspaces__workspace__evaluator_results__evaluator_result_id__get parameters: - name: workspace in: path @@ -3792,32 +3814,31 @@ paths: schema: type: string title: Workspace - - name: name + - name: evaluator_result_id in: path required: true schema: type: string - title: Name + title: Evaluator Result Id responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' - '404': - description: Experiment not found + $ref: '#/components/schemas/EvaluatorResult' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - put: + /apis/intake/v2/workspaces/{workspace}/experiment-groups: + post: tags: - - Experiments - summary: Update Experiment - operationId: update_experiment_apis_intake_v2_workspaces__workspace__experiments__name__put + - Experiment Groups + summary: Create Experiment Group + operationId: create_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups_post parameters: - name: workspace in: path @@ -3825,40 +3846,32 @@ paths: schema: type: string title: Workspace - - name: name - in: path - required: true - schema: - type: string - title: Name requestBody: required: true content: application/json: schema: - $ref: '#/components/schemas/ExperimentRequest' + $ref: '#/components/schemas/ExperimentGroupRequest' responses: - '200': + '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' - '404': - description: Experiment not found + $ref: '#/components/schemas/ExperimentGroupResponse' '409': - description: Attempt to change an immutable field + description: Experiment group already exists '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - delete: + get: tags: - - Experiments - summary: Delete Experiment - operationId: delete_experiment_apis_intake_v2_workspaces__workspace__experiments__name__delete + - Experiment Groups + summary: List Experiment Groups + operationId: list_experiment_groups_apis_intake_v2_workspaces__workspace__experiment_groups_get parameters: - name: workspace in: path @@ -3866,36 +3879,71 @@ paths: schema: type: string title: Workspace - - name: name - in: path - required: true + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: sort + in: query + required: false schema: + enum: + - -created_at + - created_at + - -updated_at + - updated_at + - -name + - name type: string - title: Name + description: Sort field; prefix with '-' for descending. + default: -created_at + title: Sort + description: Sort field; prefix with '-' for descending. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/ExperimentGroupFilter' + description: 'Filter experiment groups by name, or by a metadata key/value: + filter[metadata.]=.' responses: - '204': + '200': description: Successful Response - '404': - description: Experiment not found + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponsesPage' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin: - post: + /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: + get: tags: - - Experiments - summary: Pin Experiment - description: 'Pin an experiment to the top of the list (workspace-shared). - - - Re-pinning an already-pinned experiment refreshes ``pinned_at`` to the current - timestamp, - - which is intentional (most-recently-pinned sorts first).' - operationId: pin_experiment_apis_intake_v2_workspaces__workspace__experiments__name__pin_post + - Experiment Groups + summary: Get Experiment Group + operationId: get_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__get parameters: - name: workspace in: path @@ -3915,22 +3963,20 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' + $ref: '#/components/schemas/ExperimentGroupResponse' '404': - description: Experiment not found + description: Experiment group not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - delete: + put: tags: - - Experiments - summary: Unpin Experiment - description: 'Unpin an experiment. Idempotent: unpinning an already-unpinned - experiment is a no-op.' - operationId: unpin_experiment_apis_intake_v2_workspaces__workspace__experiments__name__pin_delete + - Experiment Groups + summary: Update Experiment Group + operationId: update_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__put parameters: - name: workspace in: path @@ -3944,27 +3990,34 @@ paths: schema: type: string title: Name + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupRequest' responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' + $ref: '#/components/schemas/ExperimentGroupResponse' '404': - description: Experiment not found + description: Experiment group not found + '409': + description: Attempt to rename the group '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiments/{name}/sessions: - get: + delete: tags: - - Experiments - summary: List Experiment Sessions - operationId: list_experiment_sessions_apis_intake_v2_workspaces__workspace__experiments__name__sessions_get + - Experiment Groups + summary: Delete Experiment Group + operationId: delete_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__delete parameters: - name: workspace in: path @@ -3978,62 +4031,11 @@ paths: schema: type: string title: Name - - name: page - in: query - required: false - schema: - type: integer - minimum: 1 - description: Page number. - default: 1 - title: Page - description: Page number. - - name: page_size - in: query - required: false - schema: - type: integer - maximum: 1000 - minimum: 1 - description: Page size. - default: 100 - title: Page Size - description: Page size. - - name: mode - in: query - required: false - schema: - enum: - - summary - - detailed - type: string - description: Response payload mode. summary keeps the same session row fields - but truncates root-span input to 1000 characters; detailed returns the - full root-span input. - default: detailed - title: Mode - description: Response payload mode. summary keeps the same session row fields - but truncates root-span input to 1000 characters; detailed returns the full - root-span input. - - in: query - name: filter - style: deepObject - required: false - explode: true - schema: - $ref: '#/components/schemas/ExperimentSessionFilter' - description: Filter sessions by test_case_id and status. responses: - '200': + '204': description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/ExperimentSessionResponsesPage' '404': - description: Experiment not found - '503': - description: ClickHouse unavailable + description: Experiment group not found '422': description: Validation Error content: @@ -4426,7 +4428,8 @@ paths: schema: $ref: '#/components/schemas/TraceFilter' description: Filter root-span-backed traces by id, session_id, root status, - root span started_at, experiment_id, and test_case_id. + root span started_at, evaluation_id (or its deprecated alias experiment_id), + and test_case_id. responses: '200': description: Successful Response @@ -10188,7 +10191,7 @@ components: properties: evaluation_id: title: Evaluation Id - description: Name of an existing Experiment entity. + description: Name of an existing Evaluation. type: string test_case_id: title: Test Case Id @@ -10204,299 +10207,48 @@ components: evaluation_run_id, metadata) keeps ingesting without error rather than being rejected.' - EvaluatorAggregate: - properties: - sum: - title: Sum - type: number - mean: - title: Mean - type: number - median: - title: Median - type: number - p90: - title: P90 - type: number - p95: - title: P95 - type: number - p99: - title: P99 - type: number - count: - type: integer - title: Count - default: 0 - type: object - title: EvaluatorAggregate - description: Aggregate statistics over evaluator scores or session-level metric - values. - EvaluatorResult: + EvaluationFilter: + additionalProperties: false + description: Filter for listing Evaluations. properties: - evaluator_result_id: - type: string - title: Evaluator Result Id - span_id: - type: string - title: Span Id - session_id: - type: string - title: Session Id - workspace: - type: string - title: Workspace name: - type: string + description: Filter evaluations by name. title: Name - value: - title: Value - type: number - string_value: - title: String Value type: string - data_type: - $ref: '#/components/schemas/EvaluatorResultDataType' - comment: - title: Comment + experiment_group_id: + description: Filter evaluations by owning group id. + title: Experiment Group Id + type: string + dataset_name: + description: Filter evaluations by dataset name. + title: Dataset Name + type: string + dataset_version: + description: Filter evaluations by dataset version. + title: Dataset Version type: string created_by: - title: Created By - type: string - created_at: - type: string - format: date-time - title: Created At - ingested_at: - type: string - format: date-time - title: Ingested At - type: object - required: - - evaluator_result_id - - span_id - - session_id - - workspace - - name - - data_type - - created_at - - ingested_at - title: EvaluatorResult - description: Response model for evaluator_results read endpoints. - EvaluatorResultDataType: - type: string - enum: - - NUMERIC - - CATEGORICAL - - BOOLEAN - - TEXT - title: EvaluatorResultDataType - EvaluatorResultFilter: - properties: - span_id: - description: Filter by target span id. - title: Span Id - type: string - session_id: - description: Filter by target session id. - title: Session Id - type: string - name: - description: Filter by evaluator/metric name. - title: Name - type: string - data_type: - allOf: - - $ref: '#/components/schemas/EvaluatorResultDataType' - description: Filter by data_type. - created_by: - description: Filter by principal/system that wrote the row. - title: Created By - type: string - value: - allOf: - - $ref: '#/components/schemas/FloatFilter' - description: Filter by numeric value (range supported). - created_at: - allOf: - - $ref: '#/components/schemas/DatetimeFilter' - description: Filter by row creation time (range supported). - title: EvaluatorResultFilter - type: object - EvaluatorResultInput: - properties: - span_id: - type: string - title: Span Id - description: Target span id. Not validated against existing spans (loose - target policy). - session_id: - type: string - title: Session Id - description: Session id the target span belongs to. Denormalized so session-scoped - reads stay fast. - name: - type: string - title: Name - description: Evaluator / metric identity (e.g. 'faithfulness/v1'). - value: - title: Value - description: Numeric value. Required when data_type is NUMERIC or BOOLEAN - (0|1). - type: number - string_value: - title: String Value - description: String value. Required when data_type is CATEGORICAL or TEXT. - type: string - data_type: - allOf: - - $ref: '#/components/schemas/EvaluatorResultDataType' - description: Discriminator for which of value / string_value carries the - payload. - comment: - title: Comment - description: Free-text rationale or explanation. - type: string - additionalProperties: false - type: object - required: - - span_id - - session_id - - name - - data_type - title: EvaluatorResultInput - description: "Request body for POST /evaluator-results.\n\nServer fills in `evaluator_result_id`,\ - \ `created_at`, `ingested_at`, and\n`created_by`. Producer supplies the target\ - \ span (loose target \u2014 not\nvalidated against the spans table), the score,\ - \ and provenance." - EvaluatorResultSortField: - type: string - enum: - - created_at - - -created_at - - value - - -value - title: EvaluatorResultSortField - EvaluatorResultsPage: - properties: - data: - items: - $ref: '#/components/schemas/EvaluatorResult' - type: array - title: Data - pagination: - allOf: - - $ref: '#/components/schemas/PaginationData' - description: Pagination information. - sort: - title: Sort - description: The field on which the results are sorted. - type: string - filter: - title: Filter - description: Filtering information. - additionalProperties: true - type: object - type: object - required: - - data - title: EvaluatorResultsPage - ExecutedAction: - properties: - action_name: - type: string - title: Action Name - description: The name of the action that was executed. - action_params: - additionalProperties: true - type: object - title: Action Params - description: The parameters for the action. - return_value: - title: Return Value - description: The value returned by the action. - llm_calls: - items: - $ref: '#/components/schemas/LLMCallInfo' - type: array - title: Llm Calls - description: Information about the LLM calls made by the action. - started_at: - title: Started At - description: Timestamp for when the action started. - type: number - finished_at: - title: Finished At - description: Timestamp for when the action finished. - type: number - duration: - title: Duration - description: How long the action took to execute, in seconds. - type: number - type: object - required: - - action_name - title: ExecutedAction - description: Information about an action that was executed. - ExperimentContext: - properties: - experiment_id: - type: string - title: Experiment Id - description: Name of an existing Experiment entity. - test_case_id: - title: Test Case Id - description: Optional producer-supplied test case id. - type: string - additionalProperties: false - type: object - required: - - experiment_id - title: ExperimentContext - description: Deprecated alias for :class:`EvaluationContext`. Producers should - send ``evaluation_context``. - ExperimentFilter: - additionalProperties: false - description: Filter for listing Experiments. - properties: - name: - description: Filter experiments by name. - title: Name - type: string - experiment_group_id: - description: Filter experiments by owning group id. - title: Experiment Group Id - type: string - dataset_name: - description: Filter experiments by dataset name. - title: Dataset Name - type: string - dataset_version: - description: Filter experiments by dataset version. - title: Dataset Version - type: string - created_by: - description: Filter experiments by the principal that created them. + description: Filter evaluations by the principal that created them. title: Created By type: string created_at: allOf: - $ref: '#/components/schemas/DatetimeFilter' - description: Filter experiments by creation timestamp; supports `$gte` and + description: Filter evaluations by creation timestamp; supports `$gte` and `$lte` for ranges. updated_at: allOf: - $ref: '#/components/schemas/DatetimeFilter' - description: Filter experiments by last-updated timestamp; supports `$gte` + description: Filter evaluations by last-updated timestamp; supports `$gte` and `$lte` for ranges. is_deleted: - description: When true, returns only soft-deleted experiments. Omit (or - false) to see only live experiments. + description: When true, returns only soft-deleted evaluations. Omit (or + false) to see only live evaluations. title: Is Deleted type: boolean is_pinned: - description: When true, returns only pinned experiments. When false, returns - only unpinned experiments. Omit to return both. + description: When true, returns only pinned evaluations. When false, returns + only unpinned evaluations. Omit to return both. title: Is Pinned type: boolean metadata: @@ -10523,189 +10275,61 @@ components: additionalProperties: $ref: '#/components/schemas/MetricStatFilters' type: object - title: ExperimentFilter + title: EvaluationFilter type: object - ExperimentGroupFilter: - additionalProperties: false - description: Filter for listing ExperimentGroups. + EvaluationRequest: properties: name: - description: Filter groups by name. + type: string title: Name + description: Producer-supplied, workspace-unique evaluation id. + experiment_group_id: type: string - is_deleted: - description: When true, returns only soft-deleted groups. Omit (or false) - to see only live groups. - title: Is Deleted - type: boolean + title: Experiment Group Id + description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ + \ group must already exist." + dataset_name: + type: string + title: Dataset Name + description: Producer-supplied dataset name. + dataset_version: + title: Dataset Version + description: Producer-supplied dataset version. + type: string + source_link: + title: Source Link + description: Optional URL for the source evaluation. + type: string + minLength: 1 + format: uri metadata: - description: Filter by a metadata key/value pair, e.g. filter[metadata.model]=claude-opus-4-8. - title: Metadata additionalProperties: type: string type: object - title: ExperimentGroupFilter - type: object - ExperimentGroupRequest: - properties: - name: - type: string - title: Name - description: Workspace-unique group name. + title: Metadata + description: Free-form producer metadata. description: title: Description - description: Human-readable purpose of the group. + description: Human-readable description. type: string - insight_id: - title: Insight Id - description: Reference to an external insight that seeded this group, if - any. + parent_evaluation_id: + title: Parent Evaluation Id + description: Entity id of the evaluation this one was derived from (e.g. + a variant of a baseline), if any. type: string - summary: - title: Summary - description: Human- or agent-authored summary of the group's findings. + parent_experiment_id: + title: Parent Experiment Id + description: Deprecated alias for parent_evaluation_id. + deprecated: true type: string - metadata: - title: Metadata - description: Free-form producer metadata for the group. - additionalProperties: - type: string - type: object - default_sort: - type: string - title: Default Sort - description: Default sort for this group's experiments list, as a `sort`-param - string (leading '-' = descending); defaults to '-created_at'. Accepts - any field the experiments list `sort` param does; clients apply it as - the list `sort` param. - default: -created_at - additionalProperties: false - type: object - required: - - name - title: ExperimentGroupRequest - description: Request body for creating an ExperimentGroup. - ExperimentGroupResponse: - properties: - id: - type: string - title: Id - name: - type: string - title: Name - workspace: - type: string - title: Workspace - description: - title: Description - type: string - insight_id: - title: Insight Id - type: string - summary: - title: Summary - type: string - metadata: - title: Metadata - additionalProperties: - type: string - type: object - default_sort: - type: string - title: Default Sort - created_at: - title: Created At - type: string - format: date-time - updated_at: - title: Updated At - type: string - format: date-time - experiment_count: - type: integer - title: Experiment Count - description: Number of live (non-soft-deleted) experiments in this group. - default: 0 - type: object - required: - - id - - name - - workspace - - default_sort - title: ExperimentGroupResponse - description: ExperimentGroup as served by the API. - ExperimentGroupResponsesPage: - properties: - data: - items: - $ref: '#/components/schemas/ExperimentGroupResponse' - type: array - title: Data - pagination: - allOf: - - $ref: '#/components/schemas/PaginationData' - description: Pagination information. - sort: - title: Sort - description: The field on which the results are sorted. - type: string - filter: - title: Filter - description: Filtering information. - additionalProperties: true - type: object - type: object - required: - - data - title: ExperimentGroupResponsesPage - ExperimentRequest: - properties: - name: - type: string - title: Name - description: Producer-supplied, workspace-unique experiment id. - experiment_group_id: - type: string - title: Experiment Group Id - description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ - \ group must already exist." - dataset_name: - type: string - title: Dataset Name - description: Producer-supplied dataset name. - dataset_version: - title: Dataset Version - description: Producer-supplied dataset version. - type: string - source_link: - title: Source Link - description: Optional URL for the source experiment. - type: string - minLength: 1 - format: uri - metadata: - additionalProperties: - type: string - type: object - title: Metadata - description: Free-form producer metadata. - description: - title: Description - description: Human-readable description. - type: string - parent_experiment_id: - title: Parent Experiment Id - description: Entity id of the experiment this one was derived from (e.g. - a variant of a baseline), if any. - type: string - status: - title: Status - description: Producer-defined lifecycle status of the experiment. - type: string - root_cause: - title: Root Cause - description: Human- or agent-authored explanation of the experiment's outcome - (e.g. why it was killed). + status: + title: Status + description: Producer-defined lifecycle status of the evaluation. + type: string + root_cause: + title: Root Cause + description: Human- or agent-authored explanation of the evaluation's outcome + (e.g. why it was killed). type: string additionalProperties: false type: object @@ -10713,9 +10337,9 @@ components: - name - experiment_group_id - dataset_name - title: ExperimentRequest - description: Request body for creating an Experiment. - ExperimentResponse: + title: EvaluationRequest + description: Request body for creating an Evaluation. + EvaluationResponse: properties: id: type: string @@ -10730,7 +10354,7 @@ components: type: string title: Experiment Group Id description: Entity id of the owning ExperimentGroup. Required for every - Experiment. + Evaluation. dataset_name: type: string title: Dataset Name @@ -10750,8 +10374,8 @@ components: description: title: Description type: string - parent_experiment_id: - title: Parent Experiment Id + parent_evaluation_id: + title: Parent Evaluation Id type: string status: title: Status @@ -10769,8 +10393,8 @@ components: format: date-time pinned_at: title: Pinned At - description: Timestamp at which the experiment was pinned, or null if unpinned. - Managed via POST/DELETE /experiments/{name}/pin. + description: Timestamp at which the evaluation was pinned, or null if unpinned. + Managed via POST/DELETE /evaluations/{name}/pin. nullable: true type: string format: date-time @@ -10786,7 +10410,7 @@ components: uniqueItems: true title: Model Names description: Distinct model names observed across ingested sessions for - this experiment. + this evaluation. agent_names: items: type: string @@ -10794,7 +10418,7 @@ components: uniqueItems: true title: Agent Names description: Distinct agent names observed across ingested sessions for - this experiment. + this evaluation. agent_versions: items: type: string @@ -10802,7 +10426,7 @@ components: uniqueItems: true title: Agent Versions description: Distinct agent versions observed across ingested sessions for - this experiment. + this evaluation. aggregate_scores: title: Aggregate Scores additionalProperties: @@ -10811,28 +10435,356 @@ components: run_count: type: integer title: Run Count - description: Number of distinct ingested experiment sessions; one session + description: Number of distinct ingested evaluation sessions; one session is treated as one run. default: 0 cost_usd: $ref: '#/components/schemas/EvaluatorAggregate' latency_ms: $ref: '#/components/schemas/EvaluatorAggregate' + parent_experiment_id: + title: Parent Experiment Id + description: Deprecated alias for parent_evaluation_id. + deprecated: true + readOnly: true + type: string + type: object + required: + - id + - name + - workspace + - experiment_group_id + - dataset_name + - parent_experiment_id + title: EvaluationResponse + description: Evaluation as served by the API, including ClickHouse-hydrated + rollups. + EvaluationResponsesPage: + properties: + data: + items: + $ref: '#/components/schemas/EvaluationResponse' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: EvaluationResponsesPage + EvaluationSessionFilter: + additionalProperties: false + description: Filter for listing EvaluationSessions. + properties: + test_case_id: + description: Filter by producer-supplied test case id. + title: Test Case Id + type: string + status: + description: Filter by root-span status (success, error, cancelled, unknown). + title: Status + type: string + title: EvaluationSessionFilter + type: object + EvaluationSessionResponse: + properties: + workspace: + type: string + title: Workspace + evaluation_name: + type: string + title: Evaluation Name + session_id: + type: string + title: Session Id + test_case_id: + title: Test Case Id + description: Producer-supplied test case identifier; null when the producer + did not set one. + type: string + trace_id: + type: string + title: Trace Id + root_span_id: + type: string + title: Root Span Id + started_at: + type: string + format: date-time + title: Started At + ended_at: + title: Ended At + type: string + format: date-time + latency_ms: + title: Latency Ms + type: number + status: + allOf: + - $ref: '#/components/schemas/SpanStatus' + description: 'Root-span status: success, error, cancelled, or unknown.' + input: + title: Input + description: Root-span input text. In summary mode this is truncated to + 1000 characters. + type: string + input_tokens: + title: Input Tokens + description: Sum of input tokens across this session's spans. + type: integer + output_tokens: + title: Output Tokens + description: Sum of output tokens across this session's spans. + type: integer + cached_tokens: + title: Cached Tokens + description: Sum of cached tokens across this session's spans. + type: integer + cost_total_usd: + title: Cost Total Usd + description: Sum of cost across this session's spans. + type: number + evaluator_scores: + additionalProperties: + type: number + type: object + title: Evaluator Scores + description: Per-evaluator session-mean score. Includes NUMERIC and BOOLEAN + evaluator results only; text/categorical results are omitted. + experiment_name: + type: string + title: Experiment Name + description: Deprecated alias for evaluation_name. + deprecated: true + readOnly: true + type: object + required: + - workspace + - evaluation_name + - session_id + - trace_id + - root_span_id + - started_at + - status + - experiment_name + title: EvaluationSessionResponse + description: "One ingested session of an Evaluation \u2014 a single test case\ + \ execution.\n\nHydrated from ClickHouse at read time by reading root/session\ + \ membership from\n``trace_index`` and joining page-bounded span/evaluator\ + \ rollups." + EvaluationSessionResponsesPage: + properties: + data: + items: + $ref: '#/components/schemas/EvaluationSessionResponse' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: EvaluationSessionResponsesPage + EvaluatorAggregate: + properties: + sum: + title: Sum + type: number + mean: + title: Mean + type: number + median: + title: Median + type: number + p90: + title: P90 + type: number + p95: + title: P95 + type: number + p99: + title: P99 + type: number + count: + type: integer + title: Count + default: 0 + type: object + title: EvaluatorAggregate + description: Aggregate statistics over evaluator scores or session-level metric + values. + EvaluatorResult: + properties: + evaluator_result_id: + type: string + title: Evaluator Result Id + span_id: + type: string + title: Span Id + session_id: + type: string + title: Session Id + workspace: + type: string + title: Workspace + name: + type: string + title: Name + value: + title: Value + type: number + string_value: + title: String Value + type: string + data_type: + $ref: '#/components/schemas/EvaluatorResultDataType' + comment: + title: Comment + type: string + created_by: + title: Created By + type: string + created_at: + type: string + format: date-time + title: Created At + ingested_at: + type: string + format: date-time + title: Ingested At + type: object + required: + - evaluator_result_id + - span_id + - session_id + - workspace + - name + - data_type + - created_at + - ingested_at + title: EvaluatorResult + description: Response model for evaluator_results read endpoints. + EvaluatorResultDataType: + type: string + enum: + - NUMERIC + - CATEGORICAL + - BOOLEAN + - TEXT + title: EvaluatorResultDataType + EvaluatorResultFilter: + properties: + span_id: + description: Filter by target span id. + title: Span Id + type: string + session_id: + description: Filter by target session id. + title: Session Id + type: string + name: + description: Filter by evaluator/metric name. + title: Name + type: string + data_type: + allOf: + - $ref: '#/components/schemas/EvaluatorResultDataType' + description: Filter by data_type. + created_by: + description: Filter by principal/system that wrote the row. + title: Created By + type: string + value: + allOf: + - $ref: '#/components/schemas/FloatFilter' + description: Filter by numeric value (range supported). + created_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Filter by row creation time (range supported). + title: EvaluatorResultFilter + type: object + EvaluatorResultInput: + properties: + span_id: + type: string + title: Span Id + description: Target span id. Not validated against existing spans (loose + target policy). + session_id: + type: string + title: Session Id + description: Session id the target span belongs to. Denormalized so session-scoped + reads stay fast. + name: + type: string + title: Name + description: Evaluator / metric identity (e.g. 'faithfulness/v1'). + value: + title: Value + description: Numeric value. Required when data_type is NUMERIC or BOOLEAN + (0|1). + type: number + string_value: + title: String Value + description: String value. Required when data_type is CATEGORICAL or TEXT. + type: string + data_type: + allOf: + - $ref: '#/components/schemas/EvaluatorResultDataType' + description: Discriminator for which of value / string_value carries the + payload. + comment: + title: Comment + description: Free-text rationale or explanation. + type: string + additionalProperties: false type: object required: - - id + - span_id + - session_id - name - - workspace - - experiment_group_id - - dataset_name - title: ExperimentResponse - description: Experiment as served by the API, including ClickHouse-hydrated - rollups. - ExperimentResponsesPage: + - data_type + title: EvaluatorResultInput + description: "Request body for POST /evaluator-results.\n\nServer fills in `evaluator_result_id`,\ + \ `created_at`, `ingested_at`, and\n`created_by`. Producer supplies the target\ + \ span (loose target \u2014 not\nvalidated against the spans table), the score,\ + \ and provenance." + EvaluatorResultSortField: + type: string + enum: + - created_at + - -created_at + - value + - -value + title: EvaluatorResultSortField + EvaluatorResultsPage: properties: data: items: - $ref: '#/components/schemas/ExperimentResponse' + $ref: '#/components/schemas/EvaluatorResult' type: array title: Data pagination: @@ -10851,105 +10803,182 @@ components: type: object required: - data - title: ExperimentResponsesPage - ExperimentSessionFilter: - additionalProperties: false - description: Filter for listing ExperimentSessions. + title: EvaluatorResultsPage + ExecutedAction: + properties: + action_name: + type: string + title: Action Name + description: The name of the action that was executed. + action_params: + additionalProperties: true + type: object + title: Action Params + description: The parameters for the action. + return_value: + title: Return Value + description: The value returned by the action. + llm_calls: + items: + $ref: '#/components/schemas/LLMCallInfo' + type: array + title: Llm Calls + description: Information about the LLM calls made by the action. + started_at: + title: Started At + description: Timestamp for when the action started. + type: number + finished_at: + title: Finished At + description: Timestamp for when the action finished. + type: number + duration: + title: Duration + description: How long the action took to execute, in seconds. + type: number + type: object + required: + - action_name + title: ExecutedAction + description: Information about an action that was executed. + ExperimentContext: properties: + experiment_id: + type: string + title: Experiment Id + description: Name of an existing Experiment entity. test_case_id: - description: Filter by producer-supplied test case id. title: Test Case Id + description: Optional producer-supplied test case id. type: string - status: - description: Filter by root-span status (success, error, cancelled, unknown). - title: Status + additionalProperties: false + type: object + required: + - experiment_id + title: ExperimentContext + description: Deprecated alias for :class:`EvaluationContext`. Producers should + send ``evaluation_context``. + ExperimentGroupFilter: + additionalProperties: false + description: Filter for listing ExperimentGroups. + properties: + name: + description: Filter groups by name. + title: Name + type: string + is_deleted: + description: When true, returns only soft-deleted groups. Omit (or false) + to see only live groups. + title: Is Deleted + type: boolean + metadata: + description: Filter by a metadata key/value pair, e.g. filter[metadata.model]=claude-opus-4-8. + title: Metadata + additionalProperties: + type: string + type: object + title: ExperimentGroupFilter + type: object + ExperimentGroupRequest: + properties: + name: + type: string + title: Name + description: Workspace-unique group name. + description: + title: Description + description: Human-readable purpose of the group. + type: string + insight_id: + title: Insight Id + description: Reference to an external insight that seeded this group, if + any. + type: string + summary: + title: Summary + description: Human- or agent-authored summary of the group's findings. + type: string + metadata: + title: Metadata + description: Free-form producer metadata for the group. + additionalProperties: + type: string + type: object + default_sort: type: string - title: ExperimentSessionFilter + title: Default Sort + description: Default sort for this group's evaluations list, as a `sort`-param + string (leading '-' = descending); defaults to '-created_at'. Accepts + any field the evaluations list `sort` param does; clients apply it as + the list `sort` param. + default: -created_at + additionalProperties: false type: object - ExperimentSessionResponse: + required: + - name + title: ExperimentGroupRequest + description: Request body for creating an ExperimentGroup. + ExperimentGroupResponse: properties: + id: + type: string + title: Id + name: + type: string + title: Name workspace: type: string title: Workspace - experiment_name: - type: string - title: Experiment Name - session_id: + description: + title: Description type: string - title: Session Id - test_case_id: - title: Test Case Id - description: Producer-supplied test case identifier; null when the producer - did not set one. + insight_id: + title: Insight Id type: string - trace_id: + summary: + title: Summary type: string - title: Trace Id - root_span_id: + metadata: + title: Metadata + additionalProperties: + type: string + type: object + default_sort: type: string - title: Root Span Id - started_at: + title: Default Sort + created_at: + title: Created At type: string format: date-time - title: Started At - ended_at: - title: Ended At + updated_at: + title: Updated At type: string format: date-time - latency_ms: - title: Latency Ms - type: number - status: - allOf: - - $ref: '#/components/schemas/SpanStatus' - description: 'Root-span status: success, error, cancelled, or unknown.' - input: - title: Input - description: Root-span input text. In summary mode this is truncated to - 1000 characters. - type: string - input_tokens: - title: Input Tokens - description: Sum of input tokens across this session's spans. - type: integer - output_tokens: - title: Output Tokens - description: Sum of output tokens across this session's spans. + evaluation_count: type: integer - cached_tokens: - title: Cached Tokens - description: Sum of cached tokens across this session's spans. + title: Evaluation Count + description: Number of live (non-soft-deleted) evaluations in this group. + default: 0 + experiment_count: type: integer - cost_total_usd: - title: Cost Total Usd - description: Sum of cost across this session's spans. - type: number - evaluator_scores: - additionalProperties: - type: number - type: object - title: Evaluator Scores - description: Per-evaluator session-mean score. Includes NUMERIC and BOOLEAN - evaluator results only; text/categorical results are omitted. + title: Experiment Count + description: Deprecated alias for evaluation_count. + deprecated: true + readOnly: true type: object required: + - id + - name - workspace - - experiment_name - - session_id - - trace_id - - root_span_id - - started_at - - status - title: ExperimentSessionResponse - description: "One ingested session of an Experiment \u2014 a single test case\ - \ execution.\n\nHydrated from ClickHouse at read time by reading root/session\ - \ membership from\n``trace_index`` and joining page-bounded span/evaluator\ - \ rollups." - ExperimentSessionResponsesPage: + - default_sort + - experiment_count + title: ExperimentGroupResponse + description: ExperimentGroup as served by the API. + ExperimentGroupResponsesPage: properties: data: items: - $ref: '#/components/schemas/ExperimentSessionResponse' + $ref: '#/components/schemas/ExperimentGroupResponse' type: array title: Data pagination: @@ -10968,7 +10997,7 @@ components: type: object required: - data - title: ExperimentSessionResponsesPage + title: ExperimentGroupResponsesPage FactCheckingRailConfig: properties: parameters: @@ -13248,7 +13277,7 @@ components: These stats must stay in sync with the runtime sort/filter grammar (``_METRIC_STATS`` in the - experiments + evaluations endpoints); a unit test guards the parity.' properties: @@ -17875,8 +17904,14 @@ components: name: title: Name type: string + evaluation_context: + $ref: '#/components/schemas/EvaluationContext' experiment_context: - $ref: '#/components/schemas/ExperimentContext' + allOf: + - $ref: '#/components/schemas/ExperimentContext' + description: Deprecated alias for evaluation_context; will be removed in + a future release. + deprecated: true started_at: type: string format: date-time @@ -17949,12 +17984,18 @@ components: allOf: - $ref: '#/components/schemas/DatetimeFilter' description: Filter by root span start timestamp. + evaluation_id: + description: Filter by root-span evaluation id. + title: Evaluation Id + type: string experiment_id: - description: Filter by root-span experiment id. + deprecated: true + description: Deprecated alias for evaluation_id. Filter by root-span evaluation + id. title: Experiment Id type: string test_case_id: - description: Filter by root-span experiment test case id. + description: Filter by root-span evaluation test case id. title: Test Case Id type: string title: TraceFilter diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index bc0343b03d..fd1e06dbb6 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -3327,12 +3327,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/evaluator-results: + /apis/intake/v2/workspaces/{workspace}/evaluations: post: tags: - - Evaluator Results - summary: Create Evaluator Result - operationId: create_evaluator_result_apis_intake_v2_workspaces__workspace__evaluator_results_post + - Evaluations + summary: Create Evaluation + operationId: create_evaluation_apis_intake_v2_workspaces__workspace__evaluations_post parameters: - name: workspace in: path @@ -3345,14 +3345,16 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/EvaluatorResultInput' + $ref: '#/components/schemas/EvaluationRequest' responses: '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/EvaluatorResult' + $ref: '#/components/schemas/EvaluationResponse' + '409': + description: Evaluation already exists '422': description: Validation Error content: @@ -3361,9 +3363,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' get: tags: - - Evaluator Results - summary: List Evaluator Results - operationId: list_evaluator_results_apis_intake_v2_workspaces__workspace__evaluator_results_get + - Evaluations + summary: List Evaluations + operationId: list_evaluations_apis_intake_v2_workspaces__workspace__evaluations_get parameters: - name: workspace in: path @@ -3389,44 +3391,65 @@ paths: maximum: 1000 minimum: 1 description: Page size. - default: 10 + default: 100 title: Page Size description: Page size. - name: sort in: query required: false schema: - allOf: - - $ref: '#/components/schemas/EvaluatorResultSortField' - default: -created_at + description: 'Field to sort by; prefix with ''-'' for descending. Sort by + an evaluation attribute (name, created_at, updated_at, pinned_at) or by + an aggregate metric: run_count, cost_usd., latency_ms., or + evaluators.., where is one of mean, median, p90, p95, + p99, sum, count. When omitted, defaults to -created_at with pinned evaluations + first.' + title: Sort + type: string + description: 'Field to sort by; prefix with ''-'' for descending. Sort by + an evaluation attribute (name, created_at, updated_at, pinned_at) or by + an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., + where is one of mean, median, p90, p95, p99, sum, count. When omitted, + defaults to -created_at with pinned evaluations first.' - in: query name: filter style: deepObject required: false explode: true schema: - $ref: '#/components/schemas/EvaluatorResultFilter' - description: Filter evaluator results by span_id, session_id, name, data_type, - created_by, value range, and created_at range. + $ref: '#/components/schemas/EvaluationFilter' + description: 'Filter evaluations by name, experiment_group_id, dataset_name, + dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true + to return only soft-deleted evaluations; omit to see only live ones. Pass + is_pinned=true (or false) to filter by pinned state; omit to return both. + Filter by a metadata key/value: filter[metadata.]=. Filter by + a rollup metric with numeric range operators ($gte/$lte/$gt/$lt/$eq): filter[run_count][$gte]=5, + filter[cost_usd.mean][$lte]=0.5, filter[latency_ms.p95][$lte]=1000, or filter[evaluators..mean][$gte]=0.8.' responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/EvaluatorResultsPage' + $ref: '#/components/schemas/EvaluationResponsesPage' + '400': + description: Unsupported sort or filter field + '413': + description: Too many evaluations selected to sort in one request + '503': + description: Telemetry store unavailable for a metric-based sort or filter '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/evaluator-results/{evaluator_result_id}: + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}: get: tags: - - Evaluator Results - summary: Get Evaluator Result - operationId: get_evaluator_result_apis_intake_v2_workspaces__workspace__evaluator_results__evaluator_result_id__get + - Evaluations + summary: Get Evaluation + operationId: get_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__get parameters: - name: workspace in: path @@ -3434,31 +3457,32 @@ paths: schema: type: string title: Workspace - - name: evaluator_result_id + - name: name in: path required: true schema: type: string - title: Evaluator Result Id + title: Name responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/EvaluatorResult' + $ref: '#/components/schemas/EvaluationResponse' + '404': + description: Evaluation not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups: - post: + put: tags: - - Experiment Groups - summary: Create Experiment Group - operationId: create_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups_post + - Evaluations + summary: Update Evaluation + operationId: update_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__put parameters: - name: workspace in: path @@ -3466,32 +3490,40 @@ paths: schema: type: string title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name requestBody: required: true content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupRequest' + $ref: '#/components/schemas/EvaluationRequest' responses: - '201': + '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/EvaluationResponse' + '404': + description: Evaluation not found '409': - description: Experiment group already exists + description: Attempt to change an immutable field '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - get: + delete: tags: - - Experiment Groups - summary: List Experiment Groups - operationId: list_experiment_groups_apis_intake_v2_workspaces__workspace__experiment_groups_get + - Evaluations + summary: Delete Evaluation + operationId: delete_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__delete parameters: - name: workspace in: path @@ -3499,71 +3531,36 @@ paths: schema: type: string title: Workspace - - name: page - in: query - required: false - schema: - type: integer - minimum: 1 - description: Page number. - default: 1 - title: Page - description: Page number. - - name: page_size - in: query - required: false - schema: - type: integer - maximum: 1000 - minimum: 1 - description: Page size. - default: 100 - title: Page Size - description: Page size. - - name: sort - in: query - required: false + - name: name + in: path + required: true schema: - enum: - - -created_at - - created_at - - -updated_at - - updated_at - - -name - - name type: string - description: Sort field; prefix with '-' for descending. - default: -created_at - title: Sort - description: Sort field; prefix with '-' for descending. - - in: query - name: filter - style: deepObject - required: false - explode: true - schema: - $ref: '#/components/schemas/ExperimentGroupFilter' - description: 'Filter experiment groups by name, or by a metadata key/value: - filter[metadata.]=.' + title: Name responses: - '200': + '204': description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/ExperimentGroupResponsesPage' + '404': + description: Evaluation not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: - get: + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin: + post: tags: - - Experiment Groups - summary: Get Experiment Group - operationId: get_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__get + - Evaluations + summary: Pin Evaluation + description: 'Pin an evaluation to the top of the list (workspace-shared). + + + Re-pinning an already-pinned evaluation refreshes ``pinned_at`` to the current + timestamp, + + which is intentional (most-recently-pinned sorts first).' + operationId: pin_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__pin_post parameters: - name: workspace in: path @@ -3583,20 +3580,22 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/EvaluationResponse' '404': - description: Experiment group not found + description: Evaluation not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - put: + delete: tags: - - Experiment Groups - summary: Update Experiment Group - operationId: update_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__put + - Evaluations + summary: Unpin Evaluation + description: 'Unpin an evaluation. Idempotent: unpinning an already-unpinned + evaluation is a no-op.' + operationId: unpin_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__pin_delete parameters: - name: workspace in: path @@ -3610,34 +3609,27 @@ paths: schema: type: string title: Name - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/ExperimentGroupRequest' responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/EvaluationResponse' '404': - description: Experiment group not found - '409': - description: Attempt to rename the group + description: Evaluation not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - delete: + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/sessions: + get: tags: - - Experiment Groups - summary: Delete Experiment Group - operationId: delete_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__delete + - Evaluations + summary: List Evaluation Sessions + operationId: list_evaluation_sessions_apis_intake_v2_workspaces__workspace__evaluations__name__sessions_get parameters: - name: workspace in: path @@ -3651,23 +3643,76 @@ paths: schema: type: string title: Name + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: mode + in: query + required: false + schema: + enum: + - summary + - detailed + type: string + description: Response payload mode. summary keeps the same session row fields + but truncates root-span input to 1000 characters; detailed returns the + full root-span input. + default: detailed + title: Mode + description: Response payload mode. summary keeps the same session row fields + but truncates root-span input to 1000 characters; detailed returns the full + root-span input. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/EvaluationSessionFilter' + description: Filter sessions by test_case_id and status. responses: - '204': + '200': description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationSessionResponsesPage' + '400': + description: Invalid filter value '404': - description: Experiment group not found + description: Evaluation not found + '503': + description: ClickHouse unavailable '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiments: + /apis/intake/v2/workspaces/{workspace}/evaluator-results: post: tags: - - Experiments - summary: Create Experiment - operationId: create_experiment_apis_intake_v2_workspaces__workspace__experiments_post + - Evaluator Results + summary: Create Evaluator Result + operationId: create_evaluator_result_apis_intake_v2_workspaces__workspace__evaluator_results_post parameters: - name: workspace in: path @@ -3680,16 +3725,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentRequest' + $ref: '#/components/schemas/EvaluatorResultInput' responses: '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' - '409': - description: Experiment already exists + $ref: '#/components/schemas/EvaluatorResult' '422': description: Validation Error content: @@ -3698,9 +3741,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' get: tags: - - Experiments - summary: List Experiments - operationId: list_experiments_apis_intake_v2_workspaces__workspace__experiments_get + - Evaluator Results + summary: List Evaluator Results + operationId: list_evaluator_results_apis_intake_v2_workspaces__workspace__evaluator_results_get parameters: - name: workspace in: path @@ -3726,65 +3769,44 @@ paths: maximum: 1000 minimum: 1 description: Page size. - default: 100 + default: 10 title: Page Size description: Page size. - name: sort in: query required: false schema: - description: 'Field to sort by; prefix with ''-'' for descending. Sort by - an experiment attribute (name, created_at, updated_at, pinned_at) or by - an aggregate metric: run_count, cost_usd., latency_ms., or - evaluators.., where is one of mean, median, p90, p95, - p99, sum, count. When omitted, defaults to -created_at with pinned experiments - first.' - title: Sort - type: string - description: 'Field to sort by; prefix with ''-'' for descending. Sort by - an experiment attribute (name, created_at, updated_at, pinned_at) or by - an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., - where is one of mean, median, p90, p95, p99, sum, count. When omitted, - defaults to -created_at with pinned experiments first.' + allOf: + - $ref: '#/components/schemas/EvaluatorResultSortField' + default: -created_at - in: query name: filter style: deepObject required: false explode: true schema: - $ref: '#/components/schemas/ExperimentFilter' - description: 'Filter experiments by name, experiment_group_id, dataset_name, - dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true - to return only soft-deleted experiments; omit to see only live ones. Pass - is_pinned=true (or false) to filter by pinned state; omit to return both. - Filter by a metadata key/value: filter[metadata.]=. Filter by - a rollup metric with numeric range operators ($gte/$lte/$gt/$lt/$eq): filter[run_count][$gte]=5, - filter[cost_usd.mean][$lte]=0.5, filter[latency_ms.p95][$lte]=1000, or filter[evaluators..mean][$gte]=0.8.' + $ref: '#/components/schemas/EvaluatorResultFilter' + description: Filter evaluator results by span_id, session_id, name, data_type, + created_by, value range, and created_at range. responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponsesPage' - '400': - description: Unsupported sort or filter field - '413': - description: Too many experiments selected to sort in one request - '503': - description: Telemetry store unavailable for a metric-based sort or filter + $ref: '#/components/schemas/EvaluatorResultsPage' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiments/{name}: + /apis/intake/v2/workspaces/{workspace}/evaluator-results/{evaluator_result_id}: get: tags: - - Experiments - summary: Get Experiment - operationId: get_experiment_apis_intake_v2_workspaces__workspace__experiments__name__get + - Evaluator Results + summary: Get Evaluator Result + operationId: get_evaluator_result_apis_intake_v2_workspaces__workspace__evaluator_results__evaluator_result_id__get parameters: - name: workspace in: path @@ -3792,32 +3814,31 @@ paths: schema: type: string title: Workspace - - name: name + - name: evaluator_result_id in: path required: true schema: type: string - title: Name + title: Evaluator Result Id responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' - '404': - description: Experiment not found + $ref: '#/components/schemas/EvaluatorResult' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - put: + /apis/intake/v2/workspaces/{workspace}/experiment-groups: + post: tags: - - Experiments - summary: Update Experiment - operationId: update_experiment_apis_intake_v2_workspaces__workspace__experiments__name__put + - Experiment Groups + summary: Create Experiment Group + operationId: create_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups_post parameters: - name: workspace in: path @@ -3825,40 +3846,32 @@ paths: schema: type: string title: Workspace - - name: name - in: path - required: true - schema: - type: string - title: Name requestBody: required: true content: application/json: schema: - $ref: '#/components/schemas/ExperimentRequest' + $ref: '#/components/schemas/ExperimentGroupRequest' responses: - '200': + '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' - '404': - description: Experiment not found + $ref: '#/components/schemas/ExperimentGroupResponse' '409': - description: Attempt to change an immutable field + description: Experiment group already exists '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - delete: + get: tags: - - Experiments - summary: Delete Experiment - operationId: delete_experiment_apis_intake_v2_workspaces__workspace__experiments__name__delete + - Experiment Groups + summary: List Experiment Groups + operationId: list_experiment_groups_apis_intake_v2_workspaces__workspace__experiment_groups_get parameters: - name: workspace in: path @@ -3866,36 +3879,71 @@ paths: schema: type: string title: Workspace - - name: name - in: path - required: true + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: sort + in: query + required: false schema: + enum: + - -created_at + - created_at + - -updated_at + - updated_at + - -name + - name type: string - title: Name + description: Sort field; prefix with '-' for descending. + default: -created_at + title: Sort + description: Sort field; prefix with '-' for descending. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/ExperimentGroupFilter' + description: 'Filter experiment groups by name, or by a metadata key/value: + filter[metadata.]=.' responses: - '204': + '200': description: Successful Response - '404': - description: Experiment not found + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponsesPage' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin: - post: + /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: + get: tags: - - Experiments - summary: Pin Experiment - description: 'Pin an experiment to the top of the list (workspace-shared). - - - Re-pinning an already-pinned experiment refreshes ``pinned_at`` to the current - timestamp, - - which is intentional (most-recently-pinned sorts first).' - operationId: pin_experiment_apis_intake_v2_workspaces__workspace__experiments__name__pin_post + - Experiment Groups + summary: Get Experiment Group + operationId: get_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__get parameters: - name: workspace in: path @@ -3915,22 +3963,20 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' + $ref: '#/components/schemas/ExperimentGroupResponse' '404': - description: Experiment not found + description: Experiment group not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - delete: + put: tags: - - Experiments - summary: Unpin Experiment - description: 'Unpin an experiment. Idempotent: unpinning an already-unpinned - experiment is a no-op.' - operationId: unpin_experiment_apis_intake_v2_workspaces__workspace__experiments__name__pin_delete + - Experiment Groups + summary: Update Experiment Group + operationId: update_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__put parameters: - name: workspace in: path @@ -3944,27 +3990,34 @@ paths: schema: type: string title: Name + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupRequest' responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' + $ref: '#/components/schemas/ExperimentGroupResponse' '404': - description: Experiment not found + description: Experiment group not found + '409': + description: Attempt to rename the group '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiments/{name}/sessions: - get: + delete: tags: - - Experiments - summary: List Experiment Sessions - operationId: list_experiment_sessions_apis_intake_v2_workspaces__workspace__experiments__name__sessions_get + - Experiment Groups + summary: Delete Experiment Group + operationId: delete_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__delete parameters: - name: workspace in: path @@ -3978,62 +4031,11 @@ paths: schema: type: string title: Name - - name: page - in: query - required: false - schema: - type: integer - minimum: 1 - description: Page number. - default: 1 - title: Page - description: Page number. - - name: page_size - in: query - required: false - schema: - type: integer - maximum: 1000 - minimum: 1 - description: Page size. - default: 100 - title: Page Size - description: Page size. - - name: mode - in: query - required: false - schema: - enum: - - summary - - detailed - type: string - description: Response payload mode. summary keeps the same session row fields - but truncates root-span input to 1000 characters; detailed returns the - full root-span input. - default: detailed - title: Mode - description: Response payload mode. summary keeps the same session row fields - but truncates root-span input to 1000 characters; detailed returns the full - root-span input. - - in: query - name: filter - style: deepObject - required: false - explode: true - schema: - $ref: '#/components/schemas/ExperimentSessionFilter' - description: Filter sessions by test_case_id and status. responses: - '200': + '204': description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/ExperimentSessionResponsesPage' '404': - description: Experiment not found - '503': - description: ClickHouse unavailable + description: Experiment group not found '422': description: Validation Error content: @@ -4426,7 +4428,8 @@ paths: schema: $ref: '#/components/schemas/TraceFilter' description: Filter root-span-backed traces by id, session_id, root status, - root span started_at, experiment_id, and test_case_id. + root span started_at, evaluation_id (or its deprecated alias experiment_id), + and test_case_id. responses: '200': description: Successful Response @@ -10188,7 +10191,7 @@ components: properties: evaluation_id: title: Evaluation Id - description: Name of an existing Experiment entity. + description: Name of an existing Evaluation. type: string test_case_id: title: Test Case Id @@ -10204,299 +10207,48 @@ components: evaluation_run_id, metadata) keeps ingesting without error rather than being rejected.' - EvaluatorAggregate: - properties: - sum: - title: Sum - type: number - mean: - title: Mean - type: number - median: - title: Median - type: number - p90: - title: P90 - type: number - p95: - title: P95 - type: number - p99: - title: P99 - type: number - count: - type: integer - title: Count - default: 0 - type: object - title: EvaluatorAggregate - description: Aggregate statistics over evaluator scores or session-level metric - values. - EvaluatorResult: + EvaluationFilter: + additionalProperties: false + description: Filter for listing Evaluations. properties: - evaluator_result_id: - type: string - title: Evaluator Result Id - span_id: - type: string - title: Span Id - session_id: - type: string - title: Session Id - workspace: - type: string - title: Workspace name: - type: string + description: Filter evaluations by name. title: Name - value: - title: Value - type: number - string_value: - title: String Value type: string - data_type: - $ref: '#/components/schemas/EvaluatorResultDataType' - comment: - title: Comment + experiment_group_id: + description: Filter evaluations by owning group id. + title: Experiment Group Id + type: string + dataset_name: + description: Filter evaluations by dataset name. + title: Dataset Name + type: string + dataset_version: + description: Filter evaluations by dataset version. + title: Dataset Version type: string created_by: - title: Created By - type: string - created_at: - type: string - format: date-time - title: Created At - ingested_at: - type: string - format: date-time - title: Ingested At - type: object - required: - - evaluator_result_id - - span_id - - session_id - - workspace - - name - - data_type - - created_at - - ingested_at - title: EvaluatorResult - description: Response model for evaluator_results read endpoints. - EvaluatorResultDataType: - type: string - enum: - - NUMERIC - - CATEGORICAL - - BOOLEAN - - TEXT - title: EvaluatorResultDataType - EvaluatorResultFilter: - properties: - span_id: - description: Filter by target span id. - title: Span Id - type: string - session_id: - description: Filter by target session id. - title: Session Id - type: string - name: - description: Filter by evaluator/metric name. - title: Name - type: string - data_type: - allOf: - - $ref: '#/components/schemas/EvaluatorResultDataType' - description: Filter by data_type. - created_by: - description: Filter by principal/system that wrote the row. - title: Created By - type: string - value: - allOf: - - $ref: '#/components/schemas/FloatFilter' - description: Filter by numeric value (range supported). - created_at: - allOf: - - $ref: '#/components/schemas/DatetimeFilter' - description: Filter by row creation time (range supported). - title: EvaluatorResultFilter - type: object - EvaluatorResultInput: - properties: - span_id: - type: string - title: Span Id - description: Target span id. Not validated against existing spans (loose - target policy). - session_id: - type: string - title: Session Id - description: Session id the target span belongs to. Denormalized so session-scoped - reads stay fast. - name: - type: string - title: Name - description: Evaluator / metric identity (e.g. 'faithfulness/v1'). - value: - title: Value - description: Numeric value. Required when data_type is NUMERIC or BOOLEAN - (0|1). - type: number - string_value: - title: String Value - description: String value. Required when data_type is CATEGORICAL or TEXT. - type: string - data_type: - allOf: - - $ref: '#/components/schemas/EvaluatorResultDataType' - description: Discriminator for which of value / string_value carries the - payload. - comment: - title: Comment - description: Free-text rationale or explanation. - type: string - additionalProperties: false - type: object - required: - - span_id - - session_id - - name - - data_type - title: EvaluatorResultInput - description: "Request body for POST /evaluator-results.\n\nServer fills in `evaluator_result_id`,\ - \ `created_at`, `ingested_at`, and\n`created_by`. Producer supplies the target\ - \ span (loose target \u2014 not\nvalidated against the spans table), the score,\ - \ and provenance." - EvaluatorResultSortField: - type: string - enum: - - created_at - - -created_at - - value - - -value - title: EvaluatorResultSortField - EvaluatorResultsPage: - properties: - data: - items: - $ref: '#/components/schemas/EvaluatorResult' - type: array - title: Data - pagination: - allOf: - - $ref: '#/components/schemas/PaginationData' - description: Pagination information. - sort: - title: Sort - description: The field on which the results are sorted. - type: string - filter: - title: Filter - description: Filtering information. - additionalProperties: true - type: object - type: object - required: - - data - title: EvaluatorResultsPage - ExecutedAction: - properties: - action_name: - type: string - title: Action Name - description: The name of the action that was executed. - action_params: - additionalProperties: true - type: object - title: Action Params - description: The parameters for the action. - return_value: - title: Return Value - description: The value returned by the action. - llm_calls: - items: - $ref: '#/components/schemas/LLMCallInfo' - type: array - title: Llm Calls - description: Information about the LLM calls made by the action. - started_at: - title: Started At - description: Timestamp for when the action started. - type: number - finished_at: - title: Finished At - description: Timestamp for when the action finished. - type: number - duration: - title: Duration - description: How long the action took to execute, in seconds. - type: number - type: object - required: - - action_name - title: ExecutedAction - description: Information about an action that was executed. - ExperimentContext: - properties: - experiment_id: - type: string - title: Experiment Id - description: Name of an existing Experiment entity. - test_case_id: - title: Test Case Id - description: Optional producer-supplied test case id. - type: string - additionalProperties: false - type: object - required: - - experiment_id - title: ExperimentContext - description: Deprecated alias for :class:`EvaluationContext`. Producers should - send ``evaluation_context``. - ExperimentFilter: - additionalProperties: false - description: Filter for listing Experiments. - properties: - name: - description: Filter experiments by name. - title: Name - type: string - experiment_group_id: - description: Filter experiments by owning group id. - title: Experiment Group Id - type: string - dataset_name: - description: Filter experiments by dataset name. - title: Dataset Name - type: string - dataset_version: - description: Filter experiments by dataset version. - title: Dataset Version - type: string - created_by: - description: Filter experiments by the principal that created them. + description: Filter evaluations by the principal that created them. title: Created By type: string created_at: allOf: - $ref: '#/components/schemas/DatetimeFilter' - description: Filter experiments by creation timestamp; supports `$gte` and + description: Filter evaluations by creation timestamp; supports `$gte` and `$lte` for ranges. updated_at: allOf: - $ref: '#/components/schemas/DatetimeFilter' - description: Filter experiments by last-updated timestamp; supports `$gte` + description: Filter evaluations by last-updated timestamp; supports `$gte` and `$lte` for ranges. is_deleted: - description: When true, returns only soft-deleted experiments. Omit (or - false) to see only live experiments. + description: When true, returns only soft-deleted evaluations. Omit (or + false) to see only live evaluations. title: Is Deleted type: boolean is_pinned: - description: When true, returns only pinned experiments. When false, returns - only unpinned experiments. Omit to return both. + description: When true, returns only pinned evaluations. When false, returns + only unpinned evaluations. Omit to return both. title: Is Pinned type: boolean metadata: @@ -10523,189 +10275,61 @@ components: additionalProperties: $ref: '#/components/schemas/MetricStatFilters' type: object - title: ExperimentFilter + title: EvaluationFilter type: object - ExperimentGroupFilter: - additionalProperties: false - description: Filter for listing ExperimentGroups. + EvaluationRequest: properties: name: - description: Filter groups by name. + type: string title: Name + description: Producer-supplied, workspace-unique evaluation id. + experiment_group_id: type: string - is_deleted: - description: When true, returns only soft-deleted groups. Omit (or false) - to see only live groups. - title: Is Deleted - type: boolean + title: Experiment Group Id + description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ + \ group must already exist." + dataset_name: + type: string + title: Dataset Name + description: Producer-supplied dataset name. + dataset_version: + title: Dataset Version + description: Producer-supplied dataset version. + type: string + source_link: + title: Source Link + description: Optional URL for the source evaluation. + type: string + minLength: 1 + format: uri metadata: - description: Filter by a metadata key/value pair, e.g. filter[metadata.model]=claude-opus-4-8. - title: Metadata additionalProperties: type: string type: object - title: ExperimentGroupFilter - type: object - ExperimentGroupRequest: - properties: - name: - type: string - title: Name - description: Workspace-unique group name. + title: Metadata + description: Free-form producer metadata. description: title: Description - description: Human-readable purpose of the group. + description: Human-readable description. type: string - insight_id: - title: Insight Id - description: Reference to an external insight that seeded this group, if - any. + parent_evaluation_id: + title: Parent Evaluation Id + description: Entity id of the evaluation this one was derived from (e.g. + a variant of a baseline), if any. type: string - summary: - title: Summary - description: Human- or agent-authored summary of the group's findings. + parent_experiment_id: + title: Parent Experiment Id + description: Deprecated alias for parent_evaluation_id. + deprecated: true type: string - metadata: - title: Metadata - description: Free-form producer metadata for the group. - additionalProperties: - type: string - type: object - default_sort: - type: string - title: Default Sort - description: Default sort for this group's experiments list, as a `sort`-param - string (leading '-' = descending); defaults to '-created_at'. Accepts - any field the experiments list `sort` param does; clients apply it as - the list `sort` param. - default: -created_at - additionalProperties: false - type: object - required: - - name - title: ExperimentGroupRequest - description: Request body for creating an ExperimentGroup. - ExperimentGroupResponse: - properties: - id: - type: string - title: Id - name: - type: string - title: Name - workspace: - type: string - title: Workspace - description: - title: Description - type: string - insight_id: - title: Insight Id - type: string - summary: - title: Summary - type: string - metadata: - title: Metadata - additionalProperties: - type: string - type: object - default_sort: - type: string - title: Default Sort - created_at: - title: Created At - type: string - format: date-time - updated_at: - title: Updated At - type: string - format: date-time - experiment_count: - type: integer - title: Experiment Count - description: Number of live (non-soft-deleted) experiments in this group. - default: 0 - type: object - required: - - id - - name - - workspace - - default_sort - title: ExperimentGroupResponse - description: ExperimentGroup as served by the API. - ExperimentGroupResponsesPage: - properties: - data: - items: - $ref: '#/components/schemas/ExperimentGroupResponse' - type: array - title: Data - pagination: - allOf: - - $ref: '#/components/schemas/PaginationData' - description: Pagination information. - sort: - title: Sort - description: The field on which the results are sorted. - type: string - filter: - title: Filter - description: Filtering information. - additionalProperties: true - type: object - type: object - required: - - data - title: ExperimentGroupResponsesPage - ExperimentRequest: - properties: - name: - type: string - title: Name - description: Producer-supplied, workspace-unique experiment id. - experiment_group_id: - type: string - title: Experiment Group Id - description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ - \ group must already exist." - dataset_name: - type: string - title: Dataset Name - description: Producer-supplied dataset name. - dataset_version: - title: Dataset Version - description: Producer-supplied dataset version. - type: string - source_link: - title: Source Link - description: Optional URL for the source experiment. - type: string - minLength: 1 - format: uri - metadata: - additionalProperties: - type: string - type: object - title: Metadata - description: Free-form producer metadata. - description: - title: Description - description: Human-readable description. - type: string - parent_experiment_id: - title: Parent Experiment Id - description: Entity id of the experiment this one was derived from (e.g. - a variant of a baseline), if any. - type: string - status: - title: Status - description: Producer-defined lifecycle status of the experiment. - type: string - root_cause: - title: Root Cause - description: Human- or agent-authored explanation of the experiment's outcome - (e.g. why it was killed). + status: + title: Status + description: Producer-defined lifecycle status of the evaluation. + type: string + root_cause: + title: Root Cause + description: Human- or agent-authored explanation of the evaluation's outcome + (e.g. why it was killed). type: string additionalProperties: false type: object @@ -10713,9 +10337,9 @@ components: - name - experiment_group_id - dataset_name - title: ExperimentRequest - description: Request body for creating an Experiment. - ExperimentResponse: + title: EvaluationRequest + description: Request body for creating an Evaluation. + EvaluationResponse: properties: id: type: string @@ -10730,7 +10354,7 @@ components: type: string title: Experiment Group Id description: Entity id of the owning ExperimentGroup. Required for every - Experiment. + Evaluation. dataset_name: type: string title: Dataset Name @@ -10750,8 +10374,8 @@ components: description: title: Description type: string - parent_experiment_id: - title: Parent Experiment Id + parent_evaluation_id: + title: Parent Evaluation Id type: string status: title: Status @@ -10769,8 +10393,8 @@ components: format: date-time pinned_at: title: Pinned At - description: Timestamp at which the experiment was pinned, or null if unpinned. - Managed via POST/DELETE /experiments/{name}/pin. + description: Timestamp at which the evaluation was pinned, or null if unpinned. + Managed via POST/DELETE /evaluations/{name}/pin. nullable: true type: string format: date-time @@ -10786,7 +10410,7 @@ components: uniqueItems: true title: Model Names description: Distinct model names observed across ingested sessions for - this experiment. + this evaluation. agent_names: items: type: string @@ -10794,7 +10418,7 @@ components: uniqueItems: true title: Agent Names description: Distinct agent names observed across ingested sessions for - this experiment. + this evaluation. agent_versions: items: type: string @@ -10802,7 +10426,7 @@ components: uniqueItems: true title: Agent Versions description: Distinct agent versions observed across ingested sessions for - this experiment. + this evaluation. aggregate_scores: title: Aggregate Scores additionalProperties: @@ -10811,28 +10435,356 @@ components: run_count: type: integer title: Run Count - description: Number of distinct ingested experiment sessions; one session + description: Number of distinct ingested evaluation sessions; one session is treated as one run. default: 0 cost_usd: $ref: '#/components/schemas/EvaluatorAggregate' latency_ms: $ref: '#/components/schemas/EvaluatorAggregate' + parent_experiment_id: + title: Parent Experiment Id + description: Deprecated alias for parent_evaluation_id. + deprecated: true + readOnly: true + type: string + type: object + required: + - id + - name + - workspace + - experiment_group_id + - dataset_name + - parent_experiment_id + title: EvaluationResponse + description: Evaluation as served by the API, including ClickHouse-hydrated + rollups. + EvaluationResponsesPage: + properties: + data: + items: + $ref: '#/components/schemas/EvaluationResponse' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: EvaluationResponsesPage + EvaluationSessionFilter: + additionalProperties: false + description: Filter for listing EvaluationSessions. + properties: + test_case_id: + description: Filter by producer-supplied test case id. + title: Test Case Id + type: string + status: + description: Filter by root-span status (success, error, cancelled, unknown). + title: Status + type: string + title: EvaluationSessionFilter + type: object + EvaluationSessionResponse: + properties: + workspace: + type: string + title: Workspace + evaluation_name: + type: string + title: Evaluation Name + session_id: + type: string + title: Session Id + test_case_id: + title: Test Case Id + description: Producer-supplied test case identifier; null when the producer + did not set one. + type: string + trace_id: + type: string + title: Trace Id + root_span_id: + type: string + title: Root Span Id + started_at: + type: string + format: date-time + title: Started At + ended_at: + title: Ended At + type: string + format: date-time + latency_ms: + title: Latency Ms + type: number + status: + allOf: + - $ref: '#/components/schemas/SpanStatus' + description: 'Root-span status: success, error, cancelled, or unknown.' + input: + title: Input + description: Root-span input text. In summary mode this is truncated to + 1000 characters. + type: string + input_tokens: + title: Input Tokens + description: Sum of input tokens across this session's spans. + type: integer + output_tokens: + title: Output Tokens + description: Sum of output tokens across this session's spans. + type: integer + cached_tokens: + title: Cached Tokens + description: Sum of cached tokens across this session's spans. + type: integer + cost_total_usd: + title: Cost Total Usd + description: Sum of cost across this session's spans. + type: number + evaluator_scores: + additionalProperties: + type: number + type: object + title: Evaluator Scores + description: Per-evaluator session-mean score. Includes NUMERIC and BOOLEAN + evaluator results only; text/categorical results are omitted. + experiment_name: + type: string + title: Experiment Name + description: Deprecated alias for evaluation_name. + deprecated: true + readOnly: true + type: object + required: + - workspace + - evaluation_name + - session_id + - trace_id + - root_span_id + - started_at + - status + - experiment_name + title: EvaluationSessionResponse + description: "One ingested session of an Evaluation \u2014 a single test case\ + \ execution.\n\nHydrated from ClickHouse at read time by reading root/session\ + \ membership from\n``trace_index`` and joining page-bounded span/evaluator\ + \ rollups." + EvaluationSessionResponsesPage: + properties: + data: + items: + $ref: '#/components/schemas/EvaluationSessionResponse' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: EvaluationSessionResponsesPage + EvaluatorAggregate: + properties: + sum: + title: Sum + type: number + mean: + title: Mean + type: number + median: + title: Median + type: number + p90: + title: P90 + type: number + p95: + title: P95 + type: number + p99: + title: P99 + type: number + count: + type: integer + title: Count + default: 0 + type: object + title: EvaluatorAggregate + description: Aggregate statistics over evaluator scores or session-level metric + values. + EvaluatorResult: + properties: + evaluator_result_id: + type: string + title: Evaluator Result Id + span_id: + type: string + title: Span Id + session_id: + type: string + title: Session Id + workspace: + type: string + title: Workspace + name: + type: string + title: Name + value: + title: Value + type: number + string_value: + title: String Value + type: string + data_type: + $ref: '#/components/schemas/EvaluatorResultDataType' + comment: + title: Comment + type: string + created_by: + title: Created By + type: string + created_at: + type: string + format: date-time + title: Created At + ingested_at: + type: string + format: date-time + title: Ingested At + type: object + required: + - evaluator_result_id + - span_id + - session_id + - workspace + - name + - data_type + - created_at + - ingested_at + title: EvaluatorResult + description: Response model for evaluator_results read endpoints. + EvaluatorResultDataType: + type: string + enum: + - NUMERIC + - CATEGORICAL + - BOOLEAN + - TEXT + title: EvaluatorResultDataType + EvaluatorResultFilter: + properties: + span_id: + description: Filter by target span id. + title: Span Id + type: string + session_id: + description: Filter by target session id. + title: Session Id + type: string + name: + description: Filter by evaluator/metric name. + title: Name + type: string + data_type: + allOf: + - $ref: '#/components/schemas/EvaluatorResultDataType' + description: Filter by data_type. + created_by: + description: Filter by principal/system that wrote the row. + title: Created By + type: string + value: + allOf: + - $ref: '#/components/schemas/FloatFilter' + description: Filter by numeric value (range supported). + created_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Filter by row creation time (range supported). + title: EvaluatorResultFilter + type: object + EvaluatorResultInput: + properties: + span_id: + type: string + title: Span Id + description: Target span id. Not validated against existing spans (loose + target policy). + session_id: + type: string + title: Session Id + description: Session id the target span belongs to. Denormalized so session-scoped + reads stay fast. + name: + type: string + title: Name + description: Evaluator / metric identity (e.g. 'faithfulness/v1'). + value: + title: Value + description: Numeric value. Required when data_type is NUMERIC or BOOLEAN + (0|1). + type: number + string_value: + title: String Value + description: String value. Required when data_type is CATEGORICAL or TEXT. + type: string + data_type: + allOf: + - $ref: '#/components/schemas/EvaluatorResultDataType' + description: Discriminator for which of value / string_value carries the + payload. + comment: + title: Comment + description: Free-text rationale or explanation. + type: string + additionalProperties: false type: object required: - - id + - span_id + - session_id - name - - workspace - - experiment_group_id - - dataset_name - title: ExperimentResponse - description: Experiment as served by the API, including ClickHouse-hydrated - rollups. - ExperimentResponsesPage: + - data_type + title: EvaluatorResultInput + description: "Request body for POST /evaluator-results.\n\nServer fills in `evaluator_result_id`,\ + \ `created_at`, `ingested_at`, and\n`created_by`. Producer supplies the target\ + \ span (loose target \u2014 not\nvalidated against the spans table), the score,\ + \ and provenance." + EvaluatorResultSortField: + type: string + enum: + - created_at + - -created_at + - value + - -value + title: EvaluatorResultSortField + EvaluatorResultsPage: properties: data: items: - $ref: '#/components/schemas/ExperimentResponse' + $ref: '#/components/schemas/EvaluatorResult' type: array title: Data pagination: @@ -10851,105 +10803,182 @@ components: type: object required: - data - title: ExperimentResponsesPage - ExperimentSessionFilter: - additionalProperties: false - description: Filter for listing ExperimentSessions. + title: EvaluatorResultsPage + ExecutedAction: + properties: + action_name: + type: string + title: Action Name + description: The name of the action that was executed. + action_params: + additionalProperties: true + type: object + title: Action Params + description: The parameters for the action. + return_value: + title: Return Value + description: The value returned by the action. + llm_calls: + items: + $ref: '#/components/schemas/LLMCallInfo' + type: array + title: Llm Calls + description: Information about the LLM calls made by the action. + started_at: + title: Started At + description: Timestamp for when the action started. + type: number + finished_at: + title: Finished At + description: Timestamp for when the action finished. + type: number + duration: + title: Duration + description: How long the action took to execute, in seconds. + type: number + type: object + required: + - action_name + title: ExecutedAction + description: Information about an action that was executed. + ExperimentContext: properties: + experiment_id: + type: string + title: Experiment Id + description: Name of an existing Experiment entity. test_case_id: - description: Filter by producer-supplied test case id. title: Test Case Id + description: Optional producer-supplied test case id. type: string - status: - description: Filter by root-span status (success, error, cancelled, unknown). - title: Status + additionalProperties: false + type: object + required: + - experiment_id + title: ExperimentContext + description: Deprecated alias for :class:`EvaluationContext`. Producers should + send ``evaluation_context``. + ExperimentGroupFilter: + additionalProperties: false + description: Filter for listing ExperimentGroups. + properties: + name: + description: Filter groups by name. + title: Name + type: string + is_deleted: + description: When true, returns only soft-deleted groups. Omit (or false) + to see only live groups. + title: Is Deleted + type: boolean + metadata: + description: Filter by a metadata key/value pair, e.g. filter[metadata.model]=claude-opus-4-8. + title: Metadata + additionalProperties: + type: string + type: object + title: ExperimentGroupFilter + type: object + ExperimentGroupRequest: + properties: + name: + type: string + title: Name + description: Workspace-unique group name. + description: + title: Description + description: Human-readable purpose of the group. + type: string + insight_id: + title: Insight Id + description: Reference to an external insight that seeded this group, if + any. + type: string + summary: + title: Summary + description: Human- or agent-authored summary of the group's findings. + type: string + metadata: + title: Metadata + description: Free-form producer metadata for the group. + additionalProperties: + type: string + type: object + default_sort: type: string - title: ExperimentSessionFilter + title: Default Sort + description: Default sort for this group's evaluations list, as a `sort`-param + string (leading '-' = descending); defaults to '-created_at'. Accepts + any field the evaluations list `sort` param does; clients apply it as + the list `sort` param. + default: -created_at + additionalProperties: false type: object - ExperimentSessionResponse: + required: + - name + title: ExperimentGroupRequest + description: Request body for creating an ExperimentGroup. + ExperimentGroupResponse: properties: + id: + type: string + title: Id + name: + type: string + title: Name workspace: type: string title: Workspace - experiment_name: - type: string - title: Experiment Name - session_id: + description: + title: Description type: string - title: Session Id - test_case_id: - title: Test Case Id - description: Producer-supplied test case identifier; null when the producer - did not set one. + insight_id: + title: Insight Id type: string - trace_id: + summary: + title: Summary type: string - title: Trace Id - root_span_id: + metadata: + title: Metadata + additionalProperties: + type: string + type: object + default_sort: type: string - title: Root Span Id - started_at: + title: Default Sort + created_at: + title: Created At type: string format: date-time - title: Started At - ended_at: - title: Ended At + updated_at: + title: Updated At type: string format: date-time - latency_ms: - title: Latency Ms - type: number - status: - allOf: - - $ref: '#/components/schemas/SpanStatus' - description: 'Root-span status: success, error, cancelled, or unknown.' - input: - title: Input - description: Root-span input text. In summary mode this is truncated to - 1000 characters. - type: string - input_tokens: - title: Input Tokens - description: Sum of input tokens across this session's spans. - type: integer - output_tokens: - title: Output Tokens - description: Sum of output tokens across this session's spans. + evaluation_count: type: integer - cached_tokens: - title: Cached Tokens - description: Sum of cached tokens across this session's spans. + title: Evaluation Count + description: Number of live (non-soft-deleted) evaluations in this group. + default: 0 + experiment_count: type: integer - cost_total_usd: - title: Cost Total Usd - description: Sum of cost across this session's spans. - type: number - evaluator_scores: - additionalProperties: - type: number - type: object - title: Evaluator Scores - description: Per-evaluator session-mean score. Includes NUMERIC and BOOLEAN - evaluator results only; text/categorical results are omitted. + title: Experiment Count + description: Deprecated alias for evaluation_count. + deprecated: true + readOnly: true type: object required: + - id + - name - workspace - - experiment_name - - session_id - - trace_id - - root_span_id - - started_at - - status - title: ExperimentSessionResponse - description: "One ingested session of an Experiment \u2014 a single test case\ - \ execution.\n\nHydrated from ClickHouse at read time by reading root/session\ - \ membership from\n``trace_index`` and joining page-bounded span/evaluator\ - \ rollups." - ExperimentSessionResponsesPage: + - default_sort + - experiment_count + title: ExperimentGroupResponse + description: ExperimentGroup as served by the API. + ExperimentGroupResponsesPage: properties: data: items: - $ref: '#/components/schemas/ExperimentSessionResponse' + $ref: '#/components/schemas/ExperimentGroupResponse' type: array title: Data pagination: @@ -10968,7 +10997,7 @@ components: type: object required: - data - title: ExperimentSessionResponsesPage + title: ExperimentGroupResponsesPage FactCheckingRailConfig: properties: parameters: @@ -13248,7 +13277,7 @@ components: These stats must stay in sync with the runtime sort/filter grammar (``_METRIC_STATS`` in the - experiments + evaluations endpoints); a unit test guards the parity.' properties: @@ -17875,8 +17904,14 @@ components: name: title: Name type: string + evaluation_context: + $ref: '#/components/schemas/EvaluationContext' experiment_context: - $ref: '#/components/schemas/ExperimentContext' + allOf: + - $ref: '#/components/schemas/ExperimentContext' + description: Deprecated alias for evaluation_context; will be removed in + a future release. + deprecated: true started_at: type: string format: date-time @@ -17949,12 +17984,18 @@ components: allOf: - $ref: '#/components/schemas/DatetimeFilter' description: Filter by root span start timestamp. + evaluation_id: + description: Filter by root-span evaluation id. + title: Evaluation Id + type: string experiment_id: - description: Filter by root-span experiment id. + deprecated: true + description: Deprecated alias for evaluation_id. Filter by root-span evaluation + id. title: Experiment Id type: string test_case_id: - description: Filter by root-span experiment test case id. + description: Filter by root-span evaluation test case id. title: Test Case Id type: string title: TraceFilter diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/traces.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/traces.py index faf924b560..38192afea8 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/traces.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/traces.py @@ -36,11 +36,14 @@ def list_traces( typer.Option( "--filter", metavar="FILTER_JSON", - help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter root-span-backed traces by id, session_id, root status, root span started_at, experiment_id, and test_case_id.", + help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter root-span-backed traces by id, session_id, root status, root span started_at, evaluation_id (or its deprecated alias experiment_id), and test_case_id.", rich_help_panel="Filter Options", ), ] = None, filter_id: Annotated[str | None, typer.Option("--filter.id", rich_help_panel="Filter Options")] = None, + filter_evaluation_id: Annotated[ + str | None, typer.Option("--filter.evaluation-id", rich_help_panel="Filter Options") + ] = None, filter_experiment_id: Annotated[ str | None, typer.Option("--filter.experiment-id", rich_help_panel="Filter Options") ] = None, @@ -85,6 +88,7 @@ def list_traces( filter=merge_filter_dict( filter, id=filter_id, + evaluation_id=filter_evaluation_id, experiment_id=filter_experiment_id, session_id=filter_session_id, status=filter_status, diff --git a/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py b/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py index c156b29737..ec82993ef8 100644 --- a/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py +++ b/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py @@ -178,7 +178,7 @@ async def test_publish_to_intake_round_trip(platform_base_url: str) -> None: group = await client.experiment_groups.create( workspace=WORKSPACE, name=GROUP_NAME, description="Intake IT", exist_ok=True ) - await client.experiments.create( + await client.evaluations.create( workspace=WORKSPACE, name=EXPERIMENT_NAME, experiment_group_id=group.id, @@ -275,7 +275,7 @@ async def test_publish_skips_nan_and_failed_scores(platform_base_url: str) -> No # should reach Intake. Only the finite, completed output should be stored. async with AsyncNeMoPlatform(base_url=platform_base_url, max_retries=2) as client: group = await client.experiment_groups.create(workspace=WORKSPACE, name=GROUP_NAME, exist_ok=True) - await client.experiments.create( + await client.evaluations.create( workspace=WORKSPACE, name=NAN_EXPERIMENT_NAME, experiment_group_id=group.id, diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index bc0343b03d..fd1e06dbb6 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -3327,12 +3327,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/evaluator-results: + /apis/intake/v2/workspaces/{workspace}/evaluations: post: tags: - - Evaluator Results - summary: Create Evaluator Result - operationId: create_evaluator_result_apis_intake_v2_workspaces__workspace__evaluator_results_post + - Evaluations + summary: Create Evaluation + operationId: create_evaluation_apis_intake_v2_workspaces__workspace__evaluations_post parameters: - name: workspace in: path @@ -3345,14 +3345,16 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/EvaluatorResultInput' + $ref: '#/components/schemas/EvaluationRequest' responses: '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/EvaluatorResult' + $ref: '#/components/schemas/EvaluationResponse' + '409': + description: Evaluation already exists '422': description: Validation Error content: @@ -3361,9 +3363,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' get: tags: - - Evaluator Results - summary: List Evaluator Results - operationId: list_evaluator_results_apis_intake_v2_workspaces__workspace__evaluator_results_get + - Evaluations + summary: List Evaluations + operationId: list_evaluations_apis_intake_v2_workspaces__workspace__evaluations_get parameters: - name: workspace in: path @@ -3389,44 +3391,65 @@ paths: maximum: 1000 minimum: 1 description: Page size. - default: 10 + default: 100 title: Page Size description: Page size. - name: sort in: query required: false schema: - allOf: - - $ref: '#/components/schemas/EvaluatorResultSortField' - default: -created_at + description: 'Field to sort by; prefix with ''-'' for descending. Sort by + an evaluation attribute (name, created_at, updated_at, pinned_at) or by + an aggregate metric: run_count, cost_usd., latency_ms., or + evaluators.., where is one of mean, median, p90, p95, + p99, sum, count. When omitted, defaults to -created_at with pinned evaluations + first.' + title: Sort + type: string + description: 'Field to sort by; prefix with ''-'' for descending. Sort by + an evaluation attribute (name, created_at, updated_at, pinned_at) or by + an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., + where is one of mean, median, p90, p95, p99, sum, count. When omitted, + defaults to -created_at with pinned evaluations first.' - in: query name: filter style: deepObject required: false explode: true schema: - $ref: '#/components/schemas/EvaluatorResultFilter' - description: Filter evaluator results by span_id, session_id, name, data_type, - created_by, value range, and created_at range. + $ref: '#/components/schemas/EvaluationFilter' + description: 'Filter evaluations by name, experiment_group_id, dataset_name, + dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true + to return only soft-deleted evaluations; omit to see only live ones. Pass + is_pinned=true (or false) to filter by pinned state; omit to return both. + Filter by a metadata key/value: filter[metadata.]=. Filter by + a rollup metric with numeric range operators ($gte/$lte/$gt/$lt/$eq): filter[run_count][$gte]=5, + filter[cost_usd.mean][$lte]=0.5, filter[latency_ms.p95][$lte]=1000, or filter[evaluators..mean][$gte]=0.8.' responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/EvaluatorResultsPage' + $ref: '#/components/schemas/EvaluationResponsesPage' + '400': + description: Unsupported sort or filter field + '413': + description: Too many evaluations selected to sort in one request + '503': + description: Telemetry store unavailable for a metric-based sort or filter '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/evaluator-results/{evaluator_result_id}: + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}: get: tags: - - Evaluator Results - summary: Get Evaluator Result - operationId: get_evaluator_result_apis_intake_v2_workspaces__workspace__evaluator_results__evaluator_result_id__get + - Evaluations + summary: Get Evaluation + operationId: get_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__get parameters: - name: workspace in: path @@ -3434,31 +3457,32 @@ paths: schema: type: string title: Workspace - - name: evaluator_result_id + - name: name in: path required: true schema: type: string - title: Evaluator Result Id + title: Name responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/EvaluatorResult' + $ref: '#/components/schemas/EvaluationResponse' + '404': + description: Evaluation not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups: - post: + put: tags: - - Experiment Groups - summary: Create Experiment Group - operationId: create_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups_post + - Evaluations + summary: Update Evaluation + operationId: update_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__put parameters: - name: workspace in: path @@ -3466,32 +3490,40 @@ paths: schema: type: string title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name requestBody: required: true content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupRequest' + $ref: '#/components/schemas/EvaluationRequest' responses: - '201': + '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/EvaluationResponse' + '404': + description: Evaluation not found '409': - description: Experiment group already exists + description: Attempt to change an immutable field '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - get: + delete: tags: - - Experiment Groups - summary: List Experiment Groups - operationId: list_experiment_groups_apis_intake_v2_workspaces__workspace__experiment_groups_get + - Evaluations + summary: Delete Evaluation + operationId: delete_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__delete parameters: - name: workspace in: path @@ -3499,71 +3531,36 @@ paths: schema: type: string title: Workspace - - name: page - in: query - required: false - schema: - type: integer - minimum: 1 - description: Page number. - default: 1 - title: Page - description: Page number. - - name: page_size - in: query - required: false - schema: - type: integer - maximum: 1000 - minimum: 1 - description: Page size. - default: 100 - title: Page Size - description: Page size. - - name: sort - in: query - required: false + - name: name + in: path + required: true schema: - enum: - - -created_at - - created_at - - -updated_at - - updated_at - - -name - - name type: string - description: Sort field; prefix with '-' for descending. - default: -created_at - title: Sort - description: Sort field; prefix with '-' for descending. - - in: query - name: filter - style: deepObject - required: false - explode: true - schema: - $ref: '#/components/schemas/ExperimentGroupFilter' - description: 'Filter experiment groups by name, or by a metadata key/value: - filter[metadata.]=.' + title: Name responses: - '200': + '204': description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/ExperimentGroupResponsesPage' + '404': + description: Evaluation not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: - get: + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin: + post: tags: - - Experiment Groups - summary: Get Experiment Group - operationId: get_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__get + - Evaluations + summary: Pin Evaluation + description: 'Pin an evaluation to the top of the list (workspace-shared). + + + Re-pinning an already-pinned evaluation refreshes ``pinned_at`` to the current + timestamp, + + which is intentional (most-recently-pinned sorts first).' + operationId: pin_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__pin_post parameters: - name: workspace in: path @@ -3583,20 +3580,22 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/EvaluationResponse' '404': - description: Experiment group not found + description: Evaluation not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - put: + delete: tags: - - Experiment Groups - summary: Update Experiment Group - operationId: update_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__put + - Evaluations + summary: Unpin Evaluation + description: 'Unpin an evaluation. Idempotent: unpinning an already-unpinned + evaluation is a no-op.' + operationId: unpin_evaluation_apis_intake_v2_workspaces__workspace__evaluations__name__pin_delete parameters: - name: workspace in: path @@ -3610,34 +3609,27 @@ paths: schema: type: string title: Name - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/ExperimentGroupRequest' responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentGroupResponse' + $ref: '#/components/schemas/EvaluationResponse' '404': - description: Experiment group not found - '409': - description: Attempt to rename the group + description: Evaluation not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - delete: + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/sessions: + get: tags: - - Experiment Groups - summary: Delete Experiment Group - operationId: delete_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__delete + - Evaluations + summary: List Evaluation Sessions + operationId: list_evaluation_sessions_apis_intake_v2_workspaces__workspace__evaluations__name__sessions_get parameters: - name: workspace in: path @@ -3651,23 +3643,76 @@ paths: schema: type: string title: Name + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: mode + in: query + required: false + schema: + enum: + - summary + - detailed + type: string + description: Response payload mode. summary keeps the same session row fields + but truncates root-span input to 1000 characters; detailed returns the + full root-span input. + default: detailed + title: Mode + description: Response payload mode. summary keeps the same session row fields + but truncates root-span input to 1000 characters; detailed returns the full + root-span input. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/EvaluationSessionFilter' + description: Filter sessions by test_case_id and status. responses: - '204': + '200': description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationSessionResponsesPage' + '400': + description: Invalid filter value '404': - description: Experiment group not found + description: Evaluation not found + '503': + description: ClickHouse unavailable '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiments: + /apis/intake/v2/workspaces/{workspace}/evaluator-results: post: tags: - - Experiments - summary: Create Experiment - operationId: create_experiment_apis_intake_v2_workspaces__workspace__experiments_post + - Evaluator Results + summary: Create Evaluator Result + operationId: create_evaluator_result_apis_intake_v2_workspaces__workspace__evaluator_results_post parameters: - name: workspace in: path @@ -3680,16 +3725,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentRequest' + $ref: '#/components/schemas/EvaluatorResultInput' responses: '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' - '409': - description: Experiment already exists + $ref: '#/components/schemas/EvaluatorResult' '422': description: Validation Error content: @@ -3698,9 +3741,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' get: tags: - - Experiments - summary: List Experiments - operationId: list_experiments_apis_intake_v2_workspaces__workspace__experiments_get + - Evaluator Results + summary: List Evaluator Results + operationId: list_evaluator_results_apis_intake_v2_workspaces__workspace__evaluator_results_get parameters: - name: workspace in: path @@ -3726,65 +3769,44 @@ paths: maximum: 1000 minimum: 1 description: Page size. - default: 100 + default: 10 title: Page Size description: Page size. - name: sort in: query required: false schema: - description: 'Field to sort by; prefix with ''-'' for descending. Sort by - an experiment attribute (name, created_at, updated_at, pinned_at) or by - an aggregate metric: run_count, cost_usd., latency_ms., or - evaluators.., where is one of mean, median, p90, p95, - p99, sum, count. When omitted, defaults to -created_at with pinned experiments - first.' - title: Sort - type: string - description: 'Field to sort by; prefix with ''-'' for descending. Sort by - an experiment attribute (name, created_at, updated_at, pinned_at) or by - an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., - where is one of mean, median, p90, p95, p99, sum, count. When omitted, - defaults to -created_at with pinned experiments first.' + allOf: + - $ref: '#/components/schemas/EvaluatorResultSortField' + default: -created_at - in: query name: filter style: deepObject required: false explode: true schema: - $ref: '#/components/schemas/ExperimentFilter' - description: 'Filter experiments by name, experiment_group_id, dataset_name, - dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true - to return only soft-deleted experiments; omit to see only live ones. Pass - is_pinned=true (or false) to filter by pinned state; omit to return both. - Filter by a metadata key/value: filter[metadata.]=. Filter by - a rollup metric with numeric range operators ($gte/$lte/$gt/$lt/$eq): filter[run_count][$gte]=5, - filter[cost_usd.mean][$lte]=0.5, filter[latency_ms.p95][$lte]=1000, or filter[evaluators..mean][$gte]=0.8.' + $ref: '#/components/schemas/EvaluatorResultFilter' + description: Filter evaluator results by span_id, session_id, name, data_type, + created_by, value range, and created_at range. responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponsesPage' - '400': - description: Unsupported sort or filter field - '413': - description: Too many experiments selected to sort in one request - '503': - description: Telemetry store unavailable for a metric-based sort or filter + $ref: '#/components/schemas/EvaluatorResultsPage' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiments/{name}: + /apis/intake/v2/workspaces/{workspace}/evaluator-results/{evaluator_result_id}: get: tags: - - Experiments - summary: Get Experiment - operationId: get_experiment_apis_intake_v2_workspaces__workspace__experiments__name__get + - Evaluator Results + summary: Get Evaluator Result + operationId: get_evaluator_result_apis_intake_v2_workspaces__workspace__evaluator_results__evaluator_result_id__get parameters: - name: workspace in: path @@ -3792,32 +3814,31 @@ paths: schema: type: string title: Workspace - - name: name + - name: evaluator_result_id in: path required: true schema: type: string - title: Name + title: Evaluator Result Id responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' - '404': - description: Experiment not found + $ref: '#/components/schemas/EvaluatorResult' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - put: + /apis/intake/v2/workspaces/{workspace}/experiment-groups: + post: tags: - - Experiments - summary: Update Experiment - operationId: update_experiment_apis_intake_v2_workspaces__workspace__experiments__name__put + - Experiment Groups + summary: Create Experiment Group + operationId: create_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups_post parameters: - name: workspace in: path @@ -3825,40 +3846,32 @@ paths: schema: type: string title: Workspace - - name: name - in: path - required: true - schema: - type: string - title: Name requestBody: required: true content: application/json: schema: - $ref: '#/components/schemas/ExperimentRequest' + $ref: '#/components/schemas/ExperimentGroupRequest' responses: - '200': + '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' - '404': - description: Experiment not found + $ref: '#/components/schemas/ExperimentGroupResponse' '409': - description: Attempt to change an immutable field + description: Experiment group already exists '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - delete: + get: tags: - - Experiments - summary: Delete Experiment - operationId: delete_experiment_apis_intake_v2_workspaces__workspace__experiments__name__delete + - Experiment Groups + summary: List Experiment Groups + operationId: list_experiment_groups_apis_intake_v2_workspaces__workspace__experiment_groups_get parameters: - name: workspace in: path @@ -3866,36 +3879,71 @@ paths: schema: type: string title: Workspace - - name: name - in: path - required: true + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 100 + title: Page Size + description: Page size. + - name: sort + in: query + required: false schema: + enum: + - -created_at + - created_at + - -updated_at + - updated_at + - -name + - name type: string - title: Name + description: Sort field; prefix with '-' for descending. + default: -created_at + title: Sort + description: Sort field; prefix with '-' for descending. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/ExperimentGroupFilter' + description: 'Filter experiment groups by name, or by a metadata key/value: + filter[metadata.]=.' responses: - '204': + '200': description: Successful Response - '404': - description: Experiment not found + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupResponsesPage' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin: - post: + /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: + get: tags: - - Experiments - summary: Pin Experiment - description: 'Pin an experiment to the top of the list (workspace-shared). - - - Re-pinning an already-pinned experiment refreshes ``pinned_at`` to the current - timestamp, - - which is intentional (most-recently-pinned sorts first).' - operationId: pin_experiment_apis_intake_v2_workspaces__workspace__experiments__name__pin_post + - Experiment Groups + summary: Get Experiment Group + operationId: get_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__get parameters: - name: workspace in: path @@ -3915,22 +3963,20 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' + $ref: '#/components/schemas/ExperimentGroupResponse' '404': - description: Experiment not found + description: Experiment group not found '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - delete: + put: tags: - - Experiments - summary: Unpin Experiment - description: 'Unpin an experiment. Idempotent: unpinning an already-unpinned - experiment is a no-op.' - operationId: unpin_experiment_apis_intake_v2_workspaces__workspace__experiments__name__pin_delete + - Experiment Groups + summary: Update Experiment Group + operationId: update_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__put parameters: - name: workspace in: path @@ -3944,27 +3990,34 @@ paths: schema: type: string title: Name + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentGroupRequest' responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/ExperimentResponse' + $ref: '#/components/schemas/ExperimentGroupResponse' '404': - description: Experiment not found + description: Experiment group not found + '409': + description: Attempt to rename the group '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiments/{name}/sessions: - get: + delete: tags: - - Experiments - summary: List Experiment Sessions - operationId: list_experiment_sessions_apis_intake_v2_workspaces__workspace__experiments__name__sessions_get + - Experiment Groups + summary: Delete Experiment Group + operationId: delete_experiment_group_apis_intake_v2_workspaces__workspace__experiment_groups__name__delete parameters: - name: workspace in: path @@ -3978,62 +4031,11 @@ paths: schema: type: string title: Name - - name: page - in: query - required: false - schema: - type: integer - minimum: 1 - description: Page number. - default: 1 - title: Page - description: Page number. - - name: page_size - in: query - required: false - schema: - type: integer - maximum: 1000 - minimum: 1 - description: Page size. - default: 100 - title: Page Size - description: Page size. - - name: mode - in: query - required: false - schema: - enum: - - summary - - detailed - type: string - description: Response payload mode. summary keeps the same session row fields - but truncates root-span input to 1000 characters; detailed returns the - full root-span input. - default: detailed - title: Mode - description: Response payload mode. summary keeps the same session row fields - but truncates root-span input to 1000 characters; detailed returns the full - root-span input. - - in: query - name: filter - style: deepObject - required: false - explode: true - schema: - $ref: '#/components/schemas/ExperimentSessionFilter' - description: Filter sessions by test_case_id and status. responses: - '200': + '204': description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/ExperimentSessionResponsesPage' '404': - description: Experiment not found - '503': - description: ClickHouse unavailable + description: Experiment group not found '422': description: Validation Error content: @@ -4426,7 +4428,8 @@ paths: schema: $ref: '#/components/schemas/TraceFilter' description: Filter root-span-backed traces by id, session_id, root status, - root span started_at, experiment_id, and test_case_id. + root span started_at, evaluation_id (or its deprecated alias experiment_id), + and test_case_id. responses: '200': description: Successful Response @@ -10188,7 +10191,7 @@ components: properties: evaluation_id: title: Evaluation Id - description: Name of an existing Experiment entity. + description: Name of an existing Evaluation. type: string test_case_id: title: Test Case Id @@ -10204,299 +10207,48 @@ components: evaluation_run_id, metadata) keeps ingesting without error rather than being rejected.' - EvaluatorAggregate: - properties: - sum: - title: Sum - type: number - mean: - title: Mean - type: number - median: - title: Median - type: number - p90: - title: P90 - type: number - p95: - title: P95 - type: number - p99: - title: P99 - type: number - count: - type: integer - title: Count - default: 0 - type: object - title: EvaluatorAggregate - description: Aggregate statistics over evaluator scores or session-level metric - values. - EvaluatorResult: + EvaluationFilter: + additionalProperties: false + description: Filter for listing Evaluations. properties: - evaluator_result_id: - type: string - title: Evaluator Result Id - span_id: - type: string - title: Span Id - session_id: - type: string - title: Session Id - workspace: - type: string - title: Workspace name: - type: string + description: Filter evaluations by name. title: Name - value: - title: Value - type: number - string_value: - title: String Value type: string - data_type: - $ref: '#/components/schemas/EvaluatorResultDataType' - comment: - title: Comment + experiment_group_id: + description: Filter evaluations by owning group id. + title: Experiment Group Id + type: string + dataset_name: + description: Filter evaluations by dataset name. + title: Dataset Name + type: string + dataset_version: + description: Filter evaluations by dataset version. + title: Dataset Version type: string created_by: - title: Created By - type: string - created_at: - type: string - format: date-time - title: Created At - ingested_at: - type: string - format: date-time - title: Ingested At - type: object - required: - - evaluator_result_id - - span_id - - session_id - - workspace - - name - - data_type - - created_at - - ingested_at - title: EvaluatorResult - description: Response model for evaluator_results read endpoints. - EvaluatorResultDataType: - type: string - enum: - - NUMERIC - - CATEGORICAL - - BOOLEAN - - TEXT - title: EvaluatorResultDataType - EvaluatorResultFilter: - properties: - span_id: - description: Filter by target span id. - title: Span Id - type: string - session_id: - description: Filter by target session id. - title: Session Id - type: string - name: - description: Filter by evaluator/metric name. - title: Name - type: string - data_type: - allOf: - - $ref: '#/components/schemas/EvaluatorResultDataType' - description: Filter by data_type. - created_by: - description: Filter by principal/system that wrote the row. - title: Created By - type: string - value: - allOf: - - $ref: '#/components/schemas/FloatFilter' - description: Filter by numeric value (range supported). - created_at: - allOf: - - $ref: '#/components/schemas/DatetimeFilter' - description: Filter by row creation time (range supported). - title: EvaluatorResultFilter - type: object - EvaluatorResultInput: - properties: - span_id: - type: string - title: Span Id - description: Target span id. Not validated against existing spans (loose - target policy). - session_id: - type: string - title: Session Id - description: Session id the target span belongs to. Denormalized so session-scoped - reads stay fast. - name: - type: string - title: Name - description: Evaluator / metric identity (e.g. 'faithfulness/v1'). - value: - title: Value - description: Numeric value. Required when data_type is NUMERIC or BOOLEAN - (0|1). - type: number - string_value: - title: String Value - description: String value. Required when data_type is CATEGORICAL or TEXT. - type: string - data_type: - allOf: - - $ref: '#/components/schemas/EvaluatorResultDataType' - description: Discriminator for which of value / string_value carries the - payload. - comment: - title: Comment - description: Free-text rationale or explanation. - type: string - additionalProperties: false - type: object - required: - - span_id - - session_id - - name - - data_type - title: EvaluatorResultInput - description: "Request body for POST /evaluator-results.\n\nServer fills in `evaluator_result_id`,\ - \ `created_at`, `ingested_at`, and\n`created_by`. Producer supplies the target\ - \ span (loose target \u2014 not\nvalidated against the spans table), the score,\ - \ and provenance." - EvaluatorResultSortField: - type: string - enum: - - created_at - - -created_at - - value - - -value - title: EvaluatorResultSortField - EvaluatorResultsPage: - properties: - data: - items: - $ref: '#/components/schemas/EvaluatorResult' - type: array - title: Data - pagination: - allOf: - - $ref: '#/components/schemas/PaginationData' - description: Pagination information. - sort: - title: Sort - description: The field on which the results are sorted. - type: string - filter: - title: Filter - description: Filtering information. - additionalProperties: true - type: object - type: object - required: - - data - title: EvaluatorResultsPage - ExecutedAction: - properties: - action_name: - type: string - title: Action Name - description: The name of the action that was executed. - action_params: - additionalProperties: true - type: object - title: Action Params - description: The parameters for the action. - return_value: - title: Return Value - description: The value returned by the action. - llm_calls: - items: - $ref: '#/components/schemas/LLMCallInfo' - type: array - title: Llm Calls - description: Information about the LLM calls made by the action. - started_at: - title: Started At - description: Timestamp for when the action started. - type: number - finished_at: - title: Finished At - description: Timestamp for when the action finished. - type: number - duration: - title: Duration - description: How long the action took to execute, in seconds. - type: number - type: object - required: - - action_name - title: ExecutedAction - description: Information about an action that was executed. - ExperimentContext: - properties: - experiment_id: - type: string - title: Experiment Id - description: Name of an existing Experiment entity. - test_case_id: - title: Test Case Id - description: Optional producer-supplied test case id. - type: string - additionalProperties: false - type: object - required: - - experiment_id - title: ExperimentContext - description: Deprecated alias for :class:`EvaluationContext`. Producers should - send ``evaluation_context``. - ExperimentFilter: - additionalProperties: false - description: Filter for listing Experiments. - properties: - name: - description: Filter experiments by name. - title: Name - type: string - experiment_group_id: - description: Filter experiments by owning group id. - title: Experiment Group Id - type: string - dataset_name: - description: Filter experiments by dataset name. - title: Dataset Name - type: string - dataset_version: - description: Filter experiments by dataset version. - title: Dataset Version - type: string - created_by: - description: Filter experiments by the principal that created them. + description: Filter evaluations by the principal that created them. title: Created By type: string created_at: allOf: - $ref: '#/components/schemas/DatetimeFilter' - description: Filter experiments by creation timestamp; supports `$gte` and + description: Filter evaluations by creation timestamp; supports `$gte` and `$lte` for ranges. updated_at: allOf: - $ref: '#/components/schemas/DatetimeFilter' - description: Filter experiments by last-updated timestamp; supports `$gte` + description: Filter evaluations by last-updated timestamp; supports `$gte` and `$lte` for ranges. is_deleted: - description: When true, returns only soft-deleted experiments. Omit (or - false) to see only live experiments. + description: When true, returns only soft-deleted evaluations. Omit (or + false) to see only live evaluations. title: Is Deleted type: boolean is_pinned: - description: When true, returns only pinned experiments. When false, returns - only unpinned experiments. Omit to return both. + description: When true, returns only pinned evaluations. When false, returns + only unpinned evaluations. Omit to return both. title: Is Pinned type: boolean metadata: @@ -10523,189 +10275,61 @@ components: additionalProperties: $ref: '#/components/schemas/MetricStatFilters' type: object - title: ExperimentFilter + title: EvaluationFilter type: object - ExperimentGroupFilter: - additionalProperties: false - description: Filter for listing ExperimentGroups. + EvaluationRequest: properties: name: - description: Filter groups by name. + type: string title: Name + description: Producer-supplied, workspace-unique evaluation id. + experiment_group_id: type: string - is_deleted: - description: When true, returns only soft-deleted groups. Omit (or false) - to see only live groups. - title: Is Deleted - type: boolean + title: Experiment Group Id + description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ + \ group must already exist." + dataset_name: + type: string + title: Dataset Name + description: Producer-supplied dataset name. + dataset_version: + title: Dataset Version + description: Producer-supplied dataset version. + type: string + source_link: + title: Source Link + description: Optional URL for the source evaluation. + type: string + minLength: 1 + format: uri metadata: - description: Filter by a metadata key/value pair, e.g. filter[metadata.model]=claude-opus-4-8. - title: Metadata additionalProperties: type: string type: object - title: ExperimentGroupFilter - type: object - ExperimentGroupRequest: - properties: - name: - type: string - title: Name - description: Workspace-unique group name. + title: Metadata + description: Free-form producer metadata. description: title: Description - description: Human-readable purpose of the group. + description: Human-readable description. type: string - insight_id: - title: Insight Id - description: Reference to an external insight that seeded this group, if - any. + parent_evaluation_id: + title: Parent Evaluation Id + description: Entity id of the evaluation this one was derived from (e.g. + a variant of a baseline), if any. type: string - summary: - title: Summary - description: Human- or agent-authored summary of the group's findings. + parent_experiment_id: + title: Parent Experiment Id + description: Deprecated alias for parent_evaluation_id. + deprecated: true type: string - metadata: - title: Metadata - description: Free-form producer metadata for the group. - additionalProperties: - type: string - type: object - default_sort: - type: string - title: Default Sort - description: Default sort for this group's experiments list, as a `sort`-param - string (leading '-' = descending); defaults to '-created_at'. Accepts - any field the experiments list `sort` param does; clients apply it as - the list `sort` param. - default: -created_at - additionalProperties: false - type: object - required: - - name - title: ExperimentGroupRequest - description: Request body for creating an ExperimentGroup. - ExperimentGroupResponse: - properties: - id: - type: string - title: Id - name: - type: string - title: Name - workspace: - type: string - title: Workspace - description: - title: Description - type: string - insight_id: - title: Insight Id - type: string - summary: - title: Summary - type: string - metadata: - title: Metadata - additionalProperties: - type: string - type: object - default_sort: - type: string - title: Default Sort - created_at: - title: Created At - type: string - format: date-time - updated_at: - title: Updated At - type: string - format: date-time - experiment_count: - type: integer - title: Experiment Count - description: Number of live (non-soft-deleted) experiments in this group. - default: 0 - type: object - required: - - id - - name - - workspace - - default_sort - title: ExperimentGroupResponse - description: ExperimentGroup as served by the API. - ExperimentGroupResponsesPage: - properties: - data: - items: - $ref: '#/components/schemas/ExperimentGroupResponse' - type: array - title: Data - pagination: - allOf: - - $ref: '#/components/schemas/PaginationData' - description: Pagination information. - sort: - title: Sort - description: The field on which the results are sorted. - type: string - filter: - title: Filter - description: Filtering information. - additionalProperties: true - type: object - type: object - required: - - data - title: ExperimentGroupResponsesPage - ExperimentRequest: - properties: - name: - type: string - title: Name - description: Producer-supplied, workspace-unique experiment id. - experiment_group_id: - type: string - title: Experiment Group Id - description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ - \ group must already exist." - dataset_name: - type: string - title: Dataset Name - description: Producer-supplied dataset name. - dataset_version: - title: Dataset Version - description: Producer-supplied dataset version. - type: string - source_link: - title: Source Link - description: Optional URL for the source experiment. - type: string - minLength: 1 - format: uri - metadata: - additionalProperties: - type: string - type: object - title: Metadata - description: Free-form producer metadata. - description: - title: Description - description: Human-readable description. - type: string - parent_experiment_id: - title: Parent Experiment Id - description: Entity id of the experiment this one was derived from (e.g. - a variant of a baseline), if any. - type: string - status: - title: Status - description: Producer-defined lifecycle status of the experiment. - type: string - root_cause: - title: Root Cause - description: Human- or agent-authored explanation of the experiment's outcome - (e.g. why it was killed). + status: + title: Status + description: Producer-defined lifecycle status of the evaluation. + type: string + root_cause: + title: Root Cause + description: Human- or agent-authored explanation of the evaluation's outcome + (e.g. why it was killed). type: string additionalProperties: false type: object @@ -10713,9 +10337,9 @@ components: - name - experiment_group_id - dataset_name - title: ExperimentRequest - description: Request body for creating an Experiment. - ExperimentResponse: + title: EvaluationRequest + description: Request body for creating an Evaluation. + EvaluationResponse: properties: id: type: string @@ -10730,7 +10354,7 @@ components: type: string title: Experiment Group Id description: Entity id of the owning ExperimentGroup. Required for every - Experiment. + Evaluation. dataset_name: type: string title: Dataset Name @@ -10750,8 +10374,8 @@ components: description: title: Description type: string - parent_experiment_id: - title: Parent Experiment Id + parent_evaluation_id: + title: Parent Evaluation Id type: string status: title: Status @@ -10769,8 +10393,8 @@ components: format: date-time pinned_at: title: Pinned At - description: Timestamp at which the experiment was pinned, or null if unpinned. - Managed via POST/DELETE /experiments/{name}/pin. + description: Timestamp at which the evaluation was pinned, or null if unpinned. + Managed via POST/DELETE /evaluations/{name}/pin. nullable: true type: string format: date-time @@ -10786,7 +10410,7 @@ components: uniqueItems: true title: Model Names description: Distinct model names observed across ingested sessions for - this experiment. + this evaluation. agent_names: items: type: string @@ -10794,7 +10418,7 @@ components: uniqueItems: true title: Agent Names description: Distinct agent names observed across ingested sessions for - this experiment. + this evaluation. agent_versions: items: type: string @@ -10802,7 +10426,7 @@ components: uniqueItems: true title: Agent Versions description: Distinct agent versions observed across ingested sessions for - this experiment. + this evaluation. aggregate_scores: title: Aggregate Scores additionalProperties: @@ -10811,28 +10435,356 @@ components: run_count: type: integer title: Run Count - description: Number of distinct ingested experiment sessions; one session + description: Number of distinct ingested evaluation sessions; one session is treated as one run. default: 0 cost_usd: $ref: '#/components/schemas/EvaluatorAggregate' latency_ms: $ref: '#/components/schemas/EvaluatorAggregate' + parent_experiment_id: + title: Parent Experiment Id + description: Deprecated alias for parent_evaluation_id. + deprecated: true + readOnly: true + type: string + type: object + required: + - id + - name + - workspace + - experiment_group_id + - dataset_name + - parent_experiment_id + title: EvaluationResponse + description: Evaluation as served by the API, including ClickHouse-hydrated + rollups. + EvaluationResponsesPage: + properties: + data: + items: + $ref: '#/components/schemas/EvaluationResponse' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: EvaluationResponsesPage + EvaluationSessionFilter: + additionalProperties: false + description: Filter for listing EvaluationSessions. + properties: + test_case_id: + description: Filter by producer-supplied test case id. + title: Test Case Id + type: string + status: + description: Filter by root-span status (success, error, cancelled, unknown). + title: Status + type: string + title: EvaluationSessionFilter + type: object + EvaluationSessionResponse: + properties: + workspace: + type: string + title: Workspace + evaluation_name: + type: string + title: Evaluation Name + session_id: + type: string + title: Session Id + test_case_id: + title: Test Case Id + description: Producer-supplied test case identifier; null when the producer + did not set one. + type: string + trace_id: + type: string + title: Trace Id + root_span_id: + type: string + title: Root Span Id + started_at: + type: string + format: date-time + title: Started At + ended_at: + title: Ended At + type: string + format: date-time + latency_ms: + title: Latency Ms + type: number + status: + allOf: + - $ref: '#/components/schemas/SpanStatus' + description: 'Root-span status: success, error, cancelled, or unknown.' + input: + title: Input + description: Root-span input text. In summary mode this is truncated to + 1000 characters. + type: string + input_tokens: + title: Input Tokens + description: Sum of input tokens across this session's spans. + type: integer + output_tokens: + title: Output Tokens + description: Sum of output tokens across this session's spans. + type: integer + cached_tokens: + title: Cached Tokens + description: Sum of cached tokens across this session's spans. + type: integer + cost_total_usd: + title: Cost Total Usd + description: Sum of cost across this session's spans. + type: number + evaluator_scores: + additionalProperties: + type: number + type: object + title: Evaluator Scores + description: Per-evaluator session-mean score. Includes NUMERIC and BOOLEAN + evaluator results only; text/categorical results are omitted. + experiment_name: + type: string + title: Experiment Name + description: Deprecated alias for evaluation_name. + deprecated: true + readOnly: true + type: object + required: + - workspace + - evaluation_name + - session_id + - trace_id + - root_span_id + - started_at + - status + - experiment_name + title: EvaluationSessionResponse + description: "One ingested session of an Evaluation \u2014 a single test case\ + \ execution.\n\nHydrated from ClickHouse at read time by reading root/session\ + \ membership from\n``trace_index`` and joining page-bounded span/evaluator\ + \ rollups." + EvaluationSessionResponsesPage: + properties: + data: + items: + $ref: '#/components/schemas/EvaluationSessionResponse' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: EvaluationSessionResponsesPage + EvaluatorAggregate: + properties: + sum: + title: Sum + type: number + mean: + title: Mean + type: number + median: + title: Median + type: number + p90: + title: P90 + type: number + p95: + title: P95 + type: number + p99: + title: P99 + type: number + count: + type: integer + title: Count + default: 0 + type: object + title: EvaluatorAggregate + description: Aggregate statistics over evaluator scores or session-level metric + values. + EvaluatorResult: + properties: + evaluator_result_id: + type: string + title: Evaluator Result Id + span_id: + type: string + title: Span Id + session_id: + type: string + title: Session Id + workspace: + type: string + title: Workspace + name: + type: string + title: Name + value: + title: Value + type: number + string_value: + title: String Value + type: string + data_type: + $ref: '#/components/schemas/EvaluatorResultDataType' + comment: + title: Comment + type: string + created_by: + title: Created By + type: string + created_at: + type: string + format: date-time + title: Created At + ingested_at: + type: string + format: date-time + title: Ingested At + type: object + required: + - evaluator_result_id + - span_id + - session_id + - workspace + - name + - data_type + - created_at + - ingested_at + title: EvaluatorResult + description: Response model for evaluator_results read endpoints. + EvaluatorResultDataType: + type: string + enum: + - NUMERIC + - CATEGORICAL + - BOOLEAN + - TEXT + title: EvaluatorResultDataType + EvaluatorResultFilter: + properties: + span_id: + description: Filter by target span id. + title: Span Id + type: string + session_id: + description: Filter by target session id. + title: Session Id + type: string + name: + description: Filter by evaluator/metric name. + title: Name + type: string + data_type: + allOf: + - $ref: '#/components/schemas/EvaluatorResultDataType' + description: Filter by data_type. + created_by: + description: Filter by principal/system that wrote the row. + title: Created By + type: string + value: + allOf: + - $ref: '#/components/schemas/FloatFilter' + description: Filter by numeric value (range supported). + created_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Filter by row creation time (range supported). + title: EvaluatorResultFilter + type: object + EvaluatorResultInput: + properties: + span_id: + type: string + title: Span Id + description: Target span id. Not validated against existing spans (loose + target policy). + session_id: + type: string + title: Session Id + description: Session id the target span belongs to. Denormalized so session-scoped + reads stay fast. + name: + type: string + title: Name + description: Evaluator / metric identity (e.g. 'faithfulness/v1'). + value: + title: Value + description: Numeric value. Required when data_type is NUMERIC or BOOLEAN + (0|1). + type: number + string_value: + title: String Value + description: String value. Required when data_type is CATEGORICAL or TEXT. + type: string + data_type: + allOf: + - $ref: '#/components/schemas/EvaluatorResultDataType' + description: Discriminator for which of value / string_value carries the + payload. + comment: + title: Comment + description: Free-text rationale or explanation. + type: string + additionalProperties: false type: object required: - - id + - span_id + - session_id - name - - workspace - - experiment_group_id - - dataset_name - title: ExperimentResponse - description: Experiment as served by the API, including ClickHouse-hydrated - rollups. - ExperimentResponsesPage: + - data_type + title: EvaluatorResultInput + description: "Request body for POST /evaluator-results.\n\nServer fills in `evaluator_result_id`,\ + \ `created_at`, `ingested_at`, and\n`created_by`. Producer supplies the target\ + \ span (loose target \u2014 not\nvalidated against the spans table), the score,\ + \ and provenance." + EvaluatorResultSortField: + type: string + enum: + - created_at + - -created_at + - value + - -value + title: EvaluatorResultSortField + EvaluatorResultsPage: properties: data: items: - $ref: '#/components/schemas/ExperimentResponse' + $ref: '#/components/schemas/EvaluatorResult' type: array title: Data pagination: @@ -10851,105 +10803,182 @@ components: type: object required: - data - title: ExperimentResponsesPage - ExperimentSessionFilter: - additionalProperties: false - description: Filter for listing ExperimentSessions. + title: EvaluatorResultsPage + ExecutedAction: + properties: + action_name: + type: string + title: Action Name + description: The name of the action that was executed. + action_params: + additionalProperties: true + type: object + title: Action Params + description: The parameters for the action. + return_value: + title: Return Value + description: The value returned by the action. + llm_calls: + items: + $ref: '#/components/schemas/LLMCallInfo' + type: array + title: Llm Calls + description: Information about the LLM calls made by the action. + started_at: + title: Started At + description: Timestamp for when the action started. + type: number + finished_at: + title: Finished At + description: Timestamp for when the action finished. + type: number + duration: + title: Duration + description: How long the action took to execute, in seconds. + type: number + type: object + required: + - action_name + title: ExecutedAction + description: Information about an action that was executed. + ExperimentContext: properties: + experiment_id: + type: string + title: Experiment Id + description: Name of an existing Experiment entity. test_case_id: - description: Filter by producer-supplied test case id. title: Test Case Id + description: Optional producer-supplied test case id. type: string - status: - description: Filter by root-span status (success, error, cancelled, unknown). - title: Status + additionalProperties: false + type: object + required: + - experiment_id + title: ExperimentContext + description: Deprecated alias for :class:`EvaluationContext`. Producers should + send ``evaluation_context``. + ExperimentGroupFilter: + additionalProperties: false + description: Filter for listing ExperimentGroups. + properties: + name: + description: Filter groups by name. + title: Name + type: string + is_deleted: + description: When true, returns only soft-deleted groups. Omit (or false) + to see only live groups. + title: Is Deleted + type: boolean + metadata: + description: Filter by a metadata key/value pair, e.g. filter[metadata.model]=claude-opus-4-8. + title: Metadata + additionalProperties: + type: string + type: object + title: ExperimentGroupFilter + type: object + ExperimentGroupRequest: + properties: + name: + type: string + title: Name + description: Workspace-unique group name. + description: + title: Description + description: Human-readable purpose of the group. + type: string + insight_id: + title: Insight Id + description: Reference to an external insight that seeded this group, if + any. + type: string + summary: + title: Summary + description: Human- or agent-authored summary of the group's findings. + type: string + metadata: + title: Metadata + description: Free-form producer metadata for the group. + additionalProperties: + type: string + type: object + default_sort: type: string - title: ExperimentSessionFilter + title: Default Sort + description: Default sort for this group's evaluations list, as a `sort`-param + string (leading '-' = descending); defaults to '-created_at'. Accepts + any field the evaluations list `sort` param does; clients apply it as + the list `sort` param. + default: -created_at + additionalProperties: false type: object - ExperimentSessionResponse: + required: + - name + title: ExperimentGroupRequest + description: Request body for creating an ExperimentGroup. + ExperimentGroupResponse: properties: + id: + type: string + title: Id + name: + type: string + title: Name workspace: type: string title: Workspace - experiment_name: - type: string - title: Experiment Name - session_id: + description: + title: Description type: string - title: Session Id - test_case_id: - title: Test Case Id - description: Producer-supplied test case identifier; null when the producer - did not set one. + insight_id: + title: Insight Id type: string - trace_id: + summary: + title: Summary type: string - title: Trace Id - root_span_id: + metadata: + title: Metadata + additionalProperties: + type: string + type: object + default_sort: type: string - title: Root Span Id - started_at: + title: Default Sort + created_at: + title: Created At type: string format: date-time - title: Started At - ended_at: - title: Ended At + updated_at: + title: Updated At type: string format: date-time - latency_ms: - title: Latency Ms - type: number - status: - allOf: - - $ref: '#/components/schemas/SpanStatus' - description: 'Root-span status: success, error, cancelled, or unknown.' - input: - title: Input - description: Root-span input text. In summary mode this is truncated to - 1000 characters. - type: string - input_tokens: - title: Input Tokens - description: Sum of input tokens across this session's spans. - type: integer - output_tokens: - title: Output Tokens - description: Sum of output tokens across this session's spans. + evaluation_count: type: integer - cached_tokens: - title: Cached Tokens - description: Sum of cached tokens across this session's spans. + title: Evaluation Count + description: Number of live (non-soft-deleted) evaluations in this group. + default: 0 + experiment_count: type: integer - cost_total_usd: - title: Cost Total Usd - description: Sum of cost across this session's spans. - type: number - evaluator_scores: - additionalProperties: - type: number - type: object - title: Evaluator Scores - description: Per-evaluator session-mean score. Includes NUMERIC and BOOLEAN - evaluator results only; text/categorical results are omitted. + title: Experiment Count + description: Deprecated alias for evaluation_count. + deprecated: true + readOnly: true type: object required: + - id + - name - workspace - - experiment_name - - session_id - - trace_id - - root_span_id - - started_at - - status - title: ExperimentSessionResponse - description: "One ingested session of an Experiment \u2014 a single test case\ - \ execution.\n\nHydrated from ClickHouse at read time by reading root/session\ - \ membership from\n``trace_index`` and joining page-bounded span/evaluator\ - \ rollups." - ExperimentSessionResponsesPage: + - default_sort + - experiment_count + title: ExperimentGroupResponse + description: ExperimentGroup as served by the API. + ExperimentGroupResponsesPage: properties: data: items: - $ref: '#/components/schemas/ExperimentSessionResponse' + $ref: '#/components/schemas/ExperimentGroupResponse' type: array title: Data pagination: @@ -10968,7 +10997,7 @@ components: type: object required: - data - title: ExperimentSessionResponsesPage + title: ExperimentGroupResponsesPage FactCheckingRailConfig: properties: parameters: @@ -13248,7 +13277,7 @@ components: These stats must stay in sync with the runtime sort/filter grammar (``_METRIC_STATS`` in the - experiments + evaluations endpoints); a unit test guards the parity.' properties: @@ -17875,8 +17904,14 @@ components: name: title: Name type: string + evaluation_context: + $ref: '#/components/schemas/EvaluationContext' experiment_context: - $ref: '#/components/schemas/ExperimentContext' + allOf: + - $ref: '#/components/schemas/ExperimentContext' + description: Deprecated alias for evaluation_context; will be removed in + a future release. + deprecated: true started_at: type: string format: date-time @@ -17949,12 +17984,18 @@ components: allOf: - $ref: '#/components/schemas/DatetimeFilter' description: Filter by root span start timestamp. + evaluation_id: + description: Filter by root-span evaluation id. + title: Evaluation Id + type: string experiment_id: - description: Filter by root-span experiment id. + deprecated: true + description: Deprecated alias for evaluation_id. Filter by root-span evaluation + id. title: Experiment Id type: string test_case_id: - description: Filter by root-span experiment test case id. + description: Filter by root-span evaluation test case id. title: Test Case Id type: string title: TraceFilter diff --git a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml index 4172d676e3..5bddc29758 100644 --- a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml @@ -912,29 +912,29 @@ resources: retrieve: get /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} update: put /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} delete: delete /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} - experiments: + evaluations: standalone_api: true models: evaluator_aggregate: EvaluatorAggregate - experiment_filter: ExperimentFilter - experiment_request: ExperimentRequest - experiment_response: ExperimentResponse - experiment_responses_page: ExperimentResponsesPage + evaluation_filter: EvaluationFilter + evaluation_request: EvaluationRequest + evaluation_response: EvaluationResponse + evaluation_responses_page: EvaluationResponsesPage metric_stat_filters: MetricStatFilters number_filter: NumberFilter methods: - create: post /apis/intake/v2/workspaces/{workspace}/experiments - list: get /apis/intake/v2/workspaces/{workspace}/experiments - retrieve: get /apis/intake/v2/workspaces/{workspace}/experiments/{name} - update: put /apis/intake/v2/workspaces/{workspace}/experiments/{name} - delete: delete /apis/intake/v2/workspaces/{workspace}/experiments/{name} - pin: post /apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin - unpin: delete /apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin + create: post /apis/intake/v2/workspaces/{workspace}/evaluations + list: get /apis/intake/v2/workspaces/{workspace}/evaluations + retrieve: get /apis/intake/v2/workspaces/{workspace}/evaluations/{name} + update: put /apis/intake/v2/workspaces/{workspace}/evaluations/{name} + delete: delete /apis/intake/v2/workspaces/{workspace}/evaluations/{name} + pin: post /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin + unpin: delete /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin subresources: sessions: models: - experiment_session_filter: ExperimentSessionFilter - experiment_session_response: ExperimentSessionResponse - experiment_session_responses_page: ExperimentSessionResponsesPage + evaluation_session_filter: EvaluationSessionFilter + evaluation_session_response: EvaluationSessionResponse + evaluation_session_responses_page: EvaluationSessionResponsesPage methods: - list: get /apis/intake/v2/workspaces/{workspace}/experiments/{name}/sessions + list: get /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/sessions diff --git a/sdk/python/nemo-platform/api.md b/sdk/python/nemo-platform/api.md index 2e24f19443..3a573a2566 100644 --- a/sdk/python/nemo-platform/api.md +++ b/sdk/python/nemo-platform/api.md @@ -70,4 +70,4 @@ from nemo_platform.types import ( # [ExperimentGroups](src/nemo_platform/resources/experiment_groups/api.md) -# [Experiments](src/nemo_platform/resources/experiments/api.md) +# [Evaluations](src/nemo_platform/resources/evaluations/api.md) diff --git a/sdk/python/nemo-platform/src/nemo_platform/_client.py b/sdk/python/nemo-platform/src/nemo_platform/_client.py index eddd268ac7..14026ae6b7 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/_client.py +++ b/sdk/python/nemo-platform/src/nemo_platform/_client.py @@ -64,7 +64,7 @@ guardrail, inference, workspaces, - experiments, + evaluations, experiment_groups, ) from .resources.iam.iam import IamResource, AsyncIamResource @@ -79,7 +79,7 @@ from .resources.guardrail.guardrail import GuardrailResource, AsyncGuardrailResource from .resources.inference.inference import InferenceResource, AsyncInferenceResource from .resources.workspaces.workspaces import WorkspacesResource, AsyncWorkspacesResource - from .resources.experiments.experiments import ExperimentsResource, AsyncExperimentsResource + from .resources.evaluations.evaluations import EvaluationsResource, AsyncEvaluationsResource from .resources.experiment_groups.experiment_groups import ExperimentGroupsResource, AsyncExperimentGroupsResource __all__ = [ @@ -299,10 +299,10 @@ def experiment_groups(self) -> ExperimentGroupsResource: return ExperimentGroupsResource(self) @cached_property - def experiments(self) -> ExperimentsResource: - from .resources.experiments import ExperimentsResource + def evaluations(self) -> EvaluationsResource: + from .resources.evaluations import EvaluationsResource - return ExperimentsResource(self) + return EvaluationsResource(self) @cached_property def with_raw_response(self) -> NeMoPlatformWithRawResponse: @@ -657,10 +657,10 @@ def experiment_groups(self) -> AsyncExperimentGroupsResource: return AsyncExperimentGroupsResource(self) @cached_property - def experiments(self) -> AsyncExperimentsResource: - from .resources.experiments import AsyncExperimentsResource + def evaluations(self) -> AsyncEvaluationsResource: + from .resources.evaluations import AsyncEvaluationsResource - return AsyncExperimentsResource(self) + return AsyncEvaluationsResource(self) @cached_property def with_raw_response(self) -> AsyncNeMoPlatformWithRawResponse: @@ -878,10 +878,10 @@ def experiment_groups(self) -> experiment_groups.ExperimentGroupsResourceWithRaw return ExperimentGroupsResourceWithRawResponse(self._client.experiment_groups) @cached_property - def experiments(self) -> experiments.ExperimentsResourceWithRawResponse: - from .resources.experiments import ExperimentsResourceWithRawResponse + def evaluations(self) -> evaluations.EvaluationsResourceWithRawResponse: + from .resources.evaluations import EvaluationsResourceWithRawResponse - return ExperimentsResourceWithRawResponse(self._client.experiments) + return EvaluationsResourceWithRawResponse(self._client.evaluations) class AsyncNeMoPlatformWithRawResponse: @@ -969,10 +969,10 @@ def experiment_groups(self) -> experiment_groups.AsyncExperimentGroupsResourceWi return AsyncExperimentGroupsResourceWithRawResponse(self._client.experiment_groups) @cached_property - def experiments(self) -> experiments.AsyncExperimentsResourceWithRawResponse: - from .resources.experiments import AsyncExperimentsResourceWithRawResponse + def evaluations(self) -> evaluations.AsyncEvaluationsResourceWithRawResponse: + from .resources.evaluations import AsyncEvaluationsResourceWithRawResponse - return AsyncExperimentsResourceWithRawResponse(self._client.experiments) + return AsyncEvaluationsResourceWithRawResponse(self._client.evaluations) class NeMoPlatformWithStreamedResponse: @@ -1060,10 +1060,10 @@ def experiment_groups(self) -> experiment_groups.ExperimentGroupsResourceWithStr return ExperimentGroupsResourceWithStreamingResponse(self._client.experiment_groups) @cached_property - def experiments(self) -> experiments.ExperimentsResourceWithStreamingResponse: - from .resources.experiments import ExperimentsResourceWithStreamingResponse + def evaluations(self) -> evaluations.EvaluationsResourceWithStreamingResponse: + from .resources.evaluations import EvaluationsResourceWithStreamingResponse - return ExperimentsResourceWithStreamingResponse(self._client.experiments) + return EvaluationsResourceWithStreamingResponse(self._client.evaluations) class AsyncNeMoPlatformWithStreamedResponse: @@ -1151,10 +1151,10 @@ def experiment_groups(self) -> experiment_groups.AsyncExperimentGroupsResourceWi return AsyncExperimentGroupsResourceWithStreamingResponse(self._client.experiment_groups) @cached_property - def experiments(self) -> experiments.AsyncExperimentsResourceWithStreamingResponse: - from .resources.experiments import AsyncExperimentsResourceWithStreamingResponse + def evaluations(self) -> evaluations.AsyncEvaluationsResourceWithStreamingResponse: + from .resources.evaluations import AsyncEvaluationsResourceWithStreamingResponse - return AsyncExperimentsResourceWithStreamingResponse(self._client.experiments) + return AsyncEvaluationsResourceWithStreamingResponse(self._client.evaluations) Client = NeMoPlatform diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/traces.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/traces.py index 21873e610e..cd96599a29 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/traces.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/traces.py @@ -36,11 +36,14 @@ def list_traces( typer.Option( "--filter", metavar="FILTER_JSON", - help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter root-span-backed traces by id, session_id, root status, root span started_at, experiment_id, and test_case_id.", + help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter root-span-backed traces by id, session_id, root status, root span started_at, evaluation_id (or its deprecated alias experiment_id), and test_case_id.", rich_help_panel="Filter Options", ), ] = None, filter_id: Annotated[str | None, typer.Option("--filter.id", rich_help_panel="Filter Options")] = None, + filter_evaluation_id: Annotated[ + str | None, typer.Option("--filter.evaluation-id", rich_help_panel="Filter Options") + ] = None, filter_experiment_id: Annotated[ str | None, typer.Option("--filter.experiment-id", rich_help_panel="Filter Options") ] = None, @@ -85,6 +88,7 @@ def list_traces( filter=merge_filter_dict( filter, id=filter_id, + evaluation_id=filter_evaluation_id, experiment_id=filter_experiment_id, session_id=filter_session_id, status=filter_status, diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/__init__.py similarity index 71% rename from sdk/python/nemo-platform/src/nemo_platform/resources/experiments/__init__.py rename to sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/__init__.py index 90ed5b2c22..e08c8f090e 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/__init__.py @@ -23,13 +23,13 @@ SessionsResourceWithStreamingResponse, AsyncSessionsResourceWithStreamingResponse, ) -from .experiments import ( - ExperimentsResource, - AsyncExperimentsResource, - ExperimentsResourceWithRawResponse, - AsyncExperimentsResourceWithRawResponse, - ExperimentsResourceWithStreamingResponse, - AsyncExperimentsResourceWithStreamingResponse, +from .evaluations import ( + EvaluationsResource, + AsyncEvaluationsResource, + EvaluationsResourceWithRawResponse, + AsyncEvaluationsResourceWithRawResponse, + EvaluationsResourceWithStreamingResponse, + AsyncEvaluationsResourceWithStreamingResponse, ) __all__ = [ @@ -39,10 +39,10 @@ "AsyncSessionsResourceWithRawResponse", "SessionsResourceWithStreamingResponse", "AsyncSessionsResourceWithStreamingResponse", - "ExperimentsResource", - "AsyncExperimentsResource", - "ExperimentsResourceWithRawResponse", - "AsyncExperimentsResourceWithRawResponse", - "ExperimentsResourceWithStreamingResponse", - "AsyncExperimentsResourceWithStreamingResponse", + "EvaluationsResource", + "AsyncEvaluationsResource", + "EvaluationsResourceWithRawResponse", + "AsyncEvaluationsResourceWithRawResponse", + "EvaluationsResourceWithStreamingResponse", + "AsyncEvaluationsResourceWithStreamingResponse", ] diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/api.md new file mode 100644 index 0000000000..485bdd1903 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/api.md @@ -0,0 +1,41 @@ +# Evaluations + +Types: + +```python +from nemo_platform.types.evaluations import ( + EvaluationFilter, + EvaluationRequest, + EvaluationResponse, + EvaluationResponsesPage, + EvaluatorAggregate, + MetricStatFilters, + NumberFilter, +) +``` + +Methods: + +- client.evaluations.create(\*, workspace, \*\*params) -> EvaluationResponse +- client.evaluations.retrieve(name, \*, workspace) -> EvaluationResponse +- client.evaluations.update(path_name, \*, workspace, \*\*params) -> EvaluationResponse +- client.evaluations.list(\*, workspace, \*\*params) -> SyncDefaultPagination[EvaluationResponse] +- client.evaluations.delete(name, \*, workspace) -> None +- client.evaluations.pin(name, \*, workspace) -> EvaluationResponse +- client.evaluations.unpin(name, \*, workspace) -> EvaluationResponse + +## Sessions + +Types: + +```python +from nemo_platform.types.evaluations import ( + EvaluationSessionFilter, + EvaluationSessionResponse, + EvaluationSessionResponsesPage, +) +``` + +Methods: + +- client.evaluations.sessions.list(name, \*, workspace, \*\*params) -> SyncDefaultPagination[EvaluationSessionResponse] diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/evaluations.py similarity index 81% rename from sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py rename to sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/evaluations.py index d512bbd26f..f2e83027ba 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/evaluations.py @@ -41,41 +41,41 @@ ) from ...pagination import SyncDefaultPagination, AsyncDefaultPagination from ..._base_client import AsyncPaginator, make_request_options -from ...types.experiments import ( - experiment_list_params, - experiment_create_params, - experiment_update_params, +from ...types.evaluations import ( + evaluation_list_params, + evaluation_create_params, + evaluation_update_params, ) -from ...types.experiments.experiment_response import ExperimentResponse -from ...types.experiments.experiment_filter_param import ExperimentFilterParam +from ...types.evaluations.evaluation_response import EvaluationResponse +from ...types.evaluations.evaluation_filter_param import EvaluationFilterParam from ..._exceptions import ConflictError -__all__ = ["ExperimentsResource", "AsyncExperimentsResource"] +__all__ = ["EvaluationsResource", "AsyncEvaluationsResource"] -class ExperimentsResource(SyncAPIResource): +class EvaluationsResource(SyncAPIResource): @cached_property def sessions(self) -> SessionsResource: return SessionsResource(self._client) @cached_property - def with_raw_response(self) -> ExperimentsResourceWithRawResponse: + def with_raw_response(self) -> EvaluationsResourceWithRawResponse: """ This property can be used as a prefix for any HTTP method call to return the raw response object instead of the parsed content. For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#accessing-raw-response-data-e-g-headers """ - return ExperimentsResourceWithRawResponse(self) + return EvaluationsResourceWithRawResponse(self) @cached_property - def with_streaming_response(self) -> ExperimentsResourceWithStreamingResponse: + def with_streaming_response(self) -> EvaluationsResourceWithStreamingResponse: """ An alternative to `.with_raw_response` that doesn't eagerly read the response body. For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#with_streaming_response """ - return ExperimentsResourceWithStreamingResponse(self) + return EvaluationsResourceWithStreamingResponse(self) def create( self, @@ -87,6 +87,7 @@ def create( dataset_version: str | Omit = omit, description: str | Omit = omit, metadata: Dict[str, str] | Omit = omit, + parent_evaluation_id: str | Omit = omit, parent_experiment_id: str | Omit = omit, root_cause: str | Omit = omit, source_link: str | Omit = omit, @@ -98,9 +99,9 @@ def create( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ExperimentResponse: + ) -> EvaluationResponse: """ - Create Experiment + Create Evaluation Args: dataset_name: Producer-supplied dataset name. @@ -108,7 +109,7 @@ def create( experiment_group_id: Entity id of the owning ExperimentGroup. Required — the group must already exist. - name: Producer-supplied, workspace-unique experiment id. + name: Producer-supplied, workspace-unique evaluation id. dataset_version: Producer-supplied dataset version. @@ -116,15 +117,17 @@ def create( metadata: Free-form producer metadata. - parent_experiment_id: Entity id of the experiment this one was derived from (e.g. a variant of a + parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a baseline), if any. - root_cause: Human- or agent-authored explanation of the experiment's outcome (e.g. why it + parent_experiment_id: Deprecated alias for parent_evaluation_id. + + root_cause: Human- or agent-authored explanation of the evaluation's outcome (e.g. why it was killed). - source_link: Optional URL for the source experiment. + source_link: Optional URL for the source evaluation. - status: Producer-defined lifecycle status of the experiment. + status: Producer-defined lifecycle status of the evaluation. exist_ok: Do not raise an error if the resource already exists. Returns the existing resource. @@ -144,7 +147,7 @@ def create( if not workspace: raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") return self._post( - path_template("/apis/intake/v2/workspaces/{workspace}/experiments", workspace=workspace), + path_template("/apis/intake/v2/workspaces/{workspace}/evaluations", workspace=workspace), body=maybe_transform( { "dataset_name": dataset_name, @@ -153,17 +156,18 @@ def create( "dataset_version": dataset_version, "description": description, "metadata": metadata, + "parent_evaluation_id": parent_evaluation_id, "parent_experiment_id": parent_experiment_id, "root_cause": root_cause, "source_link": source_link, "status": status, }, - experiment_create_params.ExperimentCreateParams, + evaluation_create_params.EvaluationCreateParams, ), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ExperimentResponse, + cast_to=EvaluationResponse, ) except ConflictError: if not exist_ok: @@ -181,9 +185,9 @@ def retrieve( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ExperimentResponse: + ) -> EvaluationResponse: """ - Get Experiment + Get Evaluation Args: extra_headers: Send extra headers @@ -201,11 +205,11 @@ def retrieve( if not name: raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") return self._get( - path_template("/apis/intake/v2/workspaces/{workspace}/experiments/{name}", workspace=workspace, name=name), + path_template("/apis/intake/v2/workspaces/{workspace}/evaluations/{name}", workspace=workspace, name=name), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ExperimentResponse, + cast_to=EvaluationResponse, ) def update( @@ -219,6 +223,7 @@ def update( dataset_version: str | Omit = omit, description: str | Omit = omit, metadata: Dict[str, str] | Omit = omit, + parent_evaluation_id: str | Omit = omit, parent_experiment_id: str | Omit = omit, root_cause: str | Omit = omit, source_link: str | Omit = omit, @@ -229,9 +234,9 @@ def update( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ExperimentResponse: + ) -> EvaluationResponse: """ - Update Experiment + Update Evaluation Args: dataset_name: Producer-supplied dataset name. @@ -239,7 +244,7 @@ def update( experiment_group_id: Entity id of the owning ExperimentGroup. Required — the group must already exist. - body_name: Producer-supplied, workspace-unique experiment id. + body_name: Producer-supplied, workspace-unique evaluation id. dataset_version: Producer-supplied dataset version. @@ -247,15 +252,17 @@ def update( metadata: Free-form producer metadata. - parent_experiment_id: Entity id of the experiment this one was derived from (e.g. a variant of a + parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a baseline), if any. - root_cause: Human- or agent-authored explanation of the experiment's outcome (e.g. why it + parent_experiment_id: Deprecated alias for parent_evaluation_id. + + root_cause: Human- or agent-authored explanation of the evaluation's outcome (e.g. why it was killed). - source_link: Optional URL for the source experiment. + source_link: Optional URL for the source evaluation. - status: Producer-defined lifecycle status of the experiment. + status: Producer-defined lifecycle status of the evaluation. extra_headers: Send extra headers @@ -273,7 +280,7 @@ def update( raise ValueError(f"Expected a non-empty value for `path_name` but received {path_name!r}") return self._put( path_template( - "/apis/intake/v2/workspaces/{workspace}/experiments/{path_name}", + "/apis/intake/v2/workspaces/{workspace}/evaluations/{path_name}", workspace=workspace, path_name=path_name, ), @@ -285,24 +292,25 @@ def update( "dataset_version": dataset_version, "description": description, "metadata": metadata, + "parent_evaluation_id": parent_evaluation_id, "parent_experiment_id": parent_experiment_id, "root_cause": root_cause, "source_link": source_link, "status": status, }, - experiment_update_params.ExperimentUpdateParams, + evaluation_update_params.EvaluationUpdateParams, ), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ExperimentResponse, + cast_to=EvaluationResponse, ) def list( self, *, workspace: str | None = None, - filter: ExperimentFilterParam | Omit = omit, + filter: EvaluationFilterParam | Omit = omit, page: int | Omit = omit, page_size: int | Omit = omit, sort: str | Omit = omit, @@ -312,14 +320,14 @@ def list( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> SyncDefaultPagination[ExperimentResponse]: + ) -> SyncDefaultPagination[EvaluationResponse]: """ - List Experiments + List Evaluations Args: - filter: Filter experiments by name, experiment_group_id, dataset_name, dataset_version, + filter: Filter evaluations by name, experiment_group_id, dataset_name, dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true to return only - soft-deleted experiments; omit to see only live ones. Pass is_pinned=true (or + soft-deleted evaluations; omit to see only live ones. Pass is_pinned=true (or false) to filter by pinned state; omit to return both. Filter by a metadata key/value: filter[metadata.]=. Filter by a rollup metric with numeric range operators ($gte/$lte/$gt/$lt/$eq): filter[run_count][$gte]=5, @@ -330,11 +338,11 @@ def list( page_size: Page size. - sort: Field to sort by; prefix with '-' for descending. Sort by an experiment + sort: Field to sort by; prefix with '-' for descending. Sort by an evaluation attribute (name, created_at, updated_at, pinned_at) or by an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., where is one of mean, median, p90, p95, p99, sum, count. When omitted, - defaults to -created_at with pinned experiments first. + defaults to -created_at with pinned evaluations first. extra_headers: Send extra headers @@ -349,8 +357,8 @@ def list( if not workspace: raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") return self._get_api_list( - path_template("/apis/intake/v2/workspaces/{workspace}/experiments", workspace=workspace), - page=SyncDefaultPagination[ExperimentResponse], + path_template("/apis/intake/v2/workspaces/{workspace}/evaluations", workspace=workspace), + page=SyncDefaultPagination[EvaluationResponse], options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, @@ -363,10 +371,10 @@ def list( "page_size": page_size, "sort": sort, }, - experiment_list_params.ExperimentListParams, + evaluation_list_params.EvaluationListParams, ), ), - model=ExperimentResponse, + model=EvaluationResponse, ) def delete( @@ -382,7 +390,7 @@ def delete( timeout: float | httpx.Timeout | None | NotGiven = not_given, ) -> None: """ - Delete Experiment + Delete Evaluation Args: extra_headers: Send extra headers @@ -401,7 +409,7 @@ def delete( raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") extra_headers = {"Accept": "*/*", **(extra_headers or {})} return self._delete( - path_template("/apis/intake/v2/workspaces/{workspace}/experiments/{name}", workspace=workspace, name=name), + path_template("/apis/intake/v2/workspaces/{workspace}/evaluations/{name}", workspace=workspace, name=name), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), @@ -419,11 +427,11 @@ def pin( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ExperimentResponse: + ) -> EvaluationResponse: """ - Pin an experiment to the top of the list (workspace-shared). + Pin an evaluation to the top of the list (workspace-shared). - Re-pinning an already-pinned experiment refreshes `pinned_at` to the current + Re-pinning an already-pinned evaluation refreshes `pinned_at` to the current timestamp, which is intentional (most-recently-pinned sorts first). Args: @@ -443,12 +451,12 @@ def pin( raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") return self._post( path_template( - "/apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin", workspace=workspace, name=name + "/apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin", workspace=workspace, name=name ), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ExperimentResponse, + cast_to=EvaluationResponse, ) def unpin( @@ -462,10 +470,10 @@ def unpin( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ExperimentResponse: - """Unpin an experiment. + ) -> EvaluationResponse: + """Unpin an evaluation. - Idempotent: unpinning an already-unpinned experiment is a + Idempotent: unpinning an already-unpinned evaluation is a no-op. Args: @@ -485,38 +493,38 @@ def unpin( raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") return self._delete( path_template( - "/apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin", workspace=workspace, name=name + "/apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin", workspace=workspace, name=name ), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ExperimentResponse, + cast_to=EvaluationResponse, ) -class AsyncExperimentsResource(AsyncAPIResource): +class AsyncEvaluationsResource(AsyncAPIResource): @cached_property def sessions(self) -> AsyncSessionsResource: return AsyncSessionsResource(self._client) @cached_property - def with_raw_response(self) -> AsyncExperimentsResourceWithRawResponse: + def with_raw_response(self) -> AsyncEvaluationsResourceWithRawResponse: """ This property can be used as a prefix for any HTTP method call to return the raw response object instead of the parsed content. For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#accessing-raw-response-data-e-g-headers """ - return AsyncExperimentsResourceWithRawResponse(self) + return AsyncEvaluationsResourceWithRawResponse(self) @cached_property - def with_streaming_response(self) -> AsyncExperimentsResourceWithStreamingResponse: + def with_streaming_response(self) -> AsyncEvaluationsResourceWithStreamingResponse: """ An alternative to `.with_raw_response` that doesn't eagerly read the response body. For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#with_streaming_response """ - return AsyncExperimentsResourceWithStreamingResponse(self) + return AsyncEvaluationsResourceWithStreamingResponse(self) async def create( self, @@ -528,6 +536,7 @@ async def create( dataset_version: str | Omit = omit, description: str | Omit = omit, metadata: Dict[str, str] | Omit = omit, + parent_evaluation_id: str | Omit = omit, parent_experiment_id: str | Omit = omit, root_cause: str | Omit = omit, source_link: str | Omit = omit, @@ -539,9 +548,9 @@ async def create( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ExperimentResponse: + ) -> EvaluationResponse: """ - Create Experiment + Create Evaluation Args: dataset_name: Producer-supplied dataset name. @@ -549,7 +558,7 @@ async def create( experiment_group_id: Entity id of the owning ExperimentGroup. Required — the group must already exist. - name: Producer-supplied, workspace-unique experiment id. + name: Producer-supplied, workspace-unique evaluation id. dataset_version: Producer-supplied dataset version. @@ -557,15 +566,17 @@ async def create( metadata: Free-form producer metadata. - parent_experiment_id: Entity id of the experiment this one was derived from (e.g. a variant of a + parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a baseline), if any. - root_cause: Human- or agent-authored explanation of the experiment's outcome (e.g. why it + parent_experiment_id: Deprecated alias for parent_evaluation_id. + + root_cause: Human- or agent-authored explanation of the evaluation's outcome (e.g. why it was killed). - source_link: Optional URL for the source experiment. + source_link: Optional URL for the source evaluation. - status: Producer-defined lifecycle status of the experiment. + status: Producer-defined lifecycle status of the evaluation. exist_ok: Do not raise an error if the resource already exists. Returns the existing resource. @@ -585,7 +596,7 @@ async def create( if not workspace: raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") return await self._post( - path_template("/apis/intake/v2/workspaces/{workspace}/experiments", workspace=workspace), + path_template("/apis/intake/v2/workspaces/{workspace}/evaluations", workspace=workspace), body=await async_maybe_transform( { "dataset_name": dataset_name, @@ -594,17 +605,18 @@ async def create( "dataset_version": dataset_version, "description": description, "metadata": metadata, + "parent_evaluation_id": parent_evaluation_id, "parent_experiment_id": parent_experiment_id, "root_cause": root_cause, "source_link": source_link, "status": status, }, - experiment_create_params.ExperimentCreateParams, + evaluation_create_params.EvaluationCreateParams, ), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ExperimentResponse, + cast_to=EvaluationResponse, ) except ConflictError: if not exist_ok: @@ -622,9 +634,9 @@ async def retrieve( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ExperimentResponse: + ) -> EvaluationResponse: """ - Get Experiment + Get Evaluation Args: extra_headers: Send extra headers @@ -642,11 +654,11 @@ async def retrieve( if not name: raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") return await self._get( - path_template("/apis/intake/v2/workspaces/{workspace}/experiments/{name}", workspace=workspace, name=name), + path_template("/apis/intake/v2/workspaces/{workspace}/evaluations/{name}", workspace=workspace, name=name), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ExperimentResponse, + cast_to=EvaluationResponse, ) async def update( @@ -660,6 +672,7 @@ async def update( dataset_version: str | Omit = omit, description: str | Omit = omit, metadata: Dict[str, str] | Omit = omit, + parent_evaluation_id: str | Omit = omit, parent_experiment_id: str | Omit = omit, root_cause: str | Omit = omit, source_link: str | Omit = omit, @@ -670,9 +683,9 @@ async def update( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ExperimentResponse: + ) -> EvaluationResponse: """ - Update Experiment + Update Evaluation Args: dataset_name: Producer-supplied dataset name. @@ -680,7 +693,7 @@ async def update( experiment_group_id: Entity id of the owning ExperimentGroup. Required — the group must already exist. - body_name: Producer-supplied, workspace-unique experiment id. + body_name: Producer-supplied, workspace-unique evaluation id. dataset_version: Producer-supplied dataset version. @@ -688,15 +701,17 @@ async def update( metadata: Free-form producer metadata. - parent_experiment_id: Entity id of the experiment this one was derived from (e.g. a variant of a + parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a baseline), if any. - root_cause: Human- or agent-authored explanation of the experiment's outcome (e.g. why it + parent_experiment_id: Deprecated alias for parent_evaluation_id. + + root_cause: Human- or agent-authored explanation of the evaluation's outcome (e.g. why it was killed). - source_link: Optional URL for the source experiment. + source_link: Optional URL for the source evaluation. - status: Producer-defined lifecycle status of the experiment. + status: Producer-defined lifecycle status of the evaluation. extra_headers: Send extra headers @@ -714,7 +729,7 @@ async def update( raise ValueError(f"Expected a non-empty value for `path_name` but received {path_name!r}") return await self._put( path_template( - "/apis/intake/v2/workspaces/{workspace}/experiments/{path_name}", + "/apis/intake/v2/workspaces/{workspace}/evaluations/{path_name}", workspace=workspace, path_name=path_name, ), @@ -726,24 +741,25 @@ async def update( "dataset_version": dataset_version, "description": description, "metadata": metadata, + "parent_evaluation_id": parent_evaluation_id, "parent_experiment_id": parent_experiment_id, "root_cause": root_cause, "source_link": source_link, "status": status, }, - experiment_update_params.ExperimentUpdateParams, + evaluation_update_params.EvaluationUpdateParams, ), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ExperimentResponse, + cast_to=EvaluationResponse, ) def list( self, *, workspace: str | None = None, - filter: ExperimentFilterParam | Omit = omit, + filter: EvaluationFilterParam | Omit = omit, page: int | Omit = omit, page_size: int | Omit = omit, sort: str | Omit = omit, @@ -753,14 +769,14 @@ def list( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> AsyncPaginator[ExperimentResponse, AsyncDefaultPagination[ExperimentResponse]]: + ) -> AsyncPaginator[EvaluationResponse, AsyncDefaultPagination[EvaluationResponse]]: """ - List Experiments + List Evaluations Args: - filter: Filter experiments by name, experiment_group_id, dataset_name, dataset_version, + filter: Filter evaluations by name, experiment_group_id, dataset_name, dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true to return only - soft-deleted experiments; omit to see only live ones. Pass is_pinned=true (or + soft-deleted evaluations; omit to see only live ones. Pass is_pinned=true (or false) to filter by pinned state; omit to return both. Filter by a metadata key/value: filter[metadata.]=. Filter by a rollup metric with numeric range operators ($gte/$lte/$gt/$lt/$eq): filter[run_count][$gte]=5, @@ -771,11 +787,11 @@ def list( page_size: Page size. - sort: Field to sort by; prefix with '-' for descending. Sort by an experiment + sort: Field to sort by; prefix with '-' for descending. Sort by an evaluation attribute (name, created_at, updated_at, pinned_at) or by an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., where is one of mean, median, p90, p95, p99, sum, count. When omitted, - defaults to -created_at with pinned experiments first. + defaults to -created_at with pinned evaluations first. extra_headers: Send extra headers @@ -790,8 +806,8 @@ def list( if not workspace: raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") return self._get_api_list( - path_template("/apis/intake/v2/workspaces/{workspace}/experiments", workspace=workspace), - page=AsyncDefaultPagination[ExperimentResponse], + path_template("/apis/intake/v2/workspaces/{workspace}/evaluations", workspace=workspace), + page=AsyncDefaultPagination[EvaluationResponse], options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, @@ -804,10 +820,10 @@ def list( "page_size": page_size, "sort": sort, }, - experiment_list_params.ExperimentListParams, + evaluation_list_params.EvaluationListParams, ), ), - model=ExperimentResponse, + model=EvaluationResponse, ) async def delete( @@ -823,7 +839,7 @@ async def delete( timeout: float | httpx.Timeout | None | NotGiven = not_given, ) -> None: """ - Delete Experiment + Delete Evaluation Args: extra_headers: Send extra headers @@ -842,7 +858,7 @@ async def delete( raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") extra_headers = {"Accept": "*/*", **(extra_headers or {})} return await self._delete( - path_template("/apis/intake/v2/workspaces/{workspace}/experiments/{name}", workspace=workspace, name=name), + path_template("/apis/intake/v2/workspaces/{workspace}/evaluations/{name}", workspace=workspace, name=name), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), @@ -860,11 +876,11 @@ async def pin( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ExperimentResponse: + ) -> EvaluationResponse: """ - Pin an experiment to the top of the list (workspace-shared). + Pin an evaluation to the top of the list (workspace-shared). - Re-pinning an already-pinned experiment refreshes `pinned_at` to the current + Re-pinning an already-pinned evaluation refreshes `pinned_at` to the current timestamp, which is intentional (most-recently-pinned sorts first). Args: @@ -884,12 +900,12 @@ async def pin( raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") return await self._post( path_template( - "/apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin", workspace=workspace, name=name + "/apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin", workspace=workspace, name=name ), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ExperimentResponse, + cast_to=EvaluationResponse, ) async def unpin( @@ -903,10 +919,10 @@ async def unpin( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ExperimentResponse: - """Unpin an experiment. + ) -> EvaluationResponse: + """Unpin an evaluation. - Idempotent: unpinning an already-unpinned experiment is a + Idempotent: unpinning an already-unpinned evaluation is a no-op. Args: @@ -926,134 +942,134 @@ async def unpin( raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") return await self._delete( path_template( - "/apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin", workspace=workspace, name=name + "/apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin", workspace=workspace, name=name ), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ExperimentResponse, + cast_to=EvaluationResponse, ) -class ExperimentsResourceWithRawResponse: - def __init__(self, experiments: ExperimentsResource) -> None: - self._experiments = experiments +class EvaluationsResourceWithRawResponse: + def __init__(self, evaluations: EvaluationsResource) -> None: + self._evaluations = evaluations self.create = to_raw_response_wrapper( - experiments.create, + evaluations.create, ) self.retrieve = to_raw_response_wrapper( - experiments.retrieve, + evaluations.retrieve, ) self.update = to_raw_response_wrapper( - experiments.update, + evaluations.update, ) self.list = to_raw_response_wrapper( - experiments.list, + evaluations.list, ) self.delete = to_raw_response_wrapper( - experiments.delete, + evaluations.delete, ) self.pin = to_raw_response_wrapper( - experiments.pin, + evaluations.pin, ) self.unpin = to_raw_response_wrapper( - experiments.unpin, + evaluations.unpin, ) @cached_property def sessions(self) -> SessionsResourceWithRawResponse: - return SessionsResourceWithRawResponse(self._experiments.sessions) + return SessionsResourceWithRawResponse(self._evaluations.sessions) -class AsyncExperimentsResourceWithRawResponse: - def __init__(self, experiments: AsyncExperimentsResource) -> None: - self._experiments = experiments +class AsyncEvaluationsResourceWithRawResponse: + def __init__(self, evaluations: AsyncEvaluationsResource) -> None: + self._evaluations = evaluations self.create = async_to_raw_response_wrapper( - experiments.create, + evaluations.create, ) self.retrieve = async_to_raw_response_wrapper( - experiments.retrieve, + evaluations.retrieve, ) self.update = async_to_raw_response_wrapper( - experiments.update, + evaluations.update, ) self.list = async_to_raw_response_wrapper( - experiments.list, + evaluations.list, ) self.delete = async_to_raw_response_wrapper( - experiments.delete, + evaluations.delete, ) self.pin = async_to_raw_response_wrapper( - experiments.pin, + evaluations.pin, ) self.unpin = async_to_raw_response_wrapper( - experiments.unpin, + evaluations.unpin, ) @cached_property def sessions(self) -> AsyncSessionsResourceWithRawResponse: - return AsyncSessionsResourceWithRawResponse(self._experiments.sessions) + return AsyncSessionsResourceWithRawResponse(self._evaluations.sessions) -class ExperimentsResourceWithStreamingResponse: - def __init__(self, experiments: ExperimentsResource) -> None: - self._experiments = experiments +class EvaluationsResourceWithStreamingResponse: + def __init__(self, evaluations: EvaluationsResource) -> None: + self._evaluations = evaluations self.create = to_streamed_response_wrapper( - experiments.create, + evaluations.create, ) self.retrieve = to_streamed_response_wrapper( - experiments.retrieve, + evaluations.retrieve, ) self.update = to_streamed_response_wrapper( - experiments.update, + evaluations.update, ) self.list = to_streamed_response_wrapper( - experiments.list, + evaluations.list, ) self.delete = to_streamed_response_wrapper( - experiments.delete, + evaluations.delete, ) self.pin = to_streamed_response_wrapper( - experiments.pin, + evaluations.pin, ) self.unpin = to_streamed_response_wrapper( - experiments.unpin, + evaluations.unpin, ) @cached_property def sessions(self) -> SessionsResourceWithStreamingResponse: - return SessionsResourceWithStreamingResponse(self._experiments.sessions) + return SessionsResourceWithStreamingResponse(self._evaluations.sessions) -class AsyncExperimentsResourceWithStreamingResponse: - def __init__(self, experiments: AsyncExperimentsResource) -> None: - self._experiments = experiments +class AsyncEvaluationsResourceWithStreamingResponse: + def __init__(self, evaluations: AsyncEvaluationsResource) -> None: + self._evaluations = evaluations self.create = async_to_streamed_response_wrapper( - experiments.create, + evaluations.create, ) self.retrieve = async_to_streamed_response_wrapper( - experiments.retrieve, + evaluations.retrieve, ) self.update = async_to_streamed_response_wrapper( - experiments.update, + evaluations.update, ) self.list = async_to_streamed_response_wrapper( - experiments.list, + evaluations.list, ) self.delete = async_to_streamed_response_wrapper( - experiments.delete, + evaluations.delete, ) self.pin = async_to_streamed_response_wrapper( - experiments.pin, + evaluations.pin, ) self.unpin = async_to_streamed_response_wrapper( - experiments.unpin, + evaluations.unpin, ) @cached_property def sessions(self) -> AsyncSessionsResourceWithStreamingResponse: - return AsyncSessionsResourceWithStreamingResponse(self._experiments.sessions) + return AsyncSessionsResourceWithStreamingResponse(self._evaluations.sessions) diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/sessions.py b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/sessions.py similarity index 89% rename from sdk/python/nemo-platform/src/nemo_platform/resources/experiments/sessions.py rename to sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/sessions.py index 84750eb748..09c23e76c0 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/sessions.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/sessions.py @@ -33,9 +33,9 @@ ) from ...pagination import SyncDefaultPagination, AsyncDefaultPagination from ..._base_client import AsyncPaginator, make_request_options -from ...types.experiments import session_list_params -from ...types.experiments.experiment_session_response import ExperimentSessionResponse -from ...types.experiments.experiment_session_filter_param import ExperimentSessionFilterParam +from ...types.evaluations import session_list_params +from ...types.evaluations.evaluation_session_response import EvaluationSessionResponse +from ...types.evaluations.evaluation_session_filter_param import EvaluationSessionFilterParam __all__ = ["SessionsResource", "AsyncSessionsResource"] @@ -65,7 +65,7 @@ def list( name: str, *, workspace: str | None = None, - filter: ExperimentSessionFilterParam | Omit = omit, + filter: EvaluationSessionFilterParam | Omit = omit, mode: Literal["summary", "detailed"] | Omit = omit, page: int | Omit = omit, page_size: int | Omit = omit, @@ -75,9 +75,9 @@ def list( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> SyncDefaultPagination[ExperimentSessionResponse]: + ) -> SyncDefaultPagination[EvaluationSessionResponse]: """ - List Experiment Sessions + List Evaluation Sessions Args: filter: Filter sessions by test_case_id and status. @@ -105,9 +105,9 @@ def list( raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") return self._get_api_list( path_template( - "/apis/intake/v2/workspaces/{workspace}/experiments/{name}/sessions", workspace=workspace, name=name + "/apis/intake/v2/workspaces/{workspace}/evaluations/{name}/sessions", workspace=workspace, name=name ), - page=SyncDefaultPagination[ExperimentSessionResponse], + page=SyncDefaultPagination[EvaluationSessionResponse], options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, @@ -123,7 +123,7 @@ def list( session_list_params.SessionListParams, ), ), - model=ExperimentSessionResponse, + model=EvaluationSessionResponse, ) @@ -152,7 +152,7 @@ def list( name: str, *, workspace: str | None = None, - filter: ExperimentSessionFilterParam | Omit = omit, + filter: EvaluationSessionFilterParam | Omit = omit, mode: Literal["summary", "detailed"] | Omit = omit, page: int | Omit = omit, page_size: int | Omit = omit, @@ -162,9 +162,9 @@ def list( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> AsyncPaginator[ExperimentSessionResponse, AsyncDefaultPagination[ExperimentSessionResponse]]: + ) -> AsyncPaginator[EvaluationSessionResponse, AsyncDefaultPagination[EvaluationSessionResponse]]: """ - List Experiment Sessions + List Evaluation Sessions Args: filter: Filter sessions by test_case_id and status. @@ -192,9 +192,9 @@ def list( raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") return self._get_api_list( path_template( - "/apis/intake/v2/workspaces/{workspace}/experiments/{name}/sessions", workspace=workspace, name=name + "/apis/intake/v2/workspaces/{workspace}/evaluations/{name}/sessions", workspace=workspace, name=name ), - page=AsyncDefaultPagination[ExperimentSessionResponse], + page=AsyncDefaultPagination[EvaluationSessionResponse], options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, @@ -210,7 +210,7 @@ def list( session_list_params.SessionListParams, ), ), - model=ExperimentSessionResponse, + model=EvaluationSessionResponse, ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py index f3f554bcf1..f4cc4f6bc7 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py @@ -90,9 +90,9 @@ def create( Args: name: Workspace-unique group name. - default_sort: Default sort for this group's experiments list, as a `sort`-param string + default_sort: Default sort for this group's evaluations list, as a `sort`-param string (leading '-' = descending); defaults to '-created_at'. Accepts any field the - experiments list `sort` param does; clients apply it as the list `sort` param. + evaluations list `sort` param does; clients apply it as the list `sort` param. description: Human-readable purpose of the group. @@ -206,9 +206,9 @@ def update( Args: body_name: Workspace-unique group name. - default_sort: Default sort for this group's experiments list, as a `sort`-param string + default_sort: Default sort for this group's evaluations list, as a `sort`-param string (leading '-' = descending); defaults to '-created_at'. Accepts any field the - experiments list `sort` param does; clients apply it as the list `sort` param. + evaluations list `sort` param does; clients apply it as the list `sort` param. description: Human-readable purpose of the group. @@ -403,9 +403,9 @@ async def create( Args: name: Workspace-unique group name. - default_sort: Default sort for this group's experiments list, as a `sort`-param string + default_sort: Default sort for this group's evaluations list, as a `sort`-param string (leading '-' = descending); defaults to '-created_at'. Accepts any field the - experiments list `sort` param does; clients apply it as the list `sort` param. + evaluations list `sort` param does; clients apply it as the list `sort` param. description: Human-readable purpose of the group. @@ -519,9 +519,9 @@ async def update( Args: body_name: Workspace-unique group name. - default_sort: Default sort for this group's experiments list, as a `sort`-param string + default_sort: Default sort for this group's evaluations list, as a `sort`-param string (leading '-' = descending); defaults to '-created_at'. Accepts any field the - experiments list `sort` param does; clients apply it as the list `sort` param. + evaluations list `sort` param does; clients apply it as the list `sort` param. description: Human-readable purpose of the group. diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md deleted file mode 100644 index 0c6e16973f..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md +++ /dev/null @@ -1,41 +0,0 @@ -# Experiments - -Types: - -```python -from nemo_platform.types.experiments import ( - EvaluatorAggregate, - ExperimentFilter, - ExperimentRequest, - ExperimentResponse, - ExperimentResponsesPage, - MetricStatFilters, - NumberFilter, -) -``` - -Methods: - -- client.experiments.create(\*, workspace, \*\*params) -> ExperimentResponse -- client.experiments.retrieve(name, \*, workspace) -> ExperimentResponse -- client.experiments.update(path_name, \*, workspace, \*\*params) -> ExperimentResponse -- client.experiments.list(\*, workspace, \*\*params) -> SyncDefaultPagination[ExperimentResponse] -- client.experiments.delete(name, \*, workspace) -> None -- client.experiments.pin(name, \*, workspace) -> ExperimentResponse -- client.experiments.unpin(name, \*, workspace) -> ExperimentResponse - -## Sessions - -Types: - -```python -from nemo_platform.types.experiments import ( - ExperimentSessionFilter, - ExperimentSessionResponse, - ExperimentSessionResponsesPage, -) -``` - -Methods: - -- client.experiments.sessions.list(name, \*, workspace, \*\*params) -> SyncDefaultPagination[ExperimentSessionResponse] diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/traces.py b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/traces.py index 1ad4d67a4c..678ceb6458 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/traces.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/traces.py @@ -128,7 +128,8 @@ def list( Args: filter: Filter root-span-backed traces by id, session_id, root status, root span - started_at, experiment_id, and test_case_id. + started_at, evaluation_id (or its deprecated alias experiment_id), and + test_case_id. mode: Use summary for root-span trace fields only, or detailed to include token, cost, and span-count rollups. @@ -259,7 +260,8 @@ def list( Args: filter: Filter root-span-backed traces by id, session_id, root status, root span - started_at, experiment_id, and test_case_id. + started_at, evaluation_id (or its deprecated alias experiment_id), and + test_case_id. mode: Use summary for root-span trace fields only, or detailed to include token, cost, and span-count rollups. diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/__init__.py similarity index 57% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiments/__init__.py rename to sdk/python/nemo-platform/src/nemo_platform/types/evaluations/__init__.py index b046d1fc20..d83c9d69a2 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/__init__.py @@ -17,16 +17,16 @@ from __future__ import annotations +from .evaluation_response import EvaluationResponse as EvaluationResponse from .evaluator_aggregate import EvaluatorAggregate as EvaluatorAggregate -from .experiment_response import ExperimentResponse as ExperimentResponse from .number_filter_param import NumberFilterParam as NumberFilterParam from .session_list_params import SessionListParams as SessionListParams -from .experiment_list_params import ExperimentListParams as ExperimentListParams -from .experiment_filter_param import ExperimentFilterParam as ExperimentFilterParam -from .experiment_create_params import ExperimentCreateParams as ExperimentCreateParams -from .experiment_update_params import ExperimentUpdateParams as ExperimentUpdateParams -from .experiment_responses_page import ExperimentResponsesPage as ExperimentResponsesPage +from .evaluation_list_params import EvaluationListParams as EvaluationListParams +from .evaluation_filter_param import EvaluationFilterParam as EvaluationFilterParam +from .evaluation_create_params import EvaluationCreateParams as EvaluationCreateParams +from .evaluation_update_params import EvaluationUpdateParams as EvaluationUpdateParams +from .evaluation_responses_page import EvaluationResponsesPage as EvaluationResponsesPage from .metric_stat_filters_param import MetricStatFiltersParam as MetricStatFiltersParam -from .experiment_session_response import ExperimentSessionResponse as ExperimentSessionResponse -from .experiment_session_filter_param import ExperimentSessionFilterParam as ExperimentSessionFilterParam -from .experiment_session_responses_page import ExperimentSessionResponsesPage as ExperimentSessionResponsesPage +from .evaluation_session_response import EvaluationSessionResponse as EvaluationSessionResponse +from .evaluation_session_filter_param import EvaluationSessionFilterParam as EvaluationSessionFilterParam +from .evaluation_session_responses_page import EvaluationSessionResponsesPage as EvaluationSessionResponsesPage diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_create_params.py similarity index 76% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py rename to sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_create_params.py index b9e30db813..956ea0db31 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_create_params.py @@ -20,10 +20,10 @@ from typing import Dict from typing_extensions import Required, TypedDict -__all__ = ["ExperimentCreateParams"] +__all__ = ["EvaluationCreateParams"] -class ExperimentCreateParams(TypedDict, total=False): +class EvaluationCreateParams(TypedDict, total=False): workspace: str dataset_name: Required[str] @@ -36,7 +36,7 @@ class ExperimentCreateParams(TypedDict, total=False): """ name: Required[str] - """Producer-supplied, workspace-unique experiment id.""" + """Producer-supplied, workspace-unique evaluation id.""" dataset_version: str """Producer-supplied dataset version.""" @@ -47,20 +47,23 @@ class ExperimentCreateParams(TypedDict, total=False): metadata: Dict[str, str] """Free-form producer metadata.""" - parent_experiment_id: str - """Entity id of the experiment this one was derived from (e.g. + parent_evaluation_id: str + """Entity id of the evaluation this one was derived from (e.g. a variant of a baseline), if any. """ + parent_experiment_id: str + """Deprecated alias for parent_evaluation_id.""" + root_cause: str - """Human- or agent-authored explanation of the experiment's outcome (e.g. + """Human- or agent-authored explanation of the evaluation's outcome (e.g. why it was killed). """ source_link: str - """Optional URL for the source experiment.""" + """Optional URL for the source evaluation.""" status: str - """Producer-defined lifecycle status of the experiment.""" + """Producer-defined lifecycle status of the evaluation.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_filter_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_filter_param.py similarity index 73% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_filter_param.py rename to sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_filter_param.py index 5f72bddad6..ecbb88e87e 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_filter_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_filter_param.py @@ -24,11 +24,11 @@ from .metric_stat_filters_param import MetricStatFiltersParam from ..shared_params.datetime_filter import DatetimeFilter -__all__ = ["ExperimentFilterParam"] +__all__ = ["EvaluationFilterParam"] -class ExperimentFilterParam(TypedDict, total=False): - """Filter for listing Experiments.""" +class EvaluationFilterParam(TypedDict, total=False): + """Filter for listing Evaluations.""" cost_usd: MetricStatFiltersParam """Numeric range filters keyed by rollup aggregate stat. @@ -36,23 +36,23 @@ class ExperimentFilterParam(TypedDict, total=False): Declaring each stat explicitly (rather than an open `dict[str, NumberFilter]`) makes the valid stats visible in the OpenAPI schema, e.g. `filter[cost_usd.mean][$lte]=0.5`. These stats must stay in sync with the - runtime sort/filter grammar (`_METRIC_STATS` in the experiments endpoints); a + runtime sort/filter grammar (`_METRIC_STATS` in the evaluations endpoints); a unit test guards the parity. """ created_at: DatetimeFilter """ - Filter experiments by creation timestamp; supports `$gte` and `$lte` for ranges. + Filter evaluations by creation timestamp; supports `$gte` and `$lte` for ranges. """ created_by: str - """Filter experiments by the principal that created them.""" + """Filter evaluations by the principal that created them.""" dataset_name: str - """Filter experiments by dataset name.""" + """Filter evaluations by dataset name.""" dataset_version: str - """Filter experiments by dataset version.""" + """Filter evaluations by dataset version.""" evaluators: Dict[str, MetricStatFiltersParam] """Filter by an evaluator rollup stat, e.g. @@ -61,18 +61,18 @@ class ExperimentFilterParam(TypedDict, total=False): """ experiment_group_id: str - """Filter experiments by owning group id.""" + """Filter evaluations by owning group id.""" is_deleted: bool - """When true, returns only soft-deleted experiments. + """When true, returns only soft-deleted evaluations. - Omit (or false) to see only live experiments. + Omit (or false) to see only live evaluations. """ is_pinned: bool - """When true, returns only pinned experiments. + """When true, returns only pinned evaluations. - When false, returns only unpinned experiments. Omit to return both. + When false, returns only unpinned evaluations. Omit to return both. """ latency_ms: MetricStatFiltersParam @@ -81,7 +81,7 @@ class ExperimentFilterParam(TypedDict, total=False): Declaring each stat explicitly (rather than an open `dict[str, NumberFilter]`) makes the valid stats visible in the OpenAPI schema, e.g. `filter[cost_usd.mean][$lte]=0.5`. These stats must stay in sync with the - runtime sort/filter grammar (`_METRIC_STATS` in the experiments endpoints); a + runtime sort/filter grammar (`_METRIC_STATS` in the evaluations endpoints); a unit test guards the parity. """ @@ -92,13 +92,13 @@ class ExperimentFilterParam(TypedDict, total=False): """ name: str - """Filter experiments by name.""" + """Filter evaluations by name.""" run_count: NumberFilterParam """Filter by run count, e.g. filter[run_count][$gte]=5.""" updated_at: DatetimeFilter """ - Filter experiments by last-updated timestamp; supports `$gte` and `$lte` for + Filter evaluations by last-updated timestamp; supports `$gte` and `$lte` for ranges. """ diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_list_params.py similarity index 81% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py rename to sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_list_params.py index 38d3864441..8322d6a3b8 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_list_params.py @@ -19,19 +19,19 @@ from typing_extensions import TypedDict -from .experiment_filter_param import ExperimentFilterParam +from .evaluation_filter_param import EvaluationFilterParam -__all__ = ["ExperimentListParams"] +__all__ = ["EvaluationListParams"] -class ExperimentListParams(TypedDict, total=False): +class EvaluationListParams(TypedDict, total=False): workspace: str - filter: ExperimentFilterParam + filter: EvaluationFilterParam """ - Filter experiments by name, experiment_group_id, dataset_name, dataset_version, + Filter evaluations by name, experiment_group_id, dataset_name, dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true to return only - soft-deleted experiments; omit to see only live ones. Pass is_pinned=true (or + soft-deleted evaluations; omit to see only live ones. Pass is_pinned=true (or false) to filter by pinned state; omit to return both. Filter by a metadata key/value: filter[metadata.]=. Filter by a rollup metric with numeric range operators ($gte/$lte/$gt/$lt/$eq): filter[run_count][$gte]=5, @@ -48,8 +48,8 @@ class ExperimentListParams(TypedDict, total=False): sort: str """Field to sort by; prefix with '-' for descending. - Sort by an experiment attribute (name, created_at, updated_at, pinned_at) or by + Sort by an evaluation attribute (name, created_at, updated_at, pinned_at) or by an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., where is one of mean, median, p90, p95, p99, - sum, count. When omitted, defaults to -created_at with pinned experiments first. + sum, count. When omitted, defaults to -created_at with pinned evaluations first. """ diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py similarity index 82% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py rename to sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py index d9a5a014ec..49349c7a6f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py @@ -22,28 +22,31 @@ from ..._models import BaseModel from .evaluator_aggregate import EvaluatorAggregate -__all__ = ["ExperimentResponse"] +__all__ = ["EvaluationResponse"] -class ExperimentResponse(BaseModel): - """Experiment as served by the API, including ClickHouse-hydrated rollups.""" +class EvaluationResponse(BaseModel): + """Evaluation as served by the API, including ClickHouse-hydrated rollups.""" id: str dataset_name: str experiment_group_id: str - """Entity id of the owning ExperimentGroup. Required for every Experiment.""" + """Entity id of the owning ExperimentGroup. Required for every Evaluation.""" name: str + parent_experiment_id: str + """Deprecated alias for parent_evaluation_id.""" + workspace: str agent_names: Optional[List[str]] = None - """Distinct agent names observed across ingested sessions for this experiment.""" + """Distinct agent names observed across ingested sessions for this evaluation.""" agent_versions: Optional[List[str]] = None - """Distinct agent versions observed across ingested sessions for this experiment.""" + """Distinct agent versions observed across ingested sessions for this evaluation.""" aggregate_scores: Optional[Dict[str, EvaluatorAggregate]] = None @@ -64,21 +67,21 @@ class ExperimentResponse(BaseModel): metadata: Optional[Dict[str, str]] = None model_names: Optional[List[str]] = None - """Distinct model names observed across ingested sessions for this experiment.""" + """Distinct model names observed across ingested sessions for this evaluation.""" - parent_experiment_id: Optional[str] = None + parent_evaluation_id: Optional[str] = None pinned_at: Optional[datetime] = None - """Timestamp at which the experiment was pinned, or null if unpinned. + """Timestamp at which the evaluation was pinned, or null if unpinned. - Managed via POST/DELETE /experiments/{name}/pin. + Managed via POST/DELETE /evaluations/{name}/pin. """ root_cause: Optional[str] = None run_count: Optional[int] = None """ - Number of distinct ingested experiment sessions; one session is treated as one + Number of distinct ingested evaluation sessions; one session is treated as one run. """ diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_responses_page.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_responses_page.py similarity index 87% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_responses_page.py rename to sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_responses_page.py index ad2f49105e..b6607d6ac5 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_responses_page.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_responses_page.py @@ -18,14 +18,14 @@ from typing import Dict, List, Optional from ..._models import BaseModel -from .experiment_response import ExperimentResponse +from .evaluation_response import EvaluationResponse from ..shared.pagination_data import PaginationData -__all__ = ["ExperimentResponsesPage"] +__all__ = ["EvaluationResponsesPage"] -class ExperimentResponsesPage(BaseModel): - data: List[ExperimentResponse] +class EvaluationResponsesPage(BaseModel): + data: List[EvaluationResponse] filter: Optional[Dict[str, object]] = None """Filtering information.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_session_filter_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_filter_param.py similarity index 86% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_session_filter_param.py rename to sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_filter_param.py index 579b250f7c..6bc7f9d8dd 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_session_filter_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_filter_param.py @@ -19,11 +19,11 @@ from typing_extensions import TypedDict -__all__ = ["ExperimentSessionFilterParam"] +__all__ = ["EvaluationSessionFilterParam"] -class ExperimentSessionFilterParam(TypedDict, total=False): - """Filter for listing ExperimentSessions.""" +class EvaluationSessionFilterParam(TypedDict, total=False): + """Filter for listing EvaluationSessions.""" status: str """Filter by root-span status (success, error, cancelled, unknown).""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_session_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_response.py similarity index 91% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_session_response.py rename to sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_response.py index fcd3305340..eb8e60bb9a 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_session_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_response.py @@ -21,17 +21,20 @@ from ..._models import BaseModel from ..intake.span_status import SpanStatus -__all__ = ["ExperimentSessionResponse"] +__all__ = ["EvaluationSessionResponse"] -class ExperimentSessionResponse(BaseModel): - """One ingested session of an Experiment — a single test case execution. +class EvaluationSessionResponse(BaseModel): + """One ingested session of an Evaluation — a single test case execution. Hydrated from ClickHouse at read time by reading root/session membership from ``trace_index`` and joining page-bounded span/evaluator rollups. """ + evaluation_name: str + experiment_name: str + """Deprecated alias for evaluation_name.""" root_span_id: str diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_session_responses_page.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_responses_page.py similarity index 85% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_session_responses_page.py rename to sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_responses_page.py index 476c161ea2..547530a3a9 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_session_responses_page.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_session_responses_page.py @@ -19,13 +19,13 @@ from ..._models import BaseModel from ..shared.pagination_data import PaginationData -from .experiment_session_response import ExperimentSessionResponse +from .evaluation_session_response import EvaluationSessionResponse -__all__ = ["ExperimentSessionResponsesPage"] +__all__ = ["EvaluationSessionResponsesPage"] -class ExperimentSessionResponsesPage(BaseModel): - data: List[ExperimentSessionResponse] +class EvaluationSessionResponsesPage(BaseModel): + data: List[EvaluationSessionResponse] filter: Optional[Dict[str, object]] = None """Filtering information.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_update_params.py similarity index 77% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py rename to sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_update_params.py index 6000e850dc..a44d66f57b 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_update_params.py @@ -22,10 +22,10 @@ from ..._utils import PropertyInfo -__all__ = ["ExperimentUpdateParams"] +__all__ = ["EvaluationUpdateParams"] -class ExperimentUpdateParams(TypedDict, total=False): +class EvaluationUpdateParams(TypedDict, total=False): workspace: str dataset_name: Required[str] @@ -38,7 +38,7 @@ class ExperimentUpdateParams(TypedDict, total=False): """ body_name: Required[Annotated[str, PropertyInfo(alias="name")]] - """Producer-supplied, workspace-unique experiment id.""" + """Producer-supplied, workspace-unique evaluation id.""" dataset_version: str """Producer-supplied dataset version.""" @@ -49,20 +49,23 @@ class ExperimentUpdateParams(TypedDict, total=False): metadata: Dict[str, str] """Free-form producer metadata.""" - parent_experiment_id: str - """Entity id of the experiment this one was derived from (e.g. + parent_evaluation_id: str + """Entity id of the evaluation this one was derived from (e.g. a variant of a baseline), if any. """ + parent_experiment_id: str + """Deprecated alias for parent_evaluation_id.""" + root_cause: str - """Human- or agent-authored explanation of the experiment's outcome (e.g. + """Human- or agent-authored explanation of the evaluation's outcome (e.g. why it was killed). """ source_link: str - """Optional URL for the source experiment.""" + """Optional URL for the source evaluation.""" status: str - """Producer-defined lifecycle status of the experiment.""" + """Producer-defined lifecycle status of the evaluation.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/evaluator_aggregate.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluator_aggregate.py similarity index 100% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiments/evaluator_aggregate.py rename to sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluator_aggregate.py diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/metric_stat_filters_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/metric_stat_filters_param.py similarity index 98% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiments/metric_stat_filters_param.py rename to sdk/python/nemo-platform/src/nemo_platform/types/evaluations/metric_stat_filters_param.py index 45ee5ddc7e..fcc58d3d77 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/metric_stat_filters_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/metric_stat_filters_param.py @@ -29,7 +29,7 @@ class MetricStatFiltersParam(TypedDict, total=False): Declaring each stat explicitly (rather than an open ``dict[str, NumberFilter]``) makes the valid stats visible in the OpenAPI schema, e.g. ``filter[cost_usd.mean][$lte]=0.5``. These stats must - stay in sync with the runtime sort/filter grammar (``_METRIC_STATS`` in the experiments + stay in sync with the runtime sort/filter grammar (``_METRIC_STATS`` in the evaluations endpoints); a unit test guards the parity. """ diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/number_filter_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/number_filter_param.py similarity index 100% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiments/number_filter_param.py rename to sdk/python/nemo-platform/src/nemo_platform/types/evaluations/number_filter_param.py diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/session_list_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/session_list_params.py similarity index 92% rename from sdk/python/nemo-platform/src/nemo_platform/types/experiments/session_list_params.py rename to sdk/python/nemo-platform/src/nemo_platform/types/evaluations/session_list_params.py index 1e556402d8..2a9ce665df 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/session_list_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/session_list_params.py @@ -19,7 +19,7 @@ from typing_extensions import Literal, TypedDict -from .experiment_session_filter_param import ExperimentSessionFilterParam +from .evaluation_session_filter_param import EvaluationSessionFilterParam __all__ = ["SessionListParams"] @@ -27,7 +27,7 @@ class SessionListParams(TypedDict, total=False): workspace: str - filter: ExperimentSessionFilterParam + filter: EvaluationSessionFilterParam """Filter sessions by test_case_id and status.""" mode: Literal["summary", "detailed"] diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py index 9066424848..1a32ce19bb 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py @@ -31,9 +31,9 @@ class ExperimentGroupCreateParams(TypedDict, total=False): default_sort: str """ - Default sort for this group's experiments list, as a `sort`-param string + Default sort for this group's evaluations list, as a `sort`-param string (leading '-' = descending); defaults to '-created_at'. Accepts any field the - experiments list `sort` param does; clients apply it as the list `sort` param. + evaluations list `sort` param does; clients apply it as the list `sort` param. """ description: str diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py index 85e766bf82..52cac70f8f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py @@ -30,6 +30,9 @@ class ExperimentGroupResponse(BaseModel): default_sort: str + experiment_count: int + """Deprecated alias for evaluation_count.""" + name: str workspace: str @@ -38,8 +41,8 @@ class ExperimentGroupResponse(BaseModel): description: Optional[str] = None - experiment_count: Optional[int] = None - """Number of live (non-soft-deleted) experiments in this group.""" + evaluation_count: Optional[int] = None + """Number of live (non-soft-deleted) evaluations in this group.""" insight_id: Optional[str] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py index 478504bdaf..9bfc12f6f8 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py @@ -33,9 +33,9 @@ class ExperimentGroupUpdateParams(TypedDict, total=False): default_sort: str """ - Default sort for this group's experiments list, as a `sort`-param string + Default sort for this group's evaluations list, as a `sort`-param string (leading '-' = descending); defaults to '-created_at'. Accepts any field the - experiments list `sort` param does; clients apply it as the list `sort` param. + evaluations list `sort` param does; clients apply it as the list `sort` param. """ description: str diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/__init__.py index 217d6fc838..db89d6420d 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/__init__.py @@ -35,6 +35,7 @@ from .trace_sort_field import TraceSortField as TraceSortField from .span_filter_param import SpanFilterParam as SpanFilterParam from .trace_list_params import TraceListParams as TraceListParams +from .evaluation_context import EvaluationContext as EvaluationContext from .experiment_context import ExperimentContext as ExperimentContext from .float_filter_param import FloatFilterParam as FloatFilterParam from .trace_filter_param import TraceFilterParam as TraceFilterParam diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context.py new file mode 100644 index 0000000000..20f7977103 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional + +from ..._models import BaseModel + +__all__ = ["EvaluationContext"] + + +class EvaluationContext(BaseModel): + """Evaluation context accepted by ingest endpoints (the canonical shape). + + ``extra="ignore"`` so a producer still sending retired keys (evaluation_sha, evaluation_run_id, + metadata) keeps ingesting without error rather than being rejected. + """ + + evaluation_id: Optional[str] = None + """Name of an existing Evaluation.""" + + test_case_id: Optional[str] = None + """Optional producer-supplied test case id.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context_param.py index a01932343a..3609d259b8 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context_param.py @@ -30,7 +30,7 @@ class EvaluationContextParam(TypedDict, total=False): """ evaluation_id: str - """Name of an existing Experiment entity.""" + """Name of an existing Evaluation.""" test_case_id: str """Optional producer-supplied test case id.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace.py index c51a5caacc..d62a2a81f2 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace.py @@ -20,6 +20,7 @@ from ..._models import BaseModel from .span_status import SpanStatus +from .evaluation_context import EvaluationContext from .experiment_context import ExperimentContext __all__ = ["Trace"] @@ -50,6 +51,14 @@ class Trace(BaseModel): error_count: Optional[int] = None + evaluation_context: Optional[EvaluationContext] = None + """Evaluation context accepted by ingest endpoints (the canonical shape). + + `extra="ignore"` so a producer still sending retired keys (evaluation_sha, + evaluation_run_id, metadata) keeps ingesting without error rather than being + rejected. + """ + experiment_context: Optional[ExperimentContext] = None """Deprecated alias for :class:`EvaluationContext`. diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_filter_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_filter_param.py index a21e61d557..78e3b2d82c 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_filter_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_filter_param.py @@ -29,8 +29,11 @@ class TraceFilterParam(TypedDict, total=False): id: str """Filter by canonical Intake trace id.""" + evaluation_id: str + """Filter by root-span evaluation id.""" + experiment_id: str - """Filter by root-span experiment id.""" + """Deprecated alias for evaluation_id. Filter by root-span evaluation id.""" session_id: str """Filter by session id.""" @@ -42,4 +45,4 @@ class TraceFilterParam(TypedDict, total=False): """Filter by root span status.""" test_case_id: str - """Filter by root-span experiment test case id.""" + """Filter by root-span evaluation test case id.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_list_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_list_params.py index 1d28a3e03a..569231a947 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_list_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_list_params.py @@ -31,7 +31,8 @@ class TraceListParams(TypedDict, total=False): filter: TraceFilterParam """ Filter root-span-backed traces by id, session_id, root status, root span - started_at, experiment_id, and test_case_id. + started_at, evaluation_id (or its deprecated alias experiment_id), and + test_case_id. """ mode: Literal["summary", "detailed"] diff --git a/sdk/python/nemo-platform/tests/api_resources/experiments/__init__.py b/sdk/python/nemo-platform/tests/api_resources/evaluations/__init__.py similarity index 100% rename from sdk/python/nemo-platform/tests/api_resources/experiments/__init__.py rename to sdk/python/nemo-platform/tests/api_resources/evaluations/__init__.py diff --git a/sdk/python/nemo-platform/tests/api_resources/experiments/test_sessions.py b/sdk/python/nemo-platform/tests/api_resources/evaluations/test_sessions.py similarity index 81% rename from sdk/python/nemo-platform/tests/api_resources/experiments/test_sessions.py rename to sdk/python/nemo-platform/tests/api_resources/evaluations/test_sessions.py index f112e3172e..dcababa5f1 100644 --- a/sdk/python/nemo-platform/tests/api_resources/experiments/test_sessions.py +++ b/sdk/python/nemo-platform/tests/api_resources/evaluations/test_sessions.py @@ -25,7 +25,7 @@ from tests.utils import assert_matches_type from nemo_platform import NeMoPlatform, AsyncNeMoPlatform from nemo_platform.pagination import SyncDefaultPagination, AsyncDefaultPagination -from nemo_platform.types.experiments import ExperimentSessionResponse +from nemo_platform.types.evaluations import EvaluationSessionResponse base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") @@ -36,16 +36,16 @@ class TestSessions: @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_list(self, client: NeMoPlatform) -> None: - session = client.experiments.sessions.list( + session = client.evaluations.sessions.list( name="name", workspace="workspace", ) - assert_matches_type(SyncDefaultPagination[ExperimentSessionResponse], session, path=["response"]) + assert_matches_type(SyncDefaultPagination[EvaluationSessionResponse], session, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: - session = client.experiments.sessions.list( + session = client.evaluations.sessions.list( name="name", workspace="workspace", filter={ @@ -56,12 +56,12 @@ def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: page=1, page_size=1, ) - assert_matches_type(SyncDefaultPagination[ExperimentSessionResponse], session, path=["response"]) + assert_matches_type(SyncDefaultPagination[EvaluationSessionResponse], session, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_raw_response_list(self, client: NeMoPlatform) -> None: - response = client.experiments.sessions.with_raw_response.list( + response = client.evaluations.sessions.with_raw_response.list( name="name", workspace="workspace", ) @@ -69,12 +69,12 @@ def test_raw_response_list(self, client: NeMoPlatform) -> None: assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" session = response.parse() - assert_matches_type(SyncDefaultPagination[ExperimentSessionResponse], session, path=["response"]) + assert_matches_type(SyncDefaultPagination[EvaluationSessionResponse], session, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_streaming_response_list(self, client: NeMoPlatform) -> None: - with client.experiments.sessions.with_streaming_response.list( + with client.evaluations.sessions.with_streaming_response.list( name="name", workspace="workspace", ) as response: @@ -82,7 +82,7 @@ def test_streaming_response_list(self, client: NeMoPlatform) -> None: assert response.http_request.headers.get("X-Stainless-Lang") == "python" session = response.parse() - assert_matches_type(SyncDefaultPagination[ExperimentSessionResponse], session, path=["response"]) + assert_matches_type(SyncDefaultPagination[EvaluationSessionResponse], session, path=["response"]) assert cast(Any, response.is_closed) is True @@ -90,13 +90,13 @@ def test_streaming_response_list(self, client: NeMoPlatform) -> None: @parametrize def test_path_params_list(self, client: NeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - client.experiments.sessions.with_raw_response.list( + client.evaluations.sessions.with_raw_response.list( name="name", workspace="", ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): - client.experiments.sessions.with_raw_response.list( + client.evaluations.sessions.with_raw_response.list( name="", workspace="workspace", ) @@ -110,16 +110,16 @@ class TestAsyncSessions: @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_list(self, async_client: AsyncNeMoPlatform) -> None: - session = await async_client.experiments.sessions.list( + session = await async_client.evaluations.sessions.list( name="name", workspace="workspace", ) - assert_matches_type(AsyncDefaultPagination[ExperimentSessionResponse], session, path=["response"]) + assert_matches_type(AsyncDefaultPagination[EvaluationSessionResponse], session, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform) -> None: - session = await async_client.experiments.sessions.list( + session = await async_client.evaluations.sessions.list( name="name", workspace="workspace", filter={ @@ -130,12 +130,12 @@ async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform page=1, page_size=1, ) - assert_matches_type(AsyncDefaultPagination[ExperimentSessionResponse], session, path=["response"]) + assert_matches_type(AsyncDefaultPagination[EvaluationSessionResponse], session, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_raw_response_list(self, async_client: AsyncNeMoPlatform) -> None: - response = await async_client.experiments.sessions.with_raw_response.list( + response = await async_client.evaluations.sessions.with_raw_response.list( name="name", workspace="workspace", ) @@ -143,12 +143,12 @@ async def test_raw_response_list(self, async_client: AsyncNeMoPlatform) -> None: assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" session = await response.parse() - assert_matches_type(AsyncDefaultPagination[ExperimentSessionResponse], session, path=["response"]) + assert_matches_type(AsyncDefaultPagination[EvaluationSessionResponse], session, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_streaming_response_list(self, async_client: AsyncNeMoPlatform) -> None: - async with async_client.experiments.sessions.with_streaming_response.list( + async with async_client.evaluations.sessions.with_streaming_response.list( name="name", workspace="workspace", ) as response: @@ -156,7 +156,7 @@ async def test_streaming_response_list(self, async_client: AsyncNeMoPlatform) -> assert response.http_request.headers.get("X-Stainless-Lang") == "python" session = await response.parse() - assert_matches_type(AsyncDefaultPagination[ExperimentSessionResponse], session, path=["response"]) + assert_matches_type(AsyncDefaultPagination[EvaluationSessionResponse], session, path=["response"]) assert cast(Any, response.is_closed) is True @@ -164,13 +164,13 @@ async def test_streaming_response_list(self, async_client: AsyncNeMoPlatform) -> @parametrize async def test_path_params_list(self, async_client: AsyncNeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - await async_client.experiments.sessions.with_raw_response.list( + await async_client.evaluations.sessions.with_raw_response.list( name="name", workspace="", ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): - await async_client.experiments.sessions.with_raw_response.list( + await async_client.evaluations.sessions.with_raw_response.list( name="", workspace="workspace", ) diff --git a/sdk/python/nemo-platform/tests/api_resources/intake/test_traces.py b/sdk/python/nemo-platform/tests/api_resources/intake/test_traces.py index 938a04bdf5..dff3f52980 100644 --- a/sdk/python/nemo-platform/tests/api_resources/intake/test_traces.py +++ b/sdk/python/nemo-platform/tests/api_resources/intake/test_traces.py @@ -111,6 +111,7 @@ def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: workspace="workspace", filter={ "id": "id", + "evaluation_id": "evaluation_id", "experiment_id": "experiment_id", "session_id": "session_id", "started_at": { @@ -244,6 +245,7 @@ async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform workspace="workspace", filter={ "id": "id", + "evaluation_id": "evaluation_id", "experiment_id": "experiment_id", "session_id": "session_id", "started_at": { diff --git a/sdk/python/nemo-platform/tests/api_resources/test_experiments.py b/sdk/python/nemo-platform/tests/api_resources/test_evaluations.py similarity index 81% rename from sdk/python/nemo-platform/tests/api_resources/test_experiments.py rename to sdk/python/nemo-platform/tests/api_resources/test_evaluations.py index 4efd976762..fd8798a6ef 100644 --- a/sdk/python/nemo-platform/tests/api_resources/test_experiments.py +++ b/sdk/python/nemo-platform/tests/api_resources/test_evaluations.py @@ -26,31 +26,31 @@ from nemo_platform import NeMoPlatform, AsyncNeMoPlatform from nemo_platform._utils import parse_datetime from nemo_platform.pagination import SyncDefaultPagination, AsyncDefaultPagination -from nemo_platform.types.experiments import ( - ExperimentResponse, +from nemo_platform.types.evaluations import ( + EvaluationResponse, ) base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") -class TestExperiments: +class TestEvaluations: parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_create(self, client: NeMoPlatform) -> None: - experiment = client.experiments.create( + evaluation = client.evaluations.create( workspace="workspace", dataset_name="dataset_name", experiment_group_id="experiment_group_id", name="name", ) - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: - experiment = client.experiments.create( + evaluation = client.evaluations.create( workspace="workspace", dataset_name="dataset_name", experiment_group_id="experiment_group_id", @@ -58,17 +58,18 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: dataset_version="dataset_version", description="description", metadata={"foo": "string"}, + parent_evaluation_id="parent_evaluation_id", parent_experiment_id="parent_experiment_id", root_cause="root_cause", source_link="https://example.com", status="status", ) - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_raw_response_create(self, client: NeMoPlatform) -> None: - response = client.experiments.with_raw_response.create( + response = client.evaluations.with_raw_response.create( workspace="workspace", dataset_name="dataset_name", experiment_group_id="experiment_group_id", @@ -77,13 +78,13 @@ def test_raw_response_create(self, client: NeMoPlatform) -> None: assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_streaming_response_create(self, client: NeMoPlatform) -> None: - with client.experiments.with_streaming_response.create( + with client.evaluations.with_streaming_response.create( workspace="workspace", dataset_name="dataset_name", experiment_group_id="experiment_group_id", @@ -92,8 +93,8 @@ def test_streaming_response_create(self, client: NeMoPlatform) -> None: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) assert cast(Any, response.is_closed) is True @@ -101,7 +102,7 @@ def test_streaming_response_create(self, client: NeMoPlatform) -> None: @parametrize def test_path_params_create(self, client: NeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - client.experiments.with_raw_response.create( + client.evaluations.with_raw_response.create( workspace="", dataset_name="dataset_name", experiment_group_id="experiment_group_id", @@ -111,37 +112,37 @@ def test_path_params_create(self, client: NeMoPlatform) -> None: @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_retrieve(self, client: NeMoPlatform) -> None: - experiment = client.experiments.retrieve( + evaluation = client.evaluations.retrieve( name="name", workspace="workspace", ) - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_raw_response_retrieve(self, client: NeMoPlatform) -> None: - response = client.experiments.with_raw_response.retrieve( + response = client.evaluations.with_raw_response.retrieve( name="name", workspace="workspace", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_streaming_response_retrieve(self, client: NeMoPlatform) -> None: - with client.experiments.with_streaming_response.retrieve( + with client.evaluations.with_streaming_response.retrieve( name="name", workspace="workspace", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) assert cast(Any, response.is_closed) is True @@ -149,13 +150,13 @@ def test_streaming_response_retrieve(self, client: NeMoPlatform) -> None: @parametrize def test_path_params_retrieve(self, client: NeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - client.experiments.with_raw_response.retrieve( + client.evaluations.with_raw_response.retrieve( name="name", workspace="", ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): - client.experiments.with_raw_response.retrieve( + client.evaluations.with_raw_response.retrieve( name="", workspace="workspace", ) @@ -163,19 +164,19 @@ def test_path_params_retrieve(self, client: NeMoPlatform) -> None: @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_update(self, client: NeMoPlatform) -> None: - experiment = client.experiments.update( + evaluation = client.evaluations.update( path_name="name", workspace="workspace", dataset_name="dataset_name", experiment_group_id="experiment_group_id", body_name="name", ) - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_update_with_all_params(self, client: NeMoPlatform) -> None: - experiment = client.experiments.update( + evaluation = client.evaluations.update( path_name="name", workspace="workspace", dataset_name="dataset_name", @@ -184,17 +185,18 @@ def test_method_update_with_all_params(self, client: NeMoPlatform) -> None: dataset_version="dataset_version", description="description", metadata={"foo": "string"}, + parent_evaluation_id="parent_evaluation_id", parent_experiment_id="parent_experiment_id", root_cause="root_cause", source_link="https://example.com", status="status", ) - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_raw_response_update(self, client: NeMoPlatform) -> None: - response = client.experiments.with_raw_response.update( + response = client.evaluations.with_raw_response.update( path_name="name", workspace="workspace", dataset_name="dataset_name", @@ -204,13 +206,13 @@ def test_raw_response_update(self, client: NeMoPlatform) -> None: assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_streaming_response_update(self, client: NeMoPlatform) -> None: - with client.experiments.with_streaming_response.update( + with client.evaluations.with_streaming_response.update( path_name="name", workspace="workspace", dataset_name="dataset_name", @@ -220,8 +222,8 @@ def test_streaming_response_update(self, client: NeMoPlatform) -> None: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) assert cast(Any, response.is_closed) is True @@ -229,7 +231,7 @@ def test_streaming_response_update(self, client: NeMoPlatform) -> None: @parametrize def test_path_params_update(self, client: NeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - client.experiments.with_raw_response.update( + client.evaluations.with_raw_response.update( path_name="name", workspace="", dataset_name="dataset_name", @@ -238,7 +240,7 @@ def test_path_params_update(self, client: NeMoPlatform) -> None: ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `path_name` but received ''"): - client.experiments.with_raw_response.update( + client.evaluations.with_raw_response.update( path_name="", workspace="workspace", dataset_name="dataset_name", @@ -249,15 +251,15 @@ def test_path_params_update(self, client: NeMoPlatform) -> None: @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_list(self, client: NeMoPlatform) -> None: - experiment = client.experiments.list( + evaluation = client.evaluations.list( workspace="workspace", ) - assert_matches_type(SyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) + assert_matches_type(SyncDefaultPagination[EvaluationResponse], evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: - experiment = client.experiments.list( + evaluation = client.evaluations.list( workspace="workspace", filter={ "cost_usd": { @@ -443,31 +445,31 @@ def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: page_size=1, sort="sort", ) - assert_matches_type(SyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) + assert_matches_type(SyncDefaultPagination[EvaluationResponse], evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_raw_response_list(self, client: NeMoPlatform) -> None: - response = client.experiments.with_raw_response.list( + response = client.evaluations.with_raw_response.list( workspace="workspace", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = response.parse() - assert_matches_type(SyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) + evaluation = response.parse() + assert_matches_type(SyncDefaultPagination[EvaluationResponse], evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_streaming_response_list(self, client: NeMoPlatform) -> None: - with client.experiments.with_streaming_response.list( + with client.evaluations.with_streaming_response.list( workspace="workspace", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = response.parse() - assert_matches_type(SyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) + evaluation = response.parse() + assert_matches_type(SyncDefaultPagination[EvaluationResponse], evaluation, path=["response"]) assert cast(Any, response.is_closed) is True @@ -475,44 +477,44 @@ def test_streaming_response_list(self, client: NeMoPlatform) -> None: @parametrize def test_path_params_list(self, client: NeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - client.experiments.with_raw_response.list( + client.evaluations.with_raw_response.list( workspace="", ) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_delete(self, client: NeMoPlatform) -> None: - experiment = client.experiments.delete( + evaluation = client.evaluations.delete( name="name", workspace="workspace", ) - assert experiment is None + assert evaluation is None @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_raw_response_delete(self, client: NeMoPlatform) -> None: - response = client.experiments.with_raw_response.delete( + response = client.evaluations.with_raw_response.delete( name="name", workspace="workspace", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = response.parse() - assert experiment is None + evaluation = response.parse() + assert evaluation is None @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_streaming_response_delete(self, client: NeMoPlatform) -> None: - with client.experiments.with_streaming_response.delete( + with client.evaluations.with_streaming_response.delete( name="name", workspace="workspace", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = response.parse() - assert experiment is None + evaluation = response.parse() + assert evaluation is None assert cast(Any, response.is_closed) is True @@ -520,13 +522,13 @@ def test_streaming_response_delete(self, client: NeMoPlatform) -> None: @parametrize def test_path_params_delete(self, client: NeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - client.experiments.with_raw_response.delete( + client.evaluations.with_raw_response.delete( name="name", workspace="", ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): - client.experiments.with_raw_response.delete( + client.evaluations.with_raw_response.delete( name="", workspace="workspace", ) @@ -534,37 +536,37 @@ def test_path_params_delete(self, client: NeMoPlatform) -> None: @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_pin(self, client: NeMoPlatform) -> None: - experiment = client.experiments.pin( + evaluation = client.evaluations.pin( name="name", workspace="workspace", ) - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_raw_response_pin(self, client: NeMoPlatform) -> None: - response = client.experiments.with_raw_response.pin( + response = client.evaluations.with_raw_response.pin( name="name", workspace="workspace", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_streaming_response_pin(self, client: NeMoPlatform) -> None: - with client.experiments.with_streaming_response.pin( + with client.evaluations.with_streaming_response.pin( name="name", workspace="workspace", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) assert cast(Any, response.is_closed) is True @@ -572,13 +574,13 @@ def test_streaming_response_pin(self, client: NeMoPlatform) -> None: @parametrize def test_path_params_pin(self, client: NeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - client.experiments.with_raw_response.pin( + client.evaluations.with_raw_response.pin( name="name", workspace="", ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): - client.experiments.with_raw_response.pin( + client.evaluations.with_raw_response.pin( name="", workspace="workspace", ) @@ -586,37 +588,37 @@ def test_path_params_pin(self, client: NeMoPlatform) -> None: @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_unpin(self, client: NeMoPlatform) -> None: - experiment = client.experiments.unpin( + evaluation = client.evaluations.unpin( name="name", workspace="workspace", ) - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_raw_response_unpin(self, client: NeMoPlatform) -> None: - response = client.experiments.with_raw_response.unpin( + response = client.evaluations.with_raw_response.unpin( name="name", workspace="workspace", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_streaming_response_unpin(self, client: NeMoPlatform) -> None: - with client.experiments.with_streaming_response.unpin( + with client.evaluations.with_streaming_response.unpin( name="name", workspace="workspace", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) assert cast(Any, response.is_closed) is True @@ -624,19 +626,19 @@ def test_streaming_response_unpin(self, client: NeMoPlatform) -> None: @parametrize def test_path_params_unpin(self, client: NeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - client.experiments.with_raw_response.unpin( + client.evaluations.with_raw_response.unpin( name="name", workspace="", ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): - client.experiments.with_raw_response.unpin( + client.evaluations.with_raw_response.unpin( name="", workspace="workspace", ) -class TestAsyncExperiments: +class TestAsyncEvaluations: parametrize = pytest.mark.parametrize( "async_client", [False, True, {"http_client": "aiohttp"}], indirect=True, ids=["loose", "strict", "aiohttp"] ) @@ -644,18 +646,18 @@ class TestAsyncExperiments: @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_create(self, async_client: AsyncNeMoPlatform) -> None: - experiment = await async_client.experiments.create( + evaluation = await async_client.evaluations.create( workspace="workspace", dataset_name="dataset_name", experiment_group_id="experiment_group_id", name="name", ) - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatform) -> None: - experiment = await async_client.experiments.create( + evaluation = await async_client.evaluations.create( workspace="workspace", dataset_name="dataset_name", experiment_group_id="experiment_group_id", @@ -663,17 +665,18 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo dataset_version="dataset_version", description="description", metadata={"foo": "string"}, + parent_evaluation_id="parent_evaluation_id", parent_experiment_id="parent_experiment_id", root_cause="root_cause", source_link="https://example.com", status="status", ) - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_raw_response_create(self, async_client: AsyncNeMoPlatform) -> None: - response = await async_client.experiments.with_raw_response.create( + response = await async_client.evaluations.with_raw_response.create( workspace="workspace", dataset_name="dataset_name", experiment_group_id="experiment_group_id", @@ -682,13 +685,13 @@ async def test_raw_response_create(self, async_client: AsyncNeMoPlatform) -> Non assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = await response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = await response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_streaming_response_create(self, async_client: AsyncNeMoPlatform) -> None: - async with async_client.experiments.with_streaming_response.create( + async with async_client.evaluations.with_streaming_response.create( workspace="workspace", dataset_name="dataset_name", experiment_group_id="experiment_group_id", @@ -697,8 +700,8 @@ async def test_streaming_response_create(self, async_client: AsyncNeMoPlatform) assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = await response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = await response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) assert cast(Any, response.is_closed) is True @@ -706,7 +709,7 @@ async def test_streaming_response_create(self, async_client: AsyncNeMoPlatform) @parametrize async def test_path_params_create(self, async_client: AsyncNeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - await async_client.experiments.with_raw_response.create( + await async_client.evaluations.with_raw_response.create( workspace="", dataset_name="dataset_name", experiment_group_id="experiment_group_id", @@ -716,37 +719,37 @@ async def test_path_params_create(self, async_client: AsyncNeMoPlatform) -> None @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_retrieve(self, async_client: AsyncNeMoPlatform) -> None: - experiment = await async_client.experiments.retrieve( + evaluation = await async_client.evaluations.retrieve( name="name", workspace="workspace", ) - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_raw_response_retrieve(self, async_client: AsyncNeMoPlatform) -> None: - response = await async_client.experiments.with_raw_response.retrieve( + response = await async_client.evaluations.with_raw_response.retrieve( name="name", workspace="workspace", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = await response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = await response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_streaming_response_retrieve(self, async_client: AsyncNeMoPlatform) -> None: - async with async_client.experiments.with_streaming_response.retrieve( + async with async_client.evaluations.with_streaming_response.retrieve( name="name", workspace="workspace", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = await response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = await response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) assert cast(Any, response.is_closed) is True @@ -754,13 +757,13 @@ async def test_streaming_response_retrieve(self, async_client: AsyncNeMoPlatform @parametrize async def test_path_params_retrieve(self, async_client: AsyncNeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - await async_client.experiments.with_raw_response.retrieve( + await async_client.evaluations.with_raw_response.retrieve( name="name", workspace="", ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): - await async_client.experiments.with_raw_response.retrieve( + await async_client.evaluations.with_raw_response.retrieve( name="", workspace="workspace", ) @@ -768,19 +771,19 @@ async def test_path_params_retrieve(self, async_client: AsyncNeMoPlatform) -> No @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_update(self, async_client: AsyncNeMoPlatform) -> None: - experiment = await async_client.experiments.update( + evaluation = await async_client.evaluations.update( path_name="name", workspace="workspace", dataset_name="dataset_name", experiment_group_id="experiment_group_id", body_name="name", ) - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_update_with_all_params(self, async_client: AsyncNeMoPlatform) -> None: - experiment = await async_client.experiments.update( + evaluation = await async_client.evaluations.update( path_name="name", workspace="workspace", dataset_name="dataset_name", @@ -789,17 +792,18 @@ async def test_method_update_with_all_params(self, async_client: AsyncNeMoPlatfo dataset_version="dataset_version", description="description", metadata={"foo": "string"}, + parent_evaluation_id="parent_evaluation_id", parent_experiment_id="parent_experiment_id", root_cause="root_cause", source_link="https://example.com", status="status", ) - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_raw_response_update(self, async_client: AsyncNeMoPlatform) -> None: - response = await async_client.experiments.with_raw_response.update( + response = await async_client.evaluations.with_raw_response.update( path_name="name", workspace="workspace", dataset_name="dataset_name", @@ -809,13 +813,13 @@ async def test_raw_response_update(self, async_client: AsyncNeMoPlatform) -> Non assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = await response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = await response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_streaming_response_update(self, async_client: AsyncNeMoPlatform) -> None: - async with async_client.experiments.with_streaming_response.update( + async with async_client.evaluations.with_streaming_response.update( path_name="name", workspace="workspace", dataset_name="dataset_name", @@ -825,8 +829,8 @@ async def test_streaming_response_update(self, async_client: AsyncNeMoPlatform) assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = await response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = await response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) assert cast(Any, response.is_closed) is True @@ -834,7 +838,7 @@ async def test_streaming_response_update(self, async_client: AsyncNeMoPlatform) @parametrize async def test_path_params_update(self, async_client: AsyncNeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - await async_client.experiments.with_raw_response.update( + await async_client.evaluations.with_raw_response.update( path_name="name", workspace="", dataset_name="dataset_name", @@ -843,7 +847,7 @@ async def test_path_params_update(self, async_client: AsyncNeMoPlatform) -> None ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `path_name` but received ''"): - await async_client.experiments.with_raw_response.update( + await async_client.evaluations.with_raw_response.update( path_name="", workspace="workspace", dataset_name="dataset_name", @@ -854,15 +858,15 @@ async def test_path_params_update(self, async_client: AsyncNeMoPlatform) -> None @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_list(self, async_client: AsyncNeMoPlatform) -> None: - experiment = await async_client.experiments.list( + evaluation = await async_client.evaluations.list( workspace="workspace", ) - assert_matches_type(AsyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) + assert_matches_type(AsyncDefaultPagination[EvaluationResponse], evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform) -> None: - experiment = await async_client.experiments.list( + evaluation = await async_client.evaluations.list( workspace="workspace", filter={ "cost_usd": { @@ -1048,31 +1052,31 @@ async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform page_size=1, sort="sort", ) - assert_matches_type(AsyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) + assert_matches_type(AsyncDefaultPagination[EvaluationResponse], evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_raw_response_list(self, async_client: AsyncNeMoPlatform) -> None: - response = await async_client.experiments.with_raw_response.list( + response = await async_client.evaluations.with_raw_response.list( workspace="workspace", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = await response.parse() - assert_matches_type(AsyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) + evaluation = await response.parse() + assert_matches_type(AsyncDefaultPagination[EvaluationResponse], evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_streaming_response_list(self, async_client: AsyncNeMoPlatform) -> None: - async with async_client.experiments.with_streaming_response.list( + async with async_client.evaluations.with_streaming_response.list( workspace="workspace", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = await response.parse() - assert_matches_type(AsyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) + evaluation = await response.parse() + assert_matches_type(AsyncDefaultPagination[EvaluationResponse], evaluation, path=["response"]) assert cast(Any, response.is_closed) is True @@ -1080,44 +1084,44 @@ async def test_streaming_response_list(self, async_client: AsyncNeMoPlatform) -> @parametrize async def test_path_params_list(self, async_client: AsyncNeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - await async_client.experiments.with_raw_response.list( + await async_client.evaluations.with_raw_response.list( workspace="", ) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_delete(self, async_client: AsyncNeMoPlatform) -> None: - experiment = await async_client.experiments.delete( + evaluation = await async_client.evaluations.delete( name="name", workspace="workspace", ) - assert experiment is None + assert evaluation is None @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_raw_response_delete(self, async_client: AsyncNeMoPlatform) -> None: - response = await async_client.experiments.with_raw_response.delete( + response = await async_client.evaluations.with_raw_response.delete( name="name", workspace="workspace", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = await response.parse() - assert experiment is None + evaluation = await response.parse() + assert evaluation is None @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_streaming_response_delete(self, async_client: AsyncNeMoPlatform) -> None: - async with async_client.experiments.with_streaming_response.delete( + async with async_client.evaluations.with_streaming_response.delete( name="name", workspace="workspace", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = await response.parse() - assert experiment is None + evaluation = await response.parse() + assert evaluation is None assert cast(Any, response.is_closed) is True @@ -1125,13 +1129,13 @@ async def test_streaming_response_delete(self, async_client: AsyncNeMoPlatform) @parametrize async def test_path_params_delete(self, async_client: AsyncNeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - await async_client.experiments.with_raw_response.delete( + await async_client.evaluations.with_raw_response.delete( name="name", workspace="", ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): - await async_client.experiments.with_raw_response.delete( + await async_client.evaluations.with_raw_response.delete( name="", workspace="workspace", ) @@ -1139,37 +1143,37 @@ async def test_path_params_delete(self, async_client: AsyncNeMoPlatform) -> None @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_pin(self, async_client: AsyncNeMoPlatform) -> None: - experiment = await async_client.experiments.pin( + evaluation = await async_client.evaluations.pin( name="name", workspace="workspace", ) - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_raw_response_pin(self, async_client: AsyncNeMoPlatform) -> None: - response = await async_client.experiments.with_raw_response.pin( + response = await async_client.evaluations.with_raw_response.pin( name="name", workspace="workspace", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = await response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = await response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_streaming_response_pin(self, async_client: AsyncNeMoPlatform) -> None: - async with async_client.experiments.with_streaming_response.pin( + async with async_client.evaluations.with_streaming_response.pin( name="name", workspace="workspace", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = await response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = await response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) assert cast(Any, response.is_closed) is True @@ -1177,13 +1181,13 @@ async def test_streaming_response_pin(self, async_client: AsyncNeMoPlatform) -> @parametrize async def test_path_params_pin(self, async_client: AsyncNeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - await async_client.experiments.with_raw_response.pin( + await async_client.evaluations.with_raw_response.pin( name="name", workspace="", ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): - await async_client.experiments.with_raw_response.pin( + await async_client.evaluations.with_raw_response.pin( name="", workspace="workspace", ) @@ -1191,37 +1195,37 @@ async def test_path_params_pin(self, async_client: AsyncNeMoPlatform) -> None: @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_unpin(self, async_client: AsyncNeMoPlatform) -> None: - experiment = await async_client.experiments.unpin( + evaluation = await async_client.evaluations.unpin( name="name", workspace="workspace", ) - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_raw_response_unpin(self, async_client: AsyncNeMoPlatform) -> None: - response = await async_client.experiments.with_raw_response.unpin( + response = await async_client.evaluations.with_raw_response.unpin( name="name", workspace="workspace", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = await response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = await response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_streaming_response_unpin(self, async_client: AsyncNeMoPlatform) -> None: - async with async_client.experiments.with_streaming_response.unpin( + async with async_client.evaluations.with_streaming_response.unpin( name="name", workspace="workspace", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment = await response.parse() - assert_matches_type(ExperimentResponse, experiment, path=["response"]) + evaluation = await response.parse() + assert_matches_type(EvaluationResponse, evaluation, path=["response"]) assert cast(Any, response.is_closed) is True @@ -1229,13 +1233,13 @@ async def test_streaming_response_unpin(self, async_client: AsyncNeMoPlatform) - @parametrize async def test_path_params_unpin(self, async_client: AsyncNeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - await async_client.experiments.with_raw_response.unpin( + await async_client.evaluations.with_raw_response.unpin( name="name", workspace="", ) with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): - await async_client.experiments.with_raw_response.unpin( + await async_client.evaluations.with_raw_response.unpin( name="", workspace="workspace", ) diff --git a/sdk/stainless.yaml b/sdk/stainless.yaml index 4172d676e3..5bddc29758 100644 --- a/sdk/stainless.yaml +++ b/sdk/stainless.yaml @@ -912,29 +912,29 @@ resources: retrieve: get /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} update: put /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} delete: delete /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} - experiments: + evaluations: standalone_api: true models: evaluator_aggregate: EvaluatorAggregate - experiment_filter: ExperimentFilter - experiment_request: ExperimentRequest - experiment_response: ExperimentResponse - experiment_responses_page: ExperimentResponsesPage + evaluation_filter: EvaluationFilter + evaluation_request: EvaluationRequest + evaluation_response: EvaluationResponse + evaluation_responses_page: EvaluationResponsesPage metric_stat_filters: MetricStatFilters number_filter: NumberFilter methods: - create: post /apis/intake/v2/workspaces/{workspace}/experiments - list: get /apis/intake/v2/workspaces/{workspace}/experiments - retrieve: get /apis/intake/v2/workspaces/{workspace}/experiments/{name} - update: put /apis/intake/v2/workspaces/{workspace}/experiments/{name} - delete: delete /apis/intake/v2/workspaces/{workspace}/experiments/{name} - pin: post /apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin - unpin: delete /apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin + create: post /apis/intake/v2/workspaces/{workspace}/evaluations + list: get /apis/intake/v2/workspaces/{workspace}/evaluations + retrieve: get /apis/intake/v2/workspaces/{workspace}/evaluations/{name} + update: put /apis/intake/v2/workspaces/{workspace}/evaluations/{name} + delete: delete /apis/intake/v2/workspaces/{workspace}/evaluations/{name} + pin: post /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin + unpin: delete /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin subresources: sessions: models: - experiment_session_filter: ExperimentSessionFilter - experiment_session_response: ExperimentSessionResponse - experiment_session_responses_page: ExperimentSessionResponsesPage + evaluation_session_filter: EvaluationSessionFilter + evaluation_session_response: EvaluationSessionResponse + evaluation_session_responses_page: EvaluationSessionResponsesPage methods: - list: get /apis/intake/v2/workspaces/{workspace}/experiments/{name}/sessions + list: get /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/sessions diff --git a/services/core/auth/scripts/auth-tools.py b/services/core/auth/scripts/auth-tools.py index 4de724a60d..925948bf86 100755 --- a/services/core/auth/scripts/auth-tools.py +++ b/services/core/auth/scripts/auth-tools.py @@ -352,6 +352,11 @@ def infer_permissions(path: str, method: str) -> List[str]: permission_prefix = f"{resource}.{sub_resource}" if sub_resource else resource + # pin/unpin mutate an existing resource in place; both belong under `update`, not the default + # post→create / delete→delete verb mapping. + if path.endswith("/pin"): + return [f"{permission_prefix}.update"] + # Map HTTP methods to permissions method_to_permission = { "get": "list" if path.endswith(resource) or path.endswith(f"{resource}s") else "read", diff --git a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml index ea8130d16c..ef7b76287e 100644 --- a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml +++ b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml @@ -137,15 +137,15 @@ authz: description: "Read intake experiment groups" update: description: "Update intake experiment groups" - experiments: + evaluations: create: - description: "Create intake experiments" + description: "Create intake evaluations" delete: - description: "Delete intake experiments" + description: "Delete intake evaluations" read: - description: "Read intake experiments" + description: "Read intake evaluations" update: - description: "Update intake experiments" + description: "Update intake evaluations" ingest: create: description: "Ingest traces into intake" @@ -302,10 +302,10 @@ authz: - inference.virtual-models.read - intake.annotations.list - intake.annotations.read + - intake.evaluations.read - intake.evaluator-results.list - intake.evaluator-results.read - intake.experiment-groups.read - - intake.experiments.read - intake.spans.list - intake.spans.read - intake.traces.read @@ -349,13 +349,13 @@ authz: - inference.virtual-models.update - intake.annotations.create - intake.annotations.delete + - intake.evaluations.create + - intake.evaluations.delete + - intake.evaluations.update - intake.evaluator-results.create - intake.experiment-groups.create - intake.experiment-groups.delete - intake.experiment-groups.update - - intake.experiments.create - - intake.experiments.delete - - intake.experiments.update - intake.ingest.create - jobs.cancel - jobs.create @@ -877,110 +877,174 @@ authz: scopes: - intake:read - platform:read - /apis/intake/v2/workspaces/{workspace}/evaluator-results: + /apis/intake/v2/workspaces/{workspace}/evaluations: get: permissions: - - intake.evaluator-results.list + - intake.evaluations.read scopes: - intake:read - platform:read post: permissions: - - intake.evaluator-results.create + - intake.evaluations.create scopes: - intake:write - platform:write - /apis/intake/v2/workspaces/{workspace}/evaluator-results/{evaluator_result_id}: - get: + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}: + delete: permissions: - - intake.evaluator-results.read + - intake.evaluations.delete scopes: - - intake:read - - platform:read - /apis/intake/v2/workspaces/{workspace}/experiment-groups: + - intake:write + - platform:write get: permissions: - - intake.experiment-groups.read + - intake.evaluations.read scopes: - intake:read - platform:read - post: + put: permissions: - - intake.experiment-groups.create + - intake.evaluations.update scopes: - intake:write - platform:write - /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin: delete: permissions: - - intake.experiment-groups.delete + - intake.evaluations.update + scopes: + - intake:write + - platform:write + post: + permissions: + - intake.evaluations.update scopes: - intake:write - platform:write + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/sessions: get: permissions: - - intake.experiment-groups.read + - intake.evaluations.read scopes: - intake:read - platform:read - put: - permissions: - - intake.experiment-groups.update - scopes: - - intake:write - - platform:write + # Deprecated /experiments* aliases: the child endpoints moved to /evaluations, but the old + # paths are kept as hidden back-compat routes (include_in_schema=False), so they are absent + # from the OpenAPI-derived bundle. Map them here to the intake.evaluations.* permissions and + # mark x-not-in-openapi so the orphan check skips them. Remove alongside the router aliases. /apis/intake/v2/workspaces/{workspace}/experiments: get: permissions: - - intake.experiments.read + - intake.evaluations.read scopes: - intake:read - platform:read + x-not-in-openapi: true post: permissions: - - intake.experiments.create + - intake.evaluations.create scopes: - intake:write - platform:write + x-not-in-openapi: true /apis/intake/v2/workspaces/{workspace}/experiments/{name}: delete: permissions: - - intake.experiments.delete + - intake.evaluations.delete scopes: - intake:write - platform:write + x-not-in-openapi: true get: permissions: - - intake.experiments.read + - intake.evaluations.read scopes: - intake:read - platform:read + x-not-in-openapi: true put: permissions: - - intake.experiments.update + - intake.evaluations.update scopes: - intake:write - platform:write + x-not-in-openapi: true /apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin: delete: permissions: - - intake.experiments.update + - intake.evaluations.update scopes: - intake:write - platform:write + x-not-in-openapi: true post: permissions: - - intake.experiments.update + - intake.evaluations.update scopes: - intake:write - platform:write + x-not-in-openapi: true /apis/intake/v2/workspaces/{workspace}/experiments/{name}/sessions: get: permissions: - - intake.experiments.read + - intake.evaluations.read + scopes: + - intake:read + - platform:read + x-not-in-openapi: true + /apis/intake/v2/workspaces/{workspace}/evaluator-results: + get: + permissions: + - intake.evaluator-results.list + scopes: + - intake:read + - platform:read + post: + permissions: + - intake.evaluator-results.create + scopes: + - intake:write + - platform:write + /apis/intake/v2/workspaces/{workspace}/evaluator-results/{evaluator_result_id}: + get: + permissions: + - intake.evaluator-results.read scopes: - intake:read - platform:read + /apis/intake/v2/workspaces/{workspace}/experiment-groups: + get: + permissions: + - intake.experiment-groups.read + scopes: + - intake:read + - platform:read + post: + permissions: + - intake.experiment-groups.create + scopes: + - intake:write + - platform:write + /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}: + delete: + permissions: + - intake.experiment-groups.delete + scopes: + - intake:write + - platform:write + get: + permissions: + - intake.experiment-groups.read + scopes: + - intake:read + - platform:read + put: + permissions: + - intake.experiment-groups.update + scopes: + - intake:write + - platform:write /apis/intake/v2/workspaces/{workspace}/ingest/atif: post: permissions: diff --git a/services/core/auth/tests/test_embedded_pdp.py b/services/core/auth/tests/test_embedded_pdp.py index b6c1d0cc7a..17bcba7e40 100644 --- a/services/core/auth/tests/test_embedded_pdp.py +++ b/services/core/auth/tests/test_embedded_pdp.py @@ -192,6 +192,39 @@ def test_authenticated_user_with_permission(self, minimal_authz_data): ) assert result["allowed"] is True + def test_legacy_experiments_url_alias_authorized_through_pdp(self, static_authz_data): + """The deprecated /experiments alias must authorize through the real bundle (not just + FastAPI's router): a normal user with intake.evaluations.* is allowed, exactly like + /evaluations, and a read-only user is denied — proving it's real authz, not a bypass. + + This guards the class of bug where hidden router aliases are absent from the + OpenAPI-derived auth bundle and therefore fail-closed for non-service principals. + """ + static_authz_data["authz"]["principals"] = { + "editor@example.com": {"workspaces": {"ws1": ["Editor"]}}, + "viewer@example.com": {"workspaces": {"ws1": ["Viewer"]}}, + } + set_policy_data(static_authz_data) + + # Editor (has intake.evaluations.create) may POST to both the canonical and the alias URL. + for path in ( + "/apis/intake/v2/workspaces/ws1/evaluations", + "/apis/intake/v2/workspaces/ws1/experiments", + ): + result = evaluate("allow", {"principal_id": "editor@example.com", "path": path, "method": "POST"}) + assert result["allowed"] is True, f"editor should be allowed to POST {path}" + + # Viewer (read-only) is denied POST to the alias — real permission check, not a blanket allow. + denied = evaluate( + "allow", + { + "principal_id": "viewer@example.com", + "path": "/apis/intake/v2/workspaces/ws1/experiments", + "method": "POST", + }, + ) + assert denied["allowed"] is False + def test_service_principal_bypass(self, minimal_authz_data): set_policy_data(minimal_authz_data) result = evaluate( diff --git a/services/intake/scripts/spans/seed_experiment_rollup_data.py b/services/intake/scripts/spans/seed_experiment_rollup_data.py index d65b8857e4..9cacabfe23 100644 --- a/services/intake/scripts/spans/seed_experiment_rollup_data.py +++ b/services/intake/scripts/spans/seed_experiment_rollup_data.py @@ -2,10 +2,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Seed local Intake with valid experiment telemetry and verify API rollups. +"""Seed local Intake with valid evaluation telemetry and verify API rollups. -A small, parameterized smoke test: seeds one experiment (attached to one group) -and ingests sessions via ATIF, then polls the experiment read endpoint until the +A small, parameterized smoke test: seeds one evaluation (attached to one group) +and ingests sessions via ATIF, then polls the evaluation read endpoint until the ClickHouse-hydrated rollup converges. Useful for verifying the rollup pipeline at varying session counts. @@ -26,7 +26,7 @@ DEFAULT_BASE_URL = "http://127.0.0.1:8080" DEFAULT_WORKSPACE = "default" -DEFAULT_EXPERIMENT = "rollup-smoke-exp" +DEFAULT_EVALUATION = "rollup-smoke-exp" DEFAULT_GROUP = "rollup-smoke-group" DATASET_NAME = "rollup-smoke-dataset" AGENT_NAME = "sample-agent" @@ -44,9 +44,9 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-url", default=DEFAULT_BASE_URL) parser.add_argument("--workspace", default=DEFAULT_WORKSPACE) - parser.add_argument("--experiment", default=DEFAULT_EXPERIMENT) + parser.add_argument("--evaluation", default=DEFAULT_EVALUATION) parser.add_argument( - "--runs", type=int, help="Generate this many synthetic experiment runs instead of the smoke set." + "--runs", type=int, help="Generate this many synthetic evaluation runs instead of the smoke set." ) parser.add_argument("--cases-per-run", type=int, default=1, help="Synthetic test cases per run when --runs is set.") args = parser.parse_args() @@ -62,14 +62,14 @@ def main() -> None: with httpx.Client(timeout=10.0) as client: group_id = _upsert_group(client, base_url, args.workspace) - _upsert_experiment(client, base_url, args.workspace, args.experiment, group_id=group_id) + _upsert_evaluation(client, base_url, args.workspace, args.evaluation, group_id=group_id) started_at = datetime.now(timezone.utc).replace(microsecond=0) for index, (run_id, test_case_id, score, cost_usd, latency_ms) in enumerate(sample_rows): response = client.post( _intake_url(base_url, args.workspace, "/ingest/atif"), json=_atif_body( started_at=started_at, - experiment_id=args.experiment, + evaluation_id=args.evaluation, run_id=run_id, test_case_id=test_case_id, score=score, @@ -83,15 +83,15 @@ def main() -> None: print(f"seeded {index + 1} sessions") expected_session_count = len(sample_rows) - experiment = _wait_for_rollup( + evaluation = _wait_for_rollup( client, base_url, args.workspace, - args.experiment, + args.evaluation, expected_run_count=expected_session_count, expected_session_count=expected_session_count, ) - print(json.dumps(experiment, indent=2, sort_keys=True)) + print(json.dumps(evaluation, indent=2, sort_keys=True)) def _preflight(base_url: str) -> None: @@ -112,24 +112,24 @@ def _upsert_group(client: httpx.Client, base_url: str, workspace: str) -> str: return response.json()["id"] -def _upsert_experiment( +def _upsert_evaluation( client: httpx.Client, base_url: str, workspace: str, - experiment: str, + evaluation: str, *, group_id: str, ) -> None: body = { - "name": experiment, + "name": evaluation, "dataset_name": DATASET_NAME, "dataset_version": "v1", "experiment_group_id": group_id, "metadata": {"seeded_by": "services/intake/scripts/spans/seed_experiment_rollup_data.py"}, } - response = client.post(_intake_url(base_url, workspace, "/experiments"), json=body) + response = client.post(_intake_url(base_url, workspace, "/evaluations"), json=body) if response.status_code == 409: - response = client.put(_intake_url(base_url, workspace, f"/experiments/{experiment}"), json=body) + response = client.put(_intake_url(base_url, workspace, f"/evaluations/{evaluation}"), json=body) response.raise_for_status() @@ -137,12 +137,12 @@ def _wait_for_rollup( client: httpx.Client, base_url: str, workspace: str, - experiment: str, + evaluation: str, *, expected_run_count: int, expected_session_count: int, ) -> dict[str, Any]: - url = _intake_url(base_url, workspace, f"/experiments/{experiment}") + url = _intake_url(base_url, workspace, f"/evaluations/{evaluation}") last_response: httpx.Response | None = None for _ in range(20): response = client.get(url) @@ -154,13 +154,13 @@ def _wait_for_rollup( return payload time.sleep(0.25) detail = last_response.text if last_response is not None else "" - raise SystemExit(f"Experiment rollup did not become visible at {url}: {detail}") + raise SystemExit(f"Evaluation rollup did not become visible at {url}: {detail}") def _atif_body( *, started_at: datetime, - experiment_id: str, + evaluation_id: str, run_id: str, test_case_id: str, score: float, @@ -170,12 +170,12 @@ def _atif_body( ) -> dict[str, Any]: session_started_at = started_at + timedelta(seconds=offset_seconds) finished_at = session_started_at + timedelta(milliseconds=latency_ms) - session_id = f"{experiment_id}-{run_id}-{test_case_id}" + session_id = f"{evaluation_id}-{run_id}-{test_case_id}" return { "schema_version": "ATIF-v1.7", "session_id": session_id, - "experiment_context": { - "experiment_id": experiment_id, + "evaluation_context": { + "evaluation_id": evaluation_id, "test_case_id": test_case_id, }, "extra": { diff --git a/services/intake/scripts/spans/seed_experiments_demo.py b/services/intake/scripts/spans/seed_experiments_demo.py index 2cfe63ae83..41fd10911c 100644 --- a/services/intake/scripts/spans/seed_experiments_demo.py +++ b/services/intake/scripts/spans/seed_experiments_demo.py @@ -2,22 +2,22 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Seed local Intake with a curated multi-group experiment dataset. +"""Seed local Intake with a curated multi-group evaluation dataset. Designed for the Studio UI team to have realistic data while building out the -Experiments surfaces. Seeds four experiment groups across multiple agents and -datasets, with varied evaluator scores, costs, and latencies. Every experiment +Evaluations surfaces. Seeds four experiment groups across multiple agents and +datasets, with varied evaluator scores, costs, and latencies. Every evaluation is attached to a group; nothing is ungrouped. Behavior: * Re-running is safe. Groups that already exist by name are left alone — none - of their experiments, sessions, or rollups are touched. -* ``--wipe-and-seed`` deletes **every** experiment group and experiment in the + of their evaluations, sessions, or rollups are touched. +* ``--wipe-and-seed`` deletes **every** experiment group and evaluation in the workspace (including ones this script didn't create) and then seeds from scratch. Note: this only removes entity-store rows. ClickHouse session data (spans, evaluator_results) is not deletable via the public API, so prior - session telemetry tagged with a re-used ``experiment.id`` will still feed + session telemetry tagged with a re-used ``evaluation.id`` will still feed the rollup after re-seeding. Usage:: @@ -25,11 +25,11 @@ uv run services/intake/scripts/spans/seed_experiments_demo.py \\ --base-url http://127.0.0.1:8000 - # Wipe ALL experiment groups + experiments in the workspace, then re-seed: + # Wipe ALL experiment groups + evaluations in the workspace, then re-seed: uv run services/intake/scripts/spans/seed_experiments_demo.py \\ --base-url http://127.0.0.1:8000 --wipe-and-seed -For the parameterized smoke test that exercises one experiment + many sessions, +For the parameterized smoke test that exercises one evaluation + many sessions, see ``seed_experiment_rollup_data.py`` in the same directory. """ @@ -54,14 +54,14 @@ @dataclass -class ExperimentSpec: +class EvaluationSpec: name: str description: str agent_name: str = "sample-agent" agent_version: str = "1.0.0" # Optional: sessions cycle through these instead of the scalar above, so a single - # experiment's rollup can surface multiple distinct agent names or versions (e.g., - # an A/B between agent versions, or a comparison of agents within one experiment). + # evaluation's rollup can surface multiple distinct agent names or versions (e.g., + # an A/B between agent versions, or a comparison of agents within one evaluation). agent_name_cycle: tuple[str, ...] | None = None agent_version_cycle: tuple[str, ...] | None = None model_name: str = "provider/sample-model" @@ -86,7 +86,7 @@ class ExperimentSpec: class GroupSpec: name: str description: str - experiments: list[ExperimentSpec] + evaluations: list[EvaluationSpec] DEMO_GROUPS: list[GroupSpec] = [ @@ -94,8 +94,8 @@ class GroupSpec: GroupSpec( name="reranker-prompt-iteration", description="Iterating on the Support-Bench RAG agent's reranker and system prompt.", - experiments=[ - ExperimentSpec( + evaluations=[ + EvaluationSpec( name="reranker-main-baseline", description="Pre-reranker baseline on the production prompt.", agent_name="codex-cli", @@ -108,7 +108,7 @@ class GroupSpec: cost_mean_usd=0.045, latency_mean_ms=2800, ), - ExperimentSpec( + EvaluationSpec( name="reranker-add-cross-encoder", description="Add a cross-encoder reranker to the retrieval step.", agent_name="codex-cli", @@ -121,7 +121,7 @@ class GroupSpec: cost_mean_usd=0.052, latency_mean_ms=3100, ), - ExperimentSpec( + EvaluationSpec( name="reranker-tightened-prompt", description="Cross-encoder reranker + tightened system prompt.", agent_name="codex-cli", @@ -134,7 +134,7 @@ class GroupSpec: cost_mean_usd=0.053, latency_mean_ms=3000, ), - ExperimentSpec( + EvaluationSpec( name="reranker-top-k-8", description="Ablation: top_k = 8 instead of the default 4.", agent_name="codex-cli", @@ -147,7 +147,7 @@ class GroupSpec: cost_mean_usd=0.075, latency_mean_ms=4100, ), - ExperimentSpec( + EvaluationSpec( name="reranker-no-reranker", description="Ablation: prompt change without the reranker.", agent_name="codex-cli", @@ -160,12 +160,12 @@ class GroupSpec: cost_mean_usd=0.038, latency_mean_ms=2400, ), - ExperimentSpec( + EvaluationSpec( name="reranker-5x-averaged", description="Cross-encoder reranker, 5 trials per case, averaged for variance.", agent_name="codex-cli", agent_version="1.2.3", - # Trials ran across a minor version bump mid-experiment. + # Trials ran across a minor version bump mid-evaluation. agent_version_cycle=("1.2.3", "1.2.4"), model_name="openai/gpt-4o-mini", dataset_name="support-bench", @@ -182,8 +182,8 @@ class GroupSpec: GroupSpec( name="coding-agent-showdown", description="Comparing three coding agents on terminal-bench-2.", - experiments=[ - ExperimentSpec( + evaluations=[ + EvaluationSpec( name="tb2-claude-code-opus", description="claude-code @ 0.125 with opus.", agent_name="claude-code", @@ -196,7 +196,7 @@ class GroupSpec: cost_mean_usd=0.42, latency_mean_ms=8200, ), - ExperimentSpec( + EvaluationSpec( name="tb2-codex-cli", description="codex-cli @ 1.4 with gpt-4o.", agent_name="codex-cli", @@ -209,7 +209,7 @@ class GroupSpec: cost_mean_usd=0.22, latency_mean_ms=5400, ), - ExperimentSpec( + EvaluationSpec( name="tb2-cursor-agent", description="cursor-agent + cursor-cli mix @ 0.4 with claude-sonnet.", agent_name="cursor-agent", @@ -230,8 +230,8 @@ class GroupSpec: GroupSpec( name="cross-agent-cross-dataset-sweep", description="Two agents × two datasets to see where each agent shines.", - experiments=[ - ExperimentSpec( + evaluations=[ + EvaluationSpec( name="claude-code-on-agentic-bench", description="claude-code on agentic-bench (long-horizon).", agent_name="claude-code", @@ -244,7 +244,7 @@ class GroupSpec: cost_mean_usd=0.55, latency_mean_ms=12000, ), - ExperimentSpec( + EvaluationSpec( name="claude-code-on-support", description="claude-code on customer-support v2.", agent_name="claude-code", @@ -257,7 +257,7 @@ class GroupSpec: cost_mean_usd=0.12, latency_mean_ms=3100, ), - ExperimentSpec( + EvaluationSpec( name="codex-cli-on-agentic-bench", description="codex-cli on agentic-bench (long-horizon).", agent_name="codex-cli", @@ -270,7 +270,7 @@ class GroupSpec: cost_mean_usd=0.31, latency_mean_ms=8500, ), - ExperimentSpec( + EvaluationSpec( name="codex-cli-on-support", description="codex-cli on customer-support v2.", agent_name="codex-cli", @@ -289,8 +289,8 @@ class GroupSpec: GroupSpec( name="claude-model-size-sweep", description="Same agent + dataset, three claude model sizes.", - experiments=[ - ExperimentSpec( + evaluations=[ + EvaluationSpec( name="tau-claude-haiku", description="claude-code with claude-haiku-4-5.", agent_name="claude-code", @@ -303,7 +303,7 @@ class GroupSpec: cost_mean_usd=0.02, latency_mean_ms=1400, ), - ExperimentSpec( + EvaluationSpec( name="tau-claude-sonnet", description="claude-code with claude-sonnet-4-6.", agent_name="claude-code", @@ -316,7 +316,7 @@ class GroupSpec: cost_mean_usd=0.11, latency_mean_ms=3200, ), - ExperimentSpec( + EvaluationSpec( name="tau-claude-opus", description="claude-code with claude-opus-4-7.", agent_name="claude-code", @@ -347,7 +347,7 @@ def main() -> None: "--wipe-and-seed", action="store_true", help=( - "DELETE every experiment group + experiment in the workspace (including ones this " + "DELETE every experiment group + evaluation in the workspace (including ones this " "script didn't create), then re-seed. Destructive — use carefully." ), ) @@ -375,27 +375,27 @@ def seed(client: httpx.Client, base_url: str, workspace: str) -> None: groups_created = 0 groups_skipped = 0 - experiments_created = 0 + evaluations_created = 0 sessions_seeded = 0 for group_spec in DEMO_GROUPS: group_id, created = _create_group_if_missing(client, base_url, workspace, group_spec) if not created or group_id is None: - print(f"\n[skip] group '{group_spec.name}' already exists; leaving it and its experiments alone") + print(f"\n[skip] group '{group_spec.name}' already exists; leaving it and its evaluations alone") groups_skipped += 1 continue groups_created += 1 - print(f"\n[group] {group_spec.name} ({len(group_spec.experiments)} experiments)") - for exp_spec in group_spec.experiments: - print(f" [experiment] {exp_spec.name} n_sessions={exp_spec.n_sessions}") - _create_experiment(client, base_url, workspace, exp_spec, group_id=group_id) + print(f"\n[group] {group_spec.name} ({len(group_spec.evaluations)} evaluations)") + for exp_spec in group_spec.evaluations: + print(f" [evaluation] {exp_spec.name} n_sessions={exp_spec.n_sessions}") + _create_evaluation(client, base_url, workspace, exp_spec, group_id=group_id) _seed_sessions(client, base_url, workspace, exp_spec, base_started_at) - experiments_created += 1 + evaluations_created += 1 sessions_seeded += exp_spec.n_sessions print( f"\n=== Done. groups: {groups_created} created, {groups_skipped} skipped. " - f"experiments: {experiments_created} created. sessions: {sessions_seeded} ingested. ===" + f"evaluations: {evaluations_created} created. sessions: {sessions_seeded} ingested. ===" ) @@ -412,20 +412,20 @@ def _create_group_if_missing( return response.json()["id"], True -def _create_experiment( +def _create_evaluation( client: httpx.Client, base_url: str, workspace: str, - spec: ExperimentSpec, + spec: EvaluationSpec, *, group_id: str, ) -> None: - """POST an experiment. Errors on conflict — callers must guarantee the experiment doesn't exist.""" - response = client.post(_intake_url(base_url, workspace, "/experiments"), json=_experiment_body(spec, group_id)) + """POST an evaluation. Errors on conflict — callers must guarantee the evaluation doesn't exist.""" + response = client.post(_intake_url(base_url, workspace, "/evaluations"), json=_evaluation_body(spec, group_id)) response.raise_for_status() -def _experiment_body(spec: ExperimentSpec, group_id: str) -> dict[str, Any]: +def _evaluation_body(spec: EvaluationSpec, group_id: str) -> dict[str, Any]: body: dict[str, Any] = { "name": spec.name, "dataset_name": spec.dataset_name, @@ -447,11 +447,11 @@ def _seed_sessions( client: httpx.Client, base_url: str, workspace: str, - spec: ExperimentSpec, + spec: EvaluationSpec, base_started_at: datetime, ) -> None: """Ingest N sessions via ATIF + per-evaluator POST /evaluator-results.""" - # Deterministic per-experiment so re-runs produce the same values. + # Deterministic per-evaluation so re-runs produce the same values. rng = random.Random(f"seed:{spec.name}") atif_url = _intake_url(base_url, workspace, "/ingest/atif") @@ -478,7 +478,7 @@ def _seed_sessions( ) atif_body = _demo_atif_body( base_started_at=base_started_at, - experiment_id=spec.name, + evaluation_id=spec.name, run_id=run_id, test_case_id=test_case_id, cost_usd=cost_usd, @@ -518,7 +518,7 @@ def _seed_sessions( def _demo_atif_body( *, base_started_at: datetime, - experiment_id: str, + evaluation_id: str, run_id: str, test_case_id: str, cost_usd: float, @@ -532,15 +532,15 @@ def _demo_atif_body( ) -> dict[str, Any]: session_started_at = base_started_at + timedelta(seconds=offset_seconds) finished_at = session_started_at + timedelta(milliseconds=latency_ms) - session_id = f"{experiment_id}-{run_id}-{test_case_id}" + session_id = f"{evaluation_id}-{run_id}-{test_case_id}" # `extra.verifier` carries the timing block (used by the rollup for session latency). # We omit `extra.verifier_result` so ATIF ingest doesn't auto-create a `harbor.verifier` # evaluator alongside our cleanly-named ones from POST /evaluator-results. return { "schema_version": "ATIF-v1.7", "session_id": session_id, - "experiment_context": { - "experiment_id": experiment_id, + "evaluation_context": { + "evaluation_id": evaluation_id, "test_case_id": test_case_id, }, "extra": { @@ -585,23 +585,23 @@ def _demo_atif_body( def _wipe_workspace(client: httpx.Client, base_url: str, workspace: str) -> None: - """Delete every experiment + experiment group in the workspace. + """Delete every evaluation + experiment group in the workspace. Iterates the existing list endpoints with pagination and DELETEs each row. - Experiments are deleted before groups so the group-level UI doesn't briefly + Evaluations are deleted before groups so the group-level UI doesn't briefly show empty groups. Only entity-store rows are removed; ClickHouse session data is untouched (no public API to delete it). """ - print(f"=== --wipe-and-seed: deleting every experiment + group in workspace '{workspace}' ===") - deleted_experiments = 0 - for name in _list_all_names(client, base_url, workspace, "/experiments"): - if _delete(client, base_url, workspace, f"/experiments/{name}"): - deleted_experiments += 1 + print(f"=== --wipe-and-seed: deleting every evaluation + group in workspace '{workspace}' ===") + deleted_evaluations = 0 + for name in _list_all_names(client, base_url, workspace, "/evaluations"): + if _delete(client, base_url, workspace, f"/evaluations/{name}"): + deleted_evaluations += 1 deleted_groups = 0 for name in _list_all_names(client, base_url, workspace, "/experiment-groups"): if _delete(client, base_url, workspace, f"/experiment-groups/{name}"): deleted_groups += 1 - print(f"deleted {deleted_experiments} experiment(s) and {deleted_groups} group(s)\n") + print(f"deleted {deleted_evaluations} evaluation(s) and {deleted_groups} group(s)\n") def _list_all_names(client: httpx.Client, base_url: str, workspace: str, suffix: str) -> list[str]: diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index c72433c0fd..6449e2118a 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Create, list, get, and delete endpoints for Experiments and ExperimentGroups. +"""Create, list, get, and delete endpoints for Evaluations and ExperimentGroups. Entity-store (Postgres) operations are wired directly onto ``EntityClient``, following the inline pattern used by the core services. PUT updates only the -mutable fields; an Experiment's identity and the dataset/agent it ran against +mutable fields; an Evaluation's identity and the dataset/agent it ran against are fixed. Rollup fields on read models are hydrated from ClickHouse. """ @@ -18,6 +18,7 @@ from typing import Annotated, Any, Literal, NamedTuple, TypeVar from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +from fastapi.routing import APIRoute from nmp.common.api.common import Page, PaginationData from nmp.common.api.filter import ComparisonOperation, FilterOperation, FilterOperator, LogicalOperation from nmp.common.api.parsed_filter import ParsedFilter, make_filter_dep @@ -25,28 +26,33 @@ from nmp.common.entities.client import EntityClient, EntityConflictError, EntityNotFoundError from nmp.common.service.dependencies import get_entity_client from nmp.intake.api.v2.experiments.schemas import ( - EXPERIMENT_SESSION_SUMMARY_INPUT_CHAR_LIMIT, + EVALUATION_SESSION_SUMMARY_INPUT_CHAR_LIMIT, + EvaluationFilter, + EvaluationRequest, + EvaluationResponse, + EvaluationSessionFilter, + EvaluationSessionMode, + EvaluationSessionResponse, EvaluatorAggregate, - ExperimentFilter, ExperimentGroupFilter, ExperimentGroupRequest, ExperimentGroupResponse, - ExperimentRequest, - ExperimentResponse, - ExperimentSessionFilter, - ExperimentSessionMode, - ExperimentSessionResponse, ) -from nmp.intake.entities.experiments import Experiment, ExperimentGroup + +# The API/Studio expose this as an "Evaluation", but it is still stored as the Experiment entity +# (entity rename + data migration deferred — see entities/experiments.py). Alias it to the name this +# layer uses; only the entity's own field names (e.g. parent_experiment_id) reference Experiment directly. +from nmp.intake.entities.experiments import Experiment as Evaluation +from nmp.intake.entities.experiments import ExperimentGroup from nmp.intake.spans.api.dependencies import require_workspace_access, validate_list_query_params from nmp.intake.spans.clickhouse_client import ClickHouseSpanClient from nmp.intake.spans.domain import SpanStatus -from nmp.intake.spans.experiment_rollup_repository import ( - ExperimentRollup, - ExperimentRollupRepository, +from nmp.intake.spans.evaluation_rollup_repository import ( + EvaluationRollup, + EvaluationRollupRepository, ScoreRollup, ) -from nmp.intake.spans.experiment_session_repository import ExperimentSessionRepository +from nmp.intake.spans.evaluation_session_repository import EvaluationSessionRepository from nmp.intake.spans.storage import make_pagination logger = logging.getLogger(__name__) @@ -59,26 +65,26 @@ def _sanitize_for_log(value: str) -> str: router = APIRouter(dependencies=[Depends(require_workspace_access)]) GROUPS_TAG = "Experiment Groups" -EXPERIMENTS_TAG = "Experiments" +EVALUATIONS_TAG = "Evaluations" ExperimentGroupSortField = Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"] -# The experiments list is sorted in the application layer (compute-on-read) so a single request can +# The evaluations list is sorted in the application layer (compute-on-read) so a single request can # sort by a ClickHouse rollup metric, not just entity columns. `sort` is therefore a free string, # validated against these: an entity column, run_count, or a `.` rollup path. _ENTITY_SORT_FIELDS = frozenset({"name", "created_at", "updated_at", "pinned_at"}) _METRIC_STATS = frozenset({"sum", "mean", "median", "p90", "p95", "p99", "count"}) -# Per-group experiment fetch bound for the in-memory merge. Groups are expected to hold at most +# Per-group evaluation fetch bound for the in-memory merge. Groups are expected to hold at most # hundreds; a query that selects more than this is rejected rather than sorted on a partial set — the # trigger to denormalize metrics into an entity-store-sortable column instead. -_MAX_GROUP_EXPERIMENTS = 1000 +_MAX_GROUP_EVALUATIONS = 1000 -EntityT = TypeVar("EntityT", Experiment, ExperimentGroup) +EntityT = TypeVar("EntityT", Evaluation, ExperimentGroup) EntityClientDep = Annotated[EntityClient, Depends(get_entity_client)] ExperimentGroupFilterDep = Annotated[ParsedFilter, Depends(make_filter_dep(ExperimentGroupFilter))] -ExperimentFilterDep = Annotated[ParsedFilter, Depends(make_filter_dep(ExperimentFilter))] -ExperimentSessionFilterDep = Annotated[ParsedFilter, Depends(make_filter_dep(ExperimentSessionFilter))] +EvaluationFilterDep = Annotated[ParsedFilter, Depends(make_filter_dep(EvaluationFilter))] +EvaluationSessionFilterDep = Annotated[ParsedFilter, Depends(make_filter_dep(EvaluationSessionFilter))] def _get_clickhouse_client(request: Request) -> ClickHouseSpanClient | None: @@ -88,23 +94,23 @@ def _get_clickhouse_client(request: Request) -> ClickHouseSpanClient | None: return getattr(service, "clickhouse_client", None) -def get_experiment_rollup_repository(request: Request) -> ExperimentRollupRepository | None: - # Rollups are enrichment only. Experiment entity reads should continue when +def get_evaluation_rollup_repository(request: Request) -> EvaluationRollupRepository | None: + # Rollups are enrichment only. Evaluation entity reads should continue when # ClickHouse is disabled or temporarily unavailable. client = _get_clickhouse_client(request) - return ExperimentRollupRepository(client) if client is not None else None + return EvaluationRollupRepository(client) if client is not None else None -ExperimentRollupRepositoryDep = Annotated[ExperimentRollupRepository | None, Depends(get_experiment_rollup_repository)] +EvaluationRollupRepositoryDep = Annotated[EvaluationRollupRepository | None, Depends(get_evaluation_rollup_repository)] -def get_experiment_session_repository(request: Request) -> ExperimentSessionRepository | None: +def get_evaluation_session_repository(request: Request) -> EvaluationSessionRepository | None: client = _get_clickhouse_client(request) - return ExperimentSessionRepository(client) if client is not None else None + return EvaluationSessionRepository(client) if client is not None else None -ExperimentSessionRepositoryDep = Annotated[ - ExperimentSessionRepository | None, Depends(get_experiment_session_repository) +EvaluationSessionRepositoryDep = Annotated[ + EvaluationSessionRepository | None, Depends(get_evaluation_session_repository) ] @@ -173,11 +179,11 @@ async def list_experiment_groups( page_size=page_size, ) responses = [ExperimentGroupResponse.from_entity(e) for e in result.data] - counts = await _count_live_experiments_by_group( + counts = await _count_live_evaluations_by_group( entity_client, workspace=workspace, group_ids=[g.id for g in result.data] ) for response in responses: - response.experiment_count = counts.get(response.id, 0) + response.evaluation_count = counts.get(response.id, 0) return Page( data=responses, pagination=PaginationData(**result.pagination.model_dump()), @@ -206,7 +212,7 @@ async def get_experiment_group( ) _reject_if_deleted(entity, workspace=workspace, name=name, label="Experiment group") response = ExperimentGroupResponse.from_entity(entity) - response.experiment_count = await _count_live_experiments_in_group( + response.evaluation_count = await _count_live_evaluations_in_group( entity_client, workspace=workspace, group_id=entity.id ) return response @@ -248,7 +254,7 @@ async def update_experiment_group( existing.default_sort = body.default_sort updated = await entity_client.update(existing) response = ExperimentGroupResponse.from_entity(updated) - response.experiment_count = await _count_live_experiments_in_group( + response.evaluation_count = await _count_live_evaluations_in_group( entity_client, workspace=workspace, group_id=updated.id ) return response @@ -267,7 +273,7 @@ async def delete_experiment_group( ) -> None: # Soft delete: flip ``is_deleted`` and rename the row so the original name is free for reuse. # The unique index on (workspace, entity_type, name) doesn't read into the JSON data column, - # so renaming on delete is what lets a new group/experiment claim the same name later. + # so renaming on delete is what lets a new group/evaluation claim the same name later. group = await _get_or_404( entity_client, ExperimentGroup, @@ -278,7 +284,7 @@ async def delete_experiment_group( _reject_if_deleted(group, workspace=workspace, name=name, label="Experiment group") # Cascade is sequential — one update per child. Linear in group size, fine for now. If - # groups routinely hold more than a few hundred experiments, add a bulk update endpoint on + # groups routinely hold more than a few hundred evaluations, add a bulk update endpoint on # the entity store rather than parallelizing here (gather hides partial-failure state # without removing the per-row API contract). # @@ -302,7 +308,7 @@ async def delete_experiment_group( ) while True: page = await entity_client.list( - Experiment, + Evaluation, workspace=workspace, filter_operation=live_children_filter, page=1, @@ -316,20 +322,20 @@ async def delete_experiment_group( @router.post( - "/v2/workspaces/{workspace}/experiments", - response_model=ExperimentResponse, + "/v2/workspaces/{workspace}/evaluations", + response_model=EvaluationResponse, status_code=status.HTTP_201_CREATED, - tags=[EXPERIMENTS_TAG], - responses={409: {"description": "Experiment already exists"}}, + tags=[EVALUATIONS_TAG], + responses={409: {"description": "Evaluation already exists"}}, ) -async def create_experiment( +async def create_evaluation( workspace: str, - body: ExperimentRequest, + body: EvaluationRequest, entity_client: EntityClientDep, -) -> ExperimentResponse: +) -> EvaluationResponse: await _validate_group_exists(entity_client, group_id=body.experiment_group_id) - await _validate_parent_experiment_exists(entity_client, parent_experiment_id=body.parent_experiment_id) - entity = Experiment( + await _validate_parent_evaluation_exists(entity_client, parent_evaluation_id=body.parent_evaluation_id) + entity = Evaluation( workspace=workspace, name=body.name, experiment_group_id=body.experiment_group_id, @@ -338,7 +344,7 @@ async def create_experiment( source_link=body.source_link, metadata=body.metadata, description=body.description, - parent_experiment_id=body.parent_experiment_id, + parent_experiment_id=body.parent_evaluation_id, status=body.status, root_cause=body.root_cause, ) @@ -347,26 +353,26 @@ async def create_experiment( except EntityConflictError as e: raise HTTPException( status_code=status.HTTP_409_CONFLICT, - detail=f"Experiment '{workspace}/{body.name}' already exists.", + detail=f"Evaluation '{workspace}/{body.name}' already exists.", ) from e - return ExperimentResponse.from_entity(created) + return EvaluationResponse.from_entity(created) @router.get( - "/v2/workspaces/{workspace}/experiments", - response_model=Page[ExperimentResponse], - tags=[EXPERIMENTS_TAG], + "/v2/workspaces/{workspace}/evaluations", + response_model=Page[EvaluationResponse], + tags=[EVALUATIONS_TAG], responses={ 400: {"description": "Unsupported sort or filter field"}, - 413: {"description": "Too many experiments selected to sort in one request"}, + 413: {"description": "Too many evaluations selected to sort in one request"}, 503: {"description": "Telemetry store unavailable for a metric-based sort or filter"}, }, openapi_extra=generate_openapi_extra_params( - filter_schema=ExperimentFilter, + filter_schema=EvaluationFilter, filter_description=( - "Filter experiments by name, experiment_group_id, " + "Filter evaluations by name, experiment_group_id, " "dataset_name, dataset_version, created_by, created_at, or updated_at. " - "Pass is_deleted=true to return only soft-deleted experiments; omit to see only live ones. " + "Pass is_deleted=true to return only soft-deleted evaluations; omit to see only live ones. " "Pass is_pinned=true (or false) to filter by pinned state; omit to return both. " "Filter by a metadata key/value: filter[metadata.]=. " "Filter by a rollup metric with numeric range operators ($gte/$lte/$gt/$lt/$eq): " @@ -375,29 +381,29 @@ async def create_experiment( ), ), ) -async def list_experiments( +async def list_evaluations( workspace: str, request: Request, entity_client: EntityClientDep, - rollup_repository: ExperimentRollupRepositoryDep, - parsed: ExperimentFilterDep, + rollup_repository: EvaluationRollupRepositoryDep, + parsed: EvaluationFilterDep, page: int = Query(default=1, ge=1, description="Page number."), page_size: int = Query(default=100, ge=1, le=1000, description="Page size."), sort: str | None = Query( default=None, description=( - "Field to sort by; prefix with '-' for descending. Sort by an experiment attribute " + "Field to sort by; prefix with '-' for descending. Sort by an evaluation attribute " "(name, created_at, updated_at, pinned_at) or by an aggregate metric: run_count, " "cost_usd., latency_ms., or evaluators.., where is one of " "mean, median, p90, p95, p99, sum, count. When omitted, defaults to -created_at with pinned " - "experiments first." + "evaluations first." ), ), -) -> Page[ExperimentResponse]: +) -> Page[EvaluationResponse]: validate_list_query_params(request) _apply_is_deleted_filter(parsed) _apply_is_pinned_filter(parsed) - # When omitted, fall back to -created_at with pinned experiments floated to the top. + # When omitted, fall back to -created_at with pinned evaluations floated to the top. if sort is not None: descending = sort.startswith("-") sort_field = sort[1:] if descending else sort @@ -416,31 +422,31 @@ async def list_experiments( entity_operation, metric_predicates = _extract_metric_predicates(parsed.operation) # Compute-on-read: fetch the whole (entity-filtered) group, hydrate every rollup, then filter, sort, # and paginate in memory so a single request can sort/filter by a ClickHouse metric that lives - # outside the entity store. Bounded to hundreds of experiments per group (see _MAX_GROUP_EXPERIMENTS). + # outside the entity store. Bounded to hundreds of evaluations per group (see _MAX_GROUP_EVALUATIONS). result = await entity_client.list( - Experiment, + Evaluation, workspace=workspace, filter_operation=entity_operation, page=1, - page_size=_MAX_GROUP_EXPERIMENTS, + page_size=_MAX_GROUP_EVALUATIONS, ) - responses = [ExperimentResponse.from_entity(e) for e in result.data] + responses = [EvaluationResponse.from_entity(e) for e in result.data] total_selected = result.pagination.total_results - if total_selected > _MAX_GROUP_EXPERIMENTS: + if total_selected > _MAX_GROUP_EVALUATIONS: # The whole filtered set is sorted in memory; anything past the fetch cap can't be sorted, so a # returned page would be silently incomplete. Fail loudly and tell the caller how to scope the # query instead (or denormalize rollup metrics for entity-store sorting once groups grow this big). logger.warning( - "Experiment list selected %d experiments, over the %d-row in-memory sort cap; refusing " + "Evaluation list selected %d evaluations, over the %d-row in-memory sort cap; refusing " "to return a partially sorted result.", total_selected, - _MAX_GROUP_EXPERIMENTS, + _MAX_GROUP_EVALUATIONS, ) raise HTTPException( status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail=( - f"This query selects {total_selected} experiments, exceeding the maximum of " - f"{_MAX_GROUP_EXPERIMENTS} that can be sorted in one request. Narrow the result with a " + f"This query selects {total_selected} evaluations, exceeding the maximum of " + f"{_MAX_GROUP_EVALUATIONS} that can be sorted in one request. Narrow the result with a " "filter (e.g. experiment_group_id)." ), ) @@ -455,11 +461,11 @@ async def list_experiments( if not hydrated and (explicit_metric_sort or metric_predicates): raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Cannot sort or filter experiments by a rollup metric: the telemetry store is unavailable.", + detail="Cannot sort or filter evaluations by a rollup metric: the telemetry store is unavailable.", ) if metric_predicates: responses = [r for r in responses if _matches_metric_predicates(r, metric_predicates)] - ordered = _sort_experiments(responses, keys=sort_keys, pinned_first=pinned_first) + ordered = _sort_evaluations(responses, keys=sort_keys, pinned_first=pinned_first) start = (page - 1) * page_size page_items = ordered[start : start + page_size] return Page( @@ -473,71 +479,71 @@ async def list_experiments( @router.get( - "/v2/workspaces/{workspace}/experiments/{name}", - response_model=ExperimentResponse, - tags=[EXPERIMENTS_TAG], - responses={404: {"description": "Experiment not found"}}, + "/v2/workspaces/{workspace}/evaluations/{name}", + response_model=EvaluationResponse, + tags=[EVALUATIONS_TAG], + responses={404: {"description": "Evaluation not found"}}, ) -async def get_experiment( +async def get_evaluation( workspace: str, name: str, entity_client: EntityClientDep, - rollup_repository: ExperimentRollupRepositoryDep, -) -> ExperimentResponse: + rollup_repository: EvaluationRollupRepositoryDep, +) -> EvaluationResponse: entity = await _get_or_404( entity_client, - Experiment, + Evaluation, workspace=workspace, name=name, - label="Experiment", + label="Evaluation", ) - _reject_if_deleted(entity, workspace=workspace, name=name, label="Experiment") - response = ExperimentResponse.from_entity(entity) + _reject_if_deleted(entity, workspace=workspace, name=name, label="Evaluation") + response = EvaluationResponse.from_entity(entity) await _hydrate_rollups(workspace=workspace, responses=[response], rollup_repository=rollup_repository) return response # Identity and the dataset it was run against are fixed for the life of an -# Experiment (see the ingest invariants); changing them means it's a different -# Experiment. PUT may only edit group membership, source link, description, metadata. -_IMMUTABLE_EXPERIMENT_FIELDS = ("name", "dataset_name", "dataset_version") +# Evaluation (see the ingest invariants); changing them means it's a different +# Evaluation. PUT may only edit group membership, source link, description, metadata. +_IMMUTABLE_EVALUATION_FIELDS = ("name", "dataset_name", "dataset_version") @router.put( - "/v2/workspaces/{workspace}/experiments/{name}", - response_model=ExperimentResponse, - tags=[EXPERIMENTS_TAG], + "/v2/workspaces/{workspace}/evaluations/{name}", + response_model=EvaluationResponse, + tags=[EVALUATIONS_TAG], responses={ - 404: {"description": "Experiment not found"}, + 404: {"description": "Evaluation not found"}, 409: {"description": "Attempt to change an immutable field"}, }, ) -async def update_experiment( +async def update_evaluation( workspace: str, name: str, - body: ExperimentRequest, + body: EvaluationRequest, entity_client: EntityClientDep, - rollup_repository: ExperimentRollupRepositoryDep, -) -> ExperimentResponse: + rollup_repository: EvaluationRollupRepositoryDep, +) -> EvaluationResponse: existing = await _get_or_404( entity_client, - Experiment, + Evaluation, workspace=workspace, name=name, - label="Experiment", + label="Evaluation", ) - _reject_if_deleted(existing, workspace=workspace, name=name, label="Experiment") + _reject_if_deleted(existing, workspace=workspace, name=name, label="Evaluation") if body.experiment_group_id != existing.experiment_group_id: await _validate_group_exists(entity_client, group_id=body.experiment_group_id) - await _validate_parent_experiment_exists(entity_client, parent_experiment_id=body.parent_experiment_id) + await _validate_parent_evaluation_exists(entity_client, parent_evaluation_id=body.parent_evaluation_id) - changed = [f for f in _IMMUTABLE_EXPERIMENT_FIELDS if getattr(body, f) != getattr(existing, f)] + changed = [f for f in _IMMUTABLE_EVALUATION_FIELDS if getattr(body, f) != getattr(existing, f)] if changed: raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail=( - f"Cannot change immutable field(s) {changed} on an existing experiment; " - "create a new experiment instead." + f"Cannot change immutable field(s) {changed} on an existing evaluation; " + "create a new evaluation instead." ), ) @@ -545,136 +551,137 @@ async def update_experiment( existing.source_link = body.source_link existing.metadata = body.metadata existing.description = body.description - existing.parent_experiment_id = body.parent_experiment_id + existing.parent_experiment_id = body.parent_evaluation_id existing.status = body.status existing.root_cause = body.root_cause updated = await entity_client.update(existing) - response = ExperimentResponse.from_entity(updated) + response = EvaluationResponse.from_entity(updated) await _hydrate_rollups(workspace=workspace, responses=[response], rollup_repository=rollup_repository) return response @router.delete( - "/v2/workspaces/{workspace}/experiments/{name}", + "/v2/workspaces/{workspace}/evaluations/{name}", status_code=status.HTTP_204_NO_CONTENT, - tags=[EXPERIMENTS_TAG], - responses={404: {"description": "Experiment not found"}}, + tags=[EVALUATIONS_TAG], + responses={404: {"description": "Evaluation not found"}}, ) -async def delete_experiment( +async def delete_evaluation( workspace: str, name: str, entity_client: EntityClientDep, ) -> None: entity = await _get_or_404( entity_client, - Experiment, + Evaluation, workspace=workspace, name=name, - label="Experiment", + label="Evaluation", ) - _reject_if_deleted(entity, workspace=workspace, name=name, label="Experiment") + _reject_if_deleted(entity, workspace=workspace, name=name, label="Evaluation") await _soft_delete(entity_client, entity) @router.post( - "/v2/workspaces/{workspace}/experiments/{name}/pin", - response_model=ExperimentResponse, - tags=[EXPERIMENTS_TAG], - responses={404: {"description": "Experiment not found"}}, + "/v2/workspaces/{workspace}/evaluations/{name}/pin", + response_model=EvaluationResponse, + tags=[EVALUATIONS_TAG], + responses={404: {"description": "Evaluation not found"}}, ) -async def pin_experiment( +async def pin_evaluation( workspace: str, name: str, entity_client: EntityClientDep, - rollup_repository: ExperimentRollupRepositoryDep, -) -> ExperimentResponse: - """Pin an experiment to the top of the list (workspace-shared). + rollup_repository: EvaluationRollupRepositoryDep, +) -> EvaluationResponse: + """Pin an evaluation to the top of the list (workspace-shared). - Re-pinning an already-pinned experiment refreshes ``pinned_at`` to the current timestamp, + Re-pinning an already-pinned evaluation refreshes ``pinned_at`` to the current timestamp, which is intentional (most-recently-pinned sorts first). """ entity = await _get_or_404( entity_client, - Experiment, + Evaluation, workspace=workspace, name=name, - label="Experiment", + label="Evaluation", ) - _reject_if_deleted(entity, workspace=workspace, name=name, label="Experiment") + _reject_if_deleted(entity, workspace=workspace, name=name, label="Evaluation") entity.pinned_at = datetime.now(timezone.utc) updated = await entity_client.update(entity) - response = ExperimentResponse.from_entity(updated) + response = EvaluationResponse.from_entity(updated) await _hydrate_rollups(workspace=workspace, responses=[response], rollup_repository=rollup_repository) return response @router.delete( - "/v2/workspaces/{workspace}/experiments/{name}/pin", - response_model=ExperimentResponse, - tags=[EXPERIMENTS_TAG], - responses={404: {"description": "Experiment not found"}}, + "/v2/workspaces/{workspace}/evaluations/{name}/pin", + response_model=EvaluationResponse, + tags=[EVALUATIONS_TAG], + responses={404: {"description": "Evaluation not found"}}, ) -async def unpin_experiment( +async def unpin_evaluation( workspace: str, name: str, entity_client: EntityClientDep, - rollup_repository: ExperimentRollupRepositoryDep, -) -> ExperimentResponse: - """Unpin an experiment. Idempotent: unpinning an already-unpinned experiment is a no-op.""" + rollup_repository: EvaluationRollupRepositoryDep, +) -> EvaluationResponse: + """Unpin an evaluation. Idempotent: unpinning an already-unpinned evaluation is a no-op.""" entity = await _get_or_404( entity_client, - Experiment, + Evaluation, workspace=workspace, name=name, - label="Experiment", + label="Evaluation", ) - _reject_if_deleted(entity, workspace=workspace, name=name, label="Experiment") + _reject_if_deleted(entity, workspace=workspace, name=name, label="Evaluation") entity.pinned_at = None updated = await entity_client.update(entity) - response = ExperimentResponse.from_entity(updated) + response = EvaluationResponse.from_entity(updated) await _hydrate_rollups(workspace=workspace, responses=[response], rollup_repository=rollup_repository) return response @router.get( - "/v2/workspaces/{workspace}/experiments/{name}/sessions", - response_model=Page[ExperimentSessionResponse], - tags=[EXPERIMENTS_TAG], + "/v2/workspaces/{workspace}/evaluations/{name}/sessions", + response_model=Page[EvaluationSessionResponse], + tags=[EVALUATIONS_TAG], responses={ - 404: {"description": "Experiment not found"}, + 400: {"description": "Invalid filter value"}, + 404: {"description": "Evaluation not found"}, 503: {"description": "ClickHouse unavailable"}, }, openapi_extra=generate_openapi_extra_params( - filter_schema=ExperimentSessionFilter, + filter_schema=EvaluationSessionFilter, filter_description="Filter sessions by test_case_id and status.", ), ) -async def list_experiment_sessions( +async def list_evaluation_sessions( workspace: str, name: str, request: Request, entity_client: EntityClientDep, - session_repository: ExperimentSessionRepositoryDep, - parsed: ExperimentSessionFilterDep, + session_repository: EvaluationSessionRepositoryDep, + parsed: EvaluationSessionFilterDep, page: int = Query(default=1, ge=1, description="Page number."), page_size: int = Query(default=100, ge=1, le=1000, description="Page size."), - mode: ExperimentSessionMode = Query( + mode: EvaluationSessionMode = Query( default="detailed", description=( "Response payload mode. summary keeps the same session row fields but truncates root-span input " "to 1000 characters; detailed returns the full root-span input." ), ), -) -> Page[ExperimentSessionResponse]: +) -> Page[EvaluationSessionResponse]: validate_list_query_params(request) - experiment = await _get_or_404( + evaluation = await _get_or_404( entity_client, - Experiment, + Evaluation, workspace=workspace, name=name, - label="Experiment", + label="Evaluation", ) - _reject_if_deleted(experiment, workspace=workspace, name=name, label="Experiment") + _reject_if_deleted(evaluation, workspace=workspace, name=name, label="Evaluation") if session_repository is None: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, @@ -692,19 +699,19 @@ async def list_experiment_sessions( try: result = await session_repository.list_sessions( workspace=workspace, - experiment_name=name, + evaluation_name=name, status=status_filter, test_case_id=test_case_id, page=page, page_size=page_size, - input_char_limit=EXPERIMENT_SESSION_SUMMARY_INPUT_CHAR_LIMIT if mode == "summary" else None, + input_char_limit=EVALUATION_SESSION_SUMMARY_INPUT_CHAR_LIMIT if mode == "summary" else None, ) except Exception as exc: # Sessions are the response payload (not enrichment), so we can't silently degrade like # _hydrate_rollups does. Convert backend failures (ClickHouse connection drop, query # timeout, etc.) to a deterministic 503 instead of letting them bubble as 500s. logger.exception( - "Per-session read failed for workspace=%s experiment=%s", + "Per-session read failed for workspace=%s evaluation=%s", _sanitize_for_log(workspace), _sanitize_for_log(name), ) @@ -712,7 +719,7 @@ async def list_experiment_sessions( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Telemetry store unavailable.", ) from exc - data = [ExperimentSessionResponse.from_row(row) for row in result.rows] + data = [EvaluationSessionResponse.from_row(row) for row in result.rows] return Page( data=data, pagination=make_pagination( @@ -743,7 +750,7 @@ async def _get_or_404( def _reject_if_deleted( - entity: Experiment | ExperimentGroup, + entity: Evaluation | ExperimentGroup, *, workspace: str, name: str, @@ -790,7 +797,7 @@ def _to_base36(value: int) -> str: return "".join(reversed(out)) -async def _soft_delete(entity_client: EntityClient, entity: Experiment | ExperimentGroup) -> None: +async def _soft_delete(entity_client: EntityClient, entity: Evaluation | ExperimentGroup) -> None: """Flip ``is_deleted`` and rename the entity in a single update.""" original_name = entity.name entity.is_deleted = True @@ -798,14 +805,14 @@ async def _soft_delete(entity_client: EntityClient, entity: Experiment | Experim await entity_client.update(entity, original_name=original_name) -async def _count_live_experiments_in_group(entity_client: EntityClient, *, workspace: str, group_id: str) -> int: - """Return the number of non-soft-deleted experiments in a single group. +async def _count_live_evaluations_in_group(entity_client: EntityClient, *, workspace: str, group_id: str) -> int: + """Return the number of non-soft-deleted evaluations in a single group. Fetches via ``list(page_size=1)`` so the response carries only ``pagination.total_results``. Used by single-group endpoints (GET, PUT). List endpoints should use the bulk variant. """ result = await entity_client.list( - Experiment, + Evaluation, workspace=workspace, filter_operation=LogicalOperation( operator=FilterOperator.AND, @@ -825,25 +832,25 @@ async def _count_live_experiments_in_group(entity_client: EntityClient, *, works return result.pagination.total_results -async def _count_live_experiments_by_group( +async def _count_live_evaluations_by_group( entity_client: EntityClient, *, workspace: str, group_ids: list[str] ) -> dict[str, int]: - """Bulk-count non-soft-deleted experiments for many groups in one (paginated) query. + """Bulk-count non-soft-deleted evaluations for many groups in one (paginated) query. Issues a single ``IN``-filter list against the entity store and tallies per group_id - client-side. Replaces N parallel ``_count_live_experiments_in_group`` calls on the + client-side. Replaces N parallel ``_count_live_evaluations_in_group`` calls on the group-list endpoint so the request shape is 1-to-1 with the entity store rather than 1-to-N (which is fragile under web-server concurrency). Returns a ``{group_id: count}`` map covering every requested group_id, with ``0`` for - groups that have no live experiments. + groups that have no live evaluations. """ counts: dict[str, int] = {group_id: 0 for group_id in group_ids} if not group_ids: return counts page = 1 # Aligned with ``EntityClient.list``'s max — paginates when a workspace's total live - # experiment count across the requested groups exceeds this. + # evaluation count across the requested groups exceeds this. page_size = 1000 filter_operation = LogicalOperation( operator=FilterOperator.AND, @@ -859,30 +866,30 @@ async def _count_live_experiments_by_group( ) while True: result = await entity_client.list( - Experiment, + Evaluation, workspace=workspace, filter_operation=filter_operation, page=page, page_size=page_size, ) - for experiment in result.data: - counts[experiment.experiment_group_id] = counts.get(experiment.experiment_group_id, 0) + 1 + for evaluation in result.data: + counts[evaluation.experiment_group_id] = counts.get(evaluation.experiment_group_id, 0) + 1 if page >= result.pagination.total_pages: break page += 1 return counts -async def _validate_parent_experiment_exists(entity_client: EntityClient, *, parent_experiment_id: str | None) -> None: - """Reject with 400 if ``parent_experiment_id`` is set but doesn't reference an existing experiment.""" - if parent_experiment_id is None: +async def _validate_parent_evaluation_exists(entity_client: EntityClient, *, parent_evaluation_id: str | None) -> None: + """Reject with 400 if ``parent_evaluation_id`` is set but doesn't reference an existing evaluation.""" + if parent_evaluation_id is None: return try: - await entity_client.get_by_id(Experiment, entity_id=parent_experiment_id) + await entity_client.get_by_id(Evaluation, entity_id=parent_evaluation_id) except EntityNotFoundError as e: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=f"parent_experiment_id '{parent_experiment_id}' does not reference an existing experiment.", + detail=f"parent_evaluation_id '{parent_evaluation_id}' does not reference an existing evaluation.", ) from e @@ -893,12 +900,12 @@ async def _validate_group_exists(entity_client: EntityClient, *, group_id: str) except EntityNotFoundError as e: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=(f"ExperimentGroup '{group_id}' must be created before an Experiment can reference it."), + detail=(f"ExperimentGroup '{group_id}' must be created before an Evaluation can reference it."), ) from e if group.is_deleted: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=f"ExperimentGroup '{group_id}' has been deleted and can no longer accept new Experiments.", + detail=f"ExperimentGroup '{group_id}' has been deleted and can no longer accept new Evaluations.", ) @@ -958,7 +965,7 @@ def _apply_is_pinned_filter(parsed: ParsedFilter) -> None: # Metric heads whose dotted sub-paths address a ClickHouse rollup (not an entity column). Declared as -# self-mapping namespaces on ExperimentFilter so paths survive filter validation untranslated. +# self-mapping namespaces on EvaluationFilter so paths survive filter validation untranslated. _METRIC_NAMESPACES = frozenset({"cost_usd", "latency_ms", "evaluators"}) _NUMERIC_FILTER_OPERATORS = frozenset( {FilterOperator.GTE, FilterOperator.LTE, FilterOperator.GT, FilterOperator.LT, FilterOperator.EQ} @@ -1071,10 +1078,10 @@ def _extract_metric_predicates( return operation, [] -def _matches_metric_predicates(response: ExperimentResponse, predicates: list[_MetricPredicate]) -> bool: +def _matches_metric_predicates(response: EvaluationResponse, predicates: list[_MetricPredicate]) -> bool: """True if the response satisfies every metric predicate. A missing metric never matches.""" for predicate in predicates: - value = _experiment_sort_value(response, predicate.field) + value = _evaluation_sort_value(response, predicate.field) if value is None or not _compare_metric(value, predicate.operator, predicate.threshold): return False return True @@ -1092,7 +1099,7 @@ def _compare_metric(value: float, operator: FilterOperator, threshold: float) -> return value == threshold # EQ -def _experiment_sort_value(response: ExperimentResponse, field: str) -> Any: +def _evaluation_sort_value(response: EvaluationResponse, field: str) -> Any: """Value for `field` on a hydrated response, or None when the metric is absent (sorts last).""" if field in _ENTITY_SORT_FIELDS: return getattr(response, field) @@ -1109,7 +1116,7 @@ def _experiment_sort_value(response: ExperimentResponse, field: str) -> Any: def _validate_default_sort(default_sort: str | None) -> None: - """Reject a default sort whose field the experiments list can't sort by. + """Reject a default sort whose field the evaluations list can't sort by. The value is a ``sort``-param string (optional leading '-' for descending), e.g. ``-cost_usd.mean``; the field must satisfy the same rule as the list ``sort`` query param. @@ -1120,23 +1127,23 @@ def _validate_default_sort(default_sort: str | None) -> None: _validate_sort_field(field) -def _sort_experiments( - responses: list[ExperimentResponse], +def _sort_evaluations( + responses: list[EvaluationResponse], *, keys: list[tuple[str, bool]], pinned_first: bool = False, -) -> list[ExperimentResponse]: +) -> list[EvaluationResponse]: """Sort by an ordered list of ``(field, descending)`` keys. Missing values sort last per key; ties break by name. Keys are applied from lowest to highest priority via successive stable sorts, so the first key dominates. With ``pinned_first``, pinned - experiments float to the top while preserving key order within the pinned and unpinned groups. + evaluations float to the top while preserving key order within the pinned and unpinned groups. """ ordered = sorted(responses, key=lambda r: r.name) # stable base tiebreak for field, descending in reversed(keys): - present = [r for r in ordered if _experiment_sort_value(r, field) is not None] - missing = [r for r in ordered if _experiment_sort_value(r, field) is None] - present.sort(key=lambda r, f=field: _experiment_sort_value(r, f), reverse=descending) + present = [r for r in ordered if _evaluation_sort_value(r, field) is not None] + missing = [r for r in ordered if _evaluation_sort_value(r, field) is None] + present.sort(key=lambda r, f=field: _evaluation_sort_value(r, f), reverse=descending) ordered = present + missing if pinned_first: # Stable: pinned (False sorts before True) float up, key order preserved within each group. @@ -1147,8 +1154,8 @@ def _sort_experiments( async def _hydrate_rollups( *, workspace: str, - responses: list[ExperimentResponse], - rollup_repository: ExperimentRollupRepository | None, + responses: list[EvaluationResponse], + rollup_repository: EvaluationRollupRepository | None, ) -> bool: """Enrich responses with ClickHouse rollups in place. @@ -1163,10 +1170,10 @@ async def _hydrate_rollups( return False try: rollups = await rollup_repository.get_rollups( - workspace=workspace, experiment_ids=[response.name for response in responses] + workspace=workspace, evaluation_ids=[response.name for response in responses] ) except Exception: - logger.exception("Skipping experiment rollup hydration because ClickHouse is unavailable") + logger.exception("Skipping evaluation rollup hydration because ClickHouse is unavailable") return False for response in responses: rollup = rollups.get(response.name) @@ -1175,7 +1182,7 @@ async def _hydrate_rollups( return True -def _apply_rollup(response: ExperimentResponse, rollup: ExperimentRollup) -> None: +def _apply_rollup(response: EvaluationResponse, rollup: EvaluationRollup) -> None: response.evaluator_names = rollup.evaluator_names response.model_names = rollup.model_names response.agent_names = rollup.agent_names @@ -1196,3 +1203,23 @@ def _aggregate(rollup: ScoreRollup) -> EvaluatorAggregate: p99=rollup.p99, count=rollup.count, ) + + +# --------------------------------------------------------------------------- +# Backwards-compatible URL aliases (TEMPORARY — remove in a follow-up PR) +# +# The child endpoints moved from `/experiments` to `/evaluations`. Register the old +# `/experiments...` paths as hidden aliases (``include_in_schema=False``) that point at the +# same handlers, so existing callers keep working until they migrate to `/evaluations`. +# --------------------------------------------------------------------------- +for _legacy_route in list(router.routes): + if isinstance(_legacy_route, APIRoute) and "/evaluations" in _legacy_route.path: + router.add_api_route( + _legacy_route.path.replace("/evaluations", "/experiments", 1), + _legacy_route.endpoint, + methods=sorted(_legacy_route.methods), + response_model=_legacy_route.response_model, + status_code=_legacy_route.status_code, + include_in_schema=False, + name=f"{_legacy_route.name}_experiments_alias", + ) diff --git a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py index feaf724fac..3ff29fa457 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Request, response, and filter schemas for the Experiments API. +"""Request, response, and filter schemas for the Evaluations API. Response models are standalone: they translate from the stored entity via ``from_entity`` and carry rollup fields hydrated from ClickHouse at read time. @@ -10,16 +10,16 @@ from __future__ import annotations from datetime import datetime -from typing import Annotated, Literal +from typing import Annotated, Literal, Self from nmp.common.entities.values import DatetimeFilter, Filter, NumberFilter, map_entity_field from nmp.intake.entities.experiments import Experiment, ExperimentGroup from nmp.intake.spans.domain import SpanStatus -from nmp.intake.spans.experiment_session_repository import ExperimentSessionRow -from pydantic import AnyUrl, BaseModel, ConfigDict, Field +from nmp.intake.spans.evaluation_session_repository import EvaluationSessionRow +from pydantic import AnyUrl, BaseModel, ConfigDict, Field, computed_field, model_validator -ExperimentSessionMode = Literal["summary", "detailed"] -EXPERIMENT_SESSION_SUMMARY_INPUT_CHAR_LIMIT = 1000 +EvaluationSessionMode = Literal["summary", "detailed"] +EVALUATION_SESSION_SUMMARY_INPUT_CHAR_LIMIT = 1000 class ExperimentGroupRequest(BaseModel): @@ -37,37 +37,54 @@ class ExperimentGroupRequest(BaseModel): default_sort: str = Field( default="-created_at", description=( - "Default sort for this group's experiments list, as a `sort`-param string (leading '-' = " - "descending); defaults to '-created_at'. Accepts any field the experiments list `sort` " + "Default sort for this group's evaluations list, as a `sort`-param string (leading '-' = " + "descending); defaults to '-created_at'. Accepts any field the evaluations list `sort` " "param does; clients apply it as the list `sort` param." ), ) -class ExperimentRequest(BaseModel): - """Request body for creating an Experiment.""" +class EvaluationRequest(BaseModel): + """Request body for creating an Evaluation.""" model_config = ConfigDict(extra="forbid") - name: str = Field(description="Producer-supplied, workspace-unique experiment id.") + name: str = Field(description="Producer-supplied, workspace-unique evaluation id.") experiment_group_id: str = Field( description="Entity id of the owning ExperimentGroup. Required — the group must already exist.", ) dataset_name: str = Field(description="Producer-supplied dataset name.") dataset_version: str | None = Field(default=None, description="Producer-supplied dataset version.") - source_link: AnyUrl | None = Field(default=None, description="Optional URL for the source experiment.") + source_link: AnyUrl | None = Field(default=None, description="Optional URL for the source evaluation.") metadata: dict[str, str] = Field(default_factory=dict, description="Free-form producer metadata.") description: str | None = Field(default=None, description="Human-readable description.") + parent_evaluation_id: str | None = Field( + default=None, + description="Entity id of the evaluation this one was derived from (e.g. a variant of a baseline), if any.", + ) parent_experiment_id: str | None = Field( default=None, - description="Entity id of the experiment this one was derived from (e.g. a variant of a baseline), if any.", + deprecated=True, + description="Deprecated alias for parent_evaluation_id.", ) - status: str | None = Field(default=None, description="Producer-defined lifecycle status of the experiment.") + status: str | None = Field(default=None, description="Producer-defined lifecycle status of the evaluation.") root_cause: str | None = Field( default=None, - description="Human- or agent-authored explanation of the experiment's outcome (e.g. why it was killed).", + description="Human- or agent-authored explanation of the evaluation's outcome (e.g. why it was killed).", ) + @model_validator(mode="after") + def _coalesce_deprecated_parent(self) -> Self: + """Accept the deprecated ``parent_experiment_id`` alias; the canonical field wins if both are set. + + Read the raw value via ``__dict__`` to avoid tripping the field's deprecation warning on + every request. + """ + deprecated_parent = self.__dict__.get("parent_experiment_id") + if self.parent_evaluation_id is None and deprecated_parent is not None: + self.parent_evaluation_id = deprecated_parent + return self + class ExperimentGroupResponse(BaseModel): """ExperimentGroup as served by the API.""" @@ -82,11 +99,16 @@ class ExperimentGroupResponse(BaseModel): default_sort: str created_at: datetime | None = None updated_at: datetime | None = None - experiment_count: int = Field( + evaluation_count: int = Field( default=0, - description="Number of live (non-soft-deleted) experiments in this group.", + description="Number of live (non-soft-deleted) evaluations in this group.", ) + @computed_field(deprecated=True, description="Deprecated alias for evaluation_count.") # type: ignore[prop-decorator] + @property + def experiment_count(self) -> int: + return self.evaluation_count + @classmethod def from_entity(cls, entity: ExperimentGroup) -> ExperimentGroupResponse: return cls( @@ -115,21 +137,21 @@ class EvaluatorAggregate(BaseModel): count: int = 0 -class ExperimentResponse(BaseModel): - """Experiment as served by the API, including ClickHouse-hydrated rollups.""" +class EvaluationResponse(BaseModel): + """Evaluation as served by the API, including ClickHouse-hydrated rollups.""" id: str name: str workspace: str experiment_group_id: str = Field( - description="Entity id of the owning ExperimentGroup. Required for every Experiment.", + description="Entity id of the owning ExperimentGroup. Required for every Evaluation.", ) dataset_name: str dataset_version: str | None = None source_link: AnyUrl | None = None metadata: dict[str, str] = Field(default_factory=dict) description: str | None = None - parent_experiment_id: str | None = None + parent_evaluation_id: str | None = None status: str | None = None root_cause: str | None = None created_at: datetime | None = None @@ -137,8 +159,8 @@ class ExperimentResponse(BaseModel): pinned_at: datetime | None = Field( default=None, description=( - "Timestamp at which the experiment was pinned, or null if unpinned. " - "Managed via POST/DELETE /experiments/{name}/pin." + "Timestamp at which the evaluation was pinned, or null if unpinned. " + "Managed via POST/DELETE /evaluations/{name}/pin." ), json_schema_extra={"nullable": True}, ) @@ -146,29 +168,34 @@ class ExperimentResponse(BaseModel): evaluator_names: list[str] = Field(default_factory=list) model_names: list[str] = Field( default_factory=list, - description="Distinct model names observed across ingested sessions for this experiment.", + description="Distinct model names observed across ingested sessions for this evaluation.", json_schema_extra={"uniqueItems": True}, ) agent_names: list[str] = Field( default_factory=list, - description="Distinct agent names observed across ingested sessions for this experiment.", + description="Distinct agent names observed across ingested sessions for this evaluation.", json_schema_extra={"uniqueItems": True}, ) agent_versions: list[str] = Field( default_factory=list, - description="Distinct agent versions observed across ingested sessions for this experiment.", + description="Distinct agent versions observed across ingested sessions for this evaluation.", json_schema_extra={"uniqueItems": True}, ) aggregate_scores: dict[str, EvaluatorAggregate] | None = None run_count: int = Field( default=0, - description="Number of distinct ingested experiment sessions; one session is treated as one run.", + description="Number of distinct ingested evaluation sessions; one session is treated as one run.", ) cost_usd: EvaluatorAggregate | None = None latency_ms: EvaluatorAggregate | None = None + @computed_field(deprecated=True, description="Deprecated alias for parent_evaluation_id.") # type: ignore[prop-decorator] + @property + def parent_experiment_id(self) -> str | None: + return self.parent_evaluation_id + @classmethod - def from_entity(cls, entity: Experiment) -> ExperimentResponse: + def from_entity(cls, entity: Experiment) -> EvaluationResponse: return cls( id=entity.id, name=entity.name, @@ -179,7 +206,7 @@ def from_entity(cls, entity: Experiment) -> ExperimentResponse: source_link=entity.source_link, metadata=entity.metadata, description=entity.description, - parent_experiment_id=entity.parent_experiment_id, + parent_evaluation_id=entity.parent_experiment_id, status=entity.status, root_cause=entity.root_cause, created_at=entity.created_at, @@ -193,7 +220,7 @@ class MetricStatFilters(BaseModel): Declaring each stat explicitly (rather than an open ``dict[str, NumberFilter]``) makes the valid stats visible in the OpenAPI schema, e.g. ``filter[cost_usd.mean][$lte]=0.5``. These stats must - stay in sync with the runtime sort/filter grammar (``_METRIC_STATS`` in the experiments + stay in sync with the runtime sort/filter grammar (``_METRIC_STATS`` in the evaluations endpoints); a unit test guards the parity. """ @@ -222,30 +249,30 @@ class ExperimentGroupFilter(Filter): ) -class ExperimentFilter(Filter): - """Filter for listing Experiments.""" +class EvaluationFilter(Filter): + """Filter for listing Evaluations.""" - name: str | None = Field(default=None, description="Filter experiments by name.") - experiment_group_id: str | None = Field(default=None, description="Filter experiments by owning group id.") - dataset_name: str | None = Field(default=None, description="Filter experiments by dataset name.") - dataset_version: str | None = Field(default=None, description="Filter experiments by dataset version.") - created_by: str | None = Field(default=None, description="Filter experiments by the principal that created them.") + name: str | None = Field(default=None, description="Filter evaluations by name.") + experiment_group_id: str | None = Field(default=None, description="Filter evaluations by owning group id.") + dataset_name: str | None = Field(default=None, description="Filter evaluations by dataset name.") + dataset_version: str | None = Field(default=None, description="Filter evaluations by dataset version.") + created_by: str | None = Field(default=None, description="Filter evaluations by the principal that created them.") created_at: DatetimeFilter | None = Field( default=None, - description="Filter experiments by creation timestamp; supports `$gte` and `$lte` for ranges.", + description="Filter evaluations by creation timestamp; supports `$gte` and `$lte` for ranges.", ) updated_at: DatetimeFilter | None = Field( default=None, - description="Filter experiments by last-updated timestamp; supports `$gte` and `$lte` for ranges.", + description="Filter evaluations by last-updated timestamp; supports `$gte` and `$lte` for ranges.", ) is_deleted: bool | None = Field( default=None, - description=("When true, returns only soft-deleted experiments. Omit (or false) to see only live experiments."), + description=("When true, returns only soft-deleted evaluations. Omit (or false) to see only live evaluations."), ) is_pinned: bool | None = Field( default=None, description=( - "When true, returns only pinned experiments. When false, returns only unpinned experiments. " + "When true, returns only pinned evaluations. When false, returns only unpinned evaluations. " "Omit to return both." ), ) @@ -272,8 +299,8 @@ class ExperimentFilter(Filter): ) -class ExperimentSessionFilter(Filter): - """Filter for listing ExperimentSessions.""" +class EvaluationSessionFilter(Filter): + """Filter for listing EvaluationSessions.""" test_case_id: str | None = Field(default=None, description="Filter by producer-supplied test case id.") status: str | None = Field( @@ -281,15 +308,21 @@ class ExperimentSessionFilter(Filter): ) -class ExperimentSessionResponse(BaseModel): - """One ingested session of an Experiment — a single test case execution. +class EvaluationSessionResponse(BaseModel): + """One ingested session of an Evaluation — a single test case execution. Hydrated from ClickHouse at read time by reading root/session membership from ``trace_index`` and joining page-bounded span/evaluator rollups. """ workspace: str - experiment_name: str + evaluation_name: str + + @computed_field(deprecated=True, description="Deprecated alias for evaluation_name.") # type: ignore[prop-decorator] + @property + def experiment_name(self) -> str: + return self.evaluation_name + session_id: str test_case_id: str | None = Field( default=None, @@ -307,7 +340,7 @@ class ExperimentSessionResponse(BaseModel): default=None, description=( "Root-span input text. In summary mode this is truncated to " - f"{EXPERIMENT_SESSION_SUMMARY_INPUT_CHAR_LIMIT} characters." + f"{EVALUATION_SESSION_SUMMARY_INPUT_CHAR_LIMIT} characters." ), ) @@ -325,10 +358,10 @@ class ExperimentSessionResponse(BaseModel): ) @classmethod - def from_row(cls, row: ExperimentSessionRow) -> ExperimentSessionResponse: + def from_row(cls, row: EvaluationSessionRow) -> EvaluationSessionResponse: return cls( workspace=row.workspace, - experiment_name=row.experiment_name, + evaluation_name=row.evaluation_name, session_id=row.session_id, test_case_id=row.test_case_id, trace_id=row.trace_id, diff --git a/services/intake/src/nmp/intake/entities/experiments.py b/services/intake/src/nmp/intake/entities/experiments.py index 2ed5dd5a49..fbec64c6a8 100644 --- a/services/intake/src/nmp/intake/entities/experiments.py +++ b/services/intake/src/nmp/intake/entities/experiments.py @@ -6,6 +6,13 @@ These are entity-store rows, distinct from ClickHouse telemetry. They hold the durable, producer-supplied metadata that organizes telemetry into leaderboard views. Rollups are derived from ClickHouse at read time. + +NOTE: The public API and Studio already call this concept an "Evaluation" — but +the entity here is intentionally still ``Experiment`` (``__entity_type__ = +"experiment"``, ``parent_experiment_id``). Renaming the entity, its +``__entity_type__``, and its stored fields is a breaking storage change that +requires a one-time data migration of existing rows, so it is deferred to a +later pass. Until then the API layer maps Evaluation ⇄ this Experiment entity. """ from __future__ import annotations @@ -79,6 +86,9 @@ class Experiment(EntityBase): """A single agent/config run against a dataset: one row on a leaderboard. ``name`` is the producer-supplied, workspace-unique experiment id. + + Exposed as "Evaluation" by the API/Studio; still stored as ``experiment`` + here pending the entity rename + data migration (see module docstring). """ __entity_type__: ClassVar[str] = "experiment" diff --git a/services/intake/src/nmp/intake/service.py b/services/intake/src/nmp/intake/service.py index db989baaf4..d744fdcfa1 100644 --- a/services/intake/src/nmp/intake/service.py +++ b/services/intake/src/nmp/intake/service.py @@ -54,7 +54,7 @@ def get_routers(self) -> List[RouterConfig]: RouterConfig( experiments.router, tag="Experiments", - description="Create, list, get, and delete Experiments and Experiment Groups", + description="Create, list, get, and delete Evaluations and Experiment Groups", ), RouterConfig(otlp.router, tag="Ingest", description="OTLP/HTTP trace ingest endpoints"), RouterConfig(atif.router, tag="Ingest", description="ATIF trajectory ingest endpoints"), diff --git a/services/intake/src/nmp/intake/spans/api/traces.py b/services/intake/src/nmp/intake/spans/api/traces.py index 8767b6f1cd..071e8d09c4 100644 --- a/services/intake/src/nmp/intake/spans/api/traces.py +++ b/services/intake/src/nmp/intake/spans/api/traces.py @@ -25,12 +25,14 @@ API_TAG = "Traces" TRACE_INDEX_FILTER_FIELDS = frozenset( { - "experiment_id", + "evaluation_id", + "experiment_id", # deprecated alias for evaluation_id "test_case_id", } ) TRACE_INDEX_FILTER_ALIASES = { - "experiment_id": "experiment_id", + "evaluation_id": "evaluation_id", + "experiment_id": "evaluation_id", # deprecated alias resolves to the evaluation_id filter "test_case_id": "test_case_id", } @@ -44,7 +46,7 @@ filter_schema=TraceFilter, filter_description=( "Filter root-span-backed traces by id, session_id, root status, root span started_at, " - "experiment_id, and test_case_id." + "evaluation_id (or its deprecated alias experiment_id), and test_case_id." ), ), ) diff --git a/services/intake/src/nmp/intake/spans/api/traces_schemas.py b/services/intake/src/nmp/intake/spans/api/traces_schemas.py index a388730d60..3386c9df2f 100644 --- a/services/intake/src/nmp/intake/spans/api/traces_schemas.py +++ b/services/intake/src/nmp/intake/spans/api/traces_schemas.py @@ -11,7 +11,7 @@ from nmp.common.entities.values import DatetimeFilter from nmp.intake.spans.domain import IntakeTrace, SpanStatus -from nmp.intake.spans.ingest.evaluation_context import ExperimentContext +from nmp.intake.spans.ingest.evaluation_context import EvaluationContext, ExperimentContext from pydantic import BaseModel, Field @@ -28,8 +28,13 @@ class TraceFilter(BaseModel): session_id: str | None = Field(default=None, description="Filter by session id.") status: SpanStatus | None = Field(default=None, description="Filter by root span status.") started_at: DatetimeFilter | None = Field(default=None, description="Filter by root span start timestamp.") - experiment_id: str | None = Field(default=None, description="Filter by root-span experiment id.") - test_case_id: str | None = Field(default=None, description="Filter by root-span experiment test case id.") + evaluation_id: str | None = Field(default=None, description="Filter by root-span evaluation id.") + experiment_id: str | None = Field( + default=None, + deprecated=True, + description="Deprecated alias for evaluation_id. Filter by root-span evaluation id.", + ) + test_case_id: str | None = Field(default=None, description="Filter by root-span evaluation test case id.") class Trace(BaseModel): @@ -38,7 +43,12 @@ class Trace(BaseModel): session_id: str workspace: str name: str | None = None - experiment_context: ExperimentContext | None = None + evaluation_context: EvaluationContext | None = None + experiment_context: ExperimentContext | None = Field( + default=None, + deprecated=True, + description="Deprecated alias for evaluation_context; will be removed in a future release.", + ) started_at: datetime ended_at: datetime | None = None duration_ms: float | None = None @@ -61,6 +71,7 @@ def from_domain(cls, trace: IntakeTrace) -> Self: session_id=trace.session_id, workspace=trace.workspace, name=trace.name, + evaluation_context=_evaluation_context(trace), experiment_context=_experiment_context(trace), started_at=trace.started_at, ended_at=trace.ended_at, @@ -78,10 +89,20 @@ def from_domain(cls, trace: IntakeTrace) -> Self: ) +def _evaluation_context(trace: IntakeTrace) -> EvaluationContext | None: + if trace.evaluation_id is None: + return None + return EvaluationContext( + evaluation_id=trace.evaluation_id, + test_case_id=trace.test_case_id, + ) + + def _experiment_context(trace: IntakeTrace) -> ExperimentContext | None: - if trace.experiment_id is None: + """Deprecated alias for ``_evaluation_context``; populated from the same evaluation id.""" + if trace.evaluation_id is None: return None return ExperimentContext( - experiment_id=trace.experiment_id, + experiment_id=trace.evaluation_id, test_case_id=trace.test_case_id, ) diff --git a/services/intake/src/nmp/intake/spans/clickhouse_migrations.py b/services/intake/src/nmp/intake/spans/clickhouse_migrations.py index ba2bea0b68..5f6a62cbfa 100644 --- a/services/intake/src/nmp/intake/spans/clickhouse_migrations.py +++ b/services/intake/src/nmp/intake/spans/clickhouse_migrations.py @@ -246,13 +246,13 @@ def _create_trace_index_schema(client, settings: ClickHouseMigrationSettings) -> table = _table(settings, "trace_index") view = _table(settings, "trace_index_mv") - client.command(f"DROP TABLE IF EXISTS {_table(settings, 'experiment_sessions_mv')}") - client.command(f"DROP TABLE IF EXISTS {_table(settings, 'experiment_sessions')}") + client.command(f"DROP TABLE IF EXISTS {_table(settings, 'evaluation_sessions_mv')}") + client.command(f"DROP TABLE IF EXISTS {_table(settings, 'evaluation_sessions')}") client.command(f"DROP TABLE IF EXISTS {view}") client.command(f"DROP TABLE IF EXISTS {table}") project_key = spec_for_field(SpanAttributeField.PROJECT).bag_key - experiment_key = spec_for_field(SpanAttributeField.EVALUATION_ID).bag_key + evaluation_key = spec_for_field(SpanAttributeField.EVALUATION_ID).bag_key test_case_key = spec_for_field(SpanAttributeField.TEST_CASE_ID).bag_key # Note this is logically a single table. CH requires creating an underlying table and then a view that writes to that table. @@ -271,7 +271,7 @@ def _create_trace_index_schema(client, settings: ClickHouseMigrationSettings) -> root_output String CODEC(ZSTD(3)), project String DEFAULT '', - experiment_id String DEFAULT '', + evaluation_id String DEFAULT '', test_case_id String DEFAULT '', root_started_at DateTime64(6) CODEC(Delta(8), ZSTD(1)), @@ -283,7 +283,7 @@ def _create_trace_index_schema(client, settings: ClickHouseMigrationSettings) -> INDEX idx_trace_id trace_id TYPE bloom_filter(0.001) GRANULARITY 1, INDEX idx_session_id session_id TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_experiment_id experiment_id TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_evaluation_id evaluation_id TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_test_case_id test_case_id TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_root_status root_status TYPE set(4) GRANULARITY 4, INDEX idx_source_format source_format TYPE set(8) GRANULARITY 4 @@ -310,7 +310,7 @@ def _create_trace_index_schema(client, settings: ClickHouseMigrationSettings) -> input AS root_input, output AS root_output, attributes_string['{project_key}'] AS project, - attributes_string['{experiment_key}'] AS experiment_id, + attributes_string['{evaluation_key}'] AS evaluation_id, attributes_string['{test_case_key}'] AS test_case_id, start_time AS root_started_at, nullIf(end_time, toDateTime64(0, 6)) AS root_ended_at, @@ -343,11 +343,17 @@ def _create_trace_index_schema(client, settings: ClickHouseMigrationSettings) -> ("ch_evaluator_results_0002", _add_evaluator_results_skip_indexes), ("ch_trace_index_0003", _create_trace_index_schema), # NeMo-owned span attribute keys moved under the ``nemo.*`` namespace - # (e.g., ``experiment.id`` → ``nemo.experiment.id``). The trace_index MV's SELECT clause + # (e.g., ``evaluation.id`` → ``nemo.evaluation.id``). The trace_index MV's SELECT clause # resolves bag keys from the catalog at creation time, so any environment that already # applied 0003 has the old keys baked in. Re-running the schema function drops and # recreates the MV with the current catalog keys. ("ch_trace_index_0004_nemo_keys", _create_trace_index_schema), + # The trace_index column ``experiment_id`` was renamed to ``evaluation_id`` (column, + # ``idx_evaluation_id``, and the MV SELECT alias). Environments that already applied 0004 still + # have the old ``experiment_id`` column, while the read path now queries ``evaluation_id`` — so + # re-run the rebuild under a new key. The function drops and recreates trace_index with the new + # column and backfills losslessly from ``spans`` (the durable source of truth). + ("ch_trace_index_0005_evaluation_id", _create_trace_index_schema), ] CURRENT_SCHEMA_VERSION = _MIGRATIONS[-1][0] diff --git a/services/intake/src/nmp/intake/spans/domain.py b/services/intake/src/nmp/intake/spans/domain.py index 84ace7c005..3b1e73c8b1 100644 --- a/services/intake/src/nmp/intake/spans/domain.py +++ b/services/intake/src/nmp/intake/spans/domain.py @@ -95,7 +95,7 @@ class TraceListFilter(BaseModel): status: SpanStatus | None = None started_at_gte: datetime | None = None started_at_lte: datetime | None = None - experiment_id: str | None = None + evaluation_id: str | None = None test_case_id: str | None = None @@ -112,7 +112,7 @@ class IntakeTrace(BaseModel): input: str | None = None output: str | None = None project: str | None = None - experiment_id: str | None = None + evaluation_id: str | None = None test_case_id: str | None = None started_at: datetime ended_at: datetime | None = None diff --git a/services/intake/src/nmp/intake/spans/experiment_rollup_repository.py b/services/intake/src/nmp/intake/spans/evaluation_rollup_repository.py similarity index 80% rename from services/intake/src/nmp/intake/spans/experiment_rollup_repository.py rename to services/intake/src/nmp/intake/spans/evaluation_rollup_repository.py index bd2f5ca993..978706357a 100644 --- a/services/intake/src/nmp/intake/spans/experiment_rollup_repository.py +++ b/services/intake/src/nmp/intake/spans/evaluation_rollup_repository.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""ClickHouse rollups for Experiment read models.""" +"""ClickHouse rollups for Evaluation read models.""" from __future__ import annotations @@ -26,8 +26,8 @@ class ScoreRollup: @dataclass -class ExperimentRollup: - experiment_id: str +class EvaluationRollup: + evaluation_id: str run_count: int = 0 model_names: list[str] = field(default_factory=list) agent_names: list[str] = field(default_factory=list) @@ -41,39 +41,39 @@ def evaluator_names(self) -> list[str]: return sorted(self.evaluator_scores) -class ExperimentRollupRepository: +class EvaluationRollupRepository: def __init__(self, client: ClickHouseSpanClient) -> None: self._client = client - async def get_rollups(self, *, workspace: str, experiment_ids: list[str]) -> dict[str, ExperimentRollup]: - experiment_ids = list(dict.fromkeys(experiment_ids)) - rollups = {experiment_id: ExperimentRollup(experiment_id=experiment_id) for experiment_id in experiment_ids} - if not experiment_ids: + async def get_rollups(self, *, workspace: str, evaluation_ids: list[str]) -> dict[str, EvaluationRollup]: + evaluation_ids = list(dict.fromkeys(evaluation_ids)) + rollups = {evaluation_id: EvaluationRollup(evaluation_id=evaluation_id) for evaluation_id in evaluation_ids} + if not evaluation_ids: return rollups - experiment_names_sql, experiment_parameters = _experiment_id_parameters(experiment_ids) - parameters = {"workspace": workspace, **experiment_parameters} + evaluation_names_sql, evaluation_parameters = _evaluation_id_parameters(evaluation_ids) + parameters = {"workspace": workspace, **evaluation_parameters} trace_index_table = self._client.table("trace_index") for row in result_rows( await self._client.query( - _run_counts_sql(trace_index_table, experiment_names_sql), + _run_counts_sql(trace_index_table, evaluation_names_sql), parameters=parameters, ) ): - rollups[row["experiment_id"]].run_count = int(row["run_count"]) + rollups[row["evaluation_id"]].run_count = int(row["run_count"]) for row in result_rows( await self._client.query( _score_rollups_sql( trace_index_table=trace_index_table, evaluator_results_table=self._client.table("evaluator_results"), - experiment_names_sql=experiment_names_sql, + evaluation_names_sql=evaluation_names_sql, ), parameters=parameters, ) ): - rollups[row["experiment_id"]].evaluator_scores[row["evaluator_name"]] = ScoreRollup( + rollups[row["evaluation_id"]].evaluator_scores[row["evaluator_name"]] = ScoreRollup( sum=float_or_none(row["sum"]), mean=float_or_none(row["mean"]), median=float_or_none(row["median"]), @@ -88,7 +88,7 @@ async def get_rollups(self, *, workspace: str, experiment_ids: list[str]) -> dic _metric_rollups_sql( trace_index_table=trace_index_table, spans_table=self._client.table("spans"), - experiment_names_sql=experiment_names_sql, + evaluation_names_sql=evaluation_names_sql, ), parameters={ **parameters, @@ -99,7 +99,7 @@ async def get_rollups(self, *, workspace: str, experiment_ids: list[str]) -> dic }, ) ): - rollup = rollups[row["experiment_id"]] + rollup = rollups[row["evaluation_id"]] rollup.model_names = _string_list(row["model_names"]) rollup.agent_names = _string_list(row["agent_names"]) rollup.agent_versions = _string_list(row["agent_versions"]) @@ -109,34 +109,34 @@ async def get_rollups(self, *, workspace: str, experiment_ids: list[str]) -> dic return rollups -def _experiment_id_parameters(experiment_ids: list[str]) -> tuple[str, dict[str, str]]: - parameters = {f"experiment_id_{index}": experiment_id for index, experiment_id in enumerate(experiment_ids)} +def _evaluation_id_parameters(evaluation_ids: list[str]) -> tuple[str, dict[str, str]]: + parameters = {f"evaluation_id_{index}": evaluation_id for index, evaluation_id in enumerate(evaluation_ids)} return ", ".join(f"%({name})s" for name in parameters), parameters -def _scoped_sessions_sql(trace_index_table: str, experiment_names_sql: str) -> str: +def _scoped_sessions_sql(trace_index_table: str, evaluation_names_sql: str) -> str: return f""" - SELECT workspace, experiment_id, session_id, latency_ms + SELECT workspace, evaluation_id, session_id, latency_ms FROM {trace_index_table} FINAL WHERE workspace = %(workspace)s AND is_deleted = 0 - AND experiment_id IN ({experiment_names_sql}) + AND evaluation_id IN ({evaluation_names_sql}) ORDER BY root_started_at ASC, root_span_id ASC - LIMIT 1 BY workspace, session_id, experiment_id + LIMIT 1 BY workspace, session_id, evaluation_id """ -def _run_counts_sql(trace_index_table: str, experiment_names_sql: str) -> str: +def _run_counts_sql(trace_index_table: str, evaluation_names_sql: str) -> str: return f""" WITH scoped_sessions AS ( - {_scoped_sessions_sql(trace_index_table, experiment_names_sql)} + {_scoped_sessions_sql(trace_index_table, evaluation_names_sql)} ) SELECT - experiment_id, + evaluation_id, count() AS run_count FROM scoped_sessions - GROUP BY experiment_id - ORDER BY experiment_id ASC + GROUP BY evaluation_id + ORDER BY evaluation_id ASC """ @@ -178,19 +178,19 @@ def stat(expr: str) -> str: return ",\n ".join(columns) -def _score_rollups_sql(*, trace_index_table: str, evaluator_results_table: str, experiment_names_sql: str) -> str: +def _score_rollups_sql(*, trace_index_table: str, evaluator_results_table: str, evaluation_names_sql: str) -> str: # Each run (session) contributes one score per evaluator, so reduce the per-span - # evaluator_results rows to a single per-(experiment, session, evaluator) value before + # evaluator_results rows to a single per-(evaluation, session, evaluator) value before # the distribution rollup. This keeps `count` aligned with run_count and the mean # run-weighted rather than weighted by spans-per-session. return f""" WITH scoped_sessions AS ( - {_scoped_sessions_sql(trace_index_table, experiment_names_sql)} + {_scoped_sessions_sql(trace_index_table, evaluation_names_sql)} ), session_scores AS ( SELECT - sessions.experiment_id AS experiment_id, + sessions.evaluation_id AS evaluation_id, results.name AS evaluator_name, avg(results.value) AS value FROM scoped_sessions AS sessions @@ -207,23 +207,23 @@ def _score_rollups_sql(*, trace_index_table: str, evaluator_results_table: str, ) AS results ON sessions.workspace = results.workspace AND sessions.session_id = results.session_id - GROUP BY sessions.experiment_id, sessions.session_id, results.name + GROUP BY sessions.evaluation_id, sessions.session_id, results.name ) SELECT - experiment_id, + evaluation_id, evaluator_name, {_stat_columns("value")} FROM session_scores - GROUP BY experiment_id, evaluator_name - ORDER BY experiment_id ASC, evaluator_name ASC + GROUP BY evaluation_id, evaluator_name + ORDER BY evaluation_id ASC, evaluator_name ASC """ -def _metric_rollups_sql(*, trace_index_table: str, spans_table: str, experiment_names_sql: str) -> str: +def _metric_rollups_sql(*, trace_index_table: str, spans_table: str, evaluation_names_sql: str) -> str: return f""" WITH scoped_sessions AS ( - {_scoped_sessions_sql(trace_index_table, experiment_names_sql)} + {_scoped_sessions_sql(trace_index_table, evaluation_names_sql)} ), current_session_spans AS ( { @@ -238,7 +238,7 @@ def _metric_rollups_sql(*, trace_index_table: str, spans_table: str, experiment_ ), session_costs AS ( SELECT - sessions.experiment_id AS experiment_id, + sessions.evaluation_id AS evaluation_id, sessions.session_id AS session_id, sessions.latency_ms AS latency_ms, if( @@ -269,18 +269,18 @@ def _metric_rollups_sql(*, trace_index_table: str, spans_table: str, experiment_ ON sessions.workspace = spans.workspace AND sessions.session_id = spans.session_id AND spans.is_deleted = 0 - GROUP BY sessions.experiment_id, sessions.session_id, sessions.latency_ms + GROUP BY sessions.evaluation_id, sessions.session_id, sessions.latency_ms ) SELECT - experiment_id, + evaluation_id, arraySort(arrayDistinct(arrayFlatten(groupArray(model_names)))) AS model_names, arraySort(arrayDistinct(arrayFlatten(groupArray(agent_names)))) AS agent_names, arraySort(arrayDistinct(arrayFlatten(groupArray(agent_versions)))) AS agent_versions, {_stat_columns("cost_usd", prefix="cost", guarded=True)}, {_stat_columns("latency_ms", prefix="latency", guarded=True)} FROM session_costs - GROUP BY experiment_id - ORDER BY experiment_id ASC + GROUP BY evaluation_id + ORDER BY evaluation_id ASC """ diff --git a/services/intake/src/nmp/intake/spans/experiment_session_repository.py b/services/intake/src/nmp/intake/spans/evaluation_session_repository.py similarity index 93% rename from services/intake/src/nmp/intake/spans/experiment_session_repository.py rename to services/intake/src/nmp/intake/spans/evaluation_session_repository.py index 0b8a155a86..1be43d8a7f 100644 --- a/services/intake/src/nmp/intake/spans/experiment_session_repository.py +++ b/services/intake/src/nmp/intake/spans/evaluation_session_repository.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""ClickHouse repository for per-session rows of an Experiment. +"""ClickHouse repository for per-session rows of an Evaluation. Returns one row per ingested session (test case execution), using ``trace_index`` for root/session membership and per-session aggregates from all spans (tokens + @@ -22,11 +22,11 @@ @dataclass(frozen=True) -class ExperimentSessionRow: - """One ingested session of an Experiment.""" +class EvaluationSessionRow: + """One ingested session of an Evaluation.""" workspace: str - experiment_name: str + evaluation_name: str session_id: str test_case_id: str | None trace_id: str @@ -44,12 +44,12 @@ class ExperimentSessionRow: @dataclass(frozen=True) -class ExperimentSessionPage: - rows: list[ExperimentSessionRow] +class EvaluationSessionPage: + rows: list[EvaluationSessionRow] total: int -class ExperimentSessionRepository: +class EvaluationSessionRepository: def __init__(self, client: ClickHouseSpanClient) -> None: self._client = client @@ -57,13 +57,13 @@ async def list_sessions( self, *, workspace: str, - experiment_name: str, + evaluation_name: str, status: SpanStatus | None = None, test_case_id: str | None = None, page: int, page_size: int, input_char_limit: int | None = None, - ) -> ExperimentSessionPage: + ) -> EvaluationSessionPage: if input_char_limit is not None and input_char_limit < 1: raise ValueError("input_char_limit must be positive when set") @@ -75,7 +75,7 @@ async def list_sessions( base_parameters: dict[str, Any] = { "workspace": workspace, - "experiment_name": experiment_name, + "evaluation_name": evaluation_name, "input_tokens_key": spec_for_field(SpanAttributeField.INPUT_TOKENS).bag_key, "output_tokens_key": spec_for_field(SpanAttributeField.OUTPUT_TOKENS).bag_key, "cached_tokens_key": spec_for_field(SpanAttributeField.CACHED_TOKENS).bag_key, @@ -92,7 +92,7 @@ async def list_sessions( ) total = int(count_result.result_rows[0][0]) if count_result.result_rows else 0 if total == 0: - return ExperimentSessionPage(rows=[], total=0) + return EvaluationSessionPage(rows=[], total=0) offset = (page - 1) * page_size list_sql = _list_sql( @@ -115,7 +115,7 @@ async def list_sessions( parameters=list_parameters, ) rows = [_row(record) for record in result_rows(list_result)] - return ExperimentSessionPage(rows=rows, total=total) + return EvaluationSessionPage(rows=rows, total=total) def _scoped_filter(*, test_case_id: str | None, status: SpanStatus | None) -> tuple[str, dict[str, Any]]: @@ -139,7 +139,7 @@ def _scoped_sessions_sql( ) -> str: select_columns = [ "workspace", - "experiment_id", + "evaluation_id", "session_id", "test_case_id", "trace_id", @@ -162,10 +162,10 @@ def _scoped_sessions_sql( FROM {trace_index_table} FINAL WHERE workspace = %(workspace)s AND is_deleted = 0 - AND experiment_id = %(experiment_name)s + AND evaluation_id = %(evaluation_name)s {scoped_filter_sql} ORDER BY root_started_at ASC, root_span_id ASC - LIMIT 1 BY workspace, session_id, experiment_id + LIMIT 1 BY workspace, session_id, evaluation_id """ @@ -208,7 +208,7 @@ def _list_sql( page_sessions AS ( SELECT workspace, - experiment_id, + evaluation_id, session_id, test_case_id, trace_id, @@ -276,7 +276,7 @@ def _list_sql( ) SELECT sessions.workspace AS workspace, - sessions.experiment_id AS experiment_id, + sessions.evaluation_id AS evaluation_id, sessions.session_id AS session_id, sessions.test_case_id AS test_case_id, sessions.trace_id AS trace_id, @@ -316,10 +316,10 @@ def _guarded_sum_sql(parameter_name: str, *, scale: int = 1) -> str: """ -def _row(record: dict[str, Any]) -> ExperimentSessionRow: - return ExperimentSessionRow( +def _row(record: dict[str, Any]) -> EvaluationSessionRow: + return EvaluationSessionRow( workspace=record["workspace"], - experiment_name=record["experiment_id"], + evaluation_name=record["evaluation_id"], session_id=record["session_id"], test_case_id=str_or_none(record["test_case_id"]), trace_id=record["trace_id"], diff --git a/services/intake/src/nmp/intake/spans/ingest/evaluation_context.py b/services/intake/src/nmp/intake/spans/ingest/evaluation_context.py index bdf1594bed..6e01b4c7fe 100644 --- a/services/intake/src/nmp/intake/spans/ingest/evaluation_context.py +++ b/services/intake/src/nmp/intake/spans/ingest/evaluation_context.py @@ -15,7 +15,7 @@ class EvaluationContext(BaseModel): metadata) keeps ingesting without error rather than being rejected. """ - evaluation_id: str | None = Field(default=None, description="Name of an existing Experiment entity.") + evaluation_id: str | None = Field(default=None, description="Name of an existing Evaluation.") test_case_id: str | None = Field(default=None, description="Optional producer-supplied test case id.") model_config = ConfigDict(extra="ignore") diff --git a/services/intake/src/nmp/intake/spans/ingest/evaluation_context_validation.py b/services/intake/src/nmp/intake/spans/ingest/evaluation_context_validation.py index 9474afb956..9cba54dfbf 100644 --- a/services/intake/src/nmp/intake/spans/ingest/evaluation_context_validation.py +++ b/services/intake/src/nmp/intake/spans/ingest/evaluation_context_validation.py @@ -25,10 +25,10 @@ async def validate_evaluation_context( except EntityNotFoundError as exc: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Experiment '{experiment_id}' must be created before it can be logged.", + detail=f"Evaluation '{experiment_id}' must be created before it can be logged.", ) from exc if experiment.is_deleted: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Experiment '{experiment_id}' has been deleted and cannot accept new sessions.", + detail=f"Evaluation '{experiment_id}' has been deleted and cannot accept new sessions.", ) diff --git a/services/intake/src/nmp/intake/spans/trace_repository.py b/services/intake/src/nmp/intake/spans/trace_repository.py index fa02c1ca2f..9bc916e95f 100644 --- a/services/intake/src/nmp/intake/spans/trace_repository.py +++ b/services/intake/src/nmp/intake/spans/trace_repository.py @@ -36,7 +36,7 @@ "root_span_id", "name", "project", - "experiment_id", + "evaluation_id", "test_case_id", "started_at", "ended_at", @@ -196,7 +196,7 @@ def _trace_select_columns(*, include_aggregates: bool) -> str: "traces.root_span_id AS root_span_id", "traces.name AS name", "traces.project AS project", - "traces.experiment_id AS experiment_id", + "traces.evaluation_id AS evaluation_id", "traces.test_case_id AS test_case_id", "traces.started_at AS started_at", "traces.ended_at AS ended_at", @@ -220,7 +220,7 @@ def _trace_index_sql(table: str, filters: TraceListFilter) -> tuple[str, dict[st nullIf(trace_roots.root_span_id, '') AS root_span_id, nullIf(trace_roots.root_name, '') AS name, nullIf(trace_roots.project, '') AS project, - nullIf(trace_roots.experiment_id, '') AS experiment_id, + nullIf(trace_roots.evaluation_id, '') AS evaluation_id, nullIf(trace_roots.test_case_id, '') AS test_case_id, trace_roots.root_started_at AS started_at, trace_roots.root_ended_at AS ended_at, @@ -276,8 +276,9 @@ def _trace_aggregates_sql(table: str) -> tuple[str, dict[str, Any]]: return query, parameters +# Maps API/filter field names to their physical trace_index columns. _TRACE_INDEX_FILTER_COLUMNS = { - "experiment_id": "experiment_id", + "evaluation_id": "evaluation_id", "test_case_id": "test_case_id", } @@ -392,7 +393,7 @@ def _row_to_trace(row: dict[str, Any]) -> IntakeTrace: input=row.get("input") or None, output=row.get("output") or None, project=row.get("project") or None, - experiment_id=row.get("experiment_id") or None, + evaluation_id=row.get("evaluation_id") or None, test_case_id=row.get("test_case_id") or None, started_at=row["started_at"], ended_at=ended_at, diff --git a/services/intake/tests/conftest.py b/services/intake/tests/conftest.py index 3844578f1c..2ccfd3f1c8 100644 --- a/services/intake/tests/conftest.py +++ b/services/intake/tests/conftest.py @@ -5,7 +5,7 @@ import pytest from fastapi.testclient import TestClient -from nmp.intake.api.v2.experiments.endpoints import get_experiment_rollup_repository +from nmp.intake.api.v2.experiments.endpoints import get_evaluation_rollup_repository from nmp.intake.service import IntakeService from nmp.testing import create_test_client @@ -16,6 +16,6 @@ def client(): with create_test_client( IntakeService, client_type=TestClient, - dependency_overrides={get_experiment_rollup_repository: lambda: None}, + dependency_overrides={get_evaluation_rollup_repository: lambda: None}, ) as tc: yield tc diff --git a/services/intake/tests/integration/spans/test_atif_ingest.py b/services/intake/tests/integration/spans/test_atif_ingest.py index 5709148c7e..2ec73bba07 100644 --- a/services/intake/tests/integration/spans/test_atif_ingest.py +++ b/services/intake/tests/integration/spans/test_atif_ingest.py @@ -847,7 +847,7 @@ def test_atif_trace_tokens_do_not_double_count_when_trajectory_and_steps_both_ca def _create_experiment(client: TestClient, name: str) -> str: group_id = _ensure_group(client) response = client.post( - "/apis/intake/v2/workspaces/default/experiments", + "/apis/intake/v2/workspaces/default/evaluations", json={ "name": name, "experiment_group_id": group_id, @@ -859,7 +859,7 @@ def _create_experiment(client: TestClient, name: str) -> str: if response.status_code == 201: return response.json()["name"] - existing = client.get(f"/apis/intake/v2/workspaces/default/experiments/{name}") + existing = client.get(f"/apis/intake/v2/workspaces/default/evaluations/{name}") assert existing.status_code == 200, existing.text return existing.json()["name"] diff --git a/services/intake/tests/integration/spans/test_chat_completions_ingest.py b/services/intake/tests/integration/spans/test_chat_completions_ingest.py index 1510ad19ab..9237092a33 100644 --- a/services/intake/tests/integration/spans/test_chat_completions_ingest.py +++ b/services/intake/tests/integration/spans/test_chat_completions_ingest.py @@ -444,7 +444,7 @@ def test_chat_completions_ingest_accepts_both_context_shapes_with_evaluation_con def _create_experiment(client: TestClient, name: str) -> str: group_id = _ensure_group(client) response = client.post( - "/apis/intake/v2/workspaces/default/experiments", + "/apis/intake/v2/workspaces/default/evaluations", json={ "name": name, "experiment_group_id": group_id, @@ -456,7 +456,7 @@ def _create_experiment(client: TestClient, name: str) -> str: if response.status_code == 201: return response.json()["name"] - existing = client.get(f"/apis/intake/v2/workspaces/default/experiments/{name}") + existing = client.get(f"/apis/intake/v2/workspaces/default/evaluations/{name}") assert existing.status_code == 200, existing.text return existing.json()["name"] diff --git a/services/intake/tests/integration/spans/test_clickhouse_bootstrap.py b/services/intake/tests/integration/spans/test_clickhouse_bootstrap.py index 20b7d76626..db26401b5b 100644 --- a/services/intake/tests/integration/spans/test_clickhouse_bootstrap.py +++ b/services/intake/tests/integration/spans/test_clickhouse_bootstrap.py @@ -28,6 +28,7 @@ def test_clickhouse_bootstrap_is_idempotent(clickhouse_client: ClickHouseSpanCli ("ch_spans_0002",), ("ch_trace_index_0003",), ("ch_trace_index_0004_nemo_keys",), + ("ch_trace_index_0005_evaluation_id",), ] diff --git a/services/intake/tests/integration/spans/test_experiment_metric_sort.py b/services/intake/tests/integration/spans/test_experiment_metric_sort.py index a08f5627ce..1669e455ef 100644 --- a/services/intake/tests/integration/spans/test_experiment_metric_sort.py +++ b/services/intake/tests/integration/spans/test_experiment_metric_sort.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""The experiments list sorts by a ClickHouse rollup metric (Option A app-merge), end to end.""" +"""The evaluations list sorts by a ClickHouse rollup metric (Option A app-merge), end to end.""" import uuid from datetime import datetime, timedelta, timezone @@ -10,7 +10,7 @@ from fastapi.testclient import TestClient ATIF_INGEST = "/apis/intake/v2/workspaces/default/ingest/atif" -EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" +EVALUATIONS = "/apis/intake/v2/workspaces/default/evaluations" GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" @@ -26,12 +26,12 @@ def _iso(value: datetime) -> str: return value.isoformat().replace("+00:00", "Z") -def _atif_body(*, started_at: datetime, experiment_id: str, cost_usd: float, offset_seconds: int) -> dict[str, Any]: +def _atif_body(*, started_at: datetime, evaluation_id: str, cost_usd: float, offset_seconds: int) -> dict[str, Any]: session_started_at = started_at + timedelta(seconds=offset_seconds) return { "schema_version": "ATIF-v1.7", - "session_id": f"{experiment_id}-session", - "experiment_context": {"experiment_id": experiment_id, "test_case_id": "case-1"}, + "session_id": f"{evaluation_id}-session", + "evaluation_context": {"evaluation_id": evaluation_id, "test_case_id": "case-1"}, "extra": {"task_name": "case-1", "verifier_result": {"rewards": {"reward": 1.0}}}, "agent": {"name": "sample-agent", "version": "1.0.0", "model_name": "provider/sample-model"}, "steps": [ @@ -47,34 +47,34 @@ def _atif_body(*, started_at: datetime, experiment_id: str, cost_usd: float, off } -def _create_experiment(client: TestClient, group_id: str, name: str) -> None: +def _create_evaluation(client: TestClient, group_id: str, name: str) -> None: response = client.post( - EXPERIMENTS, + EVALUATIONS, json={"name": name, "experiment_group_id": group_id, "dataset_name": "ds"}, ) assert response.status_code == 201, response.text def test_list_sorts_by_cost_metric_missing_last(client: TestClient) -> None: - # Unique per run so reruns/shared integration state can't collide on group or experiment names. + # Unique per run so reruns/shared integration state can't collide on group or evaluation names. suffix = uuid.uuid4().hex group_id = _ensure_group(client, name=f"metric-sort-group-{suffix}") started_at = datetime.now(timezone.utc).replace(microsecond=0) cheap, pricey, mid = f"exp-cheap-{suffix}", f"exp-pricey-{suffix}", f"exp-mid-{suffix}" for index, (name, cost) in enumerate([(cheap, 0.10), (pricey, 0.90), (mid, 0.50)]): - _create_experiment(client, group_id, name) + _create_evaluation(client, group_id, name) response = client.post( ATIF_INGEST, - json=_atif_body(started_at=started_at, experiment_id=name, cost_usd=cost, offset_seconds=index * 10), + json=_atif_body(started_at=started_at, evaluation_id=name, cost_usd=cost, offset_seconds=index * 10), ) assert response.status_code == 201, response.text # No ingest -> no cost rollup -> must sort last regardless of direction. norun = f"exp-norun-{suffix}" - _create_experiment(client, group_id, norun) + _create_evaluation(client, group_id, norun) - # Filter by this group so the assertion only inspects experiments this test created. + # Filter by this group so the assertion only inspects evaluations this test created. listed = client.get( - EXPERIMENTS, + EVALUATIONS, params={"filter[experiment_group_id]": group_id, "sort": "-cost_usd.mean", "page_size": 50}, ) assert listed.status_code == 200, listed.text @@ -85,23 +85,23 @@ def test_list_sorts_by_cost_metric_missing_last(client: TestClient) -> None: def test_list_filters_by_cost_metric(client: TestClient) -> None: # Same shape as the sort test, but filtering. Combine an entity filter (group) with two metric # filters on different fields: cost_usd.mean <= 0.50 excludes pricey; run_count >= 1 excludes the - # never-ingested experiment (whose cost rollup is also missing). Sort by cost so the order is + # never-ingested evaluation (whose cost rollup is also missing). Sort by cost so the order is # deterministic by value rather than by creation time. suffix = uuid.uuid4().hex group_id = _ensure_group(client, name=f"metric-filter-group-{suffix}") started_at = datetime.now(timezone.utc).replace(microsecond=0) cheap, pricey, mid = f"exp-cheap-{suffix}", f"exp-pricey-{suffix}", f"exp-mid-{suffix}" for index, (name, cost) in enumerate([(cheap, 0.10), (pricey, 0.90), (mid, 0.50)]): - _create_experiment(client, group_id, name) + _create_evaluation(client, group_id, name) response = client.post( ATIF_INGEST, - json=_atif_body(started_at=started_at, experiment_id=name, cost_usd=cost, offset_seconds=index * 10), + json=_atif_body(started_at=started_at, evaluation_id=name, cost_usd=cost, offset_seconds=index * 10), ) assert response.status_code == 201, response.text - _create_experiment(client, group_id, f"exp-norun-{suffix}") # no ingest -> excluded by both predicates + _create_evaluation(client, group_id, f"exp-norun-{suffix}") # no ingest -> excluded by both predicates listed = client.get( - EXPERIMENTS, + EVALUATIONS, params={ "filter[experiment_group_id]": group_id, "filter[cost_usd.mean][lte]": "0.50", @@ -116,5 +116,5 @@ def test_list_filters_by_cost_metric(client: TestClient) -> None: def test_list_rejects_unknown_sort_field(client: TestClient) -> None: - response = client.get(EXPERIMENTS, params={"sort": "bogus.field"}) + response = client.get(EVALUATIONS, params={"sort": "bogus.field"}) assert response.status_code == 400, response.text diff --git a/services/intake/tests/integration/spans/test_experiment_rollups.py b/services/intake/tests/integration/spans/test_experiment_rollups.py index 6376d890a0..c5d165f9b2 100644 --- a/services/intake/tests/integration/spans/test_experiment_rollups.py +++ b/services/intake/tests/integration/spans/test_experiment_rollups.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Experiment rollup integration tests.""" +"""Evaluation rollup integration tests.""" from __future__ import annotations @@ -12,7 +12,7 @@ from fastapi.testclient import TestClient ATIF_INGEST = "/apis/intake/v2/workspaces/default/ingest/atif" -EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" +EVALUATIONS = "/apis/intake/v2/workspaces/default/evaluations" GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" @@ -25,13 +25,13 @@ def _ensure_group(client: TestClient, name: str = "rollup-test-group") -> str: return response.json()["id"] -def test_experiment_response_hydrates_clickhouse_rollups(client: TestClient) -> None: - experiment_id = "rollup-exp" +def test_evaluation_response_hydrates_clickhouse_rollups(client: TestClient) -> None: + evaluation_id = "rollup-exp" group_id = _ensure_group(client) created = client.post( - EXPERIMENTS, + EVALUATIONS, json={ - "name": experiment_id, + "name": evaluation_id, "experiment_group_id": group_id, "dataset_name": "rollup-dataset", "dataset_version": "v1", @@ -51,7 +51,7 @@ def test_experiment_response_hydrates_clickhouse_rollups(client: TestClient) -> ATIF_INGEST, json=_atif_body( started_at=started_at, - experiment_id=experiment_id, + evaluation_id=evaluation_id, run_id=run_id, test_case_id=test_case_id, score=score, @@ -62,15 +62,15 @@ def test_experiment_response_hydrates_clickhouse_rollups(client: TestClient) -> ) assert response.status_code == 201, response.text - fetched = client.get(f"{EXPERIMENTS}/{experiment_id}") + fetched = client.get(f"{EVALUATIONS}/{evaluation_id}") assert fetched.status_code == 200, fetched.text - experiment = fetched.json() + evaluation = fetched.json() - assert experiment["run_count"] == 4 - assert experiment["evaluator_names"] == ["reward"] - assert experiment["model_names"] == ["provider/sample-model"] + assert evaluation["run_count"] == 4 + assert evaluation["evaluator_names"] == ["reward"] + assert evaluation["model_names"] == ["provider/sample-model"] - score = experiment["aggregate_scores"]["reward"] + score = evaluation["aggregate_scores"]["reward"] assert score["sum"] == pytest.approx(3.0) assert score["mean"] == pytest.approx(0.75) assert score["median"] == pytest.approx(0.8) @@ -79,7 +79,7 @@ def test_experiment_response_hydrates_clickhouse_rollups(client: TestClient) -> assert score["p99"] == pytest.approx(1.0) assert score["count"] == 4 - cost = experiment["cost_usd"] + cost = evaluation["cost_usd"] assert cost["sum"] == pytest.approx(0.65) assert cost["mean"] == pytest.approx(0.1625) assert cost["median"] == pytest.approx(0.2) @@ -88,7 +88,7 @@ def test_experiment_response_hydrates_clickhouse_rollups(client: TestClient) -> assert cost["p99"] == pytest.approx(0.3) assert cost["count"] == 4 - latency = experiment["latency_ms"] + latency = evaluation["latency_ms"] assert latency["sum"] == pytest.approx(7000.0) assert latency["mean"] == pytest.approx(1750.0) assert latency["median"] == pytest.approx(2000.0) @@ -97,33 +97,33 @@ def test_experiment_response_hydrates_clickhouse_rollups(client: TestClient) -> assert latency["p99"] == pytest.approx(3000.0) assert latency["count"] == 4 - listed = client.get(EXPERIMENTS) + listed = client.get(EVALUATIONS) assert listed.status_code == 200, listed.text - listed_experiment = next(item for item in listed.json()["data"] if item["name"] == experiment_id) - assert listed_experiment["aggregate_scores"]["reward"]["mean"] == pytest.approx(0.75) + listed_evaluation = next(item for item in listed.json()["data"] if item["name"] == evaluation_id) + assert listed_evaluation["aggregate_scores"]["reward"]["mean"] == pytest.approx(0.75) -def test_atif_ingest_rejects_deleted_experiment(client: TestClient) -> None: - experiment_id = "soft-deleted-exp" +def test_atif_ingest_rejects_deleted_evaluation(client: TestClient) -> None: + evaluation_id = "soft-deleted-exp" group_id = _ensure_group(client) created = client.post( - EXPERIMENTS, + EVALUATIONS, json={ - "name": experiment_id, + "name": evaluation_id, "experiment_group_id": group_id, "dataset_name": "rollup-dataset", "dataset_version": "v1", }, ) assert created.status_code == 201, created.text - deleted = client.delete(f"{EXPERIMENTS}/{experiment_id}") + deleted = client.delete(f"{EVALUATIONS}/{evaluation_id}") assert deleted.status_code == 204, deleted.text response = client.post( ATIF_INGEST, json=_atif_body( started_at=datetime.now(timezone.utc).replace(microsecond=0), - experiment_id=experiment_id, + evaluation_id=evaluation_id, run_id="run-1", test_case_id="case-1", score=1.0, @@ -136,12 +136,12 @@ def test_atif_ingest_rejects_deleted_experiment(client: TestClient) -> None: assert "deleted" in response.json()["detail"].lower() -def test_atif_ingest_rejects_unknown_experiment_context(client: TestClient) -> None: +def test_atif_ingest_rejects_unknown_evaluation_context(client: TestClient) -> None: response = client.post( ATIF_INGEST, json=_atif_body( started_at=datetime.now(timezone.utc).replace(microsecond=0), - experiment_id="missing-exp", + evaluation_id="missing-exp", run_id="run-1", test_case_id="case-1", score=1.0, @@ -155,13 +155,13 @@ def test_atif_ingest_rejects_unknown_experiment_context(client: TestClient) -> N assert "must be created before it can be logged" in response.json()["detail"] -def test_deprecated_evaluation_context_hydrates_experiment_rollups(client: TestClient) -> None: - experiment_id = "legacy-eval-context-exp" +def test_deprecated_evaluation_context_hydrates_evaluation_rollups(client: TestClient) -> None: + evaluation_id = "legacy-eval-context-exp" group_id = _ensure_group(client) created = client.post( - EXPERIMENTS, + EVALUATIONS, json={ - "name": experiment_id, + "name": evaluation_id, "experiment_group_id": group_id, "dataset_name": "rollup-dataset", "dataset_version": "v1", @@ -174,7 +174,7 @@ def test_deprecated_evaluation_context_hydrates_experiment_rollups(client: TestC json={ **_atif_body( started_at=datetime.now(timezone.utc).replace(microsecond=0), - experiment_id=experiment_id, + evaluation_id=evaluation_id, run_id="run-1", test_case_id="case-1", score=1.0, @@ -182,24 +182,23 @@ def test_deprecated_evaluation_context_hydrates_experiment_rollups(client: TestC latency_ms=100, offset_seconds=0, ), - "experiment_context": None, - "evaluation_context": {"evaluation_id": experiment_id, "test_case_id": "case-1"}, + "evaluation_context": {"evaluation_id": evaluation_id, "test_case_id": "case-1"}, }, ) assert response.status_code == 201, response.text - fetched = client.get(f"{EXPERIMENTS}/{experiment_id}") + fetched = client.get(f"{EVALUATIONS}/{evaluation_id}") assert fetched.status_code == 200, fetched.text - experiment = fetched.json() - assert experiment["run_count"] == 1 - assert experiment["aggregate_scores"]["reward"]["mean"] == pytest.approx(1.0) + evaluation = fetched.json() + assert evaluation["run_count"] == 1 + assert evaluation["aggregate_scores"]["reward"]["mean"] == pytest.approx(1.0) def _atif_body( *, started_at: datetime, - experiment_id: str, + evaluation_id: str, run_id: str, test_case_id: str, score: float, @@ -209,12 +208,12 @@ def _atif_body( ) -> dict[str, Any]: session_started_at = started_at + timedelta(seconds=offset_seconds) finished_at = session_started_at + timedelta(milliseconds=latency_ms) - session_id = f"{experiment_id}-{run_id}-{test_case_id}" + session_id = f"{evaluation_id}-{run_id}-{test_case_id}" return { "schema_version": "ATIF-v1.7", "session_id": session_id, - "experiment_context": { - "experiment_id": experiment_id, + "evaluation_context": { + "evaluation_id": evaluation_id, "test_case_id": test_case_id, }, "extra": { diff --git a/services/intake/tests/integration/spans/test_experiment_sessions.py b/services/intake/tests/integration/spans/test_experiment_sessions.py index fcc3650d77..b03ed04479 100644 --- a/services/intake/tests/integration/spans/test_experiment_sessions.py +++ b/services/intake/tests/integration/spans/test_experiment_sessions.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Integration tests for the per-session experiment endpoint.""" +"""Integration tests for the per-session evaluation endpoint.""" from __future__ import annotations @@ -12,7 +12,7 @@ from fastapi.testclient import TestClient ATIF_INGEST = "/apis/intake/v2/workspaces/default/ingest/atif" -EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" +EVALUATIONS = "/apis/intake/v2/workspaces/default/evaluations" GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" @@ -25,13 +25,13 @@ def _ensure_group(client: TestClient, name: str = "sessions-test-group") -> str: return response.json()["id"] -def test_list_experiment_sessions_returns_joined_session_rows(client: TestClient) -> None: - experiment_name = "sessions-exp" +def test_list_evaluation_sessions_returns_joined_session_rows(client: TestClient) -> None: + evaluation_name = "sessions-exp" group_id = _ensure_group(client) created = client.post( - EXPERIMENTS, + EVALUATIONS, json={ - "name": experiment_name, + "name": evaluation_name, "experiment_group_id": group_id, "dataset_name": "sessions-dataset", "dataset_version": "v1", @@ -58,7 +58,7 @@ def test_list_experiment_sessions_returns_joined_session_rows(client: TestClient ATIF_INGEST, json=_atif_body( started_at=started_at, - experiment_name=experiment_name, + evaluation_name=evaluation_name, test_case_id=test_case_id, score=score, cost_usd=cost_usd, @@ -71,7 +71,7 @@ def test_list_experiment_sessions_returns_joined_session_rows(client: TestClient ) assert response.status_code == 201, response.text - listed = client.get(f"{EXPERIMENTS}/{experiment_name}/sessions") + listed = client.get(f"{EVALUATIONS}/{evaluation_name}/sessions") assert listed.status_code == 200, listed.text body = listed.json() assert body["pagination"]["total_results"] == 3 @@ -81,7 +81,7 @@ def test_list_experiment_sessions_returns_joined_session_rows(client: TestClient assert set(rows_by_case) == {"case-a", "case-b", "case-c"} case_a = rows_by_case["case-a"] - assert case_a["experiment_name"] == experiment_name + assert case_a["evaluation_name"] == evaluation_name assert case_a["session_id"] assert case_a["trace_id"] assert case_a["root_span_id"] @@ -92,7 +92,7 @@ def test_list_experiment_sessions_returns_joined_session_rows(client: TestClient assert case_a["evaluator_scores"] == {"reward": pytest.approx(1.0)} assert case_a["status"] in {"success", "unknown"} - paged = client.get(f"{EXPERIMENTS}/{experiment_name}/sessions", params={"page": 2, "page_size": 1}) + paged = client.get(f"{EVALUATIONS}/{evaluation_name}/sessions", params={"page": 2, "page_size": 1}) assert paged.status_code == 200, paged.text paged_body = paged.json() assert paged_body["pagination"]["total_results"] == 3 @@ -101,13 +101,13 @@ def test_list_experiment_sessions_returns_joined_session_rows(client: TestClient assert paged_body["data"][0]["evaluator_scores"] == {"reward": pytest.approx(0.5)} -def test_list_experiment_sessions_filter_by_test_case(client: TestClient) -> None: - experiment_name = "sessions-filter-exp" +def test_list_evaluation_sessions_filter_by_test_case(client: TestClient) -> None: + evaluation_name = "sessions-filter-exp" group_id = _ensure_group(client) created = client.post( - EXPERIMENTS, + EVALUATIONS, json={ - "name": experiment_name, + "name": evaluation_name, "experiment_group_id": group_id, "dataset_name": "sessions-dataset", "dataset_version": "v1", @@ -121,7 +121,7 @@ def test_list_experiment_sessions_filter_by_test_case(client: TestClient) -> Non ATIF_INGEST, json=_atif_body( started_at=started_at, - experiment_name=experiment_name, + evaluation_name=evaluation_name, test_case_id=test_case_id, score=1.0, cost_usd=0.01, @@ -135,7 +135,7 @@ def test_list_experiment_sessions_filter_by_test_case(client: TestClient) -> Non assert response.status_code == 201, response.text filtered = client.get( - f"{EXPERIMENTS}/{experiment_name}/sessions", + f"{EVALUATIONS}/{evaluation_name}/sessions", params={"filter[test_case_id]": "alpha"}, ) assert filtered.status_code == 200, filtered.text @@ -145,16 +145,16 @@ def test_list_experiment_sessions_filter_by_test_case(client: TestClient) -> Non assert body["data"][0]["test_case_id"] == "alpha" -def test_list_experiment_sessions_filter_by_status(client: TestClient) -> None: +def test_list_evaluation_sessions_filter_by_status(client: TestClient) -> None: # ATIF ingest doesn't expose explicit per-span status, so all seeded sessions land with the # default root-span status. This test verifies the filter is wired through (no SQL break and # mismatched filters return zero) rather than per-status seeding. - experiment_name = "sessions-status-exp" + evaluation_name = "sessions-status-exp" group_id = _ensure_group(client) created = client.post( - EXPERIMENTS, + EVALUATIONS, json={ - "name": experiment_name, + "name": evaluation_name, "experiment_group_id": group_id, "dataset_name": "sessions-dataset", "dataset_version": "v1", @@ -166,7 +166,7 @@ def test_list_experiment_sessions_filter_by_status(client: TestClient) -> None: ATIF_INGEST, json=_atif_body( started_at=datetime.now(timezone.utc).replace(microsecond=0), - experiment_name=experiment_name, + evaluation_name=evaluation_name, test_case_id="case-1", score=1.0, cost_usd=0.01, @@ -179,12 +179,12 @@ def test_list_experiment_sessions_filter_by_status(client: TestClient) -> None: ) assert response.status_code == 201, response.text - listed = client.get(f"{EXPERIMENTS}/{experiment_name}/sessions") + listed = client.get(f"{EVALUATIONS}/{evaluation_name}/sessions") assert listed.status_code == 200, listed.text seeded_status = listed.json()["data"][0]["status"] matching = client.get( - f"{EXPERIMENTS}/{experiment_name}/sessions", + f"{EVALUATIONS}/{evaluation_name}/sessions", params={"filter[status]": seeded_status}, ) assert matching.status_code == 200, matching.text @@ -192,21 +192,21 @@ def test_list_experiment_sessions_filter_by_status(client: TestClient) -> None: other_status = "error" if seeded_status != "error" else "cancelled" mismatched = client.get( - f"{EXPERIMENTS}/{experiment_name}/sessions", + f"{EVALUATIONS}/{evaluation_name}/sessions", params={"filter[status]": other_status}, ) assert mismatched.status_code == 200, mismatched.text assert mismatched.json()["pagination"]["total_results"] == 0 -def test_list_experiment_sessions_returns_404_for_unknown_experiment(client: TestClient) -> None: - response = client.get(f"{EXPERIMENTS}/does-not-exist/sessions") +def test_list_evaluation_sessions_returns_404_for_unknown_evaluation(client: TestClient) -> None: + response = client.get(f"{EVALUATIONS}/does-not-exist/sessions") assert response.status_code == 404, response.text -def test_list_experiment_sessions_rejects_unknown_query_param(client: TestClient) -> None: +def test_list_evaluation_sessions_rejects_unknown_query_param(client: TestClient) -> None: response = client.get( - f"{EXPERIMENTS}/does-not-exist/sessions", + f"{EVALUATIONS}/does-not-exist/sessions", params={"test_caseid": "case-1"}, ) assert response.status_code == 400, response.text @@ -216,7 +216,7 @@ def test_list_experiment_sessions_rejects_unknown_query_param(client: TestClient def _atif_body( *, started_at: datetime, - experiment_name: str, + evaluation_name: str, test_case_id: str, score: float, cost_usd: float, @@ -228,12 +228,12 @@ def _atif_body( ) -> dict[str, Any]: session_started_at = started_at + timedelta(seconds=offset_seconds) finished_at = session_started_at + timedelta(milliseconds=latency_ms) - session_id = f"{experiment_name}-{run_id}-{test_case_id}" + session_id = f"{evaluation_name}-{run_id}-{test_case_id}" return { "schema_version": "ATIF-v1.7", "session_id": session_id, - "experiment_context": { - "experiment_id": experiment_name, + "evaluation_context": { + "evaluation_id": evaluation_name, "test_case_id": test_case_id, }, "extra": { diff --git a/services/intake/tests/integration/spans/test_traces_read.py b/services/intake/tests/integration/spans/test_traces_read.py index 2e419d6162..4137ec9d4c 100644 --- a/services/intake/tests/integration/spans/test_traces_read.py +++ b/services/intake/tests/integration/spans/test_traces_read.py @@ -65,7 +65,7 @@ def test_traces_read_returns_core_trace_summary(client: TestClient, make_otlp_re "/apis/intake/v2/workspaces/default/traces", params={ "filter[session_id]": "trace-session", - "filter[experiment_id]": "experiment-a", + "filter[evaluation_id]": "experiment-a", "page_size": 20, }, ) @@ -88,9 +88,11 @@ def test_traces_read_returns_core_trace_summary(client: TestClient, make_otlp_re assert Decimal(str(trace["cost_output_usd"])) == Decimal("0.0037") assert trace["span_count"] == 2 assert trace["error_count"] == 0 + assert trace["evaluation_context"]["evaluation_id"] == "experiment-a" + assert trace["evaluation_context"]["test_case_id"] == "case-a" assert trace["experiment_context"]["experiment_id"] == "experiment-a" assert trace["experiment_context"]["test_case_id"] == "case-a" - assert "evaluation_context" not in trace + assert "evaluation_id" not in trace assert "experiment_id" not in trace assert "test_case_id" not in trace assert "source_format" not in trace @@ -111,9 +113,11 @@ def test_traces_read_returns_core_trace_summary(client: TestClient, make_otlp_re summary_trace = summary_response.json()["data"][0] assert summary_trace["id"] == trace["id"] assert summary_trace["status"] == "success" + assert summary_trace["evaluation_context"]["evaluation_id"] == "experiment-a" + assert summary_trace["evaluation_context"]["test_case_id"] == "case-a" assert summary_trace["experiment_context"]["experiment_id"] == "experiment-a" assert summary_trace["experiment_context"]["test_case_id"] == "case-a" - assert "evaluation_context" not in summary_trace + assert "evaluation_id" not in summary_trace assert "experiment_id" not in summary_trace assert "test_case_id" not in summary_trace assert "input_tokens" not in summary_trace diff --git a/services/intake/tests/integration/test_experiments_crud.py b/services/intake/tests/integration/test_experiments_crud.py index 8d29ee246b..a60215513f 100644 --- a/services/intake/tests/integration/test_experiments_crud.py +++ b/services/intake/tests/integration/test_experiments_crud.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""CRUD tests for the Experiments and ExperimentGroups endpoints.""" +"""CRUD tests for the Evaluations and ExperimentGroups endpoints.""" from __future__ import annotations @@ -9,19 +9,21 @@ from fastapi import FastAPI from fastapi.testclient import TestClient -from nmp.intake.api.v2.experiments.endpoints import get_experiment_rollup_repository +from nmp.intake.api.v2.experiments.endpoints import get_evaluation_rollup_repository GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" -EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" +EVALUATIONS = "/apis/intake/v2/workspaces/default/evaluations" +# Deprecated URL alias: the child endpoints moved /experiments -> /evaluations. +EXPERIMENTS_ALIAS = "/apis/intake/v2/workspaces/default/experiments" -def _experiment_body(*, experiment_group_id: str, **overrides: Any) -> dict: +def _evaluation_body(*, experiment_group_id: str, **overrides: Any) -> dict: body = { "name": "terminal-bench-2_claude-code_opus_baseline", "experiment_group_id": experiment_group_id, "dataset_name": "terminal-bench-2", "dataset_version": "v1", - "source_link": "https://example.com/experiments/tb2-baseline", + "source_link": "https://example.com/evaluations/tb2-baseline", "metadata": {"job_name": "tb2-baseline"}, } body.update(overrides) @@ -75,46 +77,46 @@ def test_experiment_group_update_description(client: TestClient) -> None: assert missing.status_code == 404 -def test_experiment_update_moves_between_groups_and_edits(client: TestClient) -> None: +def test_evaluation_update_moves_between_groups_and_edits(client: TestClient) -> None: group_a = client.post(GROUPS, json={"name": "grp-a"}).json() group_b = client.post(GROUPS, json={"name": "grp-b"}).json() - client.post(EXPERIMENTS, json=_experiment_body(name="exp-a", experiment_group_id=group_a["id"])) + client.post(EVALUATIONS, json=_evaluation_body(name="exp-a", experiment_group_id=group_a["id"])) - # Move the experiment from group_a to group_b and edit its description. - body = _experiment_body(name="exp-a", experiment_group_id=group_b["id"], description="looks good") - updated = client.put(f"{EXPERIMENTS}/exp-a", json=body) + # Move the evaluation from group_a to group_b and edit its description. + body = _evaluation_body(name="exp-a", experiment_group_id=group_b["id"], description="looks good") + updated = client.put(f"{EVALUATIONS}/exp-a", json=body) assert updated.status_code == 200, updated.text assert updated.json()["experiment_group_id"] == group_b["id"] assert updated.json()["description"] == "looks good" -def test_experiment_update_rejects_immutable_change(client: TestClient) -> None: +def test_evaluation_update_rejects_immutable_change(client: TestClient) -> None: group = _create_group(client) client.post( - EXPERIMENTS, - json=_experiment_body(name="exp-a", dataset_name="tb2", experiment_group_id=group["id"]), + EVALUATIONS, + json=_evaluation_body(name="exp-a", dataset_name="tb2", experiment_group_id=group["id"]), ) - changed = _experiment_body(name="exp-a", dataset_name="tb3", experiment_group_id=group["id"]) - resp = client.put(f"{EXPERIMENTS}/exp-a", json=changed) + changed = _evaluation_body(name="exp-a", dataset_name="tb3", experiment_group_id=group["id"]) + resp = client.put(f"{EVALUATIONS}/exp-a", json=changed) assert resp.status_code == 409, resp.text assert "dataset_name" in resp.json()["detail"] missing = client.put( - f"{EXPERIMENTS}/missing", - json=_experiment_body(name="missing", experiment_group_id=group["id"]), + f"{EVALUATIONS}/missing", + json=_evaluation_body(name="missing", experiment_group_id=group["id"]), ) assert missing.status_code == 404 -def test_experiment_crud_and_empty_rollups(client: TestClient) -> None: +def test_evaluation_crud_and_empty_rollups(client: TestClient) -> None: group = client.post(GROUPS, json={"name": "grp"}).json() - created = client.post(EXPERIMENTS, json=_experiment_body(experiment_group_id=group["id"])) + created = client.post(EVALUATIONS, json=_evaluation_body(experiment_group_id=group["id"])) assert created.status_code == 201, created.text exp = created.json() assert exp["name"] == "terminal-bench-2_claude-code_opus_baseline" assert exp["experiment_group_id"] == group["id"] assert exp["dataset_name"] == "terminal-bench-2" - assert exp["source_link"] == "https://example.com/experiments/tb2-baseline" + assert exp["source_link"] == "https://example.com/evaluations/tb2-baseline" assert exp["metadata"] == {"job_name": "tb2-baseline"} # Rollups exist on the read model but are empty until ClickHouse hydration lands. @@ -126,124 +128,177 @@ def test_experiment_crud_and_empty_rollups(client: TestClient) -> None: assert exp["run_count"] == 0 -def test_experiment_read_degrades_when_rollup_hydration_fails(client: TestClient) -> None: +def test_evaluation_read_degrades_when_rollup_hydration_fails(client: TestClient) -> None: class FailingRollupRepository: - async def get_rollups(self, *, workspace: str, experiment_ids: list[str]) -> dict: + async def get_rollups(self, *, workspace: str, evaluation_ids: list[str]) -> dict: raise RuntimeError("clickhouse unavailable") group = _create_group(client) app = cast(FastAPI, client.app) - app.dependency_overrides[get_experiment_rollup_repository] = lambda: FailingRollupRepository() + app.dependency_overrides[get_evaluation_rollup_repository] = lambda: FailingRollupRepository() try: created = client.post( - EXPERIMENTS, - json=_experiment_body(name="exp-rollup-fails", experiment_group_id=group["id"]), + EVALUATIONS, + json=_evaluation_body(name="exp-rollup-fails", experiment_group_id=group["id"]), ) assert created.status_code == 201, created.text - fetched = client.get(f"{EXPERIMENTS}/exp-rollup-fails") + fetched = client.get(f"{EVALUATIONS}/exp-rollup-fails") assert fetched.status_code == 200, fetched.text assert fetched.json()["name"] == "exp-rollup-fails" assert fetched.json()["run_count"] == 0 assert fetched.json()["aggregate_scores"] is None finally: - app.dependency_overrides.pop(get_experiment_rollup_repository, None) + app.dependency_overrides.pop(get_evaluation_rollup_repository, None) -def test_experiment_create_rejects_missing_group_id(client: TestClient) -> None: - body = _experiment_body(name="exp-no-group", experiment_group_id="placeholder") +def test_evaluation_create_rejects_missing_group_id(client: TestClient) -> None: + body = _evaluation_body(name="exp-no-group", experiment_group_id="placeholder") body.pop("experiment_group_id") - response = client.post(EXPERIMENTS, json=body) + response = client.post(EVALUATIONS, json=body) assert response.status_code == 422, response.text -def test_experiment_create_rejects_unknown_group_id(client: TestClient) -> None: +def test_evaluation_create_rejects_unknown_group_id(client: TestClient) -> None: response = client.post( - EXPERIMENTS, - json=_experiment_body(name="exp-bad-group", experiment_group_id="experiment_group-does-not-exist"), + EVALUATIONS, + json=_evaluation_body(name="exp-bad-group", experiment_group_id="experiment_group-does-not-exist"), ) assert response.status_code == 400, response.text assert "must be created before" in response.json()["detail"] -def test_delete_group_cascades_to_experiments(client: TestClient) -> None: +def test_delete_group_cascades_to_evaluations(client: TestClient) -> None: group = client.post(GROUPS, json={"name": "doomed-group"}).json() - client.post(EXPERIMENTS, json=_experiment_body(name="exp-doomed-1", experiment_group_id=group["id"])) - client.post(EXPERIMENTS, json=_experiment_body(name="exp-doomed-2", experiment_group_id=group["id"])) + client.post(EVALUATIONS, json=_evaluation_body(name="exp-doomed-1", experiment_group_id=group["id"])) + client.post(EVALUATIONS, json=_evaluation_body(name="exp-doomed-2", experiment_group_id=group["id"])) deleted = client.delete(f"{GROUPS}/doomed-group") assert deleted.status_code == 204, deleted.text - # Both child experiments are gone with the group. + # Both child evaluations are gone with the group. for child_name in ("exp-doomed-1", "exp-doomed-2"): - missing = client.get(f"{EXPERIMENTS}/{child_name}") + missing = client.get(f"{EVALUATIONS}/{child_name}") assert missing.status_code == 404 -def test_experiment_conflict_and_not_found(client: TestClient) -> None: +def test_legacy_experiments_url_alias_still_works(client: TestClient) -> None: + """The child endpoints moved /experiments -> /evaluations; the old paths remain as hidden aliases.""" + group = _create_group(client, name="legacy-alias-group") + + # Create via the deprecated /experiments URL. + created = client.post( + EXPERIMENTS_ALIAS, + json=_evaluation_body(name="legacy-alias-eval", experiment_group_id=group["id"]), + ) + assert created.status_code == 201, created.text + assert created.json()["name"] == "legacy-alias-eval" + + # The deprecated URL and the canonical URL resolve to the same entity. + via_legacy = client.get(f"{EXPERIMENTS_ALIAS}/legacy-alias-eval") + via_canonical = client.get(f"{EVALUATIONS}/legacy-alias-eval") + assert via_legacy.status_code == 200, via_legacy.text + assert via_canonical.status_code == 200 + assert via_legacy.json() == via_canonical.json() + + +def test_legacy_experiments_url_aliases_are_hidden_from_schema() -> None: + from fastapi.routing import APIRoute + from nmp.intake.api.v2.experiments.endpoints import router + + alias_routes = [ + route + for route in router.routes + if isinstance(route, APIRoute) and "/experiments" in route.path and "/experiment-groups" not in route.path + ] + assert alias_routes, "expected legacy /experiments alias routes to be registered" + assert all(route.include_in_schema is False for route in alias_routes) + + +def test_deprecated_field_aliases_are_backwards_compatible(client: TestClient) -> None: + """Renamed fields keep deprecated aliases: requests accept the old name, responses return both.""" + group = _create_group(client, name="alias-compat-group") + # Group response carries both the canonical count and the deprecated experiment_count alias. + assert group["experiment_count"] == group["evaluation_count"] + + parent = client.post(EVALUATIONS, json=_evaluation_body(name="parent-eval", experiment_group_id=group["id"])).json() + + # Create a child referencing the parent via the DEPRECATED parent_experiment_id request field. + body = _evaluation_body(name="child-eval", experiment_group_id=group["id"]) + body["parent_experiment_id"] = parent["id"] + created = client.post(EVALUATIONS, json=body) + assert created.status_code == 201, created.text + + payload = created.json() + # The response echoes both the canonical and deprecated parent field, coalesced from the old input. + assert payload["parent_evaluation_id"] == parent["id"] + assert payload["parent_experiment_id"] == parent["id"] + + +def test_evaluation_conflict_and_not_found(client: TestClient) -> None: group = _create_group(client) - created = client.post(EXPERIMENTS, json=_experiment_body(experiment_group_id=group["id"])) + created = client.post(EVALUATIONS, json=_evaluation_body(experiment_group_id=group["id"])) assert created.status_code == 201 - duplicate = client.post(EXPERIMENTS, json=_experiment_body(experiment_group_id=group["id"])) + duplicate = client.post(EVALUATIONS, json=_evaluation_body(experiment_group_id=group["id"])) assert duplicate.status_code == 409 - missing = client.get(f"{EXPERIMENTS}/does-not-exist") + missing = client.get(f"{EVALUATIONS}/does-not-exist") assert missing.status_code == 404 - missing_delete = client.delete(f"{EXPERIMENTS}/does-not-exist") + missing_delete = client.delete(f"{EVALUATIONS}/does-not-exist") assert missing_delete.status_code == 404 -def test_experiment_list_and_scope_to_group(client: TestClient) -> None: +def test_evaluation_list_and_scope_to_group(client: TestClient) -> None: group_a = client.post(GROUPS, json={"name": "grp-a"}).json() group_b = client.post(GROUPS, json={"name": "grp-b"}).json() - client.post(EXPERIMENTS, json=_experiment_body(name="exp-a", experiment_group_id=group_a["id"])) - client.post(EXPERIMENTS, json=_experiment_body(name="exp-b", experiment_group_id=group_b["id"])) + client.post(EVALUATIONS, json=_evaluation_body(name="exp-a", experiment_group_id=group_a["id"])) + client.post(EVALUATIONS, json=_evaluation_body(name="exp-b", experiment_group_id=group_b["id"])) - all_resp = client.get(EXPERIMENTS) + all_resp = client.get(EVALUATIONS) assert all_resp.status_code == 200 names = {e["name"] for e in all_resp.json()["data"]} assert {"exp-a", "exp-b"} <= names - in_group_a = client.get(EXPERIMENTS, params={"filter[experiment_group_id]": group_a["id"]}) + in_group_a = client.get(EVALUATIONS, params={"filter[experiment_group_id]": group_a["id"]}) assert in_group_a.status_code == 200 assert {e["name"] for e in in_group_a.json()["data"]} == {"exp-a"} - deleted = client.delete(f"{EXPERIMENTS}/exp-a") + deleted = client.delete(f"{EVALUATIONS}/exp-a") assert deleted.status_code == 204 - missing = client.get(f"{EXPERIMENTS}/exp-a") + missing = client.get(f"{EVALUATIONS}/exp-a") assert missing.status_code == 404 -def test_experiment_filter_by_dataset_version(client: TestClient) -> None: +def test_evaluation_filter_by_dataset_version(client: TestClient) -> None: group = _create_group(client) client.post( - EXPERIMENTS, - json=_experiment_body(name="exp-v1", dataset_version="v1", experiment_group_id=group["id"]), + EVALUATIONS, + json=_evaluation_body(name="exp-v1", dataset_version="v1", experiment_group_id=group["id"]), ) client.post( - EXPERIMENTS, - json=_experiment_body(name="exp-v2", dataset_version="v1", experiment_group_id=group["id"]), + EVALUATIONS, + json=_evaluation_body(name="exp-v2", dataset_version="v1", experiment_group_id=group["id"]), ) client.post( - EXPERIMENTS, - json=_experiment_body(name="exp-v3", dataset_version="v2", experiment_group_id=group["id"]), + EVALUATIONS, + json=_evaluation_body(name="exp-v3", dataset_version="v2", experiment_group_id=group["id"]), ) - by_dataset_version = client.get(EXPERIMENTS, params={"filter[dataset_version]": "v1"}) + by_dataset_version = client.get(EVALUATIONS, params={"filter[dataset_version]": "v1"}) assert by_dataset_version.status_code == 200 assert {e["name"] for e in by_dataset_version.json()["data"]} == {"exp-v1", "exp-v2"} -def test_experiment_filter_by_created_at_range(client: TestClient) -> None: +def test_evaluation_filter_by_created_at_range(client: TestClient) -> None: from datetime import datetime, timedelta, timezone group = _create_group(client) before_create = datetime.now(timezone.utc) - timedelta(seconds=2) - client.post(EXPERIMENTS, json=_experiment_body(name="exp-recent", experiment_group_id=group["id"])) + client.post(EVALUATIONS, json=_evaluation_body(name="exp-recent", experiment_group_id=group["id"])) after_create = datetime.now(timezone.utc) + timedelta(seconds=2) - # Range that brackets the create timestamp -> the experiment is included. + # Range that brackets the create timestamp -> the evaluation is included. in_range = client.get( - EXPERIMENTS, + EVALUATIONS, params={ "filter[name]": "exp-recent", "filter[created_at][$gte]": before_create.isoformat(), @@ -255,7 +310,7 @@ def test_experiment_filter_by_created_at_range(client: TestClient) -> None: # Range entirely after the create timestamp -> excluded. future_only = client.get( - EXPERIMENTS, + EVALUATIONS, params={ "filter[name]": "exp-recent", "filter[created_at][$gte]": (after_create + timedelta(hours=1)).isoformat(), @@ -265,36 +320,36 @@ def test_experiment_filter_by_created_at_range(client: TestClient) -> None: assert future_only.json()["data"] == [] -def test_experiment_filter_by_created_by(client: TestClient) -> None: +def test_evaluation_filter_by_created_by(client: TestClient) -> None: # The test harness doesn't set an authenticated principal, so we only verify the filter # parameter is accepted and routed through the entity store without erroring. group = _create_group(client) - client.post(EXPERIMENTS, json=_experiment_body(name="exp-cb", experiment_group_id=group["id"])) - response = client.get(EXPERIMENTS, params={"filter[created_by]": "someone@example.com"}) + client.post(EVALUATIONS, json=_evaluation_body(name="exp-cb", experiment_group_id=group["id"])) + response = client.get(EVALUATIONS, params={"filter[created_by]": "someone@example.com"}) assert response.status_code == 200, response.text def test_soft_delete_frees_name_for_reuse(client: TestClient) -> None: group = _create_group(client) - first = client.post(EXPERIMENTS, json=_experiment_body(name="reusable", experiment_group_id=group["id"])) + first = client.post(EVALUATIONS, json=_evaluation_body(name="reusable", experiment_group_id=group["id"])) assert first.status_code == 201, first.text - deleted = client.delete(f"{EXPERIMENTS}/reusable") + deleted = client.delete(f"{EVALUATIONS}/reusable") assert deleted.status_code == 204, deleted.text - # The original name is now free; a new experiment can claim it. - second = client.post(EXPERIMENTS, json=_experiment_body(name="reusable", experiment_group_id=group["id"])) + # The original name is now free; a new evaluation can claim it. + second = client.post(EVALUATIONS, json=_evaluation_body(name="reusable", experiment_group_id=group["id"])) assert second.status_code == 201, second.text assert second.json()["id"] != first.json()["id"] def test_list_hides_soft_deleted_by_default(client: TestClient) -> None: group = _create_group(client) - client.post(EXPERIMENTS, json=_experiment_body(name="exp-live", experiment_group_id=group["id"])) - client.post(EXPERIMENTS, json=_experiment_body(name="exp-gone", experiment_group_id=group["id"])) - client.delete(f"{EXPERIMENTS}/exp-gone") + client.post(EVALUATIONS, json=_evaluation_body(name="exp-live", experiment_group_id=group["id"])) + client.post(EVALUATIONS, json=_evaluation_body(name="exp-gone", experiment_group_id=group["id"])) + client.delete(f"{EVALUATIONS}/exp-gone") - listed = client.get(EXPERIMENTS) + listed = client.get(EVALUATIONS) assert listed.status_code == 200 names = {e["name"] for e in listed.json()["data"]} assert "exp-live" in names @@ -304,14 +359,14 @@ def test_list_hides_soft_deleted_by_default(client: TestClient) -> None: def test_filter_is_deleted_true_returns_only_deleted(client: TestClient) -> None: group = _create_group(client) - client.post(EXPERIMENTS, json=_experiment_body(name="exp-still-here", experiment_group_id=group["id"])) - client.post(EXPERIMENTS, json=_experiment_body(name="exp-trash", experiment_group_id=group["id"])) - client.delete(f"{EXPERIMENTS}/exp-trash") + client.post(EVALUATIONS, json=_evaluation_body(name="exp-still-here", experiment_group_id=group["id"])) + client.post(EVALUATIONS, json=_evaluation_body(name="exp-trash", experiment_group_id=group["id"])) + client.delete(f"{EVALUATIONS}/exp-trash") - response = client.get(EXPERIMENTS, params={"filter[is_deleted]": "true"}) + response = client.get(EVALUATIONS, params={"filter[is_deleted]": "true"}) assert response.status_code == 200, response.text data = response.json()["data"] - # Live experiments are excluded; only deleted rows (with mangled names) appear. The + # Live evaluations are excluded; only deleted rows (with mangled names) appear. The # response body intentionally omits ``is_deleted``; the filter context and mangled name # are the signal that these are trash-bin rows. assert any(e["name"].startswith("exp-trash-deleted-") for e in data) @@ -320,8 +375,8 @@ def test_filter_is_deleted_true_returns_only_deleted(client: TestClient) -> None def test_group_soft_delete_cascades_and_frees_names(client: TestClient) -> None: group = client.post(GROUPS, json={"name": "doomed-group-v2"}).json() - client.post(EXPERIMENTS, json=_experiment_body(name="child-1", experiment_group_id=group["id"])) - client.post(EXPERIMENTS, json=_experiment_body(name="child-2", experiment_group_id=group["id"])) + client.post(EVALUATIONS, json=_evaluation_body(name="child-1", experiment_group_id=group["id"])) + client.post(EVALUATIONS, json=_evaluation_body(name="child-2", experiment_group_id=group["id"])) deleted = client.delete(f"{GROUPS}/doomed-group-v2") assert deleted.status_code == 204, deleted.text @@ -329,136 +384,136 @@ def test_group_soft_delete_cascades_and_frees_names(client: TestClient) -> None: # Group and children all read as 404 (live view). assert client.get(f"{GROUPS}/doomed-group-v2").status_code == 404 for child_name in ("child-1", "child-2"): - assert client.get(f"{EXPERIMENTS}/{child_name}").status_code == 404 + assert client.get(f"{EVALUATIONS}/{child_name}").status_code == 404 # Names are reusable in a fresh group. fresh_group = client.post(GROUPS, json={"name": "doomed-group-v2"}) assert fresh_group.status_code == 201, fresh_group.text revived = client.post( - EXPERIMENTS, json=_experiment_body(name="child-1", experiment_group_id=fresh_group.json()["id"]) + EVALUATIONS, json=_evaluation_body(name="child-1", experiment_group_id=fresh_group.json()["id"]) ) assert revived.status_code == 201, revived.text # Trash view still surfaces the cascaded rows. deleted_groups = client.get(GROUPS, params={"filter[is_deleted]": "true"}) assert any(g["name"].startswith("doomed-group-v2-deleted-") for g in deleted_groups.json()["data"]) - deleted_exps = client.get(EXPERIMENTS, params={"filter[is_deleted]": "true"}) + deleted_exps = client.get(EVALUATIONS, params={"filter[is_deleted]": "true"}) deleted_names = {e["name"] for e in deleted_exps.json()["data"]} assert any(n.startswith("child-1-deleted-") for n in deleted_names) assert any(n.startswith("child-2-deleted-") for n in deleted_names) -def test_create_experiment_in_deleted_group_rejected(client: TestClient) -> None: +def test_create_evaluation_in_deleted_group_rejected(client: TestClient) -> None: group = client.post(GROUPS, json={"name": "ephemeral-group"}).json() client.delete(f"{GROUPS}/ephemeral-group") response = client.post( - EXPERIMENTS, - json=_experiment_body(name="orphan", experiment_group_id=group["id"]), + EVALUATIONS, + json=_evaluation_body(name="orphan", experiment_group_id=group["id"]), ) assert response.status_code == 400, response.text assert "deleted" in response.json()["detail"].lower() -def test_update_or_delete_deleted_experiment_returns_404(client: TestClient) -> None: +def test_update_or_delete_deleted_evaluation_returns_404(client: TestClient) -> None: group = _create_group(client) - client.post(EXPERIMENTS, json=_experiment_body(name="exp-once", experiment_group_id=group["id"])) - client.delete(f"{EXPERIMENTS}/exp-once") + client.post(EVALUATIONS, json=_evaluation_body(name="exp-once", experiment_group_id=group["id"])) + client.delete(f"{EVALUATIONS}/exp-once") # GET, PUT, and a second DELETE all 404 on the (now-renamed) deleted row. - assert client.get(f"{EXPERIMENTS}/exp-once").status_code == 404 + assert client.get(f"{EVALUATIONS}/exp-once").status_code == 404 assert ( client.put( - f"{EXPERIMENTS}/exp-once", - json=_experiment_body(name="exp-once", experiment_group_id=group["id"]), + f"{EVALUATIONS}/exp-once", + json=_evaluation_body(name="exp-once", experiment_group_id=group["id"]), ).status_code == 404 ) - delete_response = client.delete(f"{EXPERIMENTS}/exp-once") + delete_response = client.delete(f"{EVALUATIONS}/exp-once") assert delete_response.status_code == 404 -def test_pin_and_unpin_experiment(client: TestClient) -> None: +def test_pin_and_unpin_evaluation(client: TestClient) -> None: group = _create_group(client) - client.post(EXPERIMENTS, json=_experiment_body(name="exp-pin", experiment_group_id=group["id"])) + client.post(EVALUATIONS, json=_evaluation_body(name="exp-pin", experiment_group_id=group["id"])) - # New experiments start unpinned. - fetched = client.get(f"{EXPERIMENTS}/exp-pin") + # New evaluations start unpinned. + fetched = client.get(f"{EVALUATIONS}/exp-pin") assert fetched.status_code == 200 assert fetched.json()["pinned_at"] is None # Pin sets pinned_at to a timestamp. - pinned = client.post(f"{EXPERIMENTS}/exp-pin/pin") + pinned = client.post(f"{EVALUATIONS}/exp-pin/pin") assert pinned.status_code == 200, pinned.text first_pinned_at = pinned.json()["pinned_at"] assert first_pinned_at is not None # Re-pinning refreshes pinned_at (most-recently-pinned-first ordering). - re_pinned = client.post(f"{EXPERIMENTS}/exp-pin/pin") + re_pinned = client.post(f"{EVALUATIONS}/exp-pin/pin") assert re_pinned.status_code == 200, re_pinned.text assert re_pinned.json()["pinned_at"] >= first_pinned_at # Unpin clears pinned_at. - unpinned = client.delete(f"{EXPERIMENTS}/exp-pin/pin") + unpinned = client.delete(f"{EVALUATIONS}/exp-pin/pin") assert unpinned.status_code == 200, unpinned.text assert unpinned.json()["pinned_at"] is None - # Unpin on an already-unpinned experiment is a no-op. - again = client.delete(f"{EXPERIMENTS}/exp-pin/pin") + # Unpin on an already-unpinned evaluation is a no-op. + again = client.delete(f"{EVALUATIONS}/exp-pin/pin") assert again.status_code == 200, again.text assert again.json()["pinned_at"] is None -def test_pin_unknown_experiment_returns_404(client: TestClient) -> None: - assert client.post(f"{EXPERIMENTS}/missing/pin").status_code == 404 - assert client.delete(f"{EXPERIMENTS}/missing/pin").status_code == 404 +def test_pin_unknown_evaluation_returns_404(client: TestClient) -> None: + assert client.post(f"{EVALUATIONS}/missing/pin").status_code == 404 + assert client.delete(f"{EVALUATIONS}/missing/pin").status_code == 404 -def test_pin_rejects_deleted_experiment(client: TestClient) -> None: +def test_pin_rejects_deleted_evaluation(client: TestClient) -> None: group = _create_group(client) - client.post(EXPERIMENTS, json=_experiment_body(name="exp-soft", experiment_group_id=group["id"])) - assert client.delete(f"{EXPERIMENTS}/exp-soft").status_code == 204 - assert client.post(f"{EXPERIMENTS}/exp-soft/pin").status_code == 404 + client.post(EVALUATIONS, json=_evaluation_body(name="exp-soft", experiment_group_id=group["id"])) + assert client.delete(f"{EVALUATIONS}/exp-soft").status_code == 204 + assert client.post(f"{EVALUATIONS}/exp-soft/pin").status_code == 404 def test_filter_by_is_pinned(client: TestClient) -> None: group = _create_group(client) - client.post(EXPERIMENTS, json=_experiment_body(name="exp-pinned-a", experiment_group_id=group["id"])) - client.post(EXPERIMENTS, json=_experiment_body(name="exp-pinned-b", experiment_group_id=group["id"])) - client.post(EXPERIMENTS, json=_experiment_body(name="exp-not-pinned", experiment_group_id=group["id"])) - client.post(f"{EXPERIMENTS}/exp-pinned-a/pin") - client.post(f"{EXPERIMENTS}/exp-pinned-b/pin") + client.post(EVALUATIONS, json=_evaluation_body(name="exp-pinned-a", experiment_group_id=group["id"])) + client.post(EVALUATIONS, json=_evaluation_body(name="exp-pinned-b", experiment_group_id=group["id"])) + client.post(EVALUATIONS, json=_evaluation_body(name="exp-not-pinned", experiment_group_id=group["id"])) + client.post(f"{EVALUATIONS}/exp-pinned-a/pin") + client.post(f"{EVALUATIONS}/exp-pinned-b/pin") - only_pinned = client.get(EXPERIMENTS, params={"filter[is_pinned]": "true"}) + only_pinned = client.get(EVALUATIONS, params={"filter[is_pinned]": "true"}) assert only_pinned.status_code == 200, only_pinned.text pinned_names = {e["name"] for e in only_pinned.json()["data"]} assert pinned_names == {"exp-pinned-a", "exp-pinned-b"} - only_unpinned = client.get(EXPERIMENTS, params={"filter[is_pinned]": "false"}) + only_unpinned = client.get(EVALUATIONS, params={"filter[is_pinned]": "false"}) assert only_unpinned.status_code == 200, only_unpinned.text unpinned_names = {e["name"] for e in only_unpinned.json()["data"]} assert "exp-not-pinned" in unpinned_names assert "exp-pinned-a" not in unpinned_names assert "exp-pinned-b" not in unpinned_names - no_filter = client.get(EXPERIMENTS) + no_filter = client.get(EVALUATIONS) all_names = {e["name"] for e in no_filter.json()["data"]} assert {"exp-pinned-a", "exp-pinned-b", "exp-not-pinned"} <= all_names def test_sort_by_pinned_at_most_recent_first(client: TestClient) -> None: group = _create_group(client) - client.post(EXPERIMENTS, json=_experiment_body(name="exp-old-pin", experiment_group_id=group["id"])) - client.post(EXPERIMENTS, json=_experiment_body(name="exp-new-pin", experiment_group_id=group["id"])) - client.post(f"{EXPERIMENTS}/exp-old-pin/pin") - client.post(f"{EXPERIMENTS}/exp-new-pin/pin") + client.post(EVALUATIONS, json=_evaluation_body(name="exp-old-pin", experiment_group_id=group["id"])) + client.post(EVALUATIONS, json=_evaluation_body(name="exp-new-pin", experiment_group_id=group["id"])) + client.post(f"{EVALUATIONS}/exp-old-pin/pin") + client.post(f"{EVALUATIONS}/exp-new-pin/pin") - pinned_desc = client.get(EXPERIMENTS, params={"filter[is_pinned]": "true", "sort": "-pinned_at"}) + pinned_desc = client.get(EVALUATIONS, params={"filter[is_pinned]": "true", "sort": "-pinned_at"}) assert pinned_desc.status_code == 200, pinned_desc.text names_desc = [e["name"] for e in pinned_desc.json()["data"]] assert names_desc.index("exp-new-pin") < names_desc.index("exp-old-pin") - pinned_asc = client.get(EXPERIMENTS, params={"filter[is_pinned]": "true", "sort": "pinned_at"}) + pinned_asc = client.get(EVALUATIONS, params={"filter[is_pinned]": "true", "sort": "pinned_at"}) assert pinned_asc.status_code == 200, pinned_asc.text names_asc = [e["name"] for e in pinned_asc.json()["data"]] assert names_asc.index("exp-old-pin") < names_asc.index("exp-new-pin") diff --git a/services/intake/tests/test_experiment_default_sort.py b/services/intake/tests/test_experiment_default_sort.py index 110f7d1d97..dd1a4d0612 100644 --- a/services/intake/tests/test_experiment_default_sort.py +++ b/services/intake/tests/test_experiment_default_sort.py @@ -4,8 +4,8 @@ """Experiment group default sort: string storage/validation and the sort helper. ``default_sort`` is a single ``sort``-param string (e.g. ``-cost_usd.mean`` or ``-created_at``) stored -on the group — any field the experiments list can sort by. The client reads it and applies it as the -list ``sort`` param; the list endpoint itself never consults it. The ``_sort_experiments`` helper +on the group — any field the evaluations list can sort by. The client reads it and applies it as the +list ``sort`` param; the list endpoint itself never consults it. The ``_sort_evaluations`` helper remains multi-key capable (pinned-first + tiebreaks), so its unit tests still exercise lists. """ @@ -14,10 +14,10 @@ import pytest from fastapi import HTTPException from fastapi.testclient import TestClient -from nmp.intake.api.v2.experiments.endpoints import _sort_experiments, _validate_default_sort -from nmp.intake.api.v2.experiments.schemas import EvaluatorAggregate, ExperimentResponse +from nmp.intake.api.v2.experiments.endpoints import _sort_evaluations, _validate_default_sort +from nmp.intake.api.v2.experiments.schemas import EvaluationResponse, EvaluatorAggregate -EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" +EVALUATIONS = "/apis/intake/v2/workspaces/default/evaluations" GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" @@ -28,8 +28,8 @@ def _exp( latency: float | None = None, pinned: bool = False, created: datetime | None = None, -) -> ExperimentResponse: - return ExperimentResponse( +) -> EvaluationResponse: + return EvaluationResponse( id=name, name=name, workspace="default", @@ -42,7 +42,7 @@ def _exp( ) -def _names(responses: list[ExperimentResponse]) -> list[str]: +def _names(responses: list[EvaluationResponse]) -> list[str]: return [r.name for r in responses] @@ -52,13 +52,13 @@ def _names(responses: list[ExperimentResponse]) -> list[str]: def test_multi_key_primary_then_tiebreak() -> None: # Primary cost asc; ties on cost broken by latency asc. rows = [_exp("a", cost=1.0, latency=200), _exp("b", cost=1.0, latency=100), _exp("c", cost=0.5, latency=999)] - ordered = _sort_experiments(rows, keys=[("cost_usd.mean", False), ("latency_ms.mean", False)]) + ordered = _sort_evaluations(rows, keys=[("cost_usd.mean", False), ("latency_ms.mean", False)]) assert _names(ordered) == ["c", "b", "a"] def test_pinned_floats_to_top() -> None: rows = [_exp("a", cost=0.1), _exp("pinned", cost=0.9, pinned=True), _exp("b", cost=0.5)] - ordered = _sort_experiments(rows, keys=[("cost_usd.mean", False)], pinned_first=True) + ordered = _sort_evaluations(rows, keys=[("cost_usd.mean", False)], pinned_first=True) # Pinned first regardless of metric; unpinned follow in cost order. assert _names(ordered) == ["pinned", "a", "b"] @@ -69,7 +69,7 @@ def test_falls_back_to_created_at_when_sorted_metric_missing() -> None: _exp("new", created=datetime(2026, 6, 1, tzinfo=timezone.utc)), ] # No cost rollup on either -> the appended -created_at key decides (newest first). - ordered = _sort_experiments(rows, keys=[("cost_usd.mean", False), ("created_at", True)]) + ordered = _sort_evaluations(rows, keys=[("cost_usd.mean", False), ("created_at", True)]) assert _names(ordered) == ["new", "old"] @@ -153,13 +153,13 @@ def test_default_order_floats_pinned_first(client: TestClient) -> None: group = client.post(GROUPS, json={"name": "g-pin"}).json() for name in ("exp-a", "exp-b"): created = client.post( - EXPERIMENTS, json={"name": name, "experiment_group_id": group["id"], "dataset_name": "ds"} + EVALUATIONS, json={"name": name, "experiment_group_id": group["id"], "dataset_name": "ds"} ) assert created.status_code == 201, created.text - # Pin the OLDER experiment (exp-a): by the -created_at fallback it would sort LAST, so seeing it + # Pin the OLDER evaluation (exp-a): by the -created_at fallback it would sort LAST, so seeing it # first proves pinned-first actually overrides the fallback rather than coinciding with newest-first. - assert client.post(f"{EXPERIMENTS}/exp-a/pin").status_code == 200 + assert client.post(f"{EVALUATIONS}/exp-a/pin").status_code == 200 # No explicit sort -> default path floats pinned to top (entity-only, no rollups needed). - listed = client.get(EXPERIMENTS, params={"filter[experiment_group_id]": group["id"]}) + listed = client.get(EVALUATIONS, params={"filter[experiment_group_id]": group["id"]}) assert listed.status_code == 200, listed.text assert [r["name"] for r in listed.json()["data"]] == ["exp-a", "exp-b"] diff --git a/services/intake/tests/test_experiment_metric_filter.py b/services/intake/tests/test_experiment_metric_filter.py index de56047df4..b6c23ed6c5 100644 --- a/services/intake/tests/test_experiment_metric_filter.py +++ b/services/intake/tests/test_experiment_metric_filter.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Metric filtering on the experiments list (Option A app-merge). +"""Metric filtering on the evaluations list (Option A app-merge). Two layers: pure helpers (split/validate/match) and endpoint wiring. The shared ``client`` fixture overrides the rollup repository to ``None`` (ClickHouse unavailable), so a metric filter that passes @@ -20,14 +20,14 @@ _matches_metric_predicates, _operation_references_metric, ) -from nmp.intake.api.v2.experiments.schemas import EvaluatorAggregate, ExperimentResponse, MetricStatFilters +from nmp.intake.api.v2.experiments.schemas import EvaluationResponse, EvaluatorAggregate, MetricStatFilters -EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" +EVALUATIONS = "/apis/intake/v2/workspaces/default/evaluations" GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" -def _exp(name: str, *, run_count: int = 0, cost_mean: float | None = None) -> ExperimentResponse: - return ExperimentResponse( +def _exp(name: str, *, run_count: int = 0, cost_mean: float | None = None) -> EvaluationResponse: + return EvaluationResponse( id=name, name=name, workspace="default", @@ -155,11 +155,11 @@ def test_matches_predicates_excludes_missing_metric() -> None: # ----------------------------- endpoint wiring ----------------------------- -def _make_experiment(client: TestClient, name: str = "exp-1", group: str = "grp-1") -> None: +def _make_evaluation(client: TestClient, name: str = "exp-1", group: str = "grp-1") -> None: group_resp = client.post(GROUPS, json={"name": group}) assert group_resp.status_code == 201, group_resp.text exp_resp = client.post( - EXPERIMENTS, + EVALUATIONS, json={"name": name, "experiment_group_id": group_resp.json()["id"], "dataset_name": "ds"}, ) assert exp_resp.status_code == 201, exp_resp.text @@ -169,28 +169,28 @@ def test_metric_filter_passes_validation_and_503s_without_rollups(client: TestCl # If the namespace declaration works, these paths get past field validation and reach the # metric-filter path, which 503s because the (mocked) rollup repository is None. Needs a non-empty # result set: an empty group has nothing to hydrate and correctly returns 200 empty. - _make_experiment(client) + _make_evaluation(client) for param in ( {"filter[cost_usd.mean][gte]": "0.5"}, {"filter[latency_ms.p95][lte]": "1000"}, {"filter[evaluators.harbor.verifier.mean][gte]": "0.8"}, {"filter[run_count][gte]": "5"}, ): - response = client.get(EXPERIMENTS, params=param) + response = client.get(EVALUATIONS, params=param) assert response.status_code == 503, (param, response.text) def test_metric_filter_bad_stat_returns_400(client: TestClient) -> None: - response = client.get(EXPERIMENTS, params={"filter[cost_usd.bogus][gte]": "0.5"}) + response = client.get(EVALUATIONS, params={"filter[cost_usd.bogus][gte]": "0.5"}) assert response.status_code == 400, response.text def test_metric_filter_non_numeric_value_returns_400(client: TestClient) -> None: - response = client.get(EXPERIMENTS, params={"filter[cost_usd.mean][gte]": "abc"}) + response = client.get(EVALUATIONS, params={"filter[cost_usd.mean][gte]": "abc"}) assert response.status_code == 400, response.text def test_metric_filter_under_or_returns_400(client: TestClient) -> None: json_filter = '{"$or": [{"cost_usd.mean": {"$gte": 0.5}}, {"name": {"$eq": "x"}}]}' - response = client.get(EXPERIMENTS, params={"filter": json_filter}) + response = client.get(EVALUATIONS, params={"filter": json_filter}) assert response.status_code == 400, response.text diff --git a/services/intake/tests/test_experiment_optimization_fields.py b/services/intake/tests/test_experiment_optimization_fields.py index 4a06322600..cab86e15e1 100644 --- a/services/intake/tests/test_experiment_optimization_fields.py +++ b/services/intake/tests/test_experiment_optimization_fields.py @@ -1,13 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Optional fields on ExperimentGroup (insight_id, summary, metadata) and Experiment -(parent_experiment_id, status, root_cause): round-trip through create/update, parent-reference +"""Optional fields on ExperimentGroup (insight_id, summary, metadata) and Evaluation +(parent_evaluation_id, status, root_cause): round-trip through create/update, parent-reference validation, and free-form status.""" from fastapi.testclient import TestClient -EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" +EVALUATIONS = "/apis/intake/v2/workspaces/default/evaluations" GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" @@ -17,8 +17,8 @@ def _group(client: TestClient, name: str = "grp") -> dict: return resp.json() -def _experiment(client: TestClient, group_id: str, name: str) -> dict: - resp = client.post(EXPERIMENTS, json={"name": name, "experiment_group_id": group_id, "dataset_name": "ds"}) +def _evaluation(client: TestClient, group_id: str, name: str) -> dict: + resp = client.post(EVALUATIONS, json={"name": name, "experiment_group_id": group_id, "dataset_name": "ds"}) assert resp.status_code == 201, resp.text return resp.json() @@ -35,36 +35,36 @@ def test_group_fields_round_trip(client: TestClient) -> None: assert body["metadata"] == {"k": "v"} -def test_experiment_fields_round_trip(client: TestClient) -> None: +def test_evaluation_fields_round_trip(client: TestClient) -> None: group = _group(client) - parent = _experiment(client, group["id"], "exp-parent") + parent = _evaluation(client, group["id"], "exp-parent") resp = client.post( - EXPERIMENTS, + EVALUATIONS, json={ "name": "exp-1", "experiment_group_id": group["id"], "dataset_name": "ds", - "parent_experiment_id": parent["id"], + "parent_evaluation_id": parent["id"], "status": "running", "root_cause": "still evaluating", }, ) assert resp.status_code == 201, resp.text body = resp.json() - assert body["parent_experiment_id"] == parent["id"] + assert body["parent_evaluation_id"] == parent["id"] assert body["status"] == "running" assert body["root_cause"] == "still evaluating" -def test_experiment_rejects_unknown_parent(client: TestClient) -> None: +def test_evaluation_rejects_unknown_parent(client: TestClient) -> None: group = _group(client) resp = client.post( - EXPERIMENTS, + EVALUATIONS, json={ "name": "exp-orphan", "experiment_group_id": group["id"], "dataset_name": "ds", - "parent_experiment_id": "does-not-exist", + "parent_evaluation_id": "does-not-exist", }, ) assert resp.status_code == 400, resp.text @@ -72,14 +72,14 @@ def test_experiment_rejects_unknown_parent(client: TestClient) -> None: def test_update_rejects_unknown_parent(client: TestClient) -> None: group = _group(client) - _experiment(client, group["id"], "exp-u") + _evaluation(client, group["id"], "exp-u") updated = client.put( - f"{EXPERIMENTS}/exp-u", + f"{EVALUATIONS}/exp-u", json={ "name": "exp-u", "experiment_group_id": group["id"], "dataset_name": "ds", - "parent_experiment_id": "does-not-exist", + "parent_evaluation_id": "does-not-exist", }, ) assert updated.status_code == 400, updated.text @@ -89,18 +89,18 @@ def test_status_is_a_free_string(client: TestClient) -> None: # status is producer-defined, not a fixed enum — any string is accepted. group = _group(client) resp = client.post( - EXPERIMENTS, + EVALUATIONS, json={"name": "exp-custom", "experiment_group_id": group["id"], "dataset_name": "ds", "status": "my-own-state"}, ) assert resp.status_code == 201, resp.text assert resp.json()["status"] == "my-own-state" -def test_experiment_status_and_root_cause_update(client: TestClient) -> None: +def test_evaluation_status_and_root_cause_update(client: TestClient) -> None: group = _group(client) - _experiment(client, group["id"], "exp-3") + _evaluation(client, group["id"], "exp-3") updated = client.put( - f"{EXPERIMENTS}/exp-3", + f"{EVALUATIONS}/exp-3", json={ "name": "exp-3", "experiment_group_id": group["id"], @@ -118,7 +118,7 @@ def test_filter_experiments_by_metadata(client: TestClient) -> None: group = _group(client, name="g-meta") for name, model in (("exp-claude", "claude-opus"), ("exp-gpt", "gpt-5")): resp = client.post( - EXPERIMENTS, + EVALUATIONS, json={ "name": name, "experiment_group_id": group["id"], @@ -129,12 +129,12 @@ def test_filter_experiments_by_metadata(client: TestClient) -> None: assert resp.status_code == 201, resp.text # A distinct value narrows to the one experiment that has it. - only_claude = client.get(EXPERIMENTS, params={"filter[metadata.model]": "claude-opus"}) + only_claude = client.get(EVALUATIONS, params={"filter[metadata.model]": "claude-opus"}) assert only_claude.status_code == 200, only_claude.text assert [e["name"] for e in only_claude.json()["data"]] == ["exp-claude"] # A shared value returns both. - both = client.get(EXPERIMENTS, params={"filter[metadata.lane]": "gold"}) + both = client.get(EVALUATIONS, params={"filter[metadata.lane]": "gold"}) assert {e["name"] for e in both.json()["data"]} == {"exp-claude", "exp-gpt"} @@ -153,5 +153,5 @@ def test_new_fields_are_optional(client: TestClient) -> None: gbody = _group(client, name="g-min") assert gbody["insight_id"] is None and gbody["summary"] is None and gbody["metadata"] is None - ebody = _experiment(client, gbody["id"], "exp-min") - assert ebody["parent_experiment_id"] is None and ebody["status"] is None and ebody["root_cause"] is None + ebody = _evaluation(client, gbody["id"], "exp-min") + assert ebody["parent_evaluation_id"] is None and ebody["status"] is None and ebody["root_cause"] is None diff --git a/services/intake/tests/test_experiment_rollup_repository.py b/services/intake/tests/test_experiment_rollup_repository.py index 799f73dbba..2288262c7c 100644 --- a/services/intake/tests/test_experiment_rollup_repository.py +++ b/services/intake/tests/test_experiment_rollup_repository.py @@ -1,13 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Experiment rollup repository tests.""" +"""Evaluation rollup repository tests.""" from typing import cast import pytest from nmp.intake.spans.clickhouse_client import ClickHouseSpanClient -from nmp.intake.spans.experiment_rollup_repository import ExperimentRollupRepository +from nmp.intake.spans.evaluation_rollup_repository import EvaluationRollupRepository class _QueryResult: @@ -31,21 +31,21 @@ async def query(self, query: str, *, parameters: dict[str, object]) -> _QueryRes return self.query_results.pop(0) -def _repository(client: _Client) -> ExperimentRollupRepository: - return ExperimentRollupRepository(cast(ClickHouseSpanClient, client)) +def _repository(client: _Client) -> EvaluationRollupRepository: + return EvaluationRollupRepository(cast(ClickHouseSpanClient, client)) @pytest.mark.asyncio -async def test_experiment_rollups_anchor_on_root_session_membership(): +async def test_evaluation_rollups_anchor_on_root_session_membership(): client = _Client( [ _QueryResult( [("exp-a", 3)], - ["experiment_id", "run_count"], + ["evaluation_id", "run_count"], ), _QueryResult( [("exp-a", "reward", 3.0, 0.75, 0.8, 1.0, 1.0, 1.0, 4)], - ["experiment_id", "evaluator_name", "sum", "mean", "median", "p90", "p95", "p99", "count"], + ["evaluation_id", "evaluator_name", "sum", "mean", "median", "p90", "p95", "p99", "count"], ), _QueryResult( [ @@ -71,7 +71,7 @@ async def test_experiment_rollups_anchor_on_root_session_membership(): ) ], [ - "experiment_id", + "evaluation_id", "model_names", "agent_names", "agent_versions", @@ -95,7 +95,7 @@ async def test_experiment_rollups_anchor_on_root_session_membership(): ) repository = _repository(client) - rollups = await repository.get_rollups(workspace="default", experiment_ids=["exp-a"]) + rollups = await repository.get_rollups(workspace="default", evaluation_ids=["exp-a"]) rollup = rollups["exp-a"] assert rollup.run_count == 3 @@ -130,16 +130,16 @@ async def test_experiment_rollups_anchor_on_root_session_membership(): assert len(client.queries) == 3 assert "FROM trace_index FINAL" in client.queries[0] assert "count() AS run_count" in client.queries[0] - assert "experiment_id IN (%(experiment_id_0)s)" in client.queries[0] + assert "evaluation_id IN (%(evaluation_id_0)s)" in client.queries[0] assert "ORDER BY root_started_at ASC, root_span_id ASC" in client.queries[0] assert "FROM evaluator_results FINAL" in client.queries[1] assert "quantileExact(0.5)(value) AS median" in client.queries[1] assert "quantileExact(0.99)(value) AS p99" in client.queries[1] assert "AND (workspace, session_id) IN (" in client.queries[1] assert "sessions.session_id = results.session_id" in client.queries[1] - # Scores are reduced to one value per (experiment, session, evaluator) before the + # Scores are reduced to one value per (evaluation, session, evaluator) before the # distribution rollup so count tracks runs and the mean is not span-weighted. - assert "GROUP BY sessions.experiment_id, sessions.session_id, results.name" in client.queries[1] + assert "GROUP BY sessions.evaluation_id, sessions.session_id, results.name" in client.queries[1] assert "current_session_spans AS" in client.queries[2] assert "(span_versions.workspace, span_versions.session_id) IN" in client.queries[2] assert "LEFT JOIN current_session_spans AS spans" in client.queries[2] @@ -150,5 +150,5 @@ async def test_experiment_rollups_anchor_on_root_session_membership(): assert "quantileExactIf(0.99)" in client.queries[2] assert "latency_p99" in client.queries[2] assert "sessions.trace_id = spans.trace_id" not in client.queries[2] - assert client.parameters[0]["experiment_id_0"] == "exp-a" + assert client.parameters[0]["evaluation_id_0"] == "exp-a" assert client.parameters[2]["model_key"] == "gen_ai.request.model" diff --git a/services/intake/tests/test_experiment_session_schemas.py b/services/intake/tests/test_experiment_session_schemas.py index 7f9add5a47..3f018db895 100644 --- a/services/intake/tests/test_experiment_session_schemas.py +++ b/services/intake/tests/test_experiment_session_schemas.py @@ -3,25 +3,25 @@ from datetime import datetime, timezone -from nmp.intake.api.v2.experiments.schemas import ExperimentSessionResponse +from nmp.intake.api.v2.experiments.schemas import EvaluationSessionResponse from nmp.intake.spans.domain import SpanStatus -from nmp.intake.spans.experiment_session_repository import ExperimentSessionRow +from nmp.intake.spans.evaluation_session_repository import EvaluationSessionRow -def test_experiment_session_from_row_preserves_input() -> None: +def test_evaluation_session_from_row_preserves_input() -> None: input_text = "x" * 1050 row = _session_row(input_text=input_text) - response = ExperimentSessionResponse.from_row(row) + response = EvaluationSessionResponse.from_row(row) assert response.input == input_text -def _session_row(input_text: str) -> ExperimentSessionRow: +def _session_row(input_text: str) -> EvaluationSessionRow: now = datetime.now(timezone.utc) - return ExperimentSessionRow( + return EvaluationSessionRow( workspace="default", - experiment_name="experiment", + evaluation_name="evaluation", session_id="session", test_case_id="case", trace_id="trace", diff --git a/services/intake/tests/test_experiment_sort.py b/services/intake/tests/test_experiment_sort.py index 65c323b26f..269872accf 100644 --- a/services/intake/tests/test_experiment_sort.py +++ b/services/intake/tests/test_experiment_sort.py @@ -1,14 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for the in-memory experiment sort (Option A app-merge).""" +"""Unit tests for the in-memory evaluation sort (Option A app-merge).""" from __future__ import annotations import pytest from fastapi import HTTPException -from nmp.intake.api.v2.experiments.endpoints import _sort_experiments, _validate_sort_field -from nmp.intake.api.v2.experiments.schemas import EvaluatorAggregate, ExperimentResponse +from nmp.intake.api.v2.experiments.endpoints import _sort_evaluations, _validate_sort_field +from nmp.intake.api.v2.experiments.schemas import EvaluationResponse, EvaluatorAggregate def _exp( @@ -17,8 +17,8 @@ def _exp( run_count: int = 0, cost_mean: float | None = None, evaluators: dict[str, float] | None = None, -) -> ExperimentResponse: - return ExperimentResponse( +) -> EvaluationResponse: + return EvaluationResponse( id=name, name=name, workspace="default", @@ -30,7 +30,7 @@ def _exp( ) -def _names(responses: list[ExperimentResponse]) -> list[str]: +def _names(responses: list[EvaluationResponse]) -> list[str]: return [r.name for r in responses] @@ -40,43 +40,43 @@ def test_sort_by_evaluator_mean_descending() -> None: _exp("b", evaluators={"reward": 0.9}), _exp("c", evaluators={"reward": 0.6}), ] - ordered = _sort_experiments(rows, keys=[("evaluators.reward.mean", True)]) + ordered = _sort_evaluations(rows, keys=[("evaluators.reward.mean", True)]) assert _names(ordered) == ["b", "c", "a"] def test_sort_by_cost_ascending() -> None: rows = [_exp("a", cost_mean=2.0), _exp("b", cost_mean=0.5), _exp("c", cost_mean=1.0)] - ordered = _sort_experiments(rows, keys=[("cost_usd.mean", False)]) + ordered = _sort_evaluations(rows, keys=[("cost_usd.mean", False)]) assert _names(ordered) == ["b", "c", "a"] def test_sort_by_run_count() -> None: rows = [_exp("a", run_count=3), _exp("b", run_count=10), _exp("c", run_count=1)] - assert _names(_sort_experiments(rows, keys=[("run_count", True)])) == ["b", "a", "c"] + assert _names(_sort_evaluations(rows, keys=[("run_count", True)])) == ["b", "a", "c"] def test_evaluator_name_with_dots_resolves() -> None: # "harbor.verifier" contains a dot; the stat is the last segment. rows = [_exp("a", evaluators={"harbor.verifier": 0.2}), _exp("b", evaluators={"harbor.verifier": 0.8})] - ordered = _sort_experiments(rows, keys=[("evaluators.harbor.verifier.mean", True)]) + ordered = _sort_evaluations(rows, keys=[("evaluators.harbor.verifier.mean", True)]) assert _names(ordered) == ["b", "a"] def test_missing_metric_sorts_last_in_both_directions() -> None: rows = [_exp("scored", cost_mean=1.0), _exp("unscored")] # unscored has no cost - assert _names(_sort_experiments(rows, keys=[("cost_usd.mean", True)])) == ["scored", "unscored"] - assert _names(_sort_experiments(rows, keys=[("cost_usd.mean", False)])) == ["scored", "unscored"] + assert _names(_sort_evaluations(rows, keys=[("cost_usd.mean", True)])) == ["scored", "unscored"] + assert _names(_sort_evaluations(rows, keys=[("cost_usd.mean", False)])) == ["scored", "unscored"] def test_ties_broken_by_name() -> None: rows = [_exp("c", cost_mean=1.0), _exp("a", cost_mean=1.0), _exp("b", cost_mean=1.0)] # Equal values -> deterministic ascending-name order, regardless of sort direction. - assert _names(_sort_experiments(rows, keys=[("cost_usd.mean", True)])) == ["a", "b", "c"] + assert _names(_sort_evaluations(rows, keys=[("cost_usd.mean", True)])) == ["a", "b", "c"] def test_entity_field_sort() -> None: rows = [_exp("b"), _exp("a"), _exp("c")] - assert _names(_sort_experiments(rows, keys=[("name", False)])) == ["a", "b", "c"] + assert _names(_sort_evaluations(rows, keys=[("name", False)])) == ["a", "b", "c"] def test_validate_accepts_entity_and_metric_fields() -> None: diff --git a/services/intake/tests/test_experiment_sort_endpoint.py b/services/intake/tests/test_experiment_sort_endpoint.py index c71de5ef0f..e502f82380 100644 --- a/services/intake/tests/test_experiment_sort_endpoint.py +++ b/services/intake/tests/test_experiment_sort_endpoint.py @@ -1,69 +1,69 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Endpoint-level guards for the experiments list sort (rollups unavailable / bad field). +"""Endpoint-level guards for the evaluations list sort (rollups unavailable / bad field). -The shared ``client`` fixture overrides ``get_experiment_rollup_repository`` to return ``None``, +The shared ``client`` fixture overrides ``get_evaluation_rollup_repository`` to return ``None``, which is exactly the "ClickHouse disabled / unavailable" condition. A metric-backed sort cannot be computed without rollups, so it must fail loudly rather than silently degrade to name order. """ from fastapi.testclient import TestClient -EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" +EVALUATIONS = "/apis/intake/v2/workspaces/default/evaluations" GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" -def _make_experiment(client: TestClient, name: str = "exp-1", group: str = "grp-1") -> None: +def _make_evaluation(client: TestClient, name: str = "exp-1", group: str = "grp-1") -> None: group_resp = client.post(GROUPS, json={"name": group}) assert group_resp.status_code == 201, group_resp.text exp_resp = client.post( - EXPERIMENTS, + EVALUATIONS, json={"name": name, "experiment_group_id": group_resp.json()["id"], "dataset_name": "ds"}, ) assert exp_resp.status_code == 201, exp_resp.text def test_metric_sort_returns_503_when_rollups_unavailable(client: TestClient) -> None: - _make_experiment(client) - response = client.get(EXPERIMENTS, params={"sort": "-cost_usd.mean"}) + _make_evaluation(client) + response = client.get(EVALUATIONS, params={"sort": "-cost_usd.mean"}) assert response.status_code == 503, response.text def test_run_count_sort_returns_503_when_rollups_unavailable(client: TestClient) -> None: - _make_experiment(client) - response = client.get(EXPERIMENTS, params={"sort": "run_count"}) + _make_evaluation(client) + response = client.get(EVALUATIONS, params={"sort": "run_count"}) assert response.status_code == 503, response.text def test_entity_sort_still_succeeds_without_rollups(client: TestClient) -> None: - _make_experiment(client) + _make_evaluation(client) for sort in ("name", "-created_at", "pinned_at"): - response = client.get(EXPERIMENTS, params={"sort": sort}) + response = client.get(EVALUATIONS, params={"sort": sort}) assert response.status_code == 200, response.text def test_unknown_sort_field_returns_400(client: TestClient) -> None: - response = client.get(EXPERIMENTS, params={"sort": "bogus.field"}) + response = client.get(EVALUATIONS, params={"sort": "bogus.field"}) assert response.status_code == 400, response.text -def test_too_many_experiments_to_sort_returns_413(client: TestClient, monkeypatch) -> None: +def test_too_many_evaluations_to_sort_returns_413(client: TestClient, monkeypatch) -> None: # The whole filtered set is sorted in memory; over the cap we refuse rather than return a # silently truncated result. 413 (distinct from the 400 bad-sort-field case) so a caller can tell # the two apart. Shrink the cap so the test stays fast. from nmp.intake.api.v2.experiments import endpoints - monkeypatch.setattr(endpoints, "_MAX_GROUP_EXPERIMENTS", 2) + monkeypatch.setattr(endpoints, "_MAX_GROUP_EVALUATIONS", 2) group_resp = client.post(GROUPS, json={"name": "big-grp"}) group_id = group_resp.json()["id"] for index in range(3): resp = client.post( - EXPERIMENTS, + EVALUATIONS, json={"name": f"exp-{index}", "experiment_group_id": group_id, "dataset_name": "ds"}, ) assert resp.status_code == 201, resp.text - response = client.get(EXPERIMENTS, params={"sort": "name"}) + response = client.get(EVALUATIONS, params={"sort": "name"}) assert response.status_code == 413, response.text assert "exceeding the maximum" in response.json()["detail"] diff --git a/services/intake/tests/test_spans_clickhouse_migrations.py b/services/intake/tests/test_spans_clickhouse_migrations.py index a315be4a8b..d50dbc1d2f 100644 --- a/services/intake/tests/test_spans_clickhouse_migrations.py +++ b/services/intake/tests/test_spans_clickhouse_migrations.py @@ -61,13 +61,13 @@ def test_trace_index_schema_is_root_span_projection(): assert "TO {table}" in ddl assert "INSERT INTO {table}" in source assert "WHERE external_parent_span_id = ''" in ddl - assert "attributes_string['{experiment_key}'] AS experiment_id" in ddl + assert "attributes_string['{evaluation_key}'] AS evaluation_id" in ddl assert "attributes_string['{test_case_key}'] AS test_case_id" in ddl assert "root_status LowCardinality(String)" in ddl assert "root_input String" in ddl assert "PRIMARY KEY (workspace, root_started_at)" in ddl assert "ORDER BY (workspace, root_started_at, trace_id, root_span_id)" in ddl - assert "INDEX idx_experiment_id experiment_id" in ddl + assert "INDEX idx_evaluation_id evaluation_id" in ddl assert "index_granularity = 256" in ddl diff --git a/services/intake/tests/test_spans_schemas.py b/services/intake/tests/test_spans_schemas.py index 7886fb0f2d..5b1ad25ad0 100644 --- a/services/intake/tests/test_spans_schemas.py +++ b/services/intake/tests/test_spans_schemas.py @@ -118,7 +118,7 @@ def test_trace_response_maps_core_trace_fields(): input="root input", output="root output", project="project-a", - experiment_id="experiment-a", + evaluation_id="experiment-a", test_case_id="case-a", started_at=started_at, ended_at=ended_at, @@ -158,6 +158,9 @@ def test_trace_response_maps_core_trace_fields(): assert response.cost_output_usd == 0.0037 assert response.span_count == 2 assert response.error_count == 1 + assert response.evaluation_context is not None + assert response.evaluation_context.evaluation_id == "experiment-a" + assert response.evaluation_context.test_case_id == "case-a" assert response.experiment_context is not None assert response.experiment_context.experiment_id == "experiment-a" assert response.experiment_context.test_case_id == "case-a" diff --git a/services/intake/tests/test_traces_api.py b/services/intake/tests/test_traces_api.py index 7240f3acd5..a220a0a9d3 100644 --- a/services/intake/tests/test_traces_api.py +++ b/services/intake/tests/test_traces_api.py @@ -6,8 +6,6 @@ import json from datetime import datetime, timezone -import pytest -from fastapi import HTTPException from nmp.common.api.filter import parse_json_filter from nmp.common.api.parsed_filter import ParsedFilter from nmp.intake.spans.api.traces import _trace_filter @@ -25,7 +23,7 @@ def test_trace_filter_maps_public_fields_to_repository_filter(): "session_id": "session-a", "status": "error", "started_at": {"$gte": started_at.isoformat()}, - "experiment_id": "experiment-a", + "evaluation_id": "experiment-a", "test_case_id": "case-a", } ), @@ -36,34 +34,35 @@ def test_trace_filter_maps_public_fields_to_repository_filter(): assert filters.session_id == "session-a" assert filters.status == SpanStatus.ERROR assert filters.started_at_gte == started_at - assert filters.experiment_id == "experiment-a" + assert filters.evaluation_id == "experiment-a" assert filters.test_case_id == "case-a" -def test_trace_filter_accepts_experiment_id(): +def test_trace_filter_accepts_evaluation_id(): filters = _trace_filter( "workspace-a", - _parsed_filter({"experiment_id": "experiment-a"}), + _parsed_filter({"evaluation_id": "experiment-a"}), ) - assert filters.experiment_id == "experiment-a" + assert filters.evaluation_id == "experiment-a" + +def test_trace_filter_accepts_deprecated_experiment_id_alias(): + filters = _trace_filter( + "workspace-a", + _parsed_filter({"experiment_id": "experiment-a"}), + ) -def test_trace_filter_rejects_removed_evaluation_id_filter(): - with pytest.raises(HTTPException, match="Unsupported trace filter"): - _trace_filter( - "workspace-a", - _parsed_filter({"evaluation_id": "experiment-a"}), - ) + assert filters.evaluation_id == "experiment-a" -def test_trace_filter_schema_keeps_trace_index_filters_canonical(): +def test_trace_filter_schema_exposes_evaluation_id_with_deprecated_experiment_id_alias(): properties = TraceFilter.model_json_schema()["properties"] - assert "evaluation_id" not in properties - assert properties["experiment_id"]["description"] == "Filter by root-span experiment id." - assert "deprecated" not in properties["experiment_id"] - assert properties["test_case_id"]["description"] == "Filter by root-span experiment test case id." + assert properties["evaluation_id"]["description"] == "Filter by root-span evaluation id." + assert "deprecated" not in properties["evaluation_id"] + assert properties["experiment_id"]["deprecated"] is True + assert properties["test_case_id"]["description"] == "Filter by root-span evaluation test case id." assert "deprecated" not in properties["test_case_id"] diff --git a/services/intake/tests/test_traces_clickhouse_repository.py b/services/intake/tests/test_traces_clickhouse_repository.py index 0a229fb559..83ecb1bbbe 100644 --- a/services/intake/tests/test_traces_clickhouse_repository.py +++ b/services/intake/tests/test_traces_clickhouse_repository.py @@ -130,7 +130,7 @@ async def test_list_traces_maps_detailed_row(): assert trace.output is None assert trace.duration_ms == 2500 assert trace.project == "project-a" - assert trace.experiment_id == "experiment-a" + assert trace.evaluation_id == "experiment-a" assert trace.test_case_id == "case-a" assert trace.input_tokens == 420 assert trace.output_tokens == 310 @@ -197,7 +197,7 @@ async def test_root_filters_use_trace_index_columns(): await repository.list_traces( filters=TraceListFilter( workspace="workspace-a", - experiment_id="experiment-a", + evaluation_id="experiment-a", ), page=1, page_size=10, @@ -205,9 +205,9 @@ async def test_root_filters_use_trace_index_columns(): mode="detailed", ) - assert "trace_roots.experiment_id = %(filter_experiment_id)s" in client.queries[0] + assert "trace_roots.evaluation_id = %(filter_evaluation_id)s" in client.queries[0] assert "candidate_spans" not in client.queries[0] - assert client.parameters[0]["filter_experiment_id"] == "experiment-a" + assert client.parameters[0]["filter_evaluation_id"] == "experiment-a" def _trace_row( @@ -225,7 +225,7 @@ def _trace_row( "root_span_id": "span-root", "name": "root", "project": "project-a", - "experiment_id": "experiment-a", + "evaluation_id": "experiment-a", "test_case_id": "case-a", "started_at": started_at, "ended_at": ended_at, diff --git a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/cli_generator/cli_config.yaml b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/cli_generator/cli_config.yaml index 6706b1b952..56e2e9d1f2 100644 --- a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/cli_generator/cli_config.yaml +++ b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/cli_generator/cli_config.yaml @@ -283,7 +283,7 @@ config: # Experiments are in the OpenAPI spec and SDK (for the frontend) but not yet exposed # as CLI commands; drop the skip once the feature is production-ready. -- resource: [experiments] +- resource: [evaluations] skip: true - resource: [experiment_groups] diff --git a/web/packages/common/src/hooks/useStudioDataViewState/filterFieldMap.integration.test.tsx b/web/packages/common/src/hooks/useStudioDataViewState/filterFieldMap.integration.test.tsx index 97d7414e9a..54b150a43b 100644 --- a/web/packages/common/src/hooks/useStudioDataViewState/filterFieldMap.integration.test.tsx +++ b/web/packages/common/src/hooks/useStudioDataViewState/filterFieldMap.integration.test.tsx @@ -135,7 +135,7 @@ const EVALUATOR_DATA: EvaluatorRow[] = [ { aggregate_scores: { accuracy: { mean: 0.95 } } }, ]; -// Mirrors ExperimentGroupDataView's getExperimentFilterField for the dynamic evaluator id. +// Mirrors ExperimentGroupDataView's getEvaluationFilterField for the dynamic evaluator id. const evaluatorFilterField = (id: string): string | undefined => { const match = id.match(/^evaluator-(.+)$/); return match ? `evaluators.${match[1]}.mean` : undefined; diff --git a/web/packages/studio/src/components/ExperimentGroupEditModal/index.tsx b/web/packages/studio/src/components/ExperimentGroupEditModal/index.tsx index d3d6d990df..b6a56d2403 100644 --- a/web/packages/studio/src/components/ExperimentGroupEditModal/index.tsx +++ b/web/packages/studio/src/components/ExperimentGroupEditModal/index.tsx @@ -6,7 +6,7 @@ import { useToast } from '@nemo/common/src/providers/toast/useToast'; import { getGetExperimentGroupQueryKey, getListExperimentGroupsQueryKey, - useListExperiments, + useListEvaluations, useUpdateExperimentGroup, } from '@nemo/sdk/generated/platform/api'; import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; @@ -40,7 +40,7 @@ export const ExperimentGroupEditModal: FC = ({ }, [open, group]); // Offer the group's discovered evaluators as first-class sort fields (only fetched while open). - const { data: experimentsPage } = useListExperiments( + const { data: experimentsPage } = useListEvaluations( workspace, { filter: { experiment_group_id: group.id }, page_size: 100 }, { query: { enabled: open && !!group.id } } diff --git a/web/packages/studio/src/components/IntakeDetail/IntakeComponents/traceKeyValues.test.tsx b/web/packages/studio/src/components/IntakeDetail/IntakeComponents/traceKeyValues.test.tsx index ca3c53dfe0..634c64fb06 100644 --- a/web/packages/studio/src/components/IntakeDetail/IntakeComponents/traceKeyValues.test.tsx +++ b/web/packages/studio/src/components/IntakeDetail/IntakeComponents/traceKeyValues.test.tsx @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { - buildExperimentContextEntries, + buildEvaluationContextEntries, buildTraceHighlightMetrics, buildTraceSummaryEntries, } from '@studio/components/IntakeDetail/IntakeComponents/traceKeyValues'; @@ -66,19 +66,19 @@ describe('traceKeyValues', () => { ]); }); - it('includes experiment context entries when present', () => { + it('includes evaluation context entries when present', () => { const trace = mockTraceById('trace-agent-run-001'); expect(trace).toBeDefined(); - const entries = buildExperimentContextEntries(trace!.experiment_context); + const entries = buildEvaluationContextEntries(trace!.evaluation_context); - expect(entries.map((entry) => entry.label)).toEqual(['Experiment ID', 'Test Case ID']); + expect(entries.map((entry) => entry.label)).toEqual(['Evaluation ID', 'Test Case ID']); }); - it('returns no experiment context entries when context is absent', () => { + it('returns no evaluation context entries when context is absent', () => { const trace = mockTraceById('trace-agent-run-002'); expect(trace).toBeDefined(); - expect(buildExperimentContextEntries(trace!.experiment_context)).toEqual([]); + expect(buildEvaluationContextEntries(trace!.evaluation_context)).toEqual([]); }); }); diff --git a/web/packages/studio/src/components/IntakeDetail/IntakeComponents/traceKeyValues.tsx b/web/packages/studio/src/components/IntakeDetail/IntakeComponents/traceKeyValues.tsx index 773f07788a..f3b133ecd2 100644 --- a/web/packages/studio/src/components/IntakeDetail/IntakeComponents/traceKeyValues.tsx +++ b/web/packages/studio/src/components/IntakeDetail/IntakeComponents/traceKeyValues.tsx @@ -5,7 +5,7 @@ // produce JSX such as status badges and links. Not a component module. /* eslint-disable react-refresh/only-export-components */ -import type { ExperimentContext, Trace } from '@nemo/sdk/generated/platform/schema'; +import type { EvaluationContext, Trace } from '@nemo/sdk/generated/platform/schema'; import { formatUnknownKeyValue, isMeaningfulValue, @@ -122,11 +122,11 @@ const TRACE_SUMMARY_DESCRIPTORS: readonly TraceFieldDescriptor[] = [ }, ]; -const EXPERIMENT_CONTEXT_DESCRIPTORS: readonly { - readonly key: keyof ExperimentContext | string; +const EVALUATION_CONTEXT_DESCRIPTORS: readonly { + readonly key: keyof EvaluationContext | string; readonly label: string; }[] = [ - { key: 'experiment_id', label: 'Experiment ID' }, + { key: 'evaluation_id', label: 'Evaluation ID' }, { key: 'test_case_id', label: 'Test Case ID' }, ]; @@ -159,6 +159,7 @@ const collectUnmappedTraceEntries = (trace: Trace): TraceKeyValueEntry[] => { const mappedKeys = new Set([ ...TRACE_SUMMARY_DESCRIPTORS.map((descriptor) => descriptor.key), ...TRACE_HIGHLIGHT_METRIC_KEYS, + 'evaluation_context', 'experiment_context', ]); @@ -227,17 +228,17 @@ export const buildTraceHighlightMetrics = (trace: Trace): TraceHighlightMetric[] }, ]; -export const buildExperimentContextEntries = ( - experimentContext: ExperimentContext | null | undefined +export const buildEvaluationContextEntries = ( + evaluationContext: EvaluationContext | null | undefined ): TraceKeyValueEntry[] => { - if (!experimentContext) { + if (!evaluationContext) { return []; } const mappedKeys = new Set(); - const knownEntries = EXPERIMENT_CONTEXT_DESCRIPTORS.flatMap(({ key, label }) => { + const knownEntries = EVALUATION_CONTEXT_DESCRIPTORS.flatMap(({ key, label }) => { mappedKeys.add(String(key)); - const value = experimentContext[key as keyof ExperimentContext]; + const value = evaluationContext[key as keyof EvaluationContext]; if (!isMeaningfulValue(value)) { return []; } @@ -252,7 +253,7 @@ export const buildExperimentContextEntries = ( ]; }); - const extraEntries = Object.entries(experimentContext) + const extraEntries = Object.entries(evaluationContext) .filter(([key, value]) => !mappedKeys.has(key) && isMeaningfulValue(value)) .sort(([left], [right]) => left.localeCompare(right)) .map(([key, value]) => ({ diff --git a/web/packages/studio/src/components/IntakeDetail/README.md b/web/packages/studio/src/components/IntakeDetail/README.md index 313ca97c41..5134ba395f 100644 --- a/web/packages/studio/src/components/IntakeDetail/README.md +++ b/web/packages/studio/src/components/IntakeDetail/README.md @@ -56,7 +56,7 @@ flowchart TB | Layer | Role | | ------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | | **Routes** | Thin wrappers; resolve workspace + id, set breadcrumbs. `IntakeTraceDetailContent` is exported for experiment trace reuse. | -| **IntakeTraceDetailView** | Page header, trace summary header, span explorer, trace-level Attributes / Experiment Context accordions, raw JSON debug. | +| **IntakeTraceDetailView** | Page header, trace summary header, span explorer, trace-level Attributes / Evaluation Context accordions, raw JSON debug. | | **TraceSpanAccordions** | Fetches span summaries and trace annotations; Tree/List toggle; expand/collapse toolbar; row headers + feedback. | | **SpanTreeView / SpanListView** | Layout shells for tree vs list modes; shared row chrome (`SpanTriggerLabel`, `SpanTriggerMeta`, `SpanFeedbackControls`). | | **TraceSpanAccordionContent** | Lazy `useGetSpan` when a span body is shown; merges list summary with full detail via `mergeSpanDetails`. | diff --git a/web/packages/studio/src/components/IntakeDetail/TraceDetailView.tsx b/web/packages/studio/src/components/IntakeDetail/TraceDetailView.tsx index 280398f684..f687cd275a 100644 --- a/web/packages/studio/src/components/IntakeDetail/TraceDetailView.tsx +++ b/web/packages/studio/src/components/IntakeDetail/TraceDetailView.tsx @@ -8,7 +8,7 @@ import { AccessibleTitle } from '@studio/components/AccessibleTitle'; import { KeyValueRows } from '@studio/components/IntakeDetail/IntakeComponents/KeyValueRows'; import { RawJsonDebug } from '@studio/components/IntakeDetail/IntakeComponents/RawJsonDebug'; import { - buildExperimentContextEntries, + buildEvaluationContextEntries, buildTraceSummaryEntries, } from '@studio/components/IntakeDetail/IntakeComponents/traceKeyValues'; import { TraceSummaryHeader } from '@studio/components/IntakeDetail/TraceDetailSummaryHeader'; @@ -25,7 +25,7 @@ import { CircleAlert } from 'lucide-react'; import { type FC, useEffect, useMemo } from 'react'; const TRACE_SUMMARY_SECTION = 'trace-summary'; -const EXPERIMENT_CONTEXT_SECTION = 'experiment-context'; +const EVALUATION_CONTEXT_SECTION = 'evaluation-context'; interface IntakeTraceDetailViewProps { workspace: string; @@ -57,8 +57,8 @@ export const IntakeTraceDetailView: FC = ({ () => (trace ? buildTraceSummaryEntries(trace, { workspace }) : []), [trace, workspace] ); - const experimentEntries = useMemo( - () => (trace ? buildExperimentContextEntries(trace.experiment_context) : []), + const evaluationEntries = useMemo( + () => (trace ? buildEvaluationContextEntries(trace.evaluation_context) : []), [trace] ); @@ -119,14 +119,14 @@ export const IntakeTraceDetailView: FC = ({ ), }, - ...(experimentEntries.length > 0 + ...(evaluationEntries.length > 0 ? [ { - value: EXPERIMENT_CONTEXT_SECTION, - slotLabel: Experiment Context, + value: EVALUATION_CONTEXT_SECTION, + slotLabel: Evaluation Context, slotContent: ( - + ), }, diff --git a/web/packages/studio/src/components/dataViews/ExperimentSessionsDataView/Empty.tsx b/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/Empty.tsx similarity index 100% rename from web/packages/studio/src/components/dataViews/ExperimentSessionsDataView/Empty.tsx rename to web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/Empty.tsx diff --git a/web/packages/studio/src/components/dataViews/ExperimentSessionsDataView/index.test.tsx b/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.test.tsx similarity index 85% rename from web/packages/studio/src/components/dataViews/ExperimentSessionsDataView/index.test.tsx rename to web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.test.tsx index eef294e486..f02bee68d3 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentSessionsDataView/index.test.tsx +++ b/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.test.tsx @@ -1,11 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { ExperimentSessionsDataView } from '@studio/components/dataViews/ExperimentSessionsDataView'; +import { EvaluationSessionsDataView } from '@studio/components/dataViews/EvaluationSessionsDataView'; import { PLATFORM_BASE_URL } from '@studio/constants/environment'; import { ROUTES } from '@studio/constants/routes'; import { server } from '@studio/mocks/node'; -import { getExperimentDetailRoute } from '@studio/routes/utils'; +import { getEvaluationDetailRoute } from '@studio/routes/utils'; import { renderRoute, screen, waitFor } from '@studio/tests/util/render'; import userEvent from '@testing-library/user-event'; import { http, HttpResponse } from 'msw'; @@ -19,8 +19,8 @@ const EXPERIMENT_GROUP = 'my-group'; const EXPERIMENT_NAME = 'my-experiment'; const TRACE_ID = 'trace-abc-123'; -const SESSIONS_URL = `${PLATFORM_BASE_URL}/apis/intake/v2/workspaces/:workspace/experiments/:name/sessions`; -const EXPERIMENT_URL = `${PLATFORM_BASE_URL}/apis/intake/v2/workspaces/:workspace/experiments/:name`; +const SESSIONS_URL = `${PLATFORM_BASE_URL}/apis/intake/v2/workspaces/:workspace/evaluations/:name/sessions`; +const EVALUATION_URL = `${PLATFORM_BASE_URL}/apis/intake/v2/workspaces/:workspace/evaluations/:name`; const mockSession = { workspace: WORKSPACE, @@ -38,7 +38,7 @@ const mockSessionsPage = { pagination: { page: 1, page_size: 50, current_page_size: 1, total_pages: 1, total_results: 1 }, }; -const mockExperiment = { +const mockEvaluation = { workspace: WORKSPACE, name: EXPERIMENT_NAME, experiment_group_name: EXPERIMENT_GROUP, @@ -48,31 +48,31 @@ const mockExperiment = { const renderDataView = () => renderRoute(undefined, { - history: getExperimentDetailRoute(WORKSPACE, EXPERIMENT_GROUP, EXPERIMENT_NAME), + history: getEvaluationDetailRoute(WORKSPACE, EXPERIMENT_GROUP, EXPERIMENT_NAME), routes: [ { - path: ROUTES.workspace.experimentDetail, + path: ROUTES.workspace.evaluationDetail, element: ( - ), }, { - path: ROUTES.workspace.experimentTraceDetail, + path: ROUTES.workspace.evaluationTraceDetail, element:
, }, ], }); -describe('ExperimentSessionsDataView', () => { +describe('EvaluationSessionsDataView', () => { let sessionRequestModes: Array; beforeEach(() => { sessionRequestModes = []; server.use( - http.get(EXPERIMENT_URL, () => HttpResponse.json(mockExperiment)), + http.get(EVALUATION_URL, () => HttpResponse.json(mockEvaluation)), http.get(SESSIONS_URL, ({ request }) => { sessionRequestModes.push(new URL(request.url).searchParams.get('mode')); return HttpResponse.json(mockSessionsPage); diff --git a/web/packages/studio/src/components/dataViews/ExperimentSessionsDataView/index.tsx b/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.tsx similarity index 87% rename from web/packages/studio/src/components/dataViews/ExperimentSessionsDataView/index.tsx rename to web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.tsx index 8e15c0fe74..ffe3731c1d 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentSessionsDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/EvaluationSessionsDataView/index.tsx @@ -13,20 +13,20 @@ import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataView import { formatDurationMs } from '@nemo/common/src/utils/date'; import { snakeCaseToTitleCase } from '@nemo/common/src/utils/formatters'; import { - listExperimentSessions, - useGetExperiment, - useListExperimentSessions, + listEvaluationSessions, + useGetEvaluation, + useListEvaluationSessions, } from '@nemo/sdk/generated/platform/api'; import type { - ExperimentSessionResponsesPage, - ExperimentSessionFilter, - ExperimentSessionResponse, - ListExperimentSessionsParams, + EvaluationSessionResponsesPage, + EvaluationSessionFilter, + EvaluationSessionResponse, + ListEvaluationSessionsParams, } from '@nemo/sdk/generated/platform/schema'; import { Text, Tooltip } from '@nvidia/foundations-react-core'; -import { Empty } from '@studio/components/dataViews/ExperimentSessionsDataView/Empty'; +import { Empty } from '@studio/components/dataViews/EvaluationSessionsDataView/Empty'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; -import { getExperimentTraceDetailRoute } from '@studio/routes/utils'; +import { getEvaluationTraceDetailRoute } from '@studio/routes/utils'; import { tooltipClassName } from '@studio/styles/common'; import { keepPreviousData } from '@tanstack/react-query'; import { isAxiosError } from 'axios'; @@ -34,14 +34,14 @@ import { Columns3 } from 'lucide-react'; import { type ComponentProps, type FC, useMemo } from 'react'; import { useNavigate } from 'react-router-dom'; -type SessionRow = ExperimentSessionResponse & { _rowId: string }; +type SessionRow = EvaluationSessionResponse & { _rowId: string }; -interface ExperimentSessionsDataViewProps { - experimentName: string; +interface EvaluationSessionsDataViewProps { + evaluationName: string; experimentGroupName: string; } -const mapStatusForBadge = (status: ExperimentSessionResponse['status']) => +const mapStatusForBadge = (status: EvaluationSessionResponse['status']) => status === 'success' ? 'completed' : status; const formatScore = (value: number): string => `${(value * 100).toFixed(1)}%`; @@ -53,36 +53,36 @@ const isUnsupportedModeError = (error: unknown): boolean => { return typeof detail === 'string' && detail === 'Unsupported query parameter(s): mode'; }; -const listExperimentSessionsWithModeFallback = async ( +const listEvaluationSessionsWithModeFallback = async ( workspace: string, - experimentName: string, - params: ListExperimentSessionsParams, + evaluationName: string, + params: ListEvaluationSessionsParams, signal: AbortSignal -): Promise => { +): Promise => { try { - return await listExperimentSessions(workspace, experimentName, params, signal); + return await listEvaluationSessions(workspace, evaluationName, params, signal); } catch (error) { if (params.mode !== 'summary' || !isUnsupportedModeError(error)) { throw error; } const fallbackParams = { ...params }; delete fallbackParams.mode; - return listExperimentSessions(workspace, experimentName, fallbackParams, signal); + return listEvaluationSessions(workspace, evaluationName, fallbackParams, signal); } }; -export const ExperimentSessionsDataView: FC = ({ - experimentName, +export const EvaluationSessionsDataView: FC = ({ + evaluationName, experimentGroupName, }) => { const workspace = useWorkspaceFromPath(); const navigate = useNavigate(); - const dataViewState = useStudioDataViewState({ columnVisibility: {} }); - const { data: experiment } = useGetExperiment(workspace, experimentName); + const dataViewState = useStudioDataViewState({ columnVisibility: {} }); + const { data: experiment } = useGetEvaluation(workspace, evaluationName); const page = dataViewState.pagination.state.pageIndex + 1; const pageSize = dataViewState.pagination.state.pageSize; - const sessionParams = useMemo( + const sessionParams = useMemo( () => ({ page, page_size: pageSize, @@ -97,15 +97,15 @@ export const ExperimentSessionsDataView: FC = ( [dataViewState.apiFilter.filter, dataViewState.debouncedSearchBar, page, pageSize] ); - const { data: sessionsResponse, isLoading } = useListExperimentSessions( + const { data: sessionsResponse, isLoading } = useListEvaluationSessions( workspace, - experimentName, + evaluationName, sessionParams, { query: { placeholderData: keepPreviousData, queryFn: ({ signal }) => - listExperimentSessionsWithModeFallback(workspace, experimentName, sessionParams, signal), + listEvaluationSessionsWithModeFallback(workspace, evaluationName, sessionParams, signal), }, } ); @@ -265,10 +265,10 @@ export const ExperimentSessionsDataView: FC = ( onRowClick={(row) => { if (row.trace_id) { navigate( - getExperimentTraceDetailRoute( + getEvaluationTraceDetailRoute( workspace, experimentGroupName, - experimentName, + evaluationName, row.trace_id ) ); diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/Empty.tsx b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/Empty.tsx index af41a789f8..49e7c05ad9 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/Empty.tsx +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/Empty.tsx @@ -29,7 +29,7 @@ export const Empty = ({ experimentGroupName }: EmptyProps) => { return ( } - header="No Experiments" + header="No Evaluations" emptyMessage="Run an experiment to see results for this group." actions={
diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx index 6f39df84fb..3f6cf18938 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx @@ -15,30 +15,30 @@ import { useToast } from '@nemo/common/src/providers/toast/useToast'; import { formatDurationMs } from '@nemo/common/src/utils/date'; import { snakeCaseToTitleCase } from '@nemo/common/src/utils/formatters'; import type { - ExperimentFilter, + EvaluationFilter, ExperimentGroupResponse, } from '@nemo/sdk/generated/platform/schema'; import { Button, Text, Tooltip } from '@nvidia/foundations-react-core'; import { Empty } from '@studio/components/dataViews/ExperimentGroupDataView/Empty'; import { MeanValueTooltipCell } from '@studio/components/dataViews/ExperimentGroupDataView/MeanValueTooltipCell'; import { - type ExperimentRow, - type ListExperimentsSortParam, - useExperimentGroupExperiments, -} from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupExperiments'; + type EvaluationRow, + type ListEvaluationsSortParam, + useExperimentGroupEvaluations, +} from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; import { useSortErrorRecovery } from '@studio/components/dataViews/ExperimentGroupDataView/useSortErrorRecovery'; import { deriveEvaluatorNames } from '@studio/components/dataViews/ExperimentGroupDataView/util'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; -import { getExperimentDetailRoute } from '@studio/routes/utils'; +import { getEvaluationDetailRoute } from '@studio/routes/utils'; import { tooltipClassName } from '@studio/styles/common'; import { useLocalStorage } from '@studio/util/hooks/useLocalStorage'; import { Columns3, Pin } from 'lucide-react'; import { type ComponentProps, type FC, useCallback, useEffect, useMemo } from 'react'; import { useNavigate } from 'react-router-dom'; -export type { ExperimentRow }; +export type { EvaluationRow }; -const DEFAULT_SORT: ListExperimentsSortParam = '-created_at'; +const DEFAULT_SORT: ListEvaluationsSortParam = '-created_at'; // Maps sortable static column ids to their API sort fields. const STATIC_SORT_FIELD_MAP: Readonly> = { @@ -70,8 +70,8 @@ const seedSortFromDefault = ( }; // Maps a filter column id to its dotted API rollup-stat field (required by the backend parser). -// Evaluator ids are dynamic, so derive `evaluators..mean` here, like getExperimentSortParam. -const getExperimentFilterField = (id: string): string | undefined => { +// Evaluator ids are dynamic, so derive `evaluators..mean` here, like getEvaluationSortParam. +const getEvaluationFilterField = (id: string): string | undefined => { if (id === 'cost_usd') return 'cost_usd.mean'; if (id === 'latency_ms') return 'latency_ms.mean'; const evaluatorMatch = id.match(/^evaluator-(.+)$/); @@ -79,9 +79,9 @@ const getExperimentFilterField = (id: string): string | undefined => { return undefined; }; -const getExperimentSortParam = ( +const getEvaluationSortParam = ( sortingState: { id: string; desc: boolean }[] -): ListExperimentsSortParam | undefined => { +): ListEvaluationsSortParam | undefined => { const [first] = sortingState; // No column sort -> omit `sort`; the API then defaults to -created_at with pinned first. if (!first) return undefined; @@ -93,7 +93,7 @@ const getExperimentSortParam = ( if (evaluatorMatch) field = `evaluators.${evaluatorMatch[1]}.mean`; } if (!field) return DEFAULT_SORT; - return `${first.desc ? '-' : ''}${field}` as ListExperimentsSortParam; + return `${first.desc ? '-' : ''}${field}` as ListEvaluationsSortParam; }; interface ExperimentGroupDataViewProps { @@ -130,12 +130,12 @@ export const ExperimentGroupDataView: FC = ({ grou // reference is stable across renders (until default_sort changes). const defaultSort = useMemo(() => seedSortFromDefault(group.default_sort), [group.default_sort]); - const dataViewState = useStudioDataViewState({ + const dataViewState = useStudioDataViewState({ defaultSort, columnVisibility: { created_by: false, updated_at: false }, // Keep the pin toggle reachable while horizontally scrolling this wide table. columnPinning: { left: ['pin'] }, - filterFieldMap: getExperimentFilterField, + filterFieldMap: getEvaluationFilterField, columnOrder: savedColumnOrder ?? [], }); @@ -147,7 +147,7 @@ export const ExperimentGroupDataView: FC = ({ grou const page = dataViewState.pagination.state.pageIndex + 1; const pageSize = dataViewState.pagination.state.pageSize; - const sortParam = getExperimentSortParam(dataViewState.sorting.state); + const sortParam = getEvaluationSortParam(dataViewState.sorting.state); const { rows: orderedData, @@ -156,7 +156,7 @@ export const ExperimentGroupDataView: FC = ({ grou error, isLoading, isSuccess, - } = useExperimentGroupExperiments({ + } = useExperimentGroupEvaluations({ workspace, experimentGroupId, filter: dataViewState.apiFilter.filter, @@ -197,7 +197,7 @@ export const ExperimentGroupDataView: FC = ({ grou ); const makeColumns = useCallback< - ComponentProps>['makeColumns'] + ComponentProps>['makeColumns'] >( ({ accessor, display }) => [ display({ @@ -227,7 +227,7 @@ export const ExperimentGroupDataView: FC = ({ grou kind="tertiary" color="neutral" size="small" - aria-label={isPinned ? 'Unpin experiment' : 'Pin experiment'} + aria-label={isPinned ? 'Unpin evaluation' : 'Pin evaluation'} aria-pressed={isPinned} onClick={() => togglePin(row.original)} > @@ -441,7 +441,7 @@ export const ExperimentGroupDataView: FC = ({ grou makeColumns={makeColumns} searchField="name" onRowClick={(row) => - navigate(getExperimentDetailRoute(workspace, experimentGroupName, row.name)) + navigate(getEvaluationDetailRoute(workspace, experimentGroupName, row.name)) } toolbarSlotEnd={ ({ })); const mockUseToast = vi.mocked(useToast); -const mockUseListExperiments = vi.mocked(useListExperiments); -const mockUsePinExperiment = vi.mocked(usePinExperiment); -const mockUseUnpinExperiment = vi.mocked(useUnpinExperiment); -const mockGetListExperimentsQueryKey = vi.mocked(getListExperimentsQueryKey); +const mockUseListEvaluations = vi.mocked(useListEvaluations); +const mockUsePinEvaluation = vi.mocked(usePinEvaluation); +const mockUseUnpinEvaluation = vi.mocked(useUnpinEvaluation); +const mockGetListEvaluationsQueryKey = vi.mocked(getListEvaluationsQueryKey); interface Row { id: string; @@ -47,19 +47,19 @@ const queryResult = (rows: Row[], total: number) => isFetching: false, isSuccess: true, error: null, - }) as unknown as ReturnType; + }) as unknown as ReturnType; const mockLists = ( pinned: { rows: Row[]; total: number }, unpinned: { rows: Row[]; total: number } ) => { - mockUseListExperiments.mockImplementation(((_workspace, params) => + mockUseListEvaluations.mockImplementation(((_workspace, params) => (params?.filter as { is_pinned?: boolean } | undefined)?.is_pinned ? queryResult(pinned.rows, pinned.total) - : queryResult(unpinned.rows, unpinned.total)) as typeof useListExperiments); + : queryResult(unpinned.rows, unpinned.total)) as typeof useListEvaluations); }; -const baseParams: UseExperimentGroupExperimentsParams = { +const baseParams: UseExperimentGroupEvaluationsParams = { workspace: 'ws', experimentGroupId: 'grp', filter: undefined, @@ -69,16 +69,16 @@ const baseParams: UseExperimentGroupExperimentsParams = { sort: '-created_at', }; -describe('useExperimentGroupExperiments', () => { +describe('useExperimentGroupEvaluations', () => { beforeEach(() => { vi.clearAllMocks(); mockUseToast.mockReturnValue({ error: vi.fn() } as unknown as ReturnType); - mockUsePinExperiment.mockReturnValue({ + mockUsePinEvaluation.mockReturnValue({ mutate: vi.fn(), - } as unknown as ReturnType); - mockUseUnpinExperiment.mockReturnValue({ + } as unknown as ReturnType); + mockUseUnpinEvaluation.mockReturnValue({ mutate: vi.fn(), - } as unknown as ReturnType); + } as unknown as ReturnType); }); it('paginates over the unpinned set only, so pinned rows do not inflate the page count', () => { @@ -89,7 +89,7 @@ describe('useExperimentGroupExperiments', () => { { rows: Array.from({ length: 50 }, (_unused, i) => unp(`u${i}`)), total: 50 } ); - const { result } = renderHook(() => useExperimentGroupExperiments(baseParams)); + const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); expect(result.current.totalCount).toBe(50); }); @@ -97,7 +97,7 @@ describe('useExperimentGroupExperiments', () => { it('falls back to the pinned count when nothing is unpinned so a fully-pinned group is not empty', () => { mockLists({ rows: [pin('p1'), pin('p2')], total: 2 }, { rows: [], total: 0 }); - const { result } = renderHook(() => useExperimentGroupExperiments(baseParams)); + const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); expect(result.current.totalCount).toBe(2); }); @@ -105,7 +105,7 @@ describe('useExperimentGroupExperiments', () => { it('reports a zero count only when both lists are empty', () => { mockLists({ rows: [], total: 0 }, { rows: [], total: 0 }); - const { result } = renderHook(() => useExperimentGroupExperiments(baseParams)); + const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); expect(result.current.totalCount).toBe(0); }); @@ -114,7 +114,7 @@ describe('useExperimentGroupExperiments', () => { // 'b' appears in both lists (the brief window where the two queries refetch out of step). mockLists({ rows: [pin('a'), pin('b')], total: 2 }, { rows: [unp('b'), unp('c')], total: 2 }); - const { result } = renderHook(() => useExperimentGroupExperiments(baseParams)); + const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); expect(result.current.rows.map((row) => row.name)).toEqual(['a', 'b', 'c']); }); @@ -122,9 +122,9 @@ describe('useExperimentGroupExperiments', () => { it('fetches the full pinned set in one page and paginates the unpinned set by the caller page size', () => { mockLists({ rows: [], total: 0 }, { rows: [], total: 0 }); - renderHook(() => useExperimentGroupExperiments({ ...baseParams, page: 2, pageSize: 25 })); + renderHook(() => useExperimentGroupEvaluations({ ...baseParams, page: 2, pageSize: 25 })); - const calls = mockUseListExperiments.mock.calls; + const calls = mockUseListEvaluations.mock.calls; const pinnedParams = calls.find( (call) => (call[1]?.filter as { is_pinned?: boolean } | undefined)?.is_pinned === true )?.[1]; @@ -140,7 +140,7 @@ describe('useExperimentGroupExperiments', () => { it('stays loading until both queries have loaded, not just the faster one', () => { // Pinned has returned; unpinned is still on its initial load (no data yet). - mockUseListExperiments.mockImplementation(((_workspace, params) => + mockUseListEvaluations.mockImplementation(((_workspace, params) => (params?.filter as { is_pinned?: boolean } | undefined)?.is_pinned ? queryResult([pin('p')], 1) : ({ @@ -148,9 +148,9 @@ describe('useExperimentGroupExperiments', () => { isLoading: true, isFetching: true, error: null, - } as unknown as ReturnType)) as typeof useListExperiments); + } as unknown as ReturnType)) as typeof useListEvaluations); - const { result } = renderHook(() => useExperimentGroupExperiments(baseParams)); + const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); expect(result.current.isLoading).toBe(true); }); @@ -158,14 +158,14 @@ describe('useExperimentGroupExperiments', () => { it('clears loading once both queries have responded', () => { mockLists({ rows: [pin('p')], total: 1 }, { rows: [unp('u')], total: 1 }); - const { result } = renderHook(() => useExperimentGroupExperiments(baseParams)); + const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); expect(result.current.isLoading).toBe(false); }); it('reports isSuccess from the unpinned (sortable) query, not the pinned one', () => { // Pinned has loaded; the unpinned query (which carries the sort) is still fetching its sort. - mockUseListExperiments.mockImplementation(((_workspace, params) => + mockUseListEvaluations.mockImplementation(((_workspace, params) => (params?.filter as { is_pinned?: boolean } | undefined)?.is_pinned ? queryResult([pin('p')], 1) : ({ @@ -174,9 +174,9 @@ describe('useExperimentGroupExperiments', () => { isFetching: true, isSuccess: false, error: null, - } as unknown as ReturnType)) as typeof useListExperiments); + } as unknown as ReturnType)) as typeof useListEvaluations); - const { result } = renderHook(() => useExperimentGroupExperiments(baseParams)); + const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); expect(result.current.isSuccess).toBe(false); }); @@ -184,7 +184,7 @@ describe('useExperimentGroupExperiments', () => { it('reports isSuccess once the unpinned query has loaded the current sort', () => { mockLists({ rows: [pin('p')], total: 1 }, { rows: [unp('u')], total: 1 }); - const { result } = renderHook(() => useExperimentGroupExperiments(baseParams)); + const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); expect(result.current.isSuccess).toBe(true); }); @@ -192,7 +192,7 @@ describe('useExperimentGroupExperiments', () => { it('does not report isSuccess while a new sort is in flight and the previous page is shown as placeholder', () => { // The `keepPreviousData` window: status 'success' but isPlaceholderData true. isSuccess must stay // false, else sort-error recovery banks the about-to-fail sort and the next 413/503 isn't recovered. - mockUseListExperiments.mockImplementation(((_workspace, params) => + mockUseListEvaluations.mockImplementation(((_workspace, params) => (params?.filter as { is_pinned?: boolean } | undefined)?.is_pinned ? queryResult([pin('p')], 1) : ({ @@ -202,9 +202,9 @@ describe('useExperimentGroupExperiments', () => { isSuccess: true, isPlaceholderData: true, error: null, - } as unknown as ReturnType)) as typeof useListExperiments); + } as unknown as ReturnType)) as typeof useListEvaluations); - const { result } = renderHook(() => useExperimentGroupExperiments(baseParams)); + const { result } = renderHook(() => useExperimentGroupEvaluations(baseParams)); expect(result.current.isSuccess).toBe(false); }); @@ -212,14 +212,14 @@ describe('useExperimentGroupExperiments', () => { it('scopes pin/unpin invalidation to this group, not the whole workspace', () => { mockLists({ rows: [], total: 0 }, { rows: [], total: 0 }); - renderHook(() => useExperimentGroupExperiments(baseParams)); + renderHook(() => useExperimentGroupEvaluations(baseParams)); // onSuccess is the group-scoped invalidate the hook wires into both mutations. - const onSuccess = mockUsePinExperiment.mock.calls[0]?.[0]?.mutation?.onSuccess as + const onSuccess = mockUsePinEvaluation.mock.calls[0]?.[0]?.mutation?.onSuccess as | (() => void) | undefined; onSuccess?.(); - expect(mockGetListExperimentsQueryKey).toHaveBeenCalledWith('ws', { + expect(mockGetListEvaluationsQueryKey).toHaveBeenCalledWith('ws', { filter: { experiment_group_id: 'grp' }, }); expect(invalidateQueries).toHaveBeenCalledTimes(1); diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useExperimentGroupExperiments.ts b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations.ts similarity index 81% rename from web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useExperimentGroupExperiments.ts rename to web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations.ts index fc4f248705..ce58884f83 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useExperimentGroupExperiments.ts +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations.ts @@ -3,25 +3,25 @@ import { useToast } from '@nemo/common/src/providers/toast/useToast'; import { - getListExperimentsQueryKey, - type ListExperimentsQueryError, - useListExperiments, - usePinExperiment, - useUnpinExperiment, + getListEvaluationsQueryKey, + type ListEvaluationsQueryError, + useListEvaluations, + usePinEvaluation, + useUnpinEvaluation, } from '@nemo/sdk/generated/platform/api'; import type { - ExperimentFilter, - ExperimentResponse, - ListExperimentsParams, + EvaluationFilter, + EvaluationResponse, + ListEvaluationsParams, } from '@nemo/sdk/generated/platform/schema'; import { keepPreviousData, useQueryClient } from '@tanstack/react-query'; import { useCallback, useMemo, useRef } from 'react'; /** An API response plus the stable `id` the data view needs. */ -export type ExperimentRow = ExperimentResponse & { id: string }; -export type ListExperimentsSortParam = NonNullable; +export type EvaluationRow = EvaluationResponse & { id: string }; +export type ListEvaluationsSortParam = NonNullable; -const toRows = (experiments: ExperimentResponse[] | undefined): ExperimentRow[] => +const toRows = (experiments: EvaluationResponse[] | undefined): EvaluationRow[] => (experiments ?? []).map((experiment) => ({ ...experiment, id: experiment.id ?? experiment.name ?? '', @@ -34,29 +34,29 @@ const toRows = (experiments: ExperimentResponse[] | undefined): ExperimentRow[] */ const MAX_PINNED_ROWS = 100; -export interface UseExperimentGroupExperimentsParams { +export interface UseExperimentGroupEvaluationsParams { workspace: string; experimentGroupId: string; - filter: Partial | undefined; + filter: Partial | undefined; search: string; page: number; pageSize: number; /** Omit (undefined) when no column sort is active; the API then defaults to -created_at. */ - sort?: ListExperimentsSortParam; + sort?: ListEvaluationsSortParam; } -export interface ExperimentGroupExperiments { +export interface ExperimentGroupEvaluations { /** Pinned rows first (newest-pinned first), then the current page of unpinned rows. */ - rows: ExperimentRow[]; + rows: EvaluationRow[]; /** Pins the row if unpinned, unpins it otherwise, then refetches both lists. */ - togglePin: (row: ExperimentRow) => void; + togglePin: (row: EvaluationRow) => void; /** * Row count that drives pagination: the unpinned total. Pinned rows ride atop every page and are * not paginated, so they're excluded; falls back to the pinned count when nothing is unpinned so * a fully-pinned group still renders one page instead of reading as empty. */ totalCount: number; - error: ListExperimentsQueryError | null; + error: ListEvaluationsQueryError | null; /** True until both queries have loaded once; ignores background refetches (page changes, pins). */ isLoading: boolean; /** True when either query is fetching */ @@ -76,7 +76,7 @@ export interface ExperimentGroupExperiments { * once and repeated atop every page rather than paginated. A pin/unpin persists through the API, * then invalidates both lists so the new state is refetched (no optimistic update). */ -export function useExperimentGroupExperiments({ +export function useExperimentGroupEvaluations({ workspace, experimentGroupId, filter, @@ -84,7 +84,7 @@ export function useExperimentGroupExperiments({ page, pageSize, sort, -}: UseExperimentGroupExperimentsParams): ExperimentGroupExperiments { +}: UseExperimentGroupEvaluationsParams): ExperimentGroupEvaluations { const queryClient = useQueryClient(); const toast = useToast(); @@ -106,13 +106,13 @@ export function useExperimentGroupExperiments({ isLoading: isPinnedLoading, isFetching: isPinnedFetching, error: pinnedError, - } = useListExperiments( + } = useListEvaluations( workspace, { page: 1, page_size: MAX_PINNED_ROWS, sort: '-pinned_at', - filter: { ...baseFilter, is_pinned: true } as ExperimentFilter, + filter: { ...baseFilter, is_pinned: true } as EvaluationFilter, }, listQueryOptions ); @@ -124,13 +124,13 @@ export function useExperimentGroupExperiments({ isSuccess: isUnpinnedSuccess, isPlaceholderData: isUnpinnedPlaceholder, error: unpinnedError, - } = useListExperiments( + } = useListEvaluations( workspace, { page, page_size: pageSize, sort, - filter: { ...baseFilter, is_pinned: false } as ExperimentFilter, + filter: { ...baseFilter, is_pinned: false } as EvaluationFilter, }, listQueryOptions ); @@ -149,26 +149,26 @@ export function useExperimentGroupExperiments({ const invalidateList = useCallback( () => queryClient.invalidateQueries({ - queryKey: getListExperimentsQueryKey(workspace, { + queryKey: getListEvaluationsQueryKey(workspace, { filter: { experiment_group_id: experimentGroupId }, }), }), [queryClient, workspace, experimentGroupId] ); - const { mutate: pinExperiment } = usePinExperiment({ + const { mutate: pinEvaluation } = usePinEvaluation({ mutation: { onSuccess: invalidateList, - onError: () => toast.error('Failed to pin experiment.'), + onError: () => toast.error('Failed to pin evaluation.'), onSettled: (_data, _error, { name }) => { pendingRef.current.delete(name); }, }, }); - const { mutate: unpinExperiment } = useUnpinExperiment({ + const { mutate: unpinEvaluation } = useUnpinEvaluation({ mutation: { onSuccess: invalidateList, - onError: () => toast.error('Failed to unpin experiment.'), + onError: () => toast.error('Failed to unpin evaluation.'), onSettled: (_data, _error, { name }) => { pendingRef.current.delete(name); }, @@ -176,19 +176,19 @@ export function useExperimentGroupExperiments({ }); const togglePin = useCallback( - (row: ExperimentRow) => { + (row: EvaluationRow) => { const { name } = row; if (pendingRef.current.has(name)) return; pendingRef.current.add(name); - if (row.pinned_at != null) unpinExperiment({ workspace, name }); - else pinExperiment({ workspace, name }); + if (row.pinned_at != null) unpinEvaluation({ workspace, name }); + else pinEvaluation({ workspace, name }); }, - [workspace, pinExperiment, unpinExperiment] + [workspace, pinEvaluation, unpinEvaluation] ); // Pinned first, then the current page of unpinned. Drop any unpinned row already shown as pinned — // it can appear in both server lists during the brief window where the two queries refetch out of step. - const rows = useMemo(() => { + const rows = useMemo(() => { const pinnedNames = new Set(pinned.map((row) => row.name)); return [...pinned, ...unpinned.filter((row) => !pinnedNames.has(row.name))]; }, [pinned, unpinned]); diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useSortErrorRecovery.test.ts b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useSortErrorRecovery.test.ts index 638b176515..38896bc116 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useSortErrorRecovery.test.ts +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useSortErrorRecovery.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { ListExperimentsQueryError } from '@nemo/sdk/generated/platform/api'; +import type { ListEvaluationsQueryError } from '@nemo/sdk/generated/platform/api'; import { SORT_ERROR_MESSAGES, type SortingState, @@ -12,8 +12,8 @@ import { renderHook } from '@testing-library/react'; const sort = (id: string, desc = false): SortingState => [{ id, desc }]; // Minimal Axios-shaped error: the hook only reads `response.status`. -const httpError = (status: number): ListExperimentsQueryError => - ({ response: { status } }) as unknown as ListExperimentsQueryError; +const httpError = (status: number): ListEvaluationsQueryError => + ({ response: { status } }) as unknown as ListEvaluationsQueryError; const CREATED_AT = sort('created_at', true); const LATENCY = sort('latency_ms'); diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useSortErrorRecovery.ts b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useSortErrorRecovery.ts index 7bf83018e7..81d135cf0f 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useSortErrorRecovery.ts +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/useSortErrorRecovery.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import type { TanstackTable } from '@nemo/common/src/components/DataView/internal'; -import type { ListExperimentsQueryError } from '@nemo/sdk/generated/platform/api'; +import type { ListEvaluationsQueryError } from '@nemo/sdk/generated/platform/api'; import { useEffect, useRef } from 'react'; /** The table's sorting state — at most one active column for this single-sort table. */ @@ -26,7 +26,7 @@ const isSameSort = (a: SortingState, b: SortingState): boolean => a[0]?.id === b[0]?.id && (a[0]?.desc ?? false) === (b[0]?.desc ?? false); export interface UseSortErrorRecoveryParams { - error: ListExperimentsQueryError | null; + error: ListEvaluationsQueryError | null; isSuccess: boolean; sortingState: SortingState; setSorting: (next: SortingState) => void; @@ -60,7 +60,7 @@ export const useSortErrorRecovery = ({ const isRecoverableSortError = message != null && !isSameSort(sortingState, lastGoodSortRef.current); - const handledErrorRef = useRef(null); + const handledErrorRef = useRef(null); useEffect(() => { if (!error) { handledErrorRef.current = null; diff --git a/web/packages/studio/src/constants/links.ts b/web/packages/studio/src/constants/links.ts index 043b1b3ae2..195849c777 100644 --- a/web/packages/studio/src/constants/links.ts +++ b/web/packages/studio/src/constants/links.ts @@ -52,5 +52,5 @@ export const LINK_DOCS_JOBS = `${DOCS_BASE_URL}studio#jobs`; // Secrets documentation links export const LINK_DOCS_SECRETS = `${DOCS_BASE_URL}get-started/core-concepts/manage-secrets`; -// Experiments +// Evaluations export const LINK_DOCS_EXPERIMENTS_CLI = `${DOCS_BASE_URL}reference/cli-reference`; diff --git a/web/packages/studio/src/constants/routes.ts b/web/packages/studio/src/constants/routes.ts index 1371a62d74..530be044c8 100644 --- a/web/packages/studio/src/constants/routes.ts +++ b/web/packages/studio/src/constants/routes.ts @@ -38,7 +38,7 @@ export const ROUTE_PARAMS = { /** Benchmark entity name segment under evaluation/benchmarks/:name */ benchmarkName: 'benchmarkName', experimentGroupName: 'experimentGroupName', - experimentName: 'experimentName', + evaluationName: 'evaluationName', guardrailConfigName: 'guardrailConfigName', } as const; @@ -74,8 +74,8 @@ export const ROUTES = { /** Empty landing page for the EXPERIMENT feature (gated by VITE_FF_EXPERIMENT). */ experiment: `/workspaces/:${P.workspace}/experiment`, experimentGroupDetail: `/workspaces/:${P.workspace}/experiment/:${P.experimentGroupName}`, - experimentDetail: `/workspaces/:${P.workspace}/experiment/:${P.experimentGroupName}/:${P.experimentName}`, - experimentTraceDetail: `/workspaces/:${P.workspace}/experiment/:${P.experimentGroupName}/:${P.experimentName}/traces/:${P.traceId}`, + evaluationDetail: `/workspaces/:${P.workspace}/experiment/:${P.experimentGroupName}/:${P.evaluationName}`, + evaluationTraceDetail: `/workspaces/:${P.workspace}/experiment/:${P.experimentGroupName}/:${P.evaluationName}/traces/:${P.traceId}`, customizationJobList: `/workspaces/:${P.workspace}/customizations`, customizationJobDetails: `/workspaces/:${P.workspace}/customizations/:${P.customizationJobName}`, filesets: `/workspaces/:${P.workspace}/filesets`, diff --git a/web/packages/studio/src/mocks/intake/telemetry.ts b/web/packages/studio/src/mocks/intake/telemetry.ts index 55baf26413..fe9dc1be8a 100644 --- a/web/packages/studio/src/mocks/intake/telemetry.ts +++ b/web/packages/studio/src/mocks/intake/telemetry.ts @@ -32,6 +32,10 @@ const trace1: Trace = { cost_usd: 0.0032, span_count: 4, error_count: 0, + evaluation_context: { + evaluation_id: 'support-policy-smoke', + test_case_id: 'case-0042', + }, experiment_context: { experiment_id: 'support-policy-smoke', test_case_id: 'case-0042', diff --git a/web/packages/studio/src/routes/ExperimentDetailRoute/ExperimentDetailMetrics.tsx b/web/packages/studio/src/routes/EvaluationDetailRoute/EvaluationDetailMetrics.tsx similarity index 91% rename from web/packages/studio/src/routes/ExperimentDetailRoute/ExperimentDetailMetrics.tsx rename to web/packages/studio/src/routes/EvaluationDetailRoute/EvaluationDetailMetrics.tsx index e699b3cca4..6789c960aa 100644 --- a/web/packages/studio/src/routes/ExperimentDetailRoute/ExperimentDetailMetrics.tsx +++ b/web/packages/studio/src/routes/EvaluationDetailRoute/EvaluationDetailMetrics.tsx @@ -4,19 +4,19 @@ import { KVPair } from '@nemo/common/src/components/KVPair'; import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; import { formatDurationMs } from '@nemo/common/src/utils/date'; -import { useGetExperiment } from '@nemo/sdk/generated/platform/api'; +import { useGetEvaluation } from '@nemo/sdk/generated/platform/api'; import { Divider, Flex, Text, Tooltip } from '@nvidia/foundations-react-core'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { tooltipClassName } from '@studio/styles/common'; import { type FC, type ReactNode } from 'react'; -interface ExperimentDetailMetricsProps { - experimentName: string; +interface EvaluationDetailMetricsProps { + evaluationName: string; } -export const ExperimentDetailMetrics: FC = ({ experimentName }) => { +export const EvaluationDetailMetrics: FC = ({ evaluationName }) => { const workspace = useWorkspaceFromPath(); - const { data: experiment, isLoading } = useGetExperiment(workspace, experimentName); + const { data: experiment, isLoading } = useGetEvaluation(workspace, evaluationName); const avgCost = experiment?.cost_usd?.mean != null ? `$${experiment.cost_usd.mean.toFixed(3)}` : undefined; diff --git a/web/packages/studio/src/routes/ExperimentDetailRoute/index.tsx b/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx similarity index 70% rename from web/packages/studio/src/routes/ExperimentDetailRoute/index.tsx rename to web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx index bcc4dbef15..2df2ca6d3b 100644 --- a/web/packages/studio/src/routes/ExperimentDetailRoute/index.tsx +++ b/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx @@ -1,25 +1,25 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { useGetExperiment } from '@nemo/sdk/generated/platform/api'; +import { useGetEvaluation } from '@nemo/sdk/generated/platform/api'; import { Badge, PageHeader, Stack, Text } from '@nvidia/foundations-react-core'; import { AccessibleTitle } from '@studio/components/AccessibleTitle'; -import { ExperimentSessionsDataView } from '@studio/components/dataViews/ExperimentSessionsDataView'; +import { EvaluationSessionsDataView } from '@studio/components/dataViews/EvaluationSessionsDataView'; import { ROUTE_PARAMS } from '@studio/constants/routes'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; -import { ExperimentDetailMetrics } from '@studio/routes/ExperimentDetailRoute/ExperimentDetailMetrics'; +import { EvaluationDetailMetrics } from '@studio/routes/EvaluationDetailRoute/EvaluationDetailMetrics'; import { getExperimentGroupDetailRoute, getExperimentRoute } from '@studio/routes/utils'; import { useRequiredPathParams } from '@studio/util/hooks/useRequiredPathParams'; import { type FC } from 'react'; -export const ExperimentDetailRoute: FC = () => { +export const EvaluationDetailRoute: FC = () => { const workspace = useWorkspaceFromPath(); - const { experimentGroupName, experimentName } = useRequiredPathParams([ + const { experimentGroupName, evaluationName } = useRequiredPathParams([ ROUTE_PARAMS.experimentGroupName, - ROUTE_PARAMS.experimentName, + ROUTE_PARAMS.evaluationName, ]); - const { data: experiment } = useGetExperiment(workspace, experimentName); + const { data: experiment } = useGetEvaluation(workspace, evaluationName); useBreadcrumbs({ items: [ @@ -28,19 +28,19 @@ export const ExperimentDetailRoute: FC = () => { href: getExperimentGroupDetailRoute(workspace, experimentGroupName), slotLabel: experimentGroupName, }, - { slotLabel: experimentName }, + { slotLabel: evaluationName }, ], }); return ( - + - +
Test cases @@ -50,8 +50,8 @@ export const ExperimentDetailRoute: FC = () => { )}
-
diff --git a/web/packages/studio/src/routes/ExperimentTraceDetailRoute/index.test.tsx b/web/packages/studio/src/routes/EvaluationTraceDetailRoute/index.test.tsx similarity index 74% rename from web/packages/studio/src/routes/ExperimentTraceDetailRoute/index.test.tsx rename to web/packages/studio/src/routes/EvaluationTraceDetailRoute/index.test.tsx index 9eece3fd96..beb59cdfd2 100644 --- a/web/packages/studio/src/routes/ExperimentTraceDetailRoute/index.test.tsx +++ b/web/packages/studio/src/routes/EvaluationTraceDetailRoute/index.test.tsx @@ -1,8 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { ExperimentTraceDetailRoute } from '@studio/routes/ExperimentTraceDetailRoute'; -import { getExperimentTraceDetailRoute } from '@studio/routes/utils'; +import { EvaluationTraceDetailRoute } from '@studio/routes/EvaluationTraceDetailRoute'; +import { getEvaluationTraceDetailRoute } from '@studio/routes/utils'; import { renderRoute, screen } from '@studio/tests/util/render'; const WORKSPACE = 'default'; @@ -13,25 +13,25 @@ const TRACE_ID = 'trace-agent-run-001'; const renderTraceDetail = () => renderRoute(undefined, { - history: getExperimentTraceDetailRoute(WORKSPACE, EXPERIMENT_GROUP, EXPERIMENT_NAME, TRACE_ID), + history: getEvaluationTraceDetailRoute(WORKSPACE, EXPERIMENT_GROUP, EXPERIMENT_NAME, TRACE_ID), routes: [ { - path: '/workspaces/:workspace/experiment/:experimentGroupName/:experimentName/traces/:traceId', - element: , + path: '/workspaces/:workspace/experiment/:experimentGroupName/:evaluationName/traces/:traceId', + element: , }, ], }); -describe('ExperimentTraceDetailRoute', () => { +describe('EvaluationTraceDetailRoute', () => { it('renders the trace detail content', async () => { renderTraceDetail(); expect(await screen.findByText('Trace Answer customer policy question')).toBeInTheDocument(); }); - it('renders the experiment context panel', async () => { + it('renders the evaluation context panel', async () => { renderTraceDetail(); await screen.findByText('Trace Answer customer policy question'); - expect(screen.getByText('Experiment Context')).toBeInTheDocument(); + expect(screen.getByText('Evaluation Context')).toBeInTheDocument(); }); it('does not render an Intake link in the page content', async () => { diff --git a/web/packages/studio/src/routes/ExperimentTraceDetailRoute/index.tsx b/web/packages/studio/src/routes/EvaluationTraceDetailRoute/index.tsx similarity index 77% rename from web/packages/studio/src/routes/ExperimentTraceDetailRoute/index.tsx rename to web/packages/studio/src/routes/EvaluationTraceDetailRoute/index.tsx index 4cab3bcd30..1d87d5dbfd 100644 --- a/web/packages/studio/src/routes/ExperimentTraceDetailRoute/index.tsx +++ b/web/packages/studio/src/routes/EvaluationTraceDetailRoute/index.tsx @@ -6,19 +6,19 @@ import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { type BreadcrumbsItemProps } from '@studio/providers/breadcrumbs/useBreadcrumbs'; import { IntakeTraceDetailContent } from '@studio/routes/IntakeTraceDetailRoute'; import { - getExperimentDetailRoute, + getEvaluationDetailRoute, getExperimentGroupDetailRoute, getExperimentRoute, } from '@studio/routes/utils'; import { useRequiredPathParams } from '@studio/util/hooks/useRequiredPathParams'; import { type FC, useMemo } from 'react'; -export const ExperimentTraceDetailRoute: FC = () => { +export const EvaluationTraceDetailRoute: FC = () => { const workspace = useWorkspaceFromPath(); - const { traceId, experimentGroupName, experimentName } = useRequiredPathParams([ + const { traceId, experimentGroupName, evaluationName } = useRequiredPathParams([ ROUTE_PARAMS.traceId, ROUTE_PARAMS.experimentGroupName, - ROUTE_PARAMS.experimentName, + ROUTE_PARAMS.evaluationName, ]); const parentBreadcrumbs = useMemo( @@ -29,11 +29,11 @@ export const ExperimentTraceDetailRoute: FC = () => { href: getExperimentGroupDetailRoute(workspace, experimentGroupName), }, { - slotLabel: experimentName, - href: getExperimentDetailRoute(workspace, experimentGroupName, experimentName), + slotLabel: evaluationName, + href: getEvaluationDetailRoute(workspace, experimentGroupName, evaluationName), }, ], - [workspace, experimentGroupName, experimentName] + [workspace, experimentGroupName, evaluationName] ); return ; diff --git a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx index f69d1cb59e..a5d69429e8 100644 --- a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx +++ b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx @@ -58,10 +58,10 @@ export const ExperimentGroupDetailRoute: FC = () => {
- Experiments - {group?.experiment_count !== undefined && ( + Evaluations + {group?.evaluation_count !== undefined && ( - {group.experiment_count} + {group.evaluation_count} )}
diff --git a/web/packages/studio/src/routes/ExperimentRoute/ExperimentGroupCard.tsx b/web/packages/studio/src/routes/ExperimentRoute/ExperimentGroupCard.tsx index 040beb1011..ee1b3051be 100644 --- a/web/packages/studio/src/routes/ExperimentRoute/ExperimentGroupCard.tsx +++ b/web/packages/studio/src/routes/ExperimentRoute/ExperimentGroupCard.tsx @@ -48,7 +48,7 @@ export const ExperimentGroupCard: FC = ({ group, works {/* Stats */}
- +
); diff --git a/web/packages/studio/src/routes/IntakeTraceDetailRoute/index.test.tsx b/web/packages/studio/src/routes/IntakeTraceDetailRoute/index.test.tsx index 61d719743b..a430d1ad45 100644 --- a/web/packages/studio/src/routes/IntakeTraceDetailRoute/index.test.tsx +++ b/web/packages/studio/src/routes/IntakeTraceDetailRoute/index.test.tsx @@ -171,21 +171,21 @@ describe('IntakeTraceDetailRoute', () => { expect(detailSpanIds).not.toContain('span-root-001'); }); - it('renders experiment context only when present and filters spans to the trace', async () => { + it('renders evaluation context only when present and filters spans to the trace', async () => { renderTraceDetail('trace-agent-run-001'); expect(await screen.findByText('Trace Answer customer policy question')).toBeInTheDocument(); - expect(screen.getByText('Experiment Context')).toBeInTheDocument(); + expect(screen.getByText('Evaluation Context')).toBeInTheDocument(); expect(await screen.findByText('Generate final response')).toBeInTheDocument(); expect(screen.queryByText('Retrieve deployment troubleshooting steps')).not.toBeInTheDocument(); }); - it('omits experiment context when the trace has none', async () => { + it('omits evaluation context when the trace has none', async () => { renderTraceDetail('trace-agent-run-002'); expect( await screen.findByText('Trace Retrieve deployment troubleshooting steps') ).toBeInTheDocument(); - expect(screen.queryByText('Experiment Context')).not.toBeInTheDocument(); + expect(screen.queryByText('Evaluation Context')).not.toBeInTheDocument(); }); }); diff --git a/web/packages/studio/src/routes/WorkspaceLayout/WorkspaceSideNav.tsx b/web/packages/studio/src/routes/WorkspaceLayout/WorkspaceSideNav.tsx index e0259cf788..96530be3f6 100644 --- a/web/packages/studio/src/routes/WorkspaceLayout/WorkspaceSideNav.tsx +++ b/web/packages/studio/src/routes/WorkspaceLayout/WorkspaceSideNav.tsx @@ -129,7 +129,7 @@ export const WorkspaceSideNav = ({ collapsed }: { collapsed?: boolean }) => { { id: 'experiment', slotIcon: , - slotLabel: 'Experiment', + slotLabel: 'Experiments', href: getExperimentRoute(workspace), }, ] diff --git a/web/packages/studio/src/routes/groups/experimentRoutes.tsx b/web/packages/studio/src/routes/groups/experimentRoutes.tsx index 86dff0ae02..7c4ac82f42 100644 --- a/web/packages/studio/src/routes/groups/experimentRoutes.tsx +++ b/web/packages/studio/src/routes/groups/experimentRoutes.tsx @@ -17,14 +17,14 @@ const ExperimentGroupDetailRoute = lazy(() => default: module.ExperimentGroupDetailRoute, })) ); -const ExperimentDetailRoute = lazy(() => - import('@studio/routes/ExperimentDetailRoute').then((module) => ({ - default: module.ExperimentDetailRoute, +const EvaluationDetailRoute = lazy(() => + import('@studio/routes/EvaluationDetailRoute').then((module) => ({ + default: module.EvaluationDetailRoute, })) ); -const ExperimentTraceDetailRoute = lazy(() => - import('@studio/routes/ExperimentTraceDetailRoute').then((module) => ({ - default: module.ExperimentTraceDetailRoute, +const EvaluationTraceDetailRoute = lazy(() => + import('@studio/routes/EvaluationTraceDetailRoute').then((module) => ({ + default: module.EvaluationTraceDetailRoute, })) ); @@ -32,7 +32,7 @@ export const experimentRoutes: RouteObject[] = gateExperimentRoutes([ { path: ROUTES.workspace.experiment, element: , - errorElement: , + errorElement: , }, { path: ROUTES.workspace.experimentGroupDetail, @@ -40,13 +40,13 @@ export const experimentRoutes: RouteObject[] = gateExperimentRoutes([ errorElement: , }, { - path: ROUTES.workspace.experimentDetail, - element: , - errorElement: , + path: ROUTES.workspace.evaluationDetail, + element: , + errorElement: , }, { - path: ROUTES.workspace.experimentTraceDetail, - element: , + path: ROUTES.workspace.evaluationTraceDetail, + element: , errorElement: , }, ]); diff --git a/web/packages/studio/src/routes/utils.ts b/web/packages/studio/src/routes/utils.ts index a458949a30..a734049c64 100644 --- a/web/packages/studio/src/routes/utils.ts +++ b/web/packages/studio/src/routes/utils.ts @@ -313,28 +313,28 @@ export const getExperimentGroupDetailRoute = (workspace: string, experimentGroup }); }; -export const getExperimentDetailRoute = ( +export const getEvaluationDetailRoute = ( workspace: string, experimentGroupName: string, - experimentName: string + evaluationName: string ) => { - return generatePath(ROUTES.workspace.experimentDetail, { + return generatePath(ROUTES.workspace.evaluationDetail, { workspace, experimentGroupName: encodeURIComponent(experimentGroupName), - experimentName: encodeURIComponent(experimentName), + evaluationName: encodeURIComponent(evaluationName), }); }; -export const getExperimentTraceDetailRoute = ( +export const getEvaluationTraceDetailRoute = ( workspace: string, experimentGroupName: string, - experimentName: string, + evaluationName: string, traceId: string ): string => { - return generatePath(ROUTES.workspace.experimentTraceDetail, { + return generatePath(ROUTES.workspace.evaluationTraceDetail, { workspace, experimentGroupName: encodeURIComponent(experimentGroupName), - experimentName: encodeURIComponent(experimentName), + evaluationName: encodeURIComponent(evaluationName), traceId, }); }; diff --git a/web/packages/studio/src/tests/title-change.test.tsx b/web/packages/studio/src/tests/title-change.test.tsx index 013e9192ca..1b8f61709b 100644 --- a/web/packages/studio/src/tests/title-change.test.tsx +++ b/web/packages/studio/src/tests/title-change.test.tsx @@ -34,7 +34,7 @@ const pathParams = { [RP.jobName]: 'test-job', [RP.benchmarkName]: 'test-benchmark', [RP.experimentGroupName]: 'test-experiment-group', - [RP.experimentName]: 'test-experiment', + [RP.evaluationName]: 'test-experiment', [RP.guardrailConfigName]: 'test-guardrail-config', };