diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index f16a77fcae..a07756d247 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -11285,6 +11285,11 @@ components: Accepts any field the evaluations list `sort` param does; clients apply it as the list `sort` param.' default: -created_at + pareto: + allOf: + - $ref: '#/components/schemas/ParetoConfig' + description: Default X/Y metrics for the group's Pareto view. Omit to preserve + the existing value on update; on create, defaults to cost vs. latency. additionalProperties: false type: object required: @@ -11319,6 +11324,8 @@ components: default_sort: type: string title: Default Sort + pareto: + $ref: '#/components/schemas/ParetoConfig' created_at: title: Created At type: string @@ -15399,6 +15406,25 @@ components: - recipe title: PangeaRailOptions description: Configuration data for the Pangea AI Guard API + ParetoConfig: + properties: + x_metric: + type: string + title: X Metric + description: Metric plotted on the Pareto X axis. + default: cost_usd + y_metric: + type: string + title: Y Metric + description: Metric plotted on the Pareto Y axis. + default: latency_ms + type: object + title: ParetoConfig + description: "Default X/Y metrics for a group's cost-vs-accuracy Pareto view.\n\ + \nMetric ids use the same vocabulary as the evaluations list sort/filter fields\ + \ \u2014 ``cost_usd``,\n``latency_ms``, or ``evaluators.``. Defaults\ + \ to cost (x) vs latency (y): both exist for\nevery group, so the chart always\ + \ has something to render before anyone customizes it." PatronusEvaluateApiParams: properties: success_strategy: diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index f16a77fcae..a07756d247 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -11285,6 +11285,11 @@ components: Accepts any field the evaluations list `sort` param does; clients apply it as the list `sort` param.' default: -created_at + pareto: + allOf: + - $ref: '#/components/schemas/ParetoConfig' + description: Default X/Y metrics for the group's Pareto view. Omit to preserve + the existing value on update; on create, defaults to cost vs. latency. additionalProperties: false type: object required: @@ -11319,6 +11324,8 @@ components: default_sort: type: string title: Default Sort + pareto: + $ref: '#/components/schemas/ParetoConfig' created_at: title: Created At type: string @@ -15399,6 +15406,25 @@ components: - recipe title: PangeaRailOptions description: Configuration data for the Pangea AI Guard API + ParetoConfig: + properties: + x_metric: + type: string + title: X Metric + description: Metric plotted on the Pareto X axis. + default: cost_usd + y_metric: + type: string + title: Y Metric + description: Metric plotted on the Pareto Y axis. + default: latency_ms + type: object + title: ParetoConfig + description: "Default X/Y metrics for a group's cost-vs-accuracy Pareto view.\n\ + \nMetric ids use the same vocabulary as the evaluations list sort/filter fields\ + \ \u2014 ``cost_usd``,\n``latency_ms``, or ``evaluators.``. Defaults\ + \ to cost (x) vs latency (y): both exist for\nevery group, so the chart always\ + \ has something to render before anyone customizes it." PatronusEvaluateApiParams: properties: success_strategy: diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index f16a77fcae..a07756d247 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -11285,6 +11285,11 @@ components: Accepts any field the evaluations list `sort` param does; clients apply it as the list `sort` param.' default: -created_at + pareto: + allOf: + - $ref: '#/components/schemas/ParetoConfig' + description: Default X/Y metrics for the group's Pareto view. Omit to preserve + the existing value on update; on create, defaults to cost vs. latency. additionalProperties: false type: object required: @@ -11319,6 +11324,8 @@ components: default_sort: type: string title: Default Sort + pareto: + $ref: '#/components/schemas/ParetoConfig' created_at: title: Created At type: string @@ -15399,6 +15406,25 @@ components: - recipe title: PangeaRailOptions description: Configuration data for the Pangea AI Guard API + ParetoConfig: + properties: + x_metric: + type: string + title: X Metric + description: Metric plotted on the Pareto X axis. + default: cost_usd + y_metric: + type: string + title: Y Metric + description: Metric plotted on the Pareto Y axis. + default: latency_ms + type: object + title: ParetoConfig + description: "Default X/Y metrics for a group's cost-vs-accuracy Pareto view.\n\ + \nMetric ids use the same vocabulary as the evaluations list sort/filter fields\ + \ \u2014 ``cost_usd``,\n``latency_ms``, or ``evaluators.``. Defaults\ + \ to cost (x) vs latency (y): both exist for\nevery group, so the chart always\ + \ has something to render before anyone customizes it." PatronusEvaluateApiParams: properties: success_strategy: diff --git a/plugins/nemo-customizer/openapi/openapi.yaml b/plugins/nemo-customizer/openapi/openapi.yaml index 2b7724e0aa..d852b91bc8 100644 --- a/plugins/nemo-customizer/openapi/openapi.yaml +++ b/plugins/nemo-customizer/openapi/openapi.yaml @@ -392,12 +392,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs: + /apis/customization/v2/workspaces/{workspace}/rl/jobs: post: tags: - - Unsloth Jobs + - Rl Jobs summary: Create Job - operationId: create_job_apis_customization_v2_workspaces__workspace__unsloth_jobs_post + operationId: create_job_apis_customization_v2_workspaces__workspace__rl_jobs_post parameters: - name: workspace in: path @@ -410,14 +410,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/UnslothJobsJobRequest' + $ref: '#/components/schemas/RlJobsJobRequest' responses: '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/UnslothJobsJob' + $ref: '#/components/schemas/RlJobsJob' '422': description: Validation Error content: @@ -426,9 +426,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' get: tags: - - Unsloth Jobs + - Rl Jobs summary: List Jobs - operationId: list_jobs_apis_customization_v2_workspaces__workspace__unsloth_jobs_get + operationId: list_jobs_apis_customization_v2_workspaces__workspace__rl_jobs_get parameters: - name: workspace in: path @@ -461,7 +461,7 @@ paths: required: false schema: allOf: - - $ref: '#/components/schemas/UnslothJobsJobsSortField' + - $ref: '#/components/schemas/RlJobsJobsSortField' description: The field to sort by. To sort in decreasing order, use `-` in front of the field name. default: -created_at @@ -473,7 +473,7 @@ paths: required: false explode: true schema: - $ref: '#/components/schemas/UnslothJobsJobsListFilter' + $ref: '#/components/schemas/RlJobsJobsListFilter' description: Filter jobs on various criteria. responses: '200': @@ -481,19 +481,19 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/UnslothJobsJobsPage' + $ref: '#/components/schemas/RlJobsJobsPage' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{job}/results/{name}: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{job}/results/{name}: get: tags: - - Unsloth Jobs + - Rl Jobs summary: Get Job Result - operationId: get_job_result_apis_customization_v2_workspaces__workspace__unsloth_jobs__job__results__name__get + operationId: get_job_result_apis_customization_v2_workspaces__workspace__rl_jobs__job__results__name__get parameters: - name: workspace in: path @@ -526,12 +526,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{job}/results/{name}/download: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{job}/results/{name}/download: get: tags: - - Unsloth Jobs + - Rl Jobs summary: Download Job Result - operationId: download_job_result_apis_customization_v2_workspaces__workspace__unsloth_jobs__job__results__name__download_get + operationId: download_job_result_apis_customization_v2_workspaces__workspace__rl_jobs__job__results__name__download_get parameters: - name: workspace in: path @@ -567,12 +567,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{name}: get: tags: - - Unsloth Jobs + - Rl Jobs summary: Get Job - operationId: get_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__get + operationId: get_job_apis_customization_v2_workspaces__workspace__rl_jobs__name__get parameters: - name: workspace in: path @@ -592,7 +592,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/UnslothJobsJob' + $ref: '#/components/schemas/RlJobsJob' '422': description: Validation Error content: @@ -601,9 +601,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' delete: tags: - - Unsloth Jobs + - Rl Jobs summary: Delete Job - operationId: delete_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__delete + operationId: delete_job_apis_customization_v2_workspaces__workspace__rl_jobs__name__delete parameters: - name: workspace in: path @@ -626,12 +626,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/cancel: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{name}/cancel: post: tags: - - Unsloth Jobs + - Rl Jobs summary: Cancel Job - operationId: cancel_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__cancel_post + operationId: cancel_job_apis_customization_v2_workspaces__workspace__rl_jobs__name__cancel_post parameters: - name: workspace in: path @@ -651,19 +651,19 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/UnslothJobsJob' + $ref: '#/components/schemas/RlJobsJob' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/logs: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{name}/logs: get: tags: - - Unsloth Jobs + - Rl Jobs summary: Get Job Logs - operationId: get_job_logs_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__logs_get + operationId: get_job_logs_apis_customization_v2_workspaces__workspace__rl_jobs__name__logs_get parameters: - name: workspace in: path @@ -702,12 +702,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/results: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{name}/results: get: tags: - - Unsloth Jobs + - Rl Jobs summary: List Job Results - operationId: list_job_results_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__results_get + operationId: list_job_results_apis_customization_v2_workspaces__workspace__rl_jobs__name__results_get parameters: - name: workspace in: path @@ -734,12 +734,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/status: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{name}/status: get: tags: - - Unsloth Jobs + - Rl Jobs summary: Get Job Status - operationId: get_job_status_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__status_get + operationId: get_job_status_apis_customization_v2_workspaces__workspace__rl_jobs__name__status_get parameters: - name: workspace in: path @@ -766,89 +766,463 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' -components: - schemas: - AutomodelBatchSpec: - properties: - global_batch_size: - type: integer - exclusiveMinimum: 0.0 - title: Global Batch Size - default: 8 - micro_batch_size: + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs: + post: + tags: + - Unsloth Jobs + summary: Create Job + operationId: create_job_apis_customization_v2_workspaces__workspace__unsloth_jobs_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/UnslothJobsJobRequest' + responses: + '201': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/UnslothJobsJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + get: + tags: + - Unsloth Jobs + summary: List Jobs + operationId: list_jobs_apis_customization_v2_workspaces__workspace__unsloth_jobs_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: type: integer - exclusiveMinimum: 0.0 - title: Micro Batch Size + exclusiveMinimum: 0 + description: Page number. default: 1 - sequence_packing: - type: boolean - title: Sequence Packing - default: false - sequence_packing_max_samples: + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: type: integer - exclusiveMinimum: 0.0 - title: Sequence Packing Max Samples - description: Samples analyzed to estimate the optimal pack size when packing - is enabled. - default: 1000 - additionalProperties: false - type: object - title: AutomodelBatchSpec - AutomodelDatasetSpec: - properties: - training: + exclusiveMinimum: 0 + description: Page size. + default: 10 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + allOf: + - $ref: '#/components/schemas/UnslothJobsJobsSortField' + description: The field to sort by. To sort in decreasing order, use `-` + in front of the field name. + default: -created_at + description: The field to sort by. To sort in decreasing order, use `-` in + front of the field name. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/UnslothJobsJobsListFilter' + description: Filter jobs on various criteria. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/UnslothJobsJobsPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{job}/results/{name}: + get: + tags: + - Unsloth Jobs + summary: Get Job Result + operationId: get_job_result_apis_customization_v2_workspaces__workspace__unsloth_jobs__job__results__name__get + parameters: + - name: workspace + in: path + required: true + schema: type: string - title: Training - description: Training fileset as 'name' or 'workspace/name'. - validation: - title: Validation + title: Workspace + - name: job + in: path + required: true + schema: type: string - prompt_template: - title: Prompt Template + title: Job + - name: name + in: path + required: true + schema: type: string - additionalProperties: false - type: object - required: - - training - title: AutomodelDatasetSpec - AutomodelJobInput: - properties: - name: title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobResultResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{job}/results/{name}/download: + get: + tags: + - Unsloth Jobs + summary: Download Job Result + operationId: download_job_result_apis_customization_v2_workspaces__workspace__unsloth_jobs__job__results__name__download_get + parameters: + - name: workspace + in: path + required: true + schema: type: string - model: + title: Workspace + - name: job + in: path + required: true + schema: type: string - title: Model - dataset: - $ref: '#/components/schemas/AutomodelDatasetSpec' - training: - $ref: '#/components/schemas/AutomodelTrainingSpec' - schedule: - $ref: '#/components/schemas/AutomodelScheduleSpec' - batch: - $ref: '#/components/schemas/AutomodelBatchSpec' - optimizer: - $ref: '#/components/schemas/AutomodelOptimizerSpec' - parallelism: - $ref: '#/components/schemas/AutomodelParallelismSpec' - output: - $ref: '#/components/schemas/AutomodelOutputRequest' - integrations: - $ref: '#/components/schemas/IntegrationsSpecInput' - additionalProperties: false - type: object - required: - - model - - dataset - - training - title: AutomodelJobInput - description: POST body / CLI JSON. - AutomodelJobOutput: - properties: - name: - title: Name + title: Job + - name: name + in: path + required: true + schema: type: string - model: + title: Name + responses: + '200': + description: Successful Response + content: + application/octet-stream: + schema: + type: string + format: binary + '404': + description: Not Found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}: + get: + tags: + - Unsloth Jobs + summary: Get Job + operationId: get_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/UnslothJobsJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Unsloth Jobs + summary: Delete Job + operationId: delete_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '204': + description: Successful Response + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/cancel: + post: + tags: + - Unsloth Jobs + summary: Cancel Job + operationId: cancel_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__cancel_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/UnslothJobsJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/logs: + get: + tags: + - Unsloth Jobs + summary: Get Job Logs + operationId: get_job_logs_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__logs_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + - name: limit + in: query + required: false + schema: + title: Limit + type: integer + - name: page_cursor + in: query + required: false + schema: + title: Page Cursor + type: string + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobLogPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/results: + get: + tags: + - Unsloth Jobs + summary: List Job Results + operationId: list_job_results_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__results_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobListResultResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/status: + get: + tags: + - Unsloth Jobs + summary: Get Job Status + operationId: get_job_status_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__status_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobStatusResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' +components: + schemas: + AutomodelBatchSpec: + properties: + global_batch_size: + type: integer + exclusiveMinimum: 0.0 + title: Global Batch Size + default: 8 + micro_batch_size: + type: integer + exclusiveMinimum: 0.0 + title: Micro Batch Size + default: 1 + sequence_packing: + type: boolean + title: Sequence Packing + default: false + sequence_packing_max_samples: + type: integer + exclusiveMinimum: 0.0 + title: Sequence Packing Max Samples + description: Samples analyzed to estimate the optimal pack size when packing + is enabled. + default: 1000 + additionalProperties: false + type: object + title: AutomodelBatchSpec + AutomodelDatasetSpec: + properties: + training: + type: string + title: Training + description: Training fileset as 'name' or 'workspace/name'. + validation: + title: Validation + type: string + prompt_template: + title: Prompt Template + type: string + additionalProperties: false + type: object + required: + - training + title: AutomodelDatasetSpec + AutomodelJobInput: + properties: + name: + title: Name + type: string + model: + type: string + title: Model + dataset: + $ref: '#/components/schemas/AutomodelDatasetSpec' + training: + $ref: '#/components/schemas/AutomodelTrainingSpec' + schedule: + $ref: '#/components/schemas/AutomodelScheduleSpec' + batch: + $ref: '#/components/schemas/AutomodelBatchSpec' + optimizer: + $ref: '#/components/schemas/AutomodelOptimizerSpec' + parallelism: + $ref: '#/components/schemas/AutomodelParallelismSpec' + output: + $ref: '#/components/schemas/AutomodelOutputRequest' + integrations: + $ref: '#/components/schemas/IntegrationsSpecInput' + additionalProperties: false + type: object + required: + - model + - dataset + - training + title: AutomodelJobInput + description: POST body / CLI JSON. + AutomodelJobOutput: + properties: + name: + title: Name + type: string + model: type: string title: Model dataset: @@ -1401,6 +1775,23 @@ components: To enable MLflow, provide a non-null ``mlflow`` object on :class:`IntegrationsSpec`.' + OptimizerType: + type: string + enum: + - adamw_with_cosine_annealing + - adam_with_cosine_annealing + - adamw_with_flat_lr + - adam_with_flat_lr + title: OptimizerType + description: Optimizer and scheduler combination types. + OutputNameType: + type: string + enum: + - adapter + - model + title: OutputNameType + description: "Output artifact type \u2014 adapter (LoRA only) or model (merged\ + \ / full)." PaginationData: properties: page: @@ -1425,254 +1816,701 @@ components: description: The total number of results. type: object required: - - page - - page_size - - current_page_size - - total_pages - - total_results - title: PaginationData - PlatformJobListResultResponse: + - page + - page_size + - current_page_size + - total_pages + - total_results + title: PaginationData + PlatformJobListResultResponse: + properties: + data: + items: + $ref: '#/components/schemas/PlatformJobResultResponse' + type: array + title: Data + type: object + required: + - data + title: PlatformJobListResultResponse + PlatformJobLog: + properties: + timestamp: + type: string + format: date-time + title: Timestamp + job: + type: string + title: Job + job_step: + type: string + title: Job Step + job_task: + type: string + title: Job Task + message: + type: string + title: Message + type: object + required: + - timestamp + - job + - job_step + - job_task + - message + title: PlatformJobLog + PlatformJobLogPage: + properties: + data: + items: + $ref: '#/components/schemas/PlatformJobLog' + type: array + title: Data + total: + type: integer + title: Total + next_page: + title: Next Page + type: string + prev_page: + title: Prev Page + type: string + type: object + required: + - data + - total + - next_page + - prev_page + title: PlatformJobLogPage + PlatformJobResultResponse: + properties: + name: + type: string + title: Name + job: + type: string + title: Job + workspace: + type: string + title: Workspace + project: + title: Project + type: string + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + artifact_url: + type: string + title: Artifact Url + artifact_storage_type: + $ref: '#/components/schemas/FileStorageType' + download_url: + title: Download Url + type: string + type: object + required: + - name + - job + - workspace + - artifact_url + - artifact_storage_type + title: PlatformJobResultResponse + PlatformJobStatus: + type: string + enum: + - created + - pending + - active + - cancelled + - cancelling + - error + - completed + - paused + - pausing + - resuming + title: PlatformJobStatus + description: 'Enumeration of possible job statuses. + + + This enum represents the various states a job can be in during its lifecycle, + + from creation to a terminal state.' + PlatformJobStatusResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + steps: + items: + $ref: '#/components/schemas/PlatformJobStepStatusResponse' + type: array + title: Steps + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + type: object + required: + - id + - name + - status + - status_details + - error_details + - steps + - created_at + - updated_at + title: PlatformJobStatusResponse + PlatformJobStepStatusResponse: properties: - data: + id: + type: string + title: Id + name: + type: string + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + tasks: items: - $ref: '#/components/schemas/PlatformJobResultResponse' + $ref: '#/components/schemas/PlatformJobTaskStatusResponse' type: array - title: Data + title: Tasks + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At type: object required: - - data - title: PlatformJobListResultResponse - PlatformJobLog: + - id + - name + - status + - status_details + - error_details + - tasks + - created_at + - updated_at + title: PlatformJobStepStatusResponse + PlatformJobTaskStatusResponse: properties: - timestamp: + id: type: string - format: date-time - title: Timestamp - job: + title: Id + name: type: string - title: Job - job_step: + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + error_stack: + title: Error Stack type: string - title: Job Step - job_task: + created_at: type: string - title: Job Task - message: + format: date-time + title: Created At + updated_at: type: string - title: Message + format: date-time + title: Updated At type: object required: - - timestamp - - job - - job_step - - job_task - - message - title: PlatformJobLog - PlatformJobLogPage: + - id + - name + - status + - status_details + - error_details + - error_stack + - created_at + - updated_at + title: PlatformJobTaskStatusResponse + RlDPOTraining: properties: - data: - items: - $ref: '#/components/schemas/PlatformJobLog' - type: array - title: Data - total: + optimizer_type: + allOf: + - $ref: '#/components/schemas/OptimizerType' + description: "Optimizer + LR-scheduler combination (AdamW/Adam \xD7 cosine-annealing/flat-LR).\ + \ Defaults to AdamW with cosine annealing." + learning_rate: + type: number + title: Learning Rate + description: Peak learning rate. + default: 0.0001 + min_learning_rate: + title: Min Learning Rate + description: Minimum LR for cosine decay. + type: number + weight_decay: + type: number + title: Weight Decay + description: Weight decay coefficient. + default: 0.01 + adam_beta1: + type: number + title: Adam Beta1 + description: Adam beta1. + default: 0.9 + adam_beta2: + type: number + title: Adam Beta2 + description: Adam beta2. + default: 0.999 + adam_eps: + type: number + exclusiveMinimum: 0.0 + title: Adam Eps + description: Adam epsilon (numerical stability term). + default: 1.0e-05 + warmup_steps: type: integer - title: Total - next_page: - title: Next Page + minimum: 0.0 + title: Warmup Steps + description: Linear warmup steps. + default: 0 + epochs: + type: integer + exclusiveMinimum: 0.0 + title: Epochs + description: Number of passes through the dataset. + default: 1 + max_steps: + title: Max Steps + description: Max training steps (overrides epochs if set). + type: integer + exclusiveMinimum: 0.0 + val_check_interval: + title: Val Check Interval + description: Validation interval. Float <= 1.0 is fraction of epoch; > 1.0 + is step count. + type: number + val_at_end: + type: boolean + title: Val At End + description: Run a final validation pass after the last training step. Keep + enabled so the final checkpoint carries validation metrics and best-checkpoint + selection works; set False only to skip the extra eval. + default: true + keep_top_k: + type: integer + exclusiveMinimum: 0.0 + title: Keep Top K + description: Number of best checkpoints to retain (ranked by validation + loss). + default: 1 + batch_size: + type: integer + exclusiveMinimum: 0.0 + title: Batch Size + description: Global batch size across all GPUs. + default: 32 + micro_batch_size: + type: integer + exclusiveMinimum: 0.0 + title: Micro Batch Size + description: Per-GPU micro batch size. + default: 1 + activation_checkpointing: + type: boolean + title: Activation Checkpointing + description: Recompute activations during the backward pass to reduce memory + at the cost of compute. Enable to fit larger models or longer sequences. + default: false + max_seq_length: + type: integer + exclusiveMinimum: 0.0 + title: Max Seq Length + description: Maximum token sequence length for training. + default: 2048 + seed: + title: Seed + description: Random seed for reproducibility. + type: integer + parallelism: + $ref: '#/components/schemas/RlParallelismParams' + execution_profile: + title: Execution Profile + description: Execution profile for the GPU training step (operator-configured). + Falls back to the service default when omitted. type: string - prev_page: - title: Prev Page + minLength: 1 + type: type: string + const: dpo + title: Type + default: dpo + ref_policy_kl_penalty: + type: number + minimum: 0.0 + title: Ref Policy Kl Penalty + description: KL penalty coefficient (beta in the DPO paper). + default: 0.05 + preference_average_log_probs: + type: boolean + title: Preference Average Log Probs + description: Average log probabilities for preference loss calculation. + default: false + sft_average_log_probs: + type: boolean + title: Sft Average Log Probs + description: Average log probabilities for SFT regularization loss. + default: false + preference_loss_weight: + type: number + minimum: 0.0 + title: Preference Loss Weight + description: Weight for the preference (DPO) loss term. + default: 1.0 + sft_loss_weight: + type: number + minimum: 0.0 + title: Sft Loss Weight + description: Weight for SFT regularization loss (0 = disabled). + default: 0.0 + max_grad_norm: + type: number + minimum: 0.0 + title: Max Grad Norm + description: Maximum gradient norm for clipping. + default: 1.0 + additionalProperties: false type: object - required: - - data - - total - - next_page - - prev_page - title: PlatformJobLogPage - PlatformJobResultResponse: + title: RlDPOTraining + description: "Direct Preference Optimization (full-weight only \u2014 PEFT unsupported)." + RlJobInput: properties: name: - type: string title: Name - job: - type: string - title: Job - workspace: type: string - title: Workspace - project: - title: Project + model: type: string - created_at: + title: Model + description: Model entity reference ('name' or 'workspace/name'). + dataset: type: string - format: date-time - title: Created At - updated_at: + title: Dataset + description: Preference dataset fileset reference. Must contain training.jsonl + + validation.jsonl. + training: + allOf: + - $ref: '#/components/schemas/RlDPOTraining' + description: DPO training method and hyperparameters. + integrations: + $ref: '#/components/schemas/IntegrationsSpecInput' + output: + $ref: '#/components/schemas/RlOutputRequest' + additionalProperties: false + type: object + required: + - model + - dataset + - training + title: RlJobInput + description: POST body / CLI JSON for ``nemo customization rl submit``. + RlJobOutput: + properties: + name: + title: Name + description: Optional job name; auto-generated when omitted. type: string - format: date-time - title: Updated At - artifact_url: + model: type: string - title: Artifact Url - artifact_storage_type: - $ref: '#/components/schemas/FileStorageType' - download_url: - title: Download Url + title: Model + description: Model entity reference ('name' or 'workspace/name'). + dataset: type: string + title: Dataset + description: Preference dataset fileset reference ('name' or 'workspace/name'). + training: + allOf: + - $ref: '#/components/schemas/RlDPOTraining' + description: Training method and hyperparameters (DPO). + integrations: + allOf: + - $ref: '#/components/schemas/IntegrationsSpecOutput' + description: W&B / MLflow integrations. + output: + allOf: + - $ref: '#/components/schemas/RlOutputResponse' + description: Output artifact created by this job. + additionalProperties: false type: object required: - - name - - job - - workspace - - artifact_url - - artifact_storage_type - title: PlatformJobResultResponse - PlatformJobStatus: - type: string - enum: - - created - - pending - - active - - cancelled - - cancelling - - error - - completed - - paused - - pausing - - resuming - title: PlatformJobStatus - description: 'Enumeration of possible job statuses. + - model + - dataset + - training + - output + title: RlJobOutput + description: 'Canonical NeMo-RL job spec (output of the plugin transform). - This enum represents the various states a job can be in during its lifecycle, + The ``dataset`` fileset must contain ``training.jsonl`` and ``validation.jsonl`` - from creation to a terminal state.' - PlatformJobStatusResponse: + (any of the four supported preference formats); the dataset-preparation step + + splits/normalizes them at runtime.' + RlJobsJob: properties: id: - type: string title: Id + type: string name: type: string title: Name + description: + title: Description + type: string + project: + title: Project + type: string + workspace: + title: Workspace + type: string + created_at: + title: Created At + type: string + format: date-time + updated_at: + title: Updated At + type: string + format: date-time + spec: + $ref: '#/components/schemas/RlJobOutput' status: $ref: '#/components/schemas/PlatformJobStatus' status_details: + title: Status Details additionalProperties: true type: object - title: Status Details error_details: title: Error Details additionalProperties: true type: object - steps: - items: - $ref: '#/components/schemas/PlatformJobStepStatusResponse' - type: array - title: Steps - created_at: - type: string - format: date-time - title: Created At - updated_at: - type: string - format: date-time - title: Updated At + ownership: + title: Ownership + additionalProperties: true + type: object + custom_fields: + title: Custom Fields + additionalProperties: true + type: object type: object required: - - id - name - - status - - status_details - - error_details - - steps - - created_at - - updated_at - title: PlatformJobStatusResponse - PlatformJobStepStatusResponse: + - spec + title: RlJobsJob + RlJobsJobRequest: properties: - id: - type: string - title: Id name: - type: string title: Name - status: - $ref: '#/components/schemas/PlatformJobStatus' - status_details: + type: string + description: + title: Description + type: string + project: + title: Project + type: string + spec: + $ref: '#/components/schemas/RlJobInput' + ownership: + title: Ownership additionalProperties: true type: object - title: Status Details - error_details: - title: Error Details + custom_fields: + title: Custom Fields additionalProperties: true type: object - tasks: - items: - $ref: '#/components/schemas/PlatformJobTaskStatusResponse' - type: array - title: Tasks + type: object + required: + - spec + title: RlJobsJobRequest + RlJobsJobsListFilter: + additionalProperties: false + properties: created_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Jobs created at 'gte' datetime or 'lte' datetime. + name: + anyOf: + - $ref: '#/components/schemas/StringFilter' + - type: string + description: Name of the job. + title: Name + workspace: + description: Workspace of the job. + title: Workspace type: string - format: date-time - title: Created At + project: + description: Project containing the job. + title: Project + type: string + status: + allOf: + - $ref: '#/components/schemas/PlatformJobStatus' + description: The current status. updated_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Jobs updated at 'gte' datetime or 'lte' datetime. + title: RlJobsJobsListFilter + type: object + RlJobsJobsPage: + properties: + data: + items: + $ref: '#/components/schemas/RlJobsJob' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. type: string - format: date-time - title: Updated At + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object type: object required: - - id - - name - - status - - status_details - - error_details - - tasks + - data + title: RlJobsJobsPage + RlJobsJobsSortField: + type: string + enum: - created_at + - -created_at - updated_at - title: PlatformJobStepStatusResponse - PlatformJobTaskStatusResponse: + - -updated_at + title: RlJobsJobsSortField + RlOutputRequest: properties: - id: - type: string - title: Id name: - type: string title: Name - status: - $ref: '#/components/schemas/PlatformJobStatus' - status_details: - additionalProperties: true - type: object - title: Status Details - error_details: - title: Error Details - additionalProperties: true - type: object - error_stack: - title: Error Stack type: string - created_at: + additionalProperties: false + type: object + title: RlOutputRequest + description: Submitter-facing output preferences. ``name`` is auto-derived if + omitted. + RlOutputResponse: + properties: + name: type: string - format: date-time - title: Created At - updated_at: + maxLength: 255 + title: Name + description: Name of the output artifact. Used to identify it during deployment + and inference. + examples: + - my-dpo-llama + type: + allOf: + - $ref: '#/components/schemas/OutputNameType' + description: Output artifact type. DPO is full-weight, so always `model`. + default: model + fileset: type: string - format: date-time - title: Updated At + maxLength: 255 + title: Fileset + description: FileSet name where output artifacts are stored. + additionalProperties: false type: object required: - - id - name - - status - - status_details - - error_details - - error_stack - - created_at - - updated_at - title: PlatformJobTaskStatusResponse + - fileset + title: RlOutputResponse + description: Resolved output artifact details. + RlParallelismParams: + properties: + num_gpus_per_node: + type: integer + exclusiveMinimum: 0.0 + title: Num Gpus Per Node + description: Number of GPUs per node. + default: 1 + num_nodes: + type: integer + exclusiveMinimum: 0.0 + title: Num Nodes + description: "Number of nodes (>1 \u2192 multi-node Ray cluster)." + default: 1 + tensor_parallel_size: + type: integer + exclusiveMinimum: 0.0 + title: Tensor Parallel Size + description: Tensor parallel size. + default: 1 + pipeline_parallel_size: + type: integer + exclusiveMinimum: 0.0 + title: Pipeline Parallel Size + description: Pipeline parallel size. + default: 1 + context_parallel_size: + type: integer + exclusiveMinimum: 0.0 + title: Context Parallel Size + description: Context parallel size. + default: 1 + sequence_parallel: + type: boolean + title: Sequence Parallel + description: Enable sequence parallelism. + default: false + additionalProperties: false + type: object + title: RlParallelismParams + description: 'Distributed training parallelism configuration. + + + Single-node multi-GPU uses ``num_nodes=1`` with ``num_gpus_per_node>1``; + + multi-node sets ``num_nodes>1`` and the compiler emits a distributed-GPU + + executor (see :mod:`nmp.rl.app.jobs.compiler`).' SecretRef: type: string pattern: ^[a-z0-9_-]+(/[a-z0-9_-]+)?$ diff --git a/plugins/nemo-deployments/openapi/openapi.yaml b/plugins/nemo-deployments/openapi/openapi.yaml index 4ae9fa9c47..2f4d67e76a 100644 --- a/plugins/nemo-deployments/openapi/openapi.yaml +++ b/plugins/nemo-deployments/openapi/openapi.yaml @@ -970,6 +970,17 @@ components: mount_point: title: Mount Point type: string + initChmod: + title: Initchmod + description: "When set, an init container chmods the freshly created (root-owned)\ + \ named volume to this mode (e.g. '0777') so a non-root workload (e.g.\ + \ the HF weight-puller) can write to it \u2014 the docker analogue of\ + \ a k8s fsGroup." + type: string + initImage: + title: Initimage + description: Image used for the init_chmod container (e.g. busybox). + type: string type: object title: DockerVolumeConfig DriftRecoveryPolicy: diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index f16a77fcae..a07756d247 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -11285,6 +11285,11 @@ components: Accepts any field the evaluations list `sort` param does; clients apply it as the list `sort` param.' default: -created_at + pareto: + allOf: + - $ref: '#/components/schemas/ParetoConfig' + description: Default X/Y metrics for the group's Pareto view. Omit to preserve + the existing value on update; on create, defaults to cost vs. latency. additionalProperties: false type: object required: @@ -11319,6 +11324,8 @@ components: default_sort: type: string title: Default Sort + pareto: + $ref: '#/components/schemas/ParetoConfig' created_at: title: Created At type: string @@ -15399,6 +15406,25 @@ components: - recipe title: PangeaRailOptions description: Configuration data for the Pangea AI Guard API + ParetoConfig: + properties: + x_metric: + type: string + title: X Metric + description: Metric plotted on the Pareto X axis. + default: cost_usd + y_metric: + type: string + title: Y Metric + description: Metric plotted on the Pareto Y axis. + default: latency_ms + type: object + title: ParetoConfig + description: "Default X/Y metrics for a group's cost-vs-accuracy Pareto view.\n\ + \nMetric ids use the same vocabulary as the evaluations list sort/filter fields\ + \ \u2014 ``cost_usd``,\n``latency_ms``, or ``evaluators.``. Defaults\ + \ to cost (x) vs latency (y): both exist for\nevery group, so the chart always\ + \ has something to render before anyone customizes it." PatronusEvaluateApiParams: properties: success_strategy: diff --git a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml index 8ebfadbe66..438911b5ef 100644 --- a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml @@ -919,6 +919,7 @@ resources: experiment_group_request: ExperimentGroupRequest experiment_group_response: ExperimentGroupResponse experiment_group_responses_page: ExperimentGroupResponsesPage + pareto_config: ParetoConfig methods: create: post /apis/intake/v2/workspaces/{workspace}/experiment-groups list: get /apis/intake/v2/workspaces/{workspace}/experiment-groups diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md index 877402928d..998ab3da68 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md @@ -8,6 +8,7 @@ from nemo_platform.types.experiment_groups import ( ExperimentGroupRequest, ExperimentGroupResponse, ExperimentGroupResponsesPage, + ParetoConfig, ) ``` diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py index 1fcc11f27a..728c4b0d9a 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py @@ -39,6 +39,7 @@ experiment_group_create_params, experiment_group_update_params, ) +from ...types.experiment_groups.pareto_config_param import ParetoConfigParam from ...types.experiment_groups.experiment_group_response import ExperimentGroupResponse from ...types.experiment_groups.experiment_group_filter_param import ExperimentGroupFilterParam from ..._exceptions import ConflictError @@ -75,6 +76,7 @@ def create( description: str | Omit = omit, insight_id: str | Omit = omit, metadata: Dict[str, str] | Omit = omit, + pareto: ParetoConfigParam | Omit = omit, summary: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -103,6 +105,13 @@ def create( metadata: Free-form producer metadata for the group. + pareto: Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — + `cost_usd`, `latency_ms`, or `evaluators.`. Defaults to cost (x) vs + latency (y): both exist for every group, so the chart always has something to + render before anyone customizes it. + summary: Human- or agent-authored summary of the group's findings. @@ -131,6 +140,7 @@ def create( "description": description, "insight_id": insight_id, "metadata": metadata, + "pareto": pareto, "summary": summary, }, experiment_group_create_params.ExperimentGroupCreateParams, @@ -195,6 +205,7 @@ def update( description: str | Omit = omit, insight_id: str | Omit = omit, metadata: Dict[str, str] | Omit = omit, + pareto: ParetoConfigParam | Omit = omit, summary: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -222,6 +233,13 @@ def update( metadata: Free-form producer metadata for the group. + pareto: Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — + `cost_usd`, `latency_ms`, or `evaluators.`. Defaults to cost (x) vs + latency (y): both exist for every group, so the chart always has something to + render before anyone customizes it. + summary: Human- or agent-authored summary of the group's findings. extra_headers: Send extra headers @@ -251,6 +269,7 @@ def update( "description": description, "insight_id": insight_id, "metadata": metadata, + "pareto": pareto, "summary": summary, }, experiment_group_update_params.ExperimentGroupUpdateParams, @@ -394,6 +413,7 @@ async def create( description: str | Omit = omit, insight_id: str | Omit = omit, metadata: Dict[str, str] | Omit = omit, + pareto: ParetoConfigParam | Omit = omit, summary: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -422,6 +442,13 @@ async def create( metadata: Free-form producer metadata for the group. + pareto: Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — + `cost_usd`, `latency_ms`, or `evaluators.`. Defaults to cost (x) vs + latency (y): both exist for every group, so the chart always has something to + render before anyone customizes it. + summary: Human- or agent-authored summary of the group's findings. @@ -450,6 +477,7 @@ async def create( "description": description, "insight_id": insight_id, "metadata": metadata, + "pareto": pareto, "summary": summary, }, experiment_group_create_params.ExperimentGroupCreateParams, @@ -514,6 +542,7 @@ async def update( description: str | Omit = omit, insight_id: str | Omit = omit, metadata: Dict[str, str] | Omit = omit, + pareto: ParetoConfigParam | Omit = omit, summary: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -541,6 +570,13 @@ async def update( metadata: Free-form producer metadata for the group. + pareto: Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — + `cost_usd`, `latency_ms`, or `evaluators.`. Defaults to cost (x) vs + latency (y): both exist for every group, so the chart always has something to + render before anyone customizes it. + summary: Human- or agent-authored summary of the group's findings. extra_headers: Send extra headers @@ -570,6 +606,7 @@ async def update( "description": description, "insight_id": insight_id, "metadata": metadata, + "pareto": pareto, "summary": summary, }, experiment_group_update_params.ExperimentGroupUpdateParams, diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py index 3506053fad..7fb0bced54 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py @@ -17,6 +17,8 @@ from __future__ import annotations +from .pareto_config import ParetoConfig as ParetoConfig +from .pareto_config_param import ParetoConfigParam as ParetoConfigParam from .experiment_group_response import ExperimentGroupResponse as ExperimentGroupResponse from .experiment_group_list_params import ExperimentGroupListParams as ExperimentGroupListParams from .experiment_group_filter_param import ExperimentGroupFilterParam as ExperimentGroupFilterParam diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py index cd6ebf095d..ffc2c29860 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py @@ -20,6 +20,8 @@ from typing import Dict from typing_extensions import Required, TypedDict +from .pareto_config_param import ParetoConfigParam + __all__ = ["ExperimentGroupCreateParams"] @@ -48,5 +50,14 @@ class ExperimentGroupCreateParams(TypedDict, total=False): metadata: Dict[str, str] """Free-form producer metadata for the group.""" + pareto: ParetoConfigParam + """Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — + `cost_usd`, `latency_ms`, or `evaluators.`. Defaults to cost (x) vs + latency (y): both exist for every group, so the chart always has something to + render before anyone customizes it. + """ + summary: str """Human- or agent-authored summary of the group's findings.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py index 52cac70f8f..ccc20c6731 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py @@ -19,6 +19,7 @@ from datetime import datetime from ..._models import BaseModel +from .pareto_config import ParetoConfig __all__ = ["ExperimentGroupResponse"] @@ -48,6 +49,15 @@ class ExperimentGroupResponse(BaseModel): metadata: Optional[Dict[str, str]] = None + pareto: Optional[ParetoConfig] = None + """Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — + `cost_usd`, `latency_ms`, or `evaluators.`. Defaults to cost (x) vs + latency (y): both exist for every group, so the chart always has something to + render before anyone customizes it. + """ + summary: Optional[str] = None updated_at: Optional[datetime] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py index b734af7b3a..f5e7d241e9 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py @@ -21,6 +21,7 @@ from typing_extensions import Required, Annotated, TypedDict from ..._utils import PropertyInfo +from .pareto_config_param import ParetoConfigParam __all__ = ["ExperimentGroupUpdateParams"] @@ -50,5 +51,14 @@ class ExperimentGroupUpdateParams(TypedDict, total=False): metadata: Dict[str, str] """Free-form producer metadata for the group.""" + pareto: ParetoConfigParam + """Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — + `cost_usd`, `latency_ms`, or `evaluators.`. Defaults to cost (x) vs + latency (y): both exist for every group, so the chart always has something to + render before anyone customizes it. + """ + summary: str """Human- or agent-authored summary of the group's findings.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config.py new file mode 100644 index 0000000000..7c2f16c937 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional + +from ..._models import BaseModel + +__all__ = ["ParetoConfig"] + + +class ParetoConfig(BaseModel): + """Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — ``cost_usd``, + ``latency_ms``, or ``evaluators.``. Defaults to cost (x) vs latency (y): both exist for + every group, so the chart always has something to render before anyone customizes it. + """ + + x_metric: Optional[str] = None + """Metric plotted on the Pareto X axis.""" + + y_metric: Optional[str] = None + """Metric plotted on the Pareto Y axis.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config_param.py new file mode 100644 index 0000000000..5fb748d0c0 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config_param.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import TypedDict + +__all__ = ["ParetoConfigParam"] + + +class ParetoConfigParam(TypedDict, total=False): + """Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — ``cost_usd``, + ``latency_ms``, or ``evaluators.``. Defaults to cost (x) vs latency (y): both exist for + every group, so the chart always has something to render before anyone customizes it. + """ + + x_metric: str + """Metric plotted on the Pareto X axis.""" + + y_metric: str + """Metric plotted on the Pareto Y axis.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/inference/container_executor_config.py b/sdk/python/nemo-platform/src/nemo_platform/types/inference/container_executor_config.py index 6c0d6f8120..923e586c7f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/inference/container_executor_config.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/inference/container_executor_config.py @@ -73,9 +73,12 @@ class ContainerExecutorConfig(BaseModel): override_config: Optional[Dict[str, object]] = None """ - Raw NIMService spec configuration that takes precedence over generated config - (NIM engine on k8s). Allows advanced configuration options directly. Ignored by - non-NIM engines. + Partial NIMService Spec fragments deep-merged after generated defaults and + k8s_nim_operator_config (NIM engine on k8s only). Supported keys: image, + command, args, resources, env, readinessProbe, livenessProbe, startupProbe, + nodeSelector, tolerations, userID, groupID, labels, initContainers, + sidecarContainers. Unsupported keys are rejected at compile time. Ignored by + non-NIM engines and docker runtime. """ run_as_group: Optional[int] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/inference/container_executor_config_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/inference/container_executor_config_param.py index 5ffb042043..d882711066 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/inference/container_executor_config_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/inference/container_executor_config_param.py @@ -76,9 +76,12 @@ class ContainerExecutorConfigParam(TypedDict, total=False): override_config: Dict[str, object] """ - Raw NIMService spec configuration that takes precedence over generated config - (NIM engine on k8s). Allows advanced configuration options directly. Ignored by - non-NIM engines. + Partial NIMService Spec fragments deep-merged after generated defaults and + k8s_nim_operator_config (NIM engine on k8s only). Supported keys: image, + command, args, resources, env, readinessProbe, livenessProbe, startupProbe, + nodeSelector, tolerations, userID, groupID, labels, initContainers, + sidecarContainers. Unsupported keys are rejected at compile time. Ignored by + non-NIM engines and docker runtime. """ run_as_group: int diff --git a/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py b/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py index d6e241d130..c713ab8143 100644 --- a/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py +++ b/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py @@ -54,6 +54,10 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: description="description", insight_id="insight_id", metadata={"foo": "string"}, + pareto={ + "x_metric": "x_metric", + "y_metric": "y_metric", + }, summary="summary", ) assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) @@ -168,6 +172,10 @@ def test_method_update_with_all_params(self, client: NeMoPlatform) -> None: description="description", insight_id="insight_id", metadata={"foo": "string"}, + pareto={ + "x_metric": "x_metric", + "y_metric": "y_metric", + }, summary="summary", ) assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) @@ -354,6 +362,10 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo description="description", insight_id="insight_id", metadata={"foo": "string"}, + pareto={ + "x_metric": "x_metric", + "y_metric": "y_metric", + }, summary="summary", ) assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) @@ -468,6 +480,10 @@ async def test_method_update_with_all_params(self, async_client: AsyncNeMoPlatfo description="description", insight_id="insight_id", metadata={"foo": "string"}, + pareto={ + "x_metric": "x_metric", + "y_metric": "y_metric", + }, summary="summary", ) assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) diff --git a/sdk/stainless.yaml b/sdk/stainless.yaml index 8ebfadbe66..438911b5ef 100644 --- a/sdk/stainless.yaml +++ b/sdk/stainless.yaml @@ -919,6 +919,7 @@ resources: experiment_group_request: ExperimentGroupRequest experiment_group_response: ExperimentGroupResponse experiment_group_responses_page: ExperimentGroupResponsesPage + pareto_config: ParetoConfig methods: create: post /apis/intake/v2/workspaces/{workspace}/experiment-groups list: get /apis/intake/v2/workspaces/{workspace}/experiment-groups diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index 04de88e863..96d9ecfeb0 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -149,6 +149,7 @@ async def create_experiment_group( summary=body.summary, metadata=body.metadata, default_sort=body.default_sort, + pareto=body.pareto, ) try: created = await entity_client.create(entity) @@ -266,6 +267,10 @@ async def update_experiment_group( existing.summary = body.summary existing.metadata = body.metadata existing.default_sort = body.default_sort + # Only overwrite the saved axes when the client actually sent them; an omitted `pareto` (older + # clients) must not silently reset customized axes to the cost/latency default. + if body.pareto is not None: + existing.pareto = body.pareto updated = await entity_client.update(existing) response = ExperimentGroupResponse.from_entity(updated) response.evaluation_count = await _count_live_evaluations_in_group( diff --git a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py index 2f136c7e5d..940417cc62 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py @@ -13,7 +13,7 @@ from typing import Annotated, Self from nmp.common.entities.values import DatetimeFilter, Filter, NumberFilter, map_entity_field -from nmp.intake.entities.experiments import Experiment, ExperimentGroup +from nmp.intake.entities.experiments import Experiment, ExperimentGroup, ParetoConfig from nmp.intake.spans.domain import ( INTAKE_PREVIEW_PAYLOAD_CHAR_LIMIT, IntakeResponseMode, @@ -48,6 +48,13 @@ class ExperimentGroupRequest(BaseModel): "the list `sort` param." ), ) + pareto: ParetoConfig | None = Field( + default=None, + description=( + "Default X/Y metrics for the group's Pareto view. Omit to preserve the existing value on " + "update; on create, defaults to cost vs. latency." + ), + ) class EvaluationRequest(BaseModel): @@ -165,6 +172,7 @@ class ExperimentGroupResponse(BaseModel): summary: str | None = None metadata: dict[str, str] | None = None default_sort: str + pareto: ParetoConfig = Field(default_factory=ParetoConfig) created_at: datetime | None = None updated_at: datetime | None = None evaluation_count: int = Field( @@ -188,6 +196,7 @@ def from_entity(cls, entity: ExperimentGroup) -> ExperimentGroupResponse: summary=entity.summary, metadata=entity.metadata, default_sort=entity.default_sort, + pareto=entity.pareto, created_at=entity.created_at, updated_at=entity.updated_at, ) diff --git a/services/intake/src/nmp/intake/entities/experiments.py b/services/intake/src/nmp/intake/entities/experiments.py index 8710286c0d..fc72db8603 100644 --- a/services/intake/src/nmp/intake/entities/experiments.py +++ b/services/intake/src/nmp/intake/entities/experiments.py @@ -22,7 +22,7 @@ from typing import Any, ClassVar from nmp.common.entities.client import EntityBase -from pydantic import AnyUrl, Field, field_validator, model_validator +from pydantic import AnyUrl, BaseModel, Field, field_validator, model_validator def _stringify_metadata(value: Any) -> Any: @@ -34,6 +34,31 @@ def _stringify_metadata(value: Any) -> Any: return {key: val if isinstance(val, str) else json.dumps(val) for key, val in value.items()} +class ParetoConfig(BaseModel): + """Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — ``cost_usd``, + ``latency_ms``, or ``evaluators.``. Defaults to cost (x) vs latency (y): both exist for + every group, so the chart always has something to render before anyone customizes it. + """ + + x_metric: str = Field(default="cost_usd", description="Metric plotted on the Pareto X axis.") + y_metric: str = Field(default="latency_ms", description="Metric plotted on the Pareto Y axis.") + + @field_validator("x_metric", "y_metric") + @classmethod + def _validate_metric(cls, value: str) -> str: + """Restrict axes to metrics Studio can actually plot: the two fixed metrics or a dynamic + ``evaluators.``. Evaluator names are customer-specific, so only the prefix is checked.""" + if value in ("cost_usd", "latency_ms"): + return value + if value.startswith("evaluators.") and value != "evaluators.": + return value + raise ValueError( + f"Unsupported Pareto metric {value!r}; expected 'cost_usd', 'latency_ms', or 'evaluators.'." + ) + + class ExperimentGroup(EntityBase): """A named container of Experiments pursuing a single optimization goal. @@ -74,6 +99,21 @@ def _default_sort_fallback(cls, value: Any) -> Any: schema-on-read, so coerce anything that isn't a usable string to the default on read.""" return value if isinstance(value, str) else "-created_at" + pareto: ParetoConfig = Field( + default_factory=ParetoConfig, + description=( + "Default X/Y metrics for this group's cost-vs-accuracy Pareto view. Defaults to cost (x) " + "vs latency (y) so the chart always renders before it's customized." + ), + ) + + @field_validator("pareto", mode="before") + @classmethod + def _pareto_fallback(cls, value: Any) -> Any: + """Schema-on-read: groups persisted before this field stored no ``pareto`` (or ``null``); + coerce anything that isn't a config mapping to the cost-vs-latency default.""" + return value if isinstance(value, dict | ParetoConfig) else ParetoConfig() + is_deleted: bool = Field( default=False, description=( diff --git a/services/intake/src/nmp/intake/spans/evaluation_rollup_repository.py b/services/intake/src/nmp/intake/spans/evaluation_rollup_repository.py index 227a609682..e82e5fbc89 100644 --- a/services/intake/src/nmp/intake/spans/evaluation_rollup_repository.py +++ b/services/intake/src/nmp/intake/spans/evaluation_rollup_repository.py @@ -11,7 +11,6 @@ from nmp.intake.spans.clickhouse_client import ClickHouseSpanClient from nmp.intake.spans.span_attribute_catalog import COST_SCALE, SpanAttributeField, spec_for_field from nmp.intake.spans.storage import float_or_none, result_rows -from nmp.intake.spans.trace_repository import current_spans_sql @dataclass(frozen=True) @@ -316,6 +315,35 @@ def _test_case_scores_cte() -> str: test_cases.evaluation_id, test_cases.test_case_key, evaluators.evaluator_name, test_cases.session_count""" +def _current_session_span_metrics_sql(spans_table: str) -> str: + """Latest-version spans for the scoped sessions, projected to *only* the metric inputs. + + The shared ``current_spans_sql`` emits every span column, including the full ``attributes_*`` Maps. + The metric rollup then joins that to the sessions, so the join hash table would hold every deduped + span with its maps — gigabytes on real agent workloads (long trajectories × many spans), which is + what trips ClickHouse's memory limit. Here we ``argMax`` just the cost value (plus a present-flag, + to keep an absent cost distinct from a real 0) and the model/agent strings, so the join carries a + handful of scalars per span instead. Deduping by span identity picks each span's latest version. + """ + return f""" + ( + SELECT + workspace, + argMax(session_id, (event_ts, is_deleted)) AS dedup_session_id, + argMax(attributes_number[%(cost_key)s], (event_ts, is_deleted)) AS cost_value, + argMax(has(mapKeys(attributes_number), %(cost_key)s), (event_ts, is_deleted)) AS cost_present, + argMax(attributes_string[%(model_key)s], (event_ts, is_deleted)) AS model_name, + argMax(attributes_string[%(agent_name_key)s], (event_ts, is_deleted)) AS agent_name, + argMax(attributes_string[%(agent_version_key)s], (event_ts, is_deleted)) AS agent_version, + argMax(is_deleted, (event_ts, is_deleted)) AS del_flag + FROM {spans_table} + WHERE workspace = %(workspace)s + AND (workspace, session_id) IN (SELECT DISTINCT workspace, session_id FROM scoped_sessions) + GROUP BY workspace, source_format, trace_id, external_span_id, id + ) + """ + + def _metric_rollups_sql(*, trace_index_table: str, spans_table: str, evaluation_names_sql: str) -> str: # Two-level rollup: per-attempt cost/latency, then averaged per test case (avg per attempt — the # number must not scale with k), then the distribution across test cases (test-case-weighted). @@ -326,50 +354,25 @@ def _metric_rollups_sql(*, trace_index_table: str, spans_table: str, evaluation_ scoped_sessions AS ( {_scoped_sessions_sql(trace_index_table, evaluation_names_sql)} ), - current_session_spans AS ( - { - current_spans_sql( - spans_table, - extra_where_sql=( - "(span_versions.workspace, span_versions.session_id) IN " - "(SELECT DISTINCT workspace, session_id FROM scoped_sessions)" - ), - ) - } - ), + current_session_spans AS {_current_session_span_metrics_sql(spans_table)}, session_costs AS ( SELECT sessions.evaluation_id AS evaluation_id, sessions.test_case_id AS test_case_key, sessions.latency_ms AS latency_ms, if( - countIf(has(mapKeys(spans.attributes_number), %(cost_key)s)) = 0, + countIf(spans.cost_present) = 0, NULL, - sumIf( - spans.attributes_number[%(cost_key)s], - has(mapKeys(spans.attributes_number), %(cost_key)s) - ) / {COST_SCALE} + sumIf(spans.cost_value, spans.cost_present) / {COST_SCALE} ) AS cost_usd, - groupUniqArrayIf( - spans.attributes_string[%(model_key)s], - has(mapKeys(spans.attributes_string), %(model_key)s) - AND spans.attributes_string[%(model_key)s] != '' - ) AS model_names, - groupUniqArrayIf( - spans.attributes_string[%(agent_name_key)s], - has(mapKeys(spans.attributes_string), %(agent_name_key)s) - AND spans.attributes_string[%(agent_name_key)s] != '' - ) AS agent_names, - groupUniqArrayIf( - spans.attributes_string[%(agent_version_key)s], - has(mapKeys(spans.attributes_string), %(agent_version_key)s) - AND spans.attributes_string[%(agent_version_key)s] != '' - ) AS agent_versions + groupUniqArrayIf(spans.model_name, spans.model_name != '') AS model_names, + groupUniqArrayIf(spans.agent_name, spans.agent_name != '') AS agent_names, + groupUniqArrayIf(spans.agent_version, spans.agent_version != '') AS agent_versions FROM scoped_sessions AS sessions LEFT JOIN current_session_spans AS spans ON sessions.workspace = spans.workspace - AND sessions.session_id = spans.session_id - AND spans.is_deleted = 0 + AND sessions.session_id = spans.dedup_session_id + AND spans.del_flag = 0 WHERE sessions.test_case_id != '' GROUP BY sessions.evaluation_id, sessions.session_id, sessions.test_case_id, sessions.latency_ms ), diff --git a/services/intake/tests/integration/test_experiments_crud.py b/services/intake/tests/integration/test_experiments_crud.py index 432f8fc87d..b9a0aca778 100644 --- a/services/intake/tests/integration/test_experiments_crud.py +++ b/services/intake/tests/integration/test_experiments_crud.py @@ -77,6 +77,44 @@ def test_experiment_group_update_description(client: TestClient) -> None: assert missing.status_code == 404 +def test_experiment_group_pareto_defaults_and_round_trips(client: TestClient) -> None: + # Omitting pareto defaults to cost (x) vs latency (y), so the chart always has something to render. + created = client.post(GROUPS, json={"name": "pareto-cfg"}) + assert created.status_code == 201, created.text + assert created.json()["pareto"] == {"x_metric": "cost_usd", "y_metric": "latency_ms"} + assert client.get(f"{GROUPS}/pareto-cfg").json()["pareto"] == { + "x_metric": "cost_usd", + "y_metric": "latency_ms", + } + + # A custom selection round-trips through PUT. + updated = client.put( + f"{GROUPS}/pareto-cfg", + json={"name": "pareto-cfg", "pareto": {"x_metric": "cost_usd", "y_metric": "evaluators.reward"}}, + ) + assert updated.status_code == 200, updated.text + assert updated.json()["pareto"] == {"x_metric": "cost_usd", "y_metric": "evaluators.reward"} + + # An update that omits pareto preserves the saved axes rather than resetting them to the default. + preserved = client.put(f"{GROUPS}/pareto-cfg", json={"name": "pareto-cfg", "summary": "unrelated edit"}) + assert preserved.status_code == 200, preserved.text + assert preserved.json()["pareto"] == {"x_metric": "cost_usd", "y_metric": "evaluators.reward"} + + +def test_experiment_group_pareto_rejects_unknown_metric(client: TestClient) -> None: + # Studio can only plot cost_usd, latency_ms, or evaluators.; anything else is rejected so it + # can't be persisted as an unusable chart default. + rejected = client.post( + GROUPS, json={"name": "bad-pareto", "pareto": {"x_metric": "made_up", "y_metric": "latency_ms"}} + ) + assert rejected.status_code == 422, rejected.text + # A dynamic evaluator metric is allowed. + ok = client.post( + GROUPS, json={"name": "ok-pareto", "pareto": {"x_metric": "evaluators.safety", "y_metric": "cost_usd"}} + ) + assert ok.status_code == 201, ok.text + + def test_evaluation_update_moves_between_groups_and_edits(client: TestClient) -> None: group_a = client.post(GROUPS, json={"name": "grp-a"}).json() group_b = client.post(GROUPS, json={"name": "grp-b"}).json() diff --git a/services/intake/tests/test_experiment_rollup_repository.py b/services/intake/tests/test_experiment_rollup_repository.py index e3dd105af0..d4c202e54b 100644 --- a/services/intake/tests/test_experiment_rollup_repository.py +++ b/services/intake/tests/test_experiment_rollup_repository.py @@ -149,9 +149,11 @@ async def test_evaluation_rollups_anchor_on_root_session_membership(): assert "WHERE sessions.test_case_id != ''" in client.queries[1] assert "test_case_metrics AS" in client.queries[2] assert "current_session_spans AS" in client.queries[2] - assert "(span_versions.workspace, span_versions.session_id) IN" in client.queries[2] + assert ( + "(workspace, session_id) IN (SELECT DISTINCT workspace, session_id FROM scoped_sessions)" in client.queries[2] + ) assert "LEFT JOIN current_session_spans AS spans" in client.queries[2] - assert "sessions.session_id = spans.session_id" in client.queries[2] + assert "sessions.session_id = spans.dedup_session_id" in client.queries[2] assert "arraySort(arrayDistinct(arrayFlatten(groupArray(model_names)))) AS model_names" in client.queries[2] assert "quantileExactIf(0.5)" in client.queries[2] assert "cost_median" in client.queries[2] diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx new file mode 100644 index 0000000000..7cd2d3b636 --- /dev/null +++ b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx @@ -0,0 +1,333 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { useToast } from '@nemo/common/src/providers/toast/useToast'; +import { + getGetExperimentGroupQueryKey, + useUpdateExperimentGroup, +} from '@nemo/sdk/generated/platform/api'; +import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import { + Button, + SelectContent, + SelectItem, + SelectListbox, + SelectRoot, + SelectTrigger, + Text, +} from '@nvidia/foundations-react-core'; +import { + buildParetoPoints, + deriveParetoMetrics, + metricLabel, + type ParetoMetric, + type ParetoPlotPoint, +} from '@studio/components/charts/ExperimentGroupParetoChart/paretoMetrics'; +import { useParetoEvaluations } from '@studio/components/charts/ExperimentGroupParetoChart/useParetoEvaluations'; +import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; +import { useQueryClient } from '@tanstack/react-query'; +import { Loader2, Save } from 'lucide-react'; +import { type FC, useMemo, useState } from 'react'; +import { + CartesianGrid, + ResponsiveContainer, + Scatter, + ScatterChart, + Tooltip, + XAxis, + YAxis, +} from 'recharts'; + +interface ExperimentGroupParetoChartProps { + workspace: string; + group: ExperimentGroupResponse; + /** + * The group's full evaluation set, when the caller already has it loaded (a small group that fit on + * the leaderboard's first page). Supplying it lets the chart render from the shared rows and skip its + * own all-evaluations fetch — which re-runs the same server-side rollup. + */ + preloadedEvaluations?: EvaluationRow[]; + /** + * True while the caller is still loading rows it will supply via `preloadedEvaluations`. The chart + * shows a loading state and holds off its own fetch, rather than briefly rendering empty. Only set + * for groups known (from `evaluation_count`) to fit one page — larger groups fetch immediately. + */ + preloadPending?: boolean; +} + +const CHART_HEIGHT = 360; +const DEFAULT_X_METRIC = 'cost_usd'; +const DEFAULT_Y_METRIC = 'latency_ms'; + +/** Unit-less axis tick labels (units live on the axis label + chart title). Big values are compacted + * so they don't wrap and collide with the rotated axis title (16000 -> "16K"); small values keep full + * precision so close cost/score ticks don't all round to the same number (0.05, 0.11 stay distinct). */ +const formatAxisTick = (value: number): string => + Math.abs(value) >= 1000 + ? value.toLocaleString(undefined, { notation: 'compact', maximumFractionDigits: 1 }) + : value.toLocaleString(undefined, { maximumFractionDigits: 3 }); + +/** Format a metric value for tooltips: cost as USD, latency in ms, evaluator scores as-is. */ +const formatMetricValue = (metric: ParetoMetric, value: number): string => { + if (metric.id === 'cost_usd') { + return `$${value.toLocaleString(undefined, { maximumFractionDigits: 4 })}`; + } + if (metric.id === 'latency_ms') { + return `${Math.round(value).toLocaleString()} ms`; + } + return value.toLocaleString(undefined, { maximumFractionDigits: 3 }); +}; + +interface ParetoTooltipProps { + active?: boolean; + payload?: ReadonlyArray<{ payload: ParetoPlotPoint }>; + xMetric: ParetoMetric; + yMetric: ParetoMetric; +} + +const ParetoTooltip: FC = ({ active, payload, xMetric, yMetric }) => { + const point = payload?.[0]?.payload; + if (!active || !point) return null; + return ( +
+ {point.name} +
+ + {xMetric.label}: {formatMetricValue(xMetric, point.x)} + + + {yMetric.label}: {formatMetricValue(yMetric, point.y)} + + {point.onFrontier && ( + + On the Pareto frontier + + )} +
+
+ ); +}; + +interface MetricSelectProps { + label: string; + value: string; + metrics: readonly ParetoMetric[]; + onChange: (id: string) => void; +} + +const MetricSelect: FC = ({ label, value, metrics, onChange }) => ( + +); + +/** + * Cost-vs-accuracy Pareto view for an experiment group: one point per evaluation with the Pareto + * frontier highlighted. Points come from the group's evaluations (the existing list endpoint, which + * already carries each evaluation's cost/latency/evaluator rollup means). The two axes are chosen from + * the group's available metrics (cost, latency, and each evaluator) and are **persisted on the group** + * — changing a picker saves the selection so it survives reloads and is shared across viewers. Seeds + * from the group's saved axes, defaulting to cost vs. latency (present for every group). + */ +export const ExperimentGroupParetoChart: FC = ({ + workspace, + group, + preloadedEvaluations, + preloadPending = false, +}) => { + const queryClient = useQueryClient(); + const toast = useToast(); + + // Reuse the caller's rows when it already has the whole group loaded. While it's still loading and + // about to supply them (`preloadPending`), hold off our own fetch and show a loading state instead of + // rendering empty. Otherwise fetch the evaluations ourselves (in parallel — see the `enabled` flag). + const hasPreloaded = preloadedEvaluations !== undefined; + const { + rows: fetchedRows, + isLoading: isFetching, + isError, + } = useParetoEvaluations(workspace, group.id, { enabled: !hasPreloaded && !preloadPending }); + const points = hasPreloaded ? preloadedEvaluations : fetchedRows; + const isLoading = hasPreloaded ? false : preloadPending || isFetching; + const metrics = useMemo(() => deriveParetoMetrics(points), [points]); + + // Selected axes are optimistic local state seeded from the group's saved config (available + // synchronously from the `group` prop). Changes update the chart immediately and persist below. + const [xMetricId, setXMetricId] = useState(group.pareto?.x_metric ?? DEFAULT_X_METRIC); + const [yMetricId, setYMetricId] = useState(group.pareto?.y_metric ?? DEFAULT_Y_METRIC); + + const { mutate: saveGroup, isPending: isSaving } = useUpdateExperimentGroup({ + mutation: { + onSuccess: () => { + toast.success('Saved the group default Pareto view.'); + queryClient.invalidateQueries({ + queryKey: getGetExperimentGroupQueryKey(workspace, group.name), + }); + }, + onError: () => toast.error('Failed to save the Pareto metrics.'), + }, + }); + + // Persist the axes on the group. PUT is a full replace, so send every current field — only `pareto` + // changes. + const persistAxes = (xMetric: string, yMetric: string) => { + saveGroup({ + workspace, + name: group.name, + data: { + name: group.name, + description: group.description, + insight_id: group.insight_id, + summary: group.summary, + metadata: group.metadata, + default_sort: group.default_sort, + pareto: { x_metric: xMetric, y_metric: yMetric }, + }, + }); + }; + + // Picking a metric only updates the local view; persisting the group-wide default is an explicit, + // clearly-labeled action (Save button) so users know a save is shared with everyone. + const handleXChange = (id: string) => setXMetricId(id); + const handleYChange = (id: string) => setYMetricId(id); + + const savedX = group.pareto?.x_metric ?? DEFAULT_X_METRIC; + const savedY = group.pareto?.y_metric ?? DEFAULT_Y_METRIC; + const hasUnsavedAxes = xMetricId !== savedX || yMetricId !== savedY; + + // Fall back to the first/second metric if a saved id isn't in the current data (e.g. an evaluator + // that dropped out). Cost and latency are always present, so a fallback always exists. + const xMetric = metrics.find((m) => m.id === xMetricId) ?? metrics[0]; + const yMetric = metrics.find((m) => m.id === yMetricId) ?? metrics[1] ?? metrics[0]; + + const plotPoints = useMemo( + () => (xMetric && yMetric ? buildParetoPoints(points, xMetric, yMetric) : []), + [points, xMetric, yMetric] + ); + + // Frontier points are sorted by x so Recharts draws the connecting line along the frontier curve. + const frontierPoints = useMemo( + () => plotPoints.filter((p) => p.onFrontier).sort((a, b) => a.x - b.x), + [plotPoints] + ); + const dominatedPoints = useMemo(() => plotPoints.filter((p) => !p.onFrontier), [plotPoints]); + + const renderBody = () => { + if (isError) { + return Could not load the Pareto data for this group.; + } + if (isLoading) { + return ( +
+ + + Loading evaluations… + +
+ ); + } + if (plotPoints.length === 0 || !xMetric || !yMetric) { + return ( + + No evaluations with both selected metrics to plot. + + ); + } + return ( + + + + + + } + /> + + + + + ); + }; + + return ( +
+
+ {`${metricLabel(xMetricId)} vs. ${metricLabel(yMetricId)}`} +
+ + + +
+
+ {renderBody()} +
+ ); +}; diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.test.ts b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.test.ts new file mode 100644 index 0000000000..a01111b50f --- /dev/null +++ b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.test.ts @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + buildParetoPoints, + deriveParetoMetrics, + type ParetoMetric, +} from '@studio/components/charts/ExperimentGroupParetoChart/paretoMetrics'; +import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; + +// Only the fields the Pareto accessors read (name + cost/latency/evaluator rollup means) are set; +// the rest of the rich EvaluationRow shape is irrelevant here, so build a minimal stand-in. +const point = (opts: { + name: string; + cost?: number; + latency?: number; + evaluators?: Record; +}): EvaluationRow => + ({ + name: opts.name, + id: opts.name, + cost_usd: opts.cost == null ? undefined : { mean: opts.cost }, + latency_ms: opts.latency == null ? undefined : { mean: opts.latency }, + aggregate_scores: opts.evaluators + ? Object.fromEntries(Object.entries(opts.evaluators).map(([name, mean]) => [name, { mean }])) + : undefined, + }) as unknown as EvaluationRow; + +const getMetric = (metrics: ParetoMetric[], id: string): ParetoMetric => { + const metric = metrics.find((m) => m.id === id); + if (!metric) throw new Error(`metric ${id} not found`); + return metric; +}; + +const frontierNames = (points: EvaluationRow[], x: ParetoMetric, y: ParetoMetric): string[] => + buildParetoPoints(points, x, y) + .filter((p) => p.onFrontier) + .map((p) => p.name) + .sort(); + +describe('deriveParetoMetrics', () => { + it('always offers cost and latency (minimized) plus one option per evaluator (maximized)', () => { + const points = [point({ name: 'a', evaluators: { reward: 1, safety: 1 } })]; + const metrics = deriveParetoMetrics(points); + // Evaluator ids use the API vocabulary (`evaluators.`) so they match the group's saved axes. + expect(metrics.map((m) => m.id)).toEqual([ + 'cost_usd', + 'latency_ms', + 'evaluators.reward', + 'evaluators.safety', + ]); + expect(getMetric(metrics, 'cost_usd').direction).toBe('min'); + expect(getMetric(metrics, 'latency_ms').direction).toBe('min'); + expect(getMetric(metrics, 'evaluators.reward').direction).toBe('max'); + }); + + it('offers only cost and latency when no evaluators are present', () => { + expect(deriveParetoMetrics([point({ name: 'a' })]).map((m) => m.id)).toEqual([ + 'cost_usd', + 'latency_ms', + ]); + }); +}); + +describe('buildParetoPoints', () => { + it('marks the non-dominated set for two minimized axes (cost vs latency)', () => { + const points = [ + point({ name: 'A', cost: 1, latency: 4 }), // cheapest -> frontier + point({ name: 'B', cost: 2, latency: 2 }), // balanced -> frontier + point({ name: 'C', cost: 4, latency: 1 }), // fastest -> frontier + point({ name: 'D', cost: 3, latency: 3 }), // dominated by B + ]; + const metrics = deriveParetoMetrics(points); + expect( + frontierNames(points, getMetric(metrics, 'cost_usd'), getMetric(metrics, 'latency_ms')) + ).toEqual(['A', 'B', 'C']); + }); + + it('respects mixed directions: minimize cost, maximize an evaluator score', () => { + const points = [ + point({ name: 'A', cost: 1, evaluators: { reward: 0.5 } }), // cheapest -> frontier + point({ name: 'B', cost: 2, evaluators: { reward: 0.9 } }), // most accurate -> frontier + point({ name: 'C', cost: 2, evaluators: { reward: 0.4 } }), // dominated by A + ]; + const metrics = deriveParetoMetrics(points); + expect( + frontierNames(points, getMetric(metrics, 'cost_usd'), getMetric(metrics, 'evaluators.reward')) + ).toEqual(['A', 'B']); + }); + + it('drops points missing either selected metric', () => { + const points = [ + point({ name: 'A', cost: 1, latency: 1 }), + point({ name: 'B', cost: 2 }), // no latency -> excluded + ]; + const metrics = deriveParetoMetrics(points); + const plotted = buildParetoPoints( + points, + getMetric(metrics, 'cost_usd'), + getMetric(metrics, 'latency_ms') + ); + expect(plotted.map((p) => p.name)).toEqual(['A']); + }); +}); diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.ts b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.ts new file mode 100644 index 0000000000..f31a367ed0 --- /dev/null +++ b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.ts @@ -0,0 +1,117 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; + +/** Which direction on an axis counts as "better": cost/latency minimize, evaluator scores maximize. */ +export type MetricDirection = 'min' | 'max'; + +export interface ParetoMetric { + /** + * Stable id, using the same metric vocabulary the API stores in `group.pareto` and the list + * sort/filter fields: `cost_usd`, `latency_ms`, or `evaluators.`. + */ + readonly id: string; + readonly label: string; + readonly direction: MetricDirection; + readonly accessor: (row: EvaluationRow) => number | null | undefined; +} + +const capitalize = (value: string): string => + value ? value.charAt(0).toUpperCase() + value.slice(1) : value; + +/** + * The display label for a metric id, resolvable synchronously from the id alone (no loaded points + * needed): `cost_usd` -> "Cost (USD)", `latency_ms` -> "Latency (ms)", `evaluators.` -> the + * capitalized evaluator name. Used so a saved evaluator axis renders its real label immediately + * instead of flashing the cost/latency fallback while the chart data loads. + */ +export function metricLabel(id: string): string { + if (id === 'cost_usd') return 'Cost (USD)'; + if (id === 'latency_ms') return 'Latency (ms)'; + return capitalize(id.startsWith('evaluators.') ? id.slice('evaluators.'.length) : id); +} + +const COST_METRIC: ParetoMetric = { + id: 'cost_usd', + label: metricLabel('cost_usd'), + direction: 'min', + accessor: (row) => row.cost_usd?.mean, +}; + +const LATENCY_METRIC: ParetoMetric = { + id: 'latency_ms', + label: metricLabel('latency_ms'), + direction: 'min', + accessor: (row) => row.latency_ms?.mean, +}; + +/** + * The metrics a user may plot on either axis: cost and latency (always present, minimized), plus one + * option per evaluator seen across the group's evaluations (maximized). Evaluator names are dynamic — + * they differ per customer — so they're derived from the data rather than hardcoded. + */ +export function deriveParetoMetrics(rows: readonly EvaluationRow[]): ParetoMetric[] { + const evaluatorNames = [ + ...new Set(rows.flatMap((row) => Object.keys(row.aggregate_scores ?? {}))), + ].sort(); + const evaluatorMetrics = evaluatorNames.map((name) => ({ + id: `evaluators.${name}`, + label: metricLabel(`evaluators.${name}`), + direction: 'max', + accessor: (row) => row.aggregate_scores?.[name]?.mean, + })); + return [COST_METRIC, LATENCY_METRIC, ...evaluatorMetrics]; +} + +export interface ParetoPlotPoint { + readonly name: string; + readonly x: number; + readonly y: number; + /** True when no other evaluation dominates this one on both axes. */ + readonly onFrontier: boolean; +} + +interface Coords { + readonly x: number; + readonly y: number; +} + +/** Whether `b` dominates `a`: at least as good on both axes and strictly better on at least one. */ +function dominates(a: Coords, b: Coords, xDir: MetricDirection, yDir: MetricDirection): boolean { + const atLeastAsGood = (av: number, bv: number, dir: MetricDirection): boolean => + dir === 'min' ? bv <= av : bv >= av; + const strictlyBetter = (av: number, bv: number, dir: MetricDirection): boolean => + dir === 'min' ? bv < av : bv > av; + return ( + atLeastAsGood(a.x, b.x, xDir) && + atLeastAsGood(a.y, b.y, yDir) && + (strictlyBetter(a.x, b.x, xDir) || strictlyBetter(a.y, b.y, yDir)) + ); +} + +/** + * Build plot points for two metrics and flag which lie on the Pareto frontier — the evaluations not + * dominated by any other on both axes. Points missing either metric (non-finite) are dropped. + */ +export function buildParetoPoints( + rows: readonly EvaluationRow[], + xMetric: ParetoMetric, + yMetric: ParetoMetric +): ParetoPlotPoint[] { + const coords = rows + .map((row): { name: string; x: number; y: number } | null => { + const x = xMetric.accessor(row); + const y = yMetric.accessor(row); + if (x == null || y == null || !Number.isFinite(x) || !Number.isFinite(y)) return null; + return { name: row.name, x, y }; + }) + .filter((point): point is { name: string; x: number; y: number } => point !== null); + + return coords.map((point) => ({ + ...point, + onFrontier: !coords.some( + (other) => other !== point && dominates(point, other, xMetric.direction, yMetric.direction) + ), + })); +} diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts new file mode 100644 index 0000000000..475aca56a9 --- /dev/null +++ b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { useListEvaluations } from '@nemo/sdk/generated/platform/api'; +import type { EvaluationFilter } from '@nemo/sdk/generated/platform/schema'; +import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; +import { useMemo } from 'react'; + +/** The list endpoint caps page_size at 1000; a group's evaluation set is far smaller, so one page + * covers every point the Pareto chart needs (the leaderboard, by contrast, is paginated). */ +const MAX_EVALUATIONS = 1000; + +export interface ParetoEvaluations { + rows: EvaluationRow[]; + isLoading: boolean; + isError: boolean; +} + +/** + * Loads every evaluation in a group in one unpaginated request for the Pareto chart. Reuses the + * existing list endpoint — each evaluation already carries the cost/latency/evaluator rollup means the + * chart plots — so no dedicated endpoint is needed. + */ +export function useParetoEvaluations( + workspace: string, + experimentGroupId: string, + options?: { enabled?: boolean } +): ParetoEvaluations { + // Callers disable the fetch when they already have the whole group loaded (a small group that fit on + // the leaderboard's first page), so this all-evaluations request — which re-runs the same server-side + // rollup — is skipped entirely. + const enabled = (options?.enabled ?? true) && !!experimentGroupId; + const { data, isLoading, isError } = useListEvaluations( + workspace, + { + page: 1, + page_size: MAX_EVALUATIONS, + filter: { experiment_group_id: experimentGroupId } as EvaluationFilter, + }, + { query: { enabled } } + ); + + const rows = useMemo( + () => + (data?.data ?? []).map((evaluation) => ({ + ...evaluation, + id: evaluation.id ?? evaluation.name ?? '', + })), + [data] + ); + + return { rows, isLoading, isError }; +} diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx index f584d426df..b4c39b277b 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx @@ -22,6 +22,7 @@ import type { ExperimentGroupResponse, } from '@nemo/sdk/generated/platform/schema'; import { Button, Text, Tooltip } from '@nvidia/foundations-react-core'; +import { ExperimentGroupParetoChart } from '@studio/components/charts/ExperimentGroupParetoChart'; import { AddToGroupModal } from '@studio/components/dataViews/ExperimentGroupDataView/AddToGroupModal'; import '@studio/components/dataViews/ExperimentGroupDataView/ExperimentGroupDataView.css'; import { Empty } from '@studio/components/dataViews/ExperimentGroupDataView/Empty'; @@ -129,10 +130,16 @@ interface ExperimentGroupDataViewProps { /** The loaded group, so the table's initial sort can seed from `default_sort` at first * render — the sorting state is initialized once and not reactive. */ group: ExperimentGroupResponse; + /** Whether the Pareto (cost-vs-accuracy) chart is shown. The toggle lives in the parent + * route header (next to the "Evaluations" title); this component only renders the chart. */ + paretoVisible: boolean; } /** Lists the experiments that belong to a single experiment group. */ -export const ExperimentGroupDataView: FC = ({ group }) => { +export const ExperimentGroupDataView: FC = ({ + group, + paretoVisible, +}) => { const workspace = useWorkspaceFromPath(); const navigate = useNavigate(); const toast = useToast(); @@ -453,8 +460,36 @@ export const ExperimentGroupDataView: FC = ({ grou return ; } + // When the whole group fits on the leaderboard's first page and isn't filtered, those loaded rows are + // the complete evaluation set — reuse them for the Pareto chart instead of refetching every + // evaluation. `evaluation_count` lets us decide this without waiting on the list query. While the + // page loads, `preloadPending` keeps the chart in its loading state; otherwise the chart fetches its + // own data in parallel. + const hasActiveFilter = Object.keys(dataViewState.apiFilter.filter ?? {}).length > 0; + const groupFitsOnePage = + group.evaluation_count != null && + group.evaluation_count <= pageSize && + page === 1 && + !dataViewState.debouncedSearchBar && + !hasActiveFilter; + const completeEvaluationSet = groupFitsOnePage && !isLoading ? orderedData : undefined; + const preloadPending = groupFitsOnePage && isLoading; + return ( <> + {paretoVisible && ( +
+ {/* Key by group id so the axis selection resets (re-seeds from the new group's saved + config) when navigating between groups without a route remount. */} + +
+ )} { @@ -22,6 +23,13 @@ export const ExperimentGroupDetailRoute: FC = () => { const { data: group, error } = useGetExperimentGroup(workspace, experimentGroupName); const [editOpen, setEditOpen] = useState(false); + // Pareto (cost-vs-accuracy) view visibility, persisted per group. Hidden by default. + const [storedParetoVisible, setParetoVisible] = useLocalStorage( + `nemo-studio:experiment-group-pareto:${group?.id ?? ''}`, + false + ); + const paretoVisible = storedParetoVisible ?? false; + useBreadcrumbs({ items: [ { href: getExperimentRoute(workspace), slotLabel: 'Experiment Groups' }, @@ -59,13 +67,18 @@ export const ExperimentGroupDetailRoute: FC = () => {
Evaluations - {group?.evaluation_count !== undefined && ( - - {group.evaluation_count} - + {group && ( + )}
- {group && } + {group && }
)}