From a6544127018e8f4f21753e916ea785a96cab2297 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:10:20 -0600 Subject: [PATCH 01/11] Create Pareto chart view Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- openapi/ga/individual/platform.openapi.yaml | 127 ++ openapi/ga/openapi.yaml | 127 ++ openapi/openapi.yaml | 127 ++ plugins/nemo-customizer/openapi/openapi.yaml | 1402 +++++++++++++---- plugins/nemo-deployments/openapi/openapi.yaml | 11 + plugins/nemo-evaluator/openapi/openapi.yaml | 26 +- .../nemo-platform/.nmpcontext/openapi.yaml | 127 ++ .../nemo-platform/.nmpcontext/stainless.yaml | 4 + .../resources/experiment_groups/api.md | 4 + .../experiment_groups/experiment_groups.py | 142 ++ .../types/experiment_groups/__init__.py | 4 + .../experiment_group_create_params.py | 11 + .../experiment_group_response.py | 10 + .../experiment_group_update_params.py | 10 + .../types/experiment_groups/pareto_config.py | 37 + .../experiment_groups/pareto_config_param.py | 37 + .../experiment_groups/pareto_data_response.py | 44 + .../experiment_groups/pareto_metric_point.py | 41 + .../inference/container_executor_config.py | 9 +- .../container_executor_config_param.py | 9 +- .../api_resources/test_experiment_groups.py | 121 ++ sdk/stainless.yaml | 4 + .../nmp/core/auth/assets/static-authz.yaml | 7 + .../intake/api/v2/experiments/endpoints.py | 84 + .../nmp/intake/api/v2/experiments/schemas.py | 29 +- .../src/nmp/intake/entities/experiments.py | 29 +- .../integration/test_experiments_crud.py | 63 + .../ExperimentGroupParetoChart/index.tsx | 293 ++++ .../paretoMetrics.test.ts | 101 ++ .../paretoMetrics.ts | 117 ++ .../ExperimentGroupDataView/index.tsx | 48 +- 31 files changed, 2900 insertions(+), 305 deletions(-) create mode 100644 sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config_param.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_data_response.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_metric_point.py create mode 100644 web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx create mode 100644 web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.test.ts create mode 100644 web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.ts diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index 9be005085e..b816826e0c 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -4213,6 +4213,56 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto: + get: + tags: + - Experiment Groups + summary: Get Experiment Group Pareto + description: 'Cost/latency/evaluator means for every evaluation in the group, + plus the group''s default axes. + + + Purpose-built for the Pareto chart: a slim, unpaginated projection of the + same rollups the + + leaderboard shows, so the client plots the full point set in one call and + computes the frontier + + from any two metrics without refetching (and without paging the full evaluations + list).' + operationId: get_experiment_group_pareto_apis_intake_v2_workspaces__workspace__experiment_groups__name__pareto_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ParetoDataResponse' + '404': + description: Experiment group not found + '413': + description: Group exceeds the per-request evaluation cap + '503': + description: Telemetry store unavailable for metric data + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/intake/v2/workspaces/{workspace}/ingest/atif: post: tags: @@ -11270,6 +11320,11 @@ components: Accepts any field the evaluations list `sort` param does; clients apply it as the list `sort` param.' default: -created_at + pareto: + allOf: + - $ref: '#/components/schemas/ParetoConfig' + description: Default X/Y metrics for the group's Pareto view. Defaults to + cost vs. latency. additionalProperties: false type: object required: @@ -11304,6 +11359,8 @@ components: default_sort: type: string title: Default Sort + pareto: + $ref: '#/components/schemas/ParetoConfig' created_at: title: Created At type: string @@ -15384,6 +15441,76 @@ components: - recipe title: PangeaRailOptions description: Configuration data for the Pangea AI Guard API + ParetoConfig: + properties: + x_metric: + type: string + title: X Metric + description: Metric plotted on the Pareto X axis. + default: cost_usd + y_metric: + type: string + title: Y Metric + description: Metric plotted on the Pareto Y axis. + default: latency_ms + type: object + title: ParetoConfig + description: "Default X/Y metrics for a group's cost-vs-accuracy Pareto view.\n\ + \nMetric ids use the same vocabulary as the evaluations list sort/filter fields\ + \ \u2014 ``cost_usd``,\n``latency_ms``, or ``evaluators.``. Defaults\ + \ to cost (x) vs latency (y): both exist for\nevery group, so the chart always\ + \ has something to render before anyone customizes it." + ParetoDataResponse: + properties: + pareto: + allOf: + - $ref: '#/components/schemas/ParetoConfig' + description: The group's configured default X/Y metrics. + points: + items: + $ref: '#/components/schemas/ParetoMetricPoint' + type: array + title: Points + description: One point per live evaluation in the group. + type: object + required: + - pareto + - points + title: ParetoDataResponse + description: "Everything the Pareto chart needs for a group: the configured\ + \ default axes plus one point per\nevaluation (cost/latency/evaluator means).\ + \ Unpaginated and slim \u2014 the client plots the whole set\nand computes\ + \ the frontier from any two metrics without refetching." + ParetoMetricPoint: + properties: + name: + type: string + title: Name + description: "Evaluation name \u2014 the leaderboard row id and rollup key." + evaluation_id: + type: string + title: Evaluation Id + description: Evaluation entity id, for navigation. + cost_usd: + title: Cost Usd + description: Mean cost (USD) across the evaluation's runs. + type: number + latency_ms: + title: Latency Ms + description: Mean latency (ms) across the evaluation's runs. + type: number + evaluators: + additionalProperties: + type: number + type: object + title: Evaluators + description: Per-evaluator mean score, keyed by evaluator name. + type: object + required: + - name + - evaluation_id + title: ParetoMetricPoint + description: One evaluation's plottable metric means for the Pareto view. PatronusEvaluateApiParams: properties: success_strategy: diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index 9be005085e..b816826e0c 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -4213,6 +4213,56 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto: + get: + tags: + - Experiment Groups + summary: Get Experiment Group Pareto + description: 'Cost/latency/evaluator means for every evaluation in the group, + plus the group''s default axes. + + + Purpose-built for the Pareto chart: a slim, unpaginated projection of the + same rollups the + + leaderboard shows, so the client plots the full point set in one call and + computes the frontier + + from any two metrics without refetching (and without paging the full evaluations + list).' + operationId: get_experiment_group_pareto_apis_intake_v2_workspaces__workspace__experiment_groups__name__pareto_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ParetoDataResponse' + '404': + description: Experiment group not found + '413': + description: Group exceeds the per-request evaluation cap + '503': + description: Telemetry store unavailable for metric data + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/intake/v2/workspaces/{workspace}/ingest/atif: post: tags: @@ -11270,6 +11320,11 @@ components: Accepts any field the evaluations list `sort` param does; clients apply it as the list `sort` param.' default: -created_at + pareto: + allOf: + - $ref: '#/components/schemas/ParetoConfig' + description: Default X/Y metrics for the group's Pareto view. Defaults to + cost vs. latency. additionalProperties: false type: object required: @@ -11304,6 +11359,8 @@ components: default_sort: type: string title: Default Sort + pareto: + $ref: '#/components/schemas/ParetoConfig' created_at: title: Created At type: string @@ -15384,6 +15441,76 @@ components: - recipe title: PangeaRailOptions description: Configuration data for the Pangea AI Guard API + ParetoConfig: + properties: + x_metric: + type: string + title: X Metric + description: Metric plotted on the Pareto X axis. + default: cost_usd + y_metric: + type: string + title: Y Metric + description: Metric plotted on the Pareto Y axis. + default: latency_ms + type: object + title: ParetoConfig + description: "Default X/Y metrics for a group's cost-vs-accuracy Pareto view.\n\ + \nMetric ids use the same vocabulary as the evaluations list sort/filter fields\ + \ \u2014 ``cost_usd``,\n``latency_ms``, or ``evaluators.``. Defaults\ + \ to cost (x) vs latency (y): both exist for\nevery group, so the chart always\ + \ has something to render before anyone customizes it." + ParetoDataResponse: + properties: + pareto: + allOf: + - $ref: '#/components/schemas/ParetoConfig' + description: The group's configured default X/Y metrics. + points: + items: + $ref: '#/components/schemas/ParetoMetricPoint' + type: array + title: Points + description: One point per live evaluation in the group. + type: object + required: + - pareto + - points + title: ParetoDataResponse + description: "Everything the Pareto chart needs for a group: the configured\ + \ default axes plus one point per\nevaluation (cost/latency/evaluator means).\ + \ Unpaginated and slim \u2014 the client plots the whole set\nand computes\ + \ the frontier from any two metrics without refetching." + ParetoMetricPoint: + properties: + name: + type: string + title: Name + description: "Evaluation name \u2014 the leaderboard row id and rollup key." + evaluation_id: + type: string + title: Evaluation Id + description: Evaluation entity id, for navigation. + cost_usd: + title: Cost Usd + description: Mean cost (USD) across the evaluation's runs. + type: number + latency_ms: + title: Latency Ms + description: Mean latency (ms) across the evaluation's runs. + type: number + evaluators: + additionalProperties: + type: number + type: object + title: Evaluators + description: Per-evaluator mean score, keyed by evaluator name. + type: object + required: + - name + - evaluation_id + title: ParetoMetricPoint + description: One evaluation's plottable metric means for the Pareto view. PatronusEvaluateApiParams: properties: success_strategy: diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 9be005085e..b816826e0c 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -4213,6 +4213,56 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto: + get: + tags: + - Experiment Groups + summary: Get Experiment Group Pareto + description: 'Cost/latency/evaluator means for every evaluation in the group, + plus the group''s default axes. + + + Purpose-built for the Pareto chart: a slim, unpaginated projection of the + same rollups the + + leaderboard shows, so the client plots the full point set in one call and + computes the frontier + + from any two metrics without refetching (and without paging the full evaluations + list).' + operationId: get_experiment_group_pareto_apis_intake_v2_workspaces__workspace__experiment_groups__name__pareto_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ParetoDataResponse' + '404': + description: Experiment group not found + '413': + description: Group exceeds the per-request evaluation cap + '503': + description: Telemetry store unavailable for metric data + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/intake/v2/workspaces/{workspace}/ingest/atif: post: tags: @@ -11270,6 +11320,11 @@ components: Accepts any field the evaluations list `sort` param does; clients apply it as the list `sort` param.' default: -created_at + pareto: + allOf: + - $ref: '#/components/schemas/ParetoConfig' + description: Default X/Y metrics for the group's Pareto view. Defaults to + cost vs. latency. additionalProperties: false type: object required: @@ -11304,6 +11359,8 @@ components: default_sort: type: string title: Default Sort + pareto: + $ref: '#/components/schemas/ParetoConfig' created_at: title: Created At type: string @@ -15384,6 +15441,76 @@ components: - recipe title: PangeaRailOptions description: Configuration data for the Pangea AI Guard API + ParetoConfig: + properties: + x_metric: + type: string + title: X Metric + description: Metric plotted on the Pareto X axis. + default: cost_usd + y_metric: + type: string + title: Y Metric + description: Metric plotted on the Pareto Y axis. + default: latency_ms + type: object + title: ParetoConfig + description: "Default X/Y metrics for a group's cost-vs-accuracy Pareto view.\n\ + \nMetric ids use the same vocabulary as the evaluations list sort/filter fields\ + \ \u2014 ``cost_usd``,\n``latency_ms``, or ``evaluators.``. Defaults\ + \ to cost (x) vs latency (y): both exist for\nevery group, so the chart always\ + \ has something to render before anyone customizes it." + ParetoDataResponse: + properties: + pareto: + allOf: + - $ref: '#/components/schemas/ParetoConfig' + description: The group's configured default X/Y metrics. + points: + items: + $ref: '#/components/schemas/ParetoMetricPoint' + type: array + title: Points + description: One point per live evaluation in the group. + type: object + required: + - pareto + - points + title: ParetoDataResponse + description: "Everything the Pareto chart needs for a group: the configured\ + \ default axes plus one point per\nevaluation (cost/latency/evaluator means).\ + \ Unpaginated and slim \u2014 the client plots the whole set\nand computes\ + \ the frontier from any two metrics without refetching." + ParetoMetricPoint: + properties: + name: + type: string + title: Name + description: "Evaluation name \u2014 the leaderboard row id and rollup key." + evaluation_id: + type: string + title: Evaluation Id + description: Evaluation entity id, for navigation. + cost_usd: + title: Cost Usd + description: Mean cost (USD) across the evaluation's runs. + type: number + latency_ms: + title: Latency Ms + description: Mean latency (ms) across the evaluation's runs. + type: number + evaluators: + additionalProperties: + type: number + type: object + title: Evaluators + description: Per-evaluator mean score, keyed by evaluator name. + type: object + required: + - name + - evaluation_id + title: ParetoMetricPoint + description: One evaluation's plottable metric means for the Pareto view. PatronusEvaluateApiParams: properties: success_strategy: diff --git a/plugins/nemo-customizer/openapi/openapi.yaml b/plugins/nemo-customizer/openapi/openapi.yaml index 2b7724e0aa..d852b91bc8 100644 --- a/plugins/nemo-customizer/openapi/openapi.yaml +++ b/plugins/nemo-customizer/openapi/openapi.yaml @@ -392,12 +392,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs: + /apis/customization/v2/workspaces/{workspace}/rl/jobs: post: tags: - - Unsloth Jobs + - Rl Jobs summary: Create Job - operationId: create_job_apis_customization_v2_workspaces__workspace__unsloth_jobs_post + operationId: create_job_apis_customization_v2_workspaces__workspace__rl_jobs_post parameters: - name: workspace in: path @@ -410,14 +410,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/UnslothJobsJobRequest' + $ref: '#/components/schemas/RlJobsJobRequest' responses: '201': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/UnslothJobsJob' + $ref: '#/components/schemas/RlJobsJob' '422': description: Validation Error content: @@ -426,9 +426,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' get: tags: - - Unsloth Jobs + - Rl Jobs summary: List Jobs - operationId: list_jobs_apis_customization_v2_workspaces__workspace__unsloth_jobs_get + operationId: list_jobs_apis_customization_v2_workspaces__workspace__rl_jobs_get parameters: - name: workspace in: path @@ -461,7 +461,7 @@ paths: required: false schema: allOf: - - $ref: '#/components/schemas/UnslothJobsJobsSortField' + - $ref: '#/components/schemas/RlJobsJobsSortField' description: The field to sort by. To sort in decreasing order, use `-` in front of the field name. default: -created_at @@ -473,7 +473,7 @@ paths: required: false explode: true schema: - $ref: '#/components/schemas/UnslothJobsJobsListFilter' + $ref: '#/components/schemas/RlJobsJobsListFilter' description: Filter jobs on various criteria. responses: '200': @@ -481,19 +481,19 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/UnslothJobsJobsPage' + $ref: '#/components/schemas/RlJobsJobsPage' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{job}/results/{name}: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{job}/results/{name}: get: tags: - - Unsloth Jobs + - Rl Jobs summary: Get Job Result - operationId: get_job_result_apis_customization_v2_workspaces__workspace__unsloth_jobs__job__results__name__get + operationId: get_job_result_apis_customization_v2_workspaces__workspace__rl_jobs__job__results__name__get parameters: - name: workspace in: path @@ -526,12 +526,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{job}/results/{name}/download: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{job}/results/{name}/download: get: tags: - - Unsloth Jobs + - Rl Jobs summary: Download Job Result - operationId: download_job_result_apis_customization_v2_workspaces__workspace__unsloth_jobs__job__results__name__download_get + operationId: download_job_result_apis_customization_v2_workspaces__workspace__rl_jobs__job__results__name__download_get parameters: - name: workspace in: path @@ -567,12 +567,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{name}: get: tags: - - Unsloth Jobs + - Rl Jobs summary: Get Job - operationId: get_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__get + operationId: get_job_apis_customization_v2_workspaces__workspace__rl_jobs__name__get parameters: - name: workspace in: path @@ -592,7 +592,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/UnslothJobsJob' + $ref: '#/components/schemas/RlJobsJob' '422': description: Validation Error content: @@ -601,9 +601,9 @@ paths: $ref: '#/components/schemas/HTTPValidationError' delete: tags: - - Unsloth Jobs + - Rl Jobs summary: Delete Job - operationId: delete_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__delete + operationId: delete_job_apis_customization_v2_workspaces__workspace__rl_jobs__name__delete parameters: - name: workspace in: path @@ -626,12 +626,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/cancel: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{name}/cancel: post: tags: - - Unsloth Jobs + - Rl Jobs summary: Cancel Job - operationId: cancel_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__cancel_post + operationId: cancel_job_apis_customization_v2_workspaces__workspace__rl_jobs__name__cancel_post parameters: - name: workspace in: path @@ -651,19 +651,19 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/UnslothJobsJob' + $ref: '#/components/schemas/RlJobsJob' '422': description: Validation Error content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/logs: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{name}/logs: get: tags: - - Unsloth Jobs + - Rl Jobs summary: Get Job Logs - operationId: get_job_logs_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__logs_get + operationId: get_job_logs_apis_customization_v2_workspaces__workspace__rl_jobs__name__logs_get parameters: - name: workspace in: path @@ -702,12 +702,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/results: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{name}/results: get: tags: - - Unsloth Jobs + - Rl Jobs summary: List Job Results - operationId: list_job_results_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__results_get + operationId: list_job_results_apis_customization_v2_workspaces__workspace__rl_jobs__name__results_get parameters: - name: workspace in: path @@ -734,12 +734,12 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/status: + /apis/customization/v2/workspaces/{workspace}/rl/jobs/{name}/status: get: tags: - - Unsloth Jobs + - Rl Jobs summary: Get Job Status - operationId: get_job_status_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__status_get + operationId: get_job_status_apis_customization_v2_workspaces__workspace__rl_jobs__name__status_get parameters: - name: workspace in: path @@ -766,89 +766,463 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' -components: - schemas: - AutomodelBatchSpec: - properties: - global_batch_size: - type: integer - exclusiveMinimum: 0.0 - title: Global Batch Size - default: 8 - micro_batch_size: + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs: + post: + tags: + - Unsloth Jobs + summary: Create Job + operationId: create_job_apis_customization_v2_workspaces__workspace__unsloth_jobs_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/UnslothJobsJobRequest' + responses: + '201': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/UnslothJobsJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + get: + tags: + - Unsloth Jobs + summary: List Jobs + operationId: list_jobs_apis_customization_v2_workspaces__workspace__unsloth_jobs_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: type: integer - exclusiveMinimum: 0.0 - title: Micro Batch Size + exclusiveMinimum: 0 + description: Page number. default: 1 - sequence_packing: - type: boolean - title: Sequence Packing - default: false - sequence_packing_max_samples: + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: type: integer - exclusiveMinimum: 0.0 - title: Sequence Packing Max Samples - description: Samples analyzed to estimate the optimal pack size when packing - is enabled. - default: 1000 - additionalProperties: false - type: object - title: AutomodelBatchSpec - AutomodelDatasetSpec: - properties: - training: + exclusiveMinimum: 0 + description: Page size. + default: 10 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + allOf: + - $ref: '#/components/schemas/UnslothJobsJobsSortField' + description: The field to sort by. To sort in decreasing order, use `-` + in front of the field name. + default: -created_at + description: The field to sort by. To sort in decreasing order, use `-` in + front of the field name. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/UnslothJobsJobsListFilter' + description: Filter jobs on various criteria. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/UnslothJobsJobsPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{job}/results/{name}: + get: + tags: + - Unsloth Jobs + summary: Get Job Result + operationId: get_job_result_apis_customization_v2_workspaces__workspace__unsloth_jobs__job__results__name__get + parameters: + - name: workspace + in: path + required: true + schema: type: string - title: Training - description: Training fileset as 'name' or 'workspace/name'. - validation: - title: Validation + title: Workspace + - name: job + in: path + required: true + schema: type: string - prompt_template: - title: Prompt Template + title: Job + - name: name + in: path + required: true + schema: type: string - additionalProperties: false - type: object - required: - - training - title: AutomodelDatasetSpec - AutomodelJobInput: - properties: - name: title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobResultResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{job}/results/{name}/download: + get: + tags: + - Unsloth Jobs + summary: Download Job Result + operationId: download_job_result_apis_customization_v2_workspaces__workspace__unsloth_jobs__job__results__name__download_get + parameters: + - name: workspace + in: path + required: true + schema: type: string - model: + title: Workspace + - name: job + in: path + required: true + schema: type: string - title: Model - dataset: - $ref: '#/components/schemas/AutomodelDatasetSpec' - training: - $ref: '#/components/schemas/AutomodelTrainingSpec' - schedule: - $ref: '#/components/schemas/AutomodelScheduleSpec' - batch: - $ref: '#/components/schemas/AutomodelBatchSpec' - optimizer: - $ref: '#/components/schemas/AutomodelOptimizerSpec' - parallelism: - $ref: '#/components/schemas/AutomodelParallelismSpec' - output: - $ref: '#/components/schemas/AutomodelOutputRequest' - integrations: - $ref: '#/components/schemas/IntegrationsSpecInput' - additionalProperties: false - type: object - required: - - model - - dataset - - training - title: AutomodelJobInput - description: POST body / CLI JSON. - AutomodelJobOutput: - properties: - name: - title: Name + title: Job + - name: name + in: path + required: true + schema: type: string - model: + title: Name + responses: + '200': + description: Successful Response + content: + application/octet-stream: + schema: + type: string + format: binary + '404': + description: Not Found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}: + get: + tags: + - Unsloth Jobs + summary: Get Job + operationId: get_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/UnslothJobsJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Unsloth Jobs + summary: Delete Job + operationId: delete_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '204': + description: Successful Response + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/cancel: + post: + tags: + - Unsloth Jobs + summary: Cancel Job + operationId: cancel_job_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__cancel_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/UnslothJobsJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/logs: + get: + tags: + - Unsloth Jobs + summary: Get Job Logs + operationId: get_job_logs_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__logs_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + - name: limit + in: query + required: false + schema: + title: Limit + type: integer + - name: page_cursor + in: query + required: false + schema: + title: Page Cursor + type: string + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobLogPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/results: + get: + tags: + - Unsloth Jobs + summary: List Job Results + operationId: list_job_results_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__results_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobListResultResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/customization/v2/workspaces/{workspace}/unsloth/jobs/{name}/status: + get: + tags: + - Unsloth Jobs + summary: Get Job Status + operationId: get_job_status_apis_customization_v2_workspaces__workspace__unsloth_jobs__name__status_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobStatusResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' +components: + schemas: + AutomodelBatchSpec: + properties: + global_batch_size: + type: integer + exclusiveMinimum: 0.0 + title: Global Batch Size + default: 8 + micro_batch_size: + type: integer + exclusiveMinimum: 0.0 + title: Micro Batch Size + default: 1 + sequence_packing: + type: boolean + title: Sequence Packing + default: false + sequence_packing_max_samples: + type: integer + exclusiveMinimum: 0.0 + title: Sequence Packing Max Samples + description: Samples analyzed to estimate the optimal pack size when packing + is enabled. + default: 1000 + additionalProperties: false + type: object + title: AutomodelBatchSpec + AutomodelDatasetSpec: + properties: + training: + type: string + title: Training + description: Training fileset as 'name' or 'workspace/name'. + validation: + title: Validation + type: string + prompt_template: + title: Prompt Template + type: string + additionalProperties: false + type: object + required: + - training + title: AutomodelDatasetSpec + AutomodelJobInput: + properties: + name: + title: Name + type: string + model: + type: string + title: Model + dataset: + $ref: '#/components/schemas/AutomodelDatasetSpec' + training: + $ref: '#/components/schemas/AutomodelTrainingSpec' + schedule: + $ref: '#/components/schemas/AutomodelScheduleSpec' + batch: + $ref: '#/components/schemas/AutomodelBatchSpec' + optimizer: + $ref: '#/components/schemas/AutomodelOptimizerSpec' + parallelism: + $ref: '#/components/schemas/AutomodelParallelismSpec' + output: + $ref: '#/components/schemas/AutomodelOutputRequest' + integrations: + $ref: '#/components/schemas/IntegrationsSpecInput' + additionalProperties: false + type: object + required: + - model + - dataset + - training + title: AutomodelJobInput + description: POST body / CLI JSON. + AutomodelJobOutput: + properties: + name: + title: Name + type: string + model: type: string title: Model dataset: @@ -1401,6 +1775,23 @@ components: To enable MLflow, provide a non-null ``mlflow`` object on :class:`IntegrationsSpec`.' + OptimizerType: + type: string + enum: + - adamw_with_cosine_annealing + - adam_with_cosine_annealing + - adamw_with_flat_lr + - adam_with_flat_lr + title: OptimizerType + description: Optimizer and scheduler combination types. + OutputNameType: + type: string + enum: + - adapter + - model + title: OutputNameType + description: "Output artifact type \u2014 adapter (LoRA only) or model (merged\ + \ / full)." PaginationData: properties: page: @@ -1425,254 +1816,701 @@ components: description: The total number of results. type: object required: - - page - - page_size - - current_page_size - - total_pages - - total_results - title: PaginationData - PlatformJobListResultResponse: + - page + - page_size + - current_page_size + - total_pages + - total_results + title: PaginationData + PlatformJobListResultResponse: + properties: + data: + items: + $ref: '#/components/schemas/PlatformJobResultResponse' + type: array + title: Data + type: object + required: + - data + title: PlatformJobListResultResponse + PlatformJobLog: + properties: + timestamp: + type: string + format: date-time + title: Timestamp + job: + type: string + title: Job + job_step: + type: string + title: Job Step + job_task: + type: string + title: Job Task + message: + type: string + title: Message + type: object + required: + - timestamp + - job + - job_step + - job_task + - message + title: PlatformJobLog + PlatformJobLogPage: + properties: + data: + items: + $ref: '#/components/schemas/PlatformJobLog' + type: array + title: Data + total: + type: integer + title: Total + next_page: + title: Next Page + type: string + prev_page: + title: Prev Page + type: string + type: object + required: + - data + - total + - next_page + - prev_page + title: PlatformJobLogPage + PlatformJobResultResponse: + properties: + name: + type: string + title: Name + job: + type: string + title: Job + workspace: + type: string + title: Workspace + project: + title: Project + type: string + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + artifact_url: + type: string + title: Artifact Url + artifact_storage_type: + $ref: '#/components/schemas/FileStorageType' + download_url: + title: Download Url + type: string + type: object + required: + - name + - job + - workspace + - artifact_url + - artifact_storage_type + title: PlatformJobResultResponse + PlatformJobStatus: + type: string + enum: + - created + - pending + - active + - cancelled + - cancelling + - error + - completed + - paused + - pausing + - resuming + title: PlatformJobStatus + description: 'Enumeration of possible job statuses. + + + This enum represents the various states a job can be in during its lifecycle, + + from creation to a terminal state.' + PlatformJobStatusResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + steps: + items: + $ref: '#/components/schemas/PlatformJobStepStatusResponse' + type: array + title: Steps + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + type: object + required: + - id + - name + - status + - status_details + - error_details + - steps + - created_at + - updated_at + title: PlatformJobStatusResponse + PlatformJobStepStatusResponse: properties: - data: + id: + type: string + title: Id + name: + type: string + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + tasks: items: - $ref: '#/components/schemas/PlatformJobResultResponse' + $ref: '#/components/schemas/PlatformJobTaskStatusResponse' type: array - title: Data + title: Tasks + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At type: object required: - - data - title: PlatformJobListResultResponse - PlatformJobLog: + - id + - name + - status + - status_details + - error_details + - tasks + - created_at + - updated_at + title: PlatformJobStepStatusResponse + PlatformJobTaskStatusResponse: properties: - timestamp: + id: type: string - format: date-time - title: Timestamp - job: + title: Id + name: type: string - title: Job - job_step: + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + error_stack: + title: Error Stack type: string - title: Job Step - job_task: + created_at: type: string - title: Job Task - message: + format: date-time + title: Created At + updated_at: type: string - title: Message + format: date-time + title: Updated At type: object required: - - timestamp - - job - - job_step - - job_task - - message - title: PlatformJobLog - PlatformJobLogPage: + - id + - name + - status + - status_details + - error_details + - error_stack + - created_at + - updated_at + title: PlatformJobTaskStatusResponse + RlDPOTraining: properties: - data: - items: - $ref: '#/components/schemas/PlatformJobLog' - type: array - title: Data - total: + optimizer_type: + allOf: + - $ref: '#/components/schemas/OptimizerType' + description: "Optimizer + LR-scheduler combination (AdamW/Adam \xD7 cosine-annealing/flat-LR).\ + \ Defaults to AdamW with cosine annealing." + learning_rate: + type: number + title: Learning Rate + description: Peak learning rate. + default: 0.0001 + min_learning_rate: + title: Min Learning Rate + description: Minimum LR for cosine decay. + type: number + weight_decay: + type: number + title: Weight Decay + description: Weight decay coefficient. + default: 0.01 + adam_beta1: + type: number + title: Adam Beta1 + description: Adam beta1. + default: 0.9 + adam_beta2: + type: number + title: Adam Beta2 + description: Adam beta2. + default: 0.999 + adam_eps: + type: number + exclusiveMinimum: 0.0 + title: Adam Eps + description: Adam epsilon (numerical stability term). + default: 1.0e-05 + warmup_steps: type: integer - title: Total - next_page: - title: Next Page + minimum: 0.0 + title: Warmup Steps + description: Linear warmup steps. + default: 0 + epochs: + type: integer + exclusiveMinimum: 0.0 + title: Epochs + description: Number of passes through the dataset. + default: 1 + max_steps: + title: Max Steps + description: Max training steps (overrides epochs if set). + type: integer + exclusiveMinimum: 0.0 + val_check_interval: + title: Val Check Interval + description: Validation interval. Float <= 1.0 is fraction of epoch; > 1.0 + is step count. + type: number + val_at_end: + type: boolean + title: Val At End + description: Run a final validation pass after the last training step. Keep + enabled so the final checkpoint carries validation metrics and best-checkpoint + selection works; set False only to skip the extra eval. + default: true + keep_top_k: + type: integer + exclusiveMinimum: 0.0 + title: Keep Top K + description: Number of best checkpoints to retain (ranked by validation + loss). + default: 1 + batch_size: + type: integer + exclusiveMinimum: 0.0 + title: Batch Size + description: Global batch size across all GPUs. + default: 32 + micro_batch_size: + type: integer + exclusiveMinimum: 0.0 + title: Micro Batch Size + description: Per-GPU micro batch size. + default: 1 + activation_checkpointing: + type: boolean + title: Activation Checkpointing + description: Recompute activations during the backward pass to reduce memory + at the cost of compute. Enable to fit larger models or longer sequences. + default: false + max_seq_length: + type: integer + exclusiveMinimum: 0.0 + title: Max Seq Length + description: Maximum token sequence length for training. + default: 2048 + seed: + title: Seed + description: Random seed for reproducibility. + type: integer + parallelism: + $ref: '#/components/schemas/RlParallelismParams' + execution_profile: + title: Execution Profile + description: Execution profile for the GPU training step (operator-configured). + Falls back to the service default when omitted. type: string - prev_page: - title: Prev Page + minLength: 1 + type: type: string + const: dpo + title: Type + default: dpo + ref_policy_kl_penalty: + type: number + minimum: 0.0 + title: Ref Policy Kl Penalty + description: KL penalty coefficient (beta in the DPO paper). + default: 0.05 + preference_average_log_probs: + type: boolean + title: Preference Average Log Probs + description: Average log probabilities for preference loss calculation. + default: false + sft_average_log_probs: + type: boolean + title: Sft Average Log Probs + description: Average log probabilities for SFT regularization loss. + default: false + preference_loss_weight: + type: number + minimum: 0.0 + title: Preference Loss Weight + description: Weight for the preference (DPO) loss term. + default: 1.0 + sft_loss_weight: + type: number + minimum: 0.0 + title: Sft Loss Weight + description: Weight for SFT regularization loss (0 = disabled). + default: 0.0 + max_grad_norm: + type: number + minimum: 0.0 + title: Max Grad Norm + description: Maximum gradient norm for clipping. + default: 1.0 + additionalProperties: false type: object - required: - - data - - total - - next_page - - prev_page - title: PlatformJobLogPage - PlatformJobResultResponse: + title: RlDPOTraining + description: "Direct Preference Optimization (full-weight only \u2014 PEFT unsupported)." + RlJobInput: properties: name: - type: string title: Name - job: - type: string - title: Job - workspace: type: string - title: Workspace - project: - title: Project + model: type: string - created_at: + title: Model + description: Model entity reference ('name' or 'workspace/name'). + dataset: type: string - format: date-time - title: Created At - updated_at: + title: Dataset + description: Preference dataset fileset reference. Must contain training.jsonl + + validation.jsonl. + training: + allOf: + - $ref: '#/components/schemas/RlDPOTraining' + description: DPO training method and hyperparameters. + integrations: + $ref: '#/components/schemas/IntegrationsSpecInput' + output: + $ref: '#/components/schemas/RlOutputRequest' + additionalProperties: false + type: object + required: + - model + - dataset + - training + title: RlJobInput + description: POST body / CLI JSON for ``nemo customization rl submit``. + RlJobOutput: + properties: + name: + title: Name + description: Optional job name; auto-generated when omitted. type: string - format: date-time - title: Updated At - artifact_url: + model: type: string - title: Artifact Url - artifact_storage_type: - $ref: '#/components/schemas/FileStorageType' - download_url: - title: Download Url + title: Model + description: Model entity reference ('name' or 'workspace/name'). + dataset: type: string + title: Dataset + description: Preference dataset fileset reference ('name' or 'workspace/name'). + training: + allOf: + - $ref: '#/components/schemas/RlDPOTraining' + description: Training method and hyperparameters (DPO). + integrations: + allOf: + - $ref: '#/components/schemas/IntegrationsSpecOutput' + description: W&B / MLflow integrations. + output: + allOf: + - $ref: '#/components/schemas/RlOutputResponse' + description: Output artifact created by this job. + additionalProperties: false type: object required: - - name - - job - - workspace - - artifact_url - - artifact_storage_type - title: PlatformJobResultResponse - PlatformJobStatus: - type: string - enum: - - created - - pending - - active - - cancelled - - cancelling - - error - - completed - - paused - - pausing - - resuming - title: PlatformJobStatus - description: 'Enumeration of possible job statuses. + - model + - dataset + - training + - output + title: RlJobOutput + description: 'Canonical NeMo-RL job spec (output of the plugin transform). - This enum represents the various states a job can be in during its lifecycle, + The ``dataset`` fileset must contain ``training.jsonl`` and ``validation.jsonl`` - from creation to a terminal state.' - PlatformJobStatusResponse: + (any of the four supported preference formats); the dataset-preparation step + + splits/normalizes them at runtime.' + RlJobsJob: properties: id: - type: string title: Id + type: string name: type: string title: Name + description: + title: Description + type: string + project: + title: Project + type: string + workspace: + title: Workspace + type: string + created_at: + title: Created At + type: string + format: date-time + updated_at: + title: Updated At + type: string + format: date-time + spec: + $ref: '#/components/schemas/RlJobOutput' status: $ref: '#/components/schemas/PlatformJobStatus' status_details: + title: Status Details additionalProperties: true type: object - title: Status Details error_details: title: Error Details additionalProperties: true type: object - steps: - items: - $ref: '#/components/schemas/PlatformJobStepStatusResponse' - type: array - title: Steps - created_at: - type: string - format: date-time - title: Created At - updated_at: - type: string - format: date-time - title: Updated At + ownership: + title: Ownership + additionalProperties: true + type: object + custom_fields: + title: Custom Fields + additionalProperties: true + type: object type: object required: - - id - name - - status - - status_details - - error_details - - steps - - created_at - - updated_at - title: PlatformJobStatusResponse - PlatformJobStepStatusResponse: + - spec + title: RlJobsJob + RlJobsJobRequest: properties: - id: - type: string - title: Id name: - type: string title: Name - status: - $ref: '#/components/schemas/PlatformJobStatus' - status_details: + type: string + description: + title: Description + type: string + project: + title: Project + type: string + spec: + $ref: '#/components/schemas/RlJobInput' + ownership: + title: Ownership additionalProperties: true type: object - title: Status Details - error_details: - title: Error Details + custom_fields: + title: Custom Fields additionalProperties: true type: object - tasks: - items: - $ref: '#/components/schemas/PlatformJobTaskStatusResponse' - type: array - title: Tasks + type: object + required: + - spec + title: RlJobsJobRequest + RlJobsJobsListFilter: + additionalProperties: false + properties: created_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Jobs created at 'gte' datetime or 'lte' datetime. + name: + anyOf: + - $ref: '#/components/schemas/StringFilter' + - type: string + description: Name of the job. + title: Name + workspace: + description: Workspace of the job. + title: Workspace type: string - format: date-time - title: Created At + project: + description: Project containing the job. + title: Project + type: string + status: + allOf: + - $ref: '#/components/schemas/PlatformJobStatus' + description: The current status. updated_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Jobs updated at 'gte' datetime or 'lte' datetime. + title: RlJobsJobsListFilter + type: object + RlJobsJobsPage: + properties: + data: + items: + $ref: '#/components/schemas/RlJobsJob' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. type: string - format: date-time - title: Updated At + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object type: object required: - - id - - name - - status - - status_details - - error_details - - tasks + - data + title: RlJobsJobsPage + RlJobsJobsSortField: + type: string + enum: - created_at + - -created_at - updated_at - title: PlatformJobStepStatusResponse - PlatformJobTaskStatusResponse: + - -updated_at + title: RlJobsJobsSortField + RlOutputRequest: properties: - id: - type: string - title: Id name: - type: string title: Name - status: - $ref: '#/components/schemas/PlatformJobStatus' - status_details: - additionalProperties: true - type: object - title: Status Details - error_details: - title: Error Details - additionalProperties: true - type: object - error_stack: - title: Error Stack type: string - created_at: + additionalProperties: false + type: object + title: RlOutputRequest + description: Submitter-facing output preferences. ``name`` is auto-derived if + omitted. + RlOutputResponse: + properties: + name: type: string - format: date-time - title: Created At - updated_at: + maxLength: 255 + title: Name + description: Name of the output artifact. Used to identify it during deployment + and inference. + examples: + - my-dpo-llama + type: + allOf: + - $ref: '#/components/schemas/OutputNameType' + description: Output artifact type. DPO is full-weight, so always `model`. + default: model + fileset: type: string - format: date-time - title: Updated At + maxLength: 255 + title: Fileset + description: FileSet name where output artifacts are stored. + additionalProperties: false type: object required: - - id - name - - status - - status_details - - error_details - - error_stack - - created_at - - updated_at - title: PlatformJobTaskStatusResponse + - fileset + title: RlOutputResponse + description: Resolved output artifact details. + RlParallelismParams: + properties: + num_gpus_per_node: + type: integer + exclusiveMinimum: 0.0 + title: Num Gpus Per Node + description: Number of GPUs per node. + default: 1 + num_nodes: + type: integer + exclusiveMinimum: 0.0 + title: Num Nodes + description: "Number of nodes (>1 \u2192 multi-node Ray cluster)." + default: 1 + tensor_parallel_size: + type: integer + exclusiveMinimum: 0.0 + title: Tensor Parallel Size + description: Tensor parallel size. + default: 1 + pipeline_parallel_size: + type: integer + exclusiveMinimum: 0.0 + title: Pipeline Parallel Size + description: Pipeline parallel size. + default: 1 + context_parallel_size: + type: integer + exclusiveMinimum: 0.0 + title: Context Parallel Size + description: Context parallel size. + default: 1 + sequence_parallel: + type: boolean + title: Sequence Parallel + description: Enable sequence parallelism. + default: false + additionalProperties: false + type: object + title: RlParallelismParams + description: 'Distributed training parallelism configuration. + + + Single-node multi-GPU uses ``num_nodes=1`` with ``num_gpus_per_node>1``; + + multi-node sets ``num_nodes>1`` and the compiler emits a distributed-GPU + + executor (see :mod:`nmp.rl.app.jobs.compiler`).' SecretRef: type: string pattern: ^[a-z0-9_-]+(/[a-z0-9_-]+)?$ diff --git a/plugins/nemo-deployments/openapi/openapi.yaml b/plugins/nemo-deployments/openapi/openapi.yaml index 4ae9fa9c47..2f4d67e76a 100644 --- a/plugins/nemo-deployments/openapi/openapi.yaml +++ b/plugins/nemo-deployments/openapi/openapi.yaml @@ -970,6 +970,17 @@ components: mount_point: title: Mount Point type: string + initChmod: + title: Initchmod + description: "When set, an init container chmods the freshly created (root-owned)\ + \ named volume to this mode (e.g. '0777') so a non-root workload (e.g.\ + \ the HF weight-puller) can write to it \u2014 the docker analogue of\ + \ a k8s fsGroup." + type: string + initImage: + title: Initimage + description: Image used for the init_chmod container (e.g. busybox). + type: string type: object title: DockerVolumeConfig DriftRecoveryPolicy: diff --git a/plugins/nemo-evaluator/openapi/openapi.yaml b/plugins/nemo-evaluator/openapi/openapi.yaml index 5d49d5dafe..51dc1115b1 100644 --- a/plugins/nemo-evaluator/openapi/openapi.yaml +++ b/plugins/nemo-evaluator/openapi/openapi.yaml @@ -3123,6 +3123,17 @@ components: description: Read JSON SSE data frames instead of a single JSON response body. default: false + response_aggregation: + type: string + enum: + - last + - concat + title: Response Aggregation + description: How to combine matched data-frame values when 'stream' is true. + 'last' keeps the final matched value (endpoints that emit a full snapshot + per frame); 'concat' joins matched string values in arrival order (token-delta + endpoints). + default: last additionalProperties: false type: object required: @@ -3721,9 +3732,20 @@ components: response_path: type: string title: Response Path - description: JSONPath applied to data-channel payloads; the last match is - the final output. + description: JSONPath applied to each data-channel payload to extract its + emitted value. default: $.value + response_aggregation: + type: string + enum: + - last + - concat + title: Response Aggregation + description: How to combine matched data-frame values into the final output. + NAT /generate/full emits token-level deltas, so 'concat' reconstructs + the complete response; 'last' keeps only the final matched value (for + endpoints that emit a full snapshot per frame). + default: concat additionalProperties: false type: object title: NatAgentConfig diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index 9be005085e..b816826e0c 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -4213,6 +4213,56 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto: + get: + tags: + - Experiment Groups + summary: Get Experiment Group Pareto + description: 'Cost/latency/evaluator means for every evaluation in the group, + plus the group''s default axes. + + + Purpose-built for the Pareto chart: a slim, unpaginated projection of the + same rollups the + + leaderboard shows, so the client plots the full point set in one call and + computes the frontier + + from any two metrics without refetching (and without paging the full evaluations + list).' + operationId: get_experiment_group_pareto_apis_intake_v2_workspaces__workspace__experiment_groups__name__pareto_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ParetoDataResponse' + '404': + description: Experiment group not found + '413': + description: Group exceeds the per-request evaluation cap + '503': + description: Telemetry store unavailable for metric data + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/intake/v2/workspaces/{workspace}/ingest/atif: post: tags: @@ -11270,6 +11320,11 @@ components: Accepts any field the evaluations list `sort` param does; clients apply it as the list `sort` param.' default: -created_at + pareto: + allOf: + - $ref: '#/components/schemas/ParetoConfig' + description: Default X/Y metrics for the group's Pareto view. Defaults to + cost vs. latency. additionalProperties: false type: object required: @@ -11304,6 +11359,8 @@ components: default_sort: type: string title: Default Sort + pareto: + $ref: '#/components/schemas/ParetoConfig' created_at: title: Created At type: string @@ -15384,6 +15441,76 @@ components: - recipe title: PangeaRailOptions description: Configuration data for the Pangea AI Guard API + ParetoConfig: + properties: + x_metric: + type: string + title: X Metric + description: Metric plotted on the Pareto X axis. + default: cost_usd + y_metric: + type: string + title: Y Metric + description: Metric plotted on the Pareto Y axis. + default: latency_ms + type: object + title: ParetoConfig + description: "Default X/Y metrics for a group's cost-vs-accuracy Pareto view.\n\ + \nMetric ids use the same vocabulary as the evaluations list sort/filter fields\ + \ \u2014 ``cost_usd``,\n``latency_ms``, or ``evaluators.``. Defaults\ + \ to cost (x) vs latency (y): both exist for\nevery group, so the chart always\ + \ has something to render before anyone customizes it." + ParetoDataResponse: + properties: + pareto: + allOf: + - $ref: '#/components/schemas/ParetoConfig' + description: The group's configured default X/Y metrics. + points: + items: + $ref: '#/components/schemas/ParetoMetricPoint' + type: array + title: Points + description: One point per live evaluation in the group. + type: object + required: + - pareto + - points + title: ParetoDataResponse + description: "Everything the Pareto chart needs for a group: the configured\ + \ default axes plus one point per\nevaluation (cost/latency/evaluator means).\ + \ Unpaginated and slim \u2014 the client plots the whole set\nand computes\ + \ the frontier from any two metrics without refetching." + ParetoMetricPoint: + properties: + name: + type: string + title: Name + description: "Evaluation name \u2014 the leaderboard row id and rollup key." + evaluation_id: + type: string + title: Evaluation Id + description: Evaluation entity id, for navigation. + cost_usd: + title: Cost Usd + description: Mean cost (USD) across the evaluation's runs. + type: number + latency_ms: + title: Latency Ms + description: Mean latency (ms) across the evaluation's runs. + type: number + evaluators: + additionalProperties: + type: number + type: object + title: Evaluators + description: Per-evaluator mean score, keyed by evaluator name. + type: object + required: + - name + - evaluation_id + title: ParetoMetricPoint + description: One evaluation's plottable metric means for the Pareto view. PatronusEvaluateApiParams: properties: success_strategy: diff --git a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml index 8ebfadbe66..eff59ce7b9 100644 --- a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml @@ -919,12 +919,16 @@ resources: experiment_group_request: ExperimentGroupRequest experiment_group_response: ExperimentGroupResponse experiment_group_responses_page: ExperimentGroupResponsesPage + pareto_config: ParetoConfig + pareto_data_response: ParetoDataResponse + pareto_metric_point: ParetoMetricPoint methods: create: post /apis/intake/v2/workspaces/{workspace}/experiment-groups list: get /apis/intake/v2/workspaces/{workspace}/experiment-groups retrieve: get /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} update: put /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} delete: delete /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} + pareto: get /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto evaluations: standalone_api: true models: diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md index 877402928d..bff75b76ba 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md @@ -8,6 +8,9 @@ from nemo_platform.types.experiment_groups import ( ExperimentGroupRequest, ExperimentGroupResponse, ExperimentGroupResponsesPage, + ParetoConfig, + ParetoDataResponse, + ParetoMetricPoint, ) ``` @@ -18,3 +21,4 @@ Methods: - client.experiment_groups.update(path_name, \*, workspace, \*\*params) -> ExperimentGroupResponse - client.experiment_groups.list(\*, workspace, \*\*params) -> SyncDefaultPagination[ExperimentGroupResponse] - client.experiment_groups.delete(name, \*, workspace) -> None +- client.experiment_groups.pareto(name, \*, workspace) -> ParetoDataResponse diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py index 1fcc11f27a..449a7e3c20 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py @@ -39,6 +39,8 @@ experiment_group_create_params, experiment_group_update_params, ) +from ...types.experiment_groups.pareto_config_param import ParetoConfigParam +from ...types.experiment_groups.pareto_data_response import ParetoDataResponse from ...types.experiment_groups.experiment_group_response import ExperimentGroupResponse from ...types.experiment_groups.experiment_group_filter_param import ExperimentGroupFilterParam from ..._exceptions import ConflictError @@ -75,6 +77,7 @@ def create( description: str | Omit = omit, insight_id: str | Omit = omit, metadata: Dict[str, str] | Omit = omit, + pareto: ParetoConfigParam | Omit = omit, summary: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -103,6 +106,13 @@ def create( metadata: Free-form producer metadata for the group. + pareto: Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — + `cost_usd`, `latency_ms`, or `evaluators.`. Defaults to cost (x) vs + latency (y): both exist for every group, so the chart always has something to + render before anyone customizes it. + summary: Human- or agent-authored summary of the group's findings. @@ -131,6 +141,7 @@ def create( "description": description, "insight_id": insight_id, "metadata": metadata, + "pareto": pareto, "summary": summary, }, experiment_group_create_params.ExperimentGroupCreateParams, @@ -195,6 +206,7 @@ def update( description: str | Omit = omit, insight_id: str | Omit = omit, metadata: Dict[str, str] | Omit = omit, + pareto: ParetoConfigParam | Omit = omit, summary: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -222,6 +234,13 @@ def update( metadata: Free-form producer metadata for the group. + pareto: Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — + `cost_usd`, `latency_ms`, or `evaluators.`. Defaults to cost (x) vs + latency (y): both exist for every group, so the chart always has something to + render before anyone customizes it. + summary: Human- or agent-authored summary of the group's findings. extra_headers: Send extra headers @@ -251,6 +270,7 @@ def update( "description": description, "insight_id": insight_id, "metadata": metadata, + "pareto": pareto, "summary": summary, }, experiment_group_update_params.ExperimentGroupUpdateParams, @@ -364,6 +384,52 @@ def delete( cast_to=NoneType, ) + def pareto( + self, + name: str, + *, + workspace: str | None = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ParetoDataResponse: + """ + Cost/latency/evaluator means for every evaluation in the group, plus the group's + default axes. + + Purpose-built for the Pareto chart: a slim, unpaginated projection of the same + rollups the leaderboard shows, so the client plots the full point set in one + call and computes the frontier from any two metrics without refetching (and + without paging the full evaluations list). + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + return self._get( + path_template( + "/apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto", workspace=workspace, name=name + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ParetoDataResponse, + ) + class AsyncExperimentGroupsResource(AsyncAPIResource): @cached_property @@ -394,6 +460,7 @@ async def create( description: str | Omit = omit, insight_id: str | Omit = omit, metadata: Dict[str, str] | Omit = omit, + pareto: ParetoConfigParam | Omit = omit, summary: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -422,6 +489,13 @@ async def create( metadata: Free-form producer metadata for the group. + pareto: Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — + `cost_usd`, `latency_ms`, or `evaluators.`. Defaults to cost (x) vs + latency (y): both exist for every group, so the chart always has something to + render before anyone customizes it. + summary: Human- or agent-authored summary of the group's findings. @@ -450,6 +524,7 @@ async def create( "description": description, "insight_id": insight_id, "metadata": metadata, + "pareto": pareto, "summary": summary, }, experiment_group_create_params.ExperimentGroupCreateParams, @@ -514,6 +589,7 @@ async def update( description: str | Omit = omit, insight_id: str | Omit = omit, metadata: Dict[str, str] | Omit = omit, + pareto: ParetoConfigParam | Omit = omit, summary: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -541,6 +617,13 @@ async def update( metadata: Free-form producer metadata for the group. + pareto: Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — + `cost_usd`, `latency_ms`, or `evaluators.`. Defaults to cost (x) vs + latency (y): both exist for every group, so the chart always has something to + render before anyone customizes it. + summary: Human- or agent-authored summary of the group's findings. extra_headers: Send extra headers @@ -570,6 +653,7 @@ async def update( "description": description, "insight_id": insight_id, "metadata": metadata, + "pareto": pareto, "summary": summary, }, experiment_group_update_params.ExperimentGroupUpdateParams, @@ -683,6 +767,52 @@ async def delete( cast_to=NoneType, ) + async def pareto( + self, + name: str, + *, + workspace: str | None = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ParetoDataResponse: + """ + Cost/latency/evaluator means for every evaluation in the group, plus the group's + default axes. + + Purpose-built for the Pareto chart: a slim, unpaginated projection of the same + rollups the leaderboard shows, so the client plots the full point set in one + call and computes the frontier from any two metrics without refetching (and + without paging the full evaluations list). + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + return await self._get( + path_template( + "/apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto", workspace=workspace, name=name + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ParetoDataResponse, + ) + class ExperimentGroupsResourceWithRawResponse: def __init__(self, experiment_groups: ExperimentGroupsResource) -> None: @@ -703,6 +833,9 @@ def __init__(self, experiment_groups: ExperimentGroupsResource) -> None: self.delete = to_raw_response_wrapper( experiment_groups.delete, ) + self.pareto = to_raw_response_wrapper( + experiment_groups.pareto, + ) class AsyncExperimentGroupsResourceWithRawResponse: @@ -724,6 +857,9 @@ def __init__(self, experiment_groups: AsyncExperimentGroupsResource) -> None: self.delete = async_to_raw_response_wrapper( experiment_groups.delete, ) + self.pareto = async_to_raw_response_wrapper( + experiment_groups.pareto, + ) class ExperimentGroupsResourceWithStreamingResponse: @@ -745,6 +881,9 @@ def __init__(self, experiment_groups: ExperimentGroupsResource) -> None: self.delete = to_streamed_response_wrapper( experiment_groups.delete, ) + self.pareto = to_streamed_response_wrapper( + experiment_groups.pareto, + ) class AsyncExperimentGroupsResourceWithStreamingResponse: @@ -766,3 +905,6 @@ def __init__(self, experiment_groups: AsyncExperimentGroupsResource) -> None: self.delete = async_to_streamed_response_wrapper( experiment_groups.delete, ) + self.pareto = async_to_streamed_response_wrapper( + experiment_groups.pareto, + ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py index 3506053fad..6254ce968a 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py @@ -17,6 +17,10 @@ from __future__ import annotations +from .pareto_config import ParetoConfig as ParetoConfig +from .pareto_config_param import ParetoConfigParam as ParetoConfigParam +from .pareto_metric_point import ParetoMetricPoint as ParetoMetricPoint +from .pareto_data_response import ParetoDataResponse as ParetoDataResponse from .experiment_group_response import ExperimentGroupResponse as ExperimentGroupResponse from .experiment_group_list_params import ExperimentGroupListParams as ExperimentGroupListParams from .experiment_group_filter_param import ExperimentGroupFilterParam as ExperimentGroupFilterParam diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py index cd6ebf095d..ffc2c29860 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py @@ -20,6 +20,8 @@ from typing import Dict from typing_extensions import Required, TypedDict +from .pareto_config_param import ParetoConfigParam + __all__ = ["ExperimentGroupCreateParams"] @@ -48,5 +50,14 @@ class ExperimentGroupCreateParams(TypedDict, total=False): metadata: Dict[str, str] """Free-form producer metadata for the group.""" + pareto: ParetoConfigParam + """Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — + `cost_usd`, `latency_ms`, or `evaluators.`. Defaults to cost (x) vs + latency (y): both exist for every group, so the chart always has something to + render before anyone customizes it. + """ + summary: str """Human- or agent-authored summary of the group's findings.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py index 52cac70f8f..ccc20c6731 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py @@ -19,6 +19,7 @@ from datetime import datetime from ..._models import BaseModel +from .pareto_config import ParetoConfig __all__ = ["ExperimentGroupResponse"] @@ -48,6 +49,15 @@ class ExperimentGroupResponse(BaseModel): metadata: Optional[Dict[str, str]] = None + pareto: Optional[ParetoConfig] = None + """Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — + `cost_usd`, `latency_ms`, or `evaluators.`. Defaults to cost (x) vs + latency (y): both exist for every group, so the chart always has something to + render before anyone customizes it. + """ + summary: Optional[str] = None updated_at: Optional[datetime] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py index b734af7b3a..f5e7d241e9 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py @@ -21,6 +21,7 @@ from typing_extensions import Required, Annotated, TypedDict from ..._utils import PropertyInfo +from .pareto_config_param import ParetoConfigParam __all__ = ["ExperimentGroupUpdateParams"] @@ -50,5 +51,14 @@ class ExperimentGroupUpdateParams(TypedDict, total=False): metadata: Dict[str, str] """Free-form producer metadata for the group.""" + pareto: ParetoConfigParam + """Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — + `cost_usd`, `latency_ms`, or `evaluators.`. Defaults to cost (x) vs + latency (y): both exist for every group, so the chart always has something to + render before anyone customizes it. + """ + summary: str """Human- or agent-authored summary of the group's findings.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config.py new file mode 100644 index 0000000000..7c2f16c937 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional + +from ..._models import BaseModel + +__all__ = ["ParetoConfig"] + + +class ParetoConfig(BaseModel): + """Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — ``cost_usd``, + ``latency_ms``, or ``evaluators.``. Defaults to cost (x) vs latency (y): both exist for + every group, so the chart always has something to render before anyone customizes it. + """ + + x_metric: Optional[str] = None + """Metric plotted on the Pareto X axis.""" + + y_metric: Optional[str] = None + """Metric plotted on the Pareto Y axis.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config_param.py new file mode 100644 index 0000000000..5fb748d0c0 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_config_param.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import TypedDict + +__all__ = ["ParetoConfigParam"] + + +class ParetoConfigParam(TypedDict, total=False): + """Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — ``cost_usd``, + ``latency_ms``, or ``evaluators.``. Defaults to cost (x) vs latency (y): both exist for + every group, so the chart always has something to render before anyone customizes it. + """ + + x_metric: str + """Metric plotted on the Pareto X axis.""" + + y_metric: str + """Metric plotted on the Pareto Y axis.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_data_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_data_response.py new file mode 100644 index 0000000000..f977271f92 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_data_response.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List + +from ..._models import BaseModel +from .pareto_config import ParetoConfig +from .pareto_metric_point import ParetoMetricPoint + +__all__ = ["ParetoDataResponse"] + + +class ParetoDataResponse(BaseModel): + """ + Everything the Pareto chart needs for a group: the configured default axes plus one point per + evaluation (cost/latency/evaluator means). Unpaginated and slim — the client plots the whole set + and computes the frontier from any two metrics without refetching. + """ + + pareto: ParetoConfig + """Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — + `cost_usd`, `latency_ms`, or `evaluators.`. Defaults to cost (x) vs + latency (y): both exist for every group, so the chart always has something to + render before anyone customizes it. + """ + + points: List[ParetoMetricPoint] + """One point per live evaluation in the group.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_metric_point.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_metric_point.py new file mode 100644 index 0000000000..637573834d --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_metric_point.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Dict, Optional + +from ..._models import BaseModel + +__all__ = ["ParetoMetricPoint"] + + +class ParetoMetricPoint(BaseModel): + """One evaluation's plottable metric means for the Pareto view.""" + + evaluation_id: str + """Evaluation entity id, for navigation.""" + + name: str + """Evaluation name — the leaderboard row id and rollup key.""" + + cost_usd: Optional[float] = None + """Mean cost (USD) across the evaluation's runs.""" + + evaluators: Optional[Dict[str, float]] = None + """Per-evaluator mean score, keyed by evaluator name.""" + + latency_ms: Optional[float] = None + """Mean latency (ms) across the evaluation's runs.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/inference/container_executor_config.py b/sdk/python/nemo-platform/src/nemo_platform/types/inference/container_executor_config.py index 6c0d6f8120..923e586c7f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/inference/container_executor_config.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/inference/container_executor_config.py @@ -73,9 +73,12 @@ class ContainerExecutorConfig(BaseModel): override_config: Optional[Dict[str, object]] = None """ - Raw NIMService spec configuration that takes precedence over generated config - (NIM engine on k8s). Allows advanced configuration options directly. Ignored by - non-NIM engines. + Partial NIMService Spec fragments deep-merged after generated defaults and + k8s_nim_operator_config (NIM engine on k8s only). Supported keys: image, + command, args, resources, env, readinessProbe, livenessProbe, startupProbe, + nodeSelector, tolerations, userID, groupID, labels, initContainers, + sidecarContainers. Unsupported keys are rejected at compile time. Ignored by + non-NIM engines and docker runtime. """ run_as_group: Optional[int] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/inference/container_executor_config_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/inference/container_executor_config_param.py index 5ffb042043..d882711066 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/inference/container_executor_config_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/inference/container_executor_config_param.py @@ -76,9 +76,12 @@ class ContainerExecutorConfigParam(TypedDict, total=False): override_config: Dict[str, object] """ - Raw NIMService spec configuration that takes precedence over generated config - (NIM engine on k8s). Allows advanced configuration options directly. Ignored by - non-NIM engines. + Partial NIMService Spec fragments deep-merged after generated defaults and + k8s_nim_operator_config (NIM engine on k8s only). Supported keys: image, + command, args, resources, env, readinessProbe, livenessProbe, startupProbe, + nodeSelector, tolerations, userID, groupID, labels, initContainers, + sidecarContainers. Unsupported keys are rejected at compile time. Ignored by + non-NIM engines and docker runtime. """ run_as_group: int diff --git a/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py b/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py index d6e241d130..cda489d418 100644 --- a/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py +++ b/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py @@ -26,6 +26,7 @@ from nemo_platform import NeMoPlatform, AsyncNeMoPlatform from nemo_platform.pagination import SyncDefaultPagination, AsyncDefaultPagination from nemo_platform.types.experiment_groups import ( + ParetoDataResponse, ExperimentGroupResponse, ) @@ -54,6 +55,10 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: description="description", insight_id="insight_id", metadata={"foo": "string"}, + pareto={ + "x_metric": "x_metric", + "y_metric": "y_metric", + }, summary="summary", ) assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) @@ -168,6 +173,10 @@ def test_method_update_with_all_params(self, client: NeMoPlatform) -> None: description="description", insight_id="insight_id", metadata={"foo": "string"}, + pareto={ + "x_metric": "x_metric", + "y_metric": "y_metric", + }, summary="summary", ) assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) @@ -329,6 +338,58 @@ def test_path_params_delete(self, client: NeMoPlatform) -> None: workspace="workspace", ) + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_pareto(self, client: NeMoPlatform) -> None: + experiment_group = client.experiment_groups.pareto( + name="name", + workspace="workspace", + ) + assert_matches_type(ParetoDataResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_pareto(self, client: NeMoPlatform) -> None: + response = client.experiment_groups.with_raw_response.pareto( + name="name", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment_group = response.parse() + assert_matches_type(ParetoDataResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_pareto(self, client: NeMoPlatform) -> None: + with client.experiment_groups.with_streaming_response.pareto( + name="name", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment_group = response.parse() + assert_matches_type(ParetoDataResponse, experiment_group, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_pareto(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.experiment_groups.with_raw_response.pareto( + name="name", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + client.experiment_groups.with_raw_response.pareto( + name="", + workspace="workspace", + ) + class TestAsyncExperimentGroups: parametrize = pytest.mark.parametrize( @@ -354,6 +415,10 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo description="description", insight_id="insight_id", metadata={"foo": "string"}, + pareto={ + "x_metric": "x_metric", + "y_metric": "y_metric", + }, summary="summary", ) assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) @@ -468,6 +533,10 @@ async def test_method_update_with_all_params(self, async_client: AsyncNeMoPlatfo description="description", insight_id="insight_id", metadata={"foo": "string"}, + pareto={ + "x_metric": "x_metric", + "y_metric": "y_metric", + }, summary="summary", ) assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) @@ -628,3 +697,55 @@ async def test_path_params_delete(self, async_client: AsyncNeMoPlatform) -> None name="", workspace="workspace", ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_pareto(self, async_client: AsyncNeMoPlatform) -> None: + experiment_group = await async_client.experiment_groups.pareto( + name="name", + workspace="workspace", + ) + assert_matches_type(ParetoDataResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_pareto(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.experiment_groups.with_raw_response.pareto( + name="name", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment_group = await response.parse() + assert_matches_type(ParetoDataResponse, experiment_group, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_pareto(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.experiment_groups.with_streaming_response.pareto( + name="name", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment_group = await response.parse() + assert_matches_type(ParetoDataResponse, experiment_group, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_pareto(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.experiment_groups.with_raw_response.pareto( + name="name", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + await async_client.experiment_groups.with_raw_response.pareto( + name="", + workspace="workspace", + ) diff --git a/sdk/stainless.yaml b/sdk/stainless.yaml index 8ebfadbe66..eff59ce7b9 100644 --- a/sdk/stainless.yaml +++ b/sdk/stainless.yaml @@ -919,12 +919,16 @@ resources: experiment_group_request: ExperimentGroupRequest experiment_group_response: ExperimentGroupResponse experiment_group_responses_page: ExperimentGroupResponsesPage + pareto_config: ParetoConfig + pareto_data_response: ParetoDataResponse + pareto_metric_point: ParetoMetricPoint methods: create: post /apis/intake/v2/workspaces/{workspace}/experiment-groups list: get /apis/intake/v2/workspaces/{workspace}/experiment-groups retrieve: get /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} update: put /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} delete: delete /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} + pareto: get /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto evaluations: standalone_api: true models: diff --git a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml index 3ccf30203e..b74903127d 100644 --- a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml +++ b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml @@ -1003,6 +1003,13 @@ authz: scopes: - intake:write - platform:write + /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto: + get: + permissions: + - intake.experiment-groups.read + scopes: + - intake:read + - platform:read /apis/intake/v2/workspaces/{workspace}/experiments: get: permissions: diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index 04de88e863..24a0a9c2a7 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -37,6 +37,8 @@ ExperimentGroupFilter, ExperimentGroupRequest, ExperimentGroupResponse, + ParetoDataResponse, + ParetoMetricPoint, ) # The API/Studio expose this as an "Evaluation", but it is still stored as the Experiment entity @@ -149,6 +151,7 @@ async def create_experiment_group( summary=body.summary, metadata=body.metadata, default_sort=body.default_sort, + pareto=body.pareto, ) try: created = await entity_client.create(entity) @@ -266,6 +269,7 @@ async def update_experiment_group( existing.summary = body.summary existing.metadata = body.metadata existing.default_sort = body.default_sort + existing.pareto = body.pareto updated = await entity_client.update(existing) response = ExperimentGroupResponse.from_entity(updated) response.evaluation_count = await _count_live_evaluations_in_group( @@ -274,6 +278,86 @@ async def update_experiment_group( return response +@router.get( + "/v2/workspaces/{workspace}/experiment-groups/{name}/pareto", + response_model=ParetoDataResponse, + tags=[GROUPS_TAG], + responses={ + 404: {"description": "Experiment group not found"}, + 413: {"description": "Group exceeds the per-request evaluation cap"}, + 503: {"description": "Telemetry store unavailable for metric data"}, + }, +) +async def get_experiment_group_pareto( + workspace: str, + name: str, + entity_client: EntityClientDep, + rollup_repository: EvaluationRollupRepositoryDep, +) -> ParetoDataResponse: + """Cost/latency/evaluator means for every evaluation in the group, plus the group's default axes. + + Purpose-built for the Pareto chart: a slim, unpaginated projection of the same rollups the + leaderboard shows, so the client plots the full point set in one call and computes the frontier + from any two metrics without refetching (and without paging the full evaluations list). + """ + group = await _get_or_404(entity_client, ExperimentGroup, workspace=workspace, name=name, label="Experiment group") + _reject_if_deleted(group, workspace=workspace, name=name, label="Experiment group") + + live_in_group = LogicalOperation( + operator=FilterOperator.AND, + operations=[ + _group_membership_filter(group.id), + LogicalOperation( + operator=FilterOperator.NOT, + operations=[ + ComparisonOperation(operator=FilterOperator.EQ, field="data.is_deleted", value=True), + ], + ), + ], + ) + result = await entity_client.list( + Evaluation, + workspace=workspace, + filter_operation=live_in_group, + page=1, + page_size=_MAX_GROUP_EVALUATIONS, + ) + if result.pagination.total_results > _MAX_GROUP_EVALUATIONS: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=( + f"This group has {result.pagination.total_results} evaluations, exceeding the maximum of " + f"{_MAX_GROUP_EVALUATIONS} the Pareto view can plot in one request." + ), + ) + + responses = [EvaluationResponse.from_entity(e) for e in result.data] + hydrated = await _hydrate_rollups(workspace=workspace, responses=responses, rollup_repository=rollup_repository) + # The Pareto view is metric data by definition; without rollups every point would be empty, so + # fail loudly rather than return a chart with nothing to plot. + if not hydrated: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Cannot build the Pareto view: the telemetry store is unavailable.", + ) + + points = [ + ParetoMetricPoint( + name=response.name, + evaluation_id=response.id, + cost_usd=response.cost_usd.mean if response.cost_usd else None, + latency_ms=response.latency_ms.mean if response.latency_ms else None, + evaluators={ + evaluator: aggregate.mean + for evaluator, aggregate in (response.aggregate_scores or {}).items() + if aggregate.mean is not None + }, + ) + for response in responses + ] + return ParetoDataResponse(pareto=group.pareto, points=points) + + @router.delete( "/v2/workspaces/{workspace}/experiment-groups/{name}", status_code=status.HTTP_204_NO_CONTENT, diff --git a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py index 2f136c7e5d..e6a245eb90 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py @@ -13,7 +13,7 @@ from typing import Annotated, Self from nmp.common.entities.values import DatetimeFilter, Filter, NumberFilter, map_entity_field -from nmp.intake.entities.experiments import Experiment, ExperimentGroup +from nmp.intake.entities.experiments import Experiment, ExperimentGroup, ParetoConfig from nmp.intake.spans.domain import ( INTAKE_PREVIEW_PAYLOAD_CHAR_LIMIT, IntakeResponseMode, @@ -48,6 +48,10 @@ class ExperimentGroupRequest(BaseModel): "the list `sort` param." ), ) + pareto: ParetoConfig = Field( + default_factory=ParetoConfig, + description="Default X/Y metrics for the group's Pareto view. Defaults to cost vs. latency.", + ) class EvaluationRequest(BaseModel): @@ -165,6 +169,7 @@ class ExperimentGroupResponse(BaseModel): summary: str | None = None metadata: dict[str, str] | None = None default_sort: str + pareto: ParetoConfig = Field(default_factory=ParetoConfig) created_at: datetime | None = None updated_at: datetime | None = None evaluation_count: int = Field( @@ -188,11 +193,33 @@ def from_entity(cls, entity: ExperimentGroup) -> ExperimentGroupResponse: summary=entity.summary, metadata=entity.metadata, default_sort=entity.default_sort, + pareto=entity.pareto, created_at=entity.created_at, updated_at=entity.updated_at, ) +class ParetoMetricPoint(BaseModel): + """One evaluation's plottable metric means for the Pareto view.""" + + name: str = Field(description="Evaluation name — the leaderboard row id and rollup key.") + evaluation_id: str = Field(description="Evaluation entity id, for navigation.") + cost_usd: float | None = Field(default=None, description="Mean cost (USD) across the evaluation's runs.") + latency_ms: float | None = Field(default=None, description="Mean latency (ms) across the evaluation's runs.") + evaluators: dict[str, float] = Field( + default_factory=dict, description="Per-evaluator mean score, keyed by evaluator name." + ) + + +class ParetoDataResponse(BaseModel): + """Everything the Pareto chart needs for a group: the configured default axes plus one point per + evaluation (cost/latency/evaluator means). Unpaginated and slim — the client plots the whole set + and computes the frontier from any two metrics without refetching.""" + + pareto: ParetoConfig = Field(description="The group's configured default X/Y metrics.") + points: list[ParetoMetricPoint] = Field(description="One point per live evaluation in the group.") + + class EvaluatorAggregate(BaseModel): """Aggregate statistics over evaluator scores or session-level metric values.""" diff --git a/services/intake/src/nmp/intake/entities/experiments.py b/services/intake/src/nmp/intake/entities/experiments.py index 8710286c0d..437c4fa987 100644 --- a/services/intake/src/nmp/intake/entities/experiments.py +++ b/services/intake/src/nmp/intake/entities/experiments.py @@ -22,7 +22,7 @@ from typing import Any, ClassVar from nmp.common.entities.client import EntityBase -from pydantic import AnyUrl, Field, field_validator, model_validator +from pydantic import AnyUrl, BaseModel, Field, field_validator, model_validator def _stringify_metadata(value: Any) -> Any: @@ -34,6 +34,18 @@ def _stringify_metadata(value: Any) -> Any: return {key: val if isinstance(val, str) else json.dumps(val) for key, val in value.items()} +class ParetoConfig(BaseModel): + """Default X/Y metrics for a group's cost-vs-accuracy Pareto view. + + Metric ids use the same vocabulary as the evaluations list sort/filter fields — ``cost_usd``, + ``latency_ms``, or ``evaluators.``. Defaults to cost (x) vs latency (y): both exist for + every group, so the chart always has something to render before anyone customizes it. + """ + + x_metric: str = Field(default="cost_usd", description="Metric plotted on the Pareto X axis.") + y_metric: str = Field(default="latency_ms", description="Metric plotted on the Pareto Y axis.") + + class ExperimentGroup(EntityBase): """A named container of Experiments pursuing a single optimization goal. @@ -74,6 +86,21 @@ def _default_sort_fallback(cls, value: Any) -> Any: schema-on-read, so coerce anything that isn't a usable string to the default on read.""" return value if isinstance(value, str) else "-created_at" + pareto: ParetoConfig = Field( + default_factory=ParetoConfig, + description=( + "Default X/Y metrics for this group's cost-vs-accuracy Pareto view. Defaults to cost (x) " + "vs latency (y) so the chart always renders before it's customized." + ), + ) + + @field_validator("pareto", mode="before") + @classmethod + def _pareto_fallback(cls, value: Any) -> Any: + """Schema-on-read: groups persisted before this field stored no ``pareto`` (or ``null``); + coerce anything that isn't a config mapping to the cost-vs-latency default.""" + return value if isinstance(value, dict | ParetoConfig) else ParetoConfig() + is_deleted: bool = Field( default=False, description=( diff --git a/services/intake/tests/integration/test_experiments_crud.py b/services/intake/tests/integration/test_experiments_crud.py index 432f8fc87d..ace9e77f49 100644 --- a/services/intake/tests/integration/test_experiments_crud.py +++ b/services/intake/tests/integration/test_experiments_crud.py @@ -77,6 +77,69 @@ def test_experiment_group_update_description(client: TestClient) -> None: assert missing.status_code == 404 +def test_experiment_group_pareto_defaults_and_round_trips(client: TestClient) -> None: + # Omitting pareto defaults to cost (x) vs latency (y), so the chart always has something to render. + created = client.post(GROUPS, json={"name": "pareto-cfg"}) + assert created.status_code == 201, created.text + assert created.json()["pareto"] == {"x_metric": "cost_usd", "y_metric": "latency_ms"} + assert client.get(f"{GROUPS}/pareto-cfg").json()["pareto"] == { + "x_metric": "cost_usd", + "y_metric": "latency_ms", + } + + # A custom selection round-trips through PUT. + updated = client.put( + f"{GROUPS}/pareto-cfg", + json={"name": "pareto-cfg", "pareto": {"x_metric": "cost_usd", "y_metric": "evaluators.reward"}}, + ) + assert updated.status_code == 200, updated.text + assert updated.json()["pareto"] == {"x_metric": "cost_usd", "y_metric": "evaluators.reward"} + + +def test_experiment_group_pareto_endpoint_projects_rollup_means(client: TestClient) -> None: + from nmp.intake.spans.evaluation_rollup_repository import EvaluationRollup, ScoreRollup + + def score(mean: float) -> ScoreRollup: + return ScoreRollup(sum=mean, mean=mean, median=mean, p90=mean, p95=mean, p99=mean, count=1) + + group = _create_group(client, name="pareto-points") + for name in ("eval-a", "eval-b"): + response = client.post(EVALUATIONS, json=_evaluation_body(name=name, experiment_group_id=group["id"])) + assert response.status_code == 201, response.text + + class StubRollupRepository: + async def get_rollups(self, *, workspace: str, evaluation_ids: list[str]) -> dict: + return { + "eval-a": EvaluationRollup( + evaluation_id="eval-a", + cost_usd=score(0.10), + latency_ms=score(200.0), + evaluator_scores={"reward": score(0.9)}, + ), + "eval-b": EvaluationRollup( + evaluation_id="eval-b", + cost_usd=score(0.30), + latency_ms=score(150.0), + evaluator_scores={"reward": score(0.8)}, + ), + } + + app = cast(FastAPI, client.app) + app.dependency_overrides[get_evaluation_rollup_repository] = lambda: StubRollupRepository() + try: + response = client.get(f"{GROUPS}/pareto-points/pareto") + assert response.status_code == 200, response.text + body = response.json() + assert body["pareto"] == {"x_metric": "cost_usd", "y_metric": "latency_ms"} + points = {point["name"]: point for point in body["points"]} + assert set(points) == {"eval-a", "eval-b"} + assert points["eval-a"]["cost_usd"] == 0.10 + assert points["eval-a"]["latency_ms"] == 200.0 + assert points["eval-a"]["evaluators"] == {"reward": 0.9} + finally: + app.dependency_overrides.pop(get_evaluation_rollup_repository, None) + + def test_evaluation_update_moves_between_groups_and_edits(client: TestClient) -> None: group_a = client.post(GROUPS, json={"name": "grp-a"}).json() group_b = client.post(GROUPS, json={"name": "grp-b"}).json() diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx new file mode 100644 index 0000000000..58de194c5b --- /dev/null +++ b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx @@ -0,0 +1,293 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { useToast } from '@nemo/common/src/providers/toast/useToast'; +import { + getGetExperimentGroupParetoQueryKey, + getGetExperimentGroupQueryKey, + useGetExperimentGroupPareto, + useUpdateExperimentGroup, +} from '@nemo/sdk/generated/platform/api'; +import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import { + Button, + SelectContent, + SelectItem, + SelectListbox, + SelectRoot, + SelectTrigger, + Text, +} from '@nvidia/foundations-react-core'; +import { + buildParetoPoints, + deriveParetoMetrics, + metricLabel, + type ParetoMetric, + type ParetoPlotPoint, +} from '@studio/components/charts/ExperimentGroupParetoChart/paretoMetrics'; +import { useQueryClient } from '@tanstack/react-query'; +import { Loader2, Save } from 'lucide-react'; +import { type FC, useMemo, useState } from 'react'; +import { + CartesianGrid, + ResponsiveContainer, + Scatter, + ScatterChart, + Tooltip, + XAxis, + YAxis, +} from 'recharts'; + +interface ExperimentGroupParetoChartProps { + workspace: string; + group: ExperimentGroupResponse; +} + +const CHART_HEIGHT = 360; +const DEFAULT_X_METRIC = 'cost_usd'; +const DEFAULT_Y_METRIC = 'latency_ms'; + +/** Unit-less axis tick labels (units live on the axis label + chart title). Big values are compacted + * so they don't wrap and collide with the rotated axis title (16000 -> "16K"); small values keep full + * precision so close cost/score ticks don't all round to the same number (0.05, 0.11 stay distinct). */ +const formatAxisTick = (value: number): string => + Math.abs(value) >= 1000 + ? value.toLocaleString(undefined, { notation: 'compact', maximumFractionDigits: 1 }) + : value.toLocaleString(undefined, { maximumFractionDigits: 3 }); + +/** Format a metric value for tooltips: cost as USD, latency in ms, evaluator scores as-is. */ +const formatMetricValue = (metric: ParetoMetric, value: number): string => { + if (metric.id === 'cost_usd') { + return `$${value.toLocaleString(undefined, { maximumFractionDigits: 4 })}`; + } + if (metric.id === 'latency_ms') { + return `${Math.round(value).toLocaleString()} ms`; + } + return value.toLocaleString(undefined, { maximumFractionDigits: 3 }); +}; + +interface ParetoTooltipProps { + active?: boolean; + payload?: ReadonlyArray<{ payload: ParetoPlotPoint }>; + xMetric: ParetoMetric; + yMetric: ParetoMetric; +} + +const ParetoTooltip: FC = ({ active, payload, xMetric, yMetric }) => { + const point = payload?.[0]?.payload; + if (!active || !point) return null; + return ( +
+ {point.name} +
+ + {xMetric.label}: {formatMetricValue(xMetric, point.x)} + + + {yMetric.label}: {formatMetricValue(yMetric, point.y)} + + {point.onFrontier && ( + + On the Pareto frontier + + )} +
+
+ ); +}; + +interface MetricSelectProps { + label: string; + value: string; + metrics: readonly ParetoMetric[]; + onChange: (id: string) => void; +} + +const MetricSelect: FC = ({ label, value, metrics, onChange }) => ( + +); + +/** + * Cost-vs-accuracy Pareto view for an experiment group: one point per evaluation with the Pareto + * frontier highlighted. Points come from the group's `/pareto` endpoint. The two axes are chosen from + * the group's available metrics (cost, latency, and each evaluator) and are **persisted on the group** + * — changing a picker saves the selection so it survives reloads and is shared across viewers. Seeds + * from the group's saved axes, defaulting to cost vs. latency (present for every group). + */ +export const ExperimentGroupParetoChart: FC = ({ + workspace, + group, +}) => { + const queryClient = useQueryClient(); + const toast = useToast(); + + const { data, isLoading, isError } = useGetExperimentGroupPareto(workspace, group.name); + const points = useMemo(() => data?.points ?? [], [data]); + const metrics = useMemo(() => deriveParetoMetrics(points), [points]); + + // Selected axes are optimistic local state seeded from the group's saved config (available + // synchronously from the `group` prop). Changes update the chart immediately and persist below. + const [xMetricId, setXMetricId] = useState(group.pareto?.x_metric ?? DEFAULT_X_METRIC); + const [yMetricId, setYMetricId] = useState(group.pareto?.y_metric ?? DEFAULT_Y_METRIC); + + const { mutate: saveGroup, isPending: isSaving } = useUpdateExperimentGroup({ + mutation: { + onSuccess: () => { + toast.success('Saved the group default Pareto view.'); + queryClient.invalidateQueries({ queryKey: getGetExperimentGroupQueryKey(workspace, group.name) }); + queryClient.invalidateQueries({ + queryKey: getGetExperimentGroupParetoQueryKey(workspace, group.name), + }); + }, + onError: () => toast.error('Failed to save the Pareto metrics.'), + }, + }); + + // Persist the axes on the group. PUT is a full replace, so send every current field — only `pareto` + // changes. + const persistAxes = (xMetric: string, yMetric: string) => { + saveGroup({ + workspace, + name: group.name, + data: { + name: group.name, + description: group.description, + insight_id: group.insight_id, + summary: group.summary, + metadata: group.metadata, + default_sort: group.default_sort, + pareto: { x_metric: xMetric, y_metric: yMetric }, + }, + }); + }; + + // Picking a metric only updates the local view; persisting the group-wide default is an explicit, + // clearly-labeled action (Save button) so users know a save is shared with everyone. + const handleXChange = (id: string) => setXMetricId(id); + const handleYChange = (id: string) => setYMetricId(id); + + const savedX = group.pareto?.x_metric ?? DEFAULT_X_METRIC; + const savedY = group.pareto?.y_metric ?? DEFAULT_Y_METRIC; + const hasUnsavedAxes = xMetricId !== savedX || yMetricId !== savedY; + + // Fall back to the first/second metric if a saved id isn't in the current data (e.g. an evaluator + // that dropped out). Cost and latency are always present, so a fallback always exists. + const xMetric = metrics.find((m) => m.id === xMetricId) ?? metrics[0]; + const yMetric = metrics.find((m) => m.id === yMetricId) ?? metrics[1] ?? metrics[0]; + + const plotPoints = useMemo( + () => (xMetric && yMetric ? buildParetoPoints(points, xMetric, yMetric) : []), + [points, xMetric, yMetric] + ); + + // Frontier points are sorted by x so Recharts draws the connecting line along the frontier curve. + const frontierPoints = useMemo( + () => plotPoints.filter((p) => p.onFrontier).sort((a, b) => a.x - b.x), + [plotPoints] + ); + const dominatedPoints = useMemo(() => plotPoints.filter((p) => !p.onFrontier), [plotPoints]); + + const renderBody = () => { + if (isError) { + return Could not load the Pareto data for this group.; + } + if (isLoading) { + return
; + } + if (plotPoints.length === 0 || !xMetric || !yMetric) { + return ( + + No evaluations with both selected metrics to plot. + + ); + } + return ( + + + + + + } + /> + + + + + ); + }; + + return ( +
+
+ {`${metricLabel(xMetricId)} vs. ${metricLabel(yMetricId)}`} +
+ + + +
+
+ {renderBody()} +
+ ); +}; diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.test.ts b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.test.ts new file mode 100644 index 0000000000..020e1803cd --- /dev/null +++ b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.test.ts @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ParetoMetricPoint } from '@nemo/sdk/generated/platform/schema'; +import { + buildParetoPoints, + deriveParetoMetrics, + type ParetoMetric, +} from '@studio/components/charts/ExperimentGroupParetoChart/paretoMetrics'; + +const point = (opts: { + name: string; + cost?: number; + latency?: number; + evaluators?: Record; +}): ParetoMetricPoint => ({ + name: opts.name, + evaluation_id: opts.name, + cost_usd: opts.cost, + latency_ms: opts.latency, + evaluators: opts.evaluators, +}); + +const getMetric = (metrics: ParetoMetric[], id: string): ParetoMetric => { + const metric = metrics.find((m) => m.id === id); + if (!metric) throw new Error(`metric ${id} not found`); + return metric; +}; + +const frontierNames = ( + points: ParetoMetricPoint[], + x: ParetoMetric, + y: ParetoMetric +): string[] => + buildParetoPoints(points, x, y) + .filter((p) => p.onFrontier) + .map((p) => p.name) + .sort(); + +describe('deriveParetoMetrics', () => { + it('always offers cost and latency (minimized) plus one option per evaluator (maximized)', () => { + const points = [point({ name: 'a', evaluators: { reward: 1, safety: 1 } })]; + const metrics = deriveParetoMetrics(points); + // Evaluator ids use the API vocabulary (`evaluators.`) so they match the group's saved axes. + expect(metrics.map((m) => m.id)).toEqual([ + 'cost_usd', + 'latency_ms', + 'evaluators.reward', + 'evaluators.safety', + ]); + expect(getMetric(metrics, 'cost_usd').direction).toBe('min'); + expect(getMetric(metrics, 'latency_ms').direction).toBe('min'); + expect(getMetric(metrics, 'evaluators.reward').direction).toBe('max'); + }); + + it('offers only cost and latency when no evaluators are present', () => { + expect(deriveParetoMetrics([point({ name: 'a' })]).map((m) => m.id)).toEqual([ + 'cost_usd', + 'latency_ms', + ]); + }); +}); + +describe('buildParetoPoints', () => { + it('marks the non-dominated set for two minimized axes (cost vs latency)', () => { + const points = [ + point({ name: 'A', cost: 1, latency: 4 }), // cheapest -> frontier + point({ name: 'B', cost: 2, latency: 2 }), // balanced -> frontier + point({ name: 'C', cost: 4, latency: 1 }), // fastest -> frontier + point({ name: 'D', cost: 3, latency: 3 }), // dominated by B + ]; + const metrics = deriveParetoMetrics(points); + expect(frontierNames(points, getMetric(metrics, 'cost_usd'), getMetric(metrics, 'latency_ms'))).toEqual([ + 'A', + 'B', + 'C', + ]); + }); + + it('respects mixed directions: minimize cost, maximize an evaluator score', () => { + const points = [ + point({ name: 'A', cost: 1, evaluators: { reward: 0.5 } }), // cheapest -> frontier + point({ name: 'B', cost: 2, evaluators: { reward: 0.9 } }), // most accurate -> frontier + point({ name: 'C', cost: 2, evaluators: { reward: 0.4 } }), // dominated by A + ]; + const metrics = deriveParetoMetrics(points); + expect( + frontierNames(points, getMetric(metrics, 'cost_usd'), getMetric(metrics, 'evaluators.reward')) + ).toEqual(['A', 'B']); + }); + + it('drops points missing either selected metric', () => { + const points = [ + point({ name: 'A', cost: 1, latency: 1 }), + point({ name: 'B', cost: 2 }), // no latency -> excluded + ]; + const metrics = deriveParetoMetrics(points); + const plotted = buildParetoPoints(points, getMetric(metrics, 'cost_usd'), getMetric(metrics, 'latency_ms')); + expect(plotted.map((p) => p.name)).toEqual(['A']); + }); +}); diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.ts b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.ts new file mode 100644 index 0000000000..cff0146310 --- /dev/null +++ b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.ts @@ -0,0 +1,117 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ParetoMetricPoint } from '@nemo/sdk/generated/platform/schema'; + +/** Which direction on an axis counts as "better": cost/latency minimize, evaluator scores maximize. */ +export type MetricDirection = 'min' | 'max'; + +export interface ParetoMetric { + /** + * Stable id, using the same metric vocabulary the API stores in `group.pareto` and the list + * sort/filter fields: `cost_usd`, `latency_ms`, or `evaluators.`. + */ + readonly id: string; + readonly label: string; + readonly direction: MetricDirection; + readonly accessor: (point: ParetoMetricPoint) => number | null | undefined; +} + +const capitalize = (value: string): string => + value ? value.charAt(0).toUpperCase() + value.slice(1) : value; + +/** + * The display label for a metric id, resolvable synchronously from the id alone (no loaded points + * needed): `cost_usd` -> "Cost (USD)", `latency_ms` -> "Latency (ms)", `evaluators.` -> the + * capitalized evaluator name. Used so a saved evaluator axis renders its real label immediately + * instead of flashing the cost/latency fallback while the chart data loads. + */ +export function metricLabel(id: string): string { + if (id === 'cost_usd') return 'Cost (USD)'; + if (id === 'latency_ms') return 'Latency (ms)'; + return capitalize(id.startsWith('evaluators.') ? id.slice('evaluators.'.length) : id); +} + +const COST_METRIC: ParetoMetric = { + id: 'cost_usd', + label: metricLabel('cost_usd'), + direction: 'min', + accessor: (point) => point.cost_usd, +}; + +const LATENCY_METRIC: ParetoMetric = { + id: 'latency_ms', + label: metricLabel('latency_ms'), + direction: 'min', + accessor: (point) => point.latency_ms, +}; + +/** + * The metrics a user may plot on either axis: cost and latency (always present, minimized), plus one + * option per evaluator seen across the group's points (maximized). Evaluator names are dynamic — they + * differ per customer — so they're derived from the data rather than hardcoded. + */ +export function deriveParetoMetrics(points: readonly ParetoMetricPoint[]): ParetoMetric[] { + const evaluatorNames = [ + ...new Set(points.flatMap((point) => Object.keys(point.evaluators ?? {}))), + ].sort(); + const evaluatorMetrics = evaluatorNames.map((name) => ({ + id: `evaluators.${name}`, + label: metricLabel(`evaluators.${name}`), + direction: 'max', + accessor: (point) => point.evaluators?.[name], + })); + return [COST_METRIC, LATENCY_METRIC, ...evaluatorMetrics]; +} + +export interface ParetoPlotPoint { + readonly name: string; + readonly x: number; + readonly y: number; + /** True when no other evaluation dominates this one on both axes. */ + readonly onFrontier: boolean; +} + +interface Coords { + readonly x: number; + readonly y: number; +} + +/** Whether `b` dominates `a`: at least as good on both axes and strictly better on at least one. */ +function dominates(a: Coords, b: Coords, xDir: MetricDirection, yDir: MetricDirection): boolean { + const atLeastAsGood = (av: number, bv: number, dir: MetricDirection): boolean => + dir === 'min' ? bv <= av : bv >= av; + const strictlyBetter = (av: number, bv: number, dir: MetricDirection): boolean => + dir === 'min' ? bv < av : bv > av; + return ( + atLeastAsGood(a.x, b.x, xDir) && + atLeastAsGood(a.y, b.y, yDir) && + (strictlyBetter(a.x, b.x, xDir) || strictlyBetter(a.y, b.y, yDir)) + ); +} + +/** + * Build plot points for two metrics and flag which lie on the Pareto frontier — the evaluations not + * dominated by any other on both axes. Points missing either metric (non-finite) are dropped. + */ +export function buildParetoPoints( + points: readonly ParetoMetricPoint[], + xMetric: ParetoMetric, + yMetric: ParetoMetric +): ParetoPlotPoint[] { + const coords = points + .map((point): { name: string; x: number; y: number } | null => { + const x = xMetric.accessor(point); + const y = yMetric.accessor(point); + if (x == null || y == null || !Number.isFinite(x) || !Number.isFinite(y)) return null; + return { name: point.name, x, y }; + }) + .filter((point): point is { name: string; x: number; y: number } => point !== null); + + return coords.map((point) => ({ + ...point, + onFrontier: !coords.some( + (other) => other !== point && dominates(point, other, xMetric.direction, yMetric.direction) + ), + })); +} diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx index f584d426df..139fa6a3fa 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx @@ -22,6 +22,7 @@ import type { ExperimentGroupResponse, } from '@nemo/sdk/generated/platform/schema'; import { Button, Text, Tooltip } from '@nvidia/foundations-react-core'; +import { ExperimentGroupParetoChart } from '@studio/components/charts/ExperimentGroupParetoChart'; import { AddToGroupModal } from '@studio/components/dataViews/ExperimentGroupDataView/AddToGroupModal'; import '@studio/components/dataViews/ExperimentGroupDataView/ExperimentGroupDataView.css'; import { Empty } from '@studio/components/dataViews/ExperimentGroupDataView/Empty'; @@ -37,7 +38,7 @@ import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { getEvaluationDetailRoute } from '@studio/routes/utils'; import { tooltipClassName } from '@studio/styles/common'; import { useLocalStorage } from '@studio/util/hooks/useLocalStorage'; -import { Columns3, FolderPlus, Pin } from 'lucide-react'; +import { ChartScatter, Columns3, FolderPlus, Pin } from 'lucide-react'; import { type ComponentProps, type FC, useCallback, useEffect, useMemo, useState } from 'react'; import { useNavigate } from 'react-router-dom'; @@ -152,6 +153,12 @@ export const ExperimentGroupDataView: FC = ({ grou [] ); + // Pareto (cost-vs-accuracy) view visibility, persisted per group. Hidden by default. + const [paretoVisible, setParetoVisible] = useLocalStorage( + `nemo-studio:experiment-group-pareto:${experimentGroupId}`, + false + ); + // Seed the sort from default_sort so its column header reflects the order on load. Memoized so the // reference is stable across renders (until default_sort changes). const defaultSort = useMemo(() => seedSortFromDefault(group.default_sort), [group.default_sort]); @@ -455,6 +462,11 @@ export const ExperimentGroupDataView: FC = ({ grou return ( <> + {paretoVisible && ( +
+ +
+ )} = ({ grou )} toolbarSlotEnd={ - } - > - <> - - Columns - - + <> + } + > + <> + + Columns + + + + } attributes={{ DataViewRoot: { From a21d91c7403b07fcebba366b9fac4004c2998448 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:20:25 -0600 Subject: [PATCH 02/11] pretty Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- .../ExperimentGroupParetoChart/index.tsx | 18 ++++++++++++++--- .../paretoMetrics.test.ts | 20 +++++++++---------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx index 58de194c5b..cdad7d4091 100644 --- a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx +++ b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx @@ -157,7 +157,9 @@ export const ExperimentGroupParetoChart: FC = ( mutation: { onSuccess: () => { toast.success('Saved the group default Pareto view.'); - queryClient.invalidateQueries({ queryKey: getGetExperimentGroupQueryKey(workspace, group.name) }); + queryClient.invalidateQueries({ + queryKey: getGetExperimentGroupQueryKey(workspace, group.name), + }); queryClient.invalidateQueries({ queryKey: getGetExperimentGroupParetoQueryKey(workspace, group.name), }); @@ -270,8 +272,18 @@ export const ExperimentGroupParetoChart: FC = (
{`${metricLabel(xMetricId)} vs. ${metricLabel(yMetricId)}`}
- - + + )} toolbarSlotEnd={ - <> - } - > - <> - - Columns - - - - + } + > + <> + + Columns + + } attributes={{ DataViewRoot: { diff --git a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx index a5d69429e8..138a2ed885 100644 --- a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx +++ b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx @@ -3,7 +3,7 @@ import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; import { useGetExperimentGroup } from '@nemo/sdk/generated/platform/api'; -import { Badge, Button, PageHeader, Stack, Text } from '@nvidia/foundations-react-core'; +import { Button, PageHeader, Stack, Text } from '@nvidia/foundations-react-core'; import { AccessibleTitle } from '@studio/components/AccessibleTitle'; import { ExperimentGroupDataView } from '@studio/components/dataViews/ExperimentGroupDataView'; import { ExperimentGroupEditModal } from '@studio/components/ExperimentGroupEditModal'; @@ -12,8 +12,9 @@ import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; import { ExperimentGroupMetrics } from '@studio/routes/ExperimentGroupDetailRoute/ExperimentGroupMetrics'; import { getExperimentRoute } from '@studio/routes/utils'; +import { useLocalStorage } from '@studio/util/hooks/useLocalStorage'; import { useRequiredPathParams } from '@studio/util/hooks/useRequiredPathParams'; -import { Pencil } from 'lucide-react'; +import { ChartScatter, Pencil } from 'lucide-react'; import { type FC, useState } from 'react'; export const ExperimentGroupDetailRoute: FC = () => { @@ -22,6 +23,13 @@ export const ExperimentGroupDetailRoute: FC = () => { const { data: group, error } = useGetExperimentGroup(workspace, experimentGroupName); const [editOpen, setEditOpen] = useState(false); + // Pareto (cost-vs-accuracy) view visibility, persisted per group. Hidden by default. + const [storedParetoVisible, setParetoVisible] = useLocalStorage( + `nemo-studio:experiment-group-pareto:${group?.id ?? ''}`, + false + ); + const paretoVisible = storedParetoVisible ?? false; + useBreadcrumbs({ items: [ { href: getExperimentRoute(workspace), slotLabel: 'Experiment Groups' }, @@ -59,13 +67,18 @@ export const ExperimentGroupDetailRoute: FC = () => {
Evaluations - {group?.evaluation_count !== undefined && ( - - {group.evaluation_count} - + {group && ( + )}
- {group && } + {group && }
)} From ad249a186bf59ed37e0235a4d0698073721d711a Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Fri, 24 Jul 2026 09:20:21 -0600 Subject: [PATCH 04/11] fix(experiments): drop unused import and reset Pareto axes on group change - Remove the now-unused ChartScatter import from ExperimentGroupDataView (the toggle moved to ExperimentGroupDetailRoute), fixing the Web lint + typecheck CI failures. - Key ExperimentGroupParetoChart by group.id so the axis selection re-seeds from the new group's saved config when navigating between groups without a route remount (CodeRabbit: axes could otherwise persist and be saved onto the wrong group). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- .../components/dataViews/ExperimentGroupDataView/index.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx index 0102bafb46..374386e83f 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx @@ -38,7 +38,7 @@ import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { getEvaluationDetailRoute } from '@studio/routes/utils'; import { tooltipClassName } from '@studio/styles/common'; import { useLocalStorage } from '@studio/util/hooks/useLocalStorage'; -import { ChartScatter, Columns3, FolderPlus, Pin } from 'lucide-react'; +import { Columns3, FolderPlus, Pin } from 'lucide-react'; import { type ComponentProps, type FC, useCallback, useEffect, useMemo, useState } from 'react'; import { useNavigate } from 'react-router-dom'; @@ -464,7 +464,9 @@ export const ExperimentGroupDataView: FC = ({ <> {paretoVisible && (
- + {/* Key by group id so the axis selection resets (re-seeds from the new group's saved + config) when navigating between groups without a route remount. */} +
)} Date: Fri, 24 Jul 2026 09:59:30 -0600 Subject: [PATCH 05/11] remove new endpoint Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- openapi/ga/individual/platform.openapi.yaml | 101 ----------------- openapi/ga/openapi.yaml | 101 ----------------- openapi/openapi.yaml | 101 ----------------- .../nemo-platform/.nmpcontext/openapi.yaml | 101 ----------------- .../nemo-platform/.nmpcontext/stainless.yaml | 3 - .../resources/experiment_groups/api.md | 3 - .../experiment_groups/experiment_groups.py | 105 ------------------ .../types/experiment_groups/__init__.py | 2 - .../experiment_groups/pareto_data_response.py | 44 -------- .../experiment_groups/pareto_metric_point.py | 41 ------- .../api_resources/test_experiment_groups.py | 105 ------------------ sdk/stainless.yaml | 3 - .../intake/api/v2/experiments/endpoints.py | 82 -------------- .../nmp/intake/api/v2/experiments/schemas.py | 21 ---- .../integration/test_experiments_crud.py | 44 -------- .../ExperimentGroupParetoChart/index.tsx | 12 +- .../paretoMetrics.test.ts | 23 ++-- .../paretoMetrics.ts | 30 ++--- .../useParetoEvaluations.ts | 48 ++++++++ 19 files changed, 81 insertions(+), 889 deletions(-) delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_data_response.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_metric_point.py create mode 100644 web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index b816826e0c..1cec818a3a 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -4213,56 +4213,6 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto: - get: - tags: - - Experiment Groups - summary: Get Experiment Group Pareto - description: 'Cost/latency/evaluator means for every evaluation in the group, - plus the group''s default axes. - - - Purpose-built for the Pareto chart: a slim, unpaginated projection of the - same rollups the - - leaderboard shows, so the client plots the full point set in one call and - computes the frontier - - from any two metrics without refetching (and without paging the full evaluations - list).' - operationId: get_experiment_group_pareto_apis_intake_v2_workspaces__workspace__experiment_groups__name__pareto_get - parameters: - - name: workspace - in: path - required: true - schema: - type: string - title: Workspace - - name: name - in: path - required: true - schema: - type: string - title: Name - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/ParetoDataResponse' - '404': - description: Experiment group not found - '413': - description: Group exceeds the per-request evaluation cap - '503': - description: Telemetry store unavailable for metric data - '422': - description: Validation Error - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPValidationError' /apis/intake/v2/workspaces/{workspace}/ingest/atif: post: tags: @@ -15460,57 +15410,6 @@ components: \ \u2014 ``cost_usd``,\n``latency_ms``, or ``evaluators.``. Defaults\ \ to cost (x) vs latency (y): both exist for\nevery group, so the chart always\ \ has something to render before anyone customizes it." - ParetoDataResponse: - properties: - pareto: - allOf: - - $ref: '#/components/schemas/ParetoConfig' - description: The group's configured default X/Y metrics. - points: - items: - $ref: '#/components/schemas/ParetoMetricPoint' - type: array - title: Points - description: One point per live evaluation in the group. - type: object - required: - - pareto - - points - title: ParetoDataResponse - description: "Everything the Pareto chart needs for a group: the configured\ - \ default axes plus one point per\nevaluation (cost/latency/evaluator means).\ - \ Unpaginated and slim \u2014 the client plots the whole set\nand computes\ - \ the frontier from any two metrics without refetching." - ParetoMetricPoint: - properties: - name: - type: string - title: Name - description: "Evaluation name \u2014 the leaderboard row id and rollup key." - evaluation_id: - type: string - title: Evaluation Id - description: Evaluation entity id, for navigation. - cost_usd: - title: Cost Usd - description: Mean cost (USD) across the evaluation's runs. - type: number - latency_ms: - title: Latency Ms - description: Mean latency (ms) across the evaluation's runs. - type: number - evaluators: - additionalProperties: - type: number - type: object - title: Evaluators - description: Per-evaluator mean score, keyed by evaluator name. - type: object - required: - - name - - evaluation_id - title: ParetoMetricPoint - description: One evaluation's plottable metric means for the Pareto view. PatronusEvaluateApiParams: properties: success_strategy: diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index b816826e0c..1cec818a3a 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -4213,56 +4213,6 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto: - get: - tags: - - Experiment Groups - summary: Get Experiment Group Pareto - description: 'Cost/latency/evaluator means for every evaluation in the group, - plus the group''s default axes. - - - Purpose-built for the Pareto chart: a slim, unpaginated projection of the - same rollups the - - leaderboard shows, so the client plots the full point set in one call and - computes the frontier - - from any two metrics without refetching (and without paging the full evaluations - list).' - operationId: get_experiment_group_pareto_apis_intake_v2_workspaces__workspace__experiment_groups__name__pareto_get - parameters: - - name: workspace - in: path - required: true - schema: - type: string - title: Workspace - - name: name - in: path - required: true - schema: - type: string - title: Name - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/ParetoDataResponse' - '404': - description: Experiment group not found - '413': - description: Group exceeds the per-request evaluation cap - '503': - description: Telemetry store unavailable for metric data - '422': - description: Validation Error - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPValidationError' /apis/intake/v2/workspaces/{workspace}/ingest/atif: post: tags: @@ -15460,57 +15410,6 @@ components: \ \u2014 ``cost_usd``,\n``latency_ms``, or ``evaluators.``. Defaults\ \ to cost (x) vs latency (y): both exist for\nevery group, so the chart always\ \ has something to render before anyone customizes it." - ParetoDataResponse: - properties: - pareto: - allOf: - - $ref: '#/components/schemas/ParetoConfig' - description: The group's configured default X/Y metrics. - points: - items: - $ref: '#/components/schemas/ParetoMetricPoint' - type: array - title: Points - description: One point per live evaluation in the group. - type: object - required: - - pareto - - points - title: ParetoDataResponse - description: "Everything the Pareto chart needs for a group: the configured\ - \ default axes plus one point per\nevaluation (cost/latency/evaluator means).\ - \ Unpaginated and slim \u2014 the client plots the whole set\nand computes\ - \ the frontier from any two metrics without refetching." - ParetoMetricPoint: - properties: - name: - type: string - title: Name - description: "Evaluation name \u2014 the leaderboard row id and rollup key." - evaluation_id: - type: string - title: Evaluation Id - description: Evaluation entity id, for navigation. - cost_usd: - title: Cost Usd - description: Mean cost (USD) across the evaluation's runs. - type: number - latency_ms: - title: Latency Ms - description: Mean latency (ms) across the evaluation's runs. - type: number - evaluators: - additionalProperties: - type: number - type: object - title: Evaluators - description: Per-evaluator mean score, keyed by evaluator name. - type: object - required: - - name - - evaluation_id - title: ParetoMetricPoint - description: One evaluation's plottable metric means for the Pareto view. PatronusEvaluateApiParams: properties: success_strategy: diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index b816826e0c..1cec818a3a 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -4213,56 +4213,6 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto: - get: - tags: - - Experiment Groups - summary: Get Experiment Group Pareto - description: 'Cost/latency/evaluator means for every evaluation in the group, - plus the group''s default axes. - - - Purpose-built for the Pareto chart: a slim, unpaginated projection of the - same rollups the - - leaderboard shows, so the client plots the full point set in one call and - computes the frontier - - from any two metrics without refetching (and without paging the full evaluations - list).' - operationId: get_experiment_group_pareto_apis_intake_v2_workspaces__workspace__experiment_groups__name__pareto_get - parameters: - - name: workspace - in: path - required: true - schema: - type: string - title: Workspace - - name: name - in: path - required: true - schema: - type: string - title: Name - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/ParetoDataResponse' - '404': - description: Experiment group not found - '413': - description: Group exceeds the per-request evaluation cap - '503': - description: Telemetry store unavailable for metric data - '422': - description: Validation Error - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPValidationError' /apis/intake/v2/workspaces/{workspace}/ingest/atif: post: tags: @@ -15460,57 +15410,6 @@ components: \ \u2014 ``cost_usd``,\n``latency_ms``, or ``evaluators.``. Defaults\ \ to cost (x) vs latency (y): both exist for\nevery group, so the chart always\ \ has something to render before anyone customizes it." - ParetoDataResponse: - properties: - pareto: - allOf: - - $ref: '#/components/schemas/ParetoConfig' - description: The group's configured default X/Y metrics. - points: - items: - $ref: '#/components/schemas/ParetoMetricPoint' - type: array - title: Points - description: One point per live evaluation in the group. - type: object - required: - - pareto - - points - title: ParetoDataResponse - description: "Everything the Pareto chart needs for a group: the configured\ - \ default axes plus one point per\nevaluation (cost/latency/evaluator means).\ - \ Unpaginated and slim \u2014 the client plots the whole set\nand computes\ - \ the frontier from any two metrics without refetching." - ParetoMetricPoint: - properties: - name: - type: string - title: Name - description: "Evaluation name \u2014 the leaderboard row id and rollup key." - evaluation_id: - type: string - title: Evaluation Id - description: Evaluation entity id, for navigation. - cost_usd: - title: Cost Usd - description: Mean cost (USD) across the evaluation's runs. - type: number - latency_ms: - title: Latency Ms - description: Mean latency (ms) across the evaluation's runs. - type: number - evaluators: - additionalProperties: - type: number - type: object - title: Evaluators - description: Per-evaluator mean score, keyed by evaluator name. - type: object - required: - - name - - evaluation_id - title: ParetoMetricPoint - description: One evaluation's plottable metric means for the Pareto view. PatronusEvaluateApiParams: properties: success_strategy: diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index b816826e0c..1cec818a3a 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -4213,56 +4213,6 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto: - get: - tags: - - Experiment Groups - summary: Get Experiment Group Pareto - description: 'Cost/latency/evaluator means for every evaluation in the group, - plus the group''s default axes. - - - Purpose-built for the Pareto chart: a slim, unpaginated projection of the - same rollups the - - leaderboard shows, so the client plots the full point set in one call and - computes the frontier - - from any two metrics without refetching (and without paging the full evaluations - list).' - operationId: get_experiment_group_pareto_apis_intake_v2_workspaces__workspace__experiment_groups__name__pareto_get - parameters: - - name: workspace - in: path - required: true - schema: - type: string - title: Workspace - - name: name - in: path - required: true - schema: - type: string - title: Name - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/ParetoDataResponse' - '404': - description: Experiment group not found - '413': - description: Group exceeds the per-request evaluation cap - '503': - description: Telemetry store unavailable for metric data - '422': - description: Validation Error - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPValidationError' /apis/intake/v2/workspaces/{workspace}/ingest/atif: post: tags: @@ -15460,57 +15410,6 @@ components: \ \u2014 ``cost_usd``,\n``latency_ms``, or ``evaluators.``. Defaults\ \ to cost (x) vs latency (y): both exist for\nevery group, so the chart always\ \ has something to render before anyone customizes it." - ParetoDataResponse: - properties: - pareto: - allOf: - - $ref: '#/components/schemas/ParetoConfig' - description: The group's configured default X/Y metrics. - points: - items: - $ref: '#/components/schemas/ParetoMetricPoint' - type: array - title: Points - description: One point per live evaluation in the group. - type: object - required: - - pareto - - points - title: ParetoDataResponse - description: "Everything the Pareto chart needs for a group: the configured\ - \ default axes plus one point per\nevaluation (cost/latency/evaluator means).\ - \ Unpaginated and slim \u2014 the client plots the whole set\nand computes\ - \ the frontier from any two metrics without refetching." - ParetoMetricPoint: - properties: - name: - type: string - title: Name - description: "Evaluation name \u2014 the leaderboard row id and rollup key." - evaluation_id: - type: string - title: Evaluation Id - description: Evaluation entity id, for navigation. - cost_usd: - title: Cost Usd - description: Mean cost (USD) across the evaluation's runs. - type: number - latency_ms: - title: Latency Ms - description: Mean latency (ms) across the evaluation's runs. - type: number - evaluators: - additionalProperties: - type: number - type: object - title: Evaluators - description: Per-evaluator mean score, keyed by evaluator name. - type: object - required: - - name - - evaluation_id - title: ParetoMetricPoint - description: One evaluation's plottable metric means for the Pareto view. PatronusEvaluateApiParams: properties: success_strategy: diff --git a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml index eff59ce7b9..438911b5ef 100644 --- a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml @@ -920,15 +920,12 @@ resources: experiment_group_response: ExperimentGroupResponse experiment_group_responses_page: ExperimentGroupResponsesPage pareto_config: ParetoConfig - pareto_data_response: ParetoDataResponse - pareto_metric_point: ParetoMetricPoint methods: create: post /apis/intake/v2/workspaces/{workspace}/experiment-groups list: get /apis/intake/v2/workspaces/{workspace}/experiment-groups retrieve: get /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} update: put /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} delete: delete /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} - pareto: get /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto evaluations: standalone_api: true models: diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md index bff75b76ba..998ab3da68 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md @@ -9,8 +9,6 @@ from nemo_platform.types.experiment_groups import ( ExperimentGroupResponse, ExperimentGroupResponsesPage, ParetoConfig, - ParetoDataResponse, - ParetoMetricPoint, ) ``` @@ -21,4 +19,3 @@ Methods: - client.experiment_groups.update(path_name, \*, workspace, \*\*params) -> ExperimentGroupResponse - client.experiment_groups.list(\*, workspace, \*\*params) -> SyncDefaultPagination[ExperimentGroupResponse] - client.experiment_groups.delete(name, \*, workspace) -> None -- client.experiment_groups.pareto(name, \*, workspace) -> ParetoDataResponse diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py index 449a7e3c20..728c4b0d9a 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py @@ -40,7 +40,6 @@ experiment_group_update_params, ) from ...types.experiment_groups.pareto_config_param import ParetoConfigParam -from ...types.experiment_groups.pareto_data_response import ParetoDataResponse from ...types.experiment_groups.experiment_group_response import ExperimentGroupResponse from ...types.experiment_groups.experiment_group_filter_param import ExperimentGroupFilterParam from ..._exceptions import ConflictError @@ -384,52 +383,6 @@ def delete( cast_to=NoneType, ) - def pareto( - self, - name: str, - *, - workspace: str | None = None, - # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. - # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Headers | None = None, - extra_query: Query | None = None, - extra_body: Body | None = None, - timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ParetoDataResponse: - """ - Cost/latency/evaluator means for every evaluation in the group, plus the group's - default axes. - - Purpose-built for the Pareto chart: a slim, unpaginated projection of the same - rollups the leaderboard shows, so the client plots the full point set in one - call and computes the frontier from any two metrics without refetching (and - without paging the full evaluations list). - - Args: - extra_headers: Send extra headers - - extra_query: Add additional query parameters to the request - - extra_body: Add additional JSON properties to the request - - timeout: Override the client-level default timeout for this request, in seconds - """ - if workspace is None: - workspace = self._client._get_workspace_path_param() - if not workspace: - raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") - if not name: - raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") - return self._get( - path_template( - "/apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto", workspace=workspace, name=name - ), - options=make_request_options( - extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout - ), - cast_to=ParetoDataResponse, - ) - class AsyncExperimentGroupsResource(AsyncAPIResource): @cached_property @@ -767,52 +720,6 @@ async def delete( cast_to=NoneType, ) - async def pareto( - self, - name: str, - *, - workspace: str | None = None, - # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. - # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Headers | None = None, - extra_query: Query | None = None, - extra_body: Body | None = None, - timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> ParetoDataResponse: - """ - Cost/latency/evaluator means for every evaluation in the group, plus the group's - default axes. - - Purpose-built for the Pareto chart: a slim, unpaginated projection of the same - rollups the leaderboard shows, so the client plots the full point set in one - call and computes the frontier from any two metrics without refetching (and - without paging the full evaluations list). - - Args: - extra_headers: Send extra headers - - extra_query: Add additional query parameters to the request - - extra_body: Add additional JSON properties to the request - - timeout: Override the client-level default timeout for this request, in seconds - """ - if workspace is None: - workspace = self._client._get_workspace_path_param() - if not workspace: - raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") - if not name: - raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") - return await self._get( - path_template( - "/apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto", workspace=workspace, name=name - ), - options=make_request_options( - extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout - ), - cast_to=ParetoDataResponse, - ) - class ExperimentGroupsResourceWithRawResponse: def __init__(self, experiment_groups: ExperimentGroupsResource) -> None: @@ -833,9 +740,6 @@ def __init__(self, experiment_groups: ExperimentGroupsResource) -> None: self.delete = to_raw_response_wrapper( experiment_groups.delete, ) - self.pareto = to_raw_response_wrapper( - experiment_groups.pareto, - ) class AsyncExperimentGroupsResourceWithRawResponse: @@ -857,9 +761,6 @@ def __init__(self, experiment_groups: AsyncExperimentGroupsResource) -> None: self.delete = async_to_raw_response_wrapper( experiment_groups.delete, ) - self.pareto = async_to_raw_response_wrapper( - experiment_groups.pareto, - ) class ExperimentGroupsResourceWithStreamingResponse: @@ -881,9 +782,6 @@ def __init__(self, experiment_groups: ExperimentGroupsResource) -> None: self.delete = to_streamed_response_wrapper( experiment_groups.delete, ) - self.pareto = to_streamed_response_wrapper( - experiment_groups.pareto, - ) class AsyncExperimentGroupsResourceWithStreamingResponse: @@ -905,6 +803,3 @@ def __init__(self, experiment_groups: AsyncExperimentGroupsResource) -> None: self.delete = async_to_streamed_response_wrapper( experiment_groups.delete, ) - self.pareto = async_to_streamed_response_wrapper( - experiment_groups.pareto, - ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py index 6254ce968a..7fb0bced54 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py @@ -19,8 +19,6 @@ from .pareto_config import ParetoConfig as ParetoConfig from .pareto_config_param import ParetoConfigParam as ParetoConfigParam -from .pareto_metric_point import ParetoMetricPoint as ParetoMetricPoint -from .pareto_data_response import ParetoDataResponse as ParetoDataResponse from .experiment_group_response import ExperimentGroupResponse as ExperimentGroupResponse from .experiment_group_list_params import ExperimentGroupListParams as ExperimentGroupListParams from .experiment_group_filter_param import ExperimentGroupFilterParam as ExperimentGroupFilterParam diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_data_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_data_response.py deleted file mode 100644 index f977271f92..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_data_response.py +++ /dev/null @@ -1,44 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -from typing import List - -from ..._models import BaseModel -from .pareto_config import ParetoConfig -from .pareto_metric_point import ParetoMetricPoint - -__all__ = ["ParetoDataResponse"] - - -class ParetoDataResponse(BaseModel): - """ - Everything the Pareto chart needs for a group: the configured default axes plus one point per - evaluation (cost/latency/evaluator means). Unpaginated and slim — the client plots the whole set - and computes the frontier from any two metrics without refetching. - """ - - pareto: ParetoConfig - """Default X/Y metrics for a group's cost-vs-accuracy Pareto view. - - Metric ids use the same vocabulary as the evaluations list sort/filter fields — - `cost_usd`, `latency_ms`, or `evaluators.`. Defaults to cost (x) vs - latency (y): both exist for every group, so the chart always has something to - render before anyone customizes it. - """ - - points: List[ParetoMetricPoint] - """One point per live evaluation in the group.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_metric_point.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_metric_point.py deleted file mode 100644 index 637573834d..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/pareto_metric_point.py +++ /dev/null @@ -1,41 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -from typing import Dict, Optional - -from ..._models import BaseModel - -__all__ = ["ParetoMetricPoint"] - - -class ParetoMetricPoint(BaseModel): - """One evaluation's plottable metric means for the Pareto view.""" - - evaluation_id: str - """Evaluation entity id, for navigation.""" - - name: str - """Evaluation name — the leaderboard row id and rollup key.""" - - cost_usd: Optional[float] = None - """Mean cost (USD) across the evaluation's runs.""" - - evaluators: Optional[Dict[str, float]] = None - """Per-evaluator mean score, keyed by evaluator name.""" - - latency_ms: Optional[float] = None - """Mean latency (ms) across the evaluation's runs.""" diff --git a/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py b/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py index cda489d418..c713ab8143 100644 --- a/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py +++ b/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py @@ -26,7 +26,6 @@ from nemo_platform import NeMoPlatform, AsyncNeMoPlatform from nemo_platform.pagination import SyncDefaultPagination, AsyncDefaultPagination from nemo_platform.types.experiment_groups import ( - ParetoDataResponse, ExperimentGroupResponse, ) @@ -338,58 +337,6 @@ def test_path_params_delete(self, client: NeMoPlatform) -> None: workspace="workspace", ) - @pytest.mark.skip(reason="Mock server tests are disabled") - @parametrize - def test_method_pareto(self, client: NeMoPlatform) -> None: - experiment_group = client.experiment_groups.pareto( - name="name", - workspace="workspace", - ) - assert_matches_type(ParetoDataResponse, experiment_group, path=["response"]) - - @pytest.mark.skip(reason="Mock server tests are disabled") - @parametrize - def test_raw_response_pareto(self, client: NeMoPlatform) -> None: - response = client.experiment_groups.with_raw_response.pareto( - name="name", - workspace="workspace", - ) - - assert response.is_closed is True - assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = response.parse() - assert_matches_type(ParetoDataResponse, experiment_group, path=["response"]) - - @pytest.mark.skip(reason="Mock server tests are disabled") - @parametrize - def test_streaming_response_pareto(self, client: NeMoPlatform) -> None: - with client.experiment_groups.with_streaming_response.pareto( - name="name", - workspace="workspace", - ) as response: - assert not response.is_closed - assert response.http_request.headers.get("X-Stainless-Lang") == "python" - - experiment_group = response.parse() - assert_matches_type(ParetoDataResponse, experiment_group, path=["response"]) - - assert cast(Any, response.is_closed) is True - - @pytest.mark.skip(reason="Mock server tests are disabled") - @parametrize - def test_path_params_pareto(self, client: NeMoPlatform) -> None: - with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - client.experiment_groups.with_raw_response.pareto( - name="name", - workspace="", - ) - - with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): - client.experiment_groups.with_raw_response.pareto( - name="", - workspace="workspace", - ) - class TestAsyncExperimentGroups: parametrize = pytest.mark.parametrize( @@ -697,55 +644,3 @@ async def test_path_params_delete(self, async_client: AsyncNeMoPlatform) -> None name="", workspace="workspace", ) - - @pytest.mark.skip(reason="Mock server tests are disabled") - @parametrize - async def test_method_pareto(self, async_client: AsyncNeMoPlatform) -> None: - experiment_group = await async_client.experiment_groups.pareto( - name="name", - workspace="workspace", - ) - assert_matches_type(ParetoDataResponse, experiment_group, path=["response"]) - - @pytest.mark.skip(reason="Mock server tests are disabled") - @parametrize - async def test_raw_response_pareto(self, async_client: AsyncNeMoPlatform) -> None: - response = await async_client.experiment_groups.with_raw_response.pareto( - name="name", - workspace="workspace", - ) - - assert response.is_closed is True - assert response.http_request.headers.get("X-Stainless-Lang") == "python" - experiment_group = await response.parse() - assert_matches_type(ParetoDataResponse, experiment_group, path=["response"]) - - @pytest.mark.skip(reason="Mock server tests are disabled") - @parametrize - async def test_streaming_response_pareto(self, async_client: AsyncNeMoPlatform) -> None: - async with async_client.experiment_groups.with_streaming_response.pareto( - name="name", - workspace="workspace", - ) as response: - assert not response.is_closed - assert response.http_request.headers.get("X-Stainless-Lang") == "python" - - experiment_group = await response.parse() - assert_matches_type(ParetoDataResponse, experiment_group, path=["response"]) - - assert cast(Any, response.is_closed) is True - - @pytest.mark.skip(reason="Mock server tests are disabled") - @parametrize - async def test_path_params_pareto(self, async_client: AsyncNeMoPlatform) -> None: - with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): - await async_client.experiment_groups.with_raw_response.pareto( - name="name", - workspace="", - ) - - with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): - await async_client.experiment_groups.with_raw_response.pareto( - name="", - workspace="workspace", - ) diff --git a/sdk/stainless.yaml b/sdk/stainless.yaml index eff59ce7b9..438911b5ef 100644 --- a/sdk/stainless.yaml +++ b/sdk/stainless.yaml @@ -920,15 +920,12 @@ resources: experiment_group_response: ExperimentGroupResponse experiment_group_responses_page: ExperimentGroupResponsesPage pareto_config: ParetoConfig - pareto_data_response: ParetoDataResponse - pareto_metric_point: ParetoMetricPoint methods: create: post /apis/intake/v2/workspaces/{workspace}/experiment-groups list: get /apis/intake/v2/workspaces/{workspace}/experiment-groups retrieve: get /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} update: put /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} delete: delete /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name} - pareto: get /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto evaluations: standalone_api: true models: diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index 24a0a9c2a7..2f277f10da 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -37,8 +37,6 @@ ExperimentGroupFilter, ExperimentGroupRequest, ExperimentGroupResponse, - ParetoDataResponse, - ParetoMetricPoint, ) # The API/Studio expose this as an "Evaluation", but it is still stored as the Experiment entity @@ -278,86 +276,6 @@ async def update_experiment_group( return response -@router.get( - "/v2/workspaces/{workspace}/experiment-groups/{name}/pareto", - response_model=ParetoDataResponse, - tags=[GROUPS_TAG], - responses={ - 404: {"description": "Experiment group not found"}, - 413: {"description": "Group exceeds the per-request evaluation cap"}, - 503: {"description": "Telemetry store unavailable for metric data"}, - }, -) -async def get_experiment_group_pareto( - workspace: str, - name: str, - entity_client: EntityClientDep, - rollup_repository: EvaluationRollupRepositoryDep, -) -> ParetoDataResponse: - """Cost/latency/evaluator means for every evaluation in the group, plus the group's default axes. - - Purpose-built for the Pareto chart: a slim, unpaginated projection of the same rollups the - leaderboard shows, so the client plots the full point set in one call and computes the frontier - from any two metrics without refetching (and without paging the full evaluations list). - """ - group = await _get_or_404(entity_client, ExperimentGroup, workspace=workspace, name=name, label="Experiment group") - _reject_if_deleted(group, workspace=workspace, name=name, label="Experiment group") - - live_in_group = LogicalOperation( - operator=FilterOperator.AND, - operations=[ - _group_membership_filter(group.id), - LogicalOperation( - operator=FilterOperator.NOT, - operations=[ - ComparisonOperation(operator=FilterOperator.EQ, field="data.is_deleted", value=True), - ], - ), - ], - ) - result = await entity_client.list( - Evaluation, - workspace=workspace, - filter_operation=live_in_group, - page=1, - page_size=_MAX_GROUP_EVALUATIONS, - ) - if result.pagination.total_results > _MAX_GROUP_EVALUATIONS: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=( - f"This group has {result.pagination.total_results} evaluations, exceeding the maximum of " - f"{_MAX_GROUP_EVALUATIONS} the Pareto view can plot in one request." - ), - ) - - responses = [EvaluationResponse.from_entity(e) for e in result.data] - hydrated = await _hydrate_rollups(workspace=workspace, responses=responses, rollup_repository=rollup_repository) - # The Pareto view is metric data by definition; without rollups every point would be empty, so - # fail loudly rather than return a chart with nothing to plot. - if not hydrated: - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Cannot build the Pareto view: the telemetry store is unavailable.", - ) - - points = [ - ParetoMetricPoint( - name=response.name, - evaluation_id=response.id, - cost_usd=response.cost_usd.mean if response.cost_usd else None, - latency_ms=response.latency_ms.mean if response.latency_ms else None, - evaluators={ - evaluator: aggregate.mean - for evaluator, aggregate in (response.aggregate_scores or {}).items() - if aggregate.mean is not None - }, - ) - for response in responses - ] - return ParetoDataResponse(pareto=group.pareto, points=points) - - @router.delete( "/v2/workspaces/{workspace}/experiment-groups/{name}", status_code=status.HTTP_204_NO_CONTENT, diff --git a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py index e6a245eb90..2322b5a23e 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py @@ -199,27 +199,6 @@ def from_entity(cls, entity: ExperimentGroup) -> ExperimentGroupResponse: ) -class ParetoMetricPoint(BaseModel): - """One evaluation's plottable metric means for the Pareto view.""" - - name: str = Field(description="Evaluation name — the leaderboard row id and rollup key.") - evaluation_id: str = Field(description="Evaluation entity id, for navigation.") - cost_usd: float | None = Field(default=None, description="Mean cost (USD) across the evaluation's runs.") - latency_ms: float | None = Field(default=None, description="Mean latency (ms) across the evaluation's runs.") - evaluators: dict[str, float] = Field( - default_factory=dict, description="Per-evaluator mean score, keyed by evaluator name." - ) - - -class ParetoDataResponse(BaseModel): - """Everything the Pareto chart needs for a group: the configured default axes plus one point per - evaluation (cost/latency/evaluator means). Unpaginated and slim — the client plots the whole set - and computes the frontier from any two metrics without refetching.""" - - pareto: ParetoConfig = Field(description="The group's configured default X/Y metrics.") - points: list[ParetoMetricPoint] = Field(description="One point per live evaluation in the group.") - - class EvaluatorAggregate(BaseModel): """Aggregate statistics over evaluator scores or session-level metric values.""" diff --git a/services/intake/tests/integration/test_experiments_crud.py b/services/intake/tests/integration/test_experiments_crud.py index ace9e77f49..d2253cec64 100644 --- a/services/intake/tests/integration/test_experiments_crud.py +++ b/services/intake/tests/integration/test_experiments_crud.py @@ -96,50 +96,6 @@ def test_experiment_group_pareto_defaults_and_round_trips(client: TestClient) -> assert updated.json()["pareto"] == {"x_metric": "cost_usd", "y_metric": "evaluators.reward"} -def test_experiment_group_pareto_endpoint_projects_rollup_means(client: TestClient) -> None: - from nmp.intake.spans.evaluation_rollup_repository import EvaluationRollup, ScoreRollup - - def score(mean: float) -> ScoreRollup: - return ScoreRollup(sum=mean, mean=mean, median=mean, p90=mean, p95=mean, p99=mean, count=1) - - group = _create_group(client, name="pareto-points") - for name in ("eval-a", "eval-b"): - response = client.post(EVALUATIONS, json=_evaluation_body(name=name, experiment_group_id=group["id"])) - assert response.status_code == 201, response.text - - class StubRollupRepository: - async def get_rollups(self, *, workspace: str, evaluation_ids: list[str]) -> dict: - return { - "eval-a": EvaluationRollup( - evaluation_id="eval-a", - cost_usd=score(0.10), - latency_ms=score(200.0), - evaluator_scores={"reward": score(0.9)}, - ), - "eval-b": EvaluationRollup( - evaluation_id="eval-b", - cost_usd=score(0.30), - latency_ms=score(150.0), - evaluator_scores={"reward": score(0.8)}, - ), - } - - app = cast(FastAPI, client.app) - app.dependency_overrides[get_evaluation_rollup_repository] = lambda: StubRollupRepository() - try: - response = client.get(f"{GROUPS}/pareto-points/pareto") - assert response.status_code == 200, response.text - body = response.json() - assert body["pareto"] == {"x_metric": "cost_usd", "y_metric": "latency_ms"} - points = {point["name"]: point for point in body["points"]} - assert set(points) == {"eval-a", "eval-b"} - assert points["eval-a"]["cost_usd"] == 0.10 - assert points["eval-a"]["latency_ms"] == 200.0 - assert points["eval-a"]["evaluators"] == {"reward": 0.9} - finally: - app.dependency_overrides.pop(get_evaluation_rollup_repository, None) - - def test_evaluation_update_moves_between_groups_and_edits(client: TestClient) -> None: group_a = client.post(GROUPS, json={"name": "grp-a"}).json() group_b = client.post(GROUPS, json={"name": "grp-b"}).json() diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx index 50075db99d..2914265b4b 100644 --- a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx +++ b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx @@ -3,9 +3,7 @@ import { useToast } from '@nemo/common/src/providers/toast/useToast'; import { - getGetExperimentGroupParetoQueryKey, getGetExperimentGroupQueryKey, - useGetExperimentGroupPareto, useUpdateExperimentGroup, } from '@nemo/sdk/generated/platform/api'; import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; @@ -25,6 +23,7 @@ import { type ParetoMetric, type ParetoPlotPoint, } from '@studio/components/charts/ExperimentGroupParetoChart/paretoMetrics'; +import { useParetoEvaluations } from '@studio/components/charts/ExperimentGroupParetoChart/useParetoEvaluations'; import { useQueryClient } from '@tanstack/react-query'; import { Loader2, Save } from 'lucide-react'; import { type FC, useMemo, useState } from 'react'; @@ -132,7 +131,8 @@ const MetricSelect: FC = ({ label, value, metrics, onChange } /** * Cost-vs-accuracy Pareto view for an experiment group: one point per evaluation with the Pareto - * frontier highlighted. Points come from the group's `/pareto` endpoint. The two axes are chosen from + * frontier highlighted. Points come from the group's evaluations (the existing list endpoint, which + * already carries each evaluation's cost/latency/evaluator rollup means). The two axes are chosen from * the group's available metrics (cost, latency, and each evaluator) and are **persisted on the group** * — changing a picker saves the selection so it survives reloads and is shared across viewers. Seeds * from the group's saved axes, defaulting to cost vs. latency (present for every group). @@ -144,8 +144,7 @@ export const ExperimentGroupParetoChart: FC = ( const queryClient = useQueryClient(); const toast = useToast(); - const { data, isLoading, isError } = useGetExperimentGroupPareto(workspace, group.name); - const points = useMemo(() => data?.points ?? [], [data]); + const { rows: points, isLoading, isError } = useParetoEvaluations(workspace, group.id); const metrics = useMemo(() => deriveParetoMetrics(points), [points]); // Selected axes are optimistic local state seeded from the group's saved config (available @@ -160,9 +159,6 @@ export const ExperimentGroupParetoChart: FC = ( queryClient.invalidateQueries({ queryKey: getGetExperimentGroupQueryKey(workspace, group.name), }); - queryClient.invalidateQueries({ - queryKey: getGetExperimentGroupParetoQueryKey(workspace, group.name), - }); }, onError: () => toast.error('Failed to save the Pareto metrics.'), }, diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.test.ts b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.test.ts index 0c8bc0ef16..a01111b50f 100644 --- a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.test.ts +++ b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.test.ts @@ -1,25 +1,30 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { ParetoMetricPoint } from '@nemo/sdk/generated/platform/schema'; import { buildParetoPoints, deriveParetoMetrics, type ParetoMetric, } from '@studio/components/charts/ExperimentGroupParetoChart/paretoMetrics'; +import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; +// Only the fields the Pareto accessors read (name + cost/latency/evaluator rollup means) are set; +// the rest of the rich EvaluationRow shape is irrelevant here, so build a minimal stand-in. const point = (opts: { name: string; cost?: number; latency?: number; evaluators?: Record; -}): ParetoMetricPoint => ({ - name: opts.name, - evaluation_id: opts.name, - cost_usd: opts.cost, - latency_ms: opts.latency, - evaluators: opts.evaluators, -}); +}): EvaluationRow => + ({ + name: opts.name, + id: opts.name, + cost_usd: opts.cost == null ? undefined : { mean: opts.cost }, + latency_ms: opts.latency == null ? undefined : { mean: opts.latency }, + aggregate_scores: opts.evaluators + ? Object.fromEntries(Object.entries(opts.evaluators).map(([name, mean]) => [name, { mean }])) + : undefined, + }) as unknown as EvaluationRow; const getMetric = (metrics: ParetoMetric[], id: string): ParetoMetric => { const metric = metrics.find((m) => m.id === id); @@ -27,7 +32,7 @@ const getMetric = (metrics: ParetoMetric[], id: string): ParetoMetric => { return metric; }; -const frontierNames = (points: ParetoMetricPoint[], x: ParetoMetric, y: ParetoMetric): string[] => +const frontierNames = (points: EvaluationRow[], x: ParetoMetric, y: ParetoMetric): string[] => buildParetoPoints(points, x, y) .filter((p) => p.onFrontier) .map((p) => p.name) diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.ts b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.ts index cff0146310..f31a367ed0 100644 --- a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.ts +++ b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/paretoMetrics.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { ParetoMetricPoint } from '@nemo/sdk/generated/platform/schema'; +import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; /** Which direction on an axis counts as "better": cost/latency minimize, evaluator scores maximize. */ export type MetricDirection = 'min' | 'max'; @@ -14,7 +14,7 @@ export interface ParetoMetric { readonly id: string; readonly label: string; readonly direction: MetricDirection; - readonly accessor: (point: ParetoMetricPoint) => number | null | undefined; + readonly accessor: (row: EvaluationRow) => number | null | undefined; } const capitalize = (value: string): string => @@ -36,30 +36,30 @@ const COST_METRIC: ParetoMetric = { id: 'cost_usd', label: metricLabel('cost_usd'), direction: 'min', - accessor: (point) => point.cost_usd, + accessor: (row) => row.cost_usd?.mean, }; const LATENCY_METRIC: ParetoMetric = { id: 'latency_ms', label: metricLabel('latency_ms'), direction: 'min', - accessor: (point) => point.latency_ms, + accessor: (row) => row.latency_ms?.mean, }; /** * The metrics a user may plot on either axis: cost and latency (always present, minimized), plus one - * option per evaluator seen across the group's points (maximized). Evaluator names are dynamic — they - * differ per customer — so they're derived from the data rather than hardcoded. + * option per evaluator seen across the group's evaluations (maximized). Evaluator names are dynamic — + * they differ per customer — so they're derived from the data rather than hardcoded. */ -export function deriveParetoMetrics(points: readonly ParetoMetricPoint[]): ParetoMetric[] { +export function deriveParetoMetrics(rows: readonly EvaluationRow[]): ParetoMetric[] { const evaluatorNames = [ - ...new Set(points.flatMap((point) => Object.keys(point.evaluators ?? {}))), + ...new Set(rows.flatMap((row) => Object.keys(row.aggregate_scores ?? {}))), ].sort(); const evaluatorMetrics = evaluatorNames.map((name) => ({ id: `evaluators.${name}`, label: metricLabel(`evaluators.${name}`), direction: 'max', - accessor: (point) => point.evaluators?.[name], + accessor: (row) => row.aggregate_scores?.[name]?.mean, })); return [COST_METRIC, LATENCY_METRIC, ...evaluatorMetrics]; } @@ -95,16 +95,16 @@ function dominates(a: Coords, b: Coords, xDir: MetricDirection, yDir: MetricDire * dominated by any other on both axes. Points missing either metric (non-finite) are dropped. */ export function buildParetoPoints( - points: readonly ParetoMetricPoint[], + rows: readonly EvaluationRow[], xMetric: ParetoMetric, yMetric: ParetoMetric ): ParetoPlotPoint[] { - const coords = points - .map((point): { name: string; x: number; y: number } | null => { - const x = xMetric.accessor(point); - const y = yMetric.accessor(point); + const coords = rows + .map((row): { name: string; x: number; y: number } | null => { + const x = xMetric.accessor(row); + const y = yMetric.accessor(row); if (x == null || y == null || !Number.isFinite(x) || !Number.isFinite(y)) return null; - return { name: point.name, x, y }; + return { name: row.name, x, y }; }) .filter((point): point is { name: string; x: number; y: number } => point !== null); diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts new file mode 100644 index 0000000000..470085f706 --- /dev/null +++ b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { useListEvaluations } from '@nemo/sdk/generated/platform/api'; +import type { EvaluationFilter } from '@nemo/sdk/generated/platform/schema'; +import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; +import { useMemo } from 'react'; + +/** The list endpoint caps page_size at 1000; a group's evaluation set is far smaller, so one page + * covers every point the Pareto chart needs (the leaderboard, by contrast, is paginated). */ +const MAX_EVALUATIONS = 1000; + +export interface ParetoEvaluations { + rows: EvaluationRow[]; + isLoading: boolean; + isError: boolean; +} + +/** + * Loads every evaluation in a group in one unpaginated request for the Pareto chart. Reuses the + * existing list endpoint — each evaluation already carries the cost/latency/evaluator rollup means the + * chart plots — so no dedicated endpoint is needed. + */ +export function useParetoEvaluations( + workspace: string, + experimentGroupId: string +): ParetoEvaluations { + const { data, isLoading, isError } = useListEvaluations( + workspace, + { + page: 1, + page_size: MAX_EVALUATIONS, + filter: { experiment_group_id: experimentGroupId } as EvaluationFilter, + }, + { query: { enabled: !!experimentGroupId } } + ); + + const rows = useMemo( + () => + (data?.data ?? []).map((evaluation) => ({ + ...evaluation, + id: evaluation.id ?? evaluation.name ?? '', + })), + [data] + ); + + return { rows, isLoading, isError }; +} From 7eae404c437ba3f8d1580d7cc6d0212bdb823d56 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Fri, 24 Jul 2026 10:05:42 -0600 Subject: [PATCH 06/11] lint Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- .../core/auth/src/nmp/core/auth/assets/static-authz.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml index b74903127d..3ccf30203e 100644 --- a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml +++ b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml @@ -1003,13 +1003,6 @@ authz: scopes: - intake:write - platform:write - /apis/intake/v2/workspaces/{workspace}/experiment-groups/{name}/pareto: - get: - permissions: - - intake.experiment-groups.read - scopes: - - intake:read - - platform:read /apis/intake/v2/workspaces/{workspace}/experiments: get: permissions: From 840769fbaaf5ae8c2fe0f0af988d3099331c0610 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:17:01 -0600 Subject: [PATCH 07/11] add loading state to pareto chart Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- .../charts/ExperimentGroupParetoChart/index.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx index 2914265b4b..31dbbbc984 100644 --- a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx +++ b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx @@ -213,7 +213,14 @@ export const ExperimentGroupParetoChart: FC = ( return Could not load the Pareto data for this group.; } if (isLoading) { - return
; + return ( +
+ + + Loading evaluations… + +
+ ); } if (plotPoints.length === 0 || !xMetric || !yMetric) { return ( From 9253cb4d2782fce84504de9e9173e60fd2a97ce5 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:53:43 -0600 Subject: [PATCH 08/11] query optimization Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- .../spans/evaluation_rollup_repository.py | 83 +++++++++++-------- .../test_experiment_rollup_repository.py | 10 ++- .../ExperimentGroupParetoChart/index.tsx | 27 +++++- .../useParetoEvaluations.ts | 9 +- .../ExperimentGroupDataView/index.tsx | 21 ++++- 5 files changed, 109 insertions(+), 41 deletions(-) diff --git a/services/intake/src/nmp/intake/spans/evaluation_rollup_repository.py b/services/intake/src/nmp/intake/spans/evaluation_rollup_repository.py index 227a609682..1019be6c5e 100644 --- a/services/intake/src/nmp/intake/spans/evaluation_rollup_repository.py +++ b/services/intake/src/nmp/intake/spans/evaluation_rollup_repository.py @@ -11,7 +11,16 @@ from nmp.intake.spans.clickhouse_client import ClickHouseSpanClient from nmp.intake.spans.span_attribute_catalog import COST_SCALE, SpanAttributeField, spec_for_field from nmp.intake.spans.storage import float_or_none, result_rows -from nmp.intake.spans.trace_repository import current_spans_sql + +# Let large rollups spill to disk instead of OOMing the server (ClickHouse code 241). With the lean +# span projection below these thresholds aren't reached for normal groups; they're a safety net for +# very large ones (many sessions/spans) so a heavy group degrades gracefully rather than erroring. +_ROLLUP_QUERY_SETTINGS = { + "max_bytes_before_external_group_by": 2_000_000_000, + "max_bytes_before_external_sort": 2_000_000_000, + "join_algorithm": "auto", + "max_bytes_in_join": 2_000_000_000, +} @dataclass(frozen=True) @@ -73,6 +82,7 @@ async def get_rollups(self, *, workspace: str, evaluation_ids: list[str]) -> dic evaluation_names_sql=evaluation_names_sql, ), parameters=parameters, + settings=_ROLLUP_QUERY_SETTINGS, ) ): rollups[row["evaluation_id"]].evaluator_scores[row["evaluator_name"]] = ScoreRollup( @@ -99,6 +109,7 @@ async def get_rollups(self, *, workspace: str, evaluation_ids: list[str]) -> dic "agent_name_key": spec_for_field(SpanAttributeField.AGENT_NAME).bag_key, "agent_version_key": spec_for_field(SpanAttributeField.AGENT_VERSION).bag_key, }, + settings=_ROLLUP_QUERY_SETTINGS, ) ): rollup = rollups[row["evaluation_id"]] @@ -316,6 +327,35 @@ def _test_case_scores_cte() -> str: test_cases.evaluation_id, test_cases.test_case_key, evaluators.evaluator_name, test_cases.session_count""" +def _current_session_span_metrics_sql(spans_table: str) -> str: + """Latest-version spans for the scoped sessions, projected to *only* the metric inputs. + + The shared ``current_spans_sql`` emits every span column, including the full ``attributes_*`` Maps. + The metric rollup then joins that to the sessions, so the join hash table would hold every deduped + span with its maps — gigabytes on real agent workloads (long trajectories × many spans), which is + what trips ClickHouse's memory limit. Here we ``argMax`` just the cost value (plus a present-flag, + to keep an absent cost distinct from a real 0) and the model/agent strings, so the join carries a + handful of scalars per span instead. Deduping by span identity picks each span's latest version. + """ + return f""" + ( + SELECT + workspace, + argMax(session_id, (event_ts, is_deleted)) AS dedup_session_id, + argMax(attributes_number[%(cost_key)s], (event_ts, is_deleted)) AS cost_value, + argMax(has(mapKeys(attributes_number), %(cost_key)s), (event_ts, is_deleted)) AS cost_present, + argMax(attributes_string[%(model_key)s], (event_ts, is_deleted)) AS model_name, + argMax(attributes_string[%(agent_name_key)s], (event_ts, is_deleted)) AS agent_name, + argMax(attributes_string[%(agent_version_key)s], (event_ts, is_deleted)) AS agent_version, + argMax(is_deleted, (event_ts, is_deleted)) AS del_flag + FROM {spans_table} + WHERE workspace = %(workspace)s + AND (workspace, session_id) IN (SELECT DISTINCT workspace, session_id FROM scoped_sessions) + GROUP BY workspace, source_format, trace_id, external_span_id, id + ) + """ + + def _metric_rollups_sql(*, trace_index_table: str, spans_table: str, evaluation_names_sql: str) -> str: # Two-level rollup: per-attempt cost/latency, then averaged per test case (avg per attempt — the # number must not scale with k), then the distribution across test cases (test-case-weighted). @@ -326,50 +366,25 @@ def _metric_rollups_sql(*, trace_index_table: str, spans_table: str, evaluation_ scoped_sessions AS ( {_scoped_sessions_sql(trace_index_table, evaluation_names_sql)} ), - current_session_spans AS ( - { - current_spans_sql( - spans_table, - extra_where_sql=( - "(span_versions.workspace, span_versions.session_id) IN " - "(SELECT DISTINCT workspace, session_id FROM scoped_sessions)" - ), - ) - } - ), + current_session_spans AS {_current_session_span_metrics_sql(spans_table)}, session_costs AS ( SELECT sessions.evaluation_id AS evaluation_id, sessions.test_case_id AS test_case_key, sessions.latency_ms AS latency_ms, if( - countIf(has(mapKeys(spans.attributes_number), %(cost_key)s)) = 0, + countIf(spans.cost_present) = 0, NULL, - sumIf( - spans.attributes_number[%(cost_key)s], - has(mapKeys(spans.attributes_number), %(cost_key)s) - ) / {COST_SCALE} + sumIf(spans.cost_value, spans.cost_present) / {COST_SCALE} ) AS cost_usd, - groupUniqArrayIf( - spans.attributes_string[%(model_key)s], - has(mapKeys(spans.attributes_string), %(model_key)s) - AND spans.attributes_string[%(model_key)s] != '' - ) AS model_names, - groupUniqArrayIf( - spans.attributes_string[%(agent_name_key)s], - has(mapKeys(spans.attributes_string), %(agent_name_key)s) - AND spans.attributes_string[%(agent_name_key)s] != '' - ) AS agent_names, - groupUniqArrayIf( - spans.attributes_string[%(agent_version_key)s], - has(mapKeys(spans.attributes_string), %(agent_version_key)s) - AND spans.attributes_string[%(agent_version_key)s] != '' - ) AS agent_versions + groupUniqArrayIf(spans.model_name, spans.model_name != '') AS model_names, + groupUniqArrayIf(spans.agent_name, spans.agent_name != '') AS agent_names, + groupUniqArrayIf(spans.agent_version, spans.agent_version != '') AS agent_versions FROM scoped_sessions AS sessions LEFT JOIN current_session_spans AS spans ON sessions.workspace = spans.workspace - AND sessions.session_id = spans.session_id - AND spans.is_deleted = 0 + AND sessions.session_id = spans.dedup_session_id + AND spans.del_flag = 0 WHERE sessions.test_case_id != '' GROUP BY sessions.evaluation_id, sessions.session_id, sessions.test_case_id, sessions.latency_ms ), diff --git a/services/intake/tests/test_experiment_rollup_repository.py b/services/intake/tests/test_experiment_rollup_repository.py index e3dd105af0..27a73dafea 100644 --- a/services/intake/tests/test_experiment_rollup_repository.py +++ b/services/intake/tests/test_experiment_rollup_repository.py @@ -25,7 +25,9 @@ def __init__(self, query_results: list[_QueryResult]) -> None: def table(self, name: str) -> str: return name - async def query(self, query: str, *, parameters: dict[str, object]) -> _QueryResult: + async def query( + self, query: str, *, parameters: dict[str, object], settings: dict[str, object] | None = None + ) -> _QueryResult: self.queries.append(query) self.parameters.append(parameters) return self.query_results.pop(0) @@ -149,9 +151,11 @@ async def test_evaluation_rollups_anchor_on_root_session_membership(): assert "WHERE sessions.test_case_id != ''" in client.queries[1] assert "test_case_metrics AS" in client.queries[2] assert "current_session_spans AS" in client.queries[2] - assert "(span_versions.workspace, span_versions.session_id) IN" in client.queries[2] + assert ( + "(workspace, session_id) IN (SELECT DISTINCT workspace, session_id FROM scoped_sessions)" in client.queries[2] + ) assert "LEFT JOIN current_session_spans AS spans" in client.queries[2] - assert "sessions.session_id = spans.session_id" in client.queries[2] + assert "sessions.session_id = spans.dedup_session_id" in client.queries[2] assert "arraySort(arrayDistinct(arrayFlatten(groupArray(model_names)))) AS model_names" in client.queries[2] assert "quantileExactIf(0.5)" in client.queries[2] assert "cost_median" in client.queries[2] diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx index 31dbbbc984..e85f75a2f3 100644 --- a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx +++ b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx @@ -24,6 +24,7 @@ import { type ParetoPlotPoint, } from '@studio/components/charts/ExperimentGroupParetoChart/paretoMetrics'; import { useParetoEvaluations } from '@studio/components/charts/ExperimentGroupParetoChart/useParetoEvaluations'; +import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; import { useQueryClient } from '@tanstack/react-query'; import { Loader2, Save } from 'lucide-react'; import { type FC, useMemo, useState } from 'react'; @@ -40,6 +41,18 @@ import { interface ExperimentGroupParetoChartProps { workspace: string; group: ExperimentGroupResponse; + /** + * The group's full evaluation set, when the caller already has it loaded (e.g. the leaderboard fit + * the whole group on one page). Supplying it lets the chart skip its own all-evaluations fetch — + * which re-runs the same server-side rollup — and render from the shared rows instead. + */ + preloadedEvaluations?: EvaluationRow[]; + /** + * True while the caller is still loading and is expected to supply `preloadedEvaluations` once done + * (page 1 of an unfiltered group). The chart shows a loading state and holds off its own fetch, + * rather than briefly rendering empty before the shared rows arrive. + */ + preloadPending?: boolean; } const CHART_HEIGHT = 360; @@ -140,11 +153,23 @@ const MetricSelect: FC = ({ label, value, metrics, onChange } export const ExperimentGroupParetoChart: FC = ({ workspace, group, + preloadedEvaluations, + preloadPending = false, }) => { const queryClient = useQueryClient(); const toast = useToast(); - const { rows: points, isLoading, isError } = useParetoEvaluations(workspace, group.id); + // Reuse the caller's rows when it already has the whole group loaded. While it's still loading and + // about to supply them (`preloadPending`), hold off our own fetch and show a loading state instead of + // rendering empty. Otherwise fetch the evaluations ourselves. + const hasPreloaded = preloadedEvaluations !== undefined; + const { + rows: fetchedRows, + isLoading: isFetching, + isError, + } = useParetoEvaluations(workspace, group.id, { enabled: !hasPreloaded && !preloadPending }); + const points = hasPreloaded ? preloadedEvaluations : fetchedRows; + const isLoading = hasPreloaded ? false : preloadPending || isFetching; const metrics = useMemo(() => deriveParetoMetrics(points), [points]); // Selected axes are optimistic local state seeded from the group's saved config (available diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts index 470085f706..56700d30ed 100644 --- a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts +++ b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts @@ -23,8 +23,13 @@ export interface ParetoEvaluations { */ export function useParetoEvaluations( workspace: string, - experimentGroupId: string + experimentGroupId: string, + options?: { enabled?: boolean } ): ParetoEvaluations { + // Callers can disable the fetch (e.g. the leaderboard already loaded the whole group on one page, + // so its rows are reused and this extra all-evaluations request — which re-runs the same server-side + // rollup — is redundant). + const enabled = (options?.enabled ?? true) && !!experimentGroupId; const { data, isLoading, isError } = useListEvaluations( workspace, { @@ -32,7 +37,7 @@ export function useParetoEvaluations( page_size: MAX_EVALUATIONS, filter: { experiment_group_id: experimentGroupId } as EvaluationFilter, }, - { query: { enabled: !!experimentGroupId } } + { query: { enabled } } ); const rows = useMemo( diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx index 374386e83f..4f6d3399ea 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx @@ -460,13 +460,32 @@ export const ExperimentGroupDataView: FC = ({ return ; } + // When the whole (unfiltered) group already fits on the first page, the loaded rows ARE the complete + // evaluation set — hand them to the Pareto chart so it can skip its own all-evaluations fetch (which + // re-runs the same server-side rollup). We only know the group fits one page once the list has + // loaded (`totalCount` is 0 mid-load), so gate on `!isLoading`; while loading on page 1 we signal + // `preloadPending` so the chart shows a loading state instead of the empty set. Any search/filter, + // or a group larger than one page, falls back to the chart fetching for itself. + const hasActiveFilter = Object.keys(dataViewState.apiFilter.filter ?? {}).length > 0; + const onFirstUnfilteredPage = + page === 1 && !dataViewState.debouncedSearchBar && !hasActiveFilter; + const completeEvaluationSet = + onFirstUnfilteredPage && !isLoading && totalCount <= pageSize ? orderedData : undefined; + const preloadPending = onFirstUnfilteredPage && isLoading; + return ( <> {paretoVisible && (
{/* Key by group id so the axis selection resets (re-seeds from the new group's saved config) when navigating between groups without a route remount. */} - +
)} Date: Fri, 24 Jul 2026 12:21:54 -0600 Subject: [PATCH 09/11] drop spill safety net Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- .../spans/evaluation_rollup_repository.py | 12 ---------- .../test_experiment_rollup_repository.py | 4 +--- .../ExperimentGroupParetoChart/index.tsx | 14 +++++------ .../useParetoEvaluations.ts | 6 ++--- .../ExperimentGroupDataView/index.tsx | 24 ++++++++++--------- 5 files changed, 24 insertions(+), 36 deletions(-) diff --git a/services/intake/src/nmp/intake/spans/evaluation_rollup_repository.py b/services/intake/src/nmp/intake/spans/evaluation_rollup_repository.py index 1019be6c5e..e82e5fbc89 100644 --- a/services/intake/src/nmp/intake/spans/evaluation_rollup_repository.py +++ b/services/intake/src/nmp/intake/spans/evaluation_rollup_repository.py @@ -12,16 +12,6 @@ from nmp.intake.spans.span_attribute_catalog import COST_SCALE, SpanAttributeField, spec_for_field from nmp.intake.spans.storage import float_or_none, result_rows -# Let large rollups spill to disk instead of OOMing the server (ClickHouse code 241). With the lean -# span projection below these thresholds aren't reached for normal groups; they're a safety net for -# very large ones (many sessions/spans) so a heavy group degrades gracefully rather than erroring. -_ROLLUP_QUERY_SETTINGS = { - "max_bytes_before_external_group_by": 2_000_000_000, - "max_bytes_before_external_sort": 2_000_000_000, - "join_algorithm": "auto", - "max_bytes_in_join": 2_000_000_000, -} - @dataclass(frozen=True) class ScoreRollup: @@ -82,7 +72,6 @@ async def get_rollups(self, *, workspace: str, evaluation_ids: list[str]) -> dic evaluation_names_sql=evaluation_names_sql, ), parameters=parameters, - settings=_ROLLUP_QUERY_SETTINGS, ) ): rollups[row["evaluation_id"]].evaluator_scores[row["evaluator_name"]] = ScoreRollup( @@ -109,7 +98,6 @@ async def get_rollups(self, *, workspace: str, evaluation_ids: list[str]) -> dic "agent_name_key": spec_for_field(SpanAttributeField.AGENT_NAME).bag_key, "agent_version_key": spec_for_field(SpanAttributeField.AGENT_VERSION).bag_key, }, - settings=_ROLLUP_QUERY_SETTINGS, ) ): rollup = rollups[row["evaluation_id"]] diff --git a/services/intake/tests/test_experiment_rollup_repository.py b/services/intake/tests/test_experiment_rollup_repository.py index 27a73dafea..d4c202e54b 100644 --- a/services/intake/tests/test_experiment_rollup_repository.py +++ b/services/intake/tests/test_experiment_rollup_repository.py @@ -25,9 +25,7 @@ def __init__(self, query_results: list[_QueryResult]) -> None: def table(self, name: str) -> str: return name - async def query( - self, query: str, *, parameters: dict[str, object], settings: dict[str, object] | None = None - ) -> _QueryResult: + async def query(self, query: str, *, parameters: dict[str, object]) -> _QueryResult: self.queries.append(query) self.parameters.append(parameters) return self.query_results.pop(0) diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx index e85f75a2f3..7cd2d3b636 100644 --- a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx +++ b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/index.tsx @@ -42,15 +42,15 @@ interface ExperimentGroupParetoChartProps { workspace: string; group: ExperimentGroupResponse; /** - * The group's full evaluation set, when the caller already has it loaded (e.g. the leaderboard fit - * the whole group on one page). Supplying it lets the chart skip its own all-evaluations fetch — - * which re-runs the same server-side rollup — and render from the shared rows instead. + * The group's full evaluation set, when the caller already has it loaded (a small group that fit on + * the leaderboard's first page). Supplying it lets the chart render from the shared rows and skip its + * own all-evaluations fetch — which re-runs the same server-side rollup. */ preloadedEvaluations?: EvaluationRow[]; /** - * True while the caller is still loading and is expected to supply `preloadedEvaluations` once done - * (page 1 of an unfiltered group). The chart shows a loading state and holds off its own fetch, - * rather than briefly rendering empty before the shared rows arrive. + * True while the caller is still loading rows it will supply via `preloadedEvaluations`. The chart + * shows a loading state and holds off its own fetch, rather than briefly rendering empty. Only set + * for groups known (from `evaluation_count`) to fit one page — larger groups fetch immediately. */ preloadPending?: boolean; } @@ -161,7 +161,7 @@ export const ExperimentGroupParetoChart: FC = ( // Reuse the caller's rows when it already has the whole group loaded. While it's still loading and // about to supply them (`preloadPending`), hold off our own fetch and show a loading state instead of - // rendering empty. Otherwise fetch the evaluations ourselves. + // rendering empty. Otherwise fetch the evaluations ourselves (in parallel — see the `enabled` flag). const hasPreloaded = preloadedEvaluations !== undefined; const { rows: fetchedRows, diff --git a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts index 56700d30ed..475aca56a9 100644 --- a/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts +++ b/web/packages/studio/src/components/charts/ExperimentGroupParetoChart/useParetoEvaluations.ts @@ -26,9 +26,9 @@ export function useParetoEvaluations( experimentGroupId: string, options?: { enabled?: boolean } ): ParetoEvaluations { - // Callers can disable the fetch (e.g. the leaderboard already loaded the whole group on one page, - // so its rows are reused and this extra all-evaluations request — which re-runs the same server-side - // rollup — is redundant). + // Callers disable the fetch when they already have the whole group loaded (a small group that fit on + // the leaderboard's first page), so this all-evaluations request — which re-runs the same server-side + // rollup — is skipped entirely. const enabled = (options?.enabled ?? true) && !!experimentGroupId; const { data, isLoading, isError } = useListEvaluations( workspace, diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx index 4f6d3399ea..b4c39b277b 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx @@ -460,18 +460,20 @@ export const ExperimentGroupDataView: FC = ({ return ; } - // When the whole (unfiltered) group already fits on the first page, the loaded rows ARE the complete - // evaluation set — hand them to the Pareto chart so it can skip its own all-evaluations fetch (which - // re-runs the same server-side rollup). We only know the group fits one page once the list has - // loaded (`totalCount` is 0 mid-load), so gate on `!isLoading`; while loading on page 1 we signal - // `preloadPending` so the chart shows a loading state instead of the empty set. Any search/filter, - // or a group larger than one page, falls back to the chart fetching for itself. + // When the whole group fits on the leaderboard's first page and isn't filtered, those loaded rows are + // the complete evaluation set — reuse them for the Pareto chart instead of refetching every + // evaluation. `evaluation_count` lets us decide this without waiting on the list query. While the + // page loads, `preloadPending` keeps the chart in its loading state; otherwise the chart fetches its + // own data in parallel. const hasActiveFilter = Object.keys(dataViewState.apiFilter.filter ?? {}).length > 0; - const onFirstUnfilteredPage = - page === 1 && !dataViewState.debouncedSearchBar && !hasActiveFilter; - const completeEvaluationSet = - onFirstUnfilteredPage && !isLoading && totalCount <= pageSize ? orderedData : undefined; - const preloadPending = onFirstUnfilteredPage && isLoading; + const groupFitsOnePage = + group.evaluation_count != null && + group.evaluation_count <= pageSize && + page === 1 && + !dataViewState.debouncedSearchBar && + !hasActiveFilter; + const completeEvaluationSet = groupFitsOnePage && !isLoading ? orderedData : undefined; + const preloadPending = groupFitsOnePage && isLoading; return ( <> From 07a3bee74a25462e42810e8bbde307535c987eb4 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Fri, 24 Jul 2026 12:34:21 -0600 Subject: [PATCH 10/11] fix(experiments): validate Pareto axes and preserve them on partial updates Address CodeRabbit review on PR #882: - Restrict ParetoConfig x/y metrics to cost_usd, latency_ms, or evaluators. so invalid identifiers can't be persisted as unusable chart defaults. - Make ExperimentGroupRequest.pareto optional so an update that omits it keeps the saved axes instead of resetting them to the cost/latency default. - Regenerate the OpenAPI spec; add tests covering rejection and preservation. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- openapi/ga/individual/platform.openapi.yaml | 4 ++-- openapi/ga/openapi.yaml | 4 ++-- openapi/openapi.yaml | 4 ++-- .../intake/api/v2/experiments/endpoints.py | 5 ++++- .../nmp/intake/api/v2/experiments/schemas.py | 9 ++++++--- .../src/nmp/intake/entities/experiments.py | 13 +++++++++++++ .../integration/test_experiments_crud.py | 19 +++++++++++++++++++ 7 files changed, 48 insertions(+), 10 deletions(-) diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index 1cec818a3a..41040f985f 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -11273,8 +11273,8 @@ components: pareto: allOf: - $ref: '#/components/schemas/ParetoConfig' - description: Default X/Y metrics for the group's Pareto view. Defaults to - cost vs. latency. + description: Default X/Y metrics for the group's Pareto view. Omit to preserve + the existing value on update; on create, defaults to cost vs. latency. additionalProperties: false type: object required: diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index 1cec818a3a..41040f985f 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -11273,8 +11273,8 @@ components: pareto: allOf: - $ref: '#/components/schemas/ParetoConfig' - description: Default X/Y metrics for the group's Pareto view. Defaults to - cost vs. latency. + description: Default X/Y metrics for the group's Pareto view. Omit to preserve + the existing value on update; on create, defaults to cost vs. latency. additionalProperties: false type: object required: diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 1cec818a3a..41040f985f 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -11273,8 +11273,8 @@ components: pareto: allOf: - $ref: '#/components/schemas/ParetoConfig' - description: Default X/Y metrics for the group's Pareto view. Defaults to - cost vs. latency. + description: Default X/Y metrics for the group's Pareto view. Omit to preserve + the existing value on update; on create, defaults to cost vs. latency. additionalProperties: false type: object required: diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index 2f277f10da..96d9ecfeb0 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -267,7 +267,10 @@ async def update_experiment_group( existing.summary = body.summary existing.metadata = body.metadata existing.default_sort = body.default_sort - existing.pareto = body.pareto + # Only overwrite the saved axes when the client actually sent them; an omitted `pareto` (older + # clients) must not silently reset customized axes to the cost/latency default. + if body.pareto is not None: + existing.pareto = body.pareto updated = await entity_client.update(existing) response = ExperimentGroupResponse.from_entity(updated) response.evaluation_count = await _count_live_evaluations_in_group( diff --git a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py index 2322b5a23e..940417cc62 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py @@ -48,9 +48,12 @@ class ExperimentGroupRequest(BaseModel): "the list `sort` param." ), ) - pareto: ParetoConfig = Field( - default_factory=ParetoConfig, - description="Default X/Y metrics for the group's Pareto view. Defaults to cost vs. latency.", + pareto: ParetoConfig | None = Field( + default=None, + description=( + "Default X/Y metrics for the group's Pareto view. Omit to preserve the existing value on " + "update; on create, defaults to cost vs. latency." + ), ) diff --git a/services/intake/src/nmp/intake/entities/experiments.py b/services/intake/src/nmp/intake/entities/experiments.py index 437c4fa987..fc72db8603 100644 --- a/services/intake/src/nmp/intake/entities/experiments.py +++ b/services/intake/src/nmp/intake/entities/experiments.py @@ -45,6 +45,19 @@ class ParetoConfig(BaseModel): x_metric: str = Field(default="cost_usd", description="Metric plotted on the Pareto X axis.") y_metric: str = Field(default="latency_ms", description="Metric plotted on the Pareto Y axis.") + @field_validator("x_metric", "y_metric") + @classmethod + def _validate_metric(cls, value: str) -> str: + """Restrict axes to metrics Studio can actually plot: the two fixed metrics or a dynamic + ``evaluators.``. Evaluator names are customer-specific, so only the prefix is checked.""" + if value in ("cost_usd", "latency_ms"): + return value + if value.startswith("evaluators.") and value != "evaluators.": + return value + raise ValueError( + f"Unsupported Pareto metric {value!r}; expected 'cost_usd', 'latency_ms', or 'evaluators.'." + ) + class ExperimentGroup(EntityBase): """A named container of Experiments pursuing a single optimization goal. diff --git a/services/intake/tests/integration/test_experiments_crud.py b/services/intake/tests/integration/test_experiments_crud.py index d2253cec64..b9a0aca778 100644 --- a/services/intake/tests/integration/test_experiments_crud.py +++ b/services/intake/tests/integration/test_experiments_crud.py @@ -95,6 +95,25 @@ def test_experiment_group_pareto_defaults_and_round_trips(client: TestClient) -> assert updated.status_code == 200, updated.text assert updated.json()["pareto"] == {"x_metric": "cost_usd", "y_metric": "evaluators.reward"} + # An update that omits pareto preserves the saved axes rather than resetting them to the default. + preserved = client.put(f"{GROUPS}/pareto-cfg", json={"name": "pareto-cfg", "summary": "unrelated edit"}) + assert preserved.status_code == 200, preserved.text + assert preserved.json()["pareto"] == {"x_metric": "cost_usd", "y_metric": "evaluators.reward"} + + +def test_experiment_group_pareto_rejects_unknown_metric(client: TestClient) -> None: + # Studio can only plot cost_usd, latency_ms, or evaluators.; anything else is rejected so it + # can't be persisted as an unusable chart default. + rejected = client.post( + GROUPS, json={"name": "bad-pareto", "pareto": {"x_metric": "made_up", "y_metric": "latency_ms"}} + ) + assert rejected.status_code == 422, rejected.text + # A dynamic evaluator metric is allowed. + ok = client.post( + GROUPS, json={"name": "ok-pareto", "pareto": {"x_metric": "evaluators.safety", "y_metric": "cost_usd"}} + ) + assert ok.status_code == 201, ok.text + def test_evaluation_update_moves_between_groups_and_edits(client: TestClient) -> None: group_a = client.post(GROUPS, json={"name": "grp-a"}).json() From 408999a80310873dca2991adc89fe72eacaf8999 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:16:14 -0600 Subject: [PATCH 11/11] update sdk Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- sdk/python/nemo-platform/.nmpcontext/openapi.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index 1cec818a3a..41040f985f 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -11273,8 +11273,8 @@ components: pareto: allOf: - $ref: '#/components/schemas/ParetoConfig' - description: Default X/Y metrics for the group's Pareto view. Defaults to - cost vs. latency. + description: Default X/Y metrics for the group's Pareto view. Omit to preserve + the existing value on update; on create, defaults to cost vs. latency. additionalProperties: false type: object required: