From eb2fe4c37961069f8e6b3617e3132444ffe9545f Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Mon, 6 Jul 2026 12:48:04 -0600 Subject: [PATCH 1/4] feat(experiments): add group default metric sort, applied client-side Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- openapi/ga/individual/platform.openapi.yaml | 52 ++----- openapi/ga/openapi.yaml | 52 ++----- openapi/openapi.yaml | 52 ++----- .../nemo-platform/.nmpcontext/openapi.yaml | 52 ++----- .../nemo-platform/.nmpcontext/stainless.yaml | 1 - .../resources/experiment_groups/api.md | 1 - .../experiment_groups/experiment_groups.py | 47 +++--- .../resources/experiments/experiments.py | 6 +- .../types/experiment_groups/__init__.py | 2 - .../experiment_group_create_params.py | 13 +- .../experiment_group_response.py | 5 +- .../experiment_group_update_params.py | 12 +- .../types/experiment_groups/sort_criterion.py | 37 ----- .../experiment_groups/sort_criterion_param.py | 37 ----- .../experiments/experiment_list_params.py | 3 +- .../api_resources/test_experiment_groups.py | 28 +--- sdk/stainless.yaml | 1 - .../intake/api/v2/experiments/endpoints.py | 62 +++----- .../nmp/intake/api/v2/experiments/schemas.py | 15 +- .../src/nmp/intake/entities/experiments.py | 23 +-- .../tests/test_experiment_default_sort.py | 82 ++++------ .../components/DefaultSortControl/index.tsx | 145 ++++++++++++++++++ .../DefaultSortControl/util.test.ts | 29 ++++ .../src/components/DefaultSortControl/util.ts | 30 ++++ .../ExperimentGroupCreateModal/index.tsx | 19 ++- .../ExperimentGroupEditModal/index.tsx | 121 +++++++++++++++ .../ExperimentGroupDataView/index.tsx | 54 ++++--- .../useExperimentGroupExperiments.ts | 3 +- .../ExperimentGroupDetailRoute/index.tsx | 23 ++- 29 files changed, 562 insertions(+), 445 deletions(-) delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/sort_criterion.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/sort_criterion_param.py create mode 100644 web/packages/studio/src/components/DefaultSortControl/index.tsx create mode 100644 web/packages/studio/src/components/DefaultSortControl/util.test.ts create mode 100644 web/packages/studio/src/components/DefaultSortControl/util.ts create mode 100644 web/packages/studio/src/components/ExperimentGroupEditModal/index.tsx diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index 5ec9968a2d..269b2efe17 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -3716,16 +3716,15 @@ paths: an experiment attribute (name, created_at, updated_at, pinned_at) or by an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., where is one of mean, median, p90, p95, - p99, sum, count. When omitted, the group''s configured default sort is - used (falling back to -created_at), with pinned experiments first.' + p99, sum, count. When omitted, defaults to -created_at with pinned experiments + first.' title: Sort type: string description: 'Field to sort by; prefix with ''-'' for descending. Sort by an experiment attribute (name, created_at, updated_at, pinned_at) or by an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., where is one of mean, median, p90, p95, p99, sum, count. When omitted, - the group''s configured default sort is used (falling back to -created_at), - with pinned experiments first.' + defaults to -created_at with pinned experiments first.' - in: query name: filter style: deepObject @@ -10455,15 +10454,13 @@ components: description: Free-form producer metadata for the group. additionalProperties: true type: object - default_sort: - title: Default Sort - description: 'Ordered default sort (priority order; first is primary, rest - are tiebreakers) for this group''s experiments list. Each field must be - a numeric rollup metric: run_count, cost_usd., latency_ms., - or evaluators...' - items: - $ref: '#/components/schemas/SortCriterion' - type: array + default_metric_sort: + title: Default Metric Sort + description: 'Default sort for this group''s experiments list, as a `sort`-param + string (leading ''-'' = descending), e.g. ''-cost_usd.mean''. Clients + apply it as the list `sort` param. The field must be a numeric rollup + metric: run_count, cost_usd., latency_ms., or evaluators...' + type: string additionalProperties: false type: object required: @@ -10494,11 +10491,9 @@ components: title: Metadata additionalProperties: true type: object - default_sort: - title: Default Sort - items: - $ref: '#/components/schemas/SortCriterion' - type: array + default_metric_sort: + title: Default Metric Sort + type: string created_at: title: Created At type: string @@ -17096,27 +17091,6 @@ components: - window_size title: SlidingWindowConfig description: Sliding window attention configuration. - SortCriterion: - properties: - field: - type: string - title: Field - description: Rollup-metric sort path, e.g. cost_usd.mean, latency_ms.p95, - or evaluators..mean. - direction: - type: string - enum: - - asc - - desc - title: Direction - description: Sort direction for this field. - type: object - required: - - field - - direction - title: SortCriterion - description: 'One criterion in a group''s default sort: a sortable rollup-metric - path and its direction.' Span: properties: span_id: diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index 5ec9968a2d..269b2efe17 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -3716,16 +3716,15 @@ paths: an experiment attribute (name, created_at, updated_at, pinned_at) or by an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., where is one of mean, median, p90, p95, - p99, sum, count. When omitted, the group''s configured default sort is - used (falling back to -created_at), with pinned experiments first.' + p99, sum, count. When omitted, defaults to -created_at with pinned experiments + first.' title: Sort type: string description: 'Field to sort by; prefix with ''-'' for descending. Sort by an experiment attribute (name, created_at, updated_at, pinned_at) or by an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., where is one of mean, median, p90, p95, p99, sum, count. When omitted, - the group''s configured default sort is used (falling back to -created_at), - with pinned experiments first.' + defaults to -created_at with pinned experiments first.' - in: query name: filter style: deepObject @@ -10455,15 +10454,13 @@ components: description: Free-form producer metadata for the group. additionalProperties: true type: object - default_sort: - title: Default Sort - description: 'Ordered default sort (priority order; first is primary, rest - are tiebreakers) for this group''s experiments list. Each field must be - a numeric rollup metric: run_count, cost_usd., latency_ms., - or evaluators...' - items: - $ref: '#/components/schemas/SortCriterion' - type: array + default_metric_sort: + title: Default Metric Sort + description: 'Default sort for this group''s experiments list, as a `sort`-param + string (leading ''-'' = descending), e.g. ''-cost_usd.mean''. Clients + apply it as the list `sort` param. The field must be a numeric rollup + metric: run_count, cost_usd., latency_ms., or evaluators...' + type: string additionalProperties: false type: object required: @@ -10494,11 +10491,9 @@ components: title: Metadata additionalProperties: true type: object - default_sort: - title: Default Sort - items: - $ref: '#/components/schemas/SortCriterion' - type: array + default_metric_sort: + title: Default Metric Sort + type: string created_at: title: Created At type: string @@ -17096,27 +17091,6 @@ components: - window_size title: SlidingWindowConfig description: Sliding window attention configuration. - SortCriterion: - properties: - field: - type: string - title: Field - description: Rollup-metric sort path, e.g. cost_usd.mean, latency_ms.p95, - or evaluators..mean. - direction: - type: string - enum: - - asc - - desc - title: Direction - description: Sort direction for this field. - type: object - required: - - field - - direction - title: SortCriterion - description: 'One criterion in a group''s default sort: a sortable rollup-metric - path and its direction.' Span: properties: span_id: diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 5ec9968a2d..269b2efe17 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -3716,16 +3716,15 @@ paths: an experiment attribute (name, created_at, updated_at, pinned_at) or by an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., where is one of mean, median, p90, p95, - p99, sum, count. When omitted, the group''s configured default sort is - used (falling back to -created_at), with pinned experiments first.' + p99, sum, count. When omitted, defaults to -created_at with pinned experiments + first.' title: Sort type: string description: 'Field to sort by; prefix with ''-'' for descending. Sort by an experiment attribute (name, created_at, updated_at, pinned_at) or by an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., where is one of mean, median, p90, p95, p99, sum, count. When omitted, - the group''s configured default sort is used (falling back to -created_at), - with pinned experiments first.' + defaults to -created_at with pinned experiments first.' - in: query name: filter style: deepObject @@ -10455,15 +10454,13 @@ components: description: Free-form producer metadata for the group. additionalProperties: true type: object - default_sort: - title: Default Sort - description: 'Ordered default sort (priority order; first is primary, rest - are tiebreakers) for this group''s experiments list. Each field must be - a numeric rollup metric: run_count, cost_usd., latency_ms., - or evaluators...' - items: - $ref: '#/components/schemas/SortCriterion' - type: array + default_metric_sort: + title: Default Metric Sort + description: 'Default sort for this group''s experiments list, as a `sort`-param + string (leading ''-'' = descending), e.g. ''-cost_usd.mean''. Clients + apply it as the list `sort` param. The field must be a numeric rollup + metric: run_count, cost_usd., latency_ms., or evaluators...' + type: string additionalProperties: false type: object required: @@ -10494,11 +10491,9 @@ components: title: Metadata additionalProperties: true type: object - default_sort: - title: Default Sort - items: - $ref: '#/components/schemas/SortCriterion' - type: array + default_metric_sort: + title: Default Metric Sort + type: string created_at: title: Created At type: string @@ -17096,27 +17091,6 @@ components: - window_size title: SlidingWindowConfig description: Sliding window attention configuration. - SortCriterion: - properties: - field: - type: string - title: Field - description: Rollup-metric sort path, e.g. cost_usd.mean, latency_ms.p95, - or evaluators..mean. - direction: - type: string - enum: - - asc - - desc - title: Direction - description: Sort direction for this field. - type: object - required: - - field - - direction - title: SortCriterion - description: 'One criterion in a group''s default sort: a sortable rollup-metric - path and its direction.' Span: properties: span_id: diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index 5ec9968a2d..269b2efe17 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -3716,16 +3716,15 @@ paths: an experiment attribute (name, created_at, updated_at, pinned_at) or by an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., where is one of mean, median, p90, p95, - p99, sum, count. When omitted, the group''s configured default sort is - used (falling back to -created_at), with pinned experiments first.' + p99, sum, count. When omitted, defaults to -created_at with pinned experiments + first.' title: Sort type: string description: 'Field to sort by; prefix with ''-'' for descending. Sort by an experiment attribute (name, created_at, updated_at, pinned_at) or by an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., where is one of mean, median, p90, p95, p99, sum, count. When omitted, - the group''s configured default sort is used (falling back to -created_at), - with pinned experiments first.' + defaults to -created_at with pinned experiments first.' - in: query name: filter style: deepObject @@ -10455,15 +10454,13 @@ components: description: Free-form producer metadata for the group. additionalProperties: true type: object - default_sort: - title: Default Sort - description: 'Ordered default sort (priority order; first is primary, rest - are tiebreakers) for this group''s experiments list. Each field must be - a numeric rollup metric: run_count, cost_usd., latency_ms., - or evaluators...' - items: - $ref: '#/components/schemas/SortCriterion' - type: array + default_metric_sort: + title: Default Metric Sort + description: 'Default sort for this group''s experiments list, as a `sort`-param + string (leading ''-'' = descending), e.g. ''-cost_usd.mean''. Clients + apply it as the list `sort` param. The field must be a numeric rollup + metric: run_count, cost_usd., latency_ms., or evaluators...' + type: string additionalProperties: false type: object required: @@ -10494,11 +10491,9 @@ components: title: Metadata additionalProperties: true type: object - default_sort: - title: Default Sort - items: - $ref: '#/components/schemas/SortCriterion' - type: array + default_metric_sort: + title: Default Metric Sort + type: string created_at: title: Created At type: string @@ -17096,27 +17091,6 @@ components: - window_size title: SlidingWindowConfig description: Sliding window attention configuration. - SortCriterion: - properties: - field: - type: string - title: Field - description: Rollup-metric sort path, e.g. cost_usd.mean, latency_ms.p95, - or evaluators..mean. - direction: - type: string - enum: - - asc - - desc - title: Direction - description: Sort direction for this field. - type: object - required: - - field - - direction - title: SortCriterion - description: 'One criterion in a group''s default sort: a sortable rollup-metric - path and its direction.' Span: properties: span_id: diff --git a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml index b3c8bdbd11..e0dc819e23 100644 --- a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml @@ -904,7 +904,6 @@ resources: experiment_group_request: ExperimentGroupRequest experiment_group_response: ExperimentGroupResponse experiment_group_responses_page: ExperimentGroupResponsesPage - sort_criterion: SortCriterion methods: create: post /apis/intake/v2/workspaces/{workspace}/experiment-groups list: get /apis/intake/v2/workspaces/{workspace}/experiment-groups diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md index 6aa7d698e4..877402928d 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/api.md @@ -8,7 +8,6 @@ from nemo_platform.types.experiment_groups import ( ExperimentGroupRequest, ExperimentGroupResponse, ExperimentGroupResponsesPage, - SortCriterion, ) ``` diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py index 64a28250d6..0a4ac0475c 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py @@ -17,7 +17,7 @@ from __future__ import annotations -from typing import Dict, Iterable +from typing import Dict from typing_extensions import Literal import httpx @@ -39,7 +39,6 @@ experiment_group_create_params, experiment_group_update_params, ) -from ...types.experiment_groups.sort_criterion_param import SortCriterionParam from ...types.experiment_groups.experiment_group_response import ExperimentGroupResponse from ...types.experiment_groups.experiment_group_filter_param import ExperimentGroupFilterParam from ..._exceptions import ConflictError @@ -72,7 +71,7 @@ def create( *, workspace: str | None = None, name: str, - default_sort: Iterable[SortCriterionParam] | Omit = omit, + default_metric_sort: str | Omit = omit, description: str | Omit = omit, insight_id: str | Omit = omit, metadata: Dict[str, object] | Omit = omit, @@ -91,9 +90,10 @@ def create( Args: name: Workspace-unique group name. - default_sort: Ordered default sort (priority order; first is primary, rest are tiebreakers) - for this group's experiments list. Each field must be a numeric rollup metric: - run_count, cost_usd., latency_ms., or evaluators... + default_metric_sort: Default sort for this group's experiments list, as a `sort`-param string + (leading '-' = descending), e.g. '-cost_usd.mean'. Clients apply it as the list + `sort` param. The field must be a numeric rollup metric: run_count, + cost_usd., latency_ms., or evaluators... description: Human-readable purpose of the group. @@ -125,7 +125,7 @@ def create( body=maybe_transform( { "name": name, - "default_sort": default_sort, + "default_metric_sort": default_metric_sort, "description": description, "insight_id": insight_id, "metadata": metadata, @@ -189,7 +189,7 @@ def update( *, workspace: str | None = None, body_name: str, - default_sort: Iterable[SortCriterionParam] | Omit = omit, + default_metric_sort: str | Omit = omit, description: str | Omit = omit, insight_id: str | Omit = omit, metadata: Dict[str, object] | Omit = omit, @@ -207,9 +207,10 @@ def update( Args: body_name: Workspace-unique group name. - default_sort: Ordered default sort (priority order; first is primary, rest are tiebreakers) - for this group's experiments list. Each field must be a numeric rollup metric: - run_count, cost_usd., latency_ms., or evaluators... + default_metric_sort: Default sort for this group's experiments list, as a `sort`-param string + (leading '-' = descending), e.g. '-cost_usd.mean'. Clients apply it as the list + `sort` param. The field must be a numeric rollup metric: run_count, + cost_usd., latency_ms., or evaluators... description: Human-readable purpose of the group. @@ -242,7 +243,7 @@ def update( body=maybe_transform( { "body_name": body_name, - "default_sort": default_sort, + "default_metric_sort": default_metric_sort, "description": description, "insight_id": insight_id, "metadata": metadata, @@ -383,7 +384,7 @@ async def create( *, workspace: str | None = None, name: str, - default_sort: Iterable[SortCriterionParam] | Omit = omit, + default_metric_sort: str | Omit = omit, description: str | Omit = omit, insight_id: str | Omit = omit, metadata: Dict[str, object] | Omit = omit, @@ -402,9 +403,10 @@ async def create( Args: name: Workspace-unique group name. - default_sort: Ordered default sort (priority order; first is primary, rest are tiebreakers) - for this group's experiments list. Each field must be a numeric rollup metric: - run_count, cost_usd., latency_ms., or evaluators... + default_metric_sort: Default sort for this group's experiments list, as a `sort`-param string + (leading '-' = descending), e.g. '-cost_usd.mean'. Clients apply it as the list + `sort` param. The field must be a numeric rollup metric: run_count, + cost_usd., latency_ms., or evaluators... description: Human-readable purpose of the group. @@ -436,7 +438,7 @@ async def create( body=await async_maybe_transform( { "name": name, - "default_sort": default_sort, + "default_metric_sort": default_metric_sort, "description": description, "insight_id": insight_id, "metadata": metadata, @@ -500,7 +502,7 @@ async def update( *, workspace: str | None = None, body_name: str, - default_sort: Iterable[SortCriterionParam] | Omit = omit, + default_metric_sort: str | Omit = omit, description: str | Omit = omit, insight_id: str | Omit = omit, metadata: Dict[str, object] | Omit = omit, @@ -518,9 +520,10 @@ async def update( Args: body_name: Workspace-unique group name. - default_sort: Ordered default sort (priority order; first is primary, rest are tiebreakers) - for this group's experiments list. Each field must be a numeric rollup metric: - run_count, cost_usd., latency_ms., or evaluators... + default_metric_sort: Default sort for this group's experiments list, as a `sort`-param string + (leading '-' = descending), e.g. '-cost_usd.mean'. Clients apply it as the list + `sort` param. The field must be a numeric rollup metric: run_count, + cost_usd., latency_ms., or evaluators... description: Human-readable purpose of the group. @@ -553,7 +556,7 @@ async def update( body=await async_maybe_transform( { "body_name": body_name, - "default_sort": default_sort, + "default_metric_sort": default_metric_sort, "description": description, "insight_id": insight_id, "metadata": metadata, diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py index be460ff43f..a6306c438b 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py @@ -333,8 +333,7 @@ def list( attribute (name, created_at, updated_at, pinned_at) or by an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., where is one of mean, median, p90, p95, p99, sum, count. When omitted, - the group's configured default sort is used (falling back to -created_at), with - pinned experiments first. + defaults to -created_at with pinned experiments first. extra_headers: Send extra headers @@ -774,8 +773,7 @@ def list( attribute (name, created_at, updated_at, pinned_at) or by an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., where is one of mean, median, p90, p95, p99, sum, count. When omitted, - the group's configured default sort is used (falling back to -created_at), with - pinned experiments first. + defaults to -created_at with pinned experiments first. extra_headers: Send extra headers diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py index 24251bd6f6..3506053fad 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/__init__.py @@ -17,8 +17,6 @@ from __future__ import annotations -from .sort_criterion import SortCriterion as SortCriterion -from .sort_criterion_param import SortCriterionParam as SortCriterionParam from .experiment_group_response import ExperimentGroupResponse as ExperimentGroupResponse from .experiment_group_list_params import ExperimentGroupListParams as ExperimentGroupListParams from .experiment_group_filter_param import ExperimentGroupFilterParam as ExperimentGroupFilterParam diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py index 61ff10c7ce..cb294d7d08 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py @@ -17,11 +17,9 @@ from __future__ import annotations -from typing import Dict, Iterable +from typing import Dict from typing_extensions import Required, TypedDict -from .sort_criterion_param import SortCriterionParam - __all__ = ["ExperimentGroupCreateParams"] @@ -31,11 +29,12 @@ class ExperimentGroupCreateParams(TypedDict, total=False): name: Required[str] """Workspace-unique group name.""" - default_sort: Iterable[SortCriterionParam] + default_metric_sort: str """ - Ordered default sort (priority order; first is primary, rest are tiebreakers) - for this group's experiments list. Each field must be a numeric rollup metric: - run_count, cost_usd., latency_ms., or evaluators... + Default sort for this group's experiments list, as a `sort`-param string + (leading '-' = descending), e.g. '-cost_usd.mean'. Clients apply it as the list + `sort` param. The field must be a numeric rollup metric: run_count, + cost_usd., latency_ms., or evaluators... """ description: str diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py index 59056155d2..18c8f0bebb 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py @@ -15,11 +15,10 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from typing import Dict, List, Optional +from typing import Dict, Optional from datetime import datetime from ..._models import BaseModel -from .sort_criterion import SortCriterion __all__ = ["ExperimentGroupResponse"] @@ -35,7 +34,7 @@ class ExperimentGroupResponse(BaseModel): created_at: Optional[datetime] = None - default_sort: Optional[List[SortCriterion]] = None + default_metric_sort: Optional[str] = None description: Optional[str] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py index a3018f2963..9e623a844e 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py @@ -17,11 +17,10 @@ from __future__ import annotations -from typing import Dict, Iterable +from typing import Dict from typing_extensions import Required, Annotated, TypedDict from ..._utils import PropertyInfo -from .sort_criterion_param import SortCriterionParam __all__ = ["ExperimentGroupUpdateParams"] @@ -32,11 +31,12 @@ class ExperimentGroupUpdateParams(TypedDict, total=False): body_name: Required[Annotated[str, PropertyInfo(alias="name")]] """Workspace-unique group name.""" - default_sort: Iterable[SortCriterionParam] + default_metric_sort: str """ - Ordered default sort (priority order; first is primary, rest are tiebreakers) - for this group's experiments list. Each field must be a numeric rollup metric: - run_count, cost_usd., latency_ms., or evaluators... + Default sort for this group's experiments list, as a `sort`-param string + (leading '-' = descending), e.g. '-cost_usd.mean'. Clients apply it as the list + `sort` param. The field must be a numeric rollup metric: run_count, + cost_usd., latency_ms., or evaluators... """ description: str diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/sort_criterion.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/sort_criterion.py deleted file mode 100644 index a52fd45767..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/sort_criterion.py +++ /dev/null @@ -1,37 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -from typing_extensions import Literal - -from ..._models import BaseModel - -__all__ = ["SortCriterion"] - - -class SortCriterion(BaseModel): - """ - One criterion in a group's default sort: a sortable rollup-metric path and its direction. - """ - - direction: Literal["asc", "desc"] - """Sort direction for this field.""" - - field: str - """Rollup-metric sort path, e.g. - - cost_usd.mean, latency_ms.p95, or evaluators..mean. - """ diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/sort_criterion_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/sort_criterion_param.py deleted file mode 100644 index 29d050dc02..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/sort_criterion_param.py +++ /dev/null @@ -1,37 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -from __future__ import annotations - -from typing_extensions import Literal, Required, TypedDict - -__all__ = ["SortCriterionParam"] - - -class SortCriterionParam(TypedDict, total=False): - """ - One criterion in a group's default sort: a sortable rollup-metric path and its direction. - """ - - direction: Required[Literal["asc", "desc"]] - """Sort direction for this field.""" - - field: Required[str] - """Rollup-metric sort path, e.g. - - cost_usd.mean, latency_ms.p95, or evaluators..mean. - """ diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py index 88e36efd90..cb131246eb 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py @@ -50,6 +50,5 @@ class ExperimentListParams(TypedDict, total=False): Sort by an experiment attribute (name, created_at, updated_at, pinned_at) or by an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., where is one of mean, median, p90, p95, p99, - sum, count. When omitted, the group's configured default sort is used (falling - back to -created_at), with pinned experiments first. + sum, count. When omitted, defaults to -created_at with pinned experiments first. """ diff --git a/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py b/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py index ef091e307f..316069388e 100644 --- a/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py +++ b/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py @@ -50,12 +50,7 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: experiment_group = client.experiment_groups.create( workspace="workspace", name="name", - default_sort=[ - { - "direction": "asc", - "field": "field", - } - ], + default_metric_sort="default_metric_sort", description="description", insight_id="insight_id", metadata={"foo": "bar"}, @@ -169,12 +164,7 @@ def test_method_update_with_all_params(self, client: NeMoPlatform) -> None: path_name="name", workspace="workspace", body_name="name", - default_sort=[ - { - "direction": "asc", - "field": "field", - } - ], + default_metric_sort="default_metric_sort", description="description", insight_id="insight_id", metadata={"foo": "bar"}, @@ -359,12 +349,7 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo experiment_group = await async_client.experiment_groups.create( workspace="workspace", name="name", - default_sort=[ - { - "direction": "asc", - "field": "field", - } - ], + default_metric_sort="default_metric_sort", description="description", insight_id="insight_id", metadata={"foo": "bar"}, @@ -478,12 +463,7 @@ async def test_method_update_with_all_params(self, async_client: AsyncNeMoPlatfo path_name="name", workspace="workspace", body_name="name", - default_sort=[ - { - "direction": "asc", - "field": "field", - } - ], + default_metric_sort="default_metric_sort", description="description", insight_id="insight_id", metadata={"foo": "bar"}, diff --git a/sdk/stainless.yaml b/sdk/stainless.yaml index b3c8bdbd11..e0dc819e23 100644 --- a/sdk/stainless.yaml +++ b/sdk/stainless.yaml @@ -904,7 +904,6 @@ resources: experiment_group_request: ExperimentGroupRequest experiment_group_response: ExperimentGroupResponse experiment_group_responses_page: ExperimentGroupResponsesPage - sort_criterion: SortCriterion methods: create: post /apis/intake/v2/workspaces/{workspace}/experiment-groups list: get /apis/intake/v2/workspaces/{workspace}/experiment-groups diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index 9705236129..9946537454 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -35,7 +35,7 @@ ExperimentSessionFilter, ExperimentSessionResponse, ) -from nmp.intake.entities.experiments import Experiment, ExperimentGroup, SortCriterion +from nmp.intake.entities.experiments import Experiment, ExperimentGroup from nmp.intake.spans.api.dependencies import require_workspace_access, validate_list_query_params from nmp.intake.spans.clickhouse_client import ClickHouseSpanClient from nmp.intake.spans.domain import SpanStatus @@ -118,7 +118,7 @@ async def create_experiment_group( body: ExperimentGroupRequest, entity_client: EntityClientDep, ) -> ExperimentGroupResponse: - _validate_default_sort(body.default_sort) + _validate_default_metric_sort(body.default_metric_sort) entity = ExperimentGroup( workspace=workspace, name=body.name, @@ -126,7 +126,7 @@ async def create_experiment_group( insight_id=body.insight_id, summary=body.summary, metadata=body.metadata, - default_sort=body.default_sort, + default_metric_sort=body.default_metric_sort, ) try: created = await entity_client.create(entity) @@ -236,12 +236,12 @@ async def update_experiment_group( status_code=status.HTTP_409_CONFLICT, detail="Cannot rename an experiment group; the name is its identity.", ) - _validate_default_sort(body.default_sort) + _validate_default_metric_sort(body.default_metric_sort) existing.description = body.description existing.insight_id = body.insight_id existing.summary = body.summary existing.metadata = body.metadata - existing.default_sort = body.default_sort + existing.default_metric_sort = body.default_metric_sort updated = await entity_client.update(existing) response = ExperimentGroupResponse.from_entity(updated) response.experiment_count = await _count_live_experiments_in_group( @@ -384,16 +384,15 @@ async def list_experiments( "Field to sort by; prefix with '-' for descending. Sort by an experiment attribute " "(name, created_at, updated_at, pinned_at) or by an aggregate metric: run_count, " "cost_usd., latency_ms., or evaluators.., where is one of " - "mean, median, p90, p95, p99, sum, count. When omitted, the group's configured default sort " - "is used (falling back to -created_at), with pinned experiments first." + "mean, median, p90, p95, p99, sum, count. When omitted, defaults to -created_at with pinned " + "experiments first." ), ), ) -> Page[ExperimentResponse]: validate_list_query_params(request) _apply_is_deleted_filter(parsed) _apply_is_pinned_filter(parsed) - # An explicit `sort` overrides the group's default sort. When omitted, fall back to that default - # sort (then -created_at), with pinned experiments floated to the top. + # When omitted, fall back to -created_at with pinned experiments floated to the top. if sort is not None: descending = sort.startswith("-") sort_field = sort[1:] if descending else sort @@ -402,7 +401,7 @@ async def list_experiments( pinned_first = False explicit_metric_sort = sort_field not in _ENTITY_SORT_FIELDS else: - sort_keys = await _default_sort_keys(entity_client, parsed) + sort_keys = [("created_at", True)] pinned_first = True explicit_metric_sort = False # Rollup-metric predicates live in ClickHouse, not the entity store, so they can't be pushed to @@ -1096,37 +1095,22 @@ def _experiment_sort_value(response: ExperimentResponse, field: str) -> Any: return getattr(score, stat, None) if score is not None else None -def _validate_default_sort(default_sort: list[SortCriterion] | None) -> None: - """Reject a default sort whose fields aren't numeric rollup metrics (run_count / `.`).""" - for entry in default_sort or []: - if not _is_valid_metric_path(entry.field): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Unsupported sort field: {entry.field}. Use a numeric rollup metric " - "(run_count, cost_usd., latency_ms., or evaluators..)." - ), - ) - - -async def _default_sort_keys(entity_client: EntityClient, parsed: ParsedFilter) -> list[tuple[str, bool]]: - """Resolve the default sort keys when no explicit ``sort`` is passed. +def _validate_default_metric_sort(default_metric_sort: str | None) -> None: + """Reject a default metric sort whose field isn't a numeric rollup metric. - When the query is scoped to a single experiment_group_id and that group has a default sort, use it - in priority order. ``-created_at`` is always appended as the final key so ordering falls back - gracefully when sort values are missing/unresolved (or no default sort is set). + The value is a ``sort``-param string (optional leading '-' for descending), e.g. ``-cost_usd.mean``. """ - keys: list[tuple[str, bool]] = [] - group_id = parsed.extract("experiment_group_id") - if isinstance(group_id, str): - try: - group = await entity_client.get_by_id(ExperimentGroup, entity_id=group_id) - except EntityNotFoundError: - group = None - if group is not None and group.default_sort: - keys = [(entry.field, entry.direction == "desc") for entry in group.default_sort] - keys.append(("created_at", True)) - return keys + if default_metric_sort is None: + return + field = default_metric_sort[1:] if default_metric_sort.startswith("-") else default_metric_sort + if not _is_valid_metric_path(field): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Unsupported sort field: {field}. Use a numeric rollup metric " + "(run_count, cost_usd., latency_ms., or evaluators..)." + ), + ) def _sort_experiments( diff --git a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py index 4e67ce4147..1c2857a24f 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py @@ -13,7 +13,7 @@ from typing import Annotated, Any from nmp.common.entities.values import DatetimeFilter, Filter, NumberFilter, map_entity_field -from nmp.intake.entities.experiments import Experiment, ExperimentGroup, SortCriterion +from nmp.intake.entities.experiments import Experiment, ExperimentGroup from nmp.intake.spans.domain import SpanStatus from nmp.intake.spans.experiment_session_repository import ExperimentSessionRow from pydantic import AnyUrl, BaseModel, ConfigDict, Field @@ -31,12 +31,13 @@ class ExperimentGroupRequest(BaseModel): ) summary: str | None = Field(default=None, description="Human- or agent-authored summary of the group's findings.") metadata: dict[str, Any] | None = Field(default=None, description="Free-form producer metadata for the group.") - default_sort: list[SortCriterion] | None = Field( + default_metric_sort: str | None = Field( default=None, description=( - "Ordered default sort (priority order; first is primary, rest are tiebreakers) for this " - "group's experiments list. Each field must be a numeric rollup metric: run_count, " - "cost_usd., latency_ms., or evaluators..." + "Default sort for this group's experiments list, as a `sort`-param string (leading '-' = " + "descending), e.g. '-cost_usd.mean'. Clients apply it as the list `sort` param. The field " + "must be a numeric rollup metric: run_count, cost_usd., latency_ms., or " + "evaluators..." ), ) @@ -76,7 +77,7 @@ class ExperimentGroupResponse(BaseModel): insight_id: str | None = None summary: str | None = None metadata: dict[str, Any] | None = None - default_sort: list[SortCriterion] | None = None + default_metric_sort: str | None = None created_at: datetime | None = None updated_at: datetime | None = None experiment_count: int = Field( @@ -94,7 +95,7 @@ def from_entity(cls, entity: ExperimentGroup) -> ExperimentGroupResponse: insight_id=entity.insight_id, summary=entity.summary, metadata=entity.metadata, - default_sort=entity.default_sort, + default_metric_sort=entity.default_metric_sort, created_at=entity.created_at, updated_at=entity.updated_at, ) diff --git a/services/intake/src/nmp/intake/entities/experiments.py b/services/intake/src/nmp/intake/entities/experiments.py index e0aeff751a..6404a97c73 100644 --- a/services/intake/src/nmp/intake/entities/experiments.py +++ b/services/intake/src/nmp/intake/entities/experiments.py @@ -11,19 +11,10 @@ from __future__ import annotations from datetime import datetime -from typing import Any, ClassVar, Literal +from typing import Any, ClassVar from nmp.common.entities.client import EntityBase -from pydantic import AnyUrl, BaseModel, Field - - -class SortCriterion(BaseModel): - """One criterion in a group's default sort: a sortable rollup-metric path and its direction.""" - - field: str = Field( - description="Rollup-metric sort path, e.g. cost_usd.mean, latency_ms.p95, or evaluators..mean." - ) - direction: Literal["asc", "desc"] = Field(description="Sort direction for this field.") +from pydantic import AnyUrl, Field class ExperimentGroup(EntityBase): @@ -41,12 +32,14 @@ class ExperimentGroup(EntityBase): ) summary: str | None = Field(default=None, description="Human- or agent-authored summary of the group's findings.") metadata: dict[str, Any] | None = Field(default=None, description="Free-form producer metadata for the group.") - default_sort: list[SortCriterion] | None = Field( + default_metric_sort: str | None = Field( default=None, description=( - "Ordered default sort in priority order (first is primary, the rest are tiebreakers). When " - "set, it is the default order for this group's experiments list. Each field must be a numeric " - "rollup metric: run_count, cost_usd., latency_ms., or evaluators..." + "Default sort for this group's experiments list, as a `sort`-param string (a leading '-' " + "means descending), e.g. '-cost_usd.mean'. The client reads it from the group and applies " + "it as the list's `sort` param; the list endpoint itself does not consult it. The field " + "must be a numeric rollup metric: run_count, cost_usd., latency_ms., or " + "evaluators..." ), ) is_deleted: bool = Field( diff --git a/services/intake/tests/test_experiment_default_sort.py b/services/intake/tests/test_experiment_default_sort.py index f4114d6acf..de486391d0 100644 --- a/services/intake/tests/test_experiment_default_sort.py +++ b/services/intake/tests/test_experiment_default_sort.py @@ -1,11 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Experiment group default sort: multi-key ordering, pinned-first, fallback, and validation. +"""Experiment group default metric sort: string storage/validation and the sort helper. -The shared ``client`` fixture overrides the rollup repository to ``None`` (ClickHouse unavailable), -which lets us verify that a metric-based *default* sort degrades to ``-created_at`` instead of -failing — distinct from an explicit metric sort, which still 503s. +``default_metric_sort`` is a single ``sort``-param string (e.g. ``-cost_usd.mean``) stored on the +group. The client reads it and applies it as the list ``sort`` param; the list endpoint itself never +consults it. The ``_sort_experiments`` helper remains multi-key capable (pinned-first + tiebreaks), +so its unit tests still exercise lists. """ from datetime import datetime, timezone @@ -13,9 +14,8 @@ import pytest from fastapi import HTTPException from fastapi.testclient import TestClient -from nmp.intake.api.v2.experiments.endpoints import _sort_experiments, _validate_default_sort +from nmp.intake.api.v2.experiments.endpoints import _sort_experiments, _validate_default_metric_sort from nmp.intake.api.v2.experiments.schemas import EvaluatorAggregate, ExperimentResponse -from nmp.intake.entities.experiments import SortCriterion EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" @@ -76,38 +76,47 @@ def test_falls_back_to_created_at_when_sorted_metric_missing() -> None: # ----------------------------- default-sort validation ----------------------------- -def test_validate_default_sort_accepts_metric_fields() -> None: - _validate_default_sort( - [ - SortCriterion(field="cost_usd.mean", direction="asc"), - SortCriterion(field="latency_ms.p95", direction="asc"), - SortCriterion(field="run_count", direction="desc"), - SortCriterion(field="evaluators.harbor.verifier.mean", direction="desc"), - ] - ) +def test_validate_default_metric_sort_accepts_metric_fields() -> None: + # Both ascending and descending ('-') sort-param strings are valid. + for value in ("cost_usd.mean", "-latency_ms.p95", "run_count", "-evaluators.harbor.verifier.mean"): + _validate_default_metric_sort(value) + _validate_default_metric_sort(None) # absent default sort is fine -def test_validate_default_sort_rejects_non_metric_fields() -> None: - for field in ("name", "created_at", "cost_usd.bogus", "evaluators.reward"): +def test_validate_default_metric_sort_rejects_non_metric_fields() -> None: + for value in ("name", "-created_at", "cost_usd.bogus", "evaluators.reward"): with pytest.raises(HTTPException) as exc: - _validate_default_sort([SortCriterion(field=field, direction="asc")]) + _validate_default_metric_sort(value) assert exc.value.status_code == 400 +def test_entity_ignores_legacy_default_sort_key() -> None: + # Rows persisted under the old `default_sort` list key must still deserialize; the renamed + # `default_metric_sort` field is simply absent (None), and the stale key is ignored. + from nmp.intake.entities.experiments import ExperimentGroup + + group = ExperimentGroup.model_validate( + { + "name": "g", + "workspace": "default", + "default_sort": [{"field": "cost_usd.mean", "direction": "asc"}], + } + ) + assert group.default_metric_sort is None + + # ----------------------------- endpoint wiring ----------------------------- def test_create_group_with_default_sort_round_trips(client: TestClient) -> None: - resp = client.post( - GROUPS, json={"name": "g-sort", "default_sort": [{"field": "cost_usd.mean", "direction": "asc"}]} - ) + resp = client.post(GROUPS, json={"name": "g-sort", "default_metric_sort": "-cost_usd.mean"}) assert resp.status_code == 201, resp.text - assert resp.json()["default_sort"] == [{"field": "cost_usd.mean", "direction": "asc"}] + assert resp.json()["default_metric_sort"] == "-cost_usd.mean" def test_create_group_rejects_non_metric_sort_field(client: TestClient) -> None: - for field in ("name", "created_at", "cost_usd.bogus"): - resp = client.post(GROUPS, json={"name": f"g-{field}", "default_sort": [{"field": field, "direction": "asc"}]}) + for value in ("name", "-created_at", "cost_usd.bogus"): + resp = client.post(GROUPS, json={"name": f"g-{value.lstrip('-')}", "default_metric_sort": value}) assert resp.status_code == 400, resp.text @@ -125,28 +134,3 @@ def test_default_order_floats_pinned_first(client: TestClient) -> None: listed = client.get(EXPERIMENTS, params={"filter[experiment_group_id]": group["id"]}) assert listed.status_code == 200, listed.text assert [r["name"] for r in listed.json()["data"]] == ["exp-a", "exp-b"] - - -def test_default_metric_sort_degrades_without_rollups(client: TestClient) -> None: - # Group's default sort is a metric, but rollups are unavailable. The default sort must NOT 503 — it - # falls back to -created_at (unlike an explicit metric sort, which does 503). - group = client.post( - GROUPS, json={"name": "g-deg", "default_sort": [{"field": "cost_usd.mean", "direction": "asc"}]} - ).json() - for name in ("e1", "e2", "e3"): - created = client.post( - EXPERIMENTS, json={"name": name, "experiment_group_id": group["id"], "dataset_name": "ds"} - ) - assert created.status_code == 201, created.text - - default_sorted = client.get(EXPERIMENTS, params={"filter[experiment_group_id]": group["id"]}) - assert default_sorted.status_code == 200, default_sorted.text - # The cost rollup is unset, so the default sort falls back to -created_at: newest first. - # (ISO-8601 UTC timestamps sort lexicographically == chronologically.) - created_ats = [row["created_at"] for row in default_sorted.json()["data"]] - assert created_ats == sorted(created_ats, reverse=True) - - explicit_metric = client.get( - EXPERIMENTS, params={"filter[experiment_group_id]": group["id"], "sort": "-cost_usd.mean"} - ) - assert explicit_metric.status_code == 503, explicit_metric.text diff --git a/web/packages/studio/src/components/DefaultSortControl/index.tsx b/web/packages/studio/src/components/DefaultSortControl/index.tsx new file mode 100644 index 0000000000..82210da847 --- /dev/null +++ b/web/packages/studio/src/components/DefaultSortControl/index.tsx @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { snakeCaseToTitleCase } from '@nemo/common/src/utils/formatters'; +import { + Button, + FormField, + SegmentedControl, + SelectContent, + SelectItem, + SelectListbox, + SelectRoot, + SelectTrigger, + Stack, + Text, +} from '@nvidia/foundations-react-core'; +import { + EVALUATOR_PREFIX, + STATIC_FIELDS, + evaluatorField, + evaluatorNameOf, + formatSortString, + isEvaluatorField, + parseSortString, +} from '@studio/components/DefaultSortControl/util'; +import { X } from 'lucide-react'; +import type { FC } from 'react'; + +/** + * A group's **default sort**: a single `sort`-param string (e.g. `-cost_usd.mean`) the client applies + * as the experiments list's `sort` param on load. Optional (`null` = no default). The field ranks on + * a metric's mean, matching the sort/filter API grammar. + */ + +export interface DefaultSortControlProps { + value: string | null; + onChange: (next: string | null) => void; + /** Known evaluator names to offer as first-class options (edit modal). Empty at create time. */ + evaluatorOptions?: string[]; + disabled?: boolean; +} + +export const DefaultSortControl: FC = ({ + value, + onChange, + evaluatorOptions = [], + disabled, +}) => { + const parsed = value != null ? parseSortString(value) : null; + const setField = (field: string) => onChange(formatSortString(field, parsed?.desc ?? true)); + + // Keep the currently-selected evaluator selectable even if it wasn't among the discovered options + // (e.g. a saved sort whose evaluator isn't in the sampled experiments), so it stays visible. + const currentEvaluator = + parsed && isEvaluatorField(parsed.field) ? evaluatorNameOf(parsed.field) : ''; + const evaluators = + currentEvaluator && !evaluatorOptions.includes(currentEvaluator) + ? [...evaluatorOptions, currentEvaluator] + : evaluatorOptions; + + // Map a stored field to the Select's option value (static id or `evaluator:`). + const selectValueFor = (field: string): string => + STATIC_FIELDS.some((f) => f.value === field) + ? field + : `${EVALUATOR_PREFIX}${evaluatorNameOf(field)}`; + + const labelForOption = (optionValue: string): string => { + const staticField = STATIC_FIELDS.find((f) => f.value === optionValue); + if (staticField) return staticField.label; + return `Avg ${snakeCaseToTitleCase(optionValue.slice(EVALUATOR_PREFIX.length))}`; + }; + + const onSelectField = (selected: string) => { + if (selected.startsWith(EVALUATOR_PREFIX)) { + setField(evaluatorField(selected.slice(EVALUATOR_PREFIX.length))); + } else { + setField(selected); + } + }; + + return ( + + + + The metric this group's experiments are sorted by on load. + +
+ + (typeof v === 'string' && v ? labelForOption(v) : undefined)} + /> + + + {STATIC_FIELDS.map((f) => ( + + {f.label} + + ))} + {evaluators.map((name) => ( + + {`Avg ${snakeCaseToTitleCase(name)}`} + + ))} + + + + + {parsed != null && ( + <> + + onChange(formatSortString(parsed.field, d === 'desc')) + } + items={[ + { value: 'asc', children: 'Asc' }, + { value: 'desc', children: 'Desc' }, + ]} + /> + + + + )} +
+ + + ); +}; diff --git a/web/packages/studio/src/components/DefaultSortControl/util.test.ts b/web/packages/studio/src/components/DefaultSortControl/util.test.ts new file mode 100644 index 0000000000..19b6498364 --- /dev/null +++ b/web/packages/studio/src/components/DefaultSortControl/util.test.ts @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + evaluatorField, + evaluatorNameOf, + formatSortString, + isEvaluatorField, + parseSortString, +} from '@studio/components/DefaultSortControl/util'; + +describe('DefaultSortControl util', () => { + it('builds and parses evaluator field paths', () => { + expect(evaluatorField('accuracy')).toBe('evaluators.accuracy.mean'); + expect(evaluatorNameOf('evaluators.accuracy.mean')).toBe('accuracy'); + // Evaluator names may contain dots; the .mean suffix is the anchor. + expect(evaluatorNameOf('evaluators.harbor.verifier.mean')).toBe('harbor.verifier'); + expect(evaluatorNameOf('cost_usd.mean')).toBe(''); + expect(isEvaluatorField('evaluators.accuracy.mean')).toBe(true); + expect(isEvaluatorField('cost_usd.mean')).toBe(false); + }); + + it('round-trips sort strings through parse/format', () => { + expect(parseSortString('-cost_usd.mean')).toEqual({ field: 'cost_usd.mean', desc: true }); + expect(parseSortString('run_count')).toEqual({ field: 'run_count', desc: false }); + expect(formatSortString('cost_usd.mean', true)).toBe('-cost_usd.mean'); + expect(formatSortString('run_count', false)).toBe('run_count'); + }); +}); diff --git a/web/packages/studio/src/components/DefaultSortControl/util.ts b/web/packages/studio/src/components/DefaultSortControl/util.ts new file mode 100644 index 0000000000..9bc9b29fe6 --- /dev/null +++ b/web/packages/studio/src/components/DefaultSortControl/util.ts @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Always-available sort fields (no experiments needed to know them). Metrics rank on their `.mean`. */ +export const STATIC_FIELDS: ReadonlyArray<{ value: string; label: string }> = [ + { value: 'cost_usd.mean', label: 'Avg Cost' }, + { value: 'latency_ms.mean', label: 'Avg Latency' }, + { value: 'run_count', label: 'Run count' }, +]; + +/** Prefix for a per-evaluator Select option value, e.g. `evaluator:accuracy`. */ +export const EVALUATOR_PREFIX = 'evaluator:'; + +// Evaluator names may contain dots; the `.mean` suffix is the anchor. +const EVALUATOR_FIELD = /^evaluators\.(.+)\.mean$/; + +export const evaluatorField = (name: string) => `evaluators.${name}.mean`; +export const isEvaluatorField = (field: string) => EVALUATOR_FIELD.test(field); +/** Evaluator name embedded in an `evaluators..mean` field, or '' if not that shape. */ +export const evaluatorNameOf = (field: string) => field.match(EVALUATOR_FIELD)?.[1] ?? ''; + +/** + * The control's value is a `sort`-param string matching the API grammar: an optional leading '-' + * (descending) followed by the metric field, e.g. `-cost_usd.mean`. Parsing/formatting keeps the + * field and direction as separate widget state while storing/emitting the single string. + */ +export const parseSortString = (value: string): { field: string; desc: boolean } => + value.startsWith('-') ? { field: value.slice(1), desc: true } : { field: value, desc: false }; + +export const formatSortString = (field: string, desc: boolean): string => `${desc ? '-' : ''}${field}`; diff --git a/web/packages/studio/src/components/ExperimentGroupCreateModal/index.tsx b/web/packages/studio/src/components/ExperimentGroupCreateModal/index.tsx index 63f6e34a06..6bc4d196c8 100644 --- a/web/packages/studio/src/components/ExperimentGroupCreateModal/index.tsx +++ b/web/packages/studio/src/components/ExperimentGroupCreateModal/index.tsx @@ -18,7 +18,6 @@ import { useCreateExperimentGroup, } from '@nemo/sdk/generated/platform/api'; import { - Button, CodeSnippet, FormField, Stack, @@ -30,14 +29,14 @@ import { TextInput, } from '@nvidia/foundations-react-core'; import { queryClient } from '@studio/api/queryClient'; +import { DefaultSortControl } from '@studio/components/DefaultSortControl'; import { experimentGroupCreateSchema, type ExperimentGroupCreateFormFields, } from '@studio/components/ExperimentGroupCreateModal/constants'; import { handleFormErrorsGeneric } from '@studio/util/forms/error'; import { AxiosError } from 'axios'; -import { Plus } from 'lucide-react'; -import type { FC } from 'react'; +import { useState, type FC } from 'react'; import { useForm, type SubmitHandler } from 'react-hook-form'; export interface ExperimentGroupCreateModalProps extends Pick { @@ -62,6 +61,9 @@ export const ExperimentGroupCreateModal: FC = ( }); const formDisabled = isSubmitting; + // Default sort is a custom, array-of-objects control, so it's managed outside react-hook-form's + // register() and merged into the payload on submit. + const [defaultSort, setDefaultSort] = useState(null); const toast = useToast(); @@ -75,6 +77,7 @@ export const ExperimentGroupCreateModal: FC = ( const resetAndClose = () => { reset(); + setDefaultSort(null); onClose(); }; @@ -85,6 +88,7 @@ export const ExperimentGroupCreateModal: FC = ( data: { name: data.name, description: data.description, + default_metric_sort: defaultSort ?? undefined, }, }); resetAndClose(); @@ -166,10 +170,11 @@ export const ExperimentGroupCreateModal: FC = ( {...register('description')} /> - + diff --git a/web/packages/studio/src/components/ExperimentGroupEditModal/index.tsx b/web/packages/studio/src/components/ExperimentGroupEditModal/index.tsx new file mode 100644 index 0000000000..2122faeebc --- /dev/null +++ b/web/packages/studio/src/components/ExperimentGroupEditModal/index.tsx @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { FormModal, type FormModalProps } from '@nemo/common/src/components/FormModal'; +import { useToast } from '@nemo/common/src/providers/toast/useToast'; +import { + getGetExperimentGroupQueryKey, + getListExperimentGroupsQueryKey, + useListExperiments, + useUpdateExperimentGroup, +} from '@nemo/sdk/generated/platform/api'; +import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import { FormField, Stack, TextArea, TextInput } from '@nvidia/foundations-react-core'; +import { queryClient } from '@studio/api/queryClient'; +import { DefaultSortControl } from '@studio/components/DefaultSortControl'; +import { AxiosError } from 'axios'; +import { type FC, type FormEvent, useEffect, useMemo, useState } from 'react'; + +export interface ExperimentGroupEditModalProps extends Pick { + workspace: string; + group: ExperimentGroupResponse; +} + +export const ExperimentGroupEditModal: FC = ({ + open, + onClose, + workspace, + group, +}) => { + const toast = useToast(); + const [description, setDescription] = useState(group.description ?? ''); + const [defaultSort, setDefaultSort] = useState(group.default_metric_sort ?? null); + + // Reset local form state whenever the modal (re)opens or points at a different group. + useEffect(() => { + if (open) { + setDescription(group.description ?? ''); + setDefaultSort(group.default_metric_sort ?? null); + } + }, [open, group]); + + // Offer the group's discovered evaluators as first-class sort fields (only fetched while open). + const { data: experimentsPage } = useListExperiments( + workspace, + { filter: { experiment_group_id: group.id }, page_size: 100 }, + { query: { enabled: open && !!group.id } } + ); + const evaluatorOptions = useMemo( + () => + [ + ...new Set((experimentsPage?.data ?? []).flatMap((e) => Object.keys(e.aggregate_scores ?? {}))), + ].sort(), + [experimentsPage] + ); + + const { mutateAsync: updateExperimentGroup, isPending } = useUpdateExperimentGroup({ + mutation: { + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: getGetExperimentGroupQueryKey(workspace, group.name), + }); + queryClient.invalidateQueries({ queryKey: getListExperimentGroupsQueryKey(workspace) }); + }, + }, + }); + + const onSubmit = async (e: FormEvent) => { + e.preventDefault(); // FormModal doesn't preventDefault; without RHF's handler we must. + try { + await updateExperimentGroup({ + workspace, + name: group.name, + data: { + // Name is immutable for a group; send it unchanged so the update isn't treated as a rename. + name: group.name, + description: description || undefined, + default_metric_sort: defaultSort ?? undefined, + }, + }); + onClose(); + } catch (error) { + const detail = error instanceof AxiosError ? error.response?.data?.detail : undefined; + const message = + typeof detail === 'string' ? detail : error instanceof Error ? error.message : 'Unknown error'; + toast.error(`Failed to update experiment group: ${message}`); + } + }; + + return ( + + + + + + +