diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index c64f47d4d3..361a63db74 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -3712,20 +3712,18 @@ paths: in: query required: false schema: - enum: - - -created_at - - created_at - - -updated_at - - updated_at - - -name - - name - - -pinned_at - - pinned_at type: string - description: Sort field; prefix with '-' for descending. + description: 'Field to sort by; prefix with ''-'' for descending. Sort by + an experiment attribute (name, created_at, updated_at, pinned_at) or by + an aggregate metric: run_count, cost_usd., latency_ms., or + evaluators.., where is one of mean, median, p90, p95, + p99, sum, count.' default: -created_at title: Sort - description: Sort field; prefix with '-' for descending. + description: 'Field to sort by; prefix with ''-'' for descending. Sort by + an experiment attribute (name, created_at, updated_at, pinned_at) or by + an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., + where is one of mean, median, p90, p95, p99, sum, count.' - in: query name: filter style: deepObject @@ -3744,6 +3742,12 @@ paths: application/json: schema: $ref: '#/components/schemas/ExperimentResponsesPage' + '400': + description: Unsupported sort field + '413': + description: Too many experiments selected to sort in one request + '503': + description: Telemetry store unavailable for a metric-based sort '422': description: Validation Error content: diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index c64f47d4d3..361a63db74 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -3712,20 +3712,18 @@ paths: in: query required: false schema: - enum: - - -created_at - - created_at - - -updated_at - - updated_at - - -name - - name - - -pinned_at - - pinned_at type: string - description: Sort field; prefix with '-' for descending. + description: 'Field to sort by; prefix with ''-'' for descending. Sort by + an experiment attribute (name, created_at, updated_at, pinned_at) or by + an aggregate metric: run_count, cost_usd., latency_ms., or + evaluators.., where is one of mean, median, p90, p95, + p99, sum, count.' default: -created_at title: Sort - description: Sort field; prefix with '-' for descending. + description: 'Field to sort by; prefix with ''-'' for descending. Sort by + an experiment attribute (name, created_at, updated_at, pinned_at) or by + an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., + where is one of mean, median, p90, p95, p99, sum, count.' - in: query name: filter style: deepObject @@ -3744,6 +3742,12 @@ paths: application/json: schema: $ref: '#/components/schemas/ExperimentResponsesPage' + '400': + description: Unsupported sort field + '413': + description: Too many experiments selected to sort in one request + '503': + description: Telemetry store unavailable for a metric-based sort '422': description: Validation Error content: diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index c64f47d4d3..361a63db74 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -3712,20 +3712,18 @@ paths: in: query required: false schema: - enum: - - -created_at - - created_at - - -updated_at - - updated_at - - -name - - name - - -pinned_at - - pinned_at type: string - description: Sort field; prefix with '-' for descending. + description: 'Field to sort by; prefix with ''-'' for descending. Sort by + an experiment attribute (name, created_at, updated_at, pinned_at) or by + an aggregate metric: run_count, cost_usd., latency_ms., or + evaluators.., where is one of mean, median, p90, p95, + p99, sum, count.' default: -created_at title: Sort - description: Sort field; prefix with '-' for descending. + description: 'Field to sort by; prefix with ''-'' for descending. Sort by + an experiment attribute (name, created_at, updated_at, pinned_at) or by + an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., + where is one of mean, median, p90, p95, p99, sum, count.' - in: query name: filter style: deepObject @@ -3744,6 +3742,12 @@ paths: application/json: schema: $ref: '#/components/schemas/ExperimentResponsesPage' + '400': + description: Unsupported sort field + '413': + description: Too many experiments selected to sort in one request + '503': + description: Telemetry store unavailable for a metric-based sort '422': description: Validation Error content: diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index c64f47d4d3..361a63db74 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -3712,20 +3712,18 @@ paths: in: query required: false schema: - enum: - - -created_at - - created_at - - -updated_at - - updated_at - - -name - - name - - -pinned_at - - pinned_at type: string - description: Sort field; prefix with '-' for descending. + description: 'Field to sort by; prefix with ''-'' for descending. Sort by + an experiment attribute (name, created_at, updated_at, pinned_at) or by + an aggregate metric: run_count, cost_usd., latency_ms., or + evaluators.., where is one of mean, median, p90, p95, + p99, sum, count.' default: -created_at title: Sort - description: Sort field; prefix with '-' for descending. + description: 'Field to sort by; prefix with ''-'' for descending. Sort by + an experiment attribute (name, created_at, updated_at, pinned_at) or by + an aggregate metric: run_count, cost_usd., latency_ms., or evaluators.., + where is one of mean, median, p90, p95, p99, sum, count.' - in: query name: filter style: deepObject @@ -3744,6 +3742,12 @@ paths: application/json: schema: $ref: '#/components/schemas/ExperimentResponsesPage' + '400': + description: Unsupported sort field + '413': + description: Too many experiments selected to sort in one request + '503': + description: Telemetry store unavailable for a metric-based sort '422': description: Validation Error content: diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py index 8931b898c8..dff0352ec2 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py @@ -18,7 +18,6 @@ from __future__ import annotations from typing import Dict -from typing_extensions import Literal import httpx @@ -286,10 +285,7 @@ def list( filter: ExperimentFilterParam | Omit = omit, page: int | Omit = omit, page_size: int | Omit = omit, - sort: Literal[ - "-created_at", "created_at", "-updated_at", "updated_at", "-name", "name", "-pinned_at", "pinned_at" - ] - | Omit = omit, + sort: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -310,7 +306,10 @@ def list( page_size: Page size. - sort: Sort field; prefix with '-' for descending. + sort: Field to sort by; prefix with '-' for descending. Sort by an experiment + attribute (name, created_at, updated_at, pinned_at) or by an aggregate metric: + run_count, cost_usd., latency_ms., or evaluators.., + where is one of mean, median, p90, p95, p99, sum, count. extra_headers: Send extra headers @@ -702,10 +701,7 @@ def list( filter: ExperimentFilterParam | Omit = omit, page: int | Omit = omit, page_size: int | Omit = omit, - sort: Literal[ - "-created_at", "created_at", "-updated_at", "updated_at", "-name", "name", "-pinned_at", "pinned_at" - ] - | Omit = omit, + sort: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -726,7 +722,10 @@ def list( page_size: Page size. - sort: Sort field; prefix with '-' for descending. + sort: Field to sort by; prefix with '-' for descending. Sort by an experiment + attribute (name, created_at, updated_at, pinned_at) or by an aggregate metric: + run_count, cost_usd., latency_ms., or evaluators.., + where is one of mean, median, p90, p95, p99, sum, count. extra_headers: Send extra headers diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py index 84183f259e..b46a82cf72 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py @@ -17,7 +17,7 @@ from __future__ import annotations -from typing_extensions import Literal, TypedDict +from typing_extensions import TypedDict from .experiment_filter_param import ExperimentFilterParam @@ -41,5 +41,11 @@ class ExperimentListParams(TypedDict, total=False): page_size: int """Page size.""" - sort: Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name", "-pinned_at", "pinned_at"] - """Sort field; prefix with '-' for descending.""" + sort: str + """Field to sort by; prefix with '-' for descending. + + Sort by an experiment attribute (name, created_at, updated_at, pinned_at) or by + an aggregate metric: run_count, cost_usd., latency_ms., or + evaluators.., where is one of mean, median, p90, p95, p99, + sum, count. + """ diff --git a/sdk/python/nemo-platform/tests/api_resources/test_experiments.py b/sdk/python/nemo-platform/tests/api_resources/test_experiments.py index 68f5457688..4f4fb2e308 100644 --- a/sdk/python/nemo-platform/tests/api_resources/test_experiments.py +++ b/sdk/python/nemo-platform/tests/api_resources/test_experiments.py @@ -274,7 +274,7 @@ def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: }, page=1, page_size=1, - sort="-created_at", + sort="sort", ) assert_matches_type(SyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) @@ -712,7 +712,7 @@ async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform }, page=1, page_size=1, - sort="-created_at", + sort="sort", ) assert_matches_type(AsyncDefaultPagination[ExperimentResponse], experiment, path=["response"]) diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index 6bbe156a76..f95204b1dc 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -15,7 +15,7 @@ import secrets import time from datetime import datetime, timezone -from typing import Annotated, Literal, TypeVar +from typing import Annotated, Any, Literal, TypeVar from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from nmp.common.api.common import Page, PaginationData @@ -60,16 +60,17 @@ def _sanitize_for_log(value: str) -> str: EXPERIMENTS_TAG = "Experiments" ExperimentGroupSortField = Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"] -ExperimentSortField = Literal[ - "-created_at", - "created_at", - "-updated_at", - "updated_at", - "-name", - "name", - "-pinned_at", - "pinned_at", -] + +# The experiments list is sorted in the application layer (compute-on-read) so a single request can +# sort by a ClickHouse rollup metric, not just entity columns. `sort` is therefore a free string, +# validated against these: an entity column, run_count, or a `.` rollup path. +_ENTITY_SORT_FIELDS = frozenset({"name", "created_at", "updated_at", "pinned_at"}) +_METRIC_STATS = frozenset({"sum", "mean", "median", "p90", "p95", "p99", "count"}) +# Per-group experiment fetch bound for the in-memory merge. Groups are expected to hold at most +# hundreds; a query that selects more than this is rejected rather than sorted on a partial set — the +# trigger to denormalize metrics into an entity-store-sortable column instead. +_MAX_GROUP_EXPERIMENTS = 1000 + EntityT = TypeVar("EntityT", Experiment, ExperimentGroup) EntityClientDep = Annotated[EntityClient, Depends(get_entity_client)] @@ -334,6 +335,11 @@ async def create_experiment( "/v2/workspaces/{workspace}/experiments", response_model=Page[ExperimentResponse], tags=[EXPERIMENTS_TAG], + responses={ + 400: {"description": "Unsupported sort field"}, + 413: {"description": "Too many experiments selected to sort in one request"}, + 503: {"description": "Telemetry store unavailable for a metric-based sort"}, + }, openapi_extra=generate_openapi_extra_params( filter_schema=ExperimentFilter, filter_description=( @@ -352,24 +358,70 @@ async def list_experiments( parsed: ExperimentFilterDep, page: int = Query(default=1, ge=1, description="Page number."), page_size: int = Query(default=100, ge=1, le=1000, description="Page size."), - sort: ExperimentSortField = Query(default="-created_at", description="Sort field; prefix with '-' for descending."), + sort: str = Query( + default="-created_at", + description=( + "Field to sort by; prefix with '-' for descending. Sort by an experiment attribute " + "(name, created_at, updated_at, pinned_at) or by an aggregate metric: run_count, " + "cost_usd., latency_ms., or evaluators.., where is one of " + "mean, median, p90, p95, p99, sum, count." + ), + ), ) -> Page[ExperimentResponse]: validate_list_query_params(request) + descending = sort.startswith("-") + sort_field = sort[1:] if descending else sort + _validate_sort_field(sort_field) _apply_is_deleted_filter(parsed) _apply_is_pinned_filter(parsed) + # Compute-on-read: fetch the whole (entity-filtered) group, hydrate every rollup, then sort and + # paginate in memory so a single request can sort by a ClickHouse metric that lives outside the + # entity store. Bounded to hundreds of experiments per group (see _MAX_GROUP_EXPERIMENTS). result = await entity_client.list( Experiment, workspace=workspace, filter_operation=parsed.operation, - sort=sort, - page=page, - page_size=page_size, + page=1, + page_size=_MAX_GROUP_EXPERIMENTS, ) responses = [ExperimentResponse.from_entity(e) for e in result.data] - await _hydrate_rollups(workspace=workspace, responses=responses, rollup_repository=rollup_repository) + total_selected = result.pagination.total_results + if total_selected > _MAX_GROUP_EXPERIMENTS: + # The whole filtered set is sorted in memory; anything past the fetch cap can't be sorted, so a + # returned page would be silently incomplete. Fail loudly and tell the caller how to scope the + # query instead (or denormalize rollup metrics for entity-store sorting once groups grow this big). + logger.warning( + "Experiment list selected %d experiments, over the %d-row in-memory sort cap; refusing " + "to return a partially sorted result.", + total_selected, + _MAX_GROUP_EXPERIMENTS, + ) + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=( + f"This query selects {total_selected} experiments, exceeding the maximum of " + f"{_MAX_GROUP_EXPERIMENTS} that can be sorted in one request. Narrow the result with a " + "filter (e.g. experiment_group_id)." + ), + ) + hydrated = await _hydrate_rollups(workspace=workspace, responses=responses, rollup_repository=rollup_repository) + # A metric-backed sort (anything other than an entity column) is meaningless without rollups: if + # hydration was skipped (ClickHouse disabled or down) every metric value would be unset and the + # result would silently collapse to name order. Reject the request instead of returning a + # misleading 200. Entity-column sorts still work and an empty group still hydrates fine. + if not hydrated and sort_field not in _ENTITY_SORT_FIELDS: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=f"Cannot sort experiments by '{sort_field}': the telemetry store is unavailable.", + ) + ordered = _sort_experiments(responses, field=sort_field, descending=descending) + start = (page - 1) * page_size + page_items = ordered[start : start + page_size] return Page( - data=responses, - pagination=PaginationData(**result.pagination.model_dump()), + data=page_items, + pagination=make_pagination( + page=page, page_size=page_size, current_page_size=len(page_items), total_results=len(ordered) + ), sort=sort, filter=parsed.to_response(), ) @@ -836,25 +888,76 @@ def _apply_is_pinned_filter(parsed: ParsedFilter) -> None: parsed.and_with(null_clause) +def _validate_sort_field(field: str) -> None: + """Reject a sort field that isn't an entity column or a known rollup-metric path.""" + if field in _ENTITY_SORT_FIELDS or field == "run_count": + return + head, _, rest = field.partition(".") + if head in ("cost_usd", "latency_ms") and rest in _METRIC_STATS: + return + if head == "evaluators": + # Evaluator names can contain dots (e.g. "harbor.verifier"); the stat is the last segment. + name, _, stat = rest.rpartition(".") + if name and stat in _METRIC_STATS: + return + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"Unsupported sort field: {field}") + + +def _experiment_sort_value(response: ExperimentResponse, field: str) -> Any: + """Value for `field` on a hydrated response, or None when the metric is absent (sorts last).""" + if field in _ENTITY_SORT_FIELDS: + return getattr(response, field) + if field == "run_count": + return response.run_count + head, _, rest = field.partition(".") + if head == "cost_usd": + return getattr(response.cost_usd, rest, None) if response.cost_usd is not None else None + if head == "latency_ms": + return getattr(response.latency_ms, rest, None) if response.latency_ms is not None else None + name, _, stat = rest.rpartition(".") # head == "evaluators" + score = (response.aggregate_scores or {}).get(name) + return getattr(score, stat, None) if score is not None else None + + +def _sort_experiments(responses: list[ExperimentResponse], *, field: str, descending: bool) -> list[ExperimentResponse]: + """Sort by an entity column or rollup metric; missing values sort last, ties broken by name.""" + by_name = sorted(responses, key=lambda r: r.name) # deterministic tiebreak under the stable sort below + valued = [(_experiment_sort_value(r, field), r) for r in by_name] + present = [(value, r) for value, r in valued if value is not None] + missing = [r for value, r in valued if value is None] + present.sort(key=lambda pair: pair[0], reverse=descending) + return [r for _, r in present] + missing + + async def _hydrate_rollups( *, workspace: str, responses: list[ExperimentResponse], rollup_repository: ExperimentRollupRepository | None, -) -> None: - if rollup_repository is None or not responses: - return +) -> bool: + """Enrich responses with ClickHouse rollups in place. + + Returns True when hydration completed (including the no-op empty-list case) and False when it was + skipped because the rollup store is unavailable (repository absent or query failed). Callers that + sort by a rollup metric use the flag to reject the request rather than silently degrade; callers + that only display metrics can ignore it. + """ + if not responses: + return True + if rollup_repository is None: + return False try: rollups = await rollup_repository.get_rollups( workspace=workspace, experiment_ids=[response.name for response in responses] ) except Exception: logger.exception("Skipping experiment rollup hydration because ClickHouse is unavailable") - return + return False for response in responses: rollup = rollups.get(response.name) if rollup is not None: _apply_rollup(response, rollup) + return True def _apply_rollup(response: ExperimentResponse, rollup: ExperimentRollup) -> None: diff --git a/services/intake/tests/integration/spans/test_experiment_metric_sort.py b/services/intake/tests/integration/spans/test_experiment_metric_sort.py new file mode 100644 index 0000000000..f387fbb192 --- /dev/null +++ b/services/intake/tests/integration/spans/test_experiment_metric_sort.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The experiments list sorts by a ClickHouse rollup metric (Option A app-merge), end to end.""" + +import uuid +from datetime import datetime, timedelta, timezone +from typing import Any + +from fastapi.testclient import TestClient + +ATIF_INGEST = "/apis/intake/v2/workspaces/default/ingest/atif" +EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" +GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" + + +def _ensure_group(client: TestClient, name: str) -> str: + response = client.post(GROUPS, json={"name": name}) + if response.status_code == 409: + response = client.get(f"{GROUPS}/{name}") + response.raise_for_status() + return response.json()["id"] + + +def _iso(value: datetime) -> str: + return value.isoformat().replace("+00:00", "Z") + + +def _atif_body(*, started_at: datetime, experiment_id: str, cost_usd: float, offset_seconds: int) -> dict[str, Any]: + session_started_at = started_at + timedelta(seconds=offset_seconds) + return { + "schema_version": "ATIF-v1.7", + "session_id": f"{experiment_id}-session", + "experiment_context": {"experiment_id": experiment_id, "test_case_id": "case-1"}, + "extra": {"task_name": "case-1", "verifier_result": {"rewards": {"reward": 1.0}}}, + "agent": {"name": "sample-agent", "version": "1.0.0", "model_name": "provider/sample-model"}, + "steps": [ + { + "step_id": 1, + "timestamp": _iso(session_started_at), + "source": "agent", + "model_name": "provider/sample-model", + "message": "done", + "metrics": {"prompt_tokens": 100, "completion_tokens": 10, "cost_usd": cost_usd}, + } + ], + } + + +def _create_experiment(client: TestClient, group_id: str, name: str) -> None: + response = client.post( + EXPERIMENTS, + json={"name": name, "experiment_group_id": group_id, "dataset_name": "ds"}, + ) + assert response.status_code == 201, response.text + + +def test_list_sorts_by_cost_metric_missing_last(client: TestClient) -> None: + # Unique per run so reruns/shared integration state can't collide on group or experiment names. + suffix = uuid.uuid4().hex + group_id = _ensure_group(client, name=f"metric-sort-group-{suffix}") + started_at = datetime.now(timezone.utc).replace(microsecond=0) + cheap, pricey, mid = f"exp-cheap-{suffix}", f"exp-pricey-{suffix}", f"exp-mid-{suffix}" + for index, (name, cost) in enumerate([(cheap, 0.10), (pricey, 0.90), (mid, 0.50)]): + _create_experiment(client, group_id, name) + response = client.post( + ATIF_INGEST, + json=_atif_body(started_at=started_at, experiment_id=name, cost_usd=cost, offset_seconds=index * 10), + ) + assert response.status_code == 201, response.text + # No ingest -> no cost rollup -> must sort last regardless of direction. + norun = f"exp-norun-{suffix}" + _create_experiment(client, group_id, norun) + + # Filter by this group so the assertion only inspects experiments this test created. + listed = client.get( + EXPERIMENTS, + params={"filter[experiment_group_id]": group_id, "sort": "-cost_usd.mean", "page_size": 50}, + ) + assert listed.status_code == 200, listed.text + names = [row["name"] for row in listed.json()["data"]] + assert names == [pricey, mid, cheap, norun] + + +def test_list_rejects_unknown_sort_field(client: TestClient) -> None: + response = client.get(EXPERIMENTS, params={"sort": "bogus.field"}) + assert response.status_code == 400, response.text diff --git a/services/intake/tests/test_experiment_sort.py b/services/intake/tests/test_experiment_sort.py new file mode 100644 index 0000000000..cb1129e76f --- /dev/null +++ b/services/intake/tests/test_experiment_sort.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the in-memory experiment sort (Option A app-merge).""" + +from __future__ import annotations + +import pytest +from fastapi import HTTPException +from nmp.intake.api.v2.experiments.endpoints import _sort_experiments, _validate_sort_field +from nmp.intake.api.v2.experiments.schemas import EvaluatorAggregate, ExperimentResponse + + +def _exp( + name: str, + *, + run_count: int = 0, + cost_mean: float | None = None, + evaluators: dict[str, float] | None = None, +) -> ExperimentResponse: + return ExperimentResponse( + id=name, + name=name, + workspace="default", + experiment_group_id="grp", + dataset_name="ds", + run_count=run_count, + cost_usd=EvaluatorAggregate(mean=cost_mean) if cost_mean is not None else None, + aggregate_scores={key: EvaluatorAggregate(mean=value) for key, value in (evaluators or {}).items()} or None, + ) + + +def _names(responses: list[ExperimentResponse]) -> list[str]: + return [r.name for r in responses] + + +def test_sort_by_evaluator_mean_descending() -> None: + rows = [ + _exp("a", evaluators={"reward": 0.4}), + _exp("b", evaluators={"reward": 0.9}), + _exp("c", evaluators={"reward": 0.6}), + ] + ordered = _sort_experiments(rows, field="evaluators.reward.mean", descending=True) + assert _names(ordered) == ["b", "c", "a"] + + +def test_sort_by_cost_ascending() -> None: + rows = [_exp("a", cost_mean=2.0), _exp("b", cost_mean=0.5), _exp("c", cost_mean=1.0)] + ordered = _sort_experiments(rows, field="cost_usd.mean", descending=False) + assert _names(ordered) == ["b", "c", "a"] + + +def test_sort_by_run_count() -> None: + rows = [_exp("a", run_count=3), _exp("b", run_count=10), _exp("c", run_count=1)] + assert _names(_sort_experiments(rows, field="run_count", descending=True)) == ["b", "a", "c"] + + +def test_evaluator_name_with_dots_resolves() -> None: + # "harbor.verifier" contains a dot; the stat is the last segment. + rows = [_exp("a", evaluators={"harbor.verifier": 0.2}), _exp("b", evaluators={"harbor.verifier": 0.8})] + ordered = _sort_experiments(rows, field="evaluators.harbor.verifier.mean", descending=True) + assert _names(ordered) == ["b", "a"] + + +def test_missing_metric_sorts_last_in_both_directions() -> None: + rows = [_exp("scored", cost_mean=1.0), _exp("unscored")] # unscored has no cost + assert _names(_sort_experiments(rows, field="cost_usd.mean", descending=True)) == ["scored", "unscored"] + assert _names(_sort_experiments(rows, field="cost_usd.mean", descending=False)) == ["scored", "unscored"] + + +def test_ties_broken_by_name() -> None: + rows = [_exp("c", cost_mean=1.0), _exp("a", cost_mean=1.0), _exp("b", cost_mean=1.0)] + # Equal values -> deterministic ascending-name order, regardless of sort direction. + assert _names(_sort_experiments(rows, field="cost_usd.mean", descending=True)) == ["a", "b", "c"] + + +def test_entity_field_sort() -> None: + rows = [_exp("b"), _exp("a"), _exp("c")] + assert _names(_sort_experiments(rows, field="name", descending=False)) == ["a", "b", "c"] + + +def test_validate_accepts_entity_and_metric_fields() -> None: + for field in ( + "name", + "created_at", + "pinned_at", + "run_count", + "cost_usd.mean", + "latency_ms.p95", + "evaluators.harbor.verifier.mean", + ): + _validate_sort_field(field) # no raise + + +def test_validate_rejects_unknown_field() -> None: + for field in ("bogus", "cost_usd.nope", "evaluators.reward", "evaluators..mean"): + with pytest.raises(HTTPException) as exc: + _validate_sort_field(field) + assert exc.value.status_code == 400 diff --git a/services/intake/tests/test_experiment_sort_endpoint.py b/services/intake/tests/test_experiment_sort_endpoint.py new file mode 100644 index 0000000000..c71de5ef0f --- /dev/null +++ b/services/intake/tests/test_experiment_sort_endpoint.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Endpoint-level guards for the experiments list sort (rollups unavailable / bad field). + +The shared ``client`` fixture overrides ``get_experiment_rollup_repository`` to return ``None``, +which is exactly the "ClickHouse disabled / unavailable" condition. A metric-backed sort cannot be +computed without rollups, so it must fail loudly rather than silently degrade to name order. +""" + +from fastapi.testclient import TestClient + +EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" +GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" + + +def _make_experiment(client: TestClient, name: str = "exp-1", group: str = "grp-1") -> None: + group_resp = client.post(GROUPS, json={"name": group}) + assert group_resp.status_code == 201, group_resp.text + exp_resp = client.post( + EXPERIMENTS, + json={"name": name, "experiment_group_id": group_resp.json()["id"], "dataset_name": "ds"}, + ) + assert exp_resp.status_code == 201, exp_resp.text + + +def test_metric_sort_returns_503_when_rollups_unavailable(client: TestClient) -> None: + _make_experiment(client) + response = client.get(EXPERIMENTS, params={"sort": "-cost_usd.mean"}) + assert response.status_code == 503, response.text + + +def test_run_count_sort_returns_503_when_rollups_unavailable(client: TestClient) -> None: + _make_experiment(client) + response = client.get(EXPERIMENTS, params={"sort": "run_count"}) + assert response.status_code == 503, response.text + + +def test_entity_sort_still_succeeds_without_rollups(client: TestClient) -> None: + _make_experiment(client) + for sort in ("name", "-created_at", "pinned_at"): + response = client.get(EXPERIMENTS, params={"sort": sort}) + assert response.status_code == 200, response.text + + +def test_unknown_sort_field_returns_400(client: TestClient) -> None: + response = client.get(EXPERIMENTS, params={"sort": "bogus.field"}) + assert response.status_code == 400, response.text + + +def test_too_many_experiments_to_sort_returns_413(client: TestClient, monkeypatch) -> None: + # The whole filtered set is sorted in memory; over the cap we refuse rather than return a + # silently truncated result. 413 (distinct from the 400 bad-sort-field case) so a caller can tell + # the two apart. Shrink the cap so the test stays fast. + from nmp.intake.api.v2.experiments import endpoints + + monkeypatch.setattr(endpoints, "_MAX_GROUP_EXPERIMENTS", 2) + group_resp = client.post(GROUPS, json={"name": "big-grp"}) + group_id = group_resp.json()["id"] + for index in range(3): + resp = client.post( + EXPERIMENTS, + json={"name": f"exp-{index}", "experiment_group_id": group_id, "dataset_name": "ds"}, + ) + assert resp.status_code == 201, resp.text + + response = client.get(EXPERIMENTS, params={"sort": "name"}) + assert response.status_code == 413, response.text + assert "exceeding the maximum" in response.json()["detail"]