From 6986a578bcfd5477ad95d25de1aa998cce910080 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Fri, 17 Jul 2026 02:13:05 -0600 Subject: [PATCH 01/13] feat(entities): add $contains array-membership filter operator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a CONTAINS ("$contains") comparison operator to the entity-store filter engine so callers can query "value is an element of a JSON array field" — the prerequisite for modeling an experiment's group membership as a list (ASE-559). - filter_ops.py: new FilterOperator.CONTAINS, a default contains() on the FilterRepository ABC (additive — existing repos need not implement it), and dispatch in ComparisonOperation.apply. The JSON filter parser is generic over the enum, so $contains parses with no parser change. - SQLAlchemyFilterRepository.contains: portable across SQLite (JSON) and PostgreSQL (JSONB) via a quote-delimited match on the serialized array text; wildcard-escaped and prefix-collision-safe ("g1" never matches ["g10"]). - InMemoryFilterRepository.contains: native membership, matching SQL semantics. - Parity harness extended with $contains cases incl. a prefix near-miss. Validated: 32 parity cases pass (SQLite + in-memory agree); 113 entities + 167 plugin filter tests pass; ruff + ty clean. The Postgres path is portable by design; Docker-backed integration tests confirm it in CI. Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- .../src/nemo_platform_plugin/filter_ops.py | 11 ++++++++++ .../nemo_platform_plugin/in_memory_filter.py | 11 ++++++++++ .../app/repository/sqlalchemy/filter.py | 20 +++++++++++++++++++ .../tests/test_filter_matches_sql_parity.py | 19 +++++++++++++----- 4 files changed, 56 insertions(+), 5 deletions(-) diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/filter_ops.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/filter_ops.py index 2c46cd9e026..9b9b586daee 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/filter_ops.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/filter_ops.py @@ -26,6 +26,7 @@ class FilterOperator(str, Enum): GTE = "$gte" IN = "$in" NIN = "$nin" + CONTAINS = "$contains" # Logical operators OR = "$or" @@ -71,6 +72,14 @@ def in_op(self, field: str, values: List[Any]) -> Any: def nin(self, field: str, values: List[Any]) -> Any: pass + def contains(self, field: str, value: Any) -> Any: + """Array membership: match rows where the array at ``field`` contains scalar ``value``. + + Optional — additive to the base contract, so repositories that don't support + array-valued fields may leave it unimplemented. + """ + raise NotImplementedError("$contains not supported by this repository") + @abstractmethod def and_op(self, operations: List[Any]) -> Any: pass @@ -133,6 +142,8 @@ def apply(self, repository: FilterRepository) -> Any: return repository.in_op(self.field, self.value) elif self.operator == FilterOperator.NIN: return repository.nin(self.field, self.value) + elif self.operator == FilterOperator.CONTAINS: + return repository.contains(self.field, self.value) elif self.operator == FilterOperator.EXISTS: raise NotImplementedError( "$exists requires a relationship-aware repository (use the entities service parser)" diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/in_memory_filter.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/in_memory_filter.py index 31d4798622a..cbaddb1ccb9 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/in_memory_filter.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/in_memory_filter.py @@ -117,6 +117,17 @@ def nin(self, field: str, values: List[Any]) -> bool: return False return field_value not in values + def contains(self, field: str, value: Any) -> bool: + """Array membership: true when the list at ``field`` contains ``value``. + + Absent/None/non-list fields match nothing (mirrors the SQL path, where a missing + or scalar element can't contain the token). Elements are compared by native value. + """ + field_value = self._value(field) + if field_value is _MISSING or not isinstance(field_value, (list, tuple)): + return False + return value in field_value + def and_op(self, operations: List[Any]) -> bool: return all(operations) diff --git a/services/core/entities/src/nmp/core/entities/app/repository/sqlalchemy/filter.py b/services/core/entities/src/nmp/core/entities/app/repository/sqlalchemy/filter.py index 40eb859d82d..9cb85e8e008 100644 --- a/services/core/entities/src/nmp/core/entities/app/repository/sqlalchemy/filter.py +++ b/services/core/entities/src/nmp/core/entities/app/repository/sqlalchemy/filter.py @@ -178,6 +178,26 @@ def nin(self, field: str, values: List[Any]) -> Any: return self._cast_json_to_text(column).not_in([str(v) for v in values]) return column.not_in(values) + def contains(self, field: str, value: Any) -> Any: + """Array membership: true when the JSON array at ``field`` contains scalar ``value``. + + Portable across SQLite (JSON) and PostgreSQL (JSONB) without a dialect branch: the + array element serializes as a quote-delimited token (e.g. ``"g1"``) in both backends' + text rendering, so we match that token in the serialized array text. Quoting makes it + collision-safe against prefixes (``"g1"`` does not match ``["g10"]``). ``value`` is + coerced to text and LIKE wildcards are escaped, so only exact elements match. + + Intended for array-valued JSON fields (e.g. ``data.experiment_ids``); values are + assumed to be JSON scalars without embedded double quotes (entity ids qualify). + """ + column, is_json = self._get_column(field) + if not is_json: + raise ValueError(f"$contains requires a JSON array field, got non-JSON field '{field}'") + needle = str(value) + for ch in ("\\", "%", "_"): + needle = needle.replace(ch, f"\\{ch}") + return cast(column, String).like(f'%"{needle}"%', escape="\\") + def and_op(self, operations: List[Any]) -> Any: """Logical AND.""" return and_(*operations) diff --git a/services/core/entities/tests/test_filter_matches_sql_parity.py b/services/core/entities/tests/test_filter_matches_sql_parity.py index c55716413a8..efd9a019646 100644 --- a/services/core/entities/tests/test_filter_matches_sql_parity.py +++ b/services/core/entities/tests/test_filter_matches_sql_parity.py @@ -43,11 +43,12 @@ class FakeEntity(Base): # plain-column NULL (name on row 5) and an explicit/absent ``k`` for $eq-None # coverage are the only nullable bits, and $eq agrees with SQL on both. SEED = [ - dict(id=1, name="llama", data={"score": 5, "tier": "free", "flag": True, "k": None}), - dict(id=2, name="Llama-2", data={"score": 9, "tier": "pro", "flag": False}), - dict(id=3, name="zephyr", data={"score": 10, "tier": "pro", "flag": True, "k": "v"}), - dict(id=4, name="mistral", data={"score": 100, "tier": "enterprise", "flag": False}), - dict(id=5, name=None, data={"score": 1, "tier": "free", "flag": False}), + dict(id=1, name="llama", data={"score": 5, "tier": "free", "flag": True, "k": None, "tags": ["red", "blue"]}), + dict(id=2, name="Llama-2", data={"score": 9, "tier": "pro", "flag": False, "tags": ["red"]}), + # "redish" is a deliberate prefix near-miss for "red" — quote-delimited matching must exclude it. + dict(id=3, name="zephyr", data={"score": 10, "tier": "pro", "flag": True, "k": "v", "tags": ["green", "redish"]}), + dict(id=4, name="mistral", data={"score": 100, "tier": "enterprise", "flag": False, "tags": []}), + dict(id=5, name=None, data={"score": 1, "tier": "free", "flag": False, "tags": ["blue"]}), ] @@ -96,6 +97,14 @@ def NOT(op): ("nin_name", C(FilterOperator.NIN, "name", ["llama"])), ("nin_data_tier", C(FilterOperator.NIN, "data.tier", ["pro"])), ("nin_data_score", C(FilterOperator.NIN, "data.score", [5, 9])), + ("contains_tags_red", C(FilterOperator.CONTAINS, "data.tags", "red")), + ("contains_tags_blue", C(FilterOperator.CONTAINS, "data.tags", "blue")), + ("contains_tags_absent", C(FilterOperator.CONTAINS, "data.tags", "nope")), + ("not_contains_tags_red", NOT(C(FilterOperator.CONTAINS, "data.tags", "red"))), + ( + "and_contains_tags", + AND(C(FilterOperator.CONTAINS, "data.tags", "blue"), C(FilterOperator.EQ, "data.tier", "free")), + ), ("gt_data_score", C(FilterOperator.GT, "data.score", 9)), ("gte_data_score", C(FilterOperator.GTE, "data.score", 10)), ("lt_data_score", C(FilterOperator.LT, "data.score", 10)), From f559bcedc1444c2197441da7de5b8a14c59764a5 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Fri, 17 Jul 2026 02:28:20 -0600 Subject: [PATCH 02/13] feat(intake): allow an evaluation to belong to multiple groups [ASE-559] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relax the one-group-per-evaluation constraint to many-to-many. An evaluation's single experiment_group_id becomes an experiment_ids list (>=1); everything that renders a group's runs (list view, rollups, sort) now spans the membership set. - Entity: Experiment.experiment_group_id -> experiment_ids: list[str] (min 1), with a model_validator(mode="before") coercing legacy single-id rows to [id] on read (no data migration). parent_experiment_id (lineage) untouched. - Create/update API: EvaluationRequest accepts experiment_ids (>=1); experiment_group_id kept as a deprecated coalesced alias. EvaluationResponse exposes experiment_ids with experiment_group_id as a deprecated computed alias (= experiment_ids[0]). Each group id is validated to exist + be live. - Membership queries: a shared _group_membership_filter() matches "$contains experiment_ids OR legacy scalar experiment_group_id", used by the list filter (rewriting the exposed experiment_group_id param), both count helpers, and the delete cascade — so un-migrated rows stay queryable. - Delete cascade is now reference-counted: a member whose sole membership was the deleted group is soft-deleted; a member shared with another group just drops that membership and survives there. Tests: multi-group create appears in both leaderboards; empty experiment_ids rejected; reference-counted cascade (shared member survives, sole member gone); existing single-group + deprecated-alias paths still pass. Full intake experiment/evaluation suite green (SQLite). Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- .../intake/api/v2/experiments/endpoints.py | 98 +++++++++++++++---- .../nmp/intake/api/v2/experiments/schemas.py | 41 ++++++-- .../src/nmp/intake/entities/experiments.py | 25 ++++- .../integration/test_experiments_crud.py | 52 ++++++++++ .../tests/test_experiment_default_sort.py | 2 +- .../tests/test_experiment_metric_filter.py | 2 +- services/intake/tests/test_experiment_sort.py | 2 +- 7 files changed, 188 insertions(+), 34 deletions(-) diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index 03eadaa640c..dec4984a286 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -282,21 +282,16 @@ async def delete_experiment_group( ) _reject_if_deleted(group, workspace=workspace, name=name, label="Experiment group") - # Cascade is sequential — one update per child. Linear in group size, fine for now. If - # groups routinely hold more than a few hundred evaluations, add a bulk update endpoint on - # the entity store rather than parallelizing here (gather hides partial-failure state - # without removing the per-row API contract). - # - # Each ``_soft_delete`` renames the row and flips ``is_deleted=True``, which drops it out of - # the live filter — so re-fetching page 1 keeps returning the next batch until nothing is - # left. No fixed cap on group size. - # - # ``data.experiment_group_id`` is the entity-store field name; the URL filter dep auto- - # prefixes ``data.`` but manually-constructed ComparisonOperations don't get that translation. + # Reference-counted cascade, sequential — one update per member. Linear in group size, fine for + # now. A member whose *sole* membership was this group is soft-deleted; a member also in another + # group just drops this membership and survives there. Both outcomes remove the member from the + # live-membership filter below (soft-delete renames + flips is_deleted; membership removal drops + # the group id from experiment_ids and leaves no legacy scalar), so re-fetching page 1 keeps + # returning the next unprocessed batch until nothing is left. No fixed cap on group size. live_children_filter = LogicalOperation( operator=FilterOperator.AND, operations=[ - ComparisonOperation(operator=FilterOperator.EQ, field="data.experiment_group_id", value=group.id), + _group_membership_filter(group.id), LogicalOperation( operator=FilterOperator.NOT, operations=[ @@ -316,7 +311,14 @@ async def delete_experiment_group( if not page.data: break for child in page.data: - await _soft_delete(entity_client, child) + remaining = [gid for gid in child.experiment_ids if gid != group.id] + if remaining: + # Shared with another group — drop only this membership; the member lives on there. + child.experiment_ids = remaining + await entity_client.update(child) + else: + # This group was the member's sole membership — cascade the delete. + await _soft_delete(entity_client, child) await _soft_delete(entity_client, group) @@ -332,12 +334,12 @@ async def create_evaluation( body: EvaluationRequest, entity_client: EntityClientDep, ) -> EvaluationResponse: - await _validate_group_exists(entity_client, group_id=body.experiment_group_id) + await _validate_groups_exist(entity_client, group_ids=body.experiment_ids) await _validate_parent_evaluation_exists(entity_client, parent_evaluation_id=body.parent_evaluation_id) entity = Evaluation( workspace=workspace, name=body.name, - experiment_group_id=body.experiment_group_id, + experiment_ids=body.experiment_ids, dataset_name=body.dataset_name, dataset_version=body.dataset_version, source_link=body.source_link, @@ -418,6 +420,9 @@ async def list_evaluations( # the metric ones are applied in memory after hydration. parsed (the full user filter) is left # intact so the response still echoes it. entity_operation, metric_predicates = _extract_metric_predicates(parsed.operation) + # Translate the exposed `experiment_group_id` filter into a membership match over `experiment_ids` + # (plus the legacy scalar), so listing a group returns every evaluation that belongs to it. + entity_operation = _rewrite_group_filter(entity_operation) # Compute-on-read: fetch the whole (entity-filtered) group, hydrate every rollup, then filter, sort, # and paginate in memory so a single request can sort/filter by a ClickHouse metric that lives # outside the entity store. Bounded to hundreds of evaluations per group (see _MAX_GROUP_EVALUATIONS). @@ -531,8 +536,9 @@ async def update_evaluation( label="Evaluation", ) _reject_if_deleted(existing, workspace=workspace, name=name, label="Evaluation") - if body.experiment_group_id != existing.experiment_group_id: - await _validate_group_exists(entity_client, group_id=body.experiment_group_id) + new_group_ids = [gid for gid in body.experiment_ids if gid not in existing.experiment_ids] + if new_group_ids: + await _validate_groups_exist(entity_client, group_ids=new_group_ids) await _validate_parent_evaluation_exists(entity_client, parent_evaluation_id=body.parent_evaluation_id) changed = [f for f in _IMMUTABLE_EVALUATION_FIELDS if getattr(body, f) != getattr(existing, f)] @@ -545,7 +551,7 @@ async def update_evaluation( ), ) - existing.experiment_group_id = body.experiment_group_id + existing.experiment_ids = body.experiment_ids existing.source_link = body.source_link existing.metadata = body.metadata existing.description = body.description @@ -815,7 +821,7 @@ async def _count_live_evaluations_in_group(entity_client: EntityClient, *, works filter_operation=LogicalOperation( operator=FilterOperator.AND, operations=[ - ComparisonOperation(operator=FilterOperator.EQ, field="data.experiment_group_id", value=group_id), + _group_membership_filter(group_id), LogicalOperation( operator=FilterOperator.NOT, operations=[ @@ -853,7 +859,10 @@ async def _count_live_evaluations_by_group( filter_operation = LogicalOperation( operator=FilterOperator.AND, operations=[ - ComparisonOperation(operator=FilterOperator.IN, field="data.experiment_group_id", value=list(group_ids)), + LogicalOperation( + operator=FilterOperator.OR, + operations=[_group_membership_filter(gid) for gid in dict.fromkeys(group_ids)], + ), LogicalOperation( operator=FilterOperator.NOT, operations=[ @@ -870,8 +879,11 @@ async def _count_live_evaluations_by_group( page=page, page_size=page_size, ) + # An evaluation may belong to several of the requested groups; count it once per membership. for evaluation in result.data: - counts[evaluation.experiment_group_id] = counts.get(evaluation.experiment_group_id, 0) + 1 + for gid in evaluation.experiment_ids: + if gid in counts: + counts[gid] += 1 if page >= result.pagination.total_pages: break page += 1 @@ -907,6 +919,50 @@ async def _validate_group_exists(entity_client: EntityClient, *, group_id: str) ) +async def _validate_groups_exist(entity_client: EntityClient, *, group_ids: list[str]) -> None: + """Reject with 400 if any referenced ExperimentGroup doesn't exist or is deleted (deduped).""" + for group_id in dict.fromkeys(group_ids): + await _validate_group_exists(entity_client, group_id=group_id) + + +def _group_membership_filter(group_id: str) -> LogicalOperation: + """Match evaluations that belong to ``group_id`` across both membership representations. + + New rows store membership as the ``experiment_ids`` list (matched with ``$contains``); legacy + rows that predate many-to-many still store a scalar ``experiment_group_id`` and are matched by + equality until they are next rewritten. The OR keeps un-migrated rows queryable, no migration. + """ + return LogicalOperation( + operator=FilterOperator.OR, + operations=[ + ComparisonOperation(operator=FilterOperator.CONTAINS, field="data.experiment_ids", value=group_id), + ComparisonOperation(operator=FilterOperator.EQ, field="data.experiment_group_id", value=group_id), + ], + ) + + +def _rewrite_group_filter(operation: FilterOperation | None) -> FilterOperation | None: + """Rewrite an ``experiment_group_id`` equality in a parsed filter into a membership match. + + The API still exposes an ``experiment_group_id`` filter param; with many-to-many membership it + means "belongs to this group", which spans the ``experiment_ids`` list and the legacy scalar. + """ + if operation is None: + return None + if isinstance(operation, ComparisonOperation): + if operation.field == "data.experiment_group_id" and operation.operator == FilterOperator.EQ: + return _group_membership_filter(operation.value) + return operation + if isinstance(operation, LogicalOperation): + return LogicalOperation( + operator=operation.operator, + operations=[ + rewritten for op in operation.operations if (rewritten := _rewrite_group_filter(op)) is not None + ], + ) + return operation + + def _apply_is_deleted_filter(parsed: ParsedFilter) -> None: """Append an ``is_deleted`` clause so list endpoints hide soft-deleted rows by default. diff --git a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py index 95a52e50412..32e30a759da 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py @@ -10,7 +10,7 @@ from __future__ import annotations from datetime import datetime -from typing import Annotated, Self +from typing import Annotated, Any, Self from nmp.common.entities.values import DatetimeFilter, Filter, NumberFilter, map_entity_field from nmp.intake.entities.experiments import Experiment, ExperimentGroup @@ -56,8 +56,17 @@ class EvaluationRequest(BaseModel): model_config = ConfigDict(extra="forbid") name: str = Field(description="Producer-supplied, workspace-unique evaluation id.") - experiment_group_id: str = Field( - description="Entity id of the owning ExperimentGroup. Required — the group must already exist.", + experiment_ids: list[str] = Field( + min_length=1, + description=( + "Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). Each group must " + "already exist. An Evaluation always belongs to at least one group." + ), + ) + experiment_group_id: str | None = Field( + default=None, + deprecated=True, + description="Deprecated single-group alias; provide experiment_ids instead.", ) dataset_name: str = Field(description="Producer-supplied dataset name.") dataset_version: str | None = Field(default=None, description="Producer-supplied dataset version.") @@ -79,6 +88,18 @@ class EvaluationRequest(BaseModel): description="Human- or agent-authored explanation of the evaluation's outcome (e.g. why it was killed).", ) + @model_validator(mode="before") + @classmethod + def _coalesce_group_membership(cls, data: Any) -> Any: + """Coalesce the deprecated single ``experiment_group_id`` into ``experiment_ids`` (>=1). + + A legacy client sending only ``experiment_group_id`` still validates; ``experiment_ids`` wins + when both are given. + """ + if isinstance(data, dict) and not data.get("experiment_ids") and data.get("experiment_group_id"): + return {**data, "experiment_ids": [data["experiment_group_id"]]} + return data + @model_validator(mode="after") def _coalesce_deprecated_parent(self) -> Self: """Accept the deprecated ``parent_experiment_id`` alias; the canonical field wins if both are set. @@ -149,8 +170,8 @@ class EvaluationResponse(BaseModel): id: str name: str workspace: str - experiment_group_id: str = Field( - description="Entity id of the owning ExperimentGroup. Required for every Evaluation.", + experiment_ids: list[str] = Field( + description="Entity ids of the ExperimentGroups this Evaluation belongs to (>=1).", ) dataset_name: str dataset_version: str | None = None @@ -200,13 +221,21 @@ class EvaluationResponse(BaseModel): def parent_experiment_id(self) -> str | None: return self.parent_evaluation_id + @computed_field( # type: ignore[prop-decorator] + deprecated=True, + description="Deprecated single-group alias; the first of experiment_ids. Use experiment_ids.", + ) + @property + def experiment_group_id(self) -> str: + return self.experiment_ids[0] + @classmethod def from_entity(cls, entity: Experiment) -> EvaluationResponse: return cls( id=entity.id, name=entity.name, workspace=entity.workspace, - experiment_group_id=entity.experiment_group_id, + experiment_ids=entity.experiment_ids, dataset_name=entity.dataset_name, dataset_version=entity.dataset_version, source_link=entity.source_link, diff --git a/services/intake/src/nmp/intake/entities/experiments.py b/services/intake/src/nmp/intake/entities/experiments.py index 1d1b8c150b5..8710286c0db 100644 --- a/services/intake/src/nmp/intake/entities/experiments.py +++ b/services/intake/src/nmp/intake/entities/experiments.py @@ -22,7 +22,7 @@ from typing import Any, ClassVar from nmp.common.entities.client import EntityBase -from pydantic import AnyUrl, Field, field_validator +from pydantic import AnyUrl, Field, field_validator, model_validator def _stringify_metadata(value: Any) -> Any: @@ -94,13 +94,30 @@ class Experiment(EntityBase): __entity_type__: ClassVar[str] = "experiment" - experiment_group_id: str = Field( + experiment_ids: list[str] = Field( + min_length=1, description=( - "Entity id of the owning ExperimentGroup. Required — every Experiment must belong to a Group. " - "Validated at create/update time; deleting a Group cascades to its Experiments." + "Entity ids of the ExperimentGroups this Experiment belongs to (>=1). An Experiment " + "always belongs to at least one group. Validated at create/update time. Deleting a Group " + "removes it from this set; an Experiment whose sole membership was that Group is deleted " + "with it, while one shared with another Group survives there." ), ) + @model_validator(mode="before") + @classmethod + def _coerce_group_membership(cls, data: Any) -> Any: + """Schema-on-read: coerce a legacy single ``experiment_group_id`` into ``experiment_ids``. + + Rows written before many-to-many membership stored a scalar ``experiment_group_id`` and no + ``experiment_ids``. Synthesize the one-element list so old rows read back unchanged, with no + data migration. When ``experiment_ids`` is already present it wins (a re-saved row drops the + stale scalar). Non-dict input passes through for pydantic to handle. + """ + if isinstance(data, dict) and not data.get("experiment_ids") and data.get("experiment_group_id"): + return {**data, "experiment_ids": [data["experiment_group_id"]]} + return data + dataset_name: str = Field(description="Producer-supplied dataset name.") dataset_version: str | None = Field(default=None, description="Producer-supplied dataset version.") source_link: AnyUrl | None = Field(default=None, description="Optional URL for the source experiment.") diff --git a/services/intake/tests/integration/test_experiments_crud.py b/services/intake/tests/integration/test_experiments_crud.py index a60215513f3..271ae2e73ed 100644 --- a/services/intake/tests/integration/test_experiments_crud.py +++ b/services/intake/tests/integration/test_experiments_crud.py @@ -517,3 +517,55 @@ def test_sort_by_pinned_at_most_recent_first(client: TestClient) -> None: assert pinned_asc.status_code == 200, pinned_asc.text names_asc = [e["name"] for e in pinned_asc.json()["data"]] assert names_asc.index("exp-old-pin") < names_asc.index("exp-new-pin") + + +def test_evaluation_belongs_to_multiple_groups(client: TestClient) -> None: + """An evaluation created with multiple experiment_ids appears in each group's leaderboard.""" + group_a = client.post(GROUPS, json={"name": "multi-a"}).json() + group_b = client.post(GROUPS, json={"name": "multi-b"}).json() + body = _evaluation_body(name="multi-member", experiment_group_id="placeholder") + body.pop("experiment_group_id") + body["experiment_ids"] = [group_a["id"], group_b["id"]] + + created = client.post(EVALUATIONS, json=body) + assert created.status_code == 201, created.text + assert set(created.json()["experiment_ids"]) == {group_a["id"], group_b["id"]} + # Deprecated single-group alias still resolves (to the first membership). + assert created.json()["experiment_group_id"] == group_a["id"] + + for gid in (group_a["id"], group_b["id"]): + listed = client.get(EVALUATIONS, params={"filter[experiment_group_id]": gid}) + assert listed.status_code == 200, listed.text + assert any(e["name"] == "multi-member" for e in listed.json()["data"]) + + +def test_evaluation_create_rejects_empty_experiment_ids(client: TestClient) -> None: + """The >=1-group invariant: an empty experiment_ids list is rejected.""" + body = _evaluation_body(name="no-groups", experiment_group_id="placeholder") + body.pop("experiment_group_id") + body["experiment_ids"] = [] + resp = client.post(EVALUATIONS, json=body) + assert resp.status_code == 422, resp.text + + +def test_delete_group_reference_counted_cascade(client: TestClient) -> None: + """Deleting a group deletes only its sole-membership members; shared members survive elsewhere.""" + group_a = client.post(GROUPS, json={"name": "rc-a"}).json() + group_b = client.post(GROUPS, json={"name": "rc-b"}).json() + + shared = _evaluation_body(name="rc-shared", experiment_group_id="placeholder") + shared.pop("experiment_group_id") + shared["experiment_ids"] = [group_a["id"], group_b["id"]] + client.post(EVALUATIONS, json=shared) + client.post(EVALUATIONS, json=_evaluation_body(name="rc-sole", experiment_group_id=group_a["id"])) + + assert client.delete(f"{GROUPS}/rc-a").status_code == 204 + + # Sole member gone; shared member survives, now only in group b. + assert client.get(f"{EVALUATIONS}/rc-sole").status_code == 404 + survivor = client.get(f"{EVALUATIONS}/rc-shared") + assert survivor.status_code == 200, survivor.text + assert survivor.json()["experiment_ids"] == [group_b["id"]] + + in_b = client.get(EVALUATIONS, params={"filter[experiment_group_id]": group_b["id"]}) + assert any(e["name"] == "rc-shared" for e in in_b.json()["data"]) diff --git a/services/intake/tests/test_experiment_default_sort.py b/services/intake/tests/test_experiment_default_sort.py index cc4af97e38e..002decbdf50 100644 --- a/services/intake/tests/test_experiment_default_sort.py +++ b/services/intake/tests/test_experiment_default_sort.py @@ -35,7 +35,7 @@ def _exp( id=name, name=name, workspace="default", - experiment_group_id="grp", + experiment_ids=["grp"], dataset_name="ds", cost_usd=EvaluatorAggregate(mean=cost) if cost is not None else None, latency_ms=EvaluatorAggregate(mean=latency) if latency is not None else None, diff --git a/services/intake/tests/test_experiment_metric_filter.py b/services/intake/tests/test_experiment_metric_filter.py index b6c23ed6c5c..5b97cfb621d 100644 --- a/services/intake/tests/test_experiment_metric_filter.py +++ b/services/intake/tests/test_experiment_metric_filter.py @@ -31,7 +31,7 @@ def _exp(name: str, *, run_count: int = 0, cost_mean: float | None = None) -> Ev id=name, name=name, workspace="default", - experiment_group_id="grp", + experiment_ids=["grp"], dataset_name="ds", run_count=run_count, cost_usd=EvaluatorAggregate(mean=cost_mean) if cost_mean is not None else None, diff --git a/services/intake/tests/test_experiment_sort.py b/services/intake/tests/test_experiment_sort.py index ad7f433ffa0..62c9552a352 100644 --- a/services/intake/tests/test_experiment_sort.py +++ b/services/intake/tests/test_experiment_sort.py @@ -22,7 +22,7 @@ def _exp( id=name, name=name, workspace="default", - experiment_group_id="grp", + experiment_ids=["grp"], dataset_name="ds", run_count=run_count, cost_usd=EvaluatorAggregate(mean=cost_mean) if cost_mean is not None else None, From 6ee725795d0f3125167bc6d0e06d99c8fba785eb Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Fri, 17 Jul 2026 10:08:29 -0600 Subject: [PATCH 03/13] fix(intake): make experiment_ids additive, keep experiment_group_id [ASE-559] Address review: the create request should not force the new field on clients. - EvaluationRequest.experiment_ids is now optional (default_factory=list) rather than required, so the request stays purely additive; experiment_group_id remains an accepted (deprecated) field. A model_validator requires at least one of the two and coalesces the deprecated single id into experiment_ids. - experiment_group_id is unchanged in the response body (a deprecated computed alias = experiment_ids[0]), consistent with the existing parent_experiment_id deprecation pattern. Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- .../nmp/intake/api/v2/experiments/schemas.py | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py index 32e30a759da..b05dc7c07b8 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py @@ -10,7 +10,7 @@ from __future__ import annotations from datetime import datetime -from typing import Annotated, Any, Self +from typing import Annotated, Self from nmp.common.entities.values import DatetimeFilter, Filter, NumberFilter, map_entity_field from nmp.intake.entities.experiments import Experiment, ExperimentGroup @@ -57,16 +57,19 @@ class EvaluationRequest(BaseModel): name: str = Field(description="Producer-supplied, workspace-unique evaluation id.") experiment_ids: list[str] = Field( - min_length=1, + default_factory=list, description=( - "Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). Each group must " - "already exist. An Evaluation always belongs to at least one group." + "Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). Preferred; each group " + "must already exist. When omitted, the deprecated experiment_group_id is used instead." ), ) experiment_group_id: str | None = Field( default=None, deprecated=True, - description="Deprecated single-group alias; provide experiment_ids instead.", + description=( + "Deprecated single-group field; provide experiment_ids instead. Coalesced into experiment_ids " + "when experiment_ids is omitted." + ), ) dataset_name: str = Field(description="Producer-supplied dataset name.") dataset_version: str | None = Field(default=None, description="Producer-supplied dataset version.") @@ -88,17 +91,22 @@ class EvaluationRequest(BaseModel): description="Human- or agent-authored explanation of the evaluation's outcome (e.g. why it was killed).", ) - @model_validator(mode="before") - @classmethod - def _coalesce_group_membership(cls, data: Any) -> Any: - """Coalesce the deprecated single ``experiment_group_id`` into ``experiment_ids`` (>=1). + @model_validator(mode="after") + def _resolve_group_membership(self) -> Self: + """Require >=1 group. Coalesce the deprecated ``experiment_group_id`` into ``experiment_ids`` + when the latter is omitted; reject a request that supplies neither. - A legacy client sending only ``experiment_group_id`` still validates; ``experiment_ids`` wins - when both are given. + Read the deprecated value via ``__dict__`` to avoid tripping its deprecation warning per request. """ - if isinstance(data, dict) and not data.get("experiment_ids") and data.get("experiment_group_id"): - return {**data, "experiment_ids": [data["experiment_group_id"]]} - return data + if not self.experiment_ids: + group_id = self.__dict__.get("experiment_group_id") + if group_id: + self.experiment_ids = [group_id] + if not self.experiment_ids: + raise ValueError( + "An evaluation must belong to at least one group: provide experiment_ids or experiment_group_id." + ) + return self @model_validator(mode="after") def _coalesce_deprecated_parent(self) -> Self: From 6eb78edf2277494ea4f8029d7ad80fa0a3ce4bdc Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Fri, 17 Jul 2026 10:14:47 -0600 Subject: [PATCH 04/13] feat(intake): add evaluation-to-group membership endpoints [ASE-559] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Curate existing runs into comparison groups (e.g. an all-benchmarks board) after creation: - POST /evaluations/{name}/groups/{group_id} — add a membership. Validates the group exists + is live; idempotent when already a member. - DELETE /evaluations/{name}/groups/{group_id} — remove a membership. Rejects removing the evaluation's last group with 409 (>=1-group invariant; delete the evaluation instead); idempotent when not a member. Modeled on the pin/unpin endpoints. Tests cover add (idempotent + appears on the target board), add-to-unknown-group (400), remove (drops the membership), and remove-last-group (409). Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- .../intake/api/v2/experiments/endpoints.py | 72 +++++++++++++++++++ .../integration/test_experiments_crud.py | 48 +++++++++++++ 2 files changed, 120 insertions(+) diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index dec4984a286..ea9f54d9f88 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -646,6 +646,78 @@ async def unpin_evaluation( return response +@router.post( + "/v2/workspaces/{workspace}/evaluations/{name}/groups/{group_id}", + response_model=EvaluationResponse, + tags=[EVALUATIONS_TAG], + responses={ + 400: {"description": "Experiment group does not exist or is deleted"}, + 404: {"description": "Evaluation not found"}, + }, +) +async def add_evaluation_to_group( + workspace: str, + name: str, + group_id: str, + entity_client: EntityClientDep, + rollup_repository: EvaluationRollupRepositoryDep, +) -> EvaluationResponse: + """Add an evaluation to an ExperimentGroup so it appears on that group's board. + + Curates an existing run into another comparison group (e.g. an all-benchmarks view). The group + must exist and be live. Idempotent: adding a group the evaluation already belongs to is a no-op. + """ + entity = await _get_or_404(entity_client, Evaluation, workspace=workspace, name=name, label="Evaluation") + _reject_if_deleted(entity, workspace=workspace, name=name, label="Evaluation") + await _validate_group_exists(entity_client, group_id=group_id) + if group_id not in entity.experiment_ids: + entity.experiment_ids = [*entity.experiment_ids, group_id] + entity = await entity_client.update(entity) + response = EvaluationResponse.from_entity(entity) + await _hydrate_rollups(workspace=workspace, responses=[response], rollup_repository=rollup_repository) + return response + + +@router.delete( + "/v2/workspaces/{workspace}/evaluations/{name}/groups/{group_id}", + response_model=EvaluationResponse, + tags=[EVALUATIONS_TAG], + responses={ + 404: {"description": "Evaluation not found"}, + 409: {"description": "Cannot remove the evaluation from its only group"}, + }, +) +async def remove_evaluation_from_group( + workspace: str, + name: str, + group_id: str, + entity_client: EntityClientDep, + rollup_repository: EvaluationRollupRepositoryDep, +) -> EvaluationResponse: + """Remove an evaluation from an ExperimentGroup. + + Preserves the >=1-group invariant: removing the evaluation's *last* group is rejected with 409 — + delete the evaluation instead. Idempotent: removing a group the evaluation isn't in is a no-op. + """ + entity = await _get_or_404(entity_client, Evaluation, workspace=workspace, name=name, label="Evaluation") + _reject_if_deleted(entity, workspace=workspace, name=name, label="Evaluation") + if group_id in entity.experiment_ids: + remaining = [gid for gid in entity.experiment_ids if gid != group_id] + if not remaining: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Cannot remove Evaluation '{name}' from its only group '{group_id}'; an evaluation " + "must belong to at least one group. Delete the evaluation instead." + ), + ) + entity.experiment_ids = remaining + entity = await entity_client.update(entity) + response = EvaluationResponse.from_entity(entity) + await _hydrate_rollups(workspace=workspace, responses=[response], rollup_repository=rollup_repository) + return response + + @router.get( "/v2/workspaces/{workspace}/evaluations/{name}/sessions", response_model=Page[EvaluationSessionResponse], diff --git a/services/intake/tests/integration/test_experiments_crud.py b/services/intake/tests/integration/test_experiments_crud.py index 271ae2e73ed..17643a0aa32 100644 --- a/services/intake/tests/integration/test_experiments_crud.py +++ b/services/intake/tests/integration/test_experiments_crud.py @@ -569,3 +569,51 @@ def test_delete_group_reference_counted_cascade(client: TestClient) -> None: in_b = client.get(EVALUATIONS, params={"filter[experiment_group_id]": group_b["id"]}) assert any(e["name"] == "rc-shared" for e in in_b.json()["data"]) + + +def test_add_evaluation_to_group(client: TestClient) -> None: + """POST .../groups/{id} adds a membership (idempotent) and the run appears on that board.""" + group_a = client.post(GROUPS, json={"name": "add-a"}).json() + group_b = client.post(GROUPS, json={"name": "add-b"}).json() + client.post(EVALUATIONS, json=_evaluation_body(name="add-eval", experiment_group_id=group_a["id"])) + + added = client.post(f"{EVALUATIONS}/add-eval/groups/{group_b['id']}") + assert added.status_code == 200, added.text + assert set(added.json()["experiment_ids"]) == {group_a["id"], group_b["id"]} + + again = client.post(f"{EVALUATIONS}/add-eval/groups/{group_b['id']}") # idempotent + assert set(again.json()["experiment_ids"]) == {group_a["id"], group_b["id"]} + + in_b = client.get(EVALUATIONS, params={"filter[experiment_group_id]": group_b["id"]}) + assert any(e["name"] == "add-eval" for e in in_b.json()["data"]) + + +def test_add_evaluation_to_unknown_group_rejected(client: TestClient) -> None: + group = client.post(GROUPS, json={"name": "add-known"}).json() + client.post(EVALUATIONS, json=_evaluation_body(name="add-eval-2", experiment_group_id=group["id"])) + resp = client.post(f"{EVALUATIONS}/add-eval-2/groups/experiment_group-nope") + assert resp.status_code == 400, resp.text + + +def test_remove_evaluation_from_group(client: TestClient) -> None: + group_a = client.post(GROUPS, json={"name": "rm-a"}).json() + group_b = client.post(GROUPS, json={"name": "rm-b"}).json() + body = _evaluation_body(name="rm-eval", experiment_group_id="placeholder") + body.pop("experiment_group_id") + body["experiment_ids"] = [group_a["id"], group_b["id"]] + client.post(EVALUATIONS, json=body) + + removed = client.delete(f"{EVALUATIONS}/rm-eval/groups/{group_a['id']}") + assert removed.status_code == 200, removed.text + assert removed.json()["experiment_ids"] == [group_b["id"]] + + in_a = client.get(EVALUATIONS, params={"filter[experiment_group_id]": group_a["id"]}) + assert all(e["name"] != "rm-eval" for e in in_a.json()["data"]) + + +def test_remove_evaluation_from_last_group_rejected(client: TestClient) -> None: + group = client.post(GROUPS, json={"name": "rm-last"}).json() + client.post(EVALUATIONS, json=_evaluation_body(name="rm-last-eval", experiment_group_id=group["id"])) + resp = client.delete(f"{EVALUATIONS}/rm-last-eval/groups/{group['id']}") + assert resp.status_code == 409, resp.text + assert client.get(f"{EVALUATIONS}/rm-last-eval").status_code == 200 # still there From f21825c1e4810397ac23c2deeb566182b9505996 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Fri, 17 Jul 2026 10:20:00 -0600 Subject: [PATCH 05/13] chore(intake): regenerate OpenAPI spec for experiment_ids + membership [ASE-559] Regenerated via `make refresh-openapi` after the many-to-many API changes: - EvaluationRequest/EvaluationResponse gain experiment_ids; experiment_group_id retained as a deprecated field/alias. - New POST/DELETE /evaluations/{name}/groups/{group_id} membership endpoints. Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- openapi/ga/individual/platform.openapi.yaml | 134 ++++++++++++++++++-- openapi/ga/openapi.yaml | 134 ++++++++++++++++++-- openapi/openapi.yaml | 134 ++++++++++++++++++-- 3 files changed, 372 insertions(+), 30 deletions(-) diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index 573dbdd4ef1..632d30b60be 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -3552,6 +3552,102 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/groups/{group_id}: + post: + tags: + - Evaluations + summary: Add Evaluation To Group + description: 'Add an evaluation to an ExperimentGroup so it appears on that + group''s board. + + + Curates an existing run into another comparison group (e.g. an all-benchmarks + view). The group + + must exist and be live. Idempotent: adding a group the evaluation already + belongs to is a no-op.' + operationId: add_evaluation_to_group_apis_intake_v2_workspaces__workspace__evaluations__name__groups__group_id__post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + - name: group_id + in: path + required: true + schema: + type: string + title: Group Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationResponse' + '400': + description: Experiment group does not exist or is deleted + '404': + description: Evaluation not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Evaluations + summary: Remove Evaluation From Group + description: "Remove an evaluation from an ExperimentGroup.\n\nPreserves the\ + \ >=1-group invariant: removing the evaluation's *last* group is rejected\ + \ with 409 \u2014\ndelete the evaluation instead. Idempotent: removing a group\ + \ the evaluation isn't in is a no-op." + operationId: remove_evaluation_from_group_apis_intake_v2_workspaces__workspace__evaluations__name__groups__group_id__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + - name: group_id + in: path + required: true + schema: + type: string + title: Group Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationResponse' + '404': + description: Evaluation not found + '409': + description: Cannot remove the evaluation from its only group + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin: post: tags: @@ -10301,11 +10397,20 @@ components: type: string title: Name description: Producer-supplied, workspace-unique evaluation id. + experiment_ids: + items: + type: string + type: array + title: Experiment Ids + description: Entity ids of the ExperimentGroups this Evaluation belongs + to (>=1). Preferred; each group must already exist. When omitted, the + deprecated experiment_group_id is used instead. experiment_group_id: - type: string title: Experiment Group Id - description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ - \ group must already exist." + description: Deprecated single-group field; provide experiment_ids instead. + Coalesced into experiment_ids when experiment_ids is omitted. + deprecated: true + type: string dataset_name: type: string title: Dataset Name @@ -10353,7 +10458,6 @@ components: type: object required: - name - - experiment_group_id - dataset_name title: EvaluationRequest description: Request body for creating an Evaluation. @@ -10368,11 +10472,13 @@ components: workspace: type: string title: Workspace - experiment_group_id: - type: string - title: Experiment Group Id - description: Entity id of the owning ExperimentGroup. Required for every - Evaluation. + experiment_ids: + items: + type: string + type: array + title: Experiment Ids + description: Entity ids of the ExperimentGroups this Evaluation belongs + to (>=1). dataset_name: type: string title: Dataset Name @@ -10466,14 +10572,22 @@ components: deprecated: true readOnly: true type: string + experiment_group_id: + type: string + title: Experiment Group Id + description: Deprecated single-group alias; the first of experiment_ids. + Use experiment_ids. + deprecated: true + readOnly: true type: object required: - id - name - workspace - - experiment_group_id + - experiment_ids - dataset_name - parent_experiment_id + - experiment_group_id title: EvaluationResponse description: Evaluation as served by the API, including ClickHouse-hydrated rollups. diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index 573dbdd4ef1..632d30b60be 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -3552,6 +3552,102 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/groups/{group_id}: + post: + tags: + - Evaluations + summary: Add Evaluation To Group + description: 'Add an evaluation to an ExperimentGroup so it appears on that + group''s board. + + + Curates an existing run into another comparison group (e.g. an all-benchmarks + view). The group + + must exist and be live. Idempotent: adding a group the evaluation already + belongs to is a no-op.' + operationId: add_evaluation_to_group_apis_intake_v2_workspaces__workspace__evaluations__name__groups__group_id__post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + - name: group_id + in: path + required: true + schema: + type: string + title: Group Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationResponse' + '400': + description: Experiment group does not exist or is deleted + '404': + description: Evaluation not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Evaluations + summary: Remove Evaluation From Group + description: "Remove an evaluation from an ExperimentGroup.\n\nPreserves the\ + \ >=1-group invariant: removing the evaluation's *last* group is rejected\ + \ with 409 \u2014\ndelete the evaluation instead. Idempotent: removing a group\ + \ the evaluation isn't in is a no-op." + operationId: remove_evaluation_from_group_apis_intake_v2_workspaces__workspace__evaluations__name__groups__group_id__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + - name: group_id + in: path + required: true + schema: + type: string + title: Group Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationResponse' + '404': + description: Evaluation not found + '409': + description: Cannot remove the evaluation from its only group + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin: post: tags: @@ -10301,11 +10397,20 @@ components: type: string title: Name description: Producer-supplied, workspace-unique evaluation id. + experiment_ids: + items: + type: string + type: array + title: Experiment Ids + description: Entity ids of the ExperimentGroups this Evaluation belongs + to (>=1). Preferred; each group must already exist. When omitted, the + deprecated experiment_group_id is used instead. experiment_group_id: - type: string title: Experiment Group Id - description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ - \ group must already exist." + description: Deprecated single-group field; provide experiment_ids instead. + Coalesced into experiment_ids when experiment_ids is omitted. + deprecated: true + type: string dataset_name: type: string title: Dataset Name @@ -10353,7 +10458,6 @@ components: type: object required: - name - - experiment_group_id - dataset_name title: EvaluationRequest description: Request body for creating an Evaluation. @@ -10368,11 +10472,13 @@ components: workspace: type: string title: Workspace - experiment_group_id: - type: string - title: Experiment Group Id - description: Entity id of the owning ExperimentGroup. Required for every - Evaluation. + experiment_ids: + items: + type: string + type: array + title: Experiment Ids + description: Entity ids of the ExperimentGroups this Evaluation belongs + to (>=1). dataset_name: type: string title: Dataset Name @@ -10466,14 +10572,22 @@ components: deprecated: true readOnly: true type: string + experiment_group_id: + type: string + title: Experiment Group Id + description: Deprecated single-group alias; the first of experiment_ids. + Use experiment_ids. + deprecated: true + readOnly: true type: object required: - id - name - workspace - - experiment_group_id + - experiment_ids - dataset_name - parent_experiment_id + - experiment_group_id title: EvaluationResponse description: Evaluation as served by the API, including ClickHouse-hydrated rollups. diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 573dbdd4ef1..632d30b60be 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -3552,6 +3552,102 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/groups/{group_id}: + post: + tags: + - Evaluations + summary: Add Evaluation To Group + description: 'Add an evaluation to an ExperimentGroup so it appears on that + group''s board. + + + Curates an existing run into another comparison group (e.g. an all-benchmarks + view). The group + + must exist and be live. Idempotent: adding a group the evaluation already + belongs to is a no-op.' + operationId: add_evaluation_to_group_apis_intake_v2_workspaces__workspace__evaluations__name__groups__group_id__post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + - name: group_id + in: path + required: true + schema: + type: string + title: Group Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationResponse' + '400': + description: Experiment group does not exist or is deleted + '404': + description: Evaluation not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Evaluations + summary: Remove Evaluation From Group + description: "Remove an evaluation from an ExperimentGroup.\n\nPreserves the\ + \ >=1-group invariant: removing the evaluation's *last* group is rejected\ + \ with 409 \u2014\ndelete the evaluation instead. Idempotent: removing a group\ + \ the evaluation isn't in is a no-op." + operationId: remove_evaluation_from_group_apis_intake_v2_workspaces__workspace__evaluations__name__groups__group_id__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + - name: group_id + in: path + required: true + schema: + type: string + title: Group Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationResponse' + '404': + description: Evaluation not found + '409': + description: Cannot remove the evaluation from its only group + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin: post: tags: @@ -10301,11 +10397,20 @@ components: type: string title: Name description: Producer-supplied, workspace-unique evaluation id. + experiment_ids: + items: + type: string + type: array + title: Experiment Ids + description: Entity ids of the ExperimentGroups this Evaluation belongs + to (>=1). Preferred; each group must already exist. When omitted, the + deprecated experiment_group_id is used instead. experiment_group_id: - type: string title: Experiment Group Id - description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ - \ group must already exist." + description: Deprecated single-group field; provide experiment_ids instead. + Coalesced into experiment_ids when experiment_ids is omitted. + deprecated: true + type: string dataset_name: type: string title: Dataset Name @@ -10353,7 +10458,6 @@ components: type: object required: - name - - experiment_group_id - dataset_name title: EvaluationRequest description: Request body for creating an Evaluation. @@ -10368,11 +10472,13 @@ components: workspace: type: string title: Workspace - experiment_group_id: - type: string - title: Experiment Group Id - description: Entity id of the owning ExperimentGroup. Required for every - Evaluation. + experiment_ids: + items: + type: string + type: array + title: Experiment Ids + description: Entity ids of the ExperimentGroups this Evaluation belongs + to (>=1). dataset_name: type: string title: Dataset Name @@ -10466,14 +10572,22 @@ components: deprecated: true readOnly: true type: string + experiment_group_id: + type: string + title: Experiment Group Id + description: Deprecated single-group alias; the first of experiment_ids. + Use experiment_ids. + deprecated: true + readOnly: true type: object required: - id - name - workspace - - experiment_group_id + - experiment_ids - dataset_name - parent_experiment_id + - experiment_group_id title: EvaluationResponse description: Evaluation as served by the API, including ClickHouse-hydrated rollups. From 787288a062944c73ebc1a71454f5295fdf0dd666 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:22:35 -0600 Subject: [PATCH 06/13] feat(studio): add "Add to group" action to the evaluations list [ASE-559] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A per-row "Add to group" action (FolderPlus, next to the pin toggle) in the ExperimentGroupDataView opens a modal with a searchable list of the workspace's experiment groups — excluding the ones the evaluation already belongs to — and adds the run to the selected group so it appears on that board too. Mirrors the pin action's mutation + scoped query invalidation, with success/error toasts. Lets users assemble a cross-benchmark ("all-benchmarks") board from existing runs. Uses the generated useAddEvaluationToGroup / useListExperimentGroups hooks. - AddToGroupModal.tsx (+ .test.tsx, 4 cases: lists/excludes members, filters by search, calls the add hook, empty state) - ExperimentGroupDataView: row-action button + modal wiring (pin logic unchanged) typecheck + component test + eslint clean. Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- .../AddToGroupModal.test.tsx | 123 +++++++++++++ .../AddToGroupModal.tsx | 164 ++++++++++++++++++ .../ExperimentGroupDataView/index.tsx | 160 ++++++++++------- 3 files changed, 383 insertions(+), 64 deletions(-) create mode 100644 web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.test.tsx create mode 100644 web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.tsx diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.test.tsx b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.test.tsx new file mode 100644 index 00000000000..68aa7c9d7e0 --- /dev/null +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.test.tsx @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { MockToastProvider } from '@nemo/common/src/tests/MockToastProvider'; +import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import { AddToGroupModal } from '@studio/components/dataViews/ExperimentGroupDataView/AddToGroupModal'; +import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +// Hoisted mocks for the SDK hooks the modal calls. +const { mockMutate, mockUseListExperimentGroups } = vi.hoisted(() => ({ + mockMutate: vi.fn<(...args: unknown[]) => void>(), + mockUseListExperimentGroups: vi.fn<() => { data: unknown; isLoading: boolean }>(), +})); + +vi.mock('@nemo/sdk/generated/platform/api', async () => { + const actual = await vi.importActual( + '@nemo/sdk/generated/platform/api' + ); + return { + ...actual, + useListExperimentGroups: () => mockUseListExperimentGroups(), + useAddEvaluationToGroup: () => ({ mutate: mockMutate, isPending: false }), + }; +}); + +const makeGroup = (id: string, name: string): ExperimentGroupResponse => ({ + id, + name, + workspace: 'default', + default_sort: '-created_at', + experiment_count: 0, +}); + +const GROUPS: ExperimentGroupResponse[] = [ + makeGroup('g1', 'Alpha benchmarks'), + makeGroup('g2', 'Beta benchmarks'), + makeGroup('g3', 'Gamma benchmarks'), +]; + +// The evaluation already belongs to g1, so g1 must be excluded from the picker. +const EVALUATION = { + id: 'eval-1', + name: 'eval-1', + experiment_ids: ['g1'], +} as unknown as EvaluationRow; + +function makeWrapper() { + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return ({ children }: { children: React.ReactNode }) => ( + + {children} + + ); +} + +function renderModal() { + const Wrapper = makeWrapper(); + return render( + + + + ); +} + +describe('AddToGroupModal', () => { + beforeEach(() => { + mockMutate.mockReset(); + mockUseListExperimentGroups.mockReset(); + mockUseListExperimentGroups.mockReturnValue({ + data: { data: GROUPS }, + isLoading: false, + }); + }); + + it('lists selectable groups, excluding groups the evaluation already belongs to', () => { + renderModal(); + // g2 and g3 are offered; g1 (already a member) is not. + expect(screen.getByRole('option', { name: 'Beta benchmarks' })).toBeInTheDocument(); + expect(screen.getByRole('option', { name: 'Gamma benchmarks' })).toBeInTheDocument(); + expect(screen.queryByRole('option', { name: 'Alpha benchmarks' })).not.toBeInTheDocument(); + }); + + it('filters the list by the search text (case-insensitive)', async () => { + const user = userEvent.setup(); + renderModal(); + await user.type(screen.getByRole('textbox', { name: /search groups/i }), 'gamma'); + expect(screen.getByRole('option', { name: 'Gamma benchmarks' })).toBeInTheDocument(); + expect(screen.queryByRole('option', { name: 'Beta benchmarks' })).not.toBeInTheDocument(); + }); + + it('calls the add hook with the evaluation name and selected group id on select', async () => { + const user = userEvent.setup(); + renderModal(); + await user.click(screen.getByRole('option', { name: 'Beta benchmarks' })); + expect(mockMutate).toHaveBeenCalledTimes(1); + expect(mockMutate).toHaveBeenCalledWith( + { workspace: 'default', name: 'eval-1', groupId: 'g2' }, + expect.objectContaining({ + onSuccess: expect.any(Function), + onError: expect.any(Function), + }) + ); + }); + + it('shows an empty message when every group is already a member', () => { + mockUseListExperimentGroups.mockReturnValue({ + data: { data: [makeGroup('g1', 'Alpha benchmarks')] }, + isLoading: false, + }); + renderModal(); + expect(screen.getByText('No groups found.')).toBeInTheDocument(); + expect(screen.queryByRole('option')).not.toBeInTheDocument(); + }); +}); diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.tsx b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.tsx new file mode 100644 index 00000000000..21241399821 --- /dev/null +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.tsx @@ -0,0 +1,164 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { useToast } from '@nemo/common/src/providers/toast/useToast'; +import { + getListEvaluationsQueryKey, + useAddEvaluationToGroup, + useListExperimentGroups, +} from '@nemo/sdk/generated/platform/api'; +import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import { + Button, + Flex, + ModalContent, + ModalDialog, + ModalFooter, + ModalHeading, + ModalMain, + ModalRoot, + Stack, + Text, + TextInput, +} from '@nvidia/foundations-react-core'; +import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; +import { useQueryClient } from '@tanstack/react-query'; +import { type FC, useEffect, useMemo, useState } from 'react'; + +export interface AddToGroupModalProps { + open: boolean; + onClose: () => void; + workspace: string; + /** The evaluation being curated into another group. */ + evaluation: EvaluationRow; + /** + * The group whose board is currently shown. Its evaluation list is invalidated on success so the + * board refreshes — mirroring how the pin action scopes invalidation to the current group. + */ + currentExperimentGroupId: string; +} + +/** + * Adds an evaluation to another ExperimentGroup. Shows a searchable list of the workspace's groups — + * excluding the ones the evaluation already belongs to — and adds the evaluation to whichever group + * the user selects, then refreshes the current board and toasts the result. + */ +export const AddToGroupModal: FC = ({ + open, + onClose, + workspace, + evaluation, + currentExperimentGroupId, +}) => { + const toast = useToast(); + const queryClient = useQueryClient(); + const [search, setSearch] = useState(''); + + // Clear the filter whenever the modal (re)opens so a stale search doesn't hide groups. + useEffect(() => { + if (open) setSearch(''); + }, [open]); + + // Only fetch groups while the modal is open. A single large page covers any realistic group count. + const { data: groupsPage, isLoading } = useListExperimentGroups( + workspace, + { page_size: 1000 }, + { query: { enabled: open && !!workspace } } + ); + + // The evaluation already belongs to these groups, so they're not valid add targets. + const memberIds = useMemo(() => new Set(evaluation.experiment_ids), [evaluation.experiment_ids]); + + // Exclude current members, then filter by the (case-insensitive) search text on the group name. + const groups = useMemo(() => { + const query = search.trim().toLowerCase(); + return (groupsPage?.data ?? []) + .filter((group) => !memberIds.has(group.id)) + .filter((group) => !query || group.name.toLowerCase().includes(query)); + }, [groupsPage, memberIds, search]); + + const { mutate: addEvaluationToGroup, isPending } = useAddEvaluationToGroup(); + + const handleSelect = (group: ExperimentGroupResponse) => { + if (isPending) return; + addEvaluationToGroup( + { workspace, name: evaluation.name, groupId: group.id }, + { + // Mirror the pin action's invalidation: scope to the current group's evaluation lists (any + // page/sort/filter) via a partial key match so the board refreshes without touching other + // groups' queries. + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: getListEvaluationsQueryKey(workspace, { + filter: { experiment_group_id: currentExperimentGroupId }, + }), + }); + toast.success(`Added "${evaluation.name}" to "${group.name}".`); + onClose(); + }, + onError: () => toast.error(`Failed to add "${evaluation.name}" to "${group.name}".`), + } + ); + }; + + return ( + + + + Add to group + + + + Add "{evaluation.name}" to another experiment group to compare it across boards. + + + + {isLoading ? ( + + Loading groups… + + ) : groups.length === 0 ? ( + + No groups found. + + ) : ( + groups.map((group) => ( + + )) + )} + + + + + + + + + + + + ); +}; diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx index 3b76f3ad06e..eb9f1fca903 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx @@ -19,6 +19,7 @@ import type { ExperimentGroupResponse, } from '@nemo/sdk/generated/platform/schema'; import { Button, Text, Tooltip } from '@nvidia/foundations-react-core'; +import { AddToGroupModal } from '@studio/components/dataViews/ExperimentGroupDataView/AddToGroupModal'; import { Empty } from '@studio/components/dataViews/ExperimentGroupDataView/Empty'; import { MeanValueTooltipCell } from '@studio/components/dataViews/ExperimentGroupDataView/MeanValueTooltipCell'; import { @@ -32,8 +33,8 @@ import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { getEvaluationDetailRoute } from '@studio/routes/utils'; import { tooltipClassName } from '@studio/styles/common'; import { useLocalStorage } from '@studio/util/hooks/useLocalStorage'; -import { Columns3, Pin } from 'lucide-react'; -import { type ComponentProps, type FC, useCallback, useEffect, useMemo } from 'react'; +import { Columns3, FolderPlus, Pin } from 'lucide-react'; +import { type ComponentProps, type FC, useCallback, useEffect, useMemo, useState } from 'react'; import { useNavigate } from 'react-router-dom'; export type { EvaluationRow }; @@ -144,6 +145,9 @@ export const ExperimentGroupDataView: FC = ({ grou const experimentGroupName = group.name; const experimentGroupId = group.id; + // The evaluation whose "Add to group" modal is open, or null when the modal is closed. + const [addToGroupRow, setAddToGroupRow] = useState(null); + // Persist column order to localStorage, keyed by experiment group ID. const [savedColumnOrder, saveColumnOrder] = useLocalStorage( `nemo-studio:experiment-group-columns:${experimentGroupId}`, @@ -228,41 +232,58 @@ export const ExperimentGroupDataView: FC = ({ grou ({ accessor, display }) => [ display({ id: 'pin', - header: () => Pinned, + header: () => Row actions, enableSorting: false, enableHiding: false, enableResizing: false, - size: 48, - minSize: 48, - maxSize: 48, + size: 88, + minSize: 88, + maxSize: 88, meta: { alignment: 'center', _isPrebuiltColumn: true, _isSizeInitialized: true }, cell: ({ row }) => { const { pinned_at } = row.original; const isPinned = pinned_at != null; return ( - - {isPinned ? 'Unpin for all users' : 'Pin for all users'} - - } - className={tooltipClassName} - side="right" - > - - + + + Add to group} + className={tooltipClassName} + side="right" + > + + + ); }, }), @@ -462,41 +483,52 @@ export const ExperimentGroupDataView: FC = ({ grou } return ( - - navigate(getEvaluationDetailRoute(workspace, experimentGroupName, row.name)) - } - toolbarSlotEnd={ - } - > - <> - - Columns - - - } - attributes={{ - DataViewRoot: { - data: orderedData, - totalCount, - requestStatus: isLoading ? 'loading' : undefined, - }, - DataViewTableContent: { - enableColumnReordering: true, - renderEmptyState: ({ hasFiltersApplied, hasSearchApplied }) => - hasFiltersApplied || hasSearchApplied ? null : ( - - ), - }, - }} - /> + <> + + navigate(getEvaluationDetailRoute(workspace, experimentGroupName, row.name)) + } + toolbarSlotEnd={ + } + > + <> + + Columns + + + } + attributes={{ + DataViewRoot: { + data: orderedData, + totalCount, + requestStatus: isLoading ? 'loading' : undefined, + }, + DataViewTableContent: { + enableColumnReordering: true, + renderEmptyState: ({ hasFiltersApplied, hasSearchApplied }) => + hasFiltersApplied || hasSearchApplied ? null : ( + + ), + }, + }} + /> + {addToGroupRow && ( + setAddToGroupRow(null)} + workspace={workspace} + evaluation={addToGroupRow} + currentExperimentGroupId={experimentGroupId} + /> + )} + ); }; From bf2fd4502935f189916c3f3462eaf028daa0487c Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Mon, 20 Jul 2026 13:31:43 -0600 Subject: [PATCH 07/13] FE & endpoint renames Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- openapi/ga/individual/platform.openapi.yaml | 44 ++- openapi/ga/openapi.yaml | 44 ++- openapi/openapi.yaml | 44 ++- .../nemo-platform/.github/workflows/ci.yml | 6 +- .../nemo-platform/.nmpcontext/openapi.yaml | 138 ++++++- .../nemo-platform/.nmpcontext/stainless.yaml | 4 + .../resources/evaluations/__init__.py | 14 + .../resources/evaluations/api.md | 7 + .../resources/evaluations/evaluations.py | 98 ++++- .../resources/evaluations/experiments.py | 325 ++++++++++++++++ .../evaluations/evaluation_create_params.py | 21 +- .../types/evaluations/evaluation_response.py | 5 +- .../evaluations/evaluation_update_params.py | 20 +- .../evaluations/test_experiments.py | 295 ++++++++++++++ .../tests/api_resources/test_evaluations.py | 30 +- sdk/stainless.yaml | 4 + .../nmp/core/auth/assets/static-authz.yaml | 13 + .../intake/api/v2/experiments/endpoints.py | 44 ++- .../integration/test_experiments_crud.py | 28 +- .../AddToGroupModal.test.tsx | 101 +++-- .../AddToGroupModal.tsx | 364 ++++++++++++------ .../ExperimentGroupDataView.css | 18 + .../ExperimentGroupDataView/index.tsx | 112 +++--- 23 files changed, 1414 insertions(+), 365 deletions(-) create mode 100644 sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/experiments.py create mode 100644 sdk/python/nemo-platform/tests/api_resources/evaluations/test_experiments.py create mode 100644 web/packages/studio/src/components/dataViews/ExperimentGroupDataView/ExperimentGroupDataView.css diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index 632d30b60be..17899a778b3 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -3552,21 +3552,23 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/groups/{group_id}: + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/experiments/{experiment_id}: post: tags: - Evaluations - summary: Add Evaluation To Group - description: 'Add an evaluation to an ExperimentGroup so it appears on that - group''s board. + summary: Add Evaluation To Experiment + description: 'Add an evaluation to an Experiment so it appears on that experiment''s + board. - Curates an existing run into another comparison group (e.g. an all-benchmarks - view). The group + Curates an existing run into another comparison experiment (e.g. an all-benchmarks + view). The - must exist and be live. Idempotent: adding a group the evaluation already - belongs to is a no-op.' - operationId: add_evaluation_to_group_apis_intake_v2_workspaces__workspace__evaluations__name__groups__group_id__post + experiment must exist and be live. Idempotent: adding an experiment the evaluation + already belongs + + to is a no-op.' + operationId: add_evaluation_to_experiment_apis_intake_v2_workspaces__workspace__evaluations__name__experiments__experiment_id__post parameters: - name: workspace in: path @@ -3580,12 +3582,12 @@ paths: schema: type: string title: Name - - name: group_id + - name: experiment_id in: path required: true schema: type: string - title: Group Id + title: Experiment Id responses: '200': description: Successful Response @@ -3594,7 +3596,7 @@ paths: schema: $ref: '#/components/schemas/EvaluationResponse' '400': - description: Experiment group does not exist or is deleted + description: Experiment does not exist or is deleted '404': description: Evaluation not found '422': @@ -3606,12 +3608,12 @@ paths: delete: tags: - Evaluations - summary: Remove Evaluation From Group - description: "Remove an evaluation from an ExperimentGroup.\n\nPreserves the\ - \ >=1-group invariant: removing the evaluation's *last* group is rejected\ - \ with 409 \u2014\ndelete the evaluation instead. Idempotent: removing a group\ - \ the evaluation isn't in is a no-op." - operationId: remove_evaluation_from_group_apis_intake_v2_workspaces__workspace__evaluations__name__groups__group_id__delete + summary: Remove Evaluation From Experiment + description: "Remove an evaluation from an Experiment.\n\nPreserves the >=1-experiment\ + \ invariant: removing the evaluation's *last* experiment is rejected\nwith\ + \ 409 \u2014 delete the evaluation instead. Idempotent: removing an experiment\ + \ the evaluation isn't\nin is a no-op." + operationId: remove_evaluation_from_experiment_apis_intake_v2_workspaces__workspace__evaluations__name__experiments__experiment_id__delete parameters: - name: workspace in: path @@ -3625,12 +3627,12 @@ paths: schema: type: string title: Name - - name: group_id + - name: experiment_id in: path required: true schema: type: string - title: Group Id + title: Experiment Id responses: '200': description: Successful Response @@ -3641,7 +3643,7 @@ paths: '404': description: Evaluation not found '409': - description: Cannot remove the evaluation from its only group + description: Cannot remove the evaluation from its only experiment '422': description: Validation Error content: diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index 632d30b60be..17899a778b3 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -3552,21 +3552,23 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/groups/{group_id}: + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/experiments/{experiment_id}: post: tags: - Evaluations - summary: Add Evaluation To Group - description: 'Add an evaluation to an ExperimentGroup so it appears on that - group''s board. + summary: Add Evaluation To Experiment + description: 'Add an evaluation to an Experiment so it appears on that experiment''s + board. - Curates an existing run into another comparison group (e.g. an all-benchmarks - view). The group + Curates an existing run into another comparison experiment (e.g. an all-benchmarks + view). The - must exist and be live. Idempotent: adding a group the evaluation already - belongs to is a no-op.' - operationId: add_evaluation_to_group_apis_intake_v2_workspaces__workspace__evaluations__name__groups__group_id__post + experiment must exist and be live. Idempotent: adding an experiment the evaluation + already belongs + + to is a no-op.' + operationId: add_evaluation_to_experiment_apis_intake_v2_workspaces__workspace__evaluations__name__experiments__experiment_id__post parameters: - name: workspace in: path @@ -3580,12 +3582,12 @@ paths: schema: type: string title: Name - - name: group_id + - name: experiment_id in: path required: true schema: type: string - title: Group Id + title: Experiment Id responses: '200': description: Successful Response @@ -3594,7 +3596,7 @@ paths: schema: $ref: '#/components/schemas/EvaluationResponse' '400': - description: Experiment group does not exist or is deleted + description: Experiment does not exist or is deleted '404': description: Evaluation not found '422': @@ -3606,12 +3608,12 @@ paths: delete: tags: - Evaluations - summary: Remove Evaluation From Group - description: "Remove an evaluation from an ExperimentGroup.\n\nPreserves the\ - \ >=1-group invariant: removing the evaluation's *last* group is rejected\ - \ with 409 \u2014\ndelete the evaluation instead. Idempotent: removing a group\ - \ the evaluation isn't in is a no-op." - operationId: remove_evaluation_from_group_apis_intake_v2_workspaces__workspace__evaluations__name__groups__group_id__delete + summary: Remove Evaluation From Experiment + description: "Remove an evaluation from an Experiment.\n\nPreserves the >=1-experiment\ + \ invariant: removing the evaluation's *last* experiment is rejected\nwith\ + \ 409 \u2014 delete the evaluation instead. Idempotent: removing an experiment\ + \ the evaluation isn't\nin is a no-op." + operationId: remove_evaluation_from_experiment_apis_intake_v2_workspaces__workspace__evaluations__name__experiments__experiment_id__delete parameters: - name: workspace in: path @@ -3625,12 +3627,12 @@ paths: schema: type: string title: Name - - name: group_id + - name: experiment_id in: path required: true schema: type: string - title: Group Id + title: Experiment Id responses: '200': description: Successful Response @@ -3641,7 +3643,7 @@ paths: '404': description: Evaluation not found '409': - description: Cannot remove the evaluation from its only group + description: Cannot remove the evaluation from its only experiment '422': description: Validation Error content: diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 632d30b60be..17899a778b3 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -3552,21 +3552,23 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/groups/{group_id}: + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/experiments/{experiment_id}: post: tags: - Evaluations - summary: Add Evaluation To Group - description: 'Add an evaluation to an ExperimentGroup so it appears on that - group''s board. + summary: Add Evaluation To Experiment + description: 'Add an evaluation to an Experiment so it appears on that experiment''s + board. - Curates an existing run into another comparison group (e.g. an all-benchmarks - view). The group + Curates an existing run into another comparison experiment (e.g. an all-benchmarks + view). The - must exist and be live. Idempotent: adding a group the evaluation already - belongs to is a no-op.' - operationId: add_evaluation_to_group_apis_intake_v2_workspaces__workspace__evaluations__name__groups__group_id__post + experiment must exist and be live. Idempotent: adding an experiment the evaluation + already belongs + + to is a no-op.' + operationId: add_evaluation_to_experiment_apis_intake_v2_workspaces__workspace__evaluations__name__experiments__experiment_id__post parameters: - name: workspace in: path @@ -3580,12 +3582,12 @@ paths: schema: type: string title: Name - - name: group_id + - name: experiment_id in: path required: true schema: type: string - title: Group Id + title: Experiment Id responses: '200': description: Successful Response @@ -3594,7 +3596,7 @@ paths: schema: $ref: '#/components/schemas/EvaluationResponse' '400': - description: Experiment group does not exist or is deleted + description: Experiment does not exist or is deleted '404': description: Evaluation not found '422': @@ -3606,12 +3608,12 @@ paths: delete: tags: - Evaluations - summary: Remove Evaluation From Group - description: "Remove an evaluation from an ExperimentGroup.\n\nPreserves the\ - \ >=1-group invariant: removing the evaluation's *last* group is rejected\ - \ with 409 \u2014\ndelete the evaluation instead. Idempotent: removing a group\ - \ the evaluation isn't in is a no-op." - operationId: remove_evaluation_from_group_apis_intake_v2_workspaces__workspace__evaluations__name__groups__group_id__delete + summary: Remove Evaluation From Experiment + description: "Remove an evaluation from an Experiment.\n\nPreserves the >=1-experiment\ + \ invariant: removing the evaluation's *last* experiment is rejected\nwith\ + \ 409 \u2014 delete the evaluation instead. Idempotent: removing an experiment\ + \ the evaluation isn't\nin is a no-op." + operationId: remove_evaluation_from_experiment_apis_intake_v2_workspaces__workspace__evaluations__name__experiments__experiment_id__delete parameters: - name: workspace in: path @@ -3625,12 +3627,12 @@ paths: schema: type: string title: Name - - name: group_id + - name: experiment_id in: path required: true schema: type: string - title: Group Id + title: Experiment Id responses: '200': description: Successful Response @@ -3641,7 +3643,7 @@ paths: '404': description: Evaluation not found '409': - description: Cannot remove the evaluation from its only group + description: Cannot remove the evaluation from its only experiment '422': description: Validation Error content: diff --git a/sdk/python/nemo-platform/.github/workflows/ci.yml b/sdk/python/nemo-platform/.github/workflows/ci.yml index 558389ad069..7c53993236e 100644 --- a/sdk/python/nemo-platform/.github/workflows/ci.yml +++ b/sdk/python/nemo-platform/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: lint: timeout-minutes: 10 name: lint - runs-on: ${{ github.repository == 'stainless-sdks/nemo-platform-python' && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} + runs-on: ${{ startsWith(github.repository, 'stainless-sdks/') && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} if: (github.event_name == 'push' || github.event.pull_request.head.repo.fork) && (github.event_name != 'push' || github.event.head_commit.message != 'codegen metadata') steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -41,7 +41,7 @@ jobs: permissions: contents: read id-token: write - runs-on: ${{ github.repository == 'stainless-sdks/nemo-platform-python' && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} + runs-on: ${{ startsWith(github.repository, 'stainless-sdks/') && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -78,7 +78,7 @@ jobs: test: timeout-minutes: 10 name: test - runs-on: ${{ github.repository == 'stainless-sdks/nemo-platform-python' && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} + runs-on: ${{ startsWith(github.repository, 'stainless-sdks/') && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} if: github.event_name == 'push' || github.event.pull_request.head.repo.fork steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index 8db73678612..17899a778b3 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -3552,6 +3552,104 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/experiments/{experiment_id}: + post: + tags: + - Evaluations + summary: Add Evaluation To Experiment + description: 'Add an evaluation to an Experiment so it appears on that experiment''s + board. + + + Curates an existing run into another comparison experiment (e.g. an all-benchmarks + view). The + + experiment must exist and be live. Idempotent: adding an experiment the evaluation + already belongs + + to is a no-op.' + operationId: add_evaluation_to_experiment_apis_intake_v2_workspaces__workspace__evaluations__name__experiments__experiment_id__post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + - name: experiment_id + in: path + required: true + schema: + type: string + title: Experiment Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationResponse' + '400': + description: Experiment does not exist or is deleted + '404': + description: Evaluation not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Evaluations + summary: Remove Evaluation From Experiment + description: "Remove an evaluation from an Experiment.\n\nPreserves the >=1-experiment\ + \ invariant: removing the evaluation's *last* experiment is rejected\nwith\ + \ 409 \u2014 delete the evaluation instead. Idempotent: removing an experiment\ + \ the evaluation isn't\nin is a no-op." + operationId: remove_evaluation_from_experiment_apis_intake_v2_workspaces__workspace__evaluations__name__experiments__experiment_id__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + - name: experiment_id + in: path + required: true + schema: + type: string + title: Experiment Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationResponse' + '404': + description: Evaluation not found + '409': + description: Cannot remove the evaluation from its only experiment + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin: post: tags: @@ -10301,11 +10399,20 @@ components: type: string title: Name description: Producer-supplied, workspace-unique evaluation id. + experiment_ids: + items: + type: string + type: array + title: Experiment Ids + description: Entity ids of the ExperimentGroups this Evaluation belongs + to (>=1). Preferred; each group must already exist. When omitted, the + deprecated experiment_group_id is used instead. experiment_group_id: - type: string title: Experiment Group Id - description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ - \ group must already exist." + description: Deprecated single-group field; provide experiment_ids instead. + Coalesced into experiment_ids when experiment_ids is omitted. + deprecated: true + type: string dataset_name: type: string title: Dataset Name @@ -10353,7 +10460,6 @@ components: type: object required: - name - - experiment_group_id - dataset_name title: EvaluationRequest description: Request body for creating an Evaluation. @@ -10368,11 +10474,13 @@ components: workspace: type: string title: Workspace - experiment_group_id: - type: string - title: Experiment Group Id - description: Entity id of the owning ExperimentGroup. Required for every - Evaluation. + experiment_ids: + items: + type: string + type: array + title: Experiment Ids + description: Entity ids of the ExperimentGroups this Evaluation belongs + to (>=1). dataset_name: type: string title: Dataset Name @@ -10466,14 +10574,22 @@ components: deprecated: true readOnly: true type: string + experiment_group_id: + type: string + title: Experiment Group Id + description: Deprecated single-group alias; the first of experiment_ids. + Use experiment_ids. + deprecated: true + readOnly: true type: object required: - id - name - workspace - - experiment_group_id + - experiment_ids - dataset_name - parent_experiment_id + - experiment_group_id title: EvaluationResponse description: Evaluation as served by the API, including ClickHouse-hydrated rollups. @@ -15004,7 +15120,7 @@ components: - source - -source title: PlatformJobListSortField - description: 'Sort fields for the job *list* endpoint.' + description: Sort fields for the job *list* endpoint. PlatformJobListTaskResponse: properties: data: diff --git a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml index 8506cbce070..c6baa873197 100644 --- a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml @@ -939,3 +939,7 @@ resources: evaluation_session_responses_page: EvaluationSessionResponsesPage methods: list: get /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/sessions + experiments: + methods: + reviewme_create: post /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/experiments/{experiment_id} + reviewme_delete: delete /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/experiments/{experiment_id} diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/__init__.py index e08c8f090e4..9947865d933 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/__init__.py @@ -31,6 +31,14 @@ EvaluationsResourceWithStreamingResponse, AsyncEvaluationsResourceWithStreamingResponse, ) +from .experiments import ( + ExperimentsResource, + AsyncExperimentsResource, + ExperimentsResourceWithRawResponse, + AsyncExperimentsResourceWithRawResponse, + ExperimentsResourceWithStreamingResponse, + AsyncExperimentsResourceWithStreamingResponse, +) __all__ = [ "SessionsResource", @@ -39,6 +47,12 @@ "AsyncSessionsResourceWithRawResponse", "SessionsResourceWithStreamingResponse", "AsyncSessionsResourceWithStreamingResponse", + "ExperimentsResource", + "AsyncExperimentsResource", + "ExperimentsResourceWithRawResponse", + "AsyncExperimentsResourceWithRawResponse", + "ExperimentsResourceWithStreamingResponse", + "AsyncExperimentsResourceWithStreamingResponse", "EvaluationsResource", "AsyncEvaluationsResource", "EvaluationsResourceWithRawResponse", diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/api.md index 485bdd1903d..191060c7908 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/api.md +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/api.md @@ -39,3 +39,10 @@ from nemo_platform.types.evaluations import ( Methods: - client.evaluations.sessions.list(name, \*, workspace, \*\*params) -> SyncDefaultPagination[EvaluationSessionResponse] + +## Experiments + +Methods: + +- client.evaluations.experiments.reviewme_create(experiment_id, \*, workspace, name) -> EvaluationResponse +- client.evaluations.experiments.reviewme_delete(experiment_id, \*, workspace, name) -> EvaluationResponse diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/evaluations.py b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/evaluations.py index 63339799637..b4305de350d 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/evaluations.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/evaluations.py @@ -21,7 +21,7 @@ import httpx -from ..._types import Body, Omit, Query, Headers, NoneType, NotGiven, omit, not_given +from ..._types import Body, Omit, Query, Headers, NoneType, NotGiven, SequenceNotStr, omit, not_given from ..._utils import path_template, maybe_transform, async_maybe_transform from .sessions import ( SessionsResource, @@ -39,6 +39,14 @@ async_to_raw_response_wrapper, async_to_streamed_response_wrapper, ) +from .experiments import ( + ExperimentsResource, + AsyncExperimentsResource, + ExperimentsResourceWithRawResponse, + AsyncExperimentsResourceWithRawResponse, + ExperimentsResourceWithStreamingResponse, + AsyncExperimentsResourceWithStreamingResponse, +) from ...pagination import SyncDefaultPagination, AsyncDefaultPagination from ..._base_client import AsyncPaginator, make_request_options from ...types.evaluations import ( @@ -58,6 +66,10 @@ class EvaluationsResource(SyncAPIResource): def sessions(self) -> SessionsResource: return SessionsResource(self._client) + @cached_property + def experiments(self) -> ExperimentsResource: + return ExperimentsResource(self._client) + @cached_property def with_raw_response(self) -> EvaluationsResourceWithRawResponse: """ @@ -82,10 +94,11 @@ def create( *, workspace: str | None = None, dataset_name: str, - experiment_group_id: str, name: str, dataset_version: str | Omit = omit, description: str | Omit = omit, + experiment_group_id: str | Omit = omit, + experiment_ids: SequenceNotStr[str] | Omit = omit, metadata: Dict[str, str] | Omit = omit, parent_evaluation_id: str | Omit = omit, parent_experiment_id: str | Omit = omit, @@ -106,15 +119,19 @@ def create( Args: dataset_name: Producer-supplied dataset name. - experiment_group_id: Entity id of the owning ExperimentGroup. Required — the group must already - exist. - name: Producer-supplied, workspace-unique evaluation id. dataset_version: Producer-supplied dataset version. description: Human-readable description. + experiment_group_id: Deprecated single-group field; provide experiment_ids instead. Coalesced into + experiment_ids when experiment_ids is omitted. + + experiment_ids: Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). Preferred; + each group must already exist. When omitted, the deprecated experiment_group_id + is used instead. + metadata: Free-form producer metadata. parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a @@ -151,10 +168,11 @@ def create( body=maybe_transform( { "dataset_name": dataset_name, - "experiment_group_id": experiment_group_id, "name": name, "dataset_version": dataset_version, "description": description, + "experiment_group_id": experiment_group_id, + "experiment_ids": experiment_ids, "metadata": metadata, "parent_evaluation_id": parent_evaluation_id, "parent_experiment_id": parent_experiment_id, @@ -218,10 +236,11 @@ def update( *, workspace: str | None = None, dataset_name: str, - experiment_group_id: str, body_name: str, dataset_version: str | Omit = omit, description: str | Omit = omit, + experiment_group_id: str | Omit = omit, + experiment_ids: SequenceNotStr[str] | Omit = omit, metadata: Dict[str, str] | Omit = omit, parent_evaluation_id: str | Omit = omit, parent_experiment_id: str | Omit = omit, @@ -241,15 +260,19 @@ def update( Args: dataset_name: Producer-supplied dataset name. - experiment_group_id: Entity id of the owning ExperimentGroup. Required — the group must already - exist. - body_name: Producer-supplied, workspace-unique evaluation id. dataset_version: Producer-supplied dataset version. description: Human-readable description. + experiment_group_id: Deprecated single-group field; provide experiment_ids instead. Coalesced into + experiment_ids when experiment_ids is omitted. + + experiment_ids: Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). Preferred; + each group must already exist. When omitted, the deprecated experiment_group_id + is used instead. + metadata: Free-form producer metadata. parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a @@ -287,10 +310,11 @@ def update( body=maybe_transform( { "dataset_name": dataset_name, - "experiment_group_id": experiment_group_id, "body_name": body_name, "dataset_version": dataset_version, "description": description, + "experiment_group_id": experiment_group_id, + "experiment_ids": experiment_ids, "metadata": metadata, "parent_evaluation_id": parent_evaluation_id, "parent_experiment_id": parent_experiment_id, @@ -509,6 +533,10 @@ class AsyncEvaluationsResource(AsyncAPIResource): def sessions(self) -> AsyncSessionsResource: return AsyncSessionsResource(self._client) + @cached_property + def experiments(self) -> AsyncExperimentsResource: + return AsyncExperimentsResource(self._client) + @cached_property def with_raw_response(self) -> AsyncEvaluationsResourceWithRawResponse: """ @@ -533,10 +561,11 @@ async def create( *, workspace: str | None = None, dataset_name: str, - experiment_group_id: str, name: str, dataset_version: str | Omit = omit, description: str | Omit = omit, + experiment_group_id: str | Omit = omit, + experiment_ids: SequenceNotStr[str] | Omit = omit, metadata: Dict[str, str] | Omit = omit, parent_evaluation_id: str | Omit = omit, parent_experiment_id: str | Omit = omit, @@ -557,15 +586,19 @@ async def create( Args: dataset_name: Producer-supplied dataset name. - experiment_group_id: Entity id of the owning ExperimentGroup. Required — the group must already - exist. - name: Producer-supplied, workspace-unique evaluation id. dataset_version: Producer-supplied dataset version. description: Human-readable description. + experiment_group_id: Deprecated single-group field; provide experiment_ids instead. Coalesced into + experiment_ids when experiment_ids is omitted. + + experiment_ids: Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). Preferred; + each group must already exist. When omitted, the deprecated experiment_group_id + is used instead. + metadata: Free-form producer metadata. parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a @@ -602,10 +635,11 @@ async def create( body=await async_maybe_transform( { "dataset_name": dataset_name, - "experiment_group_id": experiment_group_id, "name": name, "dataset_version": dataset_version, "description": description, + "experiment_group_id": experiment_group_id, + "experiment_ids": experiment_ids, "metadata": metadata, "parent_evaluation_id": parent_evaluation_id, "parent_experiment_id": parent_experiment_id, @@ -669,10 +703,11 @@ async def update( *, workspace: str | None = None, dataset_name: str, - experiment_group_id: str, body_name: str, dataset_version: str | Omit = omit, description: str | Omit = omit, + experiment_group_id: str | Omit = omit, + experiment_ids: SequenceNotStr[str] | Omit = omit, metadata: Dict[str, str] | Omit = omit, parent_evaluation_id: str | Omit = omit, parent_experiment_id: str | Omit = omit, @@ -692,15 +727,19 @@ async def update( Args: dataset_name: Producer-supplied dataset name. - experiment_group_id: Entity id of the owning ExperimentGroup. Required — the group must already - exist. - body_name: Producer-supplied, workspace-unique evaluation id. dataset_version: Producer-supplied dataset version. description: Human-readable description. + experiment_group_id: Deprecated single-group field; provide experiment_ids instead. Coalesced into + experiment_ids when experiment_ids is omitted. + + experiment_ids: Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). Preferred; + each group must already exist. When omitted, the deprecated experiment_group_id + is used instead. + metadata: Free-form producer metadata. parent_evaluation_id: Entity id of the evaluation this one was derived from (e.g. a variant of a @@ -738,10 +777,11 @@ async def update( body=await async_maybe_transform( { "dataset_name": dataset_name, - "experiment_group_id": experiment_group_id, "body_name": body_name, "dataset_version": dataset_version, "description": description, + "experiment_group_id": experiment_group_id, + "experiment_ids": experiment_ids, "metadata": metadata, "parent_evaluation_id": parent_evaluation_id, "parent_experiment_id": parent_experiment_id, @@ -985,6 +1025,10 @@ def __init__(self, evaluations: EvaluationsResource) -> None: def sessions(self) -> SessionsResourceWithRawResponse: return SessionsResourceWithRawResponse(self._evaluations.sessions) + @cached_property + def experiments(self) -> ExperimentsResourceWithRawResponse: + return ExperimentsResourceWithRawResponse(self._evaluations.experiments) + class AsyncEvaluationsResourceWithRawResponse: def __init__(self, evaluations: AsyncEvaluationsResource) -> None: @@ -1016,6 +1060,10 @@ def __init__(self, evaluations: AsyncEvaluationsResource) -> None: def sessions(self) -> AsyncSessionsResourceWithRawResponse: return AsyncSessionsResourceWithRawResponse(self._evaluations.sessions) + @cached_property + def experiments(self) -> AsyncExperimentsResourceWithRawResponse: + return AsyncExperimentsResourceWithRawResponse(self._evaluations.experiments) + class EvaluationsResourceWithStreamingResponse: def __init__(self, evaluations: EvaluationsResource) -> None: @@ -1047,6 +1095,10 @@ def __init__(self, evaluations: EvaluationsResource) -> None: def sessions(self) -> SessionsResourceWithStreamingResponse: return SessionsResourceWithStreamingResponse(self._evaluations.sessions) + @cached_property + def experiments(self) -> ExperimentsResourceWithStreamingResponse: + return ExperimentsResourceWithStreamingResponse(self._evaluations.experiments) + class AsyncEvaluationsResourceWithStreamingResponse: def __init__(self, evaluations: AsyncEvaluationsResource) -> None: @@ -1077,3 +1129,7 @@ def __init__(self, evaluations: AsyncEvaluationsResource) -> None: @cached_property def sessions(self) -> AsyncSessionsResourceWithStreamingResponse: return AsyncSessionsResourceWithStreamingResponse(self._evaluations.sessions) + + @cached_property + def experiments(self) -> AsyncExperimentsResourceWithStreamingResponse: + return AsyncExperimentsResourceWithStreamingResponse(self._evaluations.experiments) diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/experiments.py b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/experiments.py new file mode 100644 index 00000000000..91029869a64 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluations/experiments.py @@ -0,0 +1,325 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import httpx + +from ..._types import Body, Query, Headers, NotGiven, not_given +from ..._utils import path_template +from ..._compat import cached_property +from ..._resource import SyncAPIResource, AsyncAPIResource +from ..._response import ( + to_raw_response_wrapper, + to_streamed_response_wrapper, + async_to_raw_response_wrapper, + async_to_streamed_response_wrapper, +) +from ..._base_client import make_request_options +from ...types.evaluations.evaluation_response import EvaluationResponse + +__all__ = ["ExperimentsResource", "AsyncExperimentsResource"] + + +class ExperimentsResource(SyncAPIResource): + @cached_property + def with_raw_response(self) -> ExperimentsResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#accessing-raw-response-data-e-g-headers + """ + return ExperimentsResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> ExperimentsResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#with_streaming_response + """ + return ExperimentsResourceWithStreamingResponse(self) + + def reviewme_create( + self, + experiment_id: str, + *, + workspace: str | None = None, + name: str, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> EvaluationResponse: + """ + Add an evaluation to an Experiment so it appears on that experiment's board. + + Curates an existing run into another comparison experiment (e.g. an + all-benchmarks view). The experiment must exist and be live. Idempotent: adding + an experiment the evaluation already belongs to is a no-op. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + if not experiment_id: + raise ValueError(f"Expected a non-empty value for `experiment_id` but received {experiment_id!r}") + return self._post( + path_template( + "/apis/intake/v2/workspaces/{workspace}/evaluations/{name}/experiments/{experiment_id}", + workspace=workspace, + name=name, + experiment_id=experiment_id, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=EvaluationResponse, + ) + + def reviewme_delete( + self, + experiment_id: str, + *, + workspace: str | None = None, + name: str, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> EvaluationResponse: + """ + Remove an evaluation from an Experiment. + + Preserves the >=1-experiment invariant: removing the evaluation's _last_ + experiment is rejected with 409 — delete the evaluation instead. Idempotent: + removing an experiment the evaluation isn't in is a no-op. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + if not experiment_id: + raise ValueError(f"Expected a non-empty value for `experiment_id` but received {experiment_id!r}") + return self._delete( + path_template( + "/apis/intake/v2/workspaces/{workspace}/evaluations/{name}/experiments/{experiment_id}", + workspace=workspace, + name=name, + experiment_id=experiment_id, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=EvaluationResponse, + ) + + +class AsyncExperimentsResource(AsyncAPIResource): + @cached_property + def with_raw_response(self) -> AsyncExperimentsResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#accessing-raw-response-data-e-g-headers + """ + return AsyncExperimentsResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncExperimentsResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#with_streaming_response + """ + return AsyncExperimentsResourceWithStreamingResponse(self) + + async def reviewme_create( + self, + experiment_id: str, + *, + workspace: str | None = None, + name: str, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> EvaluationResponse: + """ + Add an evaluation to an Experiment so it appears on that experiment's board. + + Curates an existing run into another comparison experiment (e.g. an + all-benchmarks view). The experiment must exist and be live. Idempotent: adding + an experiment the evaluation already belongs to is a no-op. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + if not experiment_id: + raise ValueError(f"Expected a non-empty value for `experiment_id` but received {experiment_id!r}") + return await self._post( + path_template( + "/apis/intake/v2/workspaces/{workspace}/evaluations/{name}/experiments/{experiment_id}", + workspace=workspace, + name=name, + experiment_id=experiment_id, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=EvaluationResponse, + ) + + async def reviewme_delete( + self, + experiment_id: str, + *, + workspace: str | None = None, + name: str, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> EvaluationResponse: + """ + Remove an evaluation from an Experiment. + + Preserves the >=1-experiment invariant: removing the evaluation's _last_ + experiment is rejected with 409 — delete the evaluation instead. Idempotent: + removing an experiment the evaluation isn't in is a no-op. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + if not experiment_id: + raise ValueError(f"Expected a non-empty value for `experiment_id` but received {experiment_id!r}") + return await self._delete( + path_template( + "/apis/intake/v2/workspaces/{workspace}/evaluations/{name}/experiments/{experiment_id}", + workspace=workspace, + name=name, + experiment_id=experiment_id, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=EvaluationResponse, + ) + + +class ExperimentsResourceWithRawResponse: + def __init__(self, experiments: ExperimentsResource) -> None: + self._experiments = experiments + + self.reviewme_create = to_raw_response_wrapper( + experiments.reviewme_create, + ) + self.reviewme_delete = to_raw_response_wrapper( + experiments.reviewme_delete, + ) + + +class AsyncExperimentsResourceWithRawResponse: + def __init__(self, experiments: AsyncExperimentsResource) -> None: + self._experiments = experiments + + self.reviewme_create = async_to_raw_response_wrapper( + experiments.reviewme_create, + ) + self.reviewme_delete = async_to_raw_response_wrapper( + experiments.reviewme_delete, + ) + + +class ExperimentsResourceWithStreamingResponse: + def __init__(self, experiments: ExperimentsResource) -> None: + self._experiments = experiments + + self.reviewme_create = to_streamed_response_wrapper( + experiments.reviewme_create, + ) + self.reviewme_delete = to_streamed_response_wrapper( + experiments.reviewme_delete, + ) + + +class AsyncExperimentsResourceWithStreamingResponse: + def __init__(self, experiments: AsyncExperimentsResource) -> None: + self._experiments = experiments + + self.reviewme_create = async_to_streamed_response_wrapper( + experiments.reviewme_create, + ) + self.reviewme_delete = async_to_streamed_response_wrapper( + experiments.reviewme_delete, + ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_create_params.py index 956ea0db31b..3308bbbdcf2 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_create_params.py @@ -20,6 +20,8 @@ from typing import Dict from typing_extensions import Required, TypedDict +from ..._types import SequenceNotStr + __all__ = ["EvaluationCreateParams"] @@ -29,12 +31,6 @@ class EvaluationCreateParams(TypedDict, total=False): dataset_name: Required[str] """Producer-supplied dataset name.""" - experiment_group_id: Required[str] - """Entity id of the owning ExperimentGroup. - - Required — the group must already exist. - """ - name: Required[str] """Producer-supplied, workspace-unique evaluation id.""" @@ -44,6 +40,19 @@ class EvaluationCreateParams(TypedDict, total=False): description: str """Human-readable description.""" + experiment_group_id: str + """Deprecated single-group field; provide experiment_ids instead. + + Coalesced into experiment_ids when experiment_ids is omitted. + """ + + experiment_ids: SequenceNotStr[str] + """Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). + + Preferred; each group must already exist. When omitted, the deprecated + experiment_group_id is used instead. + """ + metadata: Dict[str, str] """Free-form producer metadata.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py index 49349c7a6f2..eeaaaf6ee8c 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_response.py @@ -33,7 +33,10 @@ class EvaluationResponse(BaseModel): dataset_name: str experiment_group_id: str - """Entity id of the owning ExperimentGroup. Required for every Evaluation.""" + """Deprecated single-group alias; the first of experiment_ids. Use experiment_ids.""" + + experiment_ids: List[str] + """Entity ids of the ExperimentGroups this Evaluation belongs to (>=1).""" name: str diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_update_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_update_params.py index a44d66f57b7..75dd13dab55 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_update_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluations/evaluation_update_params.py @@ -20,6 +20,7 @@ from typing import Dict from typing_extensions import Required, Annotated, TypedDict +from ..._types import SequenceNotStr from ..._utils import PropertyInfo __all__ = ["EvaluationUpdateParams"] @@ -31,12 +32,6 @@ class EvaluationUpdateParams(TypedDict, total=False): dataset_name: Required[str] """Producer-supplied dataset name.""" - experiment_group_id: Required[str] - """Entity id of the owning ExperimentGroup. - - Required — the group must already exist. - """ - body_name: Required[Annotated[str, PropertyInfo(alias="name")]] """Producer-supplied, workspace-unique evaluation id.""" @@ -46,6 +41,19 @@ class EvaluationUpdateParams(TypedDict, total=False): description: str """Human-readable description.""" + experiment_group_id: str + """Deprecated single-group field; provide experiment_ids instead. + + Coalesced into experiment_ids when experiment_ids is omitted. + """ + + experiment_ids: SequenceNotStr[str] + """Entity ids of the ExperimentGroups this Evaluation belongs to (>=1). + + Preferred; each group must already exist. When omitted, the deprecated + experiment_group_id is used instead. + """ + metadata: Dict[str, str] """Free-form producer metadata.""" diff --git a/sdk/python/nemo-platform/tests/api_resources/evaluations/test_experiments.py b/sdk/python/nemo-platform/tests/api_resources/evaluations/test_experiments.py new file mode 100644 index 00000000000..d266f629d19 --- /dev/null +++ b/sdk/python/nemo-platform/tests/api_resources/evaluations/test_experiments.py @@ -0,0 +1,295 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import os +from typing import Any, cast + +import pytest + +from tests.utils import assert_matches_type +from nemo_platform import NeMoPlatform, AsyncNeMoPlatform +from nemo_platform.types.evaluations import EvaluationResponse + +base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") + + +class TestExperiments: + parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_reviewme_create(self, client: NeMoPlatform) -> None: + experiment = client.evaluations.experiments.reviewme_create( + experiment_id="experiment_id", + workspace="workspace", + name="name", + ) + assert_matches_type(EvaluationResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_reviewme_create(self, client: NeMoPlatform) -> None: + response = client.evaluations.experiments.with_raw_response.reviewme_create( + experiment_id="experiment_id", + workspace="workspace", + name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = response.parse() + assert_matches_type(EvaluationResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_reviewme_create(self, client: NeMoPlatform) -> None: + with client.evaluations.experiments.with_streaming_response.reviewme_create( + experiment_id="experiment_id", + workspace="workspace", + name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = response.parse() + assert_matches_type(EvaluationResponse, experiment, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_reviewme_create(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.evaluations.experiments.with_raw_response.reviewme_create( + experiment_id="experiment_id", + workspace="", + name="name", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + client.evaluations.experiments.with_raw_response.reviewme_create( + experiment_id="experiment_id", + workspace="workspace", + name="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `experiment_id` but received ''"): + client.evaluations.experiments.with_raw_response.reviewme_create( + experiment_id="", + workspace="workspace", + name="name", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_reviewme_delete(self, client: NeMoPlatform) -> None: + experiment = client.evaluations.experiments.reviewme_delete( + experiment_id="experiment_id", + workspace="workspace", + name="name", + ) + assert_matches_type(EvaluationResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_reviewme_delete(self, client: NeMoPlatform) -> None: + response = client.evaluations.experiments.with_raw_response.reviewme_delete( + experiment_id="experiment_id", + workspace="workspace", + name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = response.parse() + assert_matches_type(EvaluationResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_reviewme_delete(self, client: NeMoPlatform) -> None: + with client.evaluations.experiments.with_streaming_response.reviewme_delete( + experiment_id="experiment_id", + workspace="workspace", + name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = response.parse() + assert_matches_type(EvaluationResponse, experiment, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_reviewme_delete(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.evaluations.experiments.with_raw_response.reviewme_delete( + experiment_id="experiment_id", + workspace="", + name="name", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + client.evaluations.experiments.with_raw_response.reviewme_delete( + experiment_id="experiment_id", + workspace="workspace", + name="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `experiment_id` but received ''"): + client.evaluations.experiments.with_raw_response.reviewme_delete( + experiment_id="", + workspace="workspace", + name="name", + ) + + +class TestAsyncExperiments: + parametrize = pytest.mark.parametrize( + "async_client", [False, True, {"http_client": "aiohttp"}], indirect=True, ids=["loose", "strict", "aiohttp"] + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_reviewme_create(self, async_client: AsyncNeMoPlatform) -> None: + experiment = await async_client.evaluations.experiments.reviewme_create( + experiment_id="experiment_id", + workspace="workspace", + name="name", + ) + assert_matches_type(EvaluationResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_reviewme_create(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.evaluations.experiments.with_raw_response.reviewme_create( + experiment_id="experiment_id", + workspace="workspace", + name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = await response.parse() + assert_matches_type(EvaluationResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_reviewme_create(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.evaluations.experiments.with_streaming_response.reviewme_create( + experiment_id="experiment_id", + workspace="workspace", + name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = await response.parse() + assert_matches_type(EvaluationResponse, experiment, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_reviewme_create(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.evaluations.experiments.with_raw_response.reviewme_create( + experiment_id="experiment_id", + workspace="", + name="name", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + await async_client.evaluations.experiments.with_raw_response.reviewme_create( + experiment_id="experiment_id", + workspace="workspace", + name="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `experiment_id` but received ''"): + await async_client.evaluations.experiments.with_raw_response.reviewme_create( + experiment_id="", + workspace="workspace", + name="name", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_reviewme_delete(self, async_client: AsyncNeMoPlatform) -> None: + experiment = await async_client.evaluations.experiments.reviewme_delete( + experiment_id="experiment_id", + workspace="workspace", + name="name", + ) + assert_matches_type(EvaluationResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_reviewme_delete(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.evaluations.experiments.with_raw_response.reviewme_delete( + experiment_id="experiment_id", + workspace="workspace", + name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = await response.parse() + assert_matches_type(EvaluationResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_reviewme_delete(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.evaluations.experiments.with_streaming_response.reviewme_delete( + experiment_id="experiment_id", + workspace="workspace", + name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = await response.parse() + assert_matches_type(EvaluationResponse, experiment, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_reviewme_delete(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.evaluations.experiments.with_raw_response.reviewme_delete( + experiment_id="experiment_id", + workspace="", + name="name", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + await async_client.evaluations.experiments.with_raw_response.reviewme_delete( + experiment_id="experiment_id", + workspace="workspace", + name="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `experiment_id` but received ''"): + await async_client.evaluations.experiments.with_raw_response.reviewme_delete( + experiment_id="", + workspace="workspace", + name="name", + ) diff --git a/sdk/python/nemo-platform/tests/api_resources/test_evaluations.py b/sdk/python/nemo-platform/tests/api_resources/test_evaluations.py index fd8798a6ef6..32e06a12a2f 100644 --- a/sdk/python/nemo-platform/tests/api_resources/test_evaluations.py +++ b/sdk/python/nemo-platform/tests/api_resources/test_evaluations.py @@ -42,7 +42,6 @@ def test_method_create(self, client: NeMoPlatform) -> None: evaluation = client.evaluations.create( workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", ) assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @@ -53,10 +52,11 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: evaluation = client.evaluations.create( workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", dataset_version="dataset_version", description="description", + experiment_group_id="experiment_group_id", + experiment_ids=["string"], metadata={"foo": "string"}, parent_evaluation_id="parent_evaluation_id", parent_experiment_id="parent_experiment_id", @@ -72,7 +72,6 @@ def test_raw_response_create(self, client: NeMoPlatform) -> None: response = client.evaluations.with_raw_response.create( workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", ) @@ -87,7 +86,6 @@ def test_streaming_response_create(self, client: NeMoPlatform) -> None: with client.evaluations.with_streaming_response.create( workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", ) as response: assert not response.is_closed @@ -105,7 +103,6 @@ def test_path_params_create(self, client: NeMoPlatform) -> None: client.evaluations.with_raw_response.create( workspace="", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", ) @@ -168,7 +165,6 @@ def test_method_update(self, client: NeMoPlatform) -> None: path_name="name", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @@ -180,10 +176,11 @@ def test_method_update_with_all_params(self, client: NeMoPlatform) -> None: path_name="name", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", dataset_version="dataset_version", description="description", + experiment_group_id="experiment_group_id", + experiment_ids=["string"], metadata={"foo": "string"}, parent_evaluation_id="parent_evaluation_id", parent_experiment_id="parent_experiment_id", @@ -200,7 +197,6 @@ def test_raw_response_update(self, client: NeMoPlatform) -> None: path_name="name", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) @@ -216,7 +212,6 @@ def test_streaming_response_update(self, client: NeMoPlatform) -> None: path_name="name", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) as response: assert not response.is_closed @@ -235,7 +230,6 @@ def test_path_params_update(self, client: NeMoPlatform) -> None: path_name="name", workspace="", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) @@ -244,7 +238,6 @@ def test_path_params_update(self, client: NeMoPlatform) -> None: path_name="", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) @@ -649,7 +642,6 @@ async def test_method_create(self, async_client: AsyncNeMoPlatform) -> None: evaluation = await async_client.evaluations.create( workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", ) assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @@ -660,10 +652,11 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo evaluation = await async_client.evaluations.create( workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", dataset_version="dataset_version", description="description", + experiment_group_id="experiment_group_id", + experiment_ids=["string"], metadata={"foo": "string"}, parent_evaluation_id="parent_evaluation_id", parent_experiment_id="parent_experiment_id", @@ -679,7 +672,6 @@ async def test_raw_response_create(self, async_client: AsyncNeMoPlatform) -> Non response = await async_client.evaluations.with_raw_response.create( workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", ) @@ -694,7 +686,6 @@ async def test_streaming_response_create(self, async_client: AsyncNeMoPlatform) async with async_client.evaluations.with_streaming_response.create( workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", ) as response: assert not response.is_closed @@ -712,7 +703,6 @@ async def test_path_params_create(self, async_client: AsyncNeMoPlatform) -> None await async_client.evaluations.with_raw_response.create( workspace="", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", name="name", ) @@ -775,7 +765,6 @@ async def test_method_update(self, async_client: AsyncNeMoPlatform) -> None: path_name="name", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) assert_matches_type(EvaluationResponse, evaluation, path=["response"]) @@ -787,10 +776,11 @@ async def test_method_update_with_all_params(self, async_client: AsyncNeMoPlatfo path_name="name", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", dataset_version="dataset_version", description="description", + experiment_group_id="experiment_group_id", + experiment_ids=["string"], metadata={"foo": "string"}, parent_evaluation_id="parent_evaluation_id", parent_experiment_id="parent_experiment_id", @@ -807,7 +797,6 @@ async def test_raw_response_update(self, async_client: AsyncNeMoPlatform) -> Non path_name="name", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) @@ -823,7 +812,6 @@ async def test_streaming_response_update(self, async_client: AsyncNeMoPlatform) path_name="name", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) as response: assert not response.is_closed @@ -842,7 +830,6 @@ async def test_path_params_update(self, async_client: AsyncNeMoPlatform) -> None path_name="name", workspace="", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) @@ -851,7 +838,6 @@ async def test_path_params_update(self, async_client: AsyncNeMoPlatform) -> None path_name="", workspace="workspace", dataset_name="dataset_name", - experiment_group_id="experiment_group_id", body_name="name", ) diff --git a/sdk/stainless.yaml b/sdk/stainless.yaml index 8506cbce070..c6baa873197 100644 --- a/sdk/stainless.yaml +++ b/sdk/stainless.yaml @@ -939,3 +939,7 @@ resources: evaluation_session_responses_page: EvaluationSessionResponsesPage methods: list: get /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/sessions + experiments: + methods: + reviewme_create: post /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/experiments/{experiment_id} + reviewme_delete: delete /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/experiments/{experiment_id} diff --git a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml index ef7b76287eb..36e804c77d7 100644 --- a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml +++ b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml @@ -909,6 +909,19 @@ authz: scopes: - intake:write - platform:write + /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/experiments/{experiment_id}: + delete: + permissions: + - intake.evaluations.delete + scopes: + - intake:write + - platform:write + post: + permissions: + - intake.evaluations.create + scopes: + - intake:write + - platform:write /apis/intake/v2/workspaces/{workspace}/evaluations/{name}/pin: delete: permissions: diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index ea9f54d9f88..5ec7742acf2 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -647,31 +647,32 @@ async def unpin_evaluation( @router.post( - "/v2/workspaces/{workspace}/evaluations/{name}/groups/{group_id}", + "/v2/workspaces/{workspace}/evaluations/{name}/experiments/{experiment_id}", response_model=EvaluationResponse, tags=[EVALUATIONS_TAG], responses={ - 400: {"description": "Experiment group does not exist or is deleted"}, + 400: {"description": "Experiment does not exist or is deleted"}, 404: {"description": "Evaluation not found"}, }, ) -async def add_evaluation_to_group( +async def add_evaluation_to_experiment( workspace: str, name: str, - group_id: str, + experiment_id: str, entity_client: EntityClientDep, rollup_repository: EvaluationRollupRepositoryDep, ) -> EvaluationResponse: - """Add an evaluation to an ExperimentGroup so it appears on that group's board. + """Add an evaluation to an Experiment so it appears on that experiment's board. - Curates an existing run into another comparison group (e.g. an all-benchmarks view). The group - must exist and be live. Idempotent: adding a group the evaluation already belongs to is a no-op. + Curates an existing run into another comparison experiment (e.g. an all-benchmarks view). The + experiment must exist and be live. Idempotent: adding an experiment the evaluation already belongs + to is a no-op. """ entity = await _get_or_404(entity_client, Evaluation, workspace=workspace, name=name, label="Evaluation") _reject_if_deleted(entity, workspace=workspace, name=name, label="Evaluation") - await _validate_group_exists(entity_client, group_id=group_id) - if group_id not in entity.experiment_ids: - entity.experiment_ids = [*entity.experiment_ids, group_id] + await _validate_group_exists(entity_client, group_id=experiment_id) + if experiment_id not in entity.experiment_ids: + entity.experiment_ids = [*entity.experiment_ids, experiment_id] entity = await entity_client.update(entity) response = EvaluationResponse.from_entity(entity) await _hydrate_rollups(workspace=workspace, responses=[response], rollup_repository=rollup_repository) @@ -679,36 +680,37 @@ async def add_evaluation_to_group( @router.delete( - "/v2/workspaces/{workspace}/evaluations/{name}/groups/{group_id}", + "/v2/workspaces/{workspace}/evaluations/{name}/experiments/{experiment_id}", response_model=EvaluationResponse, tags=[EVALUATIONS_TAG], responses={ 404: {"description": "Evaluation not found"}, - 409: {"description": "Cannot remove the evaluation from its only group"}, + 409: {"description": "Cannot remove the evaluation from its only experiment"}, }, ) -async def remove_evaluation_from_group( +async def remove_evaluation_from_experiment( workspace: str, name: str, - group_id: str, + experiment_id: str, entity_client: EntityClientDep, rollup_repository: EvaluationRollupRepositoryDep, ) -> EvaluationResponse: - """Remove an evaluation from an ExperimentGroup. + """Remove an evaluation from an Experiment. - Preserves the >=1-group invariant: removing the evaluation's *last* group is rejected with 409 — - delete the evaluation instead. Idempotent: removing a group the evaluation isn't in is a no-op. + Preserves the >=1-experiment invariant: removing the evaluation's *last* experiment is rejected + with 409 — delete the evaluation instead. Idempotent: removing an experiment the evaluation isn't + in is a no-op. """ entity = await _get_or_404(entity_client, Evaluation, workspace=workspace, name=name, label="Evaluation") _reject_if_deleted(entity, workspace=workspace, name=name, label="Evaluation") - if group_id in entity.experiment_ids: - remaining = [gid for gid in entity.experiment_ids if gid != group_id] + if experiment_id in entity.experiment_ids: + remaining = [gid for gid in entity.experiment_ids if gid != experiment_id] if not remaining: raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail=( - f"Cannot remove Evaluation '{name}' from its only group '{group_id}'; an evaluation " - "must belong to at least one group. Delete the evaluation instead." + f"Cannot remove Evaluation '{name}' from its only experiment '{experiment_id}'; an " + "evaluation must belong to at least one experiment. Delete the evaluation instead." ), ) entity.experiment_ids = remaining diff --git a/services/intake/tests/integration/test_experiments_crud.py b/services/intake/tests/integration/test_experiments_crud.py index 17643a0aa32..395ff069e5c 100644 --- a/services/intake/tests/integration/test_experiments_crud.py +++ b/services/intake/tests/integration/test_experiments_crud.py @@ -206,10 +206,16 @@ def test_legacy_experiments_url_aliases_are_hidden_from_schema() -> None: from fastapi.routing import APIRoute from nmp.intake.api.v2.experiments.endpoints import router + # Legacy aliases are the top-level `/experiments` collection routes that mirror `/evaluations`. + # Exclude `/experiment-groups` (a distinct resource) and the nested + # `/evaluations/{name}/experiments/{experiment_id}` membership routes (real, in-schema endpoints). alias_routes = [ route for route in router.routes - if isinstance(route, APIRoute) and "/experiments" in route.path and "/experiment-groups" not in route.path + if isinstance(route, APIRoute) + and "/experiments" in route.path + and "/experiment-groups" not in route.path + and "/evaluations/" not in route.path ] assert alias_routes, "expected legacy /experiments alias routes to be registered" assert all(route.include_in_schema is False for route in alias_routes) @@ -571,31 +577,31 @@ def test_delete_group_reference_counted_cascade(client: TestClient) -> None: assert any(e["name"] == "rc-shared" for e in in_b.json()["data"]) -def test_add_evaluation_to_group(client: TestClient) -> None: - """POST .../groups/{id} adds a membership (idempotent) and the run appears on that board.""" +def test_add_evaluation_to_experiment(client: TestClient) -> None: + """POST .../experiments/{id} adds a membership (idempotent) and the run appears on that board.""" group_a = client.post(GROUPS, json={"name": "add-a"}).json() group_b = client.post(GROUPS, json={"name": "add-b"}).json() client.post(EVALUATIONS, json=_evaluation_body(name="add-eval", experiment_group_id=group_a["id"])) - added = client.post(f"{EVALUATIONS}/add-eval/groups/{group_b['id']}") + added = client.post(f"{EVALUATIONS}/add-eval/experiments/{group_b['id']}") assert added.status_code == 200, added.text assert set(added.json()["experiment_ids"]) == {group_a["id"], group_b["id"]} - again = client.post(f"{EVALUATIONS}/add-eval/groups/{group_b['id']}") # idempotent + again = client.post(f"{EVALUATIONS}/add-eval/experiments/{group_b['id']}") # idempotent assert set(again.json()["experiment_ids"]) == {group_a["id"], group_b["id"]} in_b = client.get(EVALUATIONS, params={"filter[experiment_group_id]": group_b["id"]}) assert any(e["name"] == "add-eval" for e in in_b.json()["data"]) -def test_add_evaluation_to_unknown_group_rejected(client: TestClient) -> None: +def test_add_evaluation_to_unknown_experiment_rejected(client: TestClient) -> None: group = client.post(GROUPS, json={"name": "add-known"}).json() client.post(EVALUATIONS, json=_evaluation_body(name="add-eval-2", experiment_group_id=group["id"])) - resp = client.post(f"{EVALUATIONS}/add-eval-2/groups/experiment_group-nope") + resp = client.post(f"{EVALUATIONS}/add-eval-2/experiments/experiment_group-nope") assert resp.status_code == 400, resp.text -def test_remove_evaluation_from_group(client: TestClient) -> None: +def test_remove_evaluation_from_experiment(client: TestClient) -> None: group_a = client.post(GROUPS, json={"name": "rm-a"}).json() group_b = client.post(GROUPS, json={"name": "rm-b"}).json() body = _evaluation_body(name="rm-eval", experiment_group_id="placeholder") @@ -603,7 +609,7 @@ def test_remove_evaluation_from_group(client: TestClient) -> None: body["experiment_ids"] = [group_a["id"], group_b["id"]] client.post(EVALUATIONS, json=body) - removed = client.delete(f"{EVALUATIONS}/rm-eval/groups/{group_a['id']}") + removed = client.delete(f"{EVALUATIONS}/rm-eval/experiments/{group_a['id']}") assert removed.status_code == 200, removed.text assert removed.json()["experiment_ids"] == [group_b["id"]] @@ -611,9 +617,9 @@ def test_remove_evaluation_from_group(client: TestClient) -> None: assert all(e["name"] != "rm-eval" for e in in_a.json()["data"]) -def test_remove_evaluation_from_last_group_rejected(client: TestClient) -> None: +def test_remove_evaluation_from_last_experiment_rejected(client: TestClient) -> None: group = client.post(GROUPS, json={"name": "rm-last"}).json() client.post(EVALUATIONS, json=_evaluation_body(name="rm-last-eval", experiment_group_id=group["id"])) - resp = client.delete(f"{EVALUATIONS}/rm-last-eval/groups/{group['id']}") + resp = client.delete(f"{EVALUATIONS}/rm-last-eval/experiments/{group['id']}") assert resp.status_code == 409, resp.text assert client.get(f"{EVALUATIONS}/rm-last-eval").status_code == 200 # still there diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.test.tsx b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.test.tsx index 68aa7c9d7e0..022d55b651b 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.test.tsx +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.test.tsx @@ -6,12 +6,13 @@ import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schem import { AddToGroupModal } from '@studio/components/dataViews/ExperimentGroupDataView/AddToGroupModal'; import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; -import { render, screen } from '@testing-library/react'; +import { render, screen, waitFor } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; // Hoisted mocks for the SDK hooks the modal calls. -const { mockMutate, mockUseListExperimentGroups } = vi.hoisted(() => ({ - mockMutate: vi.fn<(...args: unknown[]) => void>(), +const { mockMutateAsync, mockCreateMutateAsync, mockUseListExperimentGroups } = vi.hoisted(() => ({ + mockMutateAsync: vi.fn<(...args: unknown[]) => Promise>(), + mockCreateMutateAsync: vi.fn<(...args: unknown[]) => Promise>(), mockUseListExperimentGroups: vi.fn<() => { data: unknown; isLoading: boolean }>(), })); @@ -22,7 +23,8 @@ vi.mock('@nemo/sdk/generated/platform/api', async () => { return { ...actual, useListExperimentGroups: () => mockUseListExperimentGroups(), - useAddEvaluationToGroup: () => ({ mutate: mockMutate, isPending: false }), + useAddEvaluationToExperiment: () => ({ mutateAsync: mockMutateAsync, isPending: false }), + useCreateExperimentGroup: () => ({ mutateAsync: mockCreateMutateAsync, isPending: false }), }; }); @@ -40,12 +42,12 @@ const GROUPS: ExperimentGroupResponse[] = [ makeGroup('g3', 'Gamma benchmarks'), ]; -// The evaluation already belongs to g1, so g1 must be excluded from the picker. -const EVALUATION = { - id: 'eval-1', - name: 'eval-1', - experiment_ids: ['g1'], -} as unknown as EvaluationRow; +const makeEvaluation = (name: string, experimentIds: string[]): EvaluationRow => + ({ id: name, name, experiment_ids: experimentIds }) as unknown as EvaluationRow; + +// eval-1 is only in g1; eval-2 is in g1 and g2. So g1 (all members) is excluded, but g2 (only some +// members) is still offered, as is g3 (no members). +const EVALUATIONS = [makeEvaluation('eval-1', ['g1']), makeEvaluation('eval-2', ['g1', 'g2'])]; function makeWrapper() { const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); @@ -56,24 +58,30 @@ function makeWrapper() { ); } -function renderModal() { +function renderModal(evaluations: EvaluationRow[] = EVALUATIONS) { const Wrapper = makeWrapper(); - return render( + const onSuccess = vi.fn(); + const utils = render( ); + return { ...utils, onSuccess }; } describe('AddToGroupModal', () => { beforeEach(() => { - mockMutate.mockReset(); + mockMutateAsync.mockReset(); + mockMutateAsync.mockResolvedValue(undefined); + mockCreateMutateAsync.mockReset(); + mockCreateMutateAsync.mockResolvedValue(makeGroup('g-new', 'Regression suite')); mockUseListExperimentGroups.mockReset(); mockUseListExperimentGroups.mockReturnValue({ data: { data: GROUPS }, @@ -81,43 +89,64 @@ describe('AddToGroupModal', () => { }); }); - it('lists selectable groups, excluding groups the evaluation already belongs to', () => { + it('offers "Create new group" plus only groups not every selected evaluation already belongs to', async () => { + const user = userEvent.setup(); renderModal(); - // g2 and g3 are offered; g1 (already a member) is not. + await user.click(screen.getByRole('combobox', { name: /experiment group/i })); + expect(await screen.findByRole('option', { name: /create new group/i })).toBeInTheDocument(); + // g1: both evals are members -> excluded. g2: only eval-2 is a member -> still offered. g3: none. expect(screen.getByRole('option', { name: 'Beta benchmarks' })).toBeInTheDocument(); expect(screen.getByRole('option', { name: 'Gamma benchmarks' })).toBeInTheDocument(); expect(screen.queryByRole('option', { name: 'Alpha benchmarks' })).not.toBeInTheDocument(); }); - it('filters the list by the search text (case-insensitive)', async () => { + it('adds every selected evaluation to the chosen group and clears the selection', async () => { const user = userEvent.setup(); - renderModal(); - await user.type(screen.getByRole('textbox', { name: /search groups/i }), 'gamma'); - expect(screen.getByRole('option', { name: 'Gamma benchmarks' })).toBeInTheDocument(); - expect(screen.queryByRole('option', { name: 'Beta benchmarks' })).not.toBeInTheDocument(); + const { onSuccess } = renderModal(); + + await user.click(screen.getByRole('combobox', { name: /experiment group/i })); + await user.click(await screen.findByRole('option', { name: 'Gamma benchmarks' })); + await user.click(screen.getByRole('button', { name: 'Add' })); + + expect(mockCreateMutateAsync).not.toHaveBeenCalled(); + expect(mockMutateAsync).toHaveBeenCalledTimes(2); + expect(mockMutateAsync).toHaveBeenCalledWith({ workspace: 'default', name: 'eval-1', experimentId: 'g3' }); + expect(mockMutateAsync).toHaveBeenCalledWith({ workspace: 'default', name: 'eval-2', experimentId: 'g3' }); + await waitFor(() => expect(onSuccess).toHaveBeenCalledTimes(1)); }); - it('calls the add hook with the evaluation name and selected group id on select', async () => { + it('creates a new group then adds every selected evaluation to it', async () => { const user = userEvent.setup(); - renderModal(); - await user.click(screen.getByRole('option', { name: 'Beta benchmarks' })); - expect(mockMutate).toHaveBeenCalledTimes(1); - expect(mockMutate).toHaveBeenCalledWith( - { workspace: 'default', name: 'eval-1', groupId: 'g2' }, - expect.objectContaining({ - onSuccess: expect.any(Function), - onError: expect.any(Function), - }) - ); + const { onSuccess } = renderModal(); + + await user.click(screen.getByRole('combobox', { name: /experiment group/i })); + await user.click(await screen.findByRole('option', { name: /create new group/i })); + await user.type(screen.getByRole('textbox', { name: /^name$/i }), 'regression-suite'); + await user.click(screen.getByRole('button', { name: 'Create & add' })); + + expect(mockCreateMutateAsync).toHaveBeenCalledWith({ + workspace: 'default', + data: expect.objectContaining({ name: 'regression-suite' }), + }); + // Added to the id returned by the create call. + expect(mockMutateAsync).toHaveBeenCalledTimes(2); + expect(mockMutateAsync).toHaveBeenCalledWith({ workspace: 'default', name: 'eval-1', experimentId: 'g-new' }); + expect(mockMutateAsync).toHaveBeenCalledWith({ workspace: 'default', name: 'eval-2', experimentId: 'g-new' }); + await waitFor(() => expect(onSuccess).toHaveBeenCalledTimes(1)); }); - it('shows an empty message when every group is already a member', () => { + it('still offers "Create new group" when every group already contains all selected evaluations', async () => { + const user = userEvent.setup(); mockUseListExperimentGroups.mockReturnValue({ data: { data: [makeGroup('g1', 'Alpha benchmarks')] }, isLoading: false, }); - renderModal(); - expect(screen.getByText('No groups found.')).toBeInTheDocument(); - expect(screen.queryByRole('option')).not.toBeInTheDocument(); + renderModal([makeEvaluation('eval-1', ['g1']), makeEvaluation('eval-2', ['g1'])]); + + const trigger = screen.getByRole('combobox', { name: /experiment group/i }); + expect(trigger).toBeEnabled(); + await user.click(trigger); + expect(await screen.findByRole('option', { name: /create new group/i })).toBeInTheDocument(); + expect(screen.queryByRole('option', { name: 'Alpha benchmarks' })).not.toBeInTheDocument(); }); }); diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.tsx b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.tsx index 21241399821..b97c911a822 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.tsx +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/AddToGroupModal.tsx @@ -1,63 +1,105 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { zodResolver } from '@hookform/resolvers/zod'; +import { FormModal, type FormModalProps } from '@nemo/common/src/components/FormModal'; import { useToast } from '@nemo/common/src/providers/toast/useToast'; import { getListEvaluationsQueryKey, - useAddEvaluationToGroup, + getListExperimentGroupsQueryKey, + useAddEvaluationToExperiment, + useCreateExperimentGroup, useListExperimentGroups, } from '@nemo/sdk/generated/platform/api'; -import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; import { - Button, Flex, - ModalContent, - ModalDialog, - ModalFooter, - ModalHeading, - ModalMain, - ModalRoot, + FormField, + SelectContent, + SelectItem, + SelectListbox, + SelectRoot, + SelectTrigger, Stack, - Text, + TextArea, TextInput, } from '@nvidia/foundations-react-core'; import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations'; +import { DefaultSortControl } from '@studio/components/DefaultSortControl'; +import { DEFAULT_SORT } from '@studio/components/DefaultSortControl/util'; +import { + experimentGroupCreateSchema, + type ExperimentGroupCreateFormFields, +} from '@studio/components/ExperimentGroupCreateModal/constants'; import { useQueryClient } from '@tanstack/react-query'; -import { type FC, useEffect, useMemo, useState } from 'react'; +import { AxiosError } from 'axios'; +import { Plus } from 'lucide-react'; +import { type FC, type FormEvent, useEffect, useMemo, useState } from 'react'; +import { useForm, type SubmitHandler } from 'react-hook-form'; + +// Sentinel Select value for the "create a new group" affordance. Reserved so it can't collide with a +// real group id (group ids are entity ids, never this literal). +const CREATE_NEW = '__create_new__'; -export interface AddToGroupModalProps { - open: boolean; - onClose: () => void; +export interface AddToGroupModalProps extends Pick { + /** Called after the selected evaluations are successfully added (e.g. to clear the row selection). */ + onSuccess?: () => void; workspace: string; - /** The evaluation being curated into another group. */ - evaluation: EvaluationRow; + /** The evaluations being curated into another group (the bulk row selection). */ + evaluations: EvaluationRow[]; /** * The group whose board is currently shown. Its evaluation list is invalidated on success so the - * board refreshes — mirroring how the pin action scopes invalidation to the current group. + * board refreshes. */ currentExperimentGroupId: string; } /** - * Adds an evaluation to another ExperimentGroup. Shows a searchable list of the workspace's groups — - * excluding the ones the evaluation already belongs to — and adds the evaluation to whichever group - * the user selects, then refreshes the current board and toasts the result. + * Adds one or more selected evaluations to another ExperimentGroup. Offers the workspace's groups in a + * dropdown — excluding groups every selected evaluation already belongs to — plus a "Create new group" + * option that reveals a name/description sub-form. On submit it either adds the evaluations to the + * chosen group, or creates the group first and then adds them, then refreshes the board and toasts. + * + * Membership is owned by the evaluation side (there's no atomic "create group with evaluations" + * endpoint), so the create path is two steps: create the group, then add each evaluation to it. A + * created-but-partially-populated group is a valid state, so a failed add is surfaced as a warning + * rather than rolled back. */ export const AddToGroupModal: FC = ({ open, onClose, + onSuccess, workspace, - evaluation, + evaluations, currentExperimentGroupId, }) => { const toast = useToast(); const queryClient = useQueryClient(); - const [search, setSearch] = useState(''); + const [selectedGroupId, setSelectedGroupId] = useState(''); + const isCreating = selectedGroupId === CREATE_NEW; + // Default sort is a single `sort`-param string driven by a custom control (not a registered RHF + // input), so it's managed here and merged into the create payload (mirrors ExperimentGroupCreateModal). + const [defaultSort, setDefaultSort] = useState(DEFAULT_SORT); + + const { + register, + handleSubmit, + reset, + setValue, + setError, + formState: { errors, isValid }, + } = useForm({ + resolver: zodResolver(experimentGroupCreateSchema), + mode: 'onChange', + }); - // Clear the filter whenever the modal (re)opens so a stale search doesn't hide groups. + // Reset the selection and the create sub-form whenever the modal (re)opens so nothing carries over. useEffect(() => { - if (open) setSearch(''); - }, [open]); + if (open) { + setSelectedGroupId(''); + setDefaultSort(DEFAULT_SORT); + reset(); + } + }, [open, reset]); // Only fetch groups while the modal is open. A single large page covers any realistic group count. const { data: groupsPage, isLoading } = useListExperimentGroups( @@ -66,99 +108,195 @@ export const AddToGroupModal: FC = ({ { query: { enabled: open && !!workspace } } ); - // The evaluation already belongs to these groups, so they're not valid add targets. - const memberIds = useMemo(() => new Set(evaluation.experiment_ids), [evaluation.experiment_ids]); - - // Exclude current members, then filter by the (case-insensitive) search text on the group name. - const groups = useMemo(() => { - const query = search.trim().toLowerCase(); - return (groupsPage?.data ?? []) - .filter((group) => !memberIds.has(group.id)) - .filter((group) => !query || group.name.toLowerCase().includes(query)); - }, [groupsPage, memberIds, search]); - - const { mutate: addEvaluationToGroup, isPending } = useAddEvaluationToGroup(); - - const handleSelect = (group: ExperimentGroupResponse) => { - if (isPending) return; - addEvaluationToGroup( - { workspace, name: evaluation.name, groupId: group.id }, - { - // Mirror the pin action's invalidation: scope to the current group's evaluation lists (any - // page/sort/filter) via a partial key match so the board refreshes without touching other - // groups' queries. - onSuccess: () => { - queryClient.invalidateQueries({ - queryKey: getListEvaluationsQueryKey(workspace, { - filter: { experiment_group_id: currentExperimentGroupId }, - }), - }); - toast.success(`Added "${evaluation.name}" to "${group.name}".`); - onClose(); - }, - onError: () => toast.error(`Failed to add "${evaluation.name}" to "${group.name}".`), - } + // Exclude a group only when EVERY selected evaluation already belongs to it (nothing to add there). + const groups = useMemo( + () => + (groupsPage?.data ?? []).filter( + (group) => !evaluations.every((evaluation) => evaluation.experiment_ids.includes(group.id)) + ), + [groupsPage, evaluations] + ); + + const groupNameById = useMemo( + () => new Map(groups.map((group) => [group.id, group.name])), + [groups] + ); + + const { mutateAsync: addEvaluationToExperiment, isPending: isAdding } = + useAddEvaluationToExperiment(); + const { mutateAsync: createExperimentGroup, isPending: isCreatingGroup } = + useCreateExperimentGroup(); + + const busy = isAdding || isCreatingGroup; + const count = evaluations.length; + const countLabel = `${count} ${count === 1 ? 'evaluation' : 'evaluations'}`; + + // Add every selected evaluation to `groupId`. Adding one already in the group is a server-side no-op, + // so it's safe to add all. Best-effort: returns how many failed rather than throwing on first error. + const associateEvaluations = async (experimentId: string): Promise => { + const results = await Promise.allSettled( + evaluations.map((evaluation) => + addEvaluationToExperiment({ workspace, name: evaluation.name, experimentId }) + ) ); + return results.filter((result) => result.status === 'rejected').length; + }; + + const refreshCurrentBoard = () => { + queryClient.invalidateQueries({ + queryKey: getListEvaluationsQueryKey(workspace, { + filter: { experiment_group_id: currentExperimentGroupId }, + }), + }); }; + // Shared finish for both paths once the target group's membership writes have settled. + const finishAdds = (groupName: string, failed: number, createdVerb: string) => { + if (failed === count) { + // Nothing landed — keep the modal open so the user can retry. + toast.error(`Failed to add ${countLabel} to "${groupName}".`); + return; + } + refreshCurrentBoard(); + if (failed > 0) { + toast.warning( + `${createdVerb} "${groupName}", but ${failed} of ${count} evaluations couldn't be added.` + ); + } else { + toast.success(`${createdVerb} "${groupName}" with ${countLabel}.`); + } + onSuccess?.(); + onClose(); + }; + + const addToExistingGroup = async () => { + const groupName = groupNameById.get(selectedGroupId); + if (!groupName) return; + finishAdds(groupName, await associateEvaluations(selectedGroupId), 'Added to'); + }; + + const createGroupAndAdd: SubmitHandler = async (data) => { + let created; + try { + created = await createExperimentGroup({ + workspace, + data: { name: data.name, description: data.description || undefined, default_sort: defaultSort }, + }); + } catch (error) { + // Creation failed (e.g. duplicate name) — surface inline on the name field where possible and + // keep the modal open. No group was created, so there's nothing to add. + const detail = error instanceof AxiosError ? error.response?.data?.detail : undefined; + if (detail === `Experiment group ${data.name} already exists.`) { + setError('name', { message: detail }); + return; + } + const message = + typeof detail === 'string' + ? detail + : error instanceof Error + ? error.message + : 'Unknown error'; + toast.error(`Failed to create experiment group: ${message}`); + return; + } + // Group now exists; adding evaluations is best-effort (a group with fewer evals is still valid). + queryClient.invalidateQueries({ queryKey: getListExperimentGroupsQueryKey(workspace) }); + finishAdds(created.name, await associateEvaluations(created.id), 'Created'); + }; + + const onSubmit = (e: FormEvent) => { + // FormModal renders the
; it doesn't preventDefault. RHF's handleSubmit does it for us on the + // create path; the add path has no RHF wrapper so we preventDefault ourselves. + if (isCreating) { + void handleSubmit(createGroupAndAdd)(e); + } else { + e.preventDefault(); + void addToExistingGroup(); + } + }; + + const submitButtonText = isCreating + ? isCreatingGroup + ? 'Creating…' + : 'Create & add' + : isAdding + ? 'Adding…' + : 'Add'; + return ( - - - - Add to group - - - - Add "{evaluation.name}" to another experiment group to compare it across boards. - + + + + + { + if (v === CREATE_NEW) return 'Create new group'; + return typeof v === 'string' && v ? (groupNameById.get(v) ?? undefined) : undefined; + }} + /> + + + + + + Create new group + + + {groups.map((group) => ( + + {group.name} + + ))} + + + + + + {isCreating && ( + <> + + setValue('name', (e.target as HTMLInputElement).value.replace(/[\s-]+/g, '-'), { + shouldValidate: true, + }) + } /> - - {isLoading ? ( - - Loading groups… - - ) : groups.length === 0 ? ( - - No groups found. - - ) : ( - groups.map((group) => ( - - )) - )} - - - - - - - - - - - + + +