From 75869da4e3ccffaa5ab1cfba7a39b09823b7247b Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Wed, 1 Jul 2026 14:13:35 -0600 Subject: [PATCH 1/3] feat(experiments): Add new fields to support NeMo Evaluator Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- openapi/ga/individual/platform.openapi.yaml | 59 ++++++++++++ openapi/ga/openapi.yaml | 59 ++++++++++++ openapi/openapi.yaml | 59 ++++++++++++ .../nemo-platform/.nmpcontext/openapi.yaml | 59 ++++++++++++ .../nemo-platform/.nmpcontext/stainless.yaml | 1 + .../experiment_groups/experiment_groups.py | 50 +++++++++- .../resources/experiments/api.md | 1 + .../resources/experiments/experiments.py | 58 ++++++++++++ .../experiment_group_create_params.py | 11 ++- .../experiment_group_response.py | 8 +- .../experiment_group_update_params.py | 11 ++- .../types/experiments/__init__.py | 1 + .../experiments/experiment_create_params.py | 17 ++++ .../types/experiments/experiment_response.py | 8 ++ .../types/experiments/experiment_status.py | 24 +++++ .../experiments/experiment_update_params.py | 16 ++++ .../api_resources/test_experiment_groups.py | 12 +++ .../tests/api_resources/test_experiments.py | 12 +++ sdk/stainless.yaml | 1 + .../intake/api/v2/experiments/endpoints.py | 17 +++- .../nmp/intake/api/v2/experiments/schemas.py | 28 +++++- .../src/nmp/intake/entities/experiments.py | 33 +++++++ .../tests/test_experiment_evaluator_fields.py | 91 +++++++++++++++++++ 23 files changed, 630 insertions(+), 6 deletions(-) create mode 100644 sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_status.py create mode 100644 services/intake/tests/test_experiment_evaluator_fields.py diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index f787da661e..132628b54e 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -10387,6 +10387,20 @@ components: title: Description description: Human-readable purpose of the group. type: string + insight_id: + title: Insight Id + description: Entity id of the NeMo Evaluator insight that seeded this group, + if any. + type: string + summary: + title: Summary + description: Human- or agent-authored summary of the group's findings. + type: string + metadata: + title: Metadata + description: Free-form producer metadata for the group. + additionalProperties: true + type: object default_sort: title: Default Sort description: 'Ordered default sort (priority order; first is primary, rest @@ -10416,6 +10430,16 @@ components: description: title: Description type: string + insight_id: + title: Insight Id + type: string + summary: + title: Summary + type: string + metadata: + title: Metadata + additionalProperties: true + type: object default_sort: title: Default Sort items: @@ -10499,6 +10523,20 @@ components: title: Description description: Human-readable description. type: string + parent_experiment_id: + title: Parent Experiment Id + description: Entity id of the experiment this one was derived from (e.g. + a variant of a baseline), if any. + type: string + status: + allOf: + - $ref: '#/components/schemas/ExperimentStatus' + description: Lifecycle status in the NeMo Evaluator flow. + root_cause: + title: Root Cause + description: Human- or agent-authored explanation of the experiment's outcome + (e.g. why it was killed). + type: string additionalProperties: false type: object required: @@ -10541,6 +10579,14 @@ components: description: title: Description type: string + parent_experiment_id: + title: Parent Experiment Id + type: string + status: + $ref: '#/components/schemas/ExperimentStatus' + root_cause: + title: Root Cause + type: string created_at: title: Created At type: string @@ -10750,6 +10796,19 @@ components: required: - data title: ExperimentSessionResponsesPage + ExperimentStatus: + type: string + enum: + - baseline + - running + - evaluated + - survived + - killed + - winner + - deployed + - rejected + title: ExperimentStatus + description: Lifecycle status of an Experiment in the NeMo Evaluator flow. FactCheckingRailConfig: properties: parameters: diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index f787da661e..132628b54e 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -10387,6 +10387,20 @@ components: title: Description description: Human-readable purpose of the group. type: string + insight_id: + title: Insight Id + description: Entity id of the NeMo Evaluator insight that seeded this group, + if any. + type: string + summary: + title: Summary + description: Human- or agent-authored summary of the group's findings. + type: string + metadata: + title: Metadata + description: Free-form producer metadata for the group. + additionalProperties: true + type: object default_sort: title: Default Sort description: 'Ordered default sort (priority order; first is primary, rest @@ -10416,6 +10430,16 @@ components: description: title: Description type: string + insight_id: + title: Insight Id + type: string + summary: + title: Summary + type: string + metadata: + title: Metadata + additionalProperties: true + type: object default_sort: title: Default Sort items: @@ -10499,6 +10523,20 @@ components: title: Description description: Human-readable description. type: string + parent_experiment_id: + title: Parent Experiment Id + description: Entity id of the experiment this one was derived from (e.g. + a variant of a baseline), if any. + type: string + status: + allOf: + - $ref: '#/components/schemas/ExperimentStatus' + description: Lifecycle status in the NeMo Evaluator flow. + root_cause: + title: Root Cause + description: Human- or agent-authored explanation of the experiment's outcome + (e.g. why it was killed). + type: string additionalProperties: false type: object required: @@ -10541,6 +10579,14 @@ components: description: title: Description type: string + parent_experiment_id: + title: Parent Experiment Id + type: string + status: + $ref: '#/components/schemas/ExperimentStatus' + root_cause: + title: Root Cause + type: string created_at: title: Created At type: string @@ -10750,6 +10796,19 @@ components: required: - data title: ExperimentSessionResponsesPage + ExperimentStatus: + type: string + enum: + - baseline + - running + - evaluated + - survived + - killed + - winner + - deployed + - rejected + title: ExperimentStatus + description: Lifecycle status of an Experiment in the NeMo Evaluator flow. FactCheckingRailConfig: properties: parameters: diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index f787da661e..132628b54e 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -10387,6 +10387,20 @@ components: title: Description description: Human-readable purpose of the group. type: string + insight_id: + title: Insight Id + description: Entity id of the NeMo Evaluator insight that seeded this group, + if any. + type: string + summary: + title: Summary + description: Human- or agent-authored summary of the group's findings. + type: string + metadata: + title: Metadata + description: Free-form producer metadata for the group. + additionalProperties: true + type: object default_sort: title: Default Sort description: 'Ordered default sort (priority order; first is primary, rest @@ -10416,6 +10430,16 @@ components: description: title: Description type: string + insight_id: + title: Insight Id + type: string + summary: + title: Summary + type: string + metadata: + title: Metadata + additionalProperties: true + type: object default_sort: title: Default Sort items: @@ -10499,6 +10523,20 @@ components: title: Description description: Human-readable description. type: string + parent_experiment_id: + title: Parent Experiment Id + description: Entity id of the experiment this one was derived from (e.g. + a variant of a baseline), if any. + type: string + status: + allOf: + - $ref: '#/components/schemas/ExperimentStatus' + description: Lifecycle status in the NeMo Evaluator flow. + root_cause: + title: Root Cause + description: Human- or agent-authored explanation of the experiment's outcome + (e.g. why it was killed). + type: string additionalProperties: false type: object required: @@ -10541,6 +10579,14 @@ components: description: title: Description type: string + parent_experiment_id: + title: Parent Experiment Id + type: string + status: + $ref: '#/components/schemas/ExperimentStatus' + root_cause: + title: Root Cause + type: string created_at: title: Created At type: string @@ -10750,6 +10796,19 @@ components: required: - data title: ExperimentSessionResponsesPage + ExperimentStatus: + type: string + enum: + - baseline + - running + - evaluated + - survived + - killed + - winner + - deployed + - rejected + title: ExperimentStatus + description: Lifecycle status of an Experiment in the NeMo Evaluator flow. FactCheckingRailConfig: properties: parameters: diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index f787da661e..132628b54e 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -10387,6 +10387,20 @@ components: title: Description description: Human-readable purpose of the group. type: string + insight_id: + title: Insight Id + description: Entity id of the NeMo Evaluator insight that seeded this group, + if any. + type: string + summary: + title: Summary + description: Human- or agent-authored summary of the group's findings. + type: string + metadata: + title: Metadata + description: Free-form producer metadata for the group. + additionalProperties: true + type: object default_sort: title: Default Sort description: 'Ordered default sort (priority order; first is primary, rest @@ -10416,6 +10430,16 @@ components: description: title: Description type: string + insight_id: + title: Insight Id + type: string + summary: + title: Summary + type: string + metadata: + title: Metadata + additionalProperties: true + type: object default_sort: title: Default Sort items: @@ -10499,6 +10523,20 @@ components: title: Description description: Human-readable description. type: string + parent_experiment_id: + title: Parent Experiment Id + description: Entity id of the experiment this one was derived from (e.g. + a variant of a baseline), if any. + type: string + status: + allOf: + - $ref: '#/components/schemas/ExperimentStatus' + description: Lifecycle status in the NeMo Evaluator flow. + root_cause: + title: Root Cause + description: Human- or agent-authored explanation of the experiment's outcome + (e.g. why it was killed). + type: string additionalProperties: false type: object required: @@ -10541,6 +10579,14 @@ components: description: title: Description type: string + parent_experiment_id: + title: Parent Experiment Id + type: string + status: + $ref: '#/components/schemas/ExperimentStatus' + root_cause: + title: Root Cause + type: string created_at: title: Created At type: string @@ -10750,6 +10796,19 @@ components: required: - data title: ExperimentSessionResponsesPage + ExperimentStatus: + type: string + enum: + - baseline + - running + - evaluated + - survived + - killed + - winner + - deployed + - rejected + title: ExperimentStatus + description: Lifecycle status of an Experiment in the NeMo Evaluator flow. FactCheckingRailConfig: properties: parameters: diff --git a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml index c30321bc99..53824b5640 100644 --- a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml @@ -914,6 +914,7 @@ resources: experiment_request: ExperimentRequest experiment_response: ExperimentResponse experiment_responses_page: ExperimentResponsesPage + experiment_status: ExperimentStatus metric_stat_filters: MetricStatFilters number_filter: NumberFilter methods: diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py index 7322acca00..71368d3f2f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py @@ -17,7 +17,7 @@ from __future__ import annotations -from typing import Iterable +from typing import Dict, Iterable from typing_extensions import Literal import httpx @@ -74,6 +74,9 @@ def create( name: str, default_sort: Iterable[SortCriterionParam] | Omit = omit, description: str | Omit = omit, + insight_id: str | Omit = omit, + metadata: Dict[str, object] | Omit = omit, + summary: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. exist_ok: bool = False, @@ -94,6 +97,12 @@ def create( description: Human-readable purpose of the group. + insight_id: Entity id of the NeMo Evaluator insight that seeded this group, if any. + + metadata: Free-form producer metadata for the group. + + summary: Human- or agent-authored summary of the group's findings. + exist_ok: Do not raise an error if the resource already exists. Returns the existing resource. @@ -118,6 +127,9 @@ def create( "name": name, "default_sort": default_sort, "description": description, + "insight_id": insight_id, + "metadata": metadata, + "summary": summary, }, experiment_group_create_params.ExperimentGroupCreateParams, ), @@ -179,6 +191,9 @@ def update( body_name: str, default_sort: Iterable[SortCriterionParam] | Omit = omit, description: str | Omit = omit, + insight_id: str | Omit = omit, + metadata: Dict[str, object] | Omit = omit, + summary: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -198,6 +213,12 @@ def update( description: Human-readable purpose of the group. + insight_id: Entity id of the NeMo Evaluator insight that seeded this group, if any. + + metadata: Free-form producer metadata for the group. + + summary: Human- or agent-authored summary of the group's findings. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -223,6 +244,9 @@ def update( "body_name": body_name, "default_sort": default_sort, "description": description, + "insight_id": insight_id, + "metadata": metadata, + "summary": summary, }, experiment_group_update_params.ExperimentGroupUpdateParams, ), @@ -361,6 +385,9 @@ async def create( name: str, default_sort: Iterable[SortCriterionParam] | Omit = omit, description: str | Omit = omit, + insight_id: str | Omit = omit, + metadata: Dict[str, object] | Omit = omit, + summary: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. exist_ok: bool = False, @@ -381,6 +408,12 @@ async def create( description: Human-readable purpose of the group. + insight_id: Entity id of the NeMo Evaluator insight that seeded this group, if any. + + metadata: Free-form producer metadata for the group. + + summary: Human- or agent-authored summary of the group's findings. + exist_ok: Do not raise an error if the resource already exists. Returns the existing resource. @@ -405,6 +438,9 @@ async def create( "name": name, "default_sort": default_sort, "description": description, + "insight_id": insight_id, + "metadata": metadata, + "summary": summary, }, experiment_group_create_params.ExperimentGroupCreateParams, ), @@ -466,6 +502,9 @@ async def update( body_name: str, default_sort: Iterable[SortCriterionParam] | Omit = omit, description: str | Omit = omit, + insight_id: str | Omit = omit, + metadata: Dict[str, object] | Omit = omit, + summary: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -485,6 +524,12 @@ async def update( description: Human-readable purpose of the group. + insight_id: Entity id of the NeMo Evaluator insight that seeded this group, if any. + + metadata: Free-form producer metadata for the group. + + summary: Human- or agent-authored summary of the group's findings. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -510,6 +555,9 @@ async def update( "body_name": body_name, "default_sort": default_sort, "description": description, + "insight_id": insight_id, + "metadata": metadata, + "summary": summary, }, experiment_group_update_params.ExperimentGroupUpdateParams, ), diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md index 0c6e16973f..2f70554173 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md @@ -9,6 +9,7 @@ from nemo_platform.types.experiments import ( ExperimentRequest, ExperimentResponse, ExperimentResponsesPage, + ExperimentStatus, MetricStatFilters, NumberFilter, ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py index 2f52d5a957..e7ab681acf 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py @@ -42,10 +42,12 @@ from ...pagination import SyncDefaultPagination, AsyncDefaultPagination from ..._base_client import AsyncPaginator, make_request_options from ...types.experiments import ( + ExperimentStatus, experiment_list_params, experiment_create_params, experiment_update_params, ) +from ...types.experiments.experiment_status import ExperimentStatus from ...types.experiments.experiment_response import ExperimentResponse from ...types.experiments.experiment_filter_param import ExperimentFilterParam from ..._exceptions import ConflictError @@ -87,7 +89,10 @@ def create( dataset_version: str | Omit = omit, description: str | Omit = omit, metadata: Dict[str, object] | Omit = omit, + parent_experiment_id: str | Omit = omit, + root_cause: str | Omit = omit, source_link: str | Omit = omit, + status: ExperimentStatus | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. exist_ok: bool = False, @@ -113,8 +118,16 @@ def create( metadata: Free-form producer metadata. + parent_experiment_id: Entity id of the experiment this one was derived from (e.g. a variant of a + baseline), if any. + + root_cause: Human- or agent-authored explanation of the experiment's outcome (e.g. why it + was killed). + source_link: Optional URL for the source experiment. + status: Lifecycle status of an Experiment in the NeMo Evaluator flow. + exist_ok: Do not raise an error if the resource already exists. Returns the existing resource. @@ -142,7 +155,10 @@ def create( "dataset_version": dataset_version, "description": description, "metadata": metadata, + "parent_experiment_id": parent_experiment_id, + "root_cause": root_cause, "source_link": source_link, + "status": status, }, experiment_create_params.ExperimentCreateParams, ), @@ -205,7 +221,10 @@ def update( dataset_version: str | Omit = omit, description: str | Omit = omit, metadata: Dict[str, object] | Omit = omit, + parent_experiment_id: str | Omit = omit, + root_cause: str | Omit = omit, source_link: str | Omit = omit, + status: ExperimentStatus | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -230,8 +249,16 @@ def update( metadata: Free-form producer metadata. + parent_experiment_id: Entity id of the experiment this one was derived from (e.g. a variant of a + baseline), if any. + + root_cause: Human- or agent-authored explanation of the experiment's outcome (e.g. why it + was killed). + source_link: Optional URL for the source experiment. + status: Lifecycle status of an Experiment in the NeMo Evaluator flow. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -260,7 +287,10 @@ def update( "dataset_version": dataset_version, "description": description, "metadata": metadata, + "parent_experiment_id": parent_experiment_id, + "root_cause": root_cause, "source_link": source_link, + "status": status, }, experiment_update_params.ExperimentUpdateParams, ), @@ -500,7 +530,10 @@ async def create( dataset_version: str | Omit = omit, description: str | Omit = omit, metadata: Dict[str, object] | Omit = omit, + parent_experiment_id: str | Omit = omit, + root_cause: str | Omit = omit, source_link: str | Omit = omit, + status: ExperimentStatus | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. exist_ok: bool = False, @@ -526,8 +559,16 @@ async def create( metadata: Free-form producer metadata. + parent_experiment_id: Entity id of the experiment this one was derived from (e.g. a variant of a + baseline), if any. + + root_cause: Human- or agent-authored explanation of the experiment's outcome (e.g. why it + was killed). + source_link: Optional URL for the source experiment. + status: Lifecycle status of an Experiment in the NeMo Evaluator flow. + exist_ok: Do not raise an error if the resource already exists. Returns the existing resource. @@ -555,7 +596,10 @@ async def create( "dataset_version": dataset_version, "description": description, "metadata": metadata, + "parent_experiment_id": parent_experiment_id, + "root_cause": root_cause, "source_link": source_link, + "status": status, }, experiment_create_params.ExperimentCreateParams, ), @@ -618,7 +662,10 @@ async def update( dataset_version: str | Omit = omit, description: str | Omit = omit, metadata: Dict[str, object] | Omit = omit, + parent_experiment_id: str | Omit = omit, + root_cause: str | Omit = omit, source_link: str | Omit = omit, + status: ExperimentStatus | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -643,8 +690,16 @@ async def update( metadata: Free-form producer metadata. + parent_experiment_id: Entity id of the experiment this one was derived from (e.g. a variant of a + baseline), if any. + + root_cause: Human- or agent-authored explanation of the experiment's outcome (e.g. why it + was killed). + source_link: Optional URL for the source experiment. + status: Lifecycle status of an Experiment in the NeMo Evaluator flow. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -673,7 +728,10 @@ async def update( "dataset_version": dataset_version, "description": description, "metadata": metadata, + "parent_experiment_id": parent_experiment_id, + "root_cause": root_cause, "source_link": source_link, + "status": status, }, experiment_update_params.ExperimentUpdateParams, ), diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py index ff87ff655b..6509223b43 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py @@ -17,7 +17,7 @@ from __future__ import annotations -from typing import Iterable +from typing import Dict, Iterable from typing_extensions import Required, TypedDict from .sort_criterion_param import SortCriterionParam @@ -40,3 +40,12 @@ class ExperimentGroupCreateParams(TypedDict, total=False): description: str """Human-readable purpose of the group.""" + + insight_id: str + """Entity id of the NeMo Evaluator insight that seeded this group, if any.""" + + metadata: Dict[str, object] + """Free-form producer metadata for the group.""" + + summary: str + """Human- or agent-authored summary of the group's findings.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py index 77d2734726..59056155d2 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_response.py @@ -15,7 +15,7 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from typing import List, Optional +from typing import Dict, List, Optional from datetime import datetime from ..._models import BaseModel @@ -42,4 +42,10 @@ class ExperimentGroupResponse(BaseModel): experiment_count: Optional[int] = None """Number of live (non-soft-deleted) experiments in this group.""" + insight_id: Optional[str] = None + + metadata: Optional[Dict[str, object]] = None + + summary: Optional[str] = None + updated_at: Optional[datetime] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py index a2ab8a8b07..af485b2476 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py @@ -17,7 +17,7 @@ from __future__ import annotations -from typing import Iterable +from typing import Dict, Iterable from typing_extensions import Required, Annotated, TypedDict from ..._utils import PropertyInfo @@ -41,3 +41,12 @@ class ExperimentGroupUpdateParams(TypedDict, total=False): description: str """Human-readable purpose of the group.""" + + insight_id: str + """Entity id of the NeMo Evaluator insight that seeded this group, if any.""" + + metadata: Dict[str, object] + """Free-form producer metadata for the group.""" + + summary: str + """Human- or agent-authored summary of the group's findings.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/__init__.py index b046d1fc20..7ced9e1cbc 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/__init__.py @@ -17,6 +17,7 @@ from __future__ import annotations +from .experiment_status import ExperimentStatus as ExperimentStatus from .evaluator_aggregate import EvaluatorAggregate as EvaluatorAggregate from .experiment_response import ExperimentResponse as ExperimentResponse from .number_filter_param import NumberFilterParam as NumberFilterParam diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py index 8d828f9274..efce920d6a 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py @@ -20,6 +20,8 @@ from typing import Dict from typing_extensions import Required, TypedDict +from .experiment_status import ExperimentStatus + __all__ = ["ExperimentCreateParams"] @@ -47,5 +49,20 @@ class ExperimentCreateParams(TypedDict, total=False): metadata: Dict[str, object] """Free-form producer metadata.""" + parent_experiment_id: str + """Entity id of the experiment this one was derived from (e.g. + + a variant of a baseline), if any. + """ + + root_cause: str + """Human- or agent-authored explanation of the experiment's outcome (e.g. + + why it was killed). + """ + source_link: str """Optional URL for the source experiment.""" + + status: ExperimentStatus + """Lifecycle status of an Experiment in the NeMo Evaluator flow.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py index 6072d3bcb8..617dfdf935 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py @@ -20,6 +20,7 @@ from ..._compat import PYDANTIC_V1, ConfigDict from ..._models import BaseModel +from .experiment_status import ExperimentStatus from .evaluator_aggregate import EvaluatorAggregate __all__ = ["ExperimentResponse"] @@ -66,12 +67,16 @@ class ExperimentResponse(BaseModel): model_names: Optional[List[str]] = None """Distinct model names observed across ingested sessions for this experiment.""" + parent_experiment_id: Optional[str] = None + pinned_at: Optional[datetime] = None """Timestamp at which the experiment was pinned, or null if unpinned. Managed via POST/DELETE /experiments/{name}/pin. """ + root_cause: Optional[str] = None + run_count: Optional[int] = None """ Number of distinct ingested experiment sessions; one session is treated as one @@ -80,6 +85,9 @@ class ExperimentResponse(BaseModel): source_link: Optional[str] = None + status: Optional[ExperimentStatus] = None + """Lifecycle status of an Experiment in the NeMo Evaluator flow.""" + updated_at: Optional[datetime] = None if not PYDANTIC_V1: diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_status.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_status.py new file mode 100644 index 0000000000..1b4f05f523 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_status.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal, TypeAlias + +__all__ = ["ExperimentStatus"] + +ExperimentStatus: TypeAlias = Literal[ + "baseline", "running", "evaluated", "survived", "killed", "winner", "deployed", "rejected" +] diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py index 8546227aa3..0cb9572480 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py @@ -21,6 +21,7 @@ from typing_extensions import Required, Annotated, TypedDict from ..._utils import PropertyInfo +from .experiment_status import ExperimentStatus __all__ = ["ExperimentUpdateParams"] @@ -49,5 +50,20 @@ class ExperimentUpdateParams(TypedDict, total=False): metadata: Dict[str, object] """Free-form producer metadata.""" + parent_experiment_id: str + """Entity id of the experiment this one was derived from (e.g. + + a variant of a baseline), if any. + """ + + root_cause: str + """Human- or agent-authored explanation of the experiment's outcome (e.g. + + why it was killed). + """ + source_link: str """Optional URL for the source experiment.""" + + status: ExperimentStatus + """Lifecycle status of an Experiment in the NeMo Evaluator flow.""" diff --git a/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py b/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py index ea95bb460e..ef091e307f 100644 --- a/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py +++ b/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py @@ -57,6 +57,9 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: } ], description="description", + insight_id="insight_id", + metadata={"foo": "bar"}, + summary="summary", ) assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) @@ -173,6 +176,9 @@ def test_method_update_with_all_params(self, client: NeMoPlatform) -> None: } ], description="description", + insight_id="insight_id", + metadata={"foo": "bar"}, + summary="summary", ) assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) @@ -360,6 +366,9 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo } ], description="description", + insight_id="insight_id", + metadata={"foo": "bar"}, + summary="summary", ) assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) @@ -476,6 +485,9 @@ async def test_method_update_with_all_params(self, async_client: AsyncNeMoPlatfo } ], description="description", + insight_id="insight_id", + metadata={"foo": "bar"}, + summary="summary", ) assert_matches_type(ExperimentGroupResponse, experiment_group, path=["response"]) diff --git a/sdk/python/nemo-platform/tests/api_resources/test_experiments.py b/sdk/python/nemo-platform/tests/api_resources/test_experiments.py index 4279d7521e..21a6e7245b 100644 --- a/sdk/python/nemo-platform/tests/api_resources/test_experiments.py +++ b/sdk/python/nemo-platform/tests/api_resources/test_experiments.py @@ -58,7 +58,10 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: dataset_version="dataset_version", description="description", metadata={"foo": "bar"}, + parent_experiment_id="parent_experiment_id", + root_cause="root_cause", source_link="https://example.com", + status="baseline", ) assert_matches_type(ExperimentResponse, experiment, path=["response"]) @@ -181,7 +184,10 @@ def test_method_update_with_all_params(self, client: NeMoPlatform) -> None: dataset_version="dataset_version", description="description", metadata={"foo": "bar"}, + parent_experiment_id="parent_experiment_id", + root_cause="root_cause", source_link="https://example.com", + status="baseline", ) assert_matches_type(ExperimentResponse, experiment, path=["response"]) @@ -656,7 +662,10 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo dataset_version="dataset_version", description="description", metadata={"foo": "bar"}, + parent_experiment_id="parent_experiment_id", + root_cause="root_cause", source_link="https://example.com", + status="baseline", ) assert_matches_type(ExperimentResponse, experiment, path=["response"]) @@ -779,7 +788,10 @@ async def test_method_update_with_all_params(self, async_client: AsyncNeMoPlatfo dataset_version="dataset_version", description="description", metadata={"foo": "bar"}, + parent_experiment_id="parent_experiment_id", + root_cause="root_cause", source_link="https://example.com", + status="baseline", ) assert_matches_type(ExperimentResponse, experiment, path=["response"]) diff --git a/sdk/stainless.yaml b/sdk/stainless.yaml index c30321bc99..53824b5640 100644 --- a/sdk/stainless.yaml +++ b/sdk/stainless.yaml @@ -914,6 +914,7 @@ resources: experiment_request: ExperimentRequest experiment_response: ExperimentResponse experiment_responses_page: ExperimentResponsesPage + experiment_status: ExperimentStatus metric_stat_filters: MetricStatFilters number_filter: NumberFilter methods: diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index 92f63edd2a..dcf5daafc7 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -120,7 +120,13 @@ async def create_experiment_group( ) -> ExperimentGroupResponse: _validate_default_sort(body.default_sort) entity = ExperimentGroup( - workspace=workspace, name=body.name, description=body.description, default_sort=body.default_sort + workspace=workspace, + name=body.name, + description=body.description, + insight_id=body.insight_id, + summary=body.summary, + metadata=body.metadata, + default_sort=body.default_sort, ) try: created = await entity_client.create(entity) @@ -232,6 +238,9 @@ async def update_experiment_group( ) _validate_default_sort(body.default_sort) existing.description = body.description + existing.insight_id = body.insight_id + existing.summary = body.summary + existing.metadata = body.metadata existing.default_sort = body.default_sort updated = await entity_client.update(existing) response = ExperimentGroupResponse.from_entity(updated) @@ -324,6 +333,9 @@ async def create_experiment( source_link=body.source_link, metadata=body.metadata, description=body.description, + parent_experiment_id=body.parent_experiment_id, + status=body.status, + root_cause=body.root_cause, ) try: created = await entity_client.create(entity) @@ -527,6 +539,9 @@ async def update_experiment( existing.source_link = body.source_link existing.metadata = body.metadata existing.description = body.description + existing.parent_experiment_id = body.parent_experiment_id + existing.status = body.status + existing.root_cause = body.root_cause updated = await entity_client.update(existing) response = ExperimentResponse.from_entity(updated) await _hydrate_rollups(workspace=workspace, responses=[response], rollup_repository=rollup_repository) diff --git a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py index 5e2ccaea4e..1f6fd9825c 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py @@ -13,7 +13,7 @@ from typing import Annotated, Any from nmp.common.entities.values import DatetimeFilter, Filter, NumberFilter, map_entity_field -from nmp.intake.entities.experiments import Experiment, ExperimentGroup, SortCriterion +from nmp.intake.entities.experiments import Experiment, ExperimentGroup, ExperimentStatus, SortCriterion from nmp.intake.spans.domain import SpanStatus from nmp.intake.spans.experiment_session_repository import ExperimentSessionRow from pydantic import AnyUrl, BaseModel, ConfigDict, Field @@ -26,6 +26,11 @@ class ExperimentGroupRequest(BaseModel): name: str = Field(description="Workspace-unique group name.") description: str | None = Field(default=None, description="Human-readable purpose of the group.") + insight_id: str | None = Field( + default=None, description="Entity id of the NeMo Evaluator insight that seeded this group, if any." + ) + summary: str | None = Field(default=None, description="Human- or agent-authored summary of the group's findings.") + metadata: dict[str, Any] | None = Field(default=None, description="Free-form producer metadata for the group.") default_sort: list[SortCriterion] | None = Field( default=None, description=( @@ -50,6 +55,15 @@ class ExperimentRequest(BaseModel): source_link: AnyUrl | None = Field(default=None, description="Optional URL for the source experiment.") metadata: dict[str, Any] = Field(default_factory=dict, description="Free-form producer metadata.") description: str | None = Field(default=None, description="Human-readable description.") + parent_experiment_id: str | None = Field( + default=None, + description="Entity id of the experiment this one was derived from (e.g. a variant of a baseline), if any.", + ) + status: ExperimentStatus | None = Field(default=None, description="Lifecycle status in the NeMo Evaluator flow.") + root_cause: str | None = Field( + default=None, + description="Human- or agent-authored explanation of the experiment's outcome (e.g. why it was killed).", + ) class ExperimentGroupResponse(BaseModel): @@ -59,6 +73,9 @@ class ExperimentGroupResponse(BaseModel): name: str workspace: str description: str | None = None + insight_id: str | None = None + summary: str | None = None + metadata: dict[str, Any] | None = None default_sort: list[SortCriterion] | None = None created_at: datetime | None = None updated_at: datetime | None = None @@ -74,6 +91,9 @@ def from_entity(cls, entity: ExperimentGroup) -> ExperimentGroupResponse: name=entity.name, workspace=entity.workspace, description=entity.description, + insight_id=entity.insight_id, + summary=entity.summary, + metadata=entity.metadata, default_sort=entity.default_sort, created_at=entity.created_at, updated_at=entity.updated_at, @@ -106,6 +126,9 @@ class ExperimentResponse(BaseModel): source_link: AnyUrl | None = None metadata: dict[str, Any] = Field(default_factory=dict) description: str | None = None + parent_experiment_id: str | None = None + status: ExperimentStatus | None = None + root_cause: str | None = None created_at: datetime | None = None updated_at: datetime | None = None pinned_at: datetime | None = Field( @@ -153,6 +176,9 @@ def from_entity(cls, entity: Experiment) -> ExperimentResponse: source_link=entity.source_link, metadata=entity.metadata, description=entity.description, + parent_experiment_id=entity.parent_experiment_id, + status=entity.status, + root_cause=entity.root_cause, created_at=entity.created_at, updated_at=entity.updated_at, pinned_at=entity.pinned_at, diff --git a/services/intake/src/nmp/intake/entities/experiments.py b/services/intake/src/nmp/intake/entities/experiments.py index b27c459576..e697ba09d2 100644 --- a/services/intake/src/nmp/intake/entities/experiments.py +++ b/services/intake/src/nmp/intake/entities/experiments.py @@ -11,6 +11,7 @@ from __future__ import annotations from datetime import datetime +from enum import Enum from typing import Any, ClassVar, Literal from nmp.common.entities.client import EntityBase @@ -26,6 +27,19 @@ class SortCriterion(BaseModel): direction: Literal["asc", "desc"] = Field(description="Sort direction for this field.") +class ExperimentStatus(str, Enum): + """Lifecycle status of an Experiment in the NeMo Evaluator flow.""" + + baseline = "baseline" + running = "running" + evaluated = "evaluated" + survived = "survived" + killed = "killed" + winner = "winner" + deployed = "deployed" + rejected = "rejected" + + class ExperimentGroup(EntityBase): """A named container of Experiments pursuing a single optimization goal. @@ -35,6 +49,12 @@ class ExperimentGroup(EntityBase): __entity_type__: ClassVar[str] = "experiment_group" description: str | None = Field(default=None, description="Human-readable purpose of the group.") + insight_id: str | None = Field( + default=None, + description="Entity id of the NeMo Evaluator insight that seeded this group, if any.", + ) + summary: str | None = Field(default=None, description="Human- or agent-authored summary of the group's findings.") + metadata: dict[str, Any] | None = Field(default=None, description="Free-form producer metadata for the group.") default_sort: list[SortCriterion] | None = Field( default=None, description=( @@ -78,6 +98,19 @@ class Experiment(EntityBase): description: str | None = Field(default=None, description="Human-readable description of the experiment.") + parent_experiment_id: str | None = Field( + default=None, + description="Entity id of the experiment this one was derived from (e.g. a variant of a baseline), if any.", + ) + status: ExperimentStatus | None = Field( + default=None, + description="Lifecycle status in the NeMo Evaluator flow.", + ) + root_cause: str | None = Field( + default=None, + description="Human- or agent-authored explanation of the experiment's outcome (e.g. why it was killed).", + ) + is_deleted: bool = Field( default=False, description=( diff --git a/services/intake/tests/test_experiment_evaluator_fields.py b/services/intake/tests/test_experiment_evaluator_fields.py new file mode 100644 index 0000000000..6e48b93667 --- /dev/null +++ b/services/intake/tests/test_experiment_evaluator_fields.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""NeMo Evaluator fields on ExperimentGroup (insight_id, summary, metadata) and Experiment +(parent_experiment_id, status, root_cause). All are optional and round-trip through create/update.""" + +from fastapi.testclient import TestClient + +EXPERIMENTS = "/apis/intake/v2/workspaces/default/experiments" +GROUPS = "/apis/intake/v2/workspaces/default/experiment-groups" + + +def _group(client: TestClient, name: str = "grp") -> dict: + resp = client.post(GROUPS, json={"name": name}) + assert resp.status_code == 201, resp.text + return resp.json() + + +def test_group_evaluator_fields_round_trip(client: TestClient) -> None: + resp = client.post( + GROUPS, + json={"name": "g1", "insight_id": "insight-123", "summary": "looks promising", "metadata": {"k": "v"}}, + ) + assert resp.status_code == 201, resp.text + body = resp.json() + assert body["insight_id"] == "insight-123" + assert body["summary"] == "looks promising" + assert body["metadata"] == {"k": "v"} + + +def test_experiment_evaluator_fields_round_trip(client: TestClient) -> None: + group = _group(client) + resp = client.post( + EXPERIMENTS, + json={ + "name": "exp-1", + "experiment_group_id": group["id"], + "dataset_name": "ds", + "parent_experiment_id": "exp-0", + "status": "running", + "root_cause": "still evaluating", + }, + ) + assert resp.status_code == 201, resp.text + body = resp.json() + assert body["parent_experiment_id"] == "exp-0" + assert body["status"] == "running" + assert body["root_cause"] == "still evaluating" + + +def test_experiment_rejects_invalid_status(client: TestClient) -> None: + group = _group(client) + resp = client.post( + EXPERIMENTS, + json={"name": "exp-bad", "experiment_group_id": group["id"], "dataset_name": "ds", "status": "bogus"}, + ) + assert resp.status_code == 422, resp.text + + +def test_experiment_status_and_root_cause_update(client: TestClient) -> None: + group = _group(client) + client.post( + EXPERIMENTS, + json={"name": "exp-3", "experiment_group_id": group["id"], "dataset_name": "ds", "status": "baseline"}, + ) + updated = client.put( + f"{EXPERIMENTS}/exp-3", + json={ + "name": "exp-3", + "experiment_group_id": group["id"], + "dataset_name": "ds", + "status": "winner", + "root_cause": "best cost/accuracy trade-off", + }, + ) + assert updated.status_code == 200, updated.text + assert updated.json()["status"] == "winner" + assert updated.json()["root_cause"] == "best cost/accuracy trade-off" + + +def test_new_fields_are_optional(client: TestClient) -> None: + # Omitting every new field is valid; they default to null. + group = client.post(GROUPS, json={"name": "g-min"}) + assert group.status_code == 201, group.text + gbody = group.json() + assert gbody["insight_id"] is None and gbody["summary"] is None and gbody["metadata"] is None + + exp = client.post(EXPERIMENTS, json={"name": "exp-min", "experiment_group_id": gbody["id"], "dataset_name": "ds"}) + assert exp.status_code == 201, exp.text + ebody = exp.json() + assert ebody["parent_experiment_id"] is None and ebody["status"] is None and ebody["root_cause"] is None From 3a3ecc1b2def175ceda4688407b69e56f6902942 Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Wed, 1 Jul 2026 16:49:23 -0600 Subject: [PATCH 2/3] pr comments Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- openapi/ga/individual/platform.openapi.yaml | 26 ++----- openapi/ga/openapi.yaml | 26 ++----- openapi/openapi.yaml | 26 ++----- .../intake/api/v2/experiments/endpoints.py | 17 +++++ .../nmp/intake/api/v2/experiments/schemas.py | 8 +-- .../src/nmp/intake/entities/experiments.py | 20 +----- ...=> test_experiment_optimization_fields.py} | 68 ++++++++++++++----- 7 files changed, 95 insertions(+), 96 deletions(-) rename services/intake/tests/{test_experiment_evaluator_fields.py => test_experiment_optimization_fields.py} (52%) diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index 132628b54e..c3abd938fa 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -10389,8 +10389,8 @@ components: type: string insight_id: title: Insight Id - description: Entity id of the NeMo Evaluator insight that seeded this group, - if any. + description: Reference to an external insight that seeded this group, if + any. type: string summary: title: Summary @@ -10529,9 +10529,9 @@ components: a variant of a baseline), if any. type: string status: - allOf: - - $ref: '#/components/schemas/ExperimentStatus' - description: Lifecycle status in the NeMo Evaluator flow. + title: Status + description: Producer-defined lifecycle status of the experiment. + type: string root_cause: title: Root Cause description: Human- or agent-authored explanation of the experiment's outcome @@ -10583,7 +10583,8 @@ components: title: Parent Experiment Id type: string status: - $ref: '#/components/schemas/ExperimentStatus' + title: Status + type: string root_cause: title: Root Cause type: string @@ -10796,19 +10797,6 @@ components: required: - data title: ExperimentSessionResponsesPage - ExperimentStatus: - type: string - enum: - - baseline - - running - - evaluated - - survived - - killed - - winner - - deployed - - rejected - title: ExperimentStatus - description: Lifecycle status of an Experiment in the NeMo Evaluator flow. FactCheckingRailConfig: properties: parameters: diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index 132628b54e..c3abd938fa 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -10389,8 +10389,8 @@ components: type: string insight_id: title: Insight Id - description: Entity id of the NeMo Evaluator insight that seeded this group, - if any. + description: Reference to an external insight that seeded this group, if + any. type: string summary: title: Summary @@ -10529,9 +10529,9 @@ components: a variant of a baseline), if any. type: string status: - allOf: - - $ref: '#/components/schemas/ExperimentStatus' - description: Lifecycle status in the NeMo Evaluator flow. + title: Status + description: Producer-defined lifecycle status of the experiment. + type: string root_cause: title: Root Cause description: Human- or agent-authored explanation of the experiment's outcome @@ -10583,7 +10583,8 @@ components: title: Parent Experiment Id type: string status: - $ref: '#/components/schemas/ExperimentStatus' + title: Status + type: string root_cause: title: Root Cause type: string @@ -10796,19 +10797,6 @@ components: required: - data title: ExperimentSessionResponsesPage - ExperimentStatus: - type: string - enum: - - baseline - - running - - evaluated - - survived - - killed - - winner - - deployed - - rejected - title: ExperimentStatus - description: Lifecycle status of an Experiment in the NeMo Evaluator flow. FactCheckingRailConfig: properties: parameters: diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 132628b54e..c3abd938fa 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -10389,8 +10389,8 @@ components: type: string insight_id: title: Insight Id - description: Entity id of the NeMo Evaluator insight that seeded this group, - if any. + description: Reference to an external insight that seeded this group, if + any. type: string summary: title: Summary @@ -10529,9 +10529,9 @@ components: a variant of a baseline), if any. type: string status: - allOf: - - $ref: '#/components/schemas/ExperimentStatus' - description: Lifecycle status in the NeMo Evaluator flow. + title: Status + description: Producer-defined lifecycle status of the experiment. + type: string root_cause: title: Root Cause description: Human- or agent-authored explanation of the experiment's outcome @@ -10583,7 +10583,8 @@ components: title: Parent Experiment Id type: string status: - $ref: '#/components/schemas/ExperimentStatus' + title: Status + type: string root_cause: title: Root Cause type: string @@ -10796,19 +10797,6 @@ components: required: - data title: ExperimentSessionResponsesPage - ExperimentStatus: - type: string - enum: - - baseline - - running - - evaluated - - survived - - killed - - winner - - deployed - - rejected - title: ExperimentStatus - description: Lifecycle status of an Experiment in the NeMo Evaluator flow. FactCheckingRailConfig: properties: parameters: diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index dcf5daafc7..e7c1e0afa4 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -324,6 +324,7 @@ async def create_experiment( entity_client: EntityClientDep, ) -> ExperimentResponse: await _validate_group_exists(entity_client, group_id=body.experiment_group_id) + await _validate_parent_experiment_exists(entity_client, parent_experiment_id=body.parent_experiment_id) entity = Experiment( workspace=workspace, name=body.name, @@ -524,6 +525,7 @@ async def update_experiment( _reject_if_deleted(existing, workspace=workspace, name=name, label="Experiment") if body.experiment_group_id != existing.experiment_group_id: await _validate_group_exists(entity_client, group_id=body.experiment_group_id) + await _validate_parent_experiment_exists(entity_client, parent_experiment_id=body.parent_experiment_id) changed = [f for f in _IMMUTABLE_EXPERIMENT_FIELDS if getattr(body, f) != getattr(existing, f)] if changed: @@ -859,6 +861,21 @@ async def _count_live_experiments_by_group( return counts +async def _validate_parent_experiment_exists( + entity_client: EntityClient, *, parent_experiment_id: str | None +) -> None: + """Reject with 400 if ``parent_experiment_id`` is set but doesn't reference an existing experiment.""" + if parent_experiment_id is None: + return + try: + await entity_client.get_by_id(Experiment, entity_id=parent_experiment_id) + except EntityNotFoundError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"parent_experiment_id '{parent_experiment_id}' does not reference an existing experiment.", + ) from e + + async def _validate_group_exists(entity_client: EntityClient, *, group_id: str) -> None: """Reject the request with 400 if the referenced ExperimentGroup doesn't exist or is deleted.""" try: diff --git a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py index 1f6fd9825c..4e67ce4147 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py @@ -13,7 +13,7 @@ from typing import Annotated, Any from nmp.common.entities.values import DatetimeFilter, Filter, NumberFilter, map_entity_field -from nmp.intake.entities.experiments import Experiment, ExperimentGroup, ExperimentStatus, SortCriterion +from nmp.intake.entities.experiments import Experiment, ExperimentGroup, SortCriterion from nmp.intake.spans.domain import SpanStatus from nmp.intake.spans.experiment_session_repository import ExperimentSessionRow from pydantic import AnyUrl, BaseModel, ConfigDict, Field @@ -27,7 +27,7 @@ class ExperimentGroupRequest(BaseModel): name: str = Field(description="Workspace-unique group name.") description: str | None = Field(default=None, description="Human-readable purpose of the group.") insight_id: str | None = Field( - default=None, description="Entity id of the NeMo Evaluator insight that seeded this group, if any." + default=None, description="Reference to an external insight that seeded this group, if any." ) summary: str | None = Field(default=None, description="Human- or agent-authored summary of the group's findings.") metadata: dict[str, Any] | None = Field(default=None, description="Free-form producer metadata for the group.") @@ -59,7 +59,7 @@ class ExperimentRequest(BaseModel): default=None, description="Entity id of the experiment this one was derived from (e.g. a variant of a baseline), if any.", ) - status: ExperimentStatus | None = Field(default=None, description="Lifecycle status in the NeMo Evaluator flow.") + status: str | None = Field(default=None, description="Producer-defined lifecycle status of the experiment.") root_cause: str | None = Field( default=None, description="Human- or agent-authored explanation of the experiment's outcome (e.g. why it was killed).", @@ -127,7 +127,7 @@ class ExperimentResponse(BaseModel): metadata: dict[str, Any] = Field(default_factory=dict) description: str | None = None parent_experiment_id: str | None = None - status: ExperimentStatus | None = None + status: str | None = None root_cause: str | None = None created_at: datetime | None = None updated_at: datetime | None = None diff --git a/services/intake/src/nmp/intake/entities/experiments.py b/services/intake/src/nmp/intake/entities/experiments.py index e697ba09d2..e0aeff751a 100644 --- a/services/intake/src/nmp/intake/entities/experiments.py +++ b/services/intake/src/nmp/intake/entities/experiments.py @@ -11,7 +11,6 @@ from __future__ import annotations from datetime import datetime -from enum import Enum from typing import Any, ClassVar, Literal from nmp.common.entities.client import EntityBase @@ -27,19 +26,6 @@ class SortCriterion(BaseModel): direction: Literal["asc", "desc"] = Field(description="Sort direction for this field.") -class ExperimentStatus(str, Enum): - """Lifecycle status of an Experiment in the NeMo Evaluator flow.""" - - baseline = "baseline" - running = "running" - evaluated = "evaluated" - survived = "survived" - killed = "killed" - winner = "winner" - deployed = "deployed" - rejected = "rejected" - - class ExperimentGroup(EntityBase): """A named container of Experiments pursuing a single optimization goal. @@ -51,7 +37,7 @@ class ExperimentGroup(EntityBase): description: str | None = Field(default=None, description="Human-readable purpose of the group.") insight_id: str | None = Field( default=None, - description="Entity id of the NeMo Evaluator insight that seeded this group, if any.", + description="Reference to an external insight that seeded this group, if any.", ) summary: str | None = Field(default=None, description="Human- or agent-authored summary of the group's findings.") metadata: dict[str, Any] | None = Field(default=None, description="Free-form producer metadata for the group.") @@ -102,9 +88,9 @@ class Experiment(EntityBase): default=None, description="Entity id of the experiment this one was derived from (e.g. a variant of a baseline), if any.", ) - status: ExperimentStatus | None = Field( + status: str | None = Field( default=None, - description="Lifecycle status in the NeMo Evaluator flow.", + description="Producer-defined lifecycle status of the experiment.", ) root_cause: str | None = Field( default=None, diff --git a/services/intake/tests/test_experiment_evaluator_fields.py b/services/intake/tests/test_experiment_optimization_fields.py similarity index 52% rename from services/intake/tests/test_experiment_evaluator_fields.py rename to services/intake/tests/test_experiment_optimization_fields.py index 6e48b93667..99e4203045 100644 --- a/services/intake/tests/test_experiment_evaluator_fields.py +++ b/services/intake/tests/test_experiment_optimization_fields.py @@ -1,8 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""NeMo Evaluator fields on ExperimentGroup (insight_id, summary, metadata) and Experiment -(parent_experiment_id, status, root_cause). All are optional and round-trip through create/update.""" +"""Optional fields on ExperimentGroup (insight_id, summary, metadata) and Experiment +(parent_experiment_id, status, root_cause): round-trip through create/update, parent-reference +validation, and free-form status.""" from fastapi.testclient import TestClient @@ -16,7 +17,13 @@ def _group(client: TestClient, name: str = "grp") -> dict: return resp.json() -def test_group_evaluator_fields_round_trip(client: TestClient) -> None: +def _experiment(client: TestClient, group_id: str, name: str) -> dict: + resp = client.post(EXPERIMENTS, json={"name": name, "experiment_group_id": group_id, "dataset_name": "ds"}) + assert resp.status_code == 201, resp.text + return resp.json() + + +def test_group_fields_round_trip(client: TestClient) -> None: resp = client.post( GROUPS, json={"name": "g1", "insight_id": "insight-123", "summary": "looks promising", "metadata": {"k": "v"}}, @@ -28,41 +35,70 @@ def test_group_evaluator_fields_round_trip(client: TestClient) -> None: assert body["metadata"] == {"k": "v"} -def test_experiment_evaluator_fields_round_trip(client: TestClient) -> None: +def test_experiment_fields_round_trip(client: TestClient) -> None: group = _group(client) + parent = _experiment(client, group["id"], "exp-parent") resp = client.post( EXPERIMENTS, json={ "name": "exp-1", "experiment_group_id": group["id"], "dataset_name": "ds", - "parent_experiment_id": "exp-0", + "parent_experiment_id": parent["id"], "status": "running", "root_cause": "still evaluating", }, ) assert resp.status_code == 201, resp.text body = resp.json() - assert body["parent_experiment_id"] == "exp-0" + assert body["parent_experiment_id"] == parent["id"] assert body["status"] == "running" assert body["root_cause"] == "still evaluating" -def test_experiment_rejects_invalid_status(client: TestClient) -> None: +def test_experiment_rejects_unknown_parent(client: TestClient) -> None: group = _group(client) resp = client.post( EXPERIMENTS, - json={"name": "exp-bad", "experiment_group_id": group["id"], "dataset_name": "ds", "status": "bogus"}, + json={ + "name": "exp-orphan", + "experiment_group_id": group["id"], + "dataset_name": "ds", + "parent_experiment_id": "does-not-exist", + }, ) - assert resp.status_code == 422, resp.text + assert resp.status_code == 400, resp.text -def test_experiment_status_and_root_cause_update(client: TestClient) -> None: +def test_update_rejects_unknown_parent(client: TestClient) -> None: + group = _group(client) + _experiment(client, group["id"], "exp-u") + updated = client.put( + f"{EXPERIMENTS}/exp-u", + json={ + "name": "exp-u", + "experiment_group_id": group["id"], + "dataset_name": "ds", + "parent_experiment_id": "does-not-exist", + }, + ) + assert updated.status_code == 400, updated.text + + +def test_status_is_a_free_string(client: TestClient) -> None: + # status is producer-defined, not a fixed enum — any string is accepted. group = _group(client) - client.post( + resp = client.post( EXPERIMENTS, - json={"name": "exp-3", "experiment_group_id": group["id"], "dataset_name": "ds", "status": "baseline"}, + json={"name": "exp-custom", "experiment_group_id": group["id"], "dataset_name": "ds", "status": "my-own-state"}, ) + assert resp.status_code == 201, resp.text + assert resp.json()["status"] == "my-own-state" + + +def test_experiment_status_and_root_cause_update(client: TestClient) -> None: + group = _group(client) + _experiment(client, group["id"], "exp-3") updated = client.put( f"{EXPERIMENTS}/exp-3", json={ @@ -80,12 +116,8 @@ def test_experiment_status_and_root_cause_update(client: TestClient) -> None: def test_new_fields_are_optional(client: TestClient) -> None: # Omitting every new field is valid; they default to null. - group = client.post(GROUPS, json={"name": "g-min"}) - assert group.status_code == 201, group.text - gbody = group.json() + gbody = _group(client, name="g-min") assert gbody["insight_id"] is None and gbody["summary"] is None and gbody["metadata"] is None - exp = client.post(EXPERIMENTS, json={"name": "exp-min", "experiment_group_id": gbody["id"], "dataset_name": "ds"}) - assert exp.status_code == 201, exp.text - ebody = exp.json() + ebody = _experiment(client, gbody["id"], "exp-min") assert ebody["parent_experiment_id"] is None and ebody["status"] is None and ebody["root_cause"] is None From 27683ea016002eca5ee215b1066ed0481ae2818b Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:20:46 -0600 Subject: [PATCH 3/3] lint Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- .../nemo-platform/.nmpcontext/openapi.yaml | 26 +++++-------------- .../nemo-platform/.nmpcontext/stainless.yaml | 1 - .../experiment_groups/experiment_groups.py | 8 +++--- .../resources/experiments/api.md | 1 - .../resources/experiments/experiments.py | 18 ++++++------- .../experiment_group_create_params.py | 2 +- .../experiment_group_update_params.py | 2 +- .../types/experiments/__init__.py | 1 - .../experiments/experiment_create_params.py | 6 ++--- .../types/experiments/experiment_response.py | 4 +-- .../types/experiments/experiment_status.py | 24 ----------------- .../experiments/experiment_update_params.py | 5 ++-- .../tests/api_resources/test_experiments.py | 8 +++--- sdk/stainless.yaml | 1 - .../intake/api/v2/experiments/endpoints.py | 4 +-- 15 files changed, 31 insertions(+), 80 deletions(-) delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_status.py diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index 132628b54e..c3abd938fa 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -10389,8 +10389,8 @@ components: type: string insight_id: title: Insight Id - description: Entity id of the NeMo Evaluator insight that seeded this group, - if any. + description: Reference to an external insight that seeded this group, if + any. type: string summary: title: Summary @@ -10529,9 +10529,9 @@ components: a variant of a baseline), if any. type: string status: - allOf: - - $ref: '#/components/schemas/ExperimentStatus' - description: Lifecycle status in the NeMo Evaluator flow. + title: Status + description: Producer-defined lifecycle status of the experiment. + type: string root_cause: title: Root Cause description: Human- or agent-authored explanation of the experiment's outcome @@ -10583,7 +10583,8 @@ components: title: Parent Experiment Id type: string status: - $ref: '#/components/schemas/ExperimentStatus' + title: Status + type: string root_cause: title: Root Cause type: string @@ -10796,19 +10797,6 @@ components: required: - data title: ExperimentSessionResponsesPage - ExperimentStatus: - type: string - enum: - - baseline - - running - - evaluated - - survived - - killed - - winner - - deployed - - rejected - title: ExperimentStatus - description: Lifecycle status of an Experiment in the NeMo Evaluator flow. FactCheckingRailConfig: properties: parameters: diff --git a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml index 53824b5640..c30321bc99 100644 --- a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml @@ -914,7 +914,6 @@ resources: experiment_request: ExperimentRequest experiment_response: ExperimentResponse experiment_responses_page: ExperimentResponsesPage - experiment_status: ExperimentStatus metric_stat_filters: MetricStatFilters number_filter: NumberFilter methods: diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py index 71368d3f2f..64a28250d6 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py @@ -97,7 +97,7 @@ def create( description: Human-readable purpose of the group. - insight_id: Entity id of the NeMo Evaluator insight that seeded this group, if any. + insight_id: Reference to an external insight that seeded this group, if any. metadata: Free-form producer metadata for the group. @@ -213,7 +213,7 @@ def update( description: Human-readable purpose of the group. - insight_id: Entity id of the NeMo Evaluator insight that seeded this group, if any. + insight_id: Reference to an external insight that seeded this group, if any. metadata: Free-form producer metadata for the group. @@ -408,7 +408,7 @@ async def create( description: Human-readable purpose of the group. - insight_id: Entity id of the NeMo Evaluator insight that seeded this group, if any. + insight_id: Reference to an external insight that seeded this group, if any. metadata: Free-form producer metadata for the group. @@ -524,7 +524,7 @@ async def update( description: Human-readable purpose of the group. - insight_id: Entity id of the NeMo Evaluator insight that seeded this group, if any. + insight_id: Reference to an external insight that seeded this group, if any. metadata: Free-form producer metadata for the group. diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md index 2f70554173..0c6e16973f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md @@ -9,7 +9,6 @@ from nemo_platform.types.experiments import ( ExperimentRequest, ExperimentResponse, ExperimentResponsesPage, - ExperimentStatus, MetricStatFilters, NumberFilter, ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py index e7ab681acf..be460ff43f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py @@ -42,12 +42,10 @@ from ...pagination import SyncDefaultPagination, AsyncDefaultPagination from ..._base_client import AsyncPaginator, make_request_options from ...types.experiments import ( - ExperimentStatus, experiment_list_params, experiment_create_params, experiment_update_params, ) -from ...types.experiments.experiment_status import ExperimentStatus from ...types.experiments.experiment_response import ExperimentResponse from ...types.experiments.experiment_filter_param import ExperimentFilterParam from ..._exceptions import ConflictError @@ -92,7 +90,7 @@ def create( parent_experiment_id: str | Omit = omit, root_cause: str | Omit = omit, source_link: str | Omit = omit, - status: ExperimentStatus | Omit = omit, + status: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. exist_ok: bool = False, @@ -126,7 +124,7 @@ def create( source_link: Optional URL for the source experiment. - status: Lifecycle status of an Experiment in the NeMo Evaluator flow. + status: Producer-defined lifecycle status of the experiment. exist_ok: Do not raise an error if the resource already exists. Returns the existing resource. @@ -224,7 +222,7 @@ def update( parent_experiment_id: str | Omit = omit, root_cause: str | Omit = omit, source_link: str | Omit = omit, - status: ExperimentStatus | Omit = omit, + status: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -257,7 +255,7 @@ def update( source_link: Optional URL for the source experiment. - status: Lifecycle status of an Experiment in the NeMo Evaluator flow. + status: Producer-defined lifecycle status of the experiment. extra_headers: Send extra headers @@ -533,7 +531,7 @@ async def create( parent_experiment_id: str | Omit = omit, root_cause: str | Omit = omit, source_link: str | Omit = omit, - status: ExperimentStatus | Omit = omit, + status: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. exist_ok: bool = False, @@ -567,7 +565,7 @@ async def create( source_link: Optional URL for the source experiment. - status: Lifecycle status of an Experiment in the NeMo Evaluator flow. + status: Producer-defined lifecycle status of the experiment. exist_ok: Do not raise an error if the resource already exists. Returns the existing resource. @@ -665,7 +663,7 @@ async def update( parent_experiment_id: str | Omit = omit, root_cause: str | Omit = omit, source_link: str | Omit = omit, - status: ExperimentStatus | Omit = omit, + status: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -698,7 +696,7 @@ async def update( source_link: Optional URL for the source experiment. - status: Lifecycle status of an Experiment in the NeMo Evaluator flow. + status: Producer-defined lifecycle status of the experiment. extra_headers: Send extra headers diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py index 6509223b43..61ff10c7ce 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_create_params.py @@ -42,7 +42,7 @@ class ExperimentGroupCreateParams(TypedDict, total=False): """Human-readable purpose of the group.""" insight_id: str - """Entity id of the NeMo Evaluator insight that seeded this group, if any.""" + """Reference to an external insight that seeded this group, if any.""" metadata: Dict[str, object] """Free-form producer metadata for the group.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py index af485b2476..a3018f2963 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_update_params.py @@ -43,7 +43,7 @@ class ExperimentGroupUpdateParams(TypedDict, total=False): """Human-readable purpose of the group.""" insight_id: str - """Entity id of the NeMo Evaluator insight that seeded this group, if any.""" + """Reference to an external insight that seeded this group, if any.""" metadata: Dict[str, object] """Free-form producer metadata for the group.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/__init__.py index 7ced9e1cbc..b046d1fc20 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/__init__.py @@ -17,7 +17,6 @@ from __future__ import annotations -from .experiment_status import ExperimentStatus as ExperimentStatus from .evaluator_aggregate import EvaluatorAggregate as EvaluatorAggregate from .experiment_response import ExperimentResponse as ExperimentResponse from .number_filter_param import NumberFilterParam as NumberFilterParam diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py index efce920d6a..d556b3f8c9 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py @@ -20,8 +20,6 @@ from typing import Dict from typing_extensions import Required, TypedDict -from .experiment_status import ExperimentStatus - __all__ = ["ExperimentCreateParams"] @@ -64,5 +62,5 @@ class ExperimentCreateParams(TypedDict, total=False): source_link: str """Optional URL for the source experiment.""" - status: ExperimentStatus - """Lifecycle status of an Experiment in the NeMo Evaluator flow.""" + status: str + """Producer-defined lifecycle status of the experiment.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py index 617dfdf935..2586a93b69 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py @@ -20,7 +20,6 @@ from ..._compat import PYDANTIC_V1, ConfigDict from ..._models import BaseModel -from .experiment_status import ExperimentStatus from .evaluator_aggregate import EvaluatorAggregate __all__ = ["ExperimentResponse"] @@ -85,8 +84,7 @@ class ExperimentResponse(BaseModel): source_link: Optional[str] = None - status: Optional[ExperimentStatus] = None - """Lifecycle status of an Experiment in the NeMo Evaluator flow.""" + status: Optional[str] = None updated_at: Optional[datetime] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_status.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_status.py deleted file mode 100644 index 1b4f05f523..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_status.py +++ /dev/null @@ -1,24 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -from typing_extensions import Literal, TypeAlias - -__all__ = ["ExperimentStatus"] - -ExperimentStatus: TypeAlias = Literal[ - "baseline", "running", "evaluated", "survived", "killed", "winner", "deployed", "rejected" -] diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py index 0cb9572480..903ebf3263 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py @@ -21,7 +21,6 @@ from typing_extensions import Required, Annotated, TypedDict from ..._utils import PropertyInfo -from .experiment_status import ExperimentStatus __all__ = ["ExperimentUpdateParams"] @@ -65,5 +64,5 @@ class ExperimentUpdateParams(TypedDict, total=False): source_link: str """Optional URL for the source experiment.""" - status: ExperimentStatus - """Lifecycle status of an Experiment in the NeMo Evaluator flow.""" + status: str + """Producer-defined lifecycle status of the experiment.""" diff --git a/sdk/python/nemo-platform/tests/api_resources/test_experiments.py b/sdk/python/nemo-platform/tests/api_resources/test_experiments.py index 21a6e7245b..ca920b04d8 100644 --- a/sdk/python/nemo-platform/tests/api_resources/test_experiments.py +++ b/sdk/python/nemo-platform/tests/api_resources/test_experiments.py @@ -61,7 +61,7 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: parent_experiment_id="parent_experiment_id", root_cause="root_cause", source_link="https://example.com", - status="baseline", + status="status", ) assert_matches_type(ExperimentResponse, experiment, path=["response"]) @@ -187,7 +187,7 @@ def test_method_update_with_all_params(self, client: NeMoPlatform) -> None: parent_experiment_id="parent_experiment_id", root_cause="root_cause", source_link="https://example.com", - status="baseline", + status="status", ) assert_matches_type(ExperimentResponse, experiment, path=["response"]) @@ -665,7 +665,7 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo parent_experiment_id="parent_experiment_id", root_cause="root_cause", source_link="https://example.com", - status="baseline", + status="status", ) assert_matches_type(ExperimentResponse, experiment, path=["response"]) @@ -791,7 +791,7 @@ async def test_method_update_with_all_params(self, async_client: AsyncNeMoPlatfo parent_experiment_id="parent_experiment_id", root_cause="root_cause", source_link="https://example.com", - status="baseline", + status="status", ) assert_matches_type(ExperimentResponse, experiment, path=["response"]) diff --git a/sdk/stainless.yaml b/sdk/stainless.yaml index 53824b5640..c30321bc99 100644 --- a/sdk/stainless.yaml +++ b/sdk/stainless.yaml @@ -914,7 +914,6 @@ resources: experiment_request: ExperimentRequest experiment_response: ExperimentResponse experiment_responses_page: ExperimentResponsesPage - experiment_status: ExperimentStatus metric_stat_filters: MetricStatFilters number_filter: NumberFilter methods: diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index e7c1e0afa4..9705236129 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -861,9 +861,7 @@ async def _count_live_experiments_by_group( return counts -async def _validate_parent_experiment_exists( - entity_client: EntityClient, *, parent_experiment_id: str | None -) -> None: +async def _validate_parent_experiment_exists(entity_client: EntityClient, *, parent_experiment_id: str | None) -> None: """Reject with 400 if ``parent_experiment_id`` is set but doesn't reference an existing experiment.""" if parent_experiment_id is None: return