diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index dba0e62e51b..386598e5cfe 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -3731,9 +3731,9 @@ paths: explode: true schema: $ref: '#/components/schemas/ExperimentFilter' - description: Filter experiments by name, experiment_group_id, agent_name, - agent_version, dataset_name, dataset_version, created_by, created_at, or - updated_at. + description: Filter experiments by name, experiment_group_id, dataset_name, + dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true + to return only soft-deleted experiments; omit to see only live ones. responses: '200': description: Successful Response @@ -9653,15 +9653,6 @@ components: evaluation_run_id: title: Evaluation Run Id type: string - dataset_id: - title: Dataset Id - type: string - dataset_name: - title: Dataset Name - type: string - dataset_version: - title: Dataset Version - type: string test_case_id: title: Test Case Id type: string @@ -9934,14 +9925,6 @@ components: description: Filter experiments by owning group id. title: Experiment Group Id type: string - agent_name: - description: Filter experiments by agent name. - title: Agent Name - type: string - agent_version: - description: Filter experiments by agent version. - title: Agent Version - type: string dataset_name: description: Filter experiments by dataset name. title: Dataset Name @@ -10066,14 +10049,6 @@ components: title: Experiment Group Id description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ \ group must already exist." - agent_name: - type: string - title: Agent Name - description: Name of the agent under test. - agent_version: - type: string - title: Agent Version - description: Version of the agent under test. dataset_name: type: string title: Dataset Name @@ -10106,8 +10081,6 @@ components: required: - name - experiment_group_id - - agent_name - - agent_version - dataset_name title: ExperimentRequest description: Request body for creating an Experiment. @@ -10127,12 +10100,6 @@ components: title: Experiment Group Id description: Entity id of the owning ExperimentGroup. Required for every Experiment. - agent_name: - type: string - title: Agent Name - agent_version: - type: string - title: Agent Version dataset_name: type: string title: Dataset Name @@ -10175,6 +10142,22 @@ components: title: Model Names description: Distinct model names observed across ingested sessions for this experiment. + agent_names: + items: + type: string + type: array + uniqueItems: true + title: Agent Names + description: Distinct agent names observed across ingested sessions for + this experiment. + agent_versions: + items: + type: string + type: array + uniqueItems: true + title: Agent Versions + description: Distinct agent versions observed across ingested sessions for + this experiment. aggregate_scores: title: Aggregate Scores additionalProperties: @@ -10196,8 +10179,6 @@ components: - name - workspace - experiment_group_id - - agent_name - - agent_version - dataset_name title: ExperimentResponse description: Experiment as served by the API, including ClickHouse-hydrated @@ -16131,12 +16112,6 @@ components: prompt_id: title: Prompt Id type: string - prompt_name: - title: Prompt Name - type: string - prompt_version: - title: Prompt Version - type: string agent_id: title: Agent Id type: string @@ -16216,15 +16191,6 @@ components: evaluation_run_id: title: Evaluation Run Id type: string - dataset_id: - title: Dataset Id - type: string - dataset_name: - title: Dataset Name - type: string - dataset_version: - title: Dataset Version - type: string test_case_id: title: Test Case Id type: string diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index dba0e62e51b..386598e5cfe 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -3731,9 +3731,9 @@ paths: explode: true schema: $ref: '#/components/schemas/ExperimentFilter' - description: Filter experiments by name, experiment_group_id, agent_name, - agent_version, dataset_name, dataset_version, created_by, created_at, or - updated_at. + description: Filter experiments by name, experiment_group_id, dataset_name, + dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true + to return only soft-deleted experiments; omit to see only live ones. responses: '200': description: Successful Response @@ -9653,15 +9653,6 @@ components: evaluation_run_id: title: Evaluation Run Id type: string - dataset_id: - title: Dataset Id - type: string - dataset_name: - title: Dataset Name - type: string - dataset_version: - title: Dataset Version - type: string test_case_id: title: Test Case Id type: string @@ -9934,14 +9925,6 @@ components: description: Filter experiments by owning group id. title: Experiment Group Id type: string - agent_name: - description: Filter experiments by agent name. - title: Agent Name - type: string - agent_version: - description: Filter experiments by agent version. - title: Agent Version - type: string dataset_name: description: Filter experiments by dataset name. title: Dataset Name @@ -10066,14 +10049,6 @@ components: title: Experiment Group Id description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ \ group must already exist." - agent_name: - type: string - title: Agent Name - description: Name of the agent under test. - agent_version: - type: string - title: Agent Version - description: Version of the agent under test. dataset_name: type: string title: Dataset Name @@ -10106,8 +10081,6 @@ components: required: - name - experiment_group_id - - agent_name - - agent_version - dataset_name title: ExperimentRequest description: Request body for creating an Experiment. @@ -10127,12 +10100,6 @@ components: title: Experiment Group Id description: Entity id of the owning ExperimentGroup. Required for every Experiment. - agent_name: - type: string - title: Agent Name - agent_version: - type: string - title: Agent Version dataset_name: type: string title: Dataset Name @@ -10175,6 +10142,22 @@ components: title: Model Names description: Distinct model names observed across ingested sessions for this experiment. + agent_names: + items: + type: string + type: array + uniqueItems: true + title: Agent Names + description: Distinct agent names observed across ingested sessions for + this experiment. + agent_versions: + items: + type: string + type: array + uniqueItems: true + title: Agent Versions + description: Distinct agent versions observed across ingested sessions for + this experiment. aggregate_scores: title: Aggregate Scores additionalProperties: @@ -10196,8 +10179,6 @@ components: - name - workspace - experiment_group_id - - agent_name - - agent_version - dataset_name title: ExperimentResponse description: Experiment as served by the API, including ClickHouse-hydrated @@ -16131,12 +16112,6 @@ components: prompt_id: title: Prompt Id type: string - prompt_name: - title: Prompt Name - type: string - prompt_version: - title: Prompt Version - type: string agent_id: title: Agent Id type: string @@ -16216,15 +16191,6 @@ components: evaluation_run_id: title: Evaluation Run Id type: string - dataset_id: - title: Dataset Id - type: string - dataset_name: - title: Dataset Name - type: string - dataset_version: - title: Dataset Version - type: string test_case_id: title: Test Case Id type: string diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index dba0e62e51b..386598e5cfe 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -3731,9 +3731,9 @@ paths: explode: true schema: $ref: '#/components/schemas/ExperimentFilter' - description: Filter experiments by name, experiment_group_id, agent_name, - agent_version, dataset_name, dataset_version, created_by, created_at, or - updated_at. + description: Filter experiments by name, experiment_group_id, dataset_name, + dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true + to return only soft-deleted experiments; omit to see only live ones. responses: '200': description: Successful Response @@ -9653,15 +9653,6 @@ components: evaluation_run_id: title: Evaluation Run Id type: string - dataset_id: - title: Dataset Id - type: string - dataset_name: - title: Dataset Name - type: string - dataset_version: - title: Dataset Version - type: string test_case_id: title: Test Case Id type: string @@ -9934,14 +9925,6 @@ components: description: Filter experiments by owning group id. title: Experiment Group Id type: string - agent_name: - description: Filter experiments by agent name. - title: Agent Name - type: string - agent_version: - description: Filter experiments by agent version. - title: Agent Version - type: string dataset_name: description: Filter experiments by dataset name. title: Dataset Name @@ -10066,14 +10049,6 @@ components: title: Experiment Group Id description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ \ group must already exist." - agent_name: - type: string - title: Agent Name - description: Name of the agent under test. - agent_version: - type: string - title: Agent Version - description: Version of the agent under test. dataset_name: type: string title: Dataset Name @@ -10106,8 +10081,6 @@ components: required: - name - experiment_group_id - - agent_name - - agent_version - dataset_name title: ExperimentRequest description: Request body for creating an Experiment. @@ -10127,12 +10100,6 @@ components: title: Experiment Group Id description: Entity id of the owning ExperimentGroup. Required for every Experiment. - agent_name: - type: string - title: Agent Name - agent_version: - type: string - title: Agent Version dataset_name: type: string title: Dataset Name @@ -10175,6 +10142,22 @@ components: title: Model Names description: Distinct model names observed across ingested sessions for this experiment. + agent_names: + items: + type: string + type: array + uniqueItems: true + title: Agent Names + description: Distinct agent names observed across ingested sessions for + this experiment. + agent_versions: + items: + type: string + type: array + uniqueItems: true + title: Agent Versions + description: Distinct agent versions observed across ingested sessions for + this experiment. aggregate_scores: title: Aggregate Scores additionalProperties: @@ -10196,8 +10179,6 @@ components: - name - workspace - experiment_group_id - - agent_name - - agent_version - dataset_name title: ExperimentResponse description: Experiment as served by the API, including ClickHouse-hydrated @@ -16131,12 +16112,6 @@ components: prompt_id: title: Prompt Id type: string - prompt_name: - title: Prompt Name - type: string - prompt_version: - title: Prompt Version - type: string agent_id: title: Agent Id type: string @@ -16216,15 +16191,6 @@ components: evaluation_run_id: title: Evaluation Run Id type: string - dataset_id: - title: Dataset Id - type: string - dataset_name: - title: Dataset Name - type: string - dataset_version: - title: Dataset Version - type: string test_case_id: title: Test Case Id type: string diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index dba0e62e51b..386598e5cfe 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -3731,9 +3731,9 @@ paths: explode: true schema: $ref: '#/components/schemas/ExperimentFilter' - description: Filter experiments by name, experiment_group_id, agent_name, - agent_version, dataset_name, dataset_version, created_by, created_at, or - updated_at. + description: Filter experiments by name, experiment_group_id, dataset_name, + dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true + to return only soft-deleted experiments; omit to see only live ones. responses: '200': description: Successful Response @@ -9653,15 +9653,6 @@ components: evaluation_run_id: title: Evaluation Run Id type: string - dataset_id: - title: Dataset Id - type: string - dataset_name: - title: Dataset Name - type: string - dataset_version: - title: Dataset Version - type: string test_case_id: title: Test Case Id type: string @@ -9934,14 +9925,6 @@ components: description: Filter experiments by owning group id. title: Experiment Group Id type: string - agent_name: - description: Filter experiments by agent name. - title: Agent Name - type: string - agent_version: - description: Filter experiments by agent version. - title: Agent Version - type: string dataset_name: description: Filter experiments by dataset name. title: Dataset Name @@ -10066,14 +10049,6 @@ components: title: Experiment Group Id description: "Entity id of the owning ExperimentGroup. Required \u2014 the\ \ group must already exist." - agent_name: - type: string - title: Agent Name - description: Name of the agent under test. - agent_version: - type: string - title: Agent Version - description: Version of the agent under test. dataset_name: type: string title: Dataset Name @@ -10106,8 +10081,6 @@ components: required: - name - experiment_group_id - - agent_name - - agent_version - dataset_name title: ExperimentRequest description: Request body for creating an Experiment. @@ -10127,12 +10100,6 @@ components: title: Experiment Group Id description: Entity id of the owning ExperimentGroup. Required for every Experiment. - agent_name: - type: string - title: Agent Name - agent_version: - type: string - title: Agent Version dataset_name: type: string title: Dataset Name @@ -10175,6 +10142,22 @@ components: title: Model Names description: Distinct model names observed across ingested sessions for this experiment. + agent_names: + items: + type: string + type: array + uniqueItems: true + title: Agent Names + description: Distinct agent names observed across ingested sessions for + this experiment. + agent_versions: + items: + type: string + type: array + uniqueItems: true + title: Agent Versions + description: Distinct agent versions observed across ingested sessions for + this experiment. aggregate_scores: title: Aggregate Scores additionalProperties: @@ -10196,8 +10179,6 @@ components: - name - workspace - experiment_group_id - - agent_name - - agent_version - dataset_name title: ExperimentResponse description: Experiment as served by the API, including ClickHouse-hydrated @@ -16131,12 +16112,6 @@ components: prompt_id: title: Prompt Id type: string - prompt_name: - title: Prompt Name - type: string - prompt_version: - title: Prompt Version - type: string agent_id: title: Agent Id type: string @@ -16216,15 +16191,6 @@ components: evaluation_run_id: title: Evaluation Run Id type: string - dataset_id: - title: Dataset Id - type: string - dataset_name: - title: Dataset Name - type: string - dataset_version: - title: Dataset Version - type: string test_case_id: title: Test Case Id type: string diff --git a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml index 7d5a9da0e6b..473c2c6a45f 100644 --- a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml @@ -254,8 +254,6 @@ resources: filesets: models: fileset_filter: FilesetFilter - fileset_metadata: FilesetMetadataOutput - fileset_metadata_param: FilesetMetadataInput methods: create: post /apis/files/v2/workspaces/{workspace}/filesets list: get /apis/files/v2/workspaces/{workspace}/filesets @@ -710,12 +708,12 @@ resources: auth_context: AuthContext tool_call_config: ToolCallConfig model_metadata_content: ModelMetadataContent - # DatasetMetadataContent is referenced by the shared FilesetMetadata schema. - # Keep it in $shared so the generated import is intra-package - # (shared/fileset_metadata -> shared/dataset_metadata_content) and avoids the - # circular import that occurs when it lives under the `files` resource: - # shared/fileset_metadata -> files/dataset_metadata_content -> files/__init__ - # -> files/fileset -> shared/fileset_metadata + # FilesetMetadata is referenced by multiple resources (files, models) at the source + # level. Pin both Output and Input variants in $shared so sync-models does not re-home + # them under `files`, which would break ``from nemo_platform.types.shared import + # FilesetMetadata`` imports downstream (models service tests). + fileset_metadata: FilesetMetadataOutput + fileset_metadata_param: FilesetMetadataInput dataset_metadata_content: DatasetMetadataContent tool_calling_metadata_content: ToolCallingMetadataContent backend_format: BackendFormat diff --git a/sdk/python/nemo-platform/api.md b/sdk/python/nemo-platform/api.md index 271d51d2dfc..2e24f194436 100644 --- a/sdk/python/nemo-platform/api.md +++ b/sdk/python/nemo-platform/api.md @@ -11,6 +11,7 @@ from nemo_platform.types import ( DeleteResponse, FileStorageType, FilesetMetadata, + FilesetMetadataParam, FinetuningType, GenericSortField, HTTPValidationError, diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py index 3a1dde702f7..7d6a674f4bd 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py @@ -82,8 +82,6 @@ def create( self, *, workspace: str | None = None, - agent_name: str, - agent_version: str, dataset_name: str, experiment_group_id: str, name: str, @@ -104,10 +102,6 @@ def create( Create Experiment Args: - agent_name: Name of the agent under test. - - agent_version: Version of the agent under test. - dataset_name: Producer-supplied dataset name. experiment_group_id: Entity id of the owning ExperimentGroup. Required — the group must already @@ -146,8 +140,6 @@ def create( path_template("/apis/intake/v2/workspaces/{workspace}/experiments", workspace=workspace), body=maybe_transform( { - "agent_name": agent_name, - "agent_version": agent_version, "dataset_name": dataset_name, "experiment_group_id": experiment_group_id, "name": name, @@ -212,8 +204,6 @@ def update( path_name: str, *, workspace: str | None = None, - agent_name: str, - agent_version: str, dataset_name: str, experiment_group_id: str, body_name: str, @@ -233,10 +223,6 @@ def update( Update Experiment Args: - agent_name: Name of the agent under test. - - agent_version: Version of the agent under test. - dataset_name: Producer-supplied dataset name. experiment_group_id: Entity id of the owning ExperimentGroup. Required — the group must already @@ -276,8 +262,6 @@ def update( ), body=maybe_transform( { - "agent_name": agent_name, - "agent_version": agent_version, "dataset_name": dataset_name, "experiment_group_id": experiment_group_id, "body_name": body_name, @@ -314,8 +298,9 @@ def list( List Experiments Args: - filter: Filter experiments by name, experiment_group_id, agent_name, agent_version, - dataset_name, dataset_version, created_by, created_at, or updated_at. + filter: Filter experiments by name, experiment_group_id, dataset_name, dataset_version, + created_by, created_at, or updated_at. Pass is_deleted=true to return only + soft-deleted experiments; omit to see only live ones. page: Page number. @@ -424,8 +409,6 @@ async def create( self, *, workspace: str | None = None, - agent_name: str, - agent_version: str, dataset_name: str, experiment_group_id: str, name: str, @@ -446,10 +429,6 @@ async def create( Create Experiment Args: - agent_name: Name of the agent under test. - - agent_version: Version of the agent under test. - dataset_name: Producer-supplied dataset name. experiment_group_id: Entity id of the owning ExperimentGroup. Required — the group must already @@ -488,8 +467,6 @@ async def create( path_template("/apis/intake/v2/workspaces/{workspace}/experiments", workspace=workspace), body=await async_maybe_transform( { - "agent_name": agent_name, - "agent_version": agent_version, "dataset_name": dataset_name, "experiment_group_id": experiment_group_id, "name": name, @@ -554,8 +531,6 @@ async def update( path_name: str, *, workspace: str | None = None, - agent_name: str, - agent_version: str, dataset_name: str, experiment_group_id: str, body_name: str, @@ -575,10 +550,6 @@ async def update( Update Experiment Args: - agent_name: Name of the agent under test. - - agent_version: Version of the agent under test. - dataset_name: Producer-supplied dataset name. experiment_group_id: Entity id of the owning ExperimentGroup. Required — the group must already @@ -618,8 +589,6 @@ async def update( ), body=await async_maybe_transform( { - "agent_name": agent_name, - "agent_version": agent_version, "dataset_name": dataset_name, "experiment_group_id": experiment_group_id, "body_name": body_name, @@ -656,8 +625,9 @@ def list( List Experiments Args: - filter: Filter experiments by name, experiment_group_id, agent_name, agent_version, - dataset_name, dataset_version, created_by, created_at, or updated_at. + filter: Filter experiments by name, experiment_group_id, dataset_name, dataset_version, + created_by, created_at, or updated_at. Pass is_deleted=true to return only + soft-deleted experiments; omit to see only live ones. page: Page number. diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/files/filesets.py b/sdk/python/nemo-platform/src/nemo_platform/resources/files/filesets.py index f8fb167cf29..9b7afcb95fd 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/files/filesets.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/files/filesets.py @@ -43,7 +43,7 @@ from ...types.files.fileset_purpose import FilesetPurpose from ...types.shared.generic_sort_field import GenericSortField from ...types.files.fileset_filter_param import FilesetFilterParam -from ...types.shared_params.fileset_metadata import FilesetMetadata +from ...types.shared_params.fileset_metadata_param import FilesetMetadataParam from ..._exceptions import ConflictError __all__ = ["FilesetsResource", "AsyncFilesetsResource"] @@ -77,7 +77,7 @@ def create( cache: bool | Omit = omit, custom_fields: Dict[str, object] | Omit = omit, description: str | Omit = omit, - metadata: FilesetMetadata | Omit = omit, + metadata: FilesetMetadataParam | Omit = omit, project: str | Omit = omit, purpose: FilesetPurpose | Omit = omit, storage: fileset_create_params.Storage | Omit = omit, @@ -206,7 +206,7 @@ def update( workspace: str | None = None, custom_fields: Dict[str, object] | Omit = omit, description: str | Omit = omit, - metadata: FilesetMetadata | Omit = omit, + metadata: FilesetMetadataParam | Omit = omit, project: str | Omit = omit, purpose: FilesetPurpose | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. @@ -402,7 +402,7 @@ async def create( cache: bool | Omit = omit, custom_fields: Dict[str, object] | Omit = omit, description: str | Omit = omit, - metadata: FilesetMetadata | Omit = omit, + metadata: FilesetMetadataParam | Omit = omit, project: str | Omit = omit, purpose: FilesetPurpose | Omit = omit, storage: fileset_create_params.Storage | Omit = omit, @@ -531,7 +531,7 @@ async def update( workspace: str | None = None, custom_fields: Dict[str, object] | Omit = omit, description: str | Omit = omit, - metadata: FilesetMetadata | Omit = omit, + metadata: FilesetMetadataParam | Omit = omit, project: str | Omit = omit, purpose: FilesetPurpose | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/guardrail/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/guardrail/api.md index 52c2cf31fdb..4fe014901d1 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/guardrail/api.md +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/guardrail/api.md @@ -60,14 +60,19 @@ from nemo_platform.types.guardrail import ( PangeaRailOptions, PatronusEvaluateAPIParams, PatronusEvaluateConfig, + PatronusEvaluateConfigParam, PatronusEvaluationSuccessStrategy, PatronusRailConfig, + PatronusRailConfigParam, PrivateAIDetection, PrivateAIDetectionOptions, RailStatus, Rails, RailsConfig, RailsConfigData, + RailsConfigDataParam, + RailsConfigParam, + RailsParam, ReasoningConfig, RegexDetection, RegexDetectionOptions, diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/jobs/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/jobs/api.md index 99f688895d1..f5bfd41e0c7 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/jobs/api.md +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/jobs/api.md @@ -8,8 +8,10 @@ from nemo_platform.types.jobs import ( ComputeResources, ContainerSpec, CPUExecutionProvider, + CPUExecutionProviderParam, CreatePlatformJobRequest, DistributedGPUExecutionProvider, + DistributedGPUExecutionProviderParam, DockerJobExecutionProfile, DockerJobExecutionProfileConfig, DockerJobNetworkConfig, @@ -17,6 +19,7 @@ from nemo_platform.types.jobs import ( DockerVolumeMount, E2EJobExecutionProfile, GPUExecutionProvider, + GPUExecutionProviderParam, ImagePullSecret, JobExecutionProfileConfig, KubernetesEmptyDirVolume, @@ -33,7 +36,9 @@ from nemo_platform.types.jobs import ( PlatformJobSecretEnvironmentVariableRef, PlatformJobSortField, PlatformJobSpec, + PlatformJobSpecParam, PlatformJobStepSpec, + PlatformJobStepSpecParam, PlatformJobsListFilter, StepLifecycle, SubprocessExecutionProvider, diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/jobs/jobs.py b/sdk/python/nemo-platform/src/nemo_platform/resources/jobs/jobs.py index 3a7aff24081..9c5652b2164 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/jobs/jobs.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/jobs/jobs.py @@ -58,6 +58,7 @@ from ...pagination import SyncLogsPagination, AsyncLogsPagination, SyncDefaultPagination, AsyncDefaultPagination from ...types.jobs import ( PlatformJobSortField, + PlatformJobSpecParam, job_list_params, job_create_params, job_get_logs_params, diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/__init__.py index 2d670dadaf1..eb3af5c4f78 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/__init__.py @@ -42,6 +42,7 @@ PlatformJobLogPage as PlatformJobLogPage, HTTPValidationError as HTTPValidationError, SlidingWindowConfig as SlidingWindowConfig, + FilesetMetadataParam as FilesetMetadataParam, ModelMetadataContent as ModelMetadataContent, AuthDiscoveryResponse as AuthDiscoveryResponse, OidcDiscoveryResponse as OidcDiscoveryResponse, diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py index ccbff814a15..d62b5fffaab 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_create_params.py @@ -26,12 +26,6 @@ class ExperimentCreateParams(TypedDict, total=False): workspace: str - agent_name: Required[str] - """Name of the agent under test.""" - - agent_version: Required[str] - """Version of the agent under test.""" - dataset_name: Required[str] """Producer-supplied dataset name.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_filter_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_filter_param.py index f387b9f678e..27477bf2c14 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_filter_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_filter_param.py @@ -27,12 +27,6 @@ class ExperimentFilterParam(TypedDict, total=False): """Filter for listing Experiments.""" - agent_name: str - """Filter experiments by agent name.""" - - agent_version: str - """Filter experiments by agent version.""" - created_at: DatetimeFilter """ Filter experiments by creation timestamp; supports `$gte` and `$lte` for ranges. diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py index 39395fe24fd..8c2adab1ce3 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py @@ -29,8 +29,9 @@ class ExperimentListParams(TypedDict, total=False): filter: ExperimentFilterParam """ - Filter experiments by name, experiment_group_id, agent_name, agent_version, - dataset_name, dataset_version, created_by, created_at, or updated_at. + Filter experiments by name, experiment_group_id, dataset_name, dataset_version, + created_by, created_at, or updated_at. Pass is_deleted=true to return only + soft-deleted experiments; omit to see only live ones. """ page: int diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py index 5d5607977c3..9967b185445 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py @@ -30,10 +30,6 @@ class ExperimentResponse(BaseModel): id: str - agent_name: str - - agent_version: str - dataset_name: str experiment_group_id: str @@ -43,6 +39,12 @@ class ExperimentResponse(BaseModel): workspace: str + agent_names: Optional[List[str]] = None + """Distinct agent names observed across ingested sessions for this experiment.""" + + agent_versions: Optional[List[str]] = None + """Distinct agent versions observed across ingested sessions for this experiment.""" + aggregate_scores: Optional[Dict[str, EvaluatorAggregate]] = None cost_usd: Optional[EvaluatorAggregate] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py index ac311169924..4fc243e7584 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_update_params.py @@ -28,12 +28,6 @@ class ExperimentUpdateParams(TypedDict, total=False): workspace: str - agent_name: Required[str] - """Name of the agent under test.""" - - agent_version: Required[str] - """Version of the agent under test.""" - dataset_name: Required[str] """Producer-supplied dataset name.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_create_params.py index 06715b1c74a..9836fcb477e 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_create_params.py @@ -24,8 +24,8 @@ from .s3_storage_config_param import S3StorageConfigParam from .ngc_storage_config_param import NGCStorageConfigParam from .local_storage_config_param import LocalStorageConfigParam -from ..shared_params.fileset_metadata import FilesetMetadata from .huggingface_storage_config_param import HuggingfaceStorageConfigParam +from ..shared_params.fileset_metadata_param import FilesetMetadataParam __all__ = ["FilesetCreateParams", "Storage"] @@ -49,7 +49,7 @@ class FilesetCreateParams(TypedDict, total=False): description: str """The description of the fileset.""" - metadata: FilesetMetadata + metadata: FilesetMetadataParam """Tagged metadata container - the key indicates the type. Example: metadata = FilesetMetadata( dataset=DatasetMetadataContent( diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_update_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_update_params.py index 3f8699dda8d..0d0c735d5b3 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_update_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_update_params.py @@ -21,7 +21,7 @@ from typing_extensions import TypedDict from .fileset_purpose import FilesetPurpose -from ..shared_params.fileset_metadata import FilesetMetadata +from ..shared_params.fileset_metadata_param import FilesetMetadataParam __all__ = ["FilesetUpdateParams"] @@ -35,7 +35,7 @@ class FilesetUpdateParams(TypedDict, total=False): description: str """The description of the fileset.""" - metadata: FilesetMetadata + metadata: FilesetMetadataParam """Tagged metadata container - the key indicates the type. Example: metadata = FilesetMetadata( dataset=DatasetMetadataContent( diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context_param.py index b42e1ac3c4d..9bc79b3f3ad 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/evaluation_context_param.py @@ -24,12 +24,6 @@ class EvaluationContextParam(TypedDict, total=False): - dataset_id: str - - dataset_name: str - - dataset_version: str - evaluation_id: str evaluation_run_id: str diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/span.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/span.py index fc8f42ddd77..1093757a51e 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/span.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/span.py @@ -83,10 +83,6 @@ class Span(BaseModel): prompt_id: Optional[str] = None - prompt_name: Optional[str] = None - - prompt_version: Optional[str] = None - provider: Optional[str] = None raw_attributes: Optional[str] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/span_evaluation_context.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/span_evaluation_context.py index a4a45711c5a..e27f39824a2 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/span_evaluation_context.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/span_evaluation_context.py @@ -23,12 +23,6 @@ class SpanEvaluationContext(BaseModel): - dataset_id: Optional[str] = None - - dataset_name: Optional[str] = None - - dataset_version: Optional[str] = None - evaluation_id: Optional[str] = None evaluation_run_id: Optional[str] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/shared/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/shared/__init__.py index e0178b4f49c..d16fead87f6 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/shared/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/shared/__init__.py @@ -39,6 +39,7 @@ from .http_validation_error import HTTPValidationError as HTTPValidationError from .platform_job_log_page import PlatformJobLogPage as PlatformJobLogPage from .sliding_window_config import SlidingWindowConfig as SlidingWindowConfig +from .fileset_metadata_param import FilesetMetadataParam as FilesetMetadataParam from .model_metadata_content import ModelMetadataContent as ModelMetadataContent from .auth_discovery_response import AuthDiscoveryResponse as AuthDiscoveryResponse from .oidc_discovery_response import OidcDiscoveryResponse as OidcDiscoveryResponse diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/shared/fileset_metadata_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/shared/fileset_metadata_param.py new file mode 100644 index 00000000000..7679c6973b0 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/shared/fileset_metadata_param.py @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional + +from ..._models import BaseModel +from .model_metadata_content import ModelMetadataContent +from .dataset_metadata_content import DatasetMetadataContent + +__all__ = ["FilesetMetadataParam"] + + +class FilesetMetadataParam(BaseModel): + """Tagged metadata container - the key indicates the type. + + Example: + metadata = FilesetMetadata( + dataset=DatasetMetadataContent( + schema={"columns": ["id", "name"]}, + ) + ) + """ + + dataset: Optional[DatasetMetadataContent] = None + """Content for dataset-type filesets.""" + + model: Optional[ModelMetadataContent] = None + """Content for model-type filesets. + + Contains tool calling configuration that is merged into the ModelSpec during + checkpoint analysis. + """ diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/shared_params/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/shared_params/__init__.py index 449d6c5e147..7cd7058ab8e 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/shared_params/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/shared_params/__init__.py @@ -23,7 +23,6 @@ from .backend_format import BackendFormat as BackendFormat from .datetime_filter import DatetimeFilter as DatetimeFilter from .finetuning_type import FinetuningType as FinetuningType -from .fileset_metadata import FilesetMetadata as FilesetMetadata from .inference_params import InferenceParams as InferenceParams from .tool_call_config import ToolCallConfig as ToolCallConfig from .api_endpoint_data import APIEndpointData as APIEndpointData @@ -32,6 +31,7 @@ from .generic_sort_field import GenericSortField as GenericSortField from .platform_job_status import PlatformJobStatus as PlatformJobStatus from .sliding_window_config import SlidingWindowConfig as SlidingWindowConfig +from .fileset_metadata_param import FilesetMetadataParam as FilesetMetadataParam from .model_metadata_content import ModelMetadataContent as ModelMetadataContent from .dataset_metadata_content import DatasetMetadataContent as DatasetMetadataContent from .tool_calling_metadata_content import ToolCallingMetadataContent as ToolCallingMetadataContent diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/shared_params/fileset_metadata.py b/sdk/python/nemo-platform/src/nemo_platform/types/shared_params/fileset_metadata_param.py similarity index 94% rename from sdk/python/nemo-platform/src/nemo_platform/types/shared_params/fileset_metadata.py rename to sdk/python/nemo-platform/src/nemo_platform/types/shared_params/fileset_metadata_param.py index d53a643b0db..e3f510ca6ec 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/shared_params/fileset_metadata.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/shared_params/fileset_metadata_param.py @@ -22,10 +22,10 @@ from .model_metadata_content import ModelMetadataContent from .dataset_metadata_content import DatasetMetadataContent -__all__ = ["FilesetMetadata"] +__all__ = ["FilesetMetadataParam"] -class FilesetMetadata(TypedDict, total=False): +class FilesetMetadataParam(TypedDict, total=False): """Tagged metadata container - the key indicates the type. Example: diff --git a/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_atif.py b/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_atif.py index 797c3d460a5..6fb291ce824 100644 --- a/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_atif.py +++ b/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_atif.py @@ -59,9 +59,6 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: schema_version="ATIF-v1.0", continued_trajectory_ref="continued_trajectory_ref", evaluation_context={ - "dataset_id": "dataset_id", - "dataset_name": "dataset_name", - "dataset_version": "dataset_version", "evaluation_id": "evaluation_id", "evaluation_run_id": "evaluation_run_id", "evaluation_sha": "evaluation_sha", @@ -180,9 +177,6 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo schema_version="ATIF-v1.0", continued_trajectory_ref="continued_trajectory_ref", evaluation_context={ - "dataset_id": "dataset_id", - "dataset_name": "dataset_name", - "dataset_version": "dataset_version", "evaluation_id": "evaluation_id", "evaluation_run_id": "evaluation_run_id", "evaluation_sha": "evaluation_sha", diff --git a/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_chat_completions.py b/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_chat_completions.py index 2b5b08ef11c..8a1ca34231d 100644 --- a/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_chat_completions.py +++ b/sdk/python/nemo-platform/tests/api_resources/intake/ingest/test_chat_completions.py @@ -65,9 +65,6 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: cost_output_usd=0, cost_usd=0, evaluation_context={ - "dataset_id": "dataset_id", - "dataset_name": "dataset_name", - "dataset_version": "dataset_version", "evaluation_id": "evaluation_id", "evaluation_run_id": "evaluation_run_id", "evaluation_sha": "evaluation_sha", @@ -170,9 +167,6 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo cost_output_usd=0, cost_usd=0, evaluation_context={ - "dataset_id": "dataset_id", - "dataset_name": "dataset_name", - "dataset_version": "dataset_version", "evaluation_id": "evaluation_id", "evaluation_run_id": "evaluation_run_id", "evaluation_sha": "evaluation_sha", diff --git a/sdk/python/nemo-platform/tests/api_resources/test_experiments.py b/sdk/python/nemo-platform/tests/api_resources/test_experiments.py index b0ebb710741..2aba797f84b 100644 --- a/sdk/python/nemo-platform/tests/api_resources/test_experiments.py +++ b/sdk/python/nemo-platform/tests/api_resources/test_experiments.py @@ -41,8 +41,6 @@ class TestExperiments: def test_method_create(self, client: NeMoPlatform) -> None: experiment = client.experiments.create( workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", name="name", @@ -54,8 +52,6 @@ def test_method_create(self, client: NeMoPlatform) -> None: def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: experiment = client.experiments.create( workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", name="name", @@ -72,8 +68,6 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: def test_raw_response_create(self, client: NeMoPlatform) -> None: response = client.experiments.with_raw_response.create( workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", name="name", @@ -89,8 +83,6 @@ def test_raw_response_create(self, client: NeMoPlatform) -> None: def test_streaming_response_create(self, client: NeMoPlatform) -> None: with client.experiments.with_streaming_response.create( workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", name="name", @@ -109,8 +101,6 @@ def test_path_params_create(self, client: NeMoPlatform) -> None: with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): client.experiments.with_raw_response.create( workspace="", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", name="name", @@ -174,8 +164,6 @@ def test_method_update(self, client: NeMoPlatform) -> None: experiment = client.experiments.update( path_name="name", workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", body_name="name", @@ -188,8 +176,6 @@ def test_method_update_with_all_params(self, client: NeMoPlatform) -> None: experiment = client.experiments.update( path_name="name", workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", body_name="name", @@ -207,8 +193,6 @@ def test_raw_response_update(self, client: NeMoPlatform) -> None: response = client.experiments.with_raw_response.update( path_name="name", workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", body_name="name", @@ -225,8 +209,6 @@ def test_streaming_response_update(self, client: NeMoPlatform) -> None: with client.experiments.with_streaming_response.update( path_name="name", workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", body_name="name", @@ -246,8 +228,6 @@ def test_path_params_update(self, client: NeMoPlatform) -> None: client.experiments.with_raw_response.update( path_name="name", workspace="", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", body_name="name", @@ -257,8 +237,6 @@ def test_path_params_update(self, client: NeMoPlatform) -> None: client.experiments.with_raw_response.update( path_name="", workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", body_name="name", @@ -278,8 +256,6 @@ def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: experiment = client.experiments.list( workspace="workspace", filter={ - "agent_name": "agent_name", - "agent_version": "agent_version", "created_at": { "gte": parse_datetime("2019-12-27T18:11:19.117Z"), "lte": parse_datetime("2019-12-27T18:11:19.117Z"), @@ -398,8 +374,6 @@ class TestAsyncExperiments: async def test_method_create(self, async_client: AsyncNeMoPlatform) -> None: experiment = await async_client.experiments.create( workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", name="name", @@ -411,8 +385,6 @@ async def test_method_create(self, async_client: AsyncNeMoPlatform) -> None: async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatform) -> None: experiment = await async_client.experiments.create( workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", name="name", @@ -429,8 +401,6 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo async def test_raw_response_create(self, async_client: AsyncNeMoPlatform) -> None: response = await async_client.experiments.with_raw_response.create( workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", name="name", @@ -446,8 +416,6 @@ async def test_raw_response_create(self, async_client: AsyncNeMoPlatform) -> Non async def test_streaming_response_create(self, async_client: AsyncNeMoPlatform) -> None: async with async_client.experiments.with_streaming_response.create( workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", name="name", @@ -466,8 +434,6 @@ async def test_path_params_create(self, async_client: AsyncNeMoPlatform) -> None with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): await async_client.experiments.with_raw_response.create( workspace="", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", name="name", @@ -531,8 +497,6 @@ async def test_method_update(self, async_client: AsyncNeMoPlatform) -> None: experiment = await async_client.experiments.update( path_name="name", workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", body_name="name", @@ -545,8 +509,6 @@ async def test_method_update_with_all_params(self, async_client: AsyncNeMoPlatfo experiment = await async_client.experiments.update( path_name="name", workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", body_name="name", @@ -564,8 +526,6 @@ async def test_raw_response_update(self, async_client: AsyncNeMoPlatform) -> Non response = await async_client.experiments.with_raw_response.update( path_name="name", workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", body_name="name", @@ -582,8 +542,6 @@ async def test_streaming_response_update(self, async_client: AsyncNeMoPlatform) async with async_client.experiments.with_streaming_response.update( path_name="name", workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", body_name="name", @@ -603,8 +561,6 @@ async def test_path_params_update(self, async_client: AsyncNeMoPlatform) -> None await async_client.experiments.with_raw_response.update( path_name="name", workspace="", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", body_name="name", @@ -614,8 +570,6 @@ async def test_path_params_update(self, async_client: AsyncNeMoPlatform) -> None await async_client.experiments.with_raw_response.update( path_name="", workspace="workspace", - agent_name="agent_name", - agent_version="agent_version", dataset_name="dataset_name", experiment_group_id="experiment_group_id", body_name="name", @@ -635,8 +589,6 @@ async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform experiment = await async_client.experiments.list( workspace="workspace", filter={ - "agent_name": "agent_name", - "agent_version": "agent_version", "created_at": { "gte": parse_datetime("2019-12-27T18:11:19.117Z"), "lte": parse_datetime("2019-12-27T18:11:19.117Z"), diff --git a/sdk/stainless.yaml b/sdk/stainless.yaml index 7d5a9da0e6b..473c2c6a45f 100644 --- a/sdk/stainless.yaml +++ b/sdk/stainless.yaml @@ -254,8 +254,6 @@ resources: filesets: models: fileset_filter: FilesetFilter - fileset_metadata: FilesetMetadataOutput - fileset_metadata_param: FilesetMetadataInput methods: create: post /apis/files/v2/workspaces/{workspace}/filesets list: get /apis/files/v2/workspaces/{workspace}/filesets @@ -710,12 +708,12 @@ resources: auth_context: AuthContext tool_call_config: ToolCallConfig model_metadata_content: ModelMetadataContent - # DatasetMetadataContent is referenced by the shared FilesetMetadata schema. - # Keep it in $shared so the generated import is intra-package - # (shared/fileset_metadata -> shared/dataset_metadata_content) and avoids the - # circular import that occurs when it lives under the `files` resource: - # shared/fileset_metadata -> files/dataset_metadata_content -> files/__init__ - # -> files/fileset -> shared/fileset_metadata + # FilesetMetadata is referenced by multiple resources (files, models) at the source + # level. Pin both Output and Input variants in $shared so sync-models does not re-home + # them under `files`, which would break ``from nemo_platform.types.shared import + # FilesetMetadata`` imports downstream (models service tests). + fileset_metadata: FilesetMetadataOutput + fileset_metadata_param: FilesetMetadataInput dataset_metadata_content: DatasetMetadataContent tool_calling_metadata_content: ToolCallingMetadataContent backend_format: BackendFormat diff --git a/services/intake/scripts/spans/seed_experiment_rollup_data.py b/services/intake/scripts/spans/seed_experiment_rollup_data.py index c41ffe94090..d65b8857e48 100644 --- a/services/intake/scripts/spans/seed_experiment_rollup_data.py +++ b/services/intake/scripts/spans/seed_experiment_rollup_data.py @@ -122,8 +122,6 @@ def _upsert_experiment( ) -> None: body = { "name": experiment, - "agent_name": AGENT_NAME, - "agent_version": AGENT_VERSION, "dataset_name": DATASET_NAME, "dataset_version": "v1", "experiment_group_id": group_id, diff --git a/services/intake/scripts/spans/seed_experiments_demo.py b/services/intake/scripts/spans/seed_experiments_demo.py index 56ce346add7..25e4e6b7a95 100644 --- a/services/intake/scripts/spans/seed_experiments_demo.py +++ b/services/intake/scripts/spans/seed_experiments_demo.py @@ -60,6 +60,11 @@ class ExperimentSpec: summary: str | None = None agent_name: str = "sample-agent" agent_version: str = "1.0.0" + # Optional: sessions cycle through these instead of the scalar above, so a single + # experiment's rollup can surface multiple distinct agent names or versions (e.g., + # an A/B between agent versions, or a comparison of agents within one experiment). + agent_name_cycle: tuple[str, ...] | None = None + agent_version_cycle: tuple[str, ...] | None = None model_name: str = "provider/sample-model" dataset_name: str = "sample-dataset" dataset_version: str = "v1" @@ -164,6 +169,8 @@ class GroupSpec: description="Cross-encoder reranker, 5 trials per case, averaged for variance.", agent_name="codex-cli", agent_version="1.2.3", + # Trials ran across a minor version bump mid-experiment. + agent_version_cycle=("1.2.3", "1.2.4"), model_name="openai/gpt-4o-mini", dataset_name="support-bench", dataset_version="v3", @@ -209,9 +216,11 @@ class GroupSpec: ), ExperimentSpec( name="tb2-cursor-agent", - description="cursor-agent @ 0.4 with claude-sonnet.", + description="cursor-agent + cursor-cli mix @ 0.4 with claude-sonnet.", agent_name="cursor-agent", agent_version="0.4.1", + # Mix of cursor's agent and CLI binary running the same task. + agent_name_cycle=("cursor-agent", "cursor-cli"), model_name="anthropic/claude-sonnet-4-6", dataset_name="terminal-bench-2", dataset_version="v1", @@ -377,7 +386,7 @@ def seed(client: httpx.Client, base_url: str, workspace: str) -> None: for group_spec in DEMO_GROUPS: group_id, created = _create_group_if_missing(client, base_url, workspace, group_spec) - if not created: + if not created or group_id is None: print(f"\n[skip] group '{group_spec.name}' already exists; leaving it and its experiments alone") groups_skipped += 1 continue @@ -425,8 +434,6 @@ def _create_experiment( def _experiment_body(spec: ExperimentSpec, group_id: str) -> dict[str, Any]: body: dict[str, Any] = { "name": spec.name, - "agent_name": spec.agent_name, - "agent_version": spec.agent_version, "dataset_name": spec.dataset_name, "dataset_version": spec.dataset_version, "experiment_group_id": group_id, @@ -469,6 +476,14 @@ def _seed_sessions( # Spread sessions across the ~5.5h prior to "now" so the Studio timeline looks varied. offset_seconds = (i / max(1, spec.n_sessions)) * 5.5 * 3600 + session_agent_name = ( + spec.agent_name_cycle[i % len(spec.agent_name_cycle)] if spec.agent_name_cycle else spec.agent_name + ) + session_agent_version = ( + spec.agent_version_cycle[i % len(spec.agent_version_cycle)] + if spec.agent_version_cycle + else spec.agent_version + ) atif_body = _demo_atif_body( base_started_at=base_started_at, experiment_id=spec.name, @@ -477,8 +492,8 @@ def _seed_sessions( cost_usd=cost_usd, latency_ms=latency_ms, offset_seconds=offset_seconds, - agent_name=spec.agent_name, - agent_version=spec.agent_version, + agent_name=session_agent_name, + agent_version=session_agent_version, model_name=spec.model_name, prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index 3e2513aa53c..3995ee08e34 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -286,8 +286,6 @@ async def create_experiment( workspace=workspace, name=body.name, experiment_group_id=body.experiment_group_id, - agent_name=body.agent_name, - agent_version=body.agent_version, dataset_name=body.dataset_name, dataset_version=body.dataset_version, source_link=body.source_link, @@ -312,8 +310,9 @@ async def create_experiment( openapi_extra=generate_openapi_extra_params( filter_schema=ExperimentFilter, filter_description=( - "Filter experiments by name, experiment_group_id, agent_name, agent_version, " - "dataset_name, dataset_version, created_by, created_at, or updated_at." + "Filter experiments by name, experiment_group_id, " + "dataset_name, dataset_version, created_by, created_at, or updated_at. " + "Pass is_deleted=true to return only soft-deleted experiments; omit to see only live ones." ), ), ) @@ -372,10 +371,10 @@ async def get_experiment( return response -# Identity and the dataset/agent it was run against are fixed for the life of an +# Identity and the dataset it was run against are fixed for the life of an # Experiment (see the ingest invariants); changing them means it's a different # Experiment. PUT may only edit group membership, source link, summary, description, metadata. -_IMMUTABLE_EXPERIMENT_FIELDS = ("name", "agent_name", "agent_version", "dataset_name", "dataset_version") +_IMMUTABLE_EXPERIMENT_FIELDS = ("name", "dataset_name", "dataset_version") @router.put( @@ -672,6 +671,8 @@ async def _hydrate_rollups( def _apply_rollup(response: ExperimentResponse, rollup: ExperimentRollup) -> None: response.evaluator_names = rollup.evaluator_names response.model_names = rollup.model_names + response.agent_names = rollup.agent_names + response.agent_versions = rollup.agent_versions response.aggregate_scores = {name: _aggregate(score) for name, score in rollup.evaluator_scores.items()} or None response.run_count = rollup.run_count response.cost_usd = _aggregate(rollup.cost_usd) if rollup.cost_usd is not None else None diff --git a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py index 06d3beff2de..bff2e7f0645 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py @@ -37,8 +37,6 @@ class ExperimentRequest(BaseModel): experiment_group_id: str = Field( description="Entity id of the owning ExperimentGroup. Required — the group must already exist.", ) - agent_name: str = Field(description="Name of the agent under test.") - agent_version: str = Field(description="Version of the agent under test.") dataset_name: str = Field(description="Producer-supplied dataset name.") dataset_version: str | None = Field(default=None, description="Producer-supplied dataset version.") source_link: AnyUrl | None = Field(default=None, description="Optional URL for the source experiment.") @@ -90,8 +88,6 @@ class ExperimentResponse(BaseModel): experiment_group_id: str = Field( description="Entity id of the owning ExperimentGroup. Required for every Experiment.", ) - agent_name: str - agent_version: str dataset_name: str dataset_version: str | None = None source_link: AnyUrl | None = None @@ -107,6 +103,16 @@ class ExperimentResponse(BaseModel): description="Distinct model names observed across ingested sessions for this experiment.", json_schema_extra={"uniqueItems": True}, ) + agent_names: list[str] = Field( + default_factory=list, + description="Distinct agent names observed across ingested sessions for this experiment.", + json_schema_extra={"uniqueItems": True}, + ) + agent_versions: list[str] = Field( + default_factory=list, + description="Distinct agent versions observed across ingested sessions for this experiment.", + json_schema_extra={"uniqueItems": True}, + ) aggregate_scores: dict[str, EvaluatorAggregate] | None = None run_count: int = Field( default=0, @@ -122,8 +128,6 @@ def from_entity(cls, entity: Experiment) -> ExperimentResponse: name=entity.name, workspace=entity.workspace, experiment_group_id=entity.experiment_group_id, - agent_name=entity.agent_name, - agent_version=entity.agent_version, dataset_name=entity.dataset_name, dataset_version=entity.dataset_version, source_link=entity.source_link, @@ -150,8 +154,6 @@ class ExperimentFilter(Filter): name: str | None = Field(default=None, description="Filter experiments by name.") experiment_group_id: str | None = Field(default=None, description="Filter experiments by owning group id.") - agent_name: str | None = Field(default=None, description="Filter experiments by agent name.") - agent_version: str | None = Field(default=None, description="Filter experiments by agent version.") dataset_name: str | None = Field(default=None, description="Filter experiments by dataset name.") dataset_version: str | None = Field(default=None, description="Filter experiments by dataset version.") created_by: str | None = Field(default=None, description="Filter experiments by the principal that created them.") diff --git a/services/intake/src/nmp/intake/entities/experiments.py b/services/intake/src/nmp/intake/entities/experiments.py index 5ba0fe201e0..4cd1907c451 100644 --- a/services/intake/src/nmp/intake/entities/experiments.py +++ b/services/intake/src/nmp/intake/entities/experiments.py @@ -49,9 +49,6 @@ class Experiment(EntityBase): ), ) - agent_name: str = Field(description="Name of the agent under test.") - agent_version: str = Field(description="Version of the agent under test.") - dataset_name: str = Field(description="Producer-supplied dataset name.") dataset_version: str | None = Field(default=None, description="Producer-supplied dataset version.") source_link: AnyUrl | None = Field(default=None, description="Optional URL for the source experiment.") diff --git a/services/intake/src/nmp/intake/spans/api/spans_schemas.py b/services/intake/src/nmp/intake/spans/api/spans_schemas.py index 82f7dda962d..8f6ac85bf99 100644 --- a/services/intake/src/nmp/intake/spans/api/spans_schemas.py +++ b/services/intake/src/nmp/intake/spans/api/spans_schemas.py @@ -82,9 +82,6 @@ class SpanEvaluationContext(BaseModel): evaluation_id: str | None = None evaluation_sha: str | None = None evaluation_run_id: str | None = None - dataset_id: str | None = None - dataset_name: str | None = None - dataset_version: str | None = None test_case_id: str | None = None metadata: dict[str, Any] = Field(default_factory=dict) @@ -99,9 +96,6 @@ def from_semantic_attributes( evaluation_id=attributes.evaluation_id, evaluation_sha=attributes.evaluation_sha, evaluation_run_id=attributes.evaluation_run_id, - dataset_id=attributes.dataset_id, - dataset_name=attributes.dataset_name, - dataset_version=attributes.dataset_version, test_case_id=attributes.test_case_id, metadata=metadata or {}, ) @@ -116,9 +110,6 @@ def has_scalar_values(self) -> bool: self.evaluation_id, self.evaluation_sha, self.evaluation_run_id, - self.dataset_id, - self.dataset_name, - self.dataset_version, self.test_case_id, ) ) @@ -143,8 +134,6 @@ class Span(BaseModel): provider: str | None = None model: str | None = None prompt_id: str | None = None - prompt_name: str | None = None - prompt_version: str | None = None agent_id: str | None = None agent_name: str | None = None tool_name: str | None = None @@ -191,8 +180,6 @@ def from_domain(cls, span: IntakeSpan, *, mode: SpanMode = "detailed") -> Self: provider=semantic_attributes.provider, model=semantic_attributes.model, prompt_id=semantic_attributes.prompt_id, - prompt_name=semantic_attributes.prompt_name, - prompt_version=semantic_attributes.prompt_version, agent_id=semantic_attributes.agent_id, agent_name=semantic_attributes.agent_name, tool_name=semantic_attributes.tool_name, diff --git a/services/intake/src/nmp/intake/spans/clickhouse_migrations.py b/services/intake/src/nmp/intake/spans/clickhouse_migrations.py index 7ed4cab6e8b..ba2bea0b68a 100644 --- a/services/intake/src/nmp/intake/spans/clickhouse_migrations.py +++ b/services/intake/src/nmp/intake/spans/clickhouse_migrations.py @@ -342,6 +342,12 @@ def _create_trace_index_schema(client, settings: ClickHouseMigrationSettings) -> ("ch_annotations_0001", _create_annotations_schema), ("ch_evaluator_results_0002", _add_evaluator_results_skip_indexes), ("ch_trace_index_0003", _create_trace_index_schema), + # NeMo-owned span attribute keys moved under the ``nemo.*`` namespace + # (e.g., ``experiment.id`` → ``nemo.experiment.id``). The trace_index MV's SELECT clause + # resolves bag keys from the catalog at creation time, so any environment that already + # applied 0003 has the old keys baked in. Re-running the schema function drops and + # recreates the MV with the current catalog keys. + ("ch_trace_index_0004_nemo_keys", _create_trace_index_schema), ] CURRENT_SCHEMA_VERSION = _MIGRATIONS[-1][0] diff --git a/services/intake/src/nmp/intake/spans/experiment_rollup_repository.py b/services/intake/src/nmp/intake/spans/experiment_rollup_repository.py index 6a014006122..bd2f5ca993f 100644 --- a/services/intake/src/nmp/intake/spans/experiment_rollup_repository.py +++ b/services/intake/src/nmp/intake/spans/experiment_rollup_repository.py @@ -30,6 +30,8 @@ class ExperimentRollup: experiment_id: str run_count: int = 0 model_names: list[str] = field(default_factory=list) + agent_names: list[str] = field(default_factory=list) + agent_versions: list[str] = field(default_factory=list) evaluator_scores: dict[str, ScoreRollup] = field(default_factory=dict) cost_usd: ScoreRollup | None = None latency_ms: ScoreRollup | None = None @@ -92,11 +94,15 @@ async def get_rollups(self, *, workspace: str, experiment_ids: list[str]) -> dic **parameters, "cost_key": spec_for_field(SpanAttributeField.COST_TOTAL_USD).bag_key, "model_key": spec_for_field(SpanAttributeField.MODEL).bag_key, + "agent_name_key": spec_for_field(SpanAttributeField.AGENT_NAME).bag_key, + "agent_version_key": spec_for_field(SpanAttributeField.AGENT_VERSION).bag_key, }, ) ): rollup = rollups[row["experiment_id"]] rollup.model_names = _string_list(row["model_names"]) + rollup.agent_names = _string_list(row["agent_names"]) + rollup.agent_versions = _string_list(row["agent_versions"]) rollup.cost_usd = _score_rollup(row, "cost") rollup.latency_ms = _score_rollup(row, "latency") @@ -247,7 +253,17 @@ def _metric_rollups_sql(*, trace_index_table: str, spans_table: str, experiment_ spans.attributes_string[%(model_key)s], has(mapKeys(spans.attributes_string), %(model_key)s) AND spans.attributes_string[%(model_key)s] != '' - ) AS model_names + ) AS model_names, + groupUniqArrayIf( + spans.attributes_string[%(agent_name_key)s], + has(mapKeys(spans.attributes_string), %(agent_name_key)s) + AND spans.attributes_string[%(agent_name_key)s] != '' + ) AS agent_names, + groupUniqArrayIf( + spans.attributes_string[%(agent_version_key)s], + has(mapKeys(spans.attributes_string), %(agent_version_key)s) + AND spans.attributes_string[%(agent_version_key)s] != '' + ) AS agent_versions FROM scoped_sessions AS sessions LEFT JOIN current_session_spans AS spans ON sessions.workspace = spans.workspace @@ -258,6 +274,8 @@ def _metric_rollups_sql(*, trace_index_table: str, spans_table: str, experiment_ SELECT experiment_id, arraySort(arrayDistinct(arrayFlatten(groupArray(model_names)))) AS model_names, + arraySort(arrayDistinct(arrayFlatten(groupArray(agent_names)))) AS agent_names, + arraySort(arrayDistinct(arrayFlatten(groupArray(agent_versions)))) AS agent_versions, {_stat_columns("cost_usd", prefix="cost", guarded=True)}, {_stat_columns("latency_ms", prefix="latency", guarded=True)} FROM session_costs diff --git a/services/intake/src/nmp/intake/spans/ingest/atif_mapping.py b/services/intake/src/nmp/intake/spans/ingest/atif_mapping.py index 519de7fa2d9..b0972b9af34 100644 --- a/services/intake/src/nmp/intake/spans/ingest/atif_mapping.py +++ b/services/intake/src/nmp/intake/spans/ingest/atif_mapping.py @@ -63,6 +63,7 @@ def trajectory_to_spans( workspace=workspace, default_session_id=trajectory.session_id, default_agent_name=trajectory.agent.name, + default_agent_version=trajectory.agent.version, default_model_name=trajectory.agent.model_name, external_parent_span_id=trajectory_span.external_span_id, step=step, @@ -75,6 +76,7 @@ def trajectory_to_spans( workspace=workspace, default_session_id=trajectory.session_id, default_agent_name=trajectory.agent.name, + default_agent_version=trajectory.agent.version, default_model_name=trajectory.agent.model_name, external_parent_span_id=step_span.external_span_id, step=step, @@ -90,6 +92,7 @@ def trajectory_to_spans( workspace=workspace, default_session_id=trajectory.session_id, default_agent_name=trajectory.agent.name, + default_agent_version=trajectory.agent.version, default_model_name=trajectory.agent.model_name, external_parent_span_id=step_span.external_span_id, step=step, @@ -151,6 +154,7 @@ def _trajectory_to_span( attribute_bags = _span_attributes( model=trajectory.agent.model_name, agent_name=trajectory.agent.name, + agent_version=trajectory.agent.version, evaluation_context=trajectory.evaluation_context, input_tokens=input_tokens, output_tokens=output_tokens, @@ -184,6 +188,7 @@ def _step_to_span( workspace: str, default_session_id: str, default_agent_name: str, + default_agent_version: str | None, default_model_name: str | None, external_parent_span_id: str, step: AtifStep, @@ -205,6 +210,7 @@ def _step_to_span( attribute_bags = _span_attributes( model=model_name or default_model_name, agent_name=default_agent_name, + agent_version=default_agent_version, input_tokens=input_tokens, output_tokens=output_tokens, cached_tokens=metrics.cached_tokens if metrics is not None else None, @@ -237,6 +243,7 @@ def _tool_call_to_span( workspace: str, default_session_id: str, default_agent_name: str, + default_agent_version: str | None, default_model_name: str | None, external_parent_span_id: str, step: AtifStep, @@ -260,6 +267,7 @@ def _tool_call_to_span( attribute_bags = _span_attributes( model=_step_model_name(step) or default_model_name, agent_name=default_agent_name, + agent_version=default_agent_version, tool_name=tool_call.function_name, error_message=error_message, raw_attributes={ @@ -293,6 +301,7 @@ def _subagent_ref_to_span( workspace: str, default_session_id: str, default_agent_name: str, + default_agent_version: str | None, default_model_name: str | None, external_parent_span_id: str, step: AtifStep, @@ -320,6 +329,7 @@ def _subagent_ref_to_span( attribute_bags = _span_attributes( model=_step_model_name(step) or default_model_name, agent_name=default_agent_name, + agent_version=default_agent_version, error_message=error_message, raw_attributes={ "step_id": step.step_id, @@ -413,6 +423,7 @@ def _evaluator_result_to_span( attribute_bags = _span_attributes( model=trajectory.agent.model_name, agent_name=trajectory.agent.name, + agent_version=trajectory.agent.version, raw_attributes=raw_attributes, ) return [ @@ -442,6 +453,7 @@ def _span_attributes( *, model: str | None = None, agent_name: str | None = None, + agent_version: str | None = None, evaluation_context: EvaluationContext | None = None, tool_name: str | None = None, error_message: str | None = None, @@ -455,12 +467,10 @@ def _span_attributes( semantic_attributes = SpanSemanticAttributes( model=model, agent_name=agent_name, + agent_version=agent_version, evaluation_id=evaluation_context.evaluation_id if evaluation_context is not None else None, evaluation_sha=evaluation_context.evaluation_sha if evaluation_context is not None else None, evaluation_run_id=evaluation_context.evaluation_run_id if evaluation_context is not None else None, - dataset_id=evaluation_context.dataset_id if evaluation_context is not None else None, - dataset_name=evaluation_context.dataset_name if evaluation_context is not None else None, - dataset_version=evaluation_context.dataset_version if evaluation_context is not None else None, test_case_id=evaluation_context.test_case_id if evaluation_context is not None else None, tool_name=tool_name, error_message=error_message, @@ -472,7 +482,7 @@ def _span_attributes( ) attribute_bags = semantic_attributes.to_bags() if evaluation_context is not None and evaluation_context.metadata: - attribute_bags.put_json("experiment.metadata", evaluation_context.metadata) + attribute_bags.put_json("nemo.experiment.metadata", evaluation_context.metadata) if raw_attributes is not None: attribute_bags.put_json("atif.raw", raw_attributes) return attribute_bags diff --git a/services/intake/src/nmp/intake/spans/ingest/chat_completions.py b/services/intake/src/nmp/intake/spans/ingest/chat_completions.py index 967bf841819..67d6aa9bbfc 100644 --- a/services/intake/src/nmp/intake/spans/ingest/chat_completions.py +++ b/services/intake/src/nmp/intake/spans/ingest/chat_completions.py @@ -235,9 +235,6 @@ def _build_attribute_bags( evaluation_id=evaluation_context.evaluation_id if evaluation_context is not None else None, evaluation_sha=evaluation_context.evaluation_sha if evaluation_context is not None else None, evaluation_run_id=evaluation_context.evaluation_run_id if evaluation_context is not None else None, - dataset_id=evaluation_context.dataset_id if evaluation_context is not None else None, - dataset_name=evaluation_context.dataset_name if evaluation_context is not None else None, - dataset_version=evaluation_context.dataset_version if evaluation_context is not None else None, test_case_id=evaluation_context.test_case_id if evaluation_context is not None else None, error_type=error_type, error_message=error_message, @@ -254,7 +251,7 @@ def _build_attribute_bags( ) attribute_bags = semantic.to_bags() if evaluation_context is not None and evaluation_context.metadata: - attribute_bags.put_json("experiment.metadata", evaluation_context.metadata) + attribute_bags.put_json("nemo.experiment.metadata", evaluation_context.metadata) for key, value in body.cost_details.items(): bag_key = f"cost.{key}" if bag_key in attribute_bags.number: diff --git a/services/intake/src/nmp/intake/spans/ingest/evaluation_context.py b/services/intake/src/nmp/intake/spans/ingest/evaluation_context.py index 0b1b49510b4..82870ce2888 100644 --- a/services/intake/src/nmp/intake/spans/ingest/evaluation_context.py +++ b/services/intake/src/nmp/intake/spans/ingest/evaluation_context.py @@ -30,9 +30,6 @@ class EvaluationContext(BaseModel): evaluation_id: str | None = None evaluation_sha: str | None = None evaluation_run_id: str | None = None - dataset_id: str | None = None - dataset_name: str | None = None - dataset_version: str | None = None test_case_id: str | None = None metadata: dict[str, Any] = Field(default_factory=dict) diff --git a/services/intake/src/nmp/intake/spans/span_attribute_bags.py b/services/intake/src/nmp/intake/spans/span_attribute_bags.py index 016c17eac7c..7c7744ce173 100644 --- a/services/intake/src/nmp/intake/spans/span_attribute_bags.py +++ b/services/intake/src/nmp/intake/spans/span_attribute_bags.py @@ -13,7 +13,6 @@ from nmp.intake.spans.span_attribute_catalog import ( COST_SCALE, KNOWN_BAG_KEYS, - LEGACY_BAG_KEY_ALIASES, SOURCE_ONLY_KEYS, SOURCE_ONLY_PREFIXES, AttributeBag, @@ -49,14 +48,7 @@ def from_domain_maps( def get_field(self, field: SpanAttributeField | str) -> str | int | float | bool | Decimal | None: spec = spec_for_field(field) bag = self._bag_for_spec(spec) - value = from_bag(bag.get(spec.bag_key), spec) - if value is not None: - return value - for alias in LEGACY_BAG_KEY_ALIASES.get(spec.field, ()): - value = from_bag(bag.get(alias), spec) - if value is not None: - return value - return None + return from_bag(bag.get(spec.bag_key), spec) def put_field(self, field: SpanAttributeField | str, value: Any) -> None: self.put_spec(spec_for_field(field), value) @@ -114,11 +106,11 @@ def raw_attributes_json(self) -> str | None: parsed_atif_raw = json.loads(atif_raw) if not isinstance(parsed_atif_raw, dict): raise TypeError("Expected atif.raw to contain a JSON object") - parsed_atif_raw.pop("experiment.metadata", None) + parsed_atif_raw.pop("nemo.experiment.metadata", None) raw.update(parsed_atif_raw) for key, value in self.string.items(): - if key not in {"atif.raw", "experiment.metadata"} and key not in KNOWN_BAG_KEYS: + if key not in {"atif.raw", "nemo.experiment.metadata"} and key not in KNOWN_BAG_KEYS: raw[key] = value for key, value in self.number.items(): if key not in KNOWN_BAG_KEYS: @@ -131,12 +123,12 @@ def raw_attributes_json(self) -> str | None: return json.dumps(raw, separators=(",", ":"), ensure_ascii=False) def evaluation_metadata(self) -> dict[str, Any] | None: - value = self.string.get("experiment.metadata") or self.string.get("evaluation.metadata") + value = self.string.get("nemo.experiment.metadata") if value is None: return None parsed = json.loads(value) if not isinstance(parsed, dict): - raise TypeError("Expected experiment/evaluation metadata to contain a JSON object") + raise TypeError("Expected experiment metadata to contain a JSON object") return parsed def _bag_for_spec(self, spec: AttributeSpec) -> dict[str, str] | dict[str, float] | dict[str, bool]: diff --git a/services/intake/src/nmp/intake/spans/span_attribute_catalog.py b/services/intake/src/nmp/intake/spans/span_attribute_catalog.py index e421ec612be..996337397db 100644 --- a/services/intake/src/nmp/intake/spans/span_attribute_catalog.py +++ b/services/intake/src/nmp/intake/spans/span_attribute_catalog.py @@ -31,18 +31,14 @@ class SpanAttributeField(StrEnum): MODEL = "model" PROVIDER = "provider" PROMPT_ID = "prompt_id" - PROMPT_NAME = "prompt_name" - PROMPT_VERSION = "prompt_version" AGENT_ID = "agent_id" AGENT_NAME = "agent_name" + AGENT_VERSION = "agent_version" TOOL_NAME = "tool_name" PROJECT = "project" EVALUATION_ID = "evaluation_id" EVALUATION_SHA = "evaluation_sha" EVALUATION_RUN_ID = "evaluation_run_id" - DATASET_ID = "dataset_id" - DATASET_NAME = "dataset_name" - DATASET_VERSION = "dataset_version" TEST_CASE_ID = "test_case_id" ERROR_TYPE = "error_type" ERROR_MESSAGE = "error_message" @@ -106,26 +102,6 @@ class AttributeSpec: "prompt.id", ), ), - AttributeSpec( - field=SpanAttributeField.PROMPT_NAME, - bag=AttributeBag.STRING, - bag_key="prompt.name", - source_keys=( - "gen_ai.prompt.name", - "llm.prompt_template.name", - "prompt.name", - ), - ), - AttributeSpec( - field=SpanAttributeField.PROMPT_VERSION, - bag=AttributeBag.STRING, - bag_key="prompt.version", - source_keys=( - "gen_ai.prompt.version", - "llm.prompt_template.version", - "prompt.version", - ), - ), AttributeSpec( field=SpanAttributeField.AGENT_ID, bag=AttributeBag.STRING, @@ -145,6 +121,15 @@ class AttributeSpec: "agent.name", ), ), + AttributeSpec( + field=SpanAttributeField.AGENT_VERSION, + bag=AttributeBag.STRING, + bag_key="agent.version", + source_keys=( + "gen_ai.agent.version", + "agent.version", + ), + ), AttributeSpec( field=SpanAttributeField.TOOL_NAME, bag=AttributeBag.STRING, @@ -167,72 +152,26 @@ class AttributeSpec: AttributeSpec( field=SpanAttributeField.EVALUATION_ID, bag=AttributeBag.STRING, - # Renamed from evaluation.id. - bag_key="experiment.id", - source_keys=( - "experiment.id", - "experiment_id", - "evaluation.id", - "evaluation_id", - ), + bag_key="nemo.experiment.id", + source_keys=("nemo.experiment.id",), ), AttributeSpec( field=SpanAttributeField.EVALUATION_SHA, bag=AttributeBag.STRING, - bag_key="experiment.sha", - source_keys=( - "experiment.sha", - "experiment_sha", - "evaluation.sha", - "evaluation_sha", - ), + bag_key="nemo.experiment.sha", + source_keys=("nemo.experiment.sha",), ), AttributeSpec( field=SpanAttributeField.EVALUATION_RUN_ID, bag=AttributeBag.STRING, - bag_key="experiment.run_id", - source_keys=( - "experiment.run_id", - "evaluation.run_id", - "evaluation_run_id", - ), - ), - AttributeSpec( - field=SpanAttributeField.DATASET_ID, - bag=AttributeBag.STRING, - bag_key="dataset.id", - source_keys=( - "dataset.id", - "dataset_id", - ), - ), - AttributeSpec( - field=SpanAttributeField.DATASET_NAME, - bag=AttributeBag.STRING, - bag_key="dataset.name", - source_keys=( - "dataset.name", - "dataset_name", - ), - ), - AttributeSpec( - field=SpanAttributeField.DATASET_VERSION, - bag=AttributeBag.STRING, - bag_key="dataset.version", - source_keys=( - "dataset.version", - "dataset_version", - ), + bag_key="nemo.experiment.run_id", + source_keys=("nemo.experiment.run_id",), ), AttributeSpec( field=SpanAttributeField.TEST_CASE_ID, bag=AttributeBag.STRING, - bag_key="test_case.id", - source_keys=( - "test_case.id", - "test_case_id", - "dataset.test_case_id", - ), + bag_key="nemo.test_case.id", + source_keys=("nemo.test_case.id",), ), AttributeSpec( field=SpanAttributeField.ERROR_TYPE, @@ -337,16 +276,7 @@ class AttributeSpec: SPECS_BY_FIELD = {spec.field: spec for spec in ATTRIBUTE_SPECS} SPECS_BY_FIELD_VALUE = {spec.field.value: spec for spec in ATTRIBUTE_SPECS} SPECS_BY_BAG_KEY = {spec.bag_key: spec for spec in ATTRIBUTE_SPECS} -LEGACY_BAG_KEY_ALIASES = { - SpanAttributeField.EVALUATION_ID: ("evaluation.id",), - SpanAttributeField.EVALUATION_SHA: ("evaluation.sha",), - SpanAttributeField.EVALUATION_RUN_ID: ("evaluation.run_id",), - SpanAttributeField.TEST_CASE_ID: ("dataset.test_case_id",), -} -LEGACY_KNOWN_BAG_KEYS = frozenset(key for aliases in LEGACY_BAG_KEY_ALIASES.values() for key in aliases) | frozenset( - {"evaluation.metadata"} -) -KNOWN_BAG_KEYS = frozenset(SPECS_BY_BAG_KEY) | LEGACY_KNOWN_BAG_KEYS +KNOWN_BAG_KEYS = frozenset(SPECS_BY_BAG_KEY) QUERYABLE_FIELDS = frozenset(SPECS_BY_FIELD_VALUE) # Source keys that populate top-level span fields or payloads rather than @@ -452,24 +382,11 @@ def where_clause( if bag_value is None: raise ValueError(f"Span attribute filter {field!r} does not support null values") - alias_keys = LEGACY_BAG_KEY_ALIASES.get(spec.field, ()) - if not alias_keys: - sql = ( - f"has(mapKeys({spec.bag.value}), %({key_param})s) " - f"AND {spec.bag.value}[%({key_param})s] {sql_operator} %({value_param})s" - ) - return sql, {key_param: spec.bag_key, value_param: bag_value} - - clauses: list[str] = [] - parameters: dict[str, Any] = {value_param: bag_value} - for index, bag_key in enumerate((spec.bag_key, *alias_keys)): - aliased_key_param = key_param if index == 0 else f"{key_param}_{index}" - clauses.append( - f"(has(mapKeys({spec.bag.value}), %({aliased_key_param})s) " - f"AND {spec.bag.value}[%({aliased_key_param})s] {sql_operator} %({value_param})s)" - ) - parameters[aliased_key_param] = bag_key - return f"({' OR '.join(clauses)})", parameters + sql = ( + f"has(mapKeys({spec.bag.value}), %({key_param})s) " + f"AND {spec.bag.value}[%({key_param})s] {sql_operator} %({value_param})s" + ) + return sql, {key_param: spec.bag_key, value_param: bag_value} def to_semantic_value(value: Any, spec: AttributeSpec) -> str | int | float | bool | Decimal | None: diff --git a/services/intake/src/nmp/intake/spans/span_semantic_attributes.py b/services/intake/src/nmp/intake/spans/span_semantic_attributes.py index be6e01dcba8..423d399c1c3 100644 --- a/services/intake/src/nmp/intake/spans/span_semantic_attributes.py +++ b/services/intake/src/nmp/intake/spans/span_semantic_attributes.py @@ -25,18 +25,14 @@ class SpanSemanticAttributes(BaseModel): model: str | None = None provider: str | None = None prompt_id: str | None = None - prompt_name: str | None = None - prompt_version: str | None = None agent_id: str | None = None agent_name: str | None = None + agent_version: str | None = None tool_name: str | None = None project: str | None = None evaluation_id: str | None = None evaluation_sha: str | None = None evaluation_run_id: str | None = None - dataset_id: str | None = None - dataset_name: str | None = None - dataset_version: str | None = None test_case_id: str | None = None error_type: str | None = None error_message: str | None = None diff --git a/services/intake/tests/integration/spans/test_atif_ingest.py b/services/intake/tests/integration/spans/test_atif_ingest.py index 4449113f0ac..c5862f49aff 100644 --- a/services/intake/tests/integration/spans/test_atif_ingest.py +++ b/services/intake/tests/integration/spans/test_atif_ingest.py @@ -189,9 +189,6 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( "evaluation_id": "eval-sample-agent-baseline", "evaluation_sha": "abc132901", "evaluation_run_id": evaluation_run_id, - "dataset_id": "sample-dataset", - "dataset_name": "Sample Dataset", - "dataset_version": "v1", "test_case_id": "sample-test-case-a", "metadata": {"trial": "sample-test-case-a__trial-a"}, } @@ -382,9 +379,6 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( "evaluation_id": evaluation_context["evaluation_id"], "evaluation_sha": evaluation_context["evaluation_sha"], "evaluation_run_id": evaluation_context["evaluation_run_id"], - "dataset_id": evaluation_context["dataset_id"], - "dataset_name": evaluation_context["dataset_name"], - "dataset_version": evaluation_context["dataset_version"], "test_case_id": evaluation_context["test_case_id"], "metadata": evaluation_context["metadata"], } @@ -392,7 +386,7 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( trajectory_raw = json.loads(trajectory["raw_attributes"]) assert trajectory_raw["session_id"] == body["session_id"] assert "evaluation_context" not in trajectory_raw - assert "experiment.metadata" not in trajectory_raw + assert "nemo.experiment.metadata" not in trajectory_raw assert "evaluation.metadata" not in trajectory_raw assert trajectory["started_at"] == _span_started_at(user_step_time) assert trajectory["ended_at"] == _span_ended_at(verifier_finished_at) @@ -717,8 +711,6 @@ def _create_experiment(client: TestClient, name: str) -> str: json={ "name": name, "experiment_group_id": group_id, - "agent_name": "sample-agent", - "agent_version": "1.0.0", "dataset_name": "sample-dataset", "dataset_version": "v1", }, diff --git a/services/intake/tests/integration/spans/test_chat_completions_ingest.py b/services/intake/tests/integration/spans/test_chat_completions_ingest.py index 5c0b786a3f8..70ba7b1f113 100644 --- a/services/intake/tests/integration/spans/test_chat_completions_ingest.py +++ b/services/intake/tests/integration/spans/test_chat_completions_ingest.py @@ -18,9 +18,6 @@ "evaluation_id": "chat-eval", "evaluation_sha": "chat-eval-sha", "evaluation_run_id": "evalrun-chat-001", - "dataset_id": "chat-dataset", - "dataset_name": "Chat Dataset", - "dataset_version": "v1", "test_case_id": "chat-case-001", "metadata": {"source": "chat-completions-test"}, } @@ -261,9 +258,6 @@ def test_chat_completions_ingest_accepts_deprecated_evaluation_context(client: T "evaluation_id": EVALUATION_CONTEXT["evaluation_id"], "evaluation_sha": EVALUATION_CONTEXT["evaluation_sha"], "evaluation_run_id": EVALUATION_CONTEXT["evaluation_run_id"], - "dataset_id": EVALUATION_CONTEXT["dataset_id"], - "dataset_name": EVALUATION_CONTEXT["dataset_name"], - "dataset_version": EVALUATION_CONTEXT["dataset_version"], "test_case_id": EVALUATION_CONTEXT["test_case_id"], "metadata": EVALUATION_CONTEXT["metadata"], } @@ -453,8 +447,6 @@ def _create_experiment(client: TestClient, name: str) -> str: json={ "name": name, "experiment_group_id": group_id, - "agent_name": "sample-agent", - "agent_version": "1.0.0", "dataset_name": "chat-dataset", "dataset_version": "v1", }, diff --git a/services/intake/tests/integration/spans/test_clickhouse_bootstrap.py b/services/intake/tests/integration/spans/test_clickhouse_bootstrap.py index 6b3970b0f7a..20b7d766269 100644 --- a/services/intake/tests/integration/spans/test_clickhouse_bootstrap.py +++ b/services/intake/tests/integration/spans/test_clickhouse_bootstrap.py @@ -27,6 +27,7 @@ def test_clickhouse_bootstrap_is_idempotent(clickhouse_client: ClickHouseSpanCli ("ch_evaluator_results_0002",), ("ch_spans_0002",), ("ch_trace_index_0003",), + ("ch_trace_index_0004_nemo_keys",), ] diff --git a/services/intake/tests/integration/spans/test_experiment_rollups.py b/services/intake/tests/integration/spans/test_experiment_rollups.py index 71ad6c49f68..75675ace33d 100644 --- a/services/intake/tests/integration/spans/test_experiment_rollups.py +++ b/services/intake/tests/integration/spans/test_experiment_rollups.py @@ -33,8 +33,6 @@ def test_experiment_response_hydrates_clickhouse_rollups(client: TestClient) -> json={ "name": experiment_id, "experiment_group_id": group_id, - "agent_name": "sample-agent", - "agent_version": "1.0.0", "dataset_name": "rollup-dataset", "dataset_version": "v1", }, @@ -113,8 +111,6 @@ def test_atif_ingest_rejects_deleted_experiment(client: TestClient) -> None: json={ "name": experiment_id, "experiment_group_id": group_id, - "agent_name": "sample-agent", - "agent_version": "1.0.0", "dataset_name": "rollup-dataset", "dataset_version": "v1", }, @@ -167,8 +163,6 @@ def test_deprecated_evaluation_context_hydrates_experiment_rollups(client: TestC json={ "name": experiment_id, "experiment_group_id": group_id, - "agent_name": "sample-agent", - "agent_version": "1.0.0", "dataset_name": "rollup-dataset", "dataset_version": "v1", }, diff --git a/services/intake/tests/integration/spans/test_experiment_sessions.py b/services/intake/tests/integration/spans/test_experiment_sessions.py index fe704a9a64a..0a8dcc27ae1 100644 --- a/services/intake/tests/integration/spans/test_experiment_sessions.py +++ b/services/intake/tests/integration/spans/test_experiment_sessions.py @@ -33,8 +33,6 @@ def test_list_experiment_sessions_returns_joined_session_rows(client: TestClient json={ "name": experiment_name, "experiment_group_id": group_id, - "agent_name": "sample-agent", - "agent_version": "1.0.0", "dataset_name": "sessions-dataset", "dataset_version": "v1", }, @@ -111,8 +109,6 @@ def test_list_experiment_sessions_filter_by_test_case(client: TestClient) -> Non json={ "name": experiment_name, "experiment_group_id": group_id, - "agent_name": "sample-agent", - "agent_version": "1.0.0", "dataset_name": "sessions-dataset", "dataset_version": "v1", }, @@ -160,8 +156,6 @@ def test_list_experiment_sessions_filter_by_status(client: TestClient) -> None: json={ "name": experiment_name, "experiment_group_id": group_id, - "agent_name": "sample-agent", - "agent_version": "1.0.0", "dataset_name": "sessions-dataset", "dataset_version": "v1", }, diff --git a/services/intake/tests/integration/spans/test_traces_read.py b/services/intake/tests/integration/spans/test_traces_read.py index bf207aa16e2..bd5b0d9b463 100644 --- a/services/intake/tests/integration/spans/test_traces_read.py +++ b/services/intake/tests/integration/spans/test_traces_read.py @@ -22,11 +22,10 @@ def test_traces_read_returns_core_trace_summary(client: TestClient, make_otlp_re "openinference.span.kind": "AGENT", "gen_ai.conversation.id": "trace-session", "project": "project-a", - "experiment.id": "experiment-a", - "test_case.id": "case-a", - "experiment.run_id": "run-a", - "dataset.name": "dataset-a", - "experiment.metadata": {"split": "dev"}, + "nemo.experiment.id": "experiment-a", + "nemo.test_case.id": "case-a", + "nemo.experiment.run_id": "run-a", + "nemo.experiment.metadata": {"split": "dev"}, "deployment.environment.name": "prod", "tag.tags": ["trace-read"], "metadata": {"owner": "trace-test"}, diff --git a/services/intake/tests/integration/test_experiments_crud.py b/services/intake/tests/integration/test_experiments_crud.py index 7954373c1b0..e96943ba7f9 100644 --- a/services/intake/tests/integration/test_experiments_crud.py +++ b/services/intake/tests/integration/test_experiments_crud.py @@ -19,8 +19,6 @@ def _experiment_body(*, experiment_group_id: str, **overrides: Any) -> dict: body = { "name": "terminal-bench-2_claude-code_opus_baseline", "experiment_group_id": experiment_group_id, - "agent_name": "claude-code", - "agent_version": "0.125.0", "dataset_name": "terminal-bench-2", "dataset_version": "v1", "source_link": "https://example.com/experiments/tb2-baseline", @@ -94,12 +92,12 @@ def test_experiment_update_rejects_immutable_change(client: TestClient) -> None: group = _create_group(client) client.post( EXPERIMENTS, - json=_experiment_body(name="exp-a", agent_name="claude-code", experiment_group_id=group["id"]), + json=_experiment_body(name="exp-a", dataset_name="tb2", experiment_group_id=group["id"]), ) - changed = _experiment_body(name="exp-a", agent_name="cursor", experiment_group_id=group["id"]) + changed = _experiment_body(name="exp-a", dataset_name="tb3", experiment_group_id=group["id"]) resp = client.put(f"{EXPERIMENTS}/exp-a", json=changed) assert resp.status_code == 409, resp.text - assert "agent_name" in resp.json()["detail"] + assert "dataset_name" in resp.json()["detail"] missing = client.put( f"{EXPERIMENTS}/missing", json=_experiment_body(name="missing", experiment_group_id=group["id"]), @@ -115,7 +113,6 @@ def test_experiment_crud_and_empty_rollups(client: TestClient) -> None: exp = created.json() assert exp["name"] == "terminal-bench-2_claude-code_opus_baseline" assert exp["experiment_group_id"] == group["id"] - assert exp["agent_name"] == "claude-code" assert exp["dataset_name"] == "terminal-bench-2" assert exp["source_link"] == "https://example.com/experiments/tb2-baseline" assert exp["metadata"] == {"job_name": "tb2-baseline"} @@ -123,6 +120,8 @@ def test_experiment_crud_and_empty_rollups(client: TestClient) -> None: # Rollups exist on the read model but are empty until ClickHouse hydration lands. assert exp["evaluator_names"] == [] assert exp["model_names"] == [] + assert exp["agent_names"] == [] + assert exp["agent_versions"] == [] assert exp["aggregate_scores"] is None assert exp["run_count"] == 0 @@ -214,42 +213,25 @@ def test_experiment_list_and_scope_to_group(client: TestClient) -> None: assert missing.status_code == 404 -def test_experiment_filter_by_agent_and_dataset_version(client: TestClient) -> None: +def test_experiment_filter_by_dataset_version(client: TestClient) -> None: group = _create_group(client) client.post( EXPERIMENTS, - json=_experiment_body( - name="exp-v1", agent_version="1.0.0", dataset_version="v1", experiment_group_id=group["id"] - ), + json=_experiment_body(name="exp-v1", dataset_version="v1", experiment_group_id=group["id"]), ) client.post( EXPERIMENTS, - json=_experiment_body( - name="exp-v2", agent_version="2.0.0", dataset_version="v1", experiment_group_id=group["id"] - ), + json=_experiment_body(name="exp-v2", dataset_version="v1", experiment_group_id=group["id"]), ) client.post( EXPERIMENTS, - json=_experiment_body( - name="exp-v3", agent_version="2.0.0", dataset_version="v2", experiment_group_id=group["id"] - ), + json=_experiment_body(name="exp-v3", dataset_version="v2", experiment_group_id=group["id"]), ) - by_agent_version = client.get(EXPERIMENTS, params={"filter[agent_version]": "2.0.0"}) - assert by_agent_version.status_code == 200 - assert {e["name"] for e in by_agent_version.json()["data"]} == {"exp-v2", "exp-v3"} - by_dataset_version = client.get(EXPERIMENTS, params={"filter[dataset_version]": "v1"}) assert by_dataset_version.status_code == 200 assert {e["name"] for e in by_dataset_version.json()["data"]} == {"exp-v1", "exp-v2"} - combined = client.get( - EXPERIMENTS, - params={"filter[agent_version]": "2.0.0", "filter[dataset_version]": "v2"}, - ) - assert combined.status_code == 200 - assert {e["name"] for e in combined.json()["data"]} == {"exp-v3"} - def test_experiment_filter_by_created_at_range(client: TestClient) -> None: from datetime import datetime, timedelta, timezone diff --git a/services/intake/tests/test_atif_v17.py b/services/intake/tests/test_atif_v17.py index aa5115bb11b..b3a213fb224 100644 --- a/services/intake/tests/test_atif_v17.py +++ b/services/intake/tests/test_atif_v17.py @@ -26,9 +26,6 @@ "evaluation_id": "eval-sample-agent-baseline", "evaluation_sha": "abc132901", "evaluation_run_id": "evalrun-01JZ8Q7K6V7R3X9N2M4P5A6B7C", - "dataset_id": "sample-dataset", - "dataset_name": "Sample Dataset", - "dataset_version": "v1", "test_case_id": "sample-test-case", "metadata": {"attempt": 1}, } @@ -219,37 +216,31 @@ def test_atif_mapping_writes_evaluation_context_only_on_root_span() -> None: root = next(span for span in spans if span.name == "sample-agent") child = next(span for span in spans if span.name == "user-1") - assert root.attributes_string["experiment.id"] == EVALUATION_CONTEXT["evaluation_id"] - assert root.attributes_string["experiment.sha"] == EVALUATION_CONTEXT["evaluation_sha"] - assert root.attributes_string["experiment.run_id"] == EVALUATION_CONTEXT["evaluation_run_id"] + assert root.attributes_string["nemo.experiment.id"] == EVALUATION_CONTEXT["evaluation_id"] + assert root.attributes_string["nemo.experiment.sha"] == EVALUATION_CONTEXT["evaluation_sha"] + assert root.attributes_string["nemo.experiment.run_id"] == EVALUATION_CONTEXT["evaluation_run_id"] assert "evaluation.id" not in root.attributes_string - assert root.attributes_string["test_case.id"] == EVALUATION_CONTEXT["test_case_id"] - assert root.attributes_string["dataset.id"] == EVALUATION_CONTEXT["dataset_id"] - assert root.attributes_string["dataset.name"] == EVALUATION_CONTEXT["dataset_name"] - assert root.attributes_string["dataset.version"] == EVALUATION_CONTEXT["dataset_version"] - assert json.loads(root.attributes_string["experiment.metadata"]) == EVALUATION_CONTEXT["metadata"] + assert root.attributes_string["nemo.test_case.id"] == EVALUATION_CONTEXT["test_case_id"] + assert json.loads(root.attributes_string["nemo.experiment.metadata"]) == EVALUATION_CONTEXT["metadata"] root_response = Span.from_domain(root) assert root_response.evaluation_context is not None assert root_response.evaluation_context.evaluation_id == EVALUATION_CONTEXT["evaluation_id"] assert root_response.evaluation_context.evaluation_sha == EVALUATION_CONTEXT["evaluation_sha"] assert root_response.evaluation_context.evaluation_run_id == EVALUATION_CONTEXT["evaluation_run_id"] - assert root_response.evaluation_context.dataset_id == EVALUATION_CONTEXT["dataset_id"] - assert root_response.evaluation_context.dataset_name == EVALUATION_CONTEXT["dataset_name"] - assert root_response.evaluation_context.dataset_version == EVALUATION_CONTEXT["dataset_version"] assert root_response.evaluation_context.test_case_id == EVALUATION_CONTEXT["test_case_id"] assert root_response.evaluation_context.metadata == EVALUATION_CONTEXT["metadata"] assert root_response.raw_attributes is not None root_raw = json.loads(root_response.raw_attributes) assert "evaluation_context" not in root_raw assert "evaluation.metadata" not in root_raw - assert "experiment.metadata" not in root_raw + assert "nemo.experiment.metadata" not in root_raw child_response = Span.from_domain(child) assert child_response.evaluation_context is None assert "evaluation.id" not in child.attributes_string - assert "experiment.id" not in child.attributes_string - assert "test_case.id" not in child.attributes_string + assert "nemo.experiment.id" not in child.attributes_string + assert "nemo.test_case.id" not in child.attributes_string def test_atif_mapping_writes_experiment_context_to_experiment_attributes() -> None: @@ -270,8 +261,8 @@ def test_atif_mapping_writes_experiment_context_to_experiment_attributes() -> No ) root = next(span for span in spans if span.name == "sample-agent") - assert root.attributes_string["experiment.id"] == "exp-1" - assert root.attributes_string["test_case.id"] == "case-1" + assert root.attributes_string["nemo.experiment.id"] == "exp-1" + assert root.attributes_string["nemo.test_case.id"] == "case-1" assert "evaluation.id" not in root.attributes_string diff --git a/services/intake/tests/test_experiment_rollup_repository.py b/services/intake/tests/test_experiment_rollup_repository.py index 50327694dee..dcd27093da2 100644 --- a/services/intake/tests/test_experiment_rollup_repository.py +++ b/services/intake/tests/test_experiment_rollup_repository.py @@ -52,6 +52,8 @@ async def test_experiment_rollups_anchor_on_root_session_membership(): ( "exp-a", ["model-b", "model-a"], + ["agent-a"], + ["1.0.0", "1.0.1"], 0.65, 0.1625, 0.2, @@ -71,6 +73,8 @@ async def test_experiment_rollups_anchor_on_root_session_membership(): [ "experiment_id", "model_names", + "agent_names", + "agent_versions", "cost_sum", "cost_mean", "cost_median", @@ -97,6 +101,8 @@ async def test_experiment_rollups_anchor_on_root_session_membership(): assert rollup.run_count == 3 assert rollup.evaluator_names == ["harbor.verifier"] assert rollup.model_names == ["model-a", "model-b"] + assert rollup.agent_names == ["agent-a"] + assert rollup.agent_versions == ["1.0.0", "1.0.1"] assert rollup.evaluator_scores["harbor.verifier"].sum == 3.0 assert rollup.evaluator_scores["harbor.verifier"].mean == 0.75 assert rollup.evaluator_scores["harbor.verifier"].median == 0.8 diff --git a/services/intake/tests/test_spans_clickhouse_migrations.py b/services/intake/tests/test_spans_clickhouse_migrations.py index dbd5ea6a56d..a315be4a8ba 100644 --- a/services/intake/tests/test_spans_clickhouse_migrations.py +++ b/services/intake/tests/test_spans_clickhouse_migrations.py @@ -72,5 +72,5 @@ def test_trace_index_schema_is_root_span_projection(): def test_trace_index_mv_keys_match_attribute_catalog(): - assert spec_for_field(SpanAttributeField.EVALUATION_ID).bag_key == "experiment.id" - assert spec_for_field(SpanAttributeField.TEST_CASE_ID).bag_key == "test_case.id" + assert spec_for_field(SpanAttributeField.EVALUATION_ID).bag_key == "nemo.experiment.id" + assert spec_for_field(SpanAttributeField.TEST_CASE_ID).bag_key == "nemo.test_case.id" diff --git a/services/intake/tests/test_spans_schemas.py b/services/intake/tests/test_spans_schemas.py index 4a0298d85e8..a178743f420 100644 --- a/services/intake/tests/test_spans_schemas.py +++ b/services/intake/tests/test_spans_schemas.py @@ -42,10 +42,10 @@ def test_span_response_raw_attributes_merges_atif_raw_with_unknown_attributes(): event_ts=now, attributes_string={ "atif.raw": json_dumps_preserve( - {"source_session_id": "session-a", "experiment.metadata": {"source": "atif.raw"}} + {"source_session_id": "session-a", "nemo.experiment.metadata": {"source": "atif.raw"}} ), "custom.string": "value-a", - "experiment.metadata": json.dumps({"source": "attribute.bag"}), + "nemo.experiment.metadata": json.dumps({"source": "attribute.bag"}), "gen_ai.request.model": "model-a", }, attributes_number={"custom.number": 1.25, "llm.token_count.prompt": 42}, diff --git a/services/intake/tests/test_spans_span_attribute_catalog.py b/services/intake/tests/test_spans_span_attribute_catalog.py index cb706597ee4..62bbfa8c3f7 100644 --- a/services/intake/tests/test_spans_span_attribute_catalog.py +++ b/services/intake/tests/test_spans_span_attribute_catalog.py @@ -166,17 +166,6 @@ def test_span_attribute_catalog_predicates_include_existence_guard(operator: str assert params["cost_total_usd_value"] == 1_000_000 -def test_span_attribute_catalog_groups_legacy_alias_predicates(): - sql, params = where_clause("evaluation_run_id", "$eq", "run-a", param_prefix="root_candidate_0") - - assert sql.startswith("(") - assert sql.endswith(")") - assert " OR " in sql - assert params["root_candidate_0_key"] == "experiment.run_id" - assert params["root_candidate_0_key_1"] == "evaluation.run_id" - assert params["root_candidate_0_value"] == "run-a" - - def test_span_attribute_catalog_rejects_ordering_on_string_fields(): with pytest.raises(ValueError, match="only supports equality"): where_clause("model", ">", "gpt-4") diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx index 631913416db..f5bdd28764c 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx @@ -124,21 +124,17 @@ export const ExperimentGroupDataView: FC = ({ ); }, }), - accessor('agent_name', { - header: 'Agent Name', + accessor((original) => original.agent_names?.join(', '), { + id: 'agent_names', + header: 'Agent Names', enableSorting: false, - meta: { - filter: { type: 'text', label: 'Agent Name', placeholder: 'Filter by Agent Name' }, - }, - cell: ({ row }) => {row.original.agent_name || '-'}, + cell: ({ getValue }) => {getValue() || '-'}, }), - accessor('agent_version', { - header: 'Agent Version', + accessor((original) => original.agent_versions?.join(', '), { + id: 'agent_versions', + header: 'Agent Versions', enableSorting: false, - meta: { - filter: { type: 'text', label: 'Agent Version', placeholder: 'Filter by Agent Version' }, - }, - cell: ({ row }) => {row.original.agent_version || '-'}, + cell: ({ getValue }) => {getValue() || '-'}, }), accessor('dataset_name', { header: 'Dataset Name', diff --git a/web/packages/studio/src/routes/ExperimentDetailRoute/ExperimentDetailMetrics.tsx b/web/packages/studio/src/routes/ExperimentDetailRoute/ExperimentDetailMetrics.tsx index f3211635e42..14f48d63234 100644 --- a/web/packages/studio/src/routes/ExperimentDetailRoute/ExperimentDetailMetrics.tsx +++ b/web/packages/studio/src/routes/ExperimentDetailRoute/ExperimentDetailMetrics.tsx @@ -27,8 +27,8 @@ export const ExperimentDetailMetrics: FC = ({ expe return (
diff --git a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/ExperimentGroupMetrics.tsx b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/ExperimentGroupMetrics.tsx index ba3dcc2b7c6..0fe06c0230d 100644 --- a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/ExperimentGroupMetrics.tsx +++ b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/ExperimentGroupMetrics.tsx @@ -6,8 +6,7 @@ import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; import { useGetExperimentGroup } from '@nemo/sdk/generated/platform/api'; import { Divider } from '@nvidia/foundations-react-core'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; -import { useAllGroupExperiments } from '@studio/routes/ExperimentGroupDetailRoute/useAllGroupExperiments'; -import { type FC, useMemo } from 'react'; +import { type FC } from 'react'; interface ExperimentGroupMetricsProps { experimentGroupName: string; @@ -18,34 +17,9 @@ export const ExperimentGroupMetrics: FC = ({ }) => { const workspace = useWorkspaceFromPath(); const { data: group } = useGetExperimentGroup(workspace, experimentGroupName); - const { experiments, isFetching } = useAllGroupExperiments(workspace, group?.id ?? ''); - - const agentNames = useMemo( - () => [...new Set(experiments.map((e) => e.agent_name).filter(Boolean))].join(', '), - [experiments] - ); - - const datasetNames = useMemo( - () => [...new Set(experiments.map((e) => e.dataset_name).filter(Boolean))].join(', '), - [experiments] - ); return (
- - - - : undefined} diff --git a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/useAllGroupExperiments.ts b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/useAllGroupExperiments.ts deleted file mode 100644 index a7f69b1f8fe..00000000000 --- a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/useAllGroupExperiments.ts +++ /dev/null @@ -1,55 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { listExperiments } from '@nemo/sdk/generated/platform/api'; -import type { - ExperimentResponse, - ExperimentResponsesPage, -} from '@nemo/sdk/generated/platform/schema'; -import { useInfiniteQuery } from '@tanstack/react-query'; -import { useEffect, useMemo } from 'react'; - -const PAGE_SIZE = 100; - -interface UseAllGroupExperimentsResult { - experiments: ExperimentResponse[]; - isFetching: boolean; -} - -/** Fetches every experiment belonging to a group, auto-paging until exhausted. */ -export const useAllGroupExperiments = ( - workspace: string, - experimentGroupId: string -): UseAllGroupExperimentsResult => { - const { data, isFetching, isError, hasNextPage, fetchNextPage } = useInfiniteQuery({ - queryKey: ['experiment-group-experiments', workspace, experimentGroupId], - queryFn: ({ pageParam, signal }) => - listExperiments( - workspace, - { - page: pageParam, - page_size: PAGE_SIZE, - filter: { experiment_group_id: experimentGroupId }, - }, - signal - ), - initialPageParam: 1, - getNextPageParam: (lastPage: ExperimentResponsesPage) => { - const { page, total_pages } = lastPage.pagination ?? {}; - return page !== undefined && total_pages !== undefined && page < total_pages - ? page + 1 - : undefined; - }, - enabled: !!experimentGroupId, - }); - - useEffect(() => { - if (!isFetching && !isError && hasNextPage) { - void fetchNextPage(); - } - }, [isFetching, isError, hasNextPage, fetchNextPage]); - - const experiments = useMemo(() => data?.pages.flatMap((page) => page.data) ?? [], [data?.pages]); - - return { experiments, isFetching }; -}; diff --git a/web/packages/studio/src/util/intakeTelemetry.ts b/web/packages/studio/src/util/intakeTelemetry.ts index 292659a6067..5b7baa2847d 100644 --- a/web/packages/studio/src/util/intakeTelemetry.ts +++ b/web/packages/studio/src/util/intakeTelemetry.ts @@ -50,13 +50,7 @@ export const getSpanDisplayName = (span: Span): string => { export const getSpanSubject = (span: Span): string => { return ( - span.tool_name || - span.model || - span.agent_name || - span.provider || - span.prompt_name || - span.project || - span.kind + span.tool_name || span.model || span.agent_name || span.provider || span.project || span.kind ); }; @@ -71,13 +65,7 @@ export const getEvaluationContextSummary = ( context: SpanEvaluationContext | null | undefined ): string => { if (!context) return EMPTY_VALUE; - return ( - context.evaluation_run_id || - context.evaluation_id || - context.dataset_name || - context.test_case_id || - EMPTY_VALUE - ); + return context.evaluation_run_id || context.evaluation_id || context.test_case_id || EMPTY_VALUE; }; export const hasEvaluationContext = (context: SpanEvaluationContext | null | undefined): boolean => @@ -86,9 +74,6 @@ export const hasEvaluationContext = (context: SpanEvaluationContext | null | und (context.evaluation_id || context.evaluation_sha || context.evaluation_run_id || - context.dataset_id || - context.dataset_name || - context.dataset_version || context.test_case_id || (context.metadata && Object.keys(context.metadata).length > 0)) );