From 226109c28aac185e1011e8bf6bbe48dd4677cf2b Mon Sep 17 00:00:00 2001 From: Aditya Pandey Date: Thu, 23 Jul 2026 14:41:19 -0600 Subject: [PATCH 1/6] feat(insights): add experiment-linked Studio insights Signed-off-by: Aditya Pandey --- openapi/ga/individual/platform.openapi.yaml | 9 +- openapi/ga/openapi.yaml | 9 +- openapi/openapi.yaml | 9 +- .../nemo-insights/scripts/insights_demo.py | 514 ++++++++++++++++++ .../src/nemo_insights_plugin/schema.py | 15 +- .../src/nemo_insights_plugin/service.py | 45 +- .../tests/test_insights_list_contract.py | 88 +++ .../intake/api/v2/experiments/endpoints.py | 4 +- .../nmp/intake/api/v2/experiments/schemas.py | 4 + .../src/nmp/intake/spans/clickhouse_client.py | 9 +- .../intake/src/nmp/intake/spans/service.py | 13 + .../src/nmp/intake/spans/trace_repository.py | 52 ++ .../integration/test_experiments_crud.py | 13 + .../test_traces_clickhouse_repository.py | 45 +- .../studio/src/nmp/studio/env_mappings.py | 5 + web/packages/studio/env/.env.dev.local.sample | 1 + web/packages/studio/env/.env.fastapi | 1 + web/packages/studio/src/api/optimizer.ts | 180 ++++++ web/packages/studio/src/assets/voyager.svg | 162 ++++++ .../src/components/ChangesetBadge/index.tsx | 25 + .../IntakeLists/IntakeTracesTable.tsx | 111 +--- .../IntakeLists/intakeTraceColumns.tsx | 122 +++++ .../OriginatingInsightLink/index.tsx | 36 ++ .../ExperimentGroupDataView/index.tsx | 11 + .../studio/src/constants/environment.ts | 1 + .../constants/featureFlags/featureFlags.ts | 1 + web/packages/studio/src/constants/routes.ts | 3 + .../EvaluationDetailMetrics.tsx | 12 + .../EvaluationDetailRoute/index.test.tsx | 92 ++++ .../routes/EvaluationDetailRoute/index.tsx | 33 +- .../optimizerDisabled.test.tsx | 83 +++ .../ExperimentGroupDetailRoute/index.test.tsx | 69 +++ .../ExperimentGroupDetailRoute/index.tsx | 48 +- .../WorkspaceLayout/WorkspaceSideNav.test.tsx | 25 + .../WorkspaceLayout/WorkspaceSideNav.tsx | 19 +- .../studio/src/routes/groups/index.ts | 1 + .../src/routes/groups/optimizerRoutes.tsx | 42 ++ web/packages/studio/src/routes/index.test.tsx | 13 +- web/packages/studio/src/routes/index.tsx | 2 + .../optimizer/InsightOpenModal/command.ts | 12 + .../optimizer/InsightOpenModal/index.test.ts | 19 + .../optimizer/InsightOpenModal/index.tsx | 63 +++ .../InsightTracesTable/index.test.tsx | 111 ++++ .../optimizer/InsightTracesTable/index.tsx | 106 ++++ .../InsightExperimentGroups.tsx | 139 +++++ .../OptimizerInsightRoute/index.test.tsx | 192 +++++++ .../optimizer/OptimizerInsightRoute/index.tsx | 217 ++++++++ .../optimizer/OptimizerRoute/index.test.tsx | 86 +++ .../routes/optimizer/OptimizerRoute/index.tsx | 175 ++++++ .../routes/optimizer/insightStatus.test.ts | 19 + .../src/routes/optimizer/insightStatus.ts | 42 ++ web/packages/studio/src/routes/utils.ts | 12 + .../studio/src/tests/title-change.test.tsx | 1 + 53 files changed, 2993 insertions(+), 128 deletions(-) create mode 100755 plugins/nemo-insights/scripts/insights_demo.py create mode 100644 plugins/nemo-insights/tests/test_insights_list_contract.py create mode 100644 web/packages/studio/src/api/optimizer.ts create mode 100644 web/packages/studio/src/assets/voyager.svg create mode 100644 web/packages/studio/src/components/ChangesetBadge/index.tsx create mode 100644 web/packages/studio/src/components/IntakeLists/intakeTraceColumns.tsx create mode 100644 web/packages/studio/src/components/OriginatingInsightLink/index.tsx create mode 100644 web/packages/studio/src/routes/EvaluationDetailRoute/index.test.tsx create mode 100644 web/packages/studio/src/routes/EvaluationDetailRoute/optimizerDisabled.test.tsx create mode 100644 web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.test.tsx create mode 100644 web/packages/studio/src/routes/WorkspaceLayout/WorkspaceSideNav.test.tsx create mode 100644 web/packages/studio/src/routes/groups/optimizerRoutes.tsx create mode 100644 web/packages/studio/src/routes/optimizer/InsightOpenModal/command.ts create mode 100644 web/packages/studio/src/routes/optimizer/InsightOpenModal/index.test.ts create mode 100644 web/packages/studio/src/routes/optimizer/InsightOpenModal/index.tsx create mode 100644 web/packages/studio/src/routes/optimizer/InsightTracesTable/index.test.tsx create mode 100644 web/packages/studio/src/routes/optimizer/InsightTracesTable/index.tsx create mode 100644 web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/InsightExperimentGroups.tsx create mode 100644 web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.test.tsx create mode 100644 web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.tsx create mode 100644 web/packages/studio/src/routes/optimizer/OptimizerRoute/index.test.tsx create mode 100644 web/packages/studio/src/routes/optimizer/OptimizerRoute/index.tsx create mode 100644 web/packages/studio/src/routes/optimizer/insightStatus.test.ts create mode 100644 web/packages/studio/src/routes/optimizer/insightStatus.ts diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index a07756d247..829a0150ee 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -4104,8 +4104,9 @@ paths: explode: true schema: $ref: '#/components/schemas/ExperimentGroupFilter' - description: 'Filter experiment groups by name, or by a metadata key/value: - filter[metadata.]=.' + description: Filter experiment groups by name, insight_id, is_deleted, or + a metadata key/value (filter[metadata.]=). Pass is_deleted=true + to return only soft-deleted groups; omit to see only live ones. responses: '200': description: Successful Response @@ -11237,6 +11238,10 @@ components: description: Filter groups by name. title: Name type: string + insight_id: + description: Filter groups by the id of the insight that seeded them. + title: Insight Id + type: string is_deleted: description: When true, returns only soft-deleted groups. Omit (or false) to see only live groups. diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index a07756d247..829a0150ee 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -4104,8 +4104,9 @@ paths: explode: true schema: $ref: '#/components/schemas/ExperimentGroupFilter' - description: 'Filter experiment groups by name, or by a metadata key/value: - filter[metadata.]=.' + description: Filter experiment groups by name, insight_id, is_deleted, or + a metadata key/value (filter[metadata.]=). Pass is_deleted=true + to return only soft-deleted groups; omit to see only live ones. responses: '200': description: Successful Response @@ -11237,6 +11238,10 @@ components: description: Filter groups by name. title: Name type: string + insight_id: + description: Filter groups by the id of the insight that seeded them. + title: Insight Id + type: string is_deleted: description: When true, returns only soft-deleted groups. Omit (or false) to see only live groups. diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index a07756d247..829a0150ee 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -4104,8 +4104,9 @@ paths: explode: true schema: $ref: '#/components/schemas/ExperimentGroupFilter' - description: 'Filter experiment groups by name, or by a metadata key/value: - filter[metadata.]=.' + description: Filter experiment groups by name, insight_id, is_deleted, or + a metadata key/value (filter[metadata.]=). Pass is_deleted=true + to return only soft-deleted groups; omit to see only live ones. responses: '200': description: Successful Response @@ -11237,6 +11238,10 @@ components: description: Filter groups by name. title: Name type: string + insight_id: + description: Filter groups by the id of the insight that seeded them. + title: Insight Id + type: string is_deleted: description: When true, returns only soft-deleted groups. Omit (or false) to see only live groups. diff --git a/plugins/nemo-insights/scripts/insights_demo.py b/plugins/nemo-insights/scripts/insights_demo.py new file mode 100755 index 0000000000..fa4ec803e4 --- /dev/null +++ b/plugins/nemo-insights/scripts/insights_demo.py @@ -0,0 +1,514 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Seed or clean a local NeMo Insights Studio demo through public HTTP APIs. + +From the repository root:: + + uv sync --group insights + services/intake/scripts/spans/run_clickhouse.sh + cd web && VITE_FF_OPTIMIZER_ENABLED=preview pnpm --filter nemo-studio-ui build:fastapi && cd .. + uv run nemo services run \ + --service-group all --config packages/nmp_platform/config/local.yaml + uv run plugins/nemo-insights/scripts/insights_demo.py seed + +Open ``http://localhost:8080/studio/workspaces/insights-demo/optimizer``. +Clean with ``uv run plugins/nemo-insights/scripts/insights_demo.py clean``. +This Insights UI is separate from the existing Agents ``Suggestions`` feature. +The script deletes only the fixed ``insights-demo`` workspace. Intake has no +public telemetry-delete API, so deterministic session IDs keep reseeding stable +when inaccessible ClickHouse rows remain. +""" + +from __future__ import annotations + +import argparse +import os +import sys +import time +from collections.abc import Callable +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Any, Literal + +import httpx + +DEMO_WORKSPACE = "insights-demo" +DEFAULT_BASE_URL = "http://localhost:8080" +CLICKHOUSE_RECOVERY_COMMAND = "services/intake/scripts/spans/run_clickhouse.sh" +INSIGHTS_INSTALL_COMMAND = "uv sync --group insights" +_BASE_TIME = datetime(2026, 7, 20, 12, 0, tzinfo=timezone.utc) +_SOURCE_URL = "https://github.com/NVIDIA-NeMo/nemo-platform/commit" +_SUPPORT_AGENT = ("support-agent", "nvidia/nemotron-mini") +_RETRIEVAL_AGENT = ("retrieval-agent", "nvidia/llama-3.3-nemotron-super") + + +def _workspace_path(service: str, suffix: str = "") -> str: + return f"/apis/{service}/v2/workspaces/{DEMO_WORKSPACE}{suffix}" + + +class DemoError(RuntimeError): + """A concise, actionable demo failure.""" + + +@dataclass(frozen=True) +class SessionSpec: + session_id: str + test_case_id: str + started_at: datetime + latency_ms: int + cost_usd: float + quality: float + correctness: float + + +@dataclass(frozen=True) +class EvaluationSpec: + name: str + source_link: str + agent: tuple[str, str] + sessions: tuple[SessionSpec, ...] + + +@dataclass(frozen=True) +class GroupSpec: + name: str + description: str + summary: str + default_sort: str + insight_key: str + evaluations: tuple[EvaluationSpec, ...] + + +@dataclass(frozen=True) +class InsightSpec: + key: str + title: str + description: str + status: Literal["open", "resolved", "deleted"] + trace_refs: tuple[str, ...] + + +@dataclass(frozen=True) +class DemoFixture: + insights: tuple[InsightSpec, ...] + groups: tuple[GroupSpec, ...] + + +def _evaluation( + name: str, + commit: str, + count: int, + *, + start_index: int, + latency_ms: int, + cost_usd: float, + quality: float, + correctness: float, + agent: tuple[str, str], +) -> EvaluationSpec: + return EvaluationSpec( + name=name, + source_link=f"{_SOURCE_URL}/{commit}", + agent=agent, + sessions=tuple( + SessionSpec( + session_id=f"insights-demo-{name}-{index + 1:02d}", + test_case_id=f"case-{index + 1:02d}", + started_at=_BASE_TIME + timedelta(minutes=7 * (start_index + index)), + latency_ms=latency_ms + index * 125, + cost_usd=round(cost_usd + index * 0.002, 3), + quality=round(min(1.0, quality + index * 0.02), 2), + correctness=round(min(1.0, correctness + index * 0.01), 2), + ) + for index in range(count) + ), + ) + + +def build_fixture() -> DemoFixture: + """Return the deterministic, compact demo fixture.""" + prompt_baseline = _evaluation( + "prompt-baseline", + "1111111", + 3, + start_index=0, + latency_ms=1850, + cost_usd=0.032, + quality=0.72, + correctness=0.79, + agent=_SUPPORT_AGENT, + ) + prompt_compact = _evaluation( + "prompt-compact-context", + "2222222", + 3, + start_index=3, + latency_ms=1120, + cost_usd=0.021, + quality=0.84, + correctness=0.88, + agent=_SUPPORT_AGENT, + ) + router_baseline = _evaluation( + "router-baseline", + "3333333", + 3, + start_index=6, + latency_ms=2300, + cost_usd=0.041, + quality=0.68, + correctness=0.76, + agent=_RETRIEVAL_AGENT, + ) + router_streaming = _evaluation( + "router-streaming-cache", + "4444444", + 2, + start_index=9, + latency_ms=940, + cost_usd=0.027, + quality=0.89, + correctness=0.91, + agent=_RETRIEVAL_AGENT, + ) + groups = ( + GroupSpec( + name="prompt-response-time", + description="Prompt experiments addressing slow support responses.", + summary="Compact context improved quality while reducing average latency and cost.", + default_sort="-evaluators.quality.mean,cost_usd.mean", + insight_key="slow-responses", + evaluations=(prompt_baseline, prompt_compact), + ), + GroupSpec( + name="retrieval-routing", + description="Routing experiments for retrieval-heavy support questions.", + summary="Streaming cache-aware retrieval produced the strongest latency result.", + default_sort="latency_ms.mean,-evaluators.correctness.mean", + insight_key="slow-responses", + evaluations=(router_baseline, router_streaming), + ), + ) + trace_refs = tuple( + session.session_id for group in groups for evaluation in group.evaluations for session in evaluation.sessions + ) + return DemoFixture( + insights=( + InsightSpec( + key="slow-responses", + title="Support responses are slow on retrieval-heavy requests", + description=( + "Evidence traces show retrieval and oversized prompt context dominate response time. " + "Compare prompt compaction and cache-aware routing experiments." + ), + status="open", + trace_refs=trace_refs, + ), + InsightSpec( + key="stable-quality", + title="Answer quality remains stable after prompt compaction", + description=("Resolved after compact-context evaluations preserved correctness while lowering cost."), + status="resolved", + trace_refs=trace_refs[:2], + ), + InsightSpec( + key="legacy-router", + title="Legacy routing recommendation is no longer actionable", + description="Deleted after cache-aware routing superseded the original recommendation.", + status="deleted", + trace_refs=trace_refs[-1:], + ), + ), + groups=groups, + ) + + +class DemoAPI: + """Small public-HTTP client for the APIs used by the demo.""" + + def __init__(self, base_url: str, *, client: httpx.Client | None = None) -> None: + self.base_url = base_url.rstrip("/") + self._owns_client = client is None + self.client = client or httpx.Client(timeout=15.0) + + def close(self) -> None: + if self._owns_client: + self.client.close() + + def __enter__(self) -> DemoAPI: + return self + + def __exit__(self, *_args: object) -> None: + self.close() + + def _request( + self, + method: str, + path: str, + *, + expected: tuple[int, ...] = (200,), + json: dict[str, Any] | None = None, + params: dict[str, Any] | None = None, + ) -> httpx.Response: + try: + response = self.client.request( + method, + f"{self.base_url}{path}", + json=json, + params=params, + ) + except httpx.RequestError as error: + raise DemoError(f"Platform is unavailable at {self.base_url}: {error}") from error + if response.status_code not in expected: + detail = response.text.strip() + suffix = f": {detail}" if detail else "" + raise DemoError(f"{method} {path} failed ({response.status_code}){suffix}") + return response + + def preflight(self) -> None: + self._request("GET", "/health/ready") + openapi = self._request("GET", "/openapi.json").json() + paths = openapi.get("paths", {}) + if "/apis/insights/v2/workspaces/{workspace}/insights" not in paths: + raise DemoError( + f"Insights service is unavailable. Install the optional plugin with: {INSIGHTS_INSTALL_COMMAND}" + ) + response = self._request( + "GET", + "/apis/intake/v2/workspaces/default/traces", + expected=(200, 404, 503), + params={"page": 1, "page_size": 1, "mode": "summary"}, + ) + if response.status_code != 200: + raise DemoError(f"Intake is unavailable. Start ClickHouse with: {CLICKHOUSE_RECOVERY_COMMAND}") + + def delete_workspace(self, *, sleep: Callable[[float], None]) -> None: + path = _workspace_path("entities") + self._request( + "DELETE", + path, + expected=(200, 404), + ) + for _ in range(20): + response = self._request("GET", path, expected=(200, 404)) + if response.status_code == 404: + return + sleep(0.25) + raise DemoError(f"Timed out deleting workspace '{DEMO_WORKSPACE}'") + + def create_workspace(self, *, sleep: Callable[[float], None]) -> None: + for _ in range(40): + response = self._request( + "POST", + "/apis/entities/v2/workspaces", + expected=(200, 201, 409), + json={ + "name": DEMO_WORKSPACE, + "description": "Deterministic local NeMo Insights Studio demo.", + }, + ) + if response.status_code != 409: + return + sleep(0.5) + raise DemoError(f"Timed out recreating workspace '{DEMO_WORKSPACE}'") + + def create_insight(self, insight: InsightSpec) -> str: + response = self._request( + "POST", + _workspace_path("insights", "/insights"), + expected=(201,), + json={ + "title": insight.title, + "description": insight.description, + "agent": "insights-demo-agent", + "status": insight.status, + "trace_refs": [], + }, + ) + return str(response.json()["id"]) + + def update_insight_traces(self, insight_id: str, trace_refs: tuple[str, ...]) -> None: + self._request( + "PATCH", + _workspace_path("insights", f"/insights/{insight_id}"), + json={"trace_refs": list(trace_refs)}, + ) + + def create_group(self, group: GroupSpec, insight_id: str) -> str: + response = self._request( + "POST", + _workspace_path("intake", "/experiment-groups"), + expected=(201,), + json={ + "name": group.name, + "description": group.description, + "summary": group.summary, + "insight_id": insight_id, + "default_sort": group.default_sort, + }, + ) + return str(response.json()["id"]) + + def create_evaluation(self, group_id: str, evaluation: EvaluationSpec) -> None: + self._request( + "POST", + _workspace_path("intake", "/evaluations"), + expected=(201,), + json={ + "name": evaluation.name, + "experiment_group_id": group_id, + "dataset_name": "insights-demo-cases", + "source_link": evaluation.source_link, + }, + ) + + def ingest_session(self, evaluation: EvaluationSpec, session: SessionSpec) -> None: + finished_at = session.started_at + timedelta(milliseconds=session.latency_ms) + agent_name, model_name = evaluation.agent + self._request( + "POST", + _workspace_path("intake", "/ingest/atif"), + expected=(201,), + json={ + "schema_version": "ATIF-v1.7", + "session_id": session.session_id, + "evaluation_context": { + "evaluation_id": evaluation.name, + "test_case_id": session.test_case_id, + }, + "extra": { + "verifier": { + "started_at": _iso(session.started_at), + "finished_at": _iso(finished_at), + }, + "verifier_result": { + "rewards": { + "quality": session.quality, + "correctness": session.correctness, + } + }, + }, + "agent": { + "name": agent_name, + "version": "1.0.0", + "model_name": model_name, + }, + "steps": [ + { + "step_id": 1, + "timestamp": _iso(session.started_at), + "source": "user", + "message": f"Investigate support request {session.test_case_id}.", + }, + { + "step_id": 2, + "timestamp": _iso(finished_at), + "source": "agent", + "model_name": model_name, + "message": "Retrieved current evidence and returned a grounded answer.", + "metrics": { + "prompt_tokens": 320, + "completion_tokens": 96, + "cost_usd": session.cost_usd, + }, + }, + ], + }, + ) + + def evaluation_is_ready(self, evaluation: EvaluationSpec) -> bool: + response = self._request( + "GET", + _workspace_path("intake", f"/evaluations/{evaluation.name}"), + ).json() + return ( + response.get("run_count") == len(evaluation.sessions) + and bool(response.get("aggregate_scores")) + and response.get("cost_usd") is not None + and response.get("latency_ms") is not None + ) + + +def clean( + api: DemoAPI, + *, + sleep: Callable[[float], None] = time.sleep, + quiet: bool = False, +) -> None: + """Delete only the dedicated demo workspace.""" + api.delete_workspace(sleep=sleep) + if not quiet: + print(f"Deleted workspace '{DEMO_WORKSPACE}'.") + print("ClickHouse telemetry remains physically stored; public Intake APIs expose no delete operation.") + + +def seed( + api: DemoAPI, + *, + sleep: Callable[[float], None] = time.sleep, +) -> None: + """Recreate and seed the deterministic demo workspace.""" + api.preflight() + clean(api, sleep=sleep, quiet=True) + api.create_workspace(sleep=sleep) + + fixture = build_fixture() + insight_ids = {insight.key: api.create_insight(insight) for insight in fixture.insights} + evaluations: list[EvaluationSpec] = [] + for group in fixture.groups: + group_id = api.create_group(group, insight_ids[group.insight_key]) + for evaluation in group.evaluations: + api.create_evaluation(group_id, evaluation) + evaluations.append(evaluation) + for session in evaluation.sessions: + api.ingest_session(evaluation, session) + + for insight in fixture.insights: + api.update_insight_traces(insight_ids[insight.key], insight.trace_refs) + + pending = {evaluation.name: evaluation for evaluation in evaluations} + for _ in range(30): + pending = {name: evaluation for name, evaluation in pending.items() if not api.evaluation_is_ready(evaluation)} + if not pending: + break + sleep(1.0) + if pending: + raise DemoError(f"Timed out waiting for evaluation rollups: {', '.join(sorted(pending))}") + + print( + f"Seeded workspace '{DEMO_WORKSPACE}' with {len(fixture.insights)} insights, " + f"{len(fixture.groups)} groups, {len(evaluations)} evaluations, and 11 traces." + ) + print(f"Open {api.base_url}/studio/workspaces/{DEMO_WORKSPACE}/optimizer") + + +def _iso(value: datetime) -> str: + return value.isoformat().replace("+00:00", "Z") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("seed", "clean")) + parser.add_argument( + "--base-url", + default=os.getenv("NMP_BASE_URL", DEFAULT_BASE_URL), + help=f"Platform base URL (default: NMP_BASE_URL or {DEFAULT_BASE_URL}).", + ) + args = parser.parse_args(argv) + + try: + with DemoAPI(args.base_url) as api: + if args.command == "seed": + seed(api) + else: + clean(api) + except DemoError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/schema.py b/plugins/nemo-insights/src/nemo_insights_plugin/schema.py index 0a304718fb..23aadd7104 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/schema.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/schema.py @@ -50,7 +50,20 @@ class UpdateInsightRequest(BaseModel): trace_refs: list[str] | None = None -InsightPage = NemoListResponse[Insight] +class InsightListItem(Insight, entity_type="insights_insight"): + """Insight representation used only in the paginated list response.""" + + experiment_group_count: int | None = Field( + default=None, + description="Number of live experiment groups linked to this insight.", + ) + last_seen_at: datetime | None = Field( + default=None, + description="Newest start timestamp among the insight's currently referenced traces.", + ) + + +InsightPage = NemoListResponse[InsightListItem] class UpdateAnalysisConfigRequest(BaseModel): diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/service.py b/plugins/nemo-insights/src/nemo_insights_plugin/service.py index 583ff6b18a..ae70b231f9 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/service.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/service.py @@ -24,6 +24,7 @@ AnalysisConfigPage, AnalysisRunStatusPage, CreateInsightRequest, + InsightListItem, InsightPage, UpdateAnalysisConfigRequest, UpdateAnalysisRunStatusRequest, @@ -42,10 +43,19 @@ from nemo_platform_plugin.jobs.routes import add_job_routes from nemo_platform_plugin.schema import PaginationData from nemo_platform_plugin.service import NemoService, RouterSpec +from nmp.intake.entities.experiments import ExperimentGroup +from nmp.intake.spans.api.dependencies import SpansServiceDep logger = logging.getLogger(__name__) +def _to_list_item(insight: Insight) -> InsightListItem: + item = InsightListItem.model_validate(insight.model_dump(exclude_computed_fields=True)) + if insight.__pydantic_private__ is not None: + item.__pydantic_private__ = insight.__pydantic_private__.copy() + return item + + class InsightsService(NemoService): """NeMo Insights plugin service. @@ -55,7 +65,7 @@ class InsightsService(NemoService): """ name: ClassVar[str] = "insights" - dependencies: ClassVar[list[str]] = ["entities", "jobs"] + dependencies: ClassVar[list[str]] = ["entities", "jobs", "intake"] def get_routers(self) -> list[RouterSpec]: return [ @@ -129,6 +139,7 @@ async def create_insight( @path_rule(callers=[CallerKind.PRINCIPAL], permissions=[InsightPerms.LIST]) async def list_insights( workspace: str, + spans_service: SpansServiceDep, page: int = Query(default=1, ge=1, description="Page number (1-indexed)."), page_size: int = Query(default=20, ge=1, le=100, description="Items per page."), sort: str = Query( @@ -159,9 +170,39 @@ async def list_insights( logger.exception("Failed to list insights") raise HTTPException(status_code=500, detail="Failed to list insights.") from exc + items = [_to_list_item(insight) for insight in result.data] + if items: + insight_ids = [item.id for item in items] + try: + counts = await entity_client.count_by( + ExperimentGroup, + "insight_id", + workspace=workspace, + filter_obj={ + "insight_id": {"$in": insight_ids}, + "is_deleted": False, + }, + ) + except Exception: + logger.exception("Failed to count experiment groups for insights") + else: + for item in items: + item.experiment_group_count = counts.get(item.id, 0) + + try: + last_seen_at = await spans_service.latest_trace_started_at_by_group( + workspace=workspace, + trace_refs_by_group={item.id: item.trace_refs for item in items}, + ) + except Exception: + logger.exception("Failed to find latest traces for insights") + else: + for item in items: + item.last_seen_at = last_seen_at.get(item.id) + pagination = PaginationData.model_validate(result.pagination.model_dump()) if result.pagination else None return InsightPage( - data=result.data, + data=items, pagination=pagination, sort=sort, filter=filter_obj or None, diff --git a/plugins/nemo-insights/tests/test_insights_list_contract.py b/plugins/nemo-insights/tests/test_insights_list_contract.py new file mode 100644 index 0000000000..e2162cbbcb --- /dev/null +++ b/plugins/nemo-insights/tests/test_insights_list_contract.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock + +from fastapi import FastAPI +from fastapi.testclient import TestClient +from nemo_insights_plugin.entities import Insight +from nemo_insights_plugin.service import InsightsService +from nemo_platform_plugin.entity_client import NemoPaginationInfo, get_entity_client +from nmp.intake.entities.experiments import ExperimentGroup +from nmp.intake.spans.api.dependencies import get_spans_service + + +def _insight(name: str, entity_id: str) -> Insight: + insight = Insight( + name=name, + workspace="default", + title=f"Title for {name}", + agent="test-agent", + description=f"Description for {name}", + ) + insight._id = entity_id + return insight + + +def _app(entity_client: AsyncMock, spans_service: AsyncMock) -> FastAPI: + app = FastAPI() + for spec in InsightsService().get_routers(): + app.include_router(spec.router, prefix=spec.prefix) + app.dependency_overrides[get_entity_client] = lambda: entity_client + app.dependency_overrides[get_spans_service] = lambda: spans_service + return app + + +def test_list_insights_enriches_the_page_with_counts_and_last_seen_at() -> None: + entity_client = AsyncMock() + spans_service = AsyncMock() + insights = [ + _insight("first", "insight-a"), + _insight("second", "insight-b"), + _insight("third", "insight-c"), + ] + insights[0].trace_refs = ["trace-old", "trace-new"] + insights[1].trace_refs = ["trace-missing"] + entity_client.list.return_value = SimpleNamespace( + data=insights, + pagination=NemoPaginationInfo( + page=1, + page_size=20, + current_page_size=len(insights), + total_pages=1, + total_results=len(insights), + ), + ) + entity_client.count_by.return_value = {"insight-a": 3} + latest = datetime(2026, 1, 2, tzinfo=timezone.utc) + spans_service.latest_trace_started_at_by_group.return_value = {"insight-a": latest} + + response = TestClient(_app(entity_client, spans_service)).get("/v2/workspaces/default/insights") + + assert response.status_code == 200 + assert [(item["id"], item["experiment_group_count"], item["last_seen_at"]) for item in response.json()["data"]] == [ + ("insight-a", 3, "2026-01-02T00:00:00Z"), + ("insight-b", 0, None), + ("insight-c", 0, None), + ] + entity_client.count_by.assert_awaited_once_with( + ExperimentGroup, + "insight_id", + workspace="default", + filter_obj={ + "insight_id": {"$in": ["insight-a", "insight-b", "insight-c"]}, + "is_deleted": False, + }, + ) + spans_service.latest_trace_started_at_by_group.assert_awaited_once_with( + workspace="default", + trace_refs_by_group={ + "insight-a": ["trace-old", "trace-new"], + "insight-b": ["trace-missing"], + "insight-c": [], + }, + ) diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index 96d9ecfeb0..40d4935b74 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -168,7 +168,9 @@ async def create_experiment_group( openapi_extra=generate_openapi_extra_params( filter_schema=ExperimentGroupFilter, filter_description=( - "Filter experiment groups by name, or by a metadata key/value: filter[metadata.]=." + "Filter experiment groups by name, insight_id, is_deleted, or a metadata key/value " + "(filter[metadata.]=). " + "Pass is_deleted=true to return only soft-deleted groups; omit to see only live ones." ), ), ) diff --git a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py index 940417cc62..5cf110b019 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py @@ -332,6 +332,10 @@ class ExperimentGroupFilter(Filter): """Filter for listing ExperimentGroups.""" name: str | None = Field(default=None, description="Filter groups by name.") + insight_id: str | None = Field( + default=None, + description="Filter groups by the id of the insight that seeded them.", + ) is_deleted: bool | None = Field( default=None, description="When true, returns only soft-deleted groups. Omit (or false) to see only live groups.", diff --git a/services/intake/src/nmp/intake/spans/clickhouse_client.py b/services/intake/src/nmp/intake/spans/clickhouse_client.py index f16fa6e612..43635e484f 100644 --- a/services/intake/src/nmp/intake/spans/clickhouse_client.py +++ b/services/intake/src/nmp/intake/spans/clickhouse_client.py @@ -11,6 +11,7 @@ from dataclasses import dataclass from typing import Any +from clickhouse_connect.driver.external import ExternalData from fastapi import HTTPException, Request from nmp.intake.config import IntakeConfig from nmp.intake.spans.clickhouse_migrations import ( @@ -83,10 +84,16 @@ async def query( *, parameters: Sequence[Any] | dict[str, Any] | None = None, settings: dict[str, Any] | None = None, + external_data: ExternalData | None = None, ) -> Any: await self.bootstrap_schema() raw_client = await self._get_raw_client() - return await raw_client.query(query, parameters=parameters, settings=settings) + return await raw_client.query( + query, + parameters=parameters, + settings=settings, + external_data=external_data, + ) async def insert( self, diff --git a/services/intake/src/nmp/intake/spans/service.py b/services/intake/src/nmp/intake/spans/service.py index c8b3e51708..2340b029e8 100644 --- a/services/intake/src/nmp/intake/spans/service.py +++ b/services/intake/src/nmp/intake/spans/service.py @@ -5,6 +5,8 @@ from __future__ import annotations +from datetime import datetime + from nmp.common.api.common import PaginatedResult from nmp.intake.spans.annotations_repository import AnnotationsRepository from nmp.intake.spans.domain import ( @@ -140,6 +142,17 @@ async def get_trace(self, *, workspace: str, trace_id: str, mode: TraceMode) -> raise TraceNotFoundError(workspace, trace_id) return trace + async def latest_trace_started_at_by_group( + self, + *, + workspace: str, + trace_refs_by_group: dict[str, list[str]], + ) -> dict[str, datetime]: + return await self._traces.latest_trace_started_at_by_group( + workspace=workspace, + trace_refs_by_group=trace_refs_by_group, + ) + async def get_session(self, *, workspace: str, session_id: str) -> IntakeSession: session = await self._sessions.get_session(workspace=workspace, session_id=session_id) if session is None: diff --git a/services/intake/src/nmp/intake/spans/trace_repository.py b/services/intake/src/nmp/intake/spans/trace_repository.py index f324c78390..03572c790e 100644 --- a/services/intake/src/nmp/intake/spans/trace_repository.py +++ b/services/intake/src/nmp/intake/spans/trace_repository.py @@ -5,9 +5,11 @@ from __future__ import annotations +import json from datetime import datetime, timezone from typing import Any +from clickhouse_connect.driver.external import ExternalData from nmp.common.api.common import PaginatedResult from nmp.intake.spans.clickhouse_client import ClickHouseSpanClient from nmp.intake.spans.domain import IntakeTrace, TraceListFilter, TraceMode @@ -148,6 +150,56 @@ async def get_trace(self, *, workspace: str, trace_id: str, mode: TraceMode) -> ) return result.data[0] if result.data else None + async def latest_trace_started_at_by_group( + self, + *, + workspace: str, + trace_refs_by_group: dict[str, list[str]], + ) -> dict[str, datetime]: + pairs = [ + (group_id, trace_id) + for group_id, trace_ids in trace_refs_by_group.items() + for trace_id in dict.fromkeys(trace_ids) + ] + if not pairs: + return {} + + trace_index_table = self._client.table("trace_index") + result = await self._client.query( + f""" + WITH + refs AS ( + SELECT group_id, trace_id + FROM trace_refs + ), + traces AS ( + SELECT + trace_roots.trace_id AS id, + trace_roots.root_started_at AS started_at + FROM {trace_index_table} AS trace_roots FINAL + WHERE trace_roots.workspace = %(workspace)s + AND trace_roots.is_deleted = 0 + AND trace_roots.trace_id IN (SELECT trace_id FROM refs) + ORDER BY trace_roots.root_started_at ASC, trace_roots.root_span_id ASC + LIMIT 1 BY trace_roots.workspace, trace_roots.source_format, trace_roots.trace_id + ) + SELECT refs.group_id, max(traces.started_at) AS started_at + FROM refs + INNER JOIN traces ON traces.id = refs.trace_id + GROUP BY refs.group_id + """, + parameters={"workspace": workspace}, + external_data=ExternalData( + file_name="trace_refs.jsonl", + data=b"\n".join( + json.dumps({"group_id": group_id, "trace_id": trace_id}).encode() for group_id, trace_id in pairs + ), + fmt="JSONEachRow", + structure="group_id String, trace_id String", + ), + ) + return {str(group_id): started_at for group_id, started_at in result.result_rows} + def _trace_rows_sql( *, trace_index_sql: str, spans_table: str, mode: TraceMode, sort: str diff --git a/services/intake/tests/integration/test_experiments_crud.py b/services/intake/tests/integration/test_experiments_crud.py index b9a0aca778..e912dde5cb 100644 --- a/services/intake/tests/integration/test_experiments_crud.py +++ b/services/intake/tests/integration/test_experiments_crud.py @@ -65,6 +65,19 @@ def test_experiment_group_crud(client: TestClient) -> None: assert missing.status_code == 404 +def test_filter_groups_by_insight_id(client: TestClient) -> None: + """The groups list can be filtered server-side by the seeding insight id.""" + seeded = client.post(GROUPS, json={"name": "seeded-group", "insight_id": "insight-abc"}) + assert seeded.status_code == 201, seeded.text + other = client.post(GROUPS, json={"name": "unseeded-group"}) + assert other.status_code == 201, other.text + + listed = client.get(GROUPS, params={"filter[insight_id]": "insight-abc"}) + assert listed.status_code == 200, listed.text + names = {g["name"] for g in listed.json()["data"]} + assert names == {"seeded-group"} + + def test_experiment_group_update_description(client: TestClient) -> None: client.post(GROUPS, json={"name": "grp", "description": "old"}) updated = client.put(f"{GROUPS}/grp", json={"name": "grp", "description": "new"}) diff --git a/services/intake/tests/test_traces_clickhouse_repository.py b/services/intake/tests/test_traces_clickhouse_repository.py index 23fb2bfdaf..f89f707601 100644 --- a/services/intake/tests/test_traces_clickhouse_repository.py +++ b/services/intake/tests/test_traces_clickhouse_repository.py @@ -3,6 +3,7 @@ """Trace repository tests.""" +import json from datetime import datetime, timedelta, timezone from typing import cast @@ -22,14 +23,22 @@ class _Client: def __init__(self, query_results: list[_QueryResult] | None = None) -> None: self.queries: list[str] = [] self.parameters: list[dict[str, object]] = [] + self.external_data: list[object | None] = [] self.query_results = query_results or [] def table(self, name: str) -> str: return name - async def query(self, query: str, *, parameters: dict[str, object]) -> _QueryResult: + async def query( + self, + query: str, + *, + parameters: dict[str, object], + external_data: object | None = None, + ) -> _QueryResult: self.queries.append(query) self.parameters.append(parameters) + self.external_data.append(external_data) if self.query_results: return self.query_results.pop(0) if query.lstrip().startswith("SELECT count()"): @@ -81,6 +90,40 @@ async def test_summary_mode_reads_root_spans_without_metric_aggregates(): assert "payload_char_limit" not in client.parameters[1] +@pytest.mark.asyncio +async def test_latest_trace_started_at_by_group_aggregates_all_references_in_one_query(): + latest = datetime(2026, 1, 2, tzinfo=timezone.utc) + client = _Client(query_results=[_QueryResult([("insight-a", latest)])]) + repository = _repository(client) + + result = await repository.latest_trace_started_at_by_group( + workspace="workspace-a", + trace_refs_by_group={ + "insight-a": ["trace-old", "trace-new"], + "insight-empty": [], + "insight-missing": ["trace-missing"], + }, + ) + + assert result == {"insight-a": latest} + assert len(client.queries) == 1 + assert "FROM trace_refs" in client.queries[0] + assert "max(traces.started_at) AS started_at" in client.queries[0] + assert "GROUP BY refs.group_id" in client.queries[0] + assert client.parameters[0] == {"workspace": "workspace-a"} + external_data = client.external_data[0] + assert external_data is not None + assert external_data.query_params == { + "trace_refs_format": "JSONEachRow", + "trace_refs_structure": "group_id String, trace_id String", + } + assert [json.loads(line) for line in external_data.form_data["trace_refs"][1].splitlines()] == [ + {"group_id": "insight-a", "trace_id": "trace-old"}, + {"group_id": "insight-a", "trace_id": "trace-new"}, + {"group_id": "insight-missing", "trace_id": "trace-missing"}, + ] + + @pytest.mark.asyncio async def test_preview_mode_bounds_payloads_and_adds_trace_aggregate_block(): client = _Client() diff --git a/services/studio/src/nmp/studio/env_mappings.py b/services/studio/src/nmp/studio/env_mappings.py index 4c8e2aedaa..046acff7a3 100644 --- a/services/studio/src/nmp/studio/env_mappings.py +++ b/services/studio/src/nmp/studio/env_mappings.py @@ -143,6 +143,11 @@ class EnvMapping: config_path="studio.feature_flags.model_compare_enabled", default="false", ), + EnvMapping( + marker="STUDIO_UI_VITE_FF_OPTIMIZER_ENABLED", + config_path="studio.feature_flags.optimizer_enabled", + default="false", + ), EnvMapping( marker="STUDIO_UI_VITE_FF_SAFE_SYNTHESIZER_ENABLED", config_path="studio.feature_flags.safe_synthesizer_enabled", diff --git a/web/packages/studio/env/.env.dev.local.sample b/web/packages/studio/env/.env.dev.local.sample index fe727be48d..f349922cf7 100644 --- a/web/packages/studio/env/.env.dev.local.sample +++ b/web/packages/studio/env/.env.dev.local.sample @@ -39,6 +39,7 @@ VITE_FF_INTAKE_ENABLED='true' VITE_FF_JOBS_ENABLED='true' VITE_FF_MEMBERS_ENABLED='preview' VITE_FF_MODEL_COMPARE_ENABLED='true' +VITE_FF_OPTIMIZER_ENABLED='false' VITE_FF_SAFE_SYNTHESIZER_ENABLED='true' VITE_FF_SECRETS_ENABLED='true' VITE_FF_SETTINGS_ENABLED='true' diff --git a/web/packages/studio/env/.env.fastapi b/web/packages/studio/env/.env.fastapi index 1c4be4abfc..8a9007efa9 100644 --- a/web/packages/studio/env/.env.fastapi +++ b/web/packages/studio/env/.env.fastapi @@ -33,5 +33,6 @@ VITE_FF_INFERENCE_PROVIDER_ENABLED=STUDIO_UI_VITE_FF_INFERENCE_PROVIDER_ENABLED VITE_FF_INTAKE_ENABLED=STUDIO_UI_VITE_FF_INTAKE_ENABLED VITE_FF_MEMBERS_ENABLED=STUDIO_UI_VITE_FF_MEMBERS_ENABLED VITE_FF_MODEL_COMPARE_ENABLED=STUDIO_UI_VITE_FF_MODEL_COMPARE_ENABLED +VITE_FF_OPTIMIZER_ENABLED=STUDIO_UI_VITE_FF_OPTIMIZER_ENABLED VITE_FF_SAFE_SYNTHESIZER_ENABLED=STUDIO_UI_VITE_FF_SAFE_SYNTHESIZER_ENABLED VITE_FF_SECRETS_ENABLED=STUDIO_UI_VITE_FF_SECRETS_ENABLED diff --git a/web/packages/studio/src/api/optimizer.ts b/web/packages/studio/src/api/optimizer.ts new file mode 100644 index 0000000000..dbd502d527 --- /dev/null +++ b/web/packages/studio/src/api/optimizer.ts @@ -0,0 +1,180 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { customFetch, type ErrorType } from '@nemo/sdk/generated/fetchers/platform'; +import type { HTTPValidationError, PaginationData } from '@nemo/sdk/generated/platform/schema'; +import { + type QueryClient, + type UseMutationOptions, + type UseMutationResult, + type UseQueryOptions, + type UseQueryResult, + useMutation, + useQuery, +} from '@tanstack/react-query'; + +/** + * The Insights plugin is a NeMo Platform backend plugin whose routes are NOT + * part of the generated SDK. We hand-write typed hooks against its endpoints + * using `customFetch`, which applies the same base URL + OIDC auth handling as + * the generated clients. + * + * Routes mount at `/apis/insights/v2/workspaces/{workspace}/...`. + */ + +export type InsightStatus = 'open' | 'resolved' | 'deleted'; + +export interface Insight { + /** Store-assigned id — used to fetch a single insight (`GET /insights/{id}`). */ + id: string; + /** Entity name (unique slug within the workspace). */ + name: string; + /** Short, human-readable sentence naming the core issue. */ + title: string; + /** The actionable problem statement. */ + description: string; + /** Registered agent name or local path this insight is about. */ + agent: string; + /** Lifecycle state. Defaults to `open`. */ + status: InsightStatus; + /** Intake trace ids identified as evidence for this insight. */ + trace_refs: string[]; + created_at?: string; + updated_at?: string; + [key: string]: unknown; +} + +export interface InsightListItem extends Insight { + /** Number of experiment groups linked to this insight, or null when unknown. */ + experiment_group_count: number | null; + /** Newest start timestamp among the insight's currently referenced traces. */ + last_seen_at?: string | null; +} + +export type OptimizerListInsightsParams = Record & { + page?: number; + page_size?: number; + sort?: string; + filter?: Record; +}; + +export interface InsightPage { + data?: InsightListItem[]; + pagination?: PaginationData; + [key: string]: unknown; +} + +const optimizerInsightsPath = (workspace: string, path = '') => + `/apis/insights/v2/workspaces/${encodeURIComponent(String(workspace))}/insights${path}`; + +type QueryOptions = { + query?: Partial>; +}; + +type MutationOptions = { + mutation?: UseMutationOptions; +}; + +export interface UpdateInsightRequest { + title?: string; + agent?: string; + description?: string; + status?: InsightStatus; + trace_refs?: string[]; +} + +export const optimizerListInsights = ( + workspace: string, + params?: OptimizerListInsightsParams, + signal?: AbortSignal +) => + customFetch({ + url: optimizerInsightsPath(workspace), + method: 'GET', + params, + signal, + }); + +export const getOptimizerListInsightsQueryKey = ( + workspace: string, + params?: OptimizerListInsightsParams +) => [optimizerInsightsPath(workspace), ...(params ? [params] : [])] as const; + +export const useOptimizerListInsights = >( + workspace: string, + params?: OptimizerListInsightsParams, + options?: QueryOptions>, TError>, + queryClient?: QueryClient +): UseQueryResult>, TError> => + useQuery( + { + queryKey: getOptimizerListInsightsQueryKey(workspace, params), + queryFn: ({ signal }) => optimizerListInsights(workspace, params, signal), + ...options?.query, + }, + queryClient + ); + +export const optimizerGetInsight = (workspace: string, insightId: string, signal?: AbortSignal) => + customFetch({ + url: optimizerInsightsPath(workspace, `/${encodeURIComponent(insightId)}`), + method: 'GET', + signal, + }); + +export const getOptimizerGetInsightQueryKey = (workspace: string, insightId: string) => + [optimizerInsightsPath(workspace, `/${encodeURIComponent(insightId)}`)] as const; + +export const useOptimizerGetInsight = >( + workspace: string, + insightId: string, + options?: QueryOptions>, TError>, + queryClient?: QueryClient +): UseQueryResult>, TError> => + useQuery( + { + queryKey: getOptimizerGetInsightQueryKey(workspace, insightId), + queryFn: ({ signal }) => optimizerGetInsight(workspace, insightId, signal), + enabled: !!insightId, + ...options?.query, + }, + queryClient + ); + +export const optimizerUpdateInsight = ( + workspace: string, + insightId: string, + data: UpdateInsightRequest +) => + customFetch({ + url: optimizerInsightsPath(workspace, `/${encodeURIComponent(insightId)}`), + method: 'PATCH', + data, + }); + +export const useOptimizerUpdateInsight = < + TError = ErrorType, + TContext = unknown, +>( + options?: MutationOptions< + Awaited>, + { workspace: string; insightId: string; data: UpdateInsightRequest }, + TError, + TContext + >, + queryClient?: QueryClient +): UseMutationResult< + Awaited>, + TError, + { workspace: string; insightId: string; data: UpdateInsightRequest }, + TContext +> => + useMutation( + { + mutationKey: ['optimizerUpdateInsight'], + mutationFn: ({ workspace, insightId, data }) => + optimizerUpdateInsight(workspace, insightId, data), + ...options?.mutation, + }, + queryClient + ); diff --git a/web/packages/studio/src/assets/voyager.svg b/web/packages/studio/src/assets/voyager.svg new file mode 100644 index 0000000000..cc7b82fd25 --- /dev/null +++ b/web/packages/studio/src/assets/voyager.svg @@ -0,0 +1,162 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/web/packages/studio/src/components/ChangesetBadge/index.tsx b/web/packages/studio/src/components/ChangesetBadge/index.tsx new file mode 100644 index 0000000000..6d76c4116e --- /dev/null +++ b/web/packages/studio/src/components/ChangesetBadge/index.tsx @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Anchor, Badge } from '@nvidia/foundations-react-core'; +import { SquareArrowOutUpRight } from 'lucide-react'; +import { type FC } from 'react'; + +interface ChangesetBadgeProps { + /** The experiment's `source_link` URL. */ + href: string; +} + +/** + * Blue "Changeset" badge linking to an experiment's source changeset. Used on both the experiment + * group table and the experiment detail header. The trailing external-link icon signals the link + * opens the source in a new tab. `stopPropagation` keeps the link from also triggering a clickable + * parent (e.g. a table row's row-click navigation). + */ +export const ChangesetBadge: FC = ({ href }) => ( + event.stopPropagation()}> + + Changeset + + +); diff --git a/web/packages/studio/src/components/IntakeLists/IntakeTracesTable.tsx b/web/packages/studio/src/components/IntakeLists/IntakeTracesTable.tsx index 2e2b57ede2..79e5b028f2 100644 --- a/web/packages/studio/src/components/IntakeLists/IntakeTracesTable.tsx +++ b/web/packages/studio/src/components/IntakeLists/IntakeTracesTable.tsx @@ -1,16 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { dateTimeFilter } from '@nemo/common/src/components/DataView/dateTimeFilter'; import { EditColumnsMenu } from '@nemo/common/src/components/DataView/internal'; import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; -import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; import { getSortParamWithWhitelist } from '@nemo/common/src/utils/query'; import { useListTraces } from '@nemo/sdk/generated/platform/api'; import type { Trace, TraceFilter, TraceSortField } from '@nemo/sdk/generated/platform/schema'; -import { Badge, Button } from '@nvidia/foundations-react-core'; +import { Button } from '@nvidia/foundations-react-core'; import { getErrorMessage } from '@studio/api/common/utils'; import { isDefaultStartedAtFilter, @@ -18,19 +16,13 @@ import { type StartedAtFilterEntry, useSeededStartedAtFilter, } from '@studio/components/IntakeLists/defaultStartedAtFilter'; -import { IntakePayloadPreviewCell } from '@studio/components/IntakeLists/IntakePayloadPreviewCell'; import { IntakeTelemetryDataView } from '@studio/components/IntakeLists/IntakeTelemetryDataView'; +import { makeIntakeTraceColumns } from '@studio/components/IntakeLists/intakeTraceColumns'; import { useWorkspaceFromPathIfExists } from '@studio/hooks/useWorkspaceFromPath'; import { getIntakeSessionTraceRoute } from '@studio/routes/utils'; -import { - formatCost, - formatDurationMs, - formatInteger, - getTraceDisplayName, -} from '@studio/util/intakeTelemetry'; import { keepPreviousData } from '@tanstack/react-query'; import { Columns3 } from 'lucide-react'; -import { type ComponentProps, type FC, type ReactNode, useState } from 'react'; +import { type FC, type ReactNode, useState } from 'react'; import { useNavigate } from 'react-router-dom'; export interface IntakeTracesTableProps { @@ -101,97 +93,6 @@ const SeededIntakeTracesTable: FC< } ); - const makeColumns: ComponentProps>['makeColumns'] = ({ - accessor, - }) => [ - accessor('id', { - id: 'id', - header: 'Trace', - size: 280, - enableSorting: false, - meta: { - filter: { - type: 'text' as const, - label: 'Trace ID', - placeholder: 'Filter by trace ID', - }, - }, - cell: ({ row }) => { - const trace = row.original; - const label = getTraceDisplayName(trace); - return label; - }, - }), - accessor('input', { - id: 'input', - header: 'Input', - size: 360, - enableSorting: false, - cell: ({ row }) => , - }), - accessor('output', { - id: 'output', - header: 'Output', - size: 360, - enableSorting: false, - cell: ({ row }) => , - }), - { - id: 'duration_ms', - header: 'Duration', - size: 120, - enableSorting: false, - cell: ({ row }) => formatDurationMs(row.original.duration_ms), - }, - { - id: 'span_count', - header: 'Spans', - size: 90, - enableSorting: false, - cell: ({ row }) => formatInteger(row.original.span_count), - }, - { - id: 'error_count', - header: 'Errors', - size: 90, - enableSorting: false, - cell: ({ row }) => { - const errorCount = row.original.error_count ?? 0; - return errorCount > 0 ? ( - - {formatInteger(errorCount)} - - ) : ( - formatInteger(errorCount) - ); - }, - }, - { - id: 'total_tokens', - header: 'Tokens', - size: 120, - enableSorting: false, - cell: ({ row }) => formatInteger(row.original.total_tokens), - }, - { - id: 'cost_usd', - header: 'Cost', - size: 110, - enableSorting: false, - cell: ({ row }) => formatCost(row.original.cost_usd), - }, - accessor('started_at', { - id: 'started_at', - header: 'Started', - size: 150, - enableSorting: true, - meta: { - filter: dateTimeFilter('Started At'), - }, - cell: ({ row }) => , - }), - ]; - if (error) { return ; } @@ -203,7 +104,11 @@ const SeededIntakeTracesTable: FC< return ( dataViewState={dataViewState} - makeColumns={makeColumns} + makeColumns={makeIntakeTraceColumns({ + traceIdFilter: true, + startedAtSort: true, + startedAtFilter: true, + })} slotEndPortalTargetId={slotEndPortalTargetId} toolbarSlotEnd={ >['makeColumns']; + +export interface IntakeTraceColumnOptions { + /** Expose a Trace ID text filter on the first column (workspace browse table). */ + traceIdFilter?: boolean; + /** Allow sorting by started_at (server-backed list only). */ + startedAtSort?: boolean; + /** Expose a Started At datetime filter (workspace browse table). */ + startedAtFilter?: boolean; +} + +/** + * Shared trace table columns for Intake browse (`IntakeTracesTable`) and insight evidence + * (`InsightTracesTable`). Keeps headers, sizes, and formatters in one place. + */ +export const makeIntakeTraceColumns = + ({ + traceIdFilter = false, + startedAtSort = false, + startedAtFilter = false, + }: IntakeTraceColumnOptions = {}): MakeIntakeTraceColumns => + ({ accessor }) => [ + accessor('id', { + id: 'id', + header: 'Trace', + size: 280, + enableSorting: false, + meta: traceIdFilter + ? { + filter: { + type: 'text' as const, + label: 'Trace ID', + placeholder: 'Filter by trace ID', + }, + } + : undefined, + cell: ({ row }) => getTraceDisplayName(row.original), + }), + accessor('input', { + id: 'input', + header: 'Input', + size: 360, + enableSorting: false, + cell: ({ row }) => , + }), + accessor('output', { + id: 'output', + header: 'Output', + size: 360, + enableSorting: false, + cell: ({ row }) => , + }), + { + id: 'duration_ms', + header: 'Duration', + size: 120, + enableSorting: false, + cell: ({ row }) => formatDurationMs(row.original.duration_ms), + }, + { + id: 'span_count', + header: 'Spans', + size: 90, + enableSorting: false, + cell: ({ row }) => formatInteger(row.original.span_count), + }, + { + id: 'error_count', + header: 'Errors', + size: 90, + enableSorting: false, + cell: ({ row }) => { + const errorCount = row.original.error_count ?? 0; + return errorCount > 0 ? ( + + {formatInteger(errorCount)} + + ) : ( + formatInteger(errorCount) + ); + }, + }, + { + id: 'total_tokens', + header: 'Tokens', + size: 120, + enableSorting: false, + cell: ({ row }) => formatInteger(row.original.total_tokens), + }, + { + id: 'cost_usd', + header: 'Cost', + size: 110, + enableSorting: false, + cell: ({ row }) => formatCost(row.original.cost_usd), + }, + accessor('started_at', { + id: 'started_at', + header: 'Started', + size: 150, + enableSorting: startedAtSort, + meta: startedAtFilter ? { filter: dateTimeFilter('Started At') } : undefined, + cell: ({ row }) => , + }), + ]; diff --git a/web/packages/studio/src/components/OriginatingInsightLink/index.tsx b/web/packages/studio/src/components/OriginatingInsightLink/index.tsx new file mode 100644 index 0000000000..ed3dfe3313 --- /dev/null +++ b/web/packages/studio/src/components/OriginatingInsightLink/index.tsx @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Anchor, Text } from '@nvidia/foundations-react-core'; +import voyagerArt from '@studio/assets/voyager.svg'; +import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; +import { getOptimizerInsightRoute } from '@studio/routes/utils'; +import { type FC } from 'react'; +import { Link } from 'react-router-dom'; + +interface OriginatingInsightLinkProps { + insightId: string; +} + +/** + * The Voyager artwork above an "Originating insight" label, both linking to the insight detail page. + * Shared by the experiment group detail page and the experiment detail root-cause card. + */ +export const OriginatingInsightLink: FC = ({ insightId }) => { + const workspace = useWorkspaceFromPath(); + return ( + + + + + Originating +
+ insight + + + + ); +}; diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx index b4c39b277b..aee20ea821 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx @@ -22,6 +22,7 @@ import type { ExperimentGroupResponse, } from '@nemo/sdk/generated/platform/schema'; import { Button, Text, Tooltip } from '@nvidia/foundations-react-core'; +import { ChangesetBadge } from '@studio/components/ChangesetBadge'; import { ExperimentGroupParetoChart } from '@studio/components/charts/ExperimentGroupParetoChart'; import { AddToGroupModal } from '@studio/components/dataViews/ExperimentGroupDataView/AddToGroupModal'; import '@studio/components/dataViews/ExperimentGroupDataView/ExperimentGroupDataView.css'; @@ -303,6 +304,16 @@ export const ExperimentGroupDataView: FC = ({ ); }, }), + accessor('source_link', { + header: 'Source', + enableSorting: false, + size: 140, + cell: ({ row }) => { + const { source_link } = row.original; + if (!source_link) return -; + return ; + }, + }), accessor((original) => original.agent_names?.join(', '), { id: 'agent_names', header: 'Agent Names', diff --git a/web/packages/studio/src/constants/environment.ts b/web/packages/studio/src/constants/environment.ts index 7f99302f34..169426adfc 100644 --- a/web/packages/studio/src/constants/environment.ts +++ b/web/packages/studio/src/constants/environment.ts @@ -48,6 +48,7 @@ export const INTAKE_ENABLED = featureFlags.intakeEnabled !== false; export const JOBS_ENABLED = featureFlags.jobsEnabled !== false; export const MEMBERS_ENABLED = featureFlags.membersEnabled !== false; export const MODEL_COMPARE_ENABLED = featureFlags.modelCompareEnabled !== false; +export const OPTIMIZER_ENABLED = featureFlags.optimizerEnabled !== false; export const SAFE_SYNTHESIZER_ENABLED = featureFlags.safeSynthesizerEnabled !== false; export const SECRETS_ENABLED = featureFlags.secretsEnabled !== false; export const SETTINGS_ENABLED = featureFlags.settingsEnabled !== false; diff --git a/web/packages/studio/src/constants/featureFlags/featureFlags.ts b/web/packages/studio/src/constants/featureFlags/featureFlags.ts index 4b4337c02c..8285a5842b 100644 --- a/web/packages/studio/src/constants/featureFlags/featureFlags.ts +++ b/web/packages/studio/src/constants/featureFlags/featureFlags.ts @@ -73,6 +73,7 @@ export const flagDefinitions = { jobsEnabled: previewFlag('VITE_FF_JOBS_ENABLED', true), membersEnabled: previewFlag('VITE_FF_MEMBERS_ENABLED'), modelCompareEnabled: previewFlag('VITE_FF_MODEL_COMPARE_ENABLED'), + optimizerEnabled: previewFlag('VITE_FF_OPTIMIZER_ENABLED', false), safeSynthesizerEnabled: previewFlag('VITE_FF_SAFE_SYNTHESIZER_ENABLED', true), secretsEnabled: previewFlag('VITE_FF_SECRETS_ENABLED', true), settingsEnabled: previewFlag('VITE_FF_SETTINGS_ENABLED', true), diff --git a/web/packages/studio/src/constants/routes.ts b/web/packages/studio/src/constants/routes.ts index 97d50113fa..51a00c0389 100644 --- a/web/packages/studio/src/constants/routes.ts +++ b/web/packages/studio/src/constants/routes.ts @@ -39,6 +39,7 @@ export const ROUTE_PARAMS = { /** Benchmark entity name segment under evaluation/benchmarks/:name */ benchmarkName: 'benchmarkName', experimentGroupName: 'experimentGroupName', + insightId: 'insightId', evaluationName: 'evaluationName', guardrailConfigName: 'guardrailConfigName', } as const; @@ -114,6 +115,8 @@ export const ROUTES = { anonymizerJob: `/workspaces/:${P.workspace}/anonymizer/:${P.anonymizerJobName}`, secrets: `/workspaces/:${P.workspace}/secrets`, guardrails: `/workspaces/:${P.workspace}/guardrails`, + optimizer: `/workspaces/:${P.workspace}/optimizer`, + optimizerInsight: `/workspaces/:${P.workspace}/optimizer/:${P.insightId}`, guardrailDetail: `/workspaces/:${P.workspace}/guardrails/:${P.guardrailConfigName}`, settings: `/workspaces/:${P.workspace}/settings`, /** Workspace members and role-based access (Entities role bindings) */ diff --git a/web/packages/studio/src/routes/EvaluationDetailRoute/EvaluationDetailMetrics.tsx b/web/packages/studio/src/routes/EvaluationDetailRoute/EvaluationDetailMetrics.tsx index 6789c960aa..11f718b7b1 100644 --- a/web/packages/studio/src/routes/EvaluationDetailRoute/EvaluationDetailMetrics.tsx +++ b/web/packages/studio/src/routes/EvaluationDetailRoute/EvaluationDetailMetrics.tsx @@ -6,6 +6,7 @@ import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; import { formatDurationMs } from '@nemo/common/src/utils/date'; import { useGetEvaluation } from '@nemo/sdk/generated/platform/api'; import { Divider, Flex, Text, Tooltip } from '@nvidia/foundations-react-core'; +import { ChangesetBadge } from '@studio/components/ChangesetBadge'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { tooltipClassName } from '@studio/styles/common'; import { type FC, type ReactNode } from 'react'; @@ -40,6 +41,17 @@ export const EvaluationDetailMetrics: FC = ({ eval return ( + {experiment?.source_link ? ( + <> + } + loading={isLoading} + orientation="vertical" + /> + + + ) : null} { + vi.stubEnv('VITE_FF_OPTIMIZER_ENABLED', 'true'); +}); + +const WORKSPACE = 'test-workspace'; +const GROUP_NAME = 'test-group'; +const EVALUATION_NAME = 'test-evaluation'; + +const evaluation = { + id: 'evaluation-id', + name: EVALUATION_NAME, + workspace: WORKSPACE, + experiment_group_id: 'group-id', + dataset_name: 'dataset', + description: 'Evaluation description', +} satisfies Partial; + +const group = { + id: 'group-id', + name: GROUP_NAME, + workspace: WORKSPACE, + insight_id: 'insight-id', + default_sort: '-created_at', + evaluation_count: 0, +} satisfies Partial; + +describe('EvaluationDetailRoute with Optimizer enabled', () => { + it('renders the originating insight description instead of relabeling the evaluation description', async () => { + server.use( + http.get('*/apis/intake/v2/workspaces/:workspace/evaluations/:name', () => + HttpResponse.json(evaluation) + ), + http.get('*/apis/intake/v2/workspaces/:workspace/experiment-groups/:name', () => + HttpResponse.json(group) + ), + http.get('*/apis/intake/v2/workspaces/:workspace/evaluations/:name/sessions', () => + HttpResponse.json({ + data: [], + pagination: { + page: 1, + page_size: 25, + current_page_size: 0, + total_pages: 0, + total_results: 0, + }, + }) + ), + http.get('*/apis/insights/v2/workspaces/:workspace/insights/:insightId', () => + HttpResponse.json({ + id: 'insight-id', + name: 'insight', + title: 'Insight', + description: 'Actual insight description', + agent: 'agent', + status: 'open', + trace_refs: [], + }) + ) + ); + + renderRoute(, { + history: `/workspaces/${WORKSPACE}/experiment/${GROUP_NAME}/${EVALUATION_NAME}`, + routes: [ + { + path: ROUTES.workspace.evaluationDetail, + element: , + }, + ], + }); + + expect(await screen.findByText('Actual insight description')).toBeInTheDocument(); + expect(screen.getByText('Insight description')).toBeInTheDocument(); + expect(screen.queryByText('Evaluation description')).not.toBeInTheDocument(); + expect(screen.getByRole('link', { name: /originating insight/i })).toHaveAttribute( + 'href', + `/workspaces/${WORKSPACE}/optimizer/insight-id` + ); + }); +}); diff --git a/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx b/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx index 2df2ca6d3b..6db07615b2 100644 --- a/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx +++ b/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx @@ -1,10 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { useGetEvaluation } from '@nemo/sdk/generated/platform/api'; -import { Badge, PageHeader, Stack, Text } from '@nvidia/foundations-react-core'; +import { useGetEvaluation, useGetExperimentGroup } from '@nemo/sdk/generated/platform/api'; +import { Badge, Card, Flex, PageHeader, Stack, Text } from '@nvidia/foundations-react-core'; +import { useOptimizerGetInsight } from '@studio/api/optimizer'; import { AccessibleTitle } from '@studio/components/AccessibleTitle'; import { EvaluationSessionsDataView } from '@studio/components/dataViews/EvaluationSessionsDataView'; +import { OriginatingInsightLink } from '@studio/components/OriginatingInsightLink'; +import { OPTIMIZER_ENABLED } from '@studio/constants/environment'; import { ROUTE_PARAMS } from '@studio/constants/routes'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; @@ -19,7 +22,14 @@ export const EvaluationDetailRoute: FC = () => { ROUTE_PARAMS.experimentGroupName, ROUTE_PARAMS.evaluationName, ]); - const { data: experiment } = useGetEvaluation(workspace, evaluationName); + const { data: evaluation } = useGetEvaluation(workspace, evaluationName); + // Evaluations reach their originating insight through the owning group's insight_id. + const { data: experimentGroup } = useGetExperimentGroup(workspace, experimentGroupName); + const insightId = experimentGroup?.insight_id ?? ''; + const { data: insight } = useOptimizerGetInsight(workspace, insightId, { + query: { enabled: OPTIMIZER_ENABLED && Boolean(insightId) }, + }); + const showInsightCard = Boolean(insight?.description); useBreadcrumbs({ items: [ @@ -38,15 +48,26 @@ export const EvaluationDetailRoute: FC = () => { + {showInsightCard ? ( + + + + + Insight description + {insight?.description} + + + + ) : null}
Test cases - {experiment?.run_count !== undefined && ( + {evaluation?.run_count !== undefined && ( - {experiment.run_count} + {evaluation.run_count} )}
diff --git a/web/packages/studio/src/routes/EvaluationDetailRoute/optimizerDisabled.test.tsx b/web/packages/studio/src/routes/EvaluationDetailRoute/optimizerDisabled.test.tsx new file mode 100644 index 0000000000..6f6125a36a --- /dev/null +++ b/web/packages/studio/src/routes/EvaluationDetailRoute/optimizerDisabled.test.tsx @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + EvaluationResponse, + ExperimentGroupResponse, +} from '@nemo/sdk/generated/platform/schema'; +import { ROUTES } from '@studio/constants/routes'; +import { server } from '@studio/mocks/node'; +import { EvaluationDetailRoute } from '@studio/routes/EvaluationDetailRoute'; +import { renderRoute, screen } from '@studio/tests/util/render'; +import { http, HttpResponse } from 'msw'; + +vi.hoisted(() => { + vi.stubEnv('VITE_FF_OPTIMIZER_ENABLED', 'false'); +}); + +const WORKSPACE = 'test-workspace'; +const GROUP_NAME = 'test-group'; +const EVALUATION_NAME = 'test-evaluation'; + +const evaluation = { + id: 'evaluation-id', + name: EVALUATION_NAME, + workspace: WORKSPACE, + experiment_group_id: 'group-id', + dataset_name: 'dataset', + description: 'Evaluation description', +} satisfies Partial; + +const group = { + id: 'group-id', + name: GROUP_NAME, + workspace: WORKSPACE, + insight_id: 'insight-id', + default_sort: '-created_at', + evaluation_count: 0, +} satisfies Partial; + +describe('EvaluationDetailRoute with Optimizer disabled', () => { + it('renders the evaluation description without requesting or linking to the insight', async () => { + const insightRequest = vi.fn(); + server.use( + http.get('*/apis/intake/v2/workspaces/:workspace/evaluations/:name', () => + HttpResponse.json(evaluation) + ), + http.get('*/apis/intake/v2/workspaces/:workspace/experiment-groups/:name', () => + HttpResponse.json(group) + ), + http.get('*/apis/intake/v2/workspaces/:workspace/evaluations/:name/sessions', () => + HttpResponse.json({ + data: [], + pagination: { + page: 1, + page_size: 25, + current_page_size: 0, + total_pages: 0, + total_results: 0, + }, + }) + ), + http.get('*/apis/insights/v2/workspaces/:workspace/insights/:insightId', () => { + insightRequest(); + return HttpResponse.json({}); + }) + ); + + renderRoute(, { + history: `/workspaces/${WORKSPACE}/experiment/${GROUP_NAME}/${EVALUATION_NAME}`, + routes: [ + { + path: ROUTES.workspace.evaluationDetail, + element: , + }, + ], + }); + + expect(await screen.findByText('Evaluation description')).toBeInTheDocument(); + expect(insightRequest).not.toHaveBeenCalled(); + expect(screen.queryByText('Insight description')).not.toBeInTheDocument(); + expect(screen.queryByRole('link', { name: /originating insight/i })).not.toBeInTheDocument(); + }); +}); diff --git a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.test.tsx b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.test.tsx new file mode 100644 index 0000000000..a538763633 --- /dev/null +++ b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.test.tsx @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import { ROUTES } from '@studio/constants/routes'; +import { server } from '@studio/mocks/node'; +import { ExperimentGroupDetailRoute } from '@studio/routes/ExperimentGroupDetailRoute'; +import { renderRoute, screen } from '@studio/tests/util/render'; +import { http, HttpResponse } from 'msw'; + +vi.hoisted(() => { + vi.stubEnv('VITE_FF_OPTIMIZER_ENABLED', 'false'); +}); + +const WORKSPACE = 'test-workspace'; +const GROUP_NAME = 'test-group'; + +const group = { + id: 'group-id', + name: GROUP_NAME, + workspace: WORKSPACE, + description: 'Editable group description', + summary: 'Generated group summary', + insight_id: 'insight-id', + default_sort: '-created_at', + evaluation_count: 0, +} satisfies Partial; + +describe('ExperimentGroupDetailRoute', () => { + it('renders the summary without requesting Optimizer when disabled', async () => { + const insightRequest = vi.fn(); + server.use( + http.get('*/apis/intake/v2/workspaces/:workspace/experiment-groups/:name', () => + HttpResponse.json(group) + ), + http.get('*/apis/intake/v2/workspaces/:workspace/evaluations', () => + HttpResponse.json({ + data: [], + pagination: { + page: 1, + page_size: 25, + current_page_size: 0, + total_pages: 0, + total_results: 0, + }, + }) + ), + http.get('*/apis/insights/v2/workspaces/:workspace/insights/:insightId', () => { + insightRequest(); + return HttpResponse.json({}); + }) + ); + + renderRoute(, { + history: `/workspaces/${WORKSPACE}/experiment/${GROUP_NAME}`, + routes: [ + { + path: ROUTES.workspace.experimentGroupDetail, + element: , + }, + ], + }); + + expect(await screen.findByText('Generated group summary')).toBeInTheDocument(); + expect(screen.queryByText('Editable group description')).not.toBeInTheDocument(); + expect(insightRequest).not.toHaveBeenCalled(); + expect(screen.queryByRole('link', { name: /originating insight/i })).not.toBeInTheDocument(); + }); +}); diff --git a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx index 138a2ed885..3b2aca712e 100644 --- a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx +++ b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx @@ -3,10 +3,22 @@ import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; import { useGetExperimentGroup } from '@nemo/sdk/generated/platform/api'; -import { Button, PageHeader, Stack, Text } from '@nvidia/foundations-react-core'; +import { + Anchor, + Button, + Card, + Flex, + PageHeader, + Stack, + Text, +} from '@nvidia/foundations-react-core'; +import { useOptimizerGetInsight } from '@studio/api/optimizer'; import { AccessibleTitle } from '@studio/components/AccessibleTitle'; import { ExperimentGroupDataView } from '@studio/components/dataViews/ExperimentGroupDataView'; import { ExperimentGroupEditModal } from '@studio/components/ExperimentGroupEditModal'; +import { OriginatingInsightLink } from '@studio/components/OriginatingInsightLink'; +import { OPTIMIZER_ENABLED } from '@studio/constants/environment'; +import { LINK_DOCS_STUDIO_EVALUATION } from '@studio/constants/links'; import { ROUTE_PARAMS } from '@studio/constants/routes'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; @@ -21,6 +33,10 @@ export const ExperimentGroupDetailRoute: FC = () => { const workspace = useWorkspaceFromPath(); const { experimentGroupName } = useRequiredPathParams([ROUTE_PARAMS.experimentGroupName]); const { data: group, error } = useGetExperimentGroup(workspace, experimentGroupName); + // The insight is a group-level concept, reached via the group's insight_id. + const { data: insight } = useOptimizerGetInsight(workspace, group?.insight_id ?? '', { + query: { enabled: OPTIMIZER_ENABLED && Boolean(group?.insight_id) }, + }); const [editOpen, setEditOpen] = useState(false); // Pareto (cost-vs-accuracy) view visibility, persisted per group. Hidden by default. @@ -43,7 +59,14 @@ export const ExperimentGroupDetailRoute: FC = () => { + An experiment is a group of evaluation runs aligned toward a common objective.{' '} + + Learn more + + + } slotActions={ + } + > + + + Run the following CLI command to start experiments for this insight. + + + + + + ); +}; diff --git a/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.test.tsx b/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.test.tsx new file mode 100644 index 0000000000..979300e94a --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.test.tsx @@ -0,0 +1,111 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { DEFAULT_WORKSPACE } from '@nemo/common/src/models/constants'; +import type { Trace } from '@nemo/sdk/generated/platform/schema'; +import { server } from '@studio/mocks/node'; +import { InsightTracesTable } from '@studio/routes/optimizer/InsightTracesTable'; +import { renderRoute, screen, waitFor } from '@studio/tests/util/render'; +import userEvent from '@testing-library/user-event'; +import { http, HttpResponse } from 'msw'; + +const makeTrace = (sequence: number): Trace => ({ + id: `trace-${String(sequence).padStart(2, '0')}`, + session_id: `session-${sequence}`, + workspace: DEFAULT_WORKSPACE, + name: `Trace ${String(sequence).padStart(2, '0')}`, + started_at: `2026-07-20T12:${String(sequence).padStart(2, '0')}:00Z`, + status: 'success', +}); + +const installTraceHandler = ({ + traces, + failedIds = [], +}: { + traces: Trace[]; + failedIds?: string[]; +}) => { + const requests: Array<{ id: string; mode: string | null }> = []; + const tracesById = new Map(traces.map((trace) => [trace.id, trace])); + + server.use( + http.get('*/apis/intake/v2/workspaces/:workspace/traces/:traceId', ({ params, request }) => { + const id = String(params['traceId']); + requests.push({ id, mode: new URL(request.url).searchParams.get('mode') }); + + if (failedIds.includes(id)) { + return HttpResponse.json({ detail: `Could not load ${id}` }, { status: 500 }); + } + + return HttpResponse.json(tracesById.get(id)); + }) + ); + + return requests; +}; + +describe('InsightTracesTable', () => { + it('requests and displays only the current page in reference order using preview mode', async () => { + const user = userEvent.setup(); + const traces = Array.from({ length: 11 }, (_, index) => makeTrace(index + 1)); + const traceIds = traces.map((trace) => trace.id); + const requests = installTraceHandler({ traces }); + + renderRoute(, { + history: '/optimizer?page_size=10', + }); + + await screen.findByText('Trace 01'); + await waitFor(() => + expect(requests).toEqual( + traceIds.slice(0, 10).map((id) => ({ + id, + mode: 'preview', + })) + ) + ); + expect(screen.queryByText('Trace 11')).not.toBeInTheDocument(); + const rows = screen.getAllByRole('row'); + expect(rows[1]).toHaveTextContent('Trace 01'); + expect(rows[2]).toHaveTextContent('Trace 02'); + expect(screen.getByText('1-10 of 11 items')).toBeInTheDocument(); + + await user.click(screen.getByRole('button', { name: /next page/i })); + + expect(await screen.findByText('Trace 11')).toBeInTheDocument(); + await waitFor(() => expect(requests.map(({ id }) => id)).toEqual(traceIds)); + expect(screen.queryByText('Trace 01')).not.toBeInTheDocument(); + expect(screen.getByText('11-11 of 11 items')).toBeInTheDocument(); + }); + + it('keeps successful rows visible when part of the current page fails', async () => { + const traces = [makeTrace(1), makeTrace(2)]; + installTraceHandler({ traces, failedIds: [traces[1].id] }); + + renderRoute( + trace.id)} + /> + ); + + expect(await screen.findByText('Trace 01')).toBeInTheDocument(); + expect(screen.getByText("1 of 2 traces couldn't be loaded.")).toBeInTheDocument(); + expect(screen.getByText('1-2 of 2 items')).toBeInTheDocument(); + }); + + it('shows an error instead of an empty state when every current-page request fails', async () => { + const traces = [makeTrace(1), makeTrace(2)]; + installTraceHandler({ traces, failedIds: traces.map((trace) => trace.id) }); + + renderRoute( + trace.id)} + /> + ); + + expect(await screen.findByText('Error')).toBeInTheDocument(); + expect(screen.queryByText('This insight has no linked traces.')).not.toBeInTheDocument(); + }); +}); diff --git a/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.tsx b/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.tsx new file mode 100644 index 0000000000..31f0330548 --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.tsx @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { EditColumnsMenu } from '@nemo/common/src/components/DataView/internal'; +import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; +import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; +import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; +import { getGetTraceQueryKey, getTrace } from '@nemo/sdk/generated/platform/api'; +import type { Trace } from '@nemo/sdk/generated/platform/schema'; +import { Flex, Stack, Text } from '@nvidia/foundations-react-core'; +import { getErrorMessage } from '@studio/api/common/utils'; +import { IntakeTelemetryDataView } from '@studio/components/IntakeLists/IntakeTelemetryDataView'; +import { makeIntakeTraceColumns } from '@studio/components/IntakeLists/intakeTraceColumns'; +import { getIntakeSessionTraceRoute } from '@studio/routes/utils'; +import { useQueries } from '@tanstack/react-query'; +import { Columns3, TriangleAlert } from 'lucide-react'; +import { type FC } from 'react'; +import { useNavigate } from 'react-router-dom'; + +const TRACE_PREVIEW_PARAMS = { mode: 'preview' } as const; + +export interface InsightTracesTableProps { + workspace: string; + /** Intake trace ids (the insight's `trace_refs`). */ + traceIds: string[]; +} + +/** + * Renders an insight's evidence traces using the same columns and DataView shell as + * `IntakeTracesTable`. Unlike the workspace browse table, this fetches only the referenced + * traces by id and preserves `traceIds` order (no server sort/filter). + */ +export const InsightTracesTable: FC = ({ workspace, traceIds }) => { + const navigate = useNavigate(); + const dataViewState = useStudioDataViewState(); + const { pageIndex, pageSize } = dataViewState.pagination.state; + const firstVisibleIndex = pageIndex * pageSize; + const visibleTraceIds = traceIds.slice(firstVisibleIndex, firstVisibleIndex + pageSize); + + const results = useQueries({ + queries: visibleTraceIds.map((id) => ({ + queryKey: getGetTraceQueryKey(workspace, id, TRACE_PREVIEW_PARAMS), + queryFn: ({ signal }) => getTrace(workspace, id, TRACE_PREVIEW_PARAMS, signal), + enabled: Boolean(workspace) && Boolean(id), + })), + }); + + const traces = results.map((r) => r.data).filter((t): t is Trace => Boolean(t)); + const isFetching = results.some((r) => r.isFetching); + const failedCount = results.filter((r) => r.isError).length; + const allFailed = + visibleTraceIds.length > 0 && failedCount === visibleTraceIds.length && !isFetching; + const firstError = results.find((r) => r.error)?.error; + + return ( + + {failedCount > 0 && !allFailed ? ( + + + + {failedCount} of {visibleTraceIds.length} traces couldn't be loaded. + + + ) : null} + + dataViewState={dataViewState} + makeColumns={makeIntakeTraceColumns()} + onRowClick={(trace) => + navigate(getIntakeSessionTraceRoute(workspace, trace.session_id, trace.id)) + } + toolbarSlotEnd={ + } + > + <> + + Columns + + + } + attributes={{ + DataViewRoot: { + data: traces, + totalCount: traceIds.length, + requestStatus: allFailed ? 'error' : isFetching ? 'loading' : undefined, + }, + DataViewTableContent: { + renderEmptyState: () => ( + + ), + renderErrorState: () => ( + + ), + }, + }} + /> + + ); +}; diff --git a/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/InsightExperimentGroups.tsx b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/InsightExperimentGroups.tsx new file mode 100644 index 0000000000..d2b3917b3c --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/InsightExperimentGroups.tsx @@ -0,0 +1,139 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import * as DataView from '@nemo/common/src/components/DataView/internal'; +import { useRowClick } from '@nemo/common/src/components/DataView/useRowClick'; +import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; +import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; +import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; +import { DEFAULT_PAGE_SIZE_OPTIONS } from '@nemo/common/src/constants/pagination'; +import { useListExperimentGroups } from '@nemo/sdk/generated/platform/api'; +import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import { Button, Text } from '@nvidia/foundations-react-core'; +import { getExperimentGroupDetailRoute } from '@studio/routes/utils'; +import { keepPreviousData } from '@tanstack/react-query'; +import { FlaskConical } from 'lucide-react'; +import { type ComponentProps, type FC } from 'react'; +import { useNavigate } from 'react-router-dom'; + +const makeColumns: ComponentProps>['makeColumns'] = ({ + accessor, +}) => [ + accessor('name', { + header: 'Experiments', + enableSorting: false, + size: 280, + cell: ({ getValue }) => {getValue()}, + }), + accessor('evaluation_count', { + header: 'Evaluations', + enableSorting: false, + size: 100, + cell: ({ getValue }) => {getValue() ?? 0}, + }), + accessor('updated_at', { + header: 'Updated', + enableSorting: false, + size: 120, + cell: ({ row }) => + row.original.updated_at ? ( + + ) : ( + — + ), + }), +]; + +interface InsightExperimentGroupsProps { + workspace: string; + insightId: string; + onRunExperiment: () => void; + runExperimentDisabled: boolean; +} + +export const InsightExperimentGroups: FC = ({ + workspace, + insightId, + onRunExperiment, + runExperimentDisabled, +}) => { + const navigate = useNavigate(); + const dataViewState = DataView.useDataViewState({ + pagination: { paginationOptions: DEFAULT_PAGE_SIZE_OPTIONS }, + }); + const { pageIndex, pageSize } = dataViewState.pagination.state; + const { + data: response, + isError, + isFetching, + refetch, + } = useListExperimentGroups( + workspace, + { + page: pageIndex + 1, + page_size: pageSize, + sort: '-created_at', + filter: { insight_id: insightId }, + }, + { query: { placeholderData: keepPreviousData } } + ); + const groups = response?.data ?? []; + const { wrapColumns, onClick, className } = useRowClick( + (group: ExperimentGroupResponse) => + navigate(getExperimentGroupDetailRoute(workspace, group.name)), + groups + ); + + return ( + +
+ ( + } + actions={ + + } + /> + )} + renderErrorState={() => ( + void refetch()}> + Retry + + } + /> + )} + /> + +
+ + ); +}; diff --git a/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.test.tsx b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.test.tsx new file mode 100644 index 0000000000..fe75bc0565 --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.test.tsx @@ -0,0 +1,192 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + getListEvaluationsQueryKey, + getListExperimentGroupsQueryKey, +} from '@nemo/sdk/generated/platform/api'; +import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import type { Insight } from '@studio/api/optimizer'; +import { PLATFORM_BASE_URL } from '@studio/constants/environment'; +import { ROUTES } from '@studio/constants/routes'; +import { server } from '@studio/mocks/node'; +import { OptimizerInsightRoute } from '@studio/routes/optimizer/OptimizerInsightRoute'; +import { getOptimizerInsightRoute } from '@studio/routes/utils'; +import { renderRoute, screen, waitFor, within } from '@studio/tests/util/render'; +import userEvent from '@testing-library/user-event'; +import { http, HttpResponse } from 'msw'; +import { useParams } from 'react-router-dom'; + +const WORKSPACE = 'workspace-a'; +const INSIGHT_ID = 'insight-a'; +const INSIGHT_URL = `${PLATFORM_BASE_URL}/apis/insights/v2/workspaces/:workspace/insights/:insightId`; +const GROUPS_URL = `${PLATFORM_BASE_URL}${getListExperimentGroupsQueryKey(':workspace')[0]}`; +const EVALUATIONS_URL = `${PLATFORM_BASE_URL}${getListEvaluationsQueryKey(':workspace')[0]}`; + +const insight: Insight = { + id: INSIGHT_ID, + name: INSIGHT_ID, + title: 'Slow responses', + description: 'The agent responds too slowly.', + agent: 'research-agent', + status: 'open', + trace_refs: [], +}; + +const pagination = ({ + page = 1, + pageSize = 10, + currentPageSize = 0, + totalResults = 0, +}: { + page?: number; + pageSize?: number; + currentPageSize?: number; + totalResults?: number; +} = {}) => ({ + page, + page_size: pageSize, + current_page_size: currentPageSize, + total_pages: Math.ceil(totalResults / pageSize), + total_results: totalResults, +}); + +const makeGroup = ( + id: string, + overrides: Partial = {} +): ExperimentGroupResponse => + ({ + id, + name: `${id}-name`, + workspace: WORKSPACE, + insight_id: insight.id, + default_sort: '-created_at', + summary: `${id} summary`, + description: `${id} description`, + evaluation_count: 3, + created_at: '2026-07-19T12:00:00Z', + updated_at: '2026-07-20T12:00:00Z', + ...overrides, + }) as ExperimentGroupResponse; + +const ExperimentGroupDestination = () => { + const { experimentGroupName } = useParams(); + return
{`Opened ${experimentGroupName}`}
; +}; + +const renderInsight = (history = getOptimizerInsightRoute(WORKSPACE, INSIGHT_ID)) => + renderRoute(undefined, { + history, + routes: [ + { path: ROUTES.workspace.optimizerInsight, element: }, + { + path: ROUTES.workspace.experimentGroupDetail, + element: , + }, + ], + }); + +describe('OptimizerInsightRoute experiments', () => { + beforeEach(() => { + server.use( + http.get(INSIGHT_URL, () => HttpResponse.json(insight)), + http.get(GROUPS_URL, () => HttpResponse.json({ data: [], pagination: pagination() })) + ); + }); + + it('renders a compact Experiments list without requesting Evaluations', async () => { + const group = makeGroup('latency-experiment'); + const evaluationRequest = vi.fn(() => HttpResponse.json({})); + server.use( + http.get(GROUPS_URL, ({ request }) => { + const params = new URL(request.url).searchParams; + expect(params.get('filter[insight_id]')).toBe(INSIGHT_ID); + return HttpResponse.json({ + data: [group], + pagination: pagination({ currentPageSize: 1, totalResults: 1 }), + }); + }), + http.get(EVALUATIONS_URL, evaluationRequest) + ); + + renderInsight(); + + const row = await screen.findByRole('row', { name: new RegExp(group.name) }); + expect(within(row).getByText(String(group.evaluation_count))).toBeInTheDocument(); + expect(evaluationRequest).not.toHaveBeenCalled(); + }); + + it('distinguishes a group-list failure from a successful empty page', async () => { + server.use(http.get(GROUPS_URL, () => new HttpResponse(null, { status: 500 }))); + const { unmount } = renderInsight(); + + expect(await screen.findByText('Failed to load experiments')).toBeInTheDocument(); + expect(screen.queryByText('No experiments for this insight.')).not.toBeInTheDocument(); + unmount(); + + server.use( + http.get(GROUPS_URL, () => HttpResponse.json({ data: [], pagination: pagination() })) + ); + renderInsight(); + + expect(await screen.findByText('No experiments for this insight.')).toBeInTheDocument(); + expect(screen.queryByText('Failed to load experiments')).not.toBeInTheDocument(); + }); + + it('requests and displays only the selected ExperimentGroup page', async () => { + const user = userEvent.setup(); + const firstPageGroups = Array.from({ length: 10 }, (_unused, index) => + makeGroup(`page-one-${index + 1}`) + ); + const secondPageGroup = makeGroup('page-two-1'); + const requests: Array<{ page: string | null; pageSize: string | null }> = []; + + server.use( + http.get(GROUPS_URL, ({ request }) => { + const params = new URL(request.url).searchParams; + const page = params.get('page'); + requests.push({ page, pageSize: params.get('page_size') }); + return HttpResponse.json({ + data: page === '2' ? [secondPageGroup] : firstPageGroups, + pagination: pagination({ + page: Number(page ?? 1), + currentPageSize: page === '2' ? 1 : 10, + totalResults: 11, + }), + }); + }) + ); + + renderInsight(); + + expect(await screen.findByText(firstPageGroups[0].name)).toBeInTheDocument(); + expect(screen.queryByText(secondPageGroup.name)).not.toBeInTheDocument(); + + const nextPageButton = screen + .getAllByRole('button', { name: /next page/i }) + .find((button) => !button.hasAttribute('disabled')); + if (!nextPageButton) throw new Error('Enabled Experiments next-page button not found'); + await user.click(nextPageButton); + + expect(await screen.findByText(secondPageGroup.name)).toBeInTheDocument(); + expect(screen.queryByText(firstPageGroups[0].name)).not.toBeInTheDocument(); + await waitFor(() => + expect(requests).toEqual([ + { page: '1', pageSize: '10' }, + { page: '2', pageSize: '10' }, + ]) + ); + + await user.click(screen.getByText(secondPageGroup.name)); + expect(await screen.findByText(`Opened ${secondPageGroup.name}`)).toBeInTheDocument(); + }); + + it('reports status mutation failures through the Studio toast', async () => { + server.use(http.patch(INSIGHT_URL, () => new HttpResponse(null, { status: 500 }))); + renderInsight(); + + await userEvent.click(await screen.findByRole('button', { name: 'Delete' })); + + expect(await screen.findByText('Failed to update insight.')).toBeInTheDocument(); + }); +}); diff --git a/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.tsx b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.tsx new file mode 100644 index 0000000000..6bbc162cc7 --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.tsx @@ -0,0 +1,217 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; +import { KVPair } from '@nemo/common/src/components/KVPair'; +import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; +import { useToast } from '@nemo/common/src/providers/toast/useToast'; +import { + Anchor, + Button, + Card, + Divider, + Flex, + PageHeader, + Stack, + Tag, + Text, +} from '@nvidia/foundations-react-core'; +import { + getOptimizerGetInsightQueryKey, + getOptimizerListInsightsQueryKey, + useOptimizerGetInsight, + useOptimizerUpdateInsight, + type InsightStatus, +} from '@studio/api/optimizer'; +import { AccessibleTitle } from '@studio/components/AccessibleTitle'; +import { ExpandableMessage } from '@studio/components/ExpandableMessage'; +import { FeatureFlagBadge } from '@studio/components/FeatureFlagBadge'; +import { Loading } from '@studio/components/Layouts/Loading'; +import { LINK_DOCS_STUDIO_EVALUATION } from '@studio/constants/links'; +import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; +import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; +import { InsightOpenModal } from '@studio/routes/optimizer/InsightOpenModal'; +import { insightActions, insightStatusColor } from '@studio/routes/optimizer/insightStatus'; +import { InsightTracesTable } from '@studio/routes/optimizer/InsightTracesTable'; +import { InsightExperimentGroups } from '@studio/routes/optimizer/OptimizerInsightRoute/InsightExperimentGroups'; +import { getOptimizerRoute } from '@studio/routes/utils'; +import { useQueryClient } from '@tanstack/react-query'; +import { type FC, useState } from 'react'; +import { Link, useParams } from 'react-router-dom'; + +export const OptimizerInsightRoute: FC = () => { + const workspace = useWorkspaceFromPath(); + const { insightId = '' } = useParams<{ insightId: string }>(); + const queryClient = useQueryClient(); + const toast = useToast(); + + const { + data: insight, + isLoading, + isError, + refetch, + } = useOptimizerGetInsight(workspace, insightId); + + const { mutate: updateInsight, isPending: isUpdating } = useOptimizerUpdateInsight({ + mutation: { + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: getOptimizerGetInsightQueryKey(workspace, insightId), + }); + queryClient.invalidateQueries({ + queryKey: getOptimizerListInsightsQueryKey(workspace), + }); + }, + onError: () => toast.error('Failed to update insight.'), + }, + }); + + const [openModalOpen, setOpenModalOpen] = useState(false); + + // The external agent changes the status after it creates the experiment. + const handleAction = (target: InsightStatus) => { + if (target === 'open') { + setOpenModalOpen(true); + return; + } + updateInsight({ workspace, insightId, data: { status: target } }); + }; + + useBreadcrumbs({ + items: [ + { href: getOptimizerRoute(workspace), slotLabel: 'Insights' }, + { slotLabel: insight?.title ?? insightId }, + ], + }); + + if (isLoading && !insight) { + return ; + } + + if (isError || !insight) { + return ( + + + + + + + + + } + /> + + + ); + } + + const traceRefs = insight.trace_refs ?? []; + + return ( + + + + {insight.title} + + + } + slotDescription={ + <> + Insight generated from observed sessions by the analyst agent.{' '} + + Learn more + + + } + slotActions={ + + {insightActions(insight.status).map((action) => ( + + ))} + + } + /> + +
+ + {insight.status} + + } + /> + + + + : '—'} + /> + + : '—'} + /> +
+ +
+ + + Description + {insight.description ? ( + + ) : ( + — + )} + + + + + handleAction('open')} + runExperimentDisabled={isUpdating} + /> + +
+ + + Observed Sessions ({traceRefs.length}) + + + + + setOpenModalOpen(false)} + /> + + ); +}; diff --git a/web/packages/studio/src/routes/optimizer/OptimizerRoute/index.test.tsx b/web/packages/studio/src/routes/optimizer/OptimizerRoute/index.test.tsx new file mode 100644 index 0000000000..565a0ef767 --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/OptimizerRoute/index.test.tsx @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { DEFAULT_WORKSPACE } from '@nemo/common/src/models/constants'; +import type { InsightListItem } from '@studio/api/optimizer'; +import { PLATFORM_BASE_URL } from '@studio/constants/environment'; +import { ROUTES } from '@studio/constants/routes'; +import { server } from '@studio/mocks/node'; +import { OptimizerRoute } from '@studio/routes/optimizer/OptimizerRoute'; +import { getOptimizerRoute } from '@studio/routes/utils'; +import { renderRoute, screen, within } from '@studio/tests/util/render'; +import { http, HttpResponse } from 'msw'; + +const INSIGHTS_URL = `${PLATFORM_BASE_URL}/apis/insights/v2/workspaces/:workspace/insights`; +const EXPERIMENT_GROUPS_URL = '*/apis/intake/v2/workspaces/:workspace/experiment-groups'; + +const makeInsight = (id: string, title: string): InsightListItem => ({ + id, + name: id, + title, + description: `${title} description`, + agent: 'research-agent', + status: 'open', + trace_refs: ['trace-1'], + experiment_group_count: null, + created_at: '2026-07-20T12:00:00Z', + updated_at: '2026-07-20T12:00:00Z', +}); + +const insightsPage = (data: InsightListItem[]) => ({ + data, + pagination: { + page: 1, + page_size: 50, + current_page_size: data.length, + total_pages: 1, + total_results: data.length, + }, +}); + +const renderList = () => + renderRoute(undefined, { + history: getOptimizerRoute(DEFAULT_WORKSPACE), + routes: [{ path: ROUTES.workspace.optimizer, element: }], + }); + +const findCell = async (insightTitle: string, columnName: string): Promise => { + const row = await screen.findByRole('row', { name: new RegExp(insightTitle) }); + + const headers = screen.getAllByRole('columnheader'); + const column = headers.findIndex((header) => header.textContent?.includes(columnName)); + if (column < 0) throw new Error(`${columnName} column not found`); + + return within(row).getAllByRole('cell')[column]; +}; + +describe('OptimizerRoute', () => { + it('renders server-provided list metadata without per-row requests', async () => { + const experimentGroupRequest = vi.fn(() => HttpResponse.json({})); + const insights = [ + { + ...makeInsight('positive', 'Positive count'), + experiment_group_count: 7, + last_seen_at: '2026-07-21T12:00:00Z', + }, + { ...makeInsight('zero', 'Zero count'), experiment_group_count: 0 }, + makeInsight('null', 'Null count'), + ]; + server.use( + http.get(INSIGHTS_URL, () => HttpResponse.json(insightsPage(insights))), + http.get(EXPERIMENT_GROUPS_URL, experimentGroupRequest) + ); + + renderList(); + + expect(await findCell('Positive count', 'Experiments')).toHaveTextContent('7'); + expect(await findCell('Zero count', 'Experiments')).toHaveTextContent('0'); + expect(await findCell('Null count', 'Experiments')).toHaveTextContent('—'); + expect((await findCell('Positive count', 'Last Seen')).querySelector('time')).toHaveAttribute( + 'datetime', + '2026-07-21T12:00:00Z' + ); + expect(await findCell('Zero count', 'Last Seen')).toHaveTextContent('—'); + expect(experimentGroupRequest).not.toHaveBeenCalled(); + }); +}); diff --git a/web/packages/studio/src/routes/optimizer/OptimizerRoute/index.tsx b/web/packages/studio/src/routes/optimizer/OptimizerRoute/index.tsx new file mode 100644 index 0000000000..7c50bcc2c2 --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/OptimizerRoute/index.tsx @@ -0,0 +1,175 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + ROW_ACTIONS_COLUMN_SIZE, + StudioDataView, +} from '@nemo/common/src/components/DataView/StudioDataView'; +import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; +import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; +import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; +import { Flex, PageHeader, Stack, Tag, Text } from '@nvidia/foundations-react-core'; +import { getErrorMessage } from '@studio/api/common/utils'; +import { type InsightListItem, useOptimizerListInsights } from '@studio/api/optimizer'; +import { AccessibleTitle } from '@studio/components/AccessibleTitle'; +import { ErrorPanel } from '@studio/components/ErrorPanel'; +import { FeatureFlagBadge } from '@studio/components/FeatureFlagBadge'; +import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; +import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; +import { insightStatusColor } from '@studio/routes/optimizer/insightStatus'; +import { getOptimizerInsightRoute, getOptimizerRoute } from '@studio/routes/utils'; +import { keepPreviousData } from '@tanstack/react-query'; +import { Lightbulb } from 'lucide-react'; +import { type ComponentProps, type FC } from 'react'; +import { useNavigate } from 'react-router-dom'; + +export const OptimizerRoute: FC = () => { + const workspace = useWorkspaceFromPath(); + + useBreadcrumbs({ + items: [{ href: getOptimizerRoute(workspace), slotLabel: 'Insights' }], + }); + + const navigate = useNavigate(); + + const dataViewState = useStudioDataViewState({ + defaultSort: [{ id: 'created_at', desc: true }], + }); + + const sortState = dataViewState.sorting.state[0]; + const sortParam = sortState ? `${sortState.desc ? '-' : ''}${sortState.id}` : '-created_at'; + + const { data, isFetching, error } = useOptimizerListInsights( + workspace, + { + page: dataViewState.pagination.state.pageIndex + 1, + page_size: dataViewState.pagination.state.pageSize, + sort: sortParam, + }, + { query: { placeholderData: keepPreviousData } } + ); + + const makeColumns: ComponentProps>['makeColumns'] = ( + { accessor }, + { rowActionsColumn } + ) => [ + accessor('status', { + header: 'Status', + enableSorting: false, + size: 110, + cell({ row }) { + const status = row.original.status; + return ( + + {status} + + ); + }, + }), + accessor('title', { + header: 'Insight', + enableSorting: false, + size: 240, + cell({ row }) { + return {row.original.title}; + }, + }), + accessor('agent', { + header: 'Agent', + enableSorting: false, + size: 160, + cell({ row }) { + return {row.original.agent || '—'}; + }, + }), + accessor('trace_refs', { + id: 'traces', + header: 'Traces', + enableSorting: false, + size: 80, + cell({ row }) { + return {row.original.trace_refs?.length ?? 0}; + }, + }), + accessor('experiment_group_count', { + header: 'Experiments', + enableSorting: false, + size: 110, + cell({ row }) { + return {row.original.experiment_group_count ?? '—'}; + }, + }), + accessor('created_at', { + header: 'Created', + enableSorting: true, + size: 140, + cell({ row }) { + return row.original.created_at ? ( + + ) : ( + — + ); + }, + }), + accessor('last_seen_at', { + header: 'Last Seen', + enableSorting: false, + size: 140, + cell({ row }) { + return row.original.last_seen_at ? ( + + ) : ( + — + ); + }, + }), + rowActionsColumn({ + size: ROW_ACTIONS_COLUMN_SIZE, + enableResizing: false, + rowActions: () => [], + }), + ]; + + return ( + + + + Insights + + + } + slotDescription="Leverage the optimizer agent to review your code and traces and suggest insights." + /> + navigate(getOptimizerInsightRoute(workspace, row.id))} + attributes={{ + DataViewRoot: { + data: data?.data ?? [], + totalCount: data?.pagination?.total_results, + requestStatus: error ? 'error' : isFetching ? 'loading' : undefined, + }, + DataViewTableContent: { + renderEmptyState: () => ( + } + header="No insights yet" + emptyMessage="Run an optimizer analysis on an agent to surface insights here." + /> + ), + renderErrorState: () => ( + + ), + }, + }} + /> + + + ); +}; diff --git a/web/packages/studio/src/routes/optimizer/insightStatus.test.ts b/web/packages/studio/src/routes/optimizer/insightStatus.test.ts new file mode 100644 index 0000000000..8d7e8867d7 --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/insightStatus.test.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { insightActions } from '@studio/routes/optimizer/insightStatus'; + +describe('insightStatus', () => { + it('only offers transitions accepted by the optimizer API', () => { + expect(insightActions('open')).toEqual([ + { label: 'Delete', target: 'deleted', kind: 'secondary' }, + { label: 'Resolve', target: 'resolved', kind: 'primary' }, + ]); + expect(insightActions('resolved')).toEqual([ + { label: 'Run experiment', target: 'open', kind: 'primary', color: 'brand' }, + ]); + expect(insightActions('deleted')).toEqual([ + { label: 'Run experiment', target: 'open', kind: 'primary', color: 'brand' }, + ]); + }); +}); diff --git a/web/packages/studio/src/routes/optimizer/insightStatus.ts b/web/packages/studio/src/routes/optimizer/insightStatus.ts new file mode 100644 index 0000000000..6bb7072272 --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/insightStatus.ts @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { Button, Tag } from '@nvidia/foundations-react-core'; +import type { InsightStatus } from '@studio/api/optimizer'; +import type { ComponentProps } from 'react'; + +type TagColor = ComponentProps['color']; + +/** Tag color per insight status. Exhaustive over InsightStatus. */ +export const INSIGHT_STATUS_COLOR: Record = { + open: 'blue', + resolved: 'green', + deleted: 'gray', +}; + +export const insightStatusColor = (status: InsightStatus): TagColor => + INSIGHT_STATUS_COLOR[status] ?? 'gray'; + +/** A status-change action rendered as a button on the insight page. */ +export interface InsightAction { + label: string; + /** Status the button transitions the insight to. */ + target: InsightStatus; + kind: 'primary' | 'secondary'; + /** Optional button color; 'brand' renders the green primary CTA. */ + color?: ComponentProps['color']; +} + +// Canonical order: secondary actions first, Run experiment (green brand CTA) on the right. +const OPEN: InsightAction = { + label: 'Run experiment', + target: 'open', + kind: 'primary', + color: 'brand', +}; +const DELETE: InsightAction = { label: 'Delete', target: 'deleted', kind: 'secondary' }; +const RESOLVE: InsightAction = { label: 'Resolve', target: 'resolved', kind: 'primary' }; + +export const insightActions = (status: InsightStatus): InsightAction[] => { + return status === 'open' ? [DELETE, RESOLVE] : [OPEN]; +}; diff --git a/web/packages/studio/src/routes/utils.ts b/web/packages/studio/src/routes/utils.ts index 02aa05752a..dd11d506f1 100644 --- a/web/packages/studio/src/routes/utils.ts +++ b/web/packages/studio/src/routes/utils.ts @@ -22,6 +22,7 @@ import { JOBS_ENABLED, MEMBERS_ENABLED, MODEL_COMPARE_ENABLED, + OPTIMIZER_ENABLED, SAFE_SYNTHESIZER_ENABLED, SECRETS_ENABLED, SETTINGS_ENABLED, @@ -102,6 +103,9 @@ export const gateDeploymentsRoutes = (routes: RouteObject | RouteObject[]) => export const gateModelCompareRoutes = (routes: RouteObject | RouteObject[]) => gateRoutes(MODEL_COMPARE_ENABLED, routes); +export const gateOptimizerRoutes = (routes: RouteObject | RouteObject[]) => + gateRoutes(OPTIMIZER_ENABLED, routes); + type WorkspacePathParams = { workspace: string; }; @@ -390,6 +394,14 @@ export const getGuardrailsRoute = (workspace: string) => { return generatePath(ROUTES.workspace.guardrails, { workspace }); }; +export const getOptimizerRoute = (workspace: string) => { + return generatePath(ROUTES.workspace.optimizer, { workspace }); +}; + +export const getOptimizerInsightRoute = (workspace: string, insightId: string) => { + return generatePath(ROUTES.workspace.optimizerInsight, { workspace, insightId }); +}; + export const getGuardrailDetailRoute = (workspace: string, guardrailConfigName: string) => { return generatePath(ROUTES.workspace.guardrailDetail, { workspace, diff --git a/web/packages/studio/src/tests/title-change.test.tsx b/web/packages/studio/src/tests/title-change.test.tsx index bf5b391c0e..2ebfeaa519 100644 --- a/web/packages/studio/src/tests/title-change.test.tsx +++ b/web/packages/studio/src/tests/title-change.test.tsx @@ -36,6 +36,7 @@ const pathParams = { [RP.benchmarkName]: 'test-benchmark', [RP.experimentGroupName]: 'test-experiment-group', [RP.evaluationName]: 'test-experiment', + [RP.insightId]: 'test-insight', [RP.guardrailConfigName]: 'test-guardrail-config', }; From 0dd2f51c1f51c8a77f74c92c2687302bfe08f878 Mon Sep 17 00:00:00 2001 From: Aditya Pandey Date: Thu, 23 Jul 2026 14:46:34 -0600 Subject: [PATCH 2/6] test(studio): clarify optimizer route assertion Signed-off-by: Aditya Pandey --- web/packages/studio/src/routes/index.test.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/packages/studio/src/routes/index.test.tsx b/web/packages/studio/src/routes/index.test.tsx index f6a91c6dc7..e8ae6c7f05 100644 --- a/web/packages/studio/src/routes/index.test.tsx +++ b/web/packages/studio/src/routes/index.test.tsx @@ -195,9 +195,9 @@ describe('Routes', () => { vi.stubEnv('VITE_FF_OPTIMIZER_ENABLED', 'preview'); const { routes } = await import('./index'); - optimizerRoutes.forEach((route) => { - expect(findIfRouteExists(routes, route)).toBe(true); - }); + const missingRoutes = optimizerRoutes.filter((route) => !findIfRouteExists(routes, route)); + + expect(missingRoutes).toHaveLength(0); }); }); }); From ad4396e12d4a1280f161985bb7d780789b204896 Mon Sep 17 00:00:00 2001 From: Aditya Pandey Date: Thu, 23 Jul 2026 15:05:28 -0600 Subject: [PATCH 3/6] chore(sdk): sync insights API changes Signed-off-by: Aditya Pandey --- sdk/python/nemo-platform/.nmpcontext/openapi.yaml | 9 +++++++-- .../resources/experiment_groups/experiment_groups.py | 12 ++++++------ .../experiment_group_filter_param.py | 3 +++ .../experiment_group_list_params.py | 5 +++-- .../tests/api_resources/test_experiment_groups.py | 2 ++ 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index a07756d247..829a0150ee 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -4104,8 +4104,9 @@ paths: explode: true schema: $ref: '#/components/schemas/ExperimentGroupFilter' - description: 'Filter experiment groups by name, or by a metadata key/value: - filter[metadata.]=.' + description: Filter experiment groups by name, insight_id, is_deleted, or + a metadata key/value (filter[metadata.]=). Pass is_deleted=true + to return only soft-deleted groups; omit to see only live ones. responses: '200': description: Successful Response @@ -11237,6 +11238,10 @@ components: description: Filter groups by name. title: Name type: string + insight_id: + description: Filter groups by the id of the insight that seeded them. + title: Insight Id + type: string is_deleted: description: When true, returns only soft-deleted groups. Omit (or false) to see only live groups. diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py index 728c4b0d9a..7d8ea35079 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiment_groups/experiment_groups.py @@ -299,9 +299,9 @@ def list( List Experiment Groups Args: - filter: - Filter experiment groups by name, or by a metadata key/value: - filter[metadata.]=. + filter: Filter experiment groups by name, insight_id, is_deleted, or a metadata + key/value (filter[metadata.]=). Pass is_deleted=true to return only + soft-deleted groups; omit to see only live ones. page: Page number. @@ -636,9 +636,9 @@ def list( List Experiment Groups Args: - filter: - Filter experiment groups by name, or by a metadata key/value: - filter[metadata.]=. + filter: Filter experiment groups by name, insight_id, is_deleted, or a metadata + key/value (filter[metadata.]=). Pass is_deleted=true to return only + soft-deleted groups; omit to see only live ones. page: Page number. diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_filter_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_filter_param.py index 382aaf78f4..b7eef3e8a5 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_filter_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_filter_param.py @@ -26,6 +26,9 @@ class ExperimentGroupFilterParam(TypedDict, total=False): """Filter for listing ExperimentGroups.""" + insight_id: str + """Filter groups by the id of the insight that seeded them.""" + is_deleted: bool """When true, returns only soft-deleted groups. diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_list_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_list_params.py index 914c80cbd2..2acfe15417 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_list_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiment_groups/experiment_group_list_params.py @@ -29,8 +29,9 @@ class ExperimentGroupListParams(TypedDict, total=False): filter: ExperimentGroupFilterParam """ - Filter experiment groups by name, or by a metadata key/value: - filter[metadata.]=. + Filter experiment groups by name, insight_id, is_deleted, or a metadata + key/value (filter[metadata.]=). Pass is_deleted=true to return only + soft-deleted groups; omit to see only live ones. """ page: int diff --git a/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py b/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py index c713ab8143..673df6bcc2 100644 --- a/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py +++ b/sdk/python/nemo-platform/tests/api_resources/test_experiment_groups.py @@ -241,6 +241,7 @@ def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: experiment_group = client.experiment_groups.list( workspace="workspace", filter={ + "insight_id": "insight_id", "is_deleted": True, "metadata": {"foo": "string"}, "name": "name", @@ -549,6 +550,7 @@ async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform experiment_group = await async_client.experiment_groups.list( workspace="workspace", filter={ + "insight_id": "insight_id", "is_deleted": True, "metadata": {"foo": "string"}, "name": "name", From f503a4d6b50954e392be68f4e1b15ece49b31777 Mon Sep 17 00:00:00 2001 From: Aditya Pandey Date: Fri, 24 Jul 2026 11:39:46 -0600 Subject: [PATCH 4/6] fix(insights): batch insight trace loading Load each visible trace page in one Intake request to avoid per-row API fan-out while preserving evidence order and failure feedback. Signed-off-by: Aditya Pandey --- .../src/nmp/intake/spans/api/query_filters.py | 10 +++ .../intake/src/nmp/intake/spans/api/traces.py | 3 +- .../intake/src/nmp/intake/spans/domain.py | 2 +- .../src/nmp/intake/spans/trace_repository.py | 8 +- services/intake/tests/test_traces_api.py | 5 +- .../test_traces_clickhouse_repository.py | 4 +- .../InsightTracesTable/index.test.tsx | 77 ++++++++----------- .../optimizer/InsightTracesTable/index.tsx | 48 ++++++------ 8 files changed, 81 insertions(+), 76 deletions(-) diff --git a/services/intake/src/nmp/intake/spans/api/query_filters.py b/services/intake/src/nmp/intake/spans/api/query_filters.py index 09feee38b5..f388eb5d58 100644 --- a/services/intake/src/nmp/intake/spans/api/query_filters.py +++ b/services/intake/src/nmp/intake/spans/api/query_filters.py @@ -38,6 +38,16 @@ def require_string_value(comparison: ComparisonOperation) -> str: return value +def require_string_or_list_value(comparison: ComparisonOperation) -> list[str]: + """Accept an equality string or an `$in` list of strings, normalized to a list.""" + if comparison.operator != FilterOperator.IN: + return [require_string_value(comparison)] + value = comparison.value + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise _bad_filter(f"Filter field {comparison.field!r} with $in must be a list of strings.") + return value + + def require_enum_value(comparison: ComparisonOperation, enum_type: type[_TEnum]) -> _TEnum: value = require_eq_value(comparison) try: diff --git a/services/intake/src/nmp/intake/spans/api/traces.py b/services/intake/src/nmp/intake/spans/api/traces.py index 4baab00a58..8928f79f9c 100644 --- a/services/intake/src/nmp/intake/spans/api/traces.py +++ b/services/intake/src/nmp/intake/spans/api/traces.py @@ -15,6 +15,7 @@ filter_comparisons, require_datetime_value, require_enum_value, + require_string_or_list_value, require_string_value, ) from nmp.intake.spans.api.traces_schemas import Trace, TraceFilter, TraceMode, TraceSortField @@ -113,7 +114,7 @@ def _trace_filter(workspace: str, parsed: ParsedFilter) -> TraceListFilter: filters = TraceListFilter(workspace=workspace) for comparison in filter_comparisons(parsed): if comparison.field == "id": - filters.trace_id = require_string_value(comparison) + filters.trace_ids = require_string_or_list_value(comparison) elif comparison.field == "session_id": filters.session_id = require_string_value(comparison) elif comparison.field == "status": diff --git a/services/intake/src/nmp/intake/spans/domain.py b/services/intake/src/nmp/intake/spans/domain.py index 6708514049..cfc0067534 100644 --- a/services/intake/src/nmp/intake/spans/domain.py +++ b/services/intake/src/nmp/intake/spans/domain.py @@ -89,7 +89,7 @@ class SpanGroup(BaseModel): class TraceListFilter(BaseModel): workspace: str - trace_id: str | None = None + trace_ids: list[str] | None = None session_id: str | None = None source_format: str | None = None status: SpanStatus | None = None diff --git a/services/intake/src/nmp/intake/spans/trace_repository.py b/services/intake/src/nmp/intake/spans/trace_repository.py index 03572c790e..8a1d75e6a6 100644 --- a/services/intake/src/nmp/intake/spans/trace_repository.py +++ b/services/intake/src/nmp/intake/spans/trace_repository.py @@ -142,7 +142,7 @@ async def list_traces( async def get_trace(self, *, workspace: str, trace_id: str, mode: TraceMode) -> IntakeTrace | None: result = await self.list_traces( - filters=TraceListFilter(workspace=workspace, trace_id=trace_id), + filters=TraceListFilter(workspace=workspace, trace_ids=[trace_id]), page=1, page_size=1, sort="-started_at", @@ -370,9 +370,9 @@ def column(name: str) -> str: clauses = [f"{column('workspace')} = %(workspace)s", f"{column('is_deleted')} = 0"] parameters: dict[str, Any] = {"workspace": filters.workspace} - if filters.trace_id is not None: - clauses.append(f"{column('trace_id')} = %(trace_id)s") - parameters["trace_id"] = filters.trace_id + if filters.trace_ids is not None: + clauses.append(f"{column('trace_id')} IN %(trace_ids)s") + parameters["trace_ids"] = filters.trace_ids if filters.session_id is not None: clauses.append(f"{column('session_id')} = %(session_id)s") parameters["session_id"] = filters.session_id diff --git a/services/intake/tests/test_traces_api.py b/services/intake/tests/test_traces_api.py index a220a0a9d3..9b5ba1870f 100644 --- a/services/intake/tests/test_traces_api.py +++ b/services/intake/tests/test_traces_api.py @@ -19,7 +19,7 @@ def test_trace_filter_maps_public_fields_to_repository_filter(): "workspace-a", _parsed_filter( { - "id": "trace-a", + "id": {"$in": ["trace-a", "trace-b"]}, "session_id": "session-a", "status": "error", "started_at": {"$gte": started_at.isoformat()}, @@ -30,7 +30,7 @@ def test_trace_filter_maps_public_fields_to_repository_filter(): ) assert filters.workspace == "workspace-a" - assert filters.trace_id == "trace-a" + assert filters.trace_ids == ["trace-a", "trace-b"] assert filters.session_id == "session-a" assert filters.status == SpanStatus.ERROR assert filters.started_at_gte == started_at @@ -69,6 +69,7 @@ def test_trace_filter_schema_exposes_evaluation_id_with_deprecated_experiment_id def test_trace_filter_applies_no_implicit_time_bound(): filters = _trace_filter("workspace-a", _parsed_filter({"id": "trace-a"})) + assert filters.trace_ids == ["trace-a"] assert filters.started_at_gte is None assert filters.started_at_lte is None diff --git a/services/intake/tests/test_traces_clickhouse_repository.py b/services/intake/tests/test_traces_clickhouse_repository.py index f89f707601..7c6782da1f 100644 --- a/services/intake/tests/test_traces_clickhouse_repository.py +++ b/services/intake/tests/test_traces_clickhouse_repository.py @@ -66,7 +66,7 @@ async def test_summary_mode_reads_root_spans_without_metric_aggregates(): repository = _repository(client) await repository.list_traces( - filters=TraceListFilter(workspace="workspace-a"), + filters=TraceListFilter(workspace="workspace-a", trace_ids=["trace-a", "trace-b"]), page=1, page_size=10, sort="started_at", @@ -80,6 +80,8 @@ async def test_summary_mode_reads_root_spans_without_metric_aggregates(): assert "trace_roots.root_output" not in client.queries[0] assert "LIMIT 1 BY trace_roots.workspace, trace_roots.source_format, trace_roots.trace_id" in client.queries[0] assert "span_versions" not in client.queries[0] + assert "trace_roots.trace_id IN %(trace_ids)s" in client.queries[0] + assert client.parameters[0]["trace_ids"] == ["trace-a", "trace-b"] assert "sumIf" not in client.queries[1] assert "groupUniqArrayIf" not in client.queries[1] assert "span_versions" not in client.queries[1] diff --git a/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.test.tsx b/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.test.tsx index 979300e94a..4dda793484 100644 --- a/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.test.tsx +++ b/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.test.tsx @@ -20,24 +20,28 @@ const makeTrace = (sequence: number): Trace => ({ const installTraceHandler = ({ traces, - failedIds = [], + missingIds = [], }: { traces: Trace[]; - failedIds?: string[]; + missingIds?: string[]; }) => { - const requests: Array<{ id: string; mode: string | null }> = []; + const requests: URL[] = []; const tracesById = new Map(traces.map((trace) => [trace.id, trace])); server.use( - http.get('*/apis/intake/v2/workspaces/:workspace/traces/:traceId', ({ params, request }) => { - const id = String(params['traceId']); - requests.push({ id, mode: new URL(request.url).searchParams.get('mode') }); - - if (failedIds.includes(id)) { - return HttpResponse.json({ detail: `Could not load ${id}` }, { status: 500 }); - } - - return HttpResponse.json(tracesById.get(id)); + http.get('*/apis/intake/v2/workspaces/:workspace/traces', ({ request }) => { + const url = new URL(request.url); + requests.push(url); + const requestedIds = url.searchParams.getAll('filter[id][$in]'); + const data = requestedIds + .filter((id) => !missingIds.includes(id)) + .flatMap((id) => { + const trace = tracesById.get(id); + return trace ? [trace] : []; + }) + .reverse(); + + return HttpResponse.json({ data }); }) ); @@ -49,21 +53,21 @@ describe('InsightTracesTable', () => { const user = userEvent.setup(); const traces = Array.from({ length: 11 }, (_, index) => makeTrace(index + 1)); const traceIds = traces.map((trace) => trace.id); - const requests = installTraceHandler({ traces }); + const missingId = traces[9].id; + const requests = installTraceHandler({ traces, missingIds: [missingId] }); renderRoute(, { history: '/optimizer?page_size=10', }); + expect(screen.queryByText("10 of 10 traces couldn't be loaded.")).not.toBeInTheDocument(); + await screen.findByText('Trace 01'); - await waitFor(() => - expect(requests).toEqual( - traceIds.slice(0, 10).map((id) => ({ - id, - mode: 'preview', - })) - ) - ); + await waitFor(() => expect(requests).toHaveLength(1)); + expect(requests[0].searchParams.get('mode')).toBe('preview'); + expect(requests[0].searchParams.getAll('filter[id][$in]')).toEqual(traceIds.slice(0, 10)); + expect(screen.getByText("1 of 10 traces couldn't be loaded.")).toBeInTheDocument(); + expect(screen.queryByText('Trace 10')).not.toBeInTheDocument(); expect(screen.queryByText('Trace 11')).not.toBeInTheDocument(); const rows = screen.getAllByRole('row'); expect(rows[1]).toHaveTextContent('Trace 01'); @@ -73,36 +77,21 @@ describe('InsightTracesTable', () => { await user.click(screen.getByRole('button', { name: /next page/i })); expect(await screen.findByText('Trace 11')).toBeInTheDocument(); - await waitFor(() => expect(requests.map(({ id }) => id)).toEqual(traceIds)); + await waitFor(() => expect(requests).toHaveLength(2)); + expect(requests[1].searchParams.getAll('filter[id][$in]')).toEqual(['trace-11']); expect(screen.queryByText('Trace 01')).not.toBeInTheDocument(); expect(screen.getByText('11-11 of 11 items')).toBeInTheDocument(); }); - it('keeps successful rows visible when part of the current page fails', async () => { - const traces = [makeTrace(1), makeTrace(2)]; - installTraceHandler({ traces, failedIds: [traces[1].id] }); - - renderRoute( - trace.id)} - /> + it('shows an error instead of an empty state when the list request fails', async () => { + server.use( + http.get('*/apis/intake/v2/workspaces/:workspace/traces', () => + HttpResponse.json({ detail: 'Could not load traces' }, { status: 500 }) + ) ); - expect(await screen.findByText('Trace 01')).toBeInTheDocument(); - expect(screen.getByText("1 of 2 traces couldn't be loaded.")).toBeInTheDocument(); - expect(screen.getByText('1-2 of 2 items')).toBeInTheDocument(); - }); - - it('shows an error instead of an empty state when every current-page request fails', async () => { - const traces = [makeTrace(1), makeTrace(2)]; - installTraceHandler({ traces, failedIds: traces.map((trace) => trace.id) }); - renderRoute( - trace.id)} - /> + ); expect(await screen.findByText('Error')).toBeInTheDocument(); diff --git a/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.tsx b/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.tsx index 31f0330548..333b724133 100644 --- a/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.tsx +++ b/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.tsx @@ -1,24 +1,22 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { withOperators } from '@nemo/common/src/api/filterOperators'; import { EditColumnsMenu } from '@nemo/common/src/components/DataView/internal'; import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; -import { getGetTraceQueryKey, getTrace } from '@nemo/sdk/generated/platform/api'; -import type { Trace } from '@nemo/sdk/generated/platform/schema'; +import { useListTraces } from '@nemo/sdk/generated/platform/api'; +import type { Trace, TraceFilter } from '@nemo/sdk/generated/platform/schema'; import { Flex, Stack, Text } from '@nvidia/foundations-react-core'; import { getErrorMessage } from '@studio/api/common/utils'; import { IntakeTelemetryDataView } from '@studio/components/IntakeLists/IntakeTelemetryDataView'; import { makeIntakeTraceColumns } from '@studio/components/IntakeLists/intakeTraceColumns'; import { getIntakeSessionTraceRoute } from '@studio/routes/utils'; -import { useQueries } from '@tanstack/react-query'; import { Columns3, TriangleAlert } from 'lucide-react'; import { type FC } from 'react'; import { useNavigate } from 'react-router-dom'; -const TRACE_PREVIEW_PARAMS = { mode: 'preview' } as const; - export interface InsightTracesTableProps { workspace: string; /** Intake trace ids (the insight's `trace_refs`). */ @@ -37,24 +35,30 @@ export const InsightTracesTable: FC = ({ workspace, tra const firstVisibleIndex = pageIndex * pageSize; const visibleTraceIds = traceIds.slice(firstVisibleIndex, firstVisibleIndex + pageSize); - const results = useQueries({ - queries: visibleTraceIds.map((id) => ({ - queryKey: getGetTraceQueryKey(workspace, id, TRACE_PREVIEW_PARAMS), - queryFn: ({ signal }) => getTrace(workspace, id, TRACE_PREVIEW_PARAMS, signal), - enabled: Boolean(workspace) && Boolean(id), - })), - }); + const { data, error, isFetching } = useListTraces( + workspace, + { + filter: withOperators({ id: { $in: visibleTraceIds } }), + mode: 'preview', + page: 1, + page_size: pageSize, + }, + { + query: { + enabled: Boolean(workspace) && visibleTraceIds.length > 0, + }, + } + ); - const traces = results.map((r) => r.data).filter((t): t is Trace => Boolean(t)); - const isFetching = results.some((r) => r.isFetching); - const failedCount = results.filter((r) => r.isError).length; - const allFailed = - visibleTraceIds.length > 0 && failedCount === visibleTraceIds.length && !isFetching; - const firstError = results.find((r) => r.error)?.error; + const tracesById = new Map((data?.data ?? []).map((trace) => [trace.id, trace])); + const traces = visibleTraceIds + .map((id) => tracesById.get(id)) + .filter((trace): trace is Trace => trace !== undefined); + const failedCount = visibleTraceIds.length - traces.length; return ( - {failedCount > 0 && !allFailed ? ( + {failedCount > 0 && !error && !isFetching ? ( @@ -84,7 +88,7 @@ export const InsightTracesTable: FC = ({ workspace, tra DataViewRoot: { data: traces, totalCount: traceIds.length, - requestStatus: allFailed ? 'error' : isFetching ? 'loading' : undefined, + requestStatus: error ? 'error' : isFetching ? 'loading' : undefined, }, DataViewTableContent: { renderEmptyState: () => ( @@ -94,9 +98,7 @@ export const InsightTracesTable: FC = ({ workspace, tra /> ), renderErrorState: () => ( - + ), }, }} From 0de946cf441f00d9144053a321af2d8e46376992 Mon Sep 17 00:00:00 2001 From: Aditya Pandey Date: Fri, 24 Jul 2026 11:56:32 -0600 Subject: [PATCH 5/6] style(studio): format insight traces table Apply repository Prettier formatting so the Studio format check passes. Signed-off-by: Aditya Pandey --- .../src/routes/optimizer/InsightTracesTable/index.test.tsx | 4 +--- .../studio/src/routes/optimizer/InsightTracesTable/index.tsx | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.test.tsx b/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.test.tsx index 4dda793484..3d9d31c6bd 100644 --- a/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.test.tsx +++ b/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.test.tsx @@ -90,9 +90,7 @@ describe('InsightTracesTable', () => { ) ); - renderRoute( - - ); + renderRoute(); expect(await screen.findByText('Error')).toBeInTheDocument(); expect(screen.queryByText('This insight has no linked traces.')).not.toBeInTheDocument(); diff --git a/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.tsx b/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.tsx index 333b724133..e115ebc9bc 100644 --- a/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.tsx +++ b/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.tsx @@ -98,7 +98,9 @@ export const InsightTracesTable: FC = ({ workspace, tra /> ), renderErrorState: () => ( - + ), }, }} From 5d7e38a3b9de3d4d9d7c32b5989789febfaa5564 Mon Sep 17 00:00:00 2001 From: Aditya Pandey Date: Fri, 24 Jul 2026 14:28:54 -0600 Subject: [PATCH 6/6] fix(studio): restore experiment group description on the detail page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The insights work replaced the group detail header's description with a static blurb, so the group's editable description was rendered nowhere, and it gave group.summary a permanent panel that reads "—" for every group no producer has summarized. Show the description as the page subtitle again and render the summary panel only when it has content. The group update endpoint is a full replace, so also resend the summary, insight link, and metadata the edit form does not touch; saving a description edit was clearing all three. Move the originating insight link into the insight card's header as a compact mark and label so the description text uses the full card width. Signed-off-by: Aditya Pandey --- .../ExperimentGroupEditModal/index.test.tsx | 66 +++++++++++++++ .../ExperimentGroupEditModal/index.tsx | 5 ++ .../OriginatingInsightLink/index.tsx | 15 ++-- .../routes/EvaluationDetailRoute/index.tsx | 12 +-- .../ExperimentGroupDetailRoute/index.test.tsx | 81 +++++++++++-------- .../ExperimentGroupDetailRoute/index.tsx | 60 ++++++-------- 6 files changed, 155 insertions(+), 84 deletions(-) create mode 100644 web/packages/studio/src/components/ExperimentGroupEditModal/index.test.tsx diff --git a/web/packages/studio/src/components/ExperimentGroupEditModal/index.test.tsx b/web/packages/studio/src/components/ExperimentGroupEditModal/index.test.tsx new file mode 100644 index 0000000000..5deaa229ce --- /dev/null +++ b/web/packages/studio/src/components/ExperimentGroupEditModal/index.test.tsx @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import { ExperimentGroupEditModal } from '@studio/components/ExperimentGroupEditModal'; +import { server } from '@studio/mocks/node'; +import { render, screen } from '@studio/tests/util/render'; +import { waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { http, HttpResponse } from 'msw'; + +const WORKSPACE = 'test-workspace'; + +const group: ExperimentGroupResponse = { + id: 'group-id', + name: 'opt-group', + workspace: WORKSPACE, + description: 'Editable group description', + summary: 'Optimizer-authored summary', + insight_id: 'insight-id', + metadata: { producer: 'optimizer' }, + default_sort: '-created_at', + evaluation_count: 0, + experiment_count: 0, +}; + +describe('ExperimentGroupEditModal', () => { + it('resends the fields it does not edit, which a full-replace update would otherwise clear', async () => { + let body: Record | undefined; + server.use( + http.get('*/apis/intake/v2/workspaces/:workspace/evaluations', () => + HttpResponse.json({ + data: [], + pagination: { + page: 1, + page_size: 100, + current_page_size: 0, + total_pages: 0, + total_results: 0, + }, + }) + ), + http.put( + '*/apis/intake/v2/workspaces/:workspace/experiment-groups/:name', + async ({ request }) => { + body = (await request.json()) as Record; + return HttpResponse.json(group); + } + ) + ); + + render(); + + await userEvent.click(await screen.findByRole('button', { name: 'Save' })); + + await waitFor(() => expect(body).toBeDefined()); + expect(body).toMatchObject({ + name: group.name, + description: group.description, + default_sort: group.default_sort, + insight_id: group.insight_id, + summary: group.summary, + metadata: group.metadata, + }); + }); +}); diff --git a/web/packages/studio/src/components/ExperimentGroupEditModal/index.tsx b/web/packages/studio/src/components/ExperimentGroupEditModal/index.tsx index b6a56d2403..cdf86e8926 100644 --- a/web/packages/studio/src/components/ExperimentGroupEditModal/index.tsx +++ b/web/packages/studio/src/components/ExperimentGroupEditModal/index.tsx @@ -77,6 +77,11 @@ export const ExperimentGroupEditModal: FC = ({ name: group.name, description: description || undefined, default_sort: defaultSort, + // The endpoint replaces the whole group, so resend the producer-owned fields this + // form doesn't edit; omitting them clears the group's summary and insight link. + insight_id: group.insight_id, + summary: group.summary, + metadata: group.metadata, }, }); onClose(); diff --git a/web/packages/studio/src/components/OriginatingInsightLink/index.tsx b/web/packages/studio/src/components/OriginatingInsightLink/index.tsx index ed3dfe3313..985c63c751 100644 --- a/web/packages/studio/src/components/OriginatingInsightLink/index.tsx +++ b/web/packages/studio/src/components/OriginatingInsightLink/index.tsx @@ -13,8 +13,8 @@ interface OriginatingInsightLinkProps { } /** - * The Voyager artwork above an "Originating insight" label, both linking to the insight detail page. - * Shared by the experiment group detail page and the experiment detail root-cause card. + * The Voyager mark and an "Originating insight" label on one line, both linking to the insight + * detail page. Sits in the top-right corner of the insight cards that reference it. */ export const OriginatingInsightLink: FC = ({ insightId }) => { const workspace = useWorkspaceFromPath(); @@ -22,13 +22,12 @@ export const OriginatingInsightLink: FC = ({ insigh - - - Originating -
- insight + {/* The artwork is authored at 90x78 and stretches to its box, so keep that ratio. */} + + + Originating insight diff --git a/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx b/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx index 6db07615b2..dccc9645f1 100644 --- a/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx +++ b/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx @@ -53,13 +53,13 @@ export const EvaluationDetailRoute: FC = () => { {showInsightCard ? ( - - - + + Insight description - {insight?.description} - - + + + {insight?.description} + ) : null}
diff --git a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.test.tsx b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.test.tsx index a538763633..b786824948 100644 --- a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.test.tsx +++ b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.test.tsx @@ -26,44 +26,57 @@ const group = { evaluation_count: 0, } satisfies Partial; -describe('ExperimentGroupDetailRoute', () => { - it('renders the summary without requesting Optimizer when disabled', async () => { - const insightRequest = vi.fn(); - server.use( - http.get('*/apis/intake/v2/workspaces/:workspace/experiment-groups/:name', () => - HttpResponse.json(group) - ), - http.get('*/apis/intake/v2/workspaces/:workspace/evaluations', () => - HttpResponse.json({ - data: [], - pagination: { - page: 1, - page_size: 25, - current_page_size: 0, - total_pages: 0, - total_results: 0, - }, - }) - ), - http.get('*/apis/insights/v2/workspaces/:workspace/insights/:insightId', () => { - insightRequest(); - return HttpResponse.json({}); +const mockGroup = (overrides?: Partial) => { + const insightRequest = vi.fn(); + server.use( + http.get('*/apis/intake/v2/workspaces/:workspace/experiment-groups/:name', () => + HttpResponse.json({ ...group, ...overrides }) + ), + http.get('*/apis/intake/v2/workspaces/:workspace/evaluations', () => + HttpResponse.json({ + data: [], + pagination: { + page: 1, + page_size: 25, + current_page_size: 0, + total_pages: 0, + total_results: 0, + }, }) - ); + ), + http.get('*/apis/insights/v2/workspaces/:workspace/insights/:insightId', () => { + insightRequest(); + return HttpResponse.json({}); + }) + ); - renderRoute(, { - history: `/workspaces/${WORKSPACE}/experiment/${GROUP_NAME}`, - routes: [ - { - path: ROUTES.workspace.experimentGroupDetail, - element: , - }, - ], - }); + renderRoute(, { + history: `/workspaces/${WORKSPACE}/experiment/${GROUP_NAME}`, + routes: [ + { + path: ROUTES.workspace.experimentGroupDetail, + element: , + }, + ], + }); + + return insightRequest; +}; - expect(await screen.findByText('Generated group summary')).toBeInTheDocument(); - expect(screen.queryByText('Editable group description')).not.toBeInTheDocument(); +describe('ExperimentGroupDetailRoute', () => { + it('renders the group description and summary without requesting Optimizer when disabled', async () => { + const insightRequest = mockGroup(); + + expect(await screen.findByText('Editable group description')).toBeInTheDocument(); + expect(screen.getByText('Generated group summary')).toBeInTheDocument(); expect(insightRequest).not.toHaveBeenCalled(); expect(screen.queryByRole('link', { name: /originating insight/i })).not.toBeInTheDocument(); }); + + it('omits the summary panel for groups no producer has summarized', async () => { + mockGroup({ summary: undefined }); + + expect(await screen.findByText('Editable group description')).toBeInTheDocument(); + expect(screen.queryByText('Summary')).not.toBeInTheDocument(); + }); }); diff --git a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx index 3b2aca712e..11c89ccf44 100644 --- a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx +++ b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx @@ -3,22 +3,13 @@ import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; import { useGetExperimentGroup } from '@nemo/sdk/generated/platform/api'; -import { - Anchor, - Button, - Card, - Flex, - PageHeader, - Stack, - Text, -} from '@nvidia/foundations-react-core'; +import { Button, Card, Flex, PageHeader, Stack, Text } from '@nvidia/foundations-react-core'; import { useOptimizerGetInsight } from '@studio/api/optimizer'; import { AccessibleTitle } from '@studio/components/AccessibleTitle'; import { ExperimentGroupDataView } from '@studio/components/dataViews/ExperimentGroupDataView'; import { ExperimentGroupEditModal } from '@studio/components/ExperimentGroupEditModal'; import { OriginatingInsightLink } from '@studio/components/OriginatingInsightLink'; import { OPTIMIZER_ENABLED } from '@studio/constants/environment'; -import { LINK_DOCS_STUDIO_EVALUATION } from '@studio/constants/links'; import { ROUTE_PARAMS } from '@studio/constants/routes'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; @@ -59,14 +50,7 @@ export const ExperimentGroupDetailRoute: FC = () => { - An experiment is a group of evaluation runs aligned toward a common objective.{' '} - - Learn more - - - } + slotDescription={group?.description || undefined} slotActions={