diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index f91c0ade04..4b9cff5d30 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -699,6 +699,16 @@ paths: default: -created_at title: Sort description: Sort field + - name: count_by + in: query + required: false + schema: + description: Optional direct string data field whose matching values should + be counted. + title: Count By + type: string + description: Optional direct string data field whose matching values should + be counted. - name: filter in: query required: false @@ -4045,8 +4055,9 @@ paths: explode: true schema: $ref: '#/components/schemas/ExperimentGroupFilter' - description: 'Filter experiment groups by name, or by a metadata key/value: - filter[metadata.]=.' + description: Filter experiment groups by name, insight_id, is_deleted, or + a metadata key/value (filter[metadata.]=). Pass is_deleted=true + to return only soft-deleted groups; omit to see only live ones. responses: '200': description: Successful Response @@ -10276,6 +10287,11 @@ components: description: Filtering information. additionalProperties: true type: object + group_counts: + title: Group Counts + additionalProperties: + type: integer + type: object type: object required: - data @@ -11106,6 +11122,10 @@ components: description: Filter groups by name. title: Name type: string + insight_id: + description: Filter groups by the id of the insight that seeded them. + title: Insight Id + type: string is_deleted: description: When true, returns only soft-deleted groups. Omit (or false) to see only live groups. diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index f91c0ade04..4b9cff5d30 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -699,6 +699,16 @@ paths: default: -created_at title: Sort description: Sort field + - name: count_by + in: query + required: false + schema: + description: Optional direct string data field whose matching values should + be counted. + title: Count By + type: string + description: Optional direct string data field whose matching values should + be counted. - name: filter in: query required: false @@ -4045,8 +4055,9 @@ paths: explode: true schema: $ref: '#/components/schemas/ExperimentGroupFilter' - description: 'Filter experiment groups by name, or by a metadata key/value: - filter[metadata.]=.' + description: Filter experiment groups by name, insight_id, is_deleted, or + a metadata key/value (filter[metadata.]=). Pass is_deleted=true + to return only soft-deleted groups; omit to see only live ones. responses: '200': description: Successful Response @@ -10276,6 +10287,11 @@ components: description: Filtering information. additionalProperties: true type: object + group_counts: + title: Group Counts + additionalProperties: + type: integer + type: object type: object required: - data @@ -11106,6 +11122,10 @@ components: description: Filter groups by name. title: Name type: string + insight_id: + description: Filter groups by the id of the insight that seeded them. + title: Insight Id + type: string is_deleted: description: When true, returns only soft-deleted groups. Omit (or false) to see only live groups. diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index f91c0ade04..4b9cff5d30 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -699,6 +699,16 @@ paths: default: -created_at title: Sort description: Sort field + - name: count_by + in: query + required: false + schema: + description: Optional direct string data field whose matching values should + be counted. + title: Count By + type: string + description: Optional direct string data field whose matching values should + be counted. - name: filter in: query required: false @@ -4045,8 +4055,9 @@ paths: explode: true schema: $ref: '#/components/schemas/ExperimentGroupFilter' - description: 'Filter experiment groups by name, or by a metadata key/value: - filter[metadata.]=.' + description: Filter experiment groups by name, insight_id, is_deleted, or + a metadata key/value (filter[metadata.]=). Pass is_deleted=true + to return only soft-deleted groups; omit to see only live ones. responses: '200': description: Successful Response @@ -10276,6 +10287,11 @@ components: description: Filtering information. additionalProperties: true type: object + group_counts: + title: Group Counts + additionalProperties: + type: integer + type: object type: object required: - data @@ -11106,6 +11122,10 @@ components: description: Filter groups by name. title: Name type: string + insight_id: + description: Filter groups by the id of the insight that seeded them. + title: Insight Id + type: string is_deleted: description: When true, returns only soft-deleted groups. Omit (or false) to see only live groups. diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/entities.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/entities.py index 56dde5bd12..0351d6e3e0 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/entities.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/entities.py @@ -233,6 +233,14 @@ async def list( page: int = 1, page_size: int = 100, ) -> ListResponse[EntityT]: ... + async def count_by( + self, + entity_type: EntityTypeLike, + field: str, + *, + workspace: str = DEFAULT_WORKSPACE, + filter_obj: dict[str, Any] | None = None, + ) -> dict[str, int]: ... async def get(self, entity_type: EntityTypeLike, name: str, *, workspace: Optional[str] = None) -> EntityT: ... async def get_by_id(self, entity_type: EntityTypeLike, entity_id: str) -> EntityT: ... async def update(self, entity: EntityT, *, original_name: str | None = None) -> EntityT: ... @@ -493,6 +501,34 @@ async def list( return ListResponse(data=entities, pagination=pagination) + async def count_by( + self, + entity_type: EntityTypeLike, + field: str, + *, + workspace: str = DEFAULT_WORKSPACE, + filter_obj: dict[str, Any] | None = None, + ) -> dict[str, int]: + """Return the number of matching entities grouped by ``field``.""" + if not field.isidentifier(): + raise ValueError(f"Field '{field}' is not a direct entity data field") + + filter_dict = _convert_filter_obj_to_filter_str(filter_obj) if filter_obj else {} + effective_filter = json.dumps(filter_dict) if filter_dict else omit + + response = await self.entities_api.list( + _get_entity_type(entity_type), + workspace=workspace, + filter=effective_filter, + page=1, + page_size=1, + extra_query={"count_by": f"data.{field}"}, + ) + group_counts = getattr(response, "group_counts", None) + if group_counts is None: + raise EntityStoreError("Grouped counts not found in response") + return TypeAdapter(dict[str, int]).validate_python(group_counts) + async def create(self, entity: EntityT) -> EntityT: """Create a new entity. diff --git a/packages/nemo_platform_plugin/tests/test_entity_client.py b/packages/nemo_platform_plugin/tests/test_entity_client.py new file mode 100644 index 0000000000..3847de42ab --- /dev/null +++ b/packages/nemo_platform_plugin/tests/test_entity_client.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from unittest.mock import AsyncMock, Mock + +import pytest +from nemo_platform.types.entities import EntitiesPage +from nemo_platform.types.shared.pagination_data import PaginationData +from nemo_platform_plugin.entities import EntityClient, EntityStoreError + + +class ExperimentGroup: + __entity_type__ = "experiment_group" + + +def _entities_page(group_counts: dict[str, int] | None = None) -> EntitiesPage: + return EntitiesPage.model_construct( + data=[], + pagination=PaginationData(page=1, page_size=1, current_page_size=0, total_pages=0, total_results=0), + group_counts=group_counts, + ) + + +@pytest.mark.asyncio +async def test_count_by_returns_grouped_counts_for_shorthand_filter() -> None: + mock_api = Mock() + mock_api.list = AsyncMock(return_value=_entities_page(group_counts={"insight-a": 2})) + client = EntityClient(mock_api) + + counts = await client.count_by( + ExperimentGroup, + "insight_id", + filter_obj={ + "insight_id": {"$in": ["insight-a"]}, + "is_deleted": False, + }, + ) + + assert counts == {"insight-a": 2} + assert mock_api.list.await_args.kwargs["filter"] == ( + '{"data.insight_id": {"$in": ["insight-a"]}, "data.is_deleted": false}' + ) + assert mock_api.list.await_args.kwargs["extra_query"] == {"count_by": "data.insight_id"} + + +@pytest.mark.asyncio +async def test_count_by_rejects_response_without_grouped_counts() -> None: + mock_api = Mock() + mock_api.list = AsyncMock(return_value=_entities_page()) + client = EntityClient(mock_api) + + with pytest.raises(EntityStoreError, match="Grouped counts not found"): + await client.count_by(ExperimentGroup, "insight_id") + + +@pytest.mark.asyncio +async def test_count_by_rejects_non_direct_field() -> None: + mock_api = Mock() + client = EntityClient(mock_api) + + with pytest.raises(ValueError, match="direct entity data field"): + await client.count_by(ExperimentGroup, "data.insight_id") diff --git a/plugins/nemo-insights/scripts/insights_demo.py b/plugins/nemo-insights/scripts/insights_demo.py new file mode 100755 index 0000000000..fa4ec803e4 --- /dev/null +++ b/plugins/nemo-insights/scripts/insights_demo.py @@ -0,0 +1,514 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Seed or clean a local NeMo Insights Studio demo through public HTTP APIs. + +From the repository root:: + + uv sync --group insights + services/intake/scripts/spans/run_clickhouse.sh + cd web && VITE_FF_OPTIMIZER_ENABLED=preview pnpm --filter nemo-studio-ui build:fastapi && cd .. + uv run nemo services run \ + --service-group all --config packages/nmp_platform/config/local.yaml + uv run plugins/nemo-insights/scripts/insights_demo.py seed + +Open ``http://localhost:8080/studio/workspaces/insights-demo/optimizer``. +Clean with ``uv run plugins/nemo-insights/scripts/insights_demo.py clean``. +This Insights UI is separate from the existing Agents ``Suggestions`` feature. +The script deletes only the fixed ``insights-demo`` workspace. Intake has no +public telemetry-delete API, so deterministic session IDs keep reseeding stable +when inaccessible ClickHouse rows remain. +""" + +from __future__ import annotations + +import argparse +import os +import sys +import time +from collections.abc import Callable +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Any, Literal + +import httpx + +DEMO_WORKSPACE = "insights-demo" +DEFAULT_BASE_URL = "http://localhost:8080" +CLICKHOUSE_RECOVERY_COMMAND = "services/intake/scripts/spans/run_clickhouse.sh" +INSIGHTS_INSTALL_COMMAND = "uv sync --group insights" +_BASE_TIME = datetime(2026, 7, 20, 12, 0, tzinfo=timezone.utc) +_SOURCE_URL = "https://github.com/NVIDIA-NeMo/nemo-platform/commit" +_SUPPORT_AGENT = ("support-agent", "nvidia/nemotron-mini") +_RETRIEVAL_AGENT = ("retrieval-agent", "nvidia/llama-3.3-nemotron-super") + + +def _workspace_path(service: str, suffix: str = "") -> str: + return f"/apis/{service}/v2/workspaces/{DEMO_WORKSPACE}{suffix}" + + +class DemoError(RuntimeError): + """A concise, actionable demo failure.""" + + +@dataclass(frozen=True) +class SessionSpec: + session_id: str + test_case_id: str + started_at: datetime + latency_ms: int + cost_usd: float + quality: float + correctness: float + + +@dataclass(frozen=True) +class EvaluationSpec: + name: str + source_link: str + agent: tuple[str, str] + sessions: tuple[SessionSpec, ...] + + +@dataclass(frozen=True) +class GroupSpec: + name: str + description: str + summary: str + default_sort: str + insight_key: str + evaluations: tuple[EvaluationSpec, ...] + + +@dataclass(frozen=True) +class InsightSpec: + key: str + title: str + description: str + status: Literal["open", "resolved", "deleted"] + trace_refs: tuple[str, ...] + + +@dataclass(frozen=True) +class DemoFixture: + insights: tuple[InsightSpec, ...] + groups: tuple[GroupSpec, ...] + + +def _evaluation( + name: str, + commit: str, + count: int, + *, + start_index: int, + latency_ms: int, + cost_usd: float, + quality: float, + correctness: float, + agent: tuple[str, str], +) -> EvaluationSpec: + return EvaluationSpec( + name=name, + source_link=f"{_SOURCE_URL}/{commit}", + agent=agent, + sessions=tuple( + SessionSpec( + session_id=f"insights-demo-{name}-{index + 1:02d}", + test_case_id=f"case-{index + 1:02d}", + started_at=_BASE_TIME + timedelta(minutes=7 * (start_index + index)), + latency_ms=latency_ms + index * 125, + cost_usd=round(cost_usd + index * 0.002, 3), + quality=round(min(1.0, quality + index * 0.02), 2), + correctness=round(min(1.0, correctness + index * 0.01), 2), + ) + for index in range(count) + ), + ) + + +def build_fixture() -> DemoFixture: + """Return the deterministic, compact demo fixture.""" + prompt_baseline = _evaluation( + "prompt-baseline", + "1111111", + 3, + start_index=0, + latency_ms=1850, + cost_usd=0.032, + quality=0.72, + correctness=0.79, + agent=_SUPPORT_AGENT, + ) + prompt_compact = _evaluation( + "prompt-compact-context", + "2222222", + 3, + start_index=3, + latency_ms=1120, + cost_usd=0.021, + quality=0.84, + correctness=0.88, + agent=_SUPPORT_AGENT, + ) + router_baseline = _evaluation( + "router-baseline", + "3333333", + 3, + start_index=6, + latency_ms=2300, + cost_usd=0.041, + quality=0.68, + correctness=0.76, + agent=_RETRIEVAL_AGENT, + ) + router_streaming = _evaluation( + "router-streaming-cache", + "4444444", + 2, + start_index=9, + latency_ms=940, + cost_usd=0.027, + quality=0.89, + correctness=0.91, + agent=_RETRIEVAL_AGENT, + ) + groups = ( + GroupSpec( + name="prompt-response-time", + description="Prompt experiments addressing slow support responses.", + summary="Compact context improved quality while reducing average latency and cost.", + default_sort="-evaluators.quality.mean,cost_usd.mean", + insight_key="slow-responses", + evaluations=(prompt_baseline, prompt_compact), + ), + GroupSpec( + name="retrieval-routing", + description="Routing experiments for retrieval-heavy support questions.", + summary="Streaming cache-aware retrieval produced the strongest latency result.", + default_sort="latency_ms.mean,-evaluators.correctness.mean", + insight_key="slow-responses", + evaluations=(router_baseline, router_streaming), + ), + ) + trace_refs = tuple( + session.session_id for group in groups for evaluation in group.evaluations for session in evaluation.sessions + ) + return DemoFixture( + insights=( + InsightSpec( + key="slow-responses", + title="Support responses are slow on retrieval-heavy requests", + description=( + "Evidence traces show retrieval and oversized prompt context dominate response time. " + "Compare prompt compaction and cache-aware routing experiments." + ), + status="open", + trace_refs=trace_refs, + ), + InsightSpec( + key="stable-quality", + title="Answer quality remains stable after prompt compaction", + description=("Resolved after compact-context evaluations preserved correctness while lowering cost."), + status="resolved", + trace_refs=trace_refs[:2], + ), + InsightSpec( + key="legacy-router", + title="Legacy routing recommendation is no longer actionable", + description="Deleted after cache-aware routing superseded the original recommendation.", + status="deleted", + trace_refs=trace_refs[-1:], + ), + ), + groups=groups, + ) + + +class DemoAPI: + """Small public-HTTP client for the APIs used by the demo.""" + + def __init__(self, base_url: str, *, client: httpx.Client | None = None) -> None: + self.base_url = base_url.rstrip("/") + self._owns_client = client is None + self.client = client or httpx.Client(timeout=15.0) + + def close(self) -> None: + if self._owns_client: + self.client.close() + + def __enter__(self) -> DemoAPI: + return self + + def __exit__(self, *_args: object) -> None: + self.close() + + def _request( + self, + method: str, + path: str, + *, + expected: tuple[int, ...] = (200,), + json: dict[str, Any] | None = None, + params: dict[str, Any] | None = None, + ) -> httpx.Response: + try: + response = self.client.request( + method, + f"{self.base_url}{path}", + json=json, + params=params, + ) + except httpx.RequestError as error: + raise DemoError(f"Platform is unavailable at {self.base_url}: {error}") from error + if response.status_code not in expected: + detail = response.text.strip() + suffix = f": {detail}" if detail else "" + raise DemoError(f"{method} {path} failed ({response.status_code}){suffix}") + return response + + def preflight(self) -> None: + self._request("GET", "/health/ready") + openapi = self._request("GET", "/openapi.json").json() + paths = openapi.get("paths", {}) + if "/apis/insights/v2/workspaces/{workspace}/insights" not in paths: + raise DemoError( + f"Insights service is unavailable. Install the optional plugin with: {INSIGHTS_INSTALL_COMMAND}" + ) + response = self._request( + "GET", + "/apis/intake/v2/workspaces/default/traces", + expected=(200, 404, 503), + params={"page": 1, "page_size": 1, "mode": "summary"}, + ) + if response.status_code != 200: + raise DemoError(f"Intake is unavailable. Start ClickHouse with: {CLICKHOUSE_RECOVERY_COMMAND}") + + def delete_workspace(self, *, sleep: Callable[[float], None]) -> None: + path = _workspace_path("entities") + self._request( + "DELETE", + path, + expected=(200, 404), + ) + for _ in range(20): + response = self._request("GET", path, expected=(200, 404)) + if response.status_code == 404: + return + sleep(0.25) + raise DemoError(f"Timed out deleting workspace '{DEMO_WORKSPACE}'") + + def create_workspace(self, *, sleep: Callable[[float], None]) -> None: + for _ in range(40): + response = self._request( + "POST", + "/apis/entities/v2/workspaces", + expected=(200, 201, 409), + json={ + "name": DEMO_WORKSPACE, + "description": "Deterministic local NeMo Insights Studio demo.", + }, + ) + if response.status_code != 409: + return + sleep(0.5) + raise DemoError(f"Timed out recreating workspace '{DEMO_WORKSPACE}'") + + def create_insight(self, insight: InsightSpec) -> str: + response = self._request( + "POST", + _workspace_path("insights", "/insights"), + expected=(201,), + json={ + "title": insight.title, + "description": insight.description, + "agent": "insights-demo-agent", + "status": insight.status, + "trace_refs": [], + }, + ) + return str(response.json()["id"]) + + def update_insight_traces(self, insight_id: str, trace_refs: tuple[str, ...]) -> None: + self._request( + "PATCH", + _workspace_path("insights", f"/insights/{insight_id}"), + json={"trace_refs": list(trace_refs)}, + ) + + def create_group(self, group: GroupSpec, insight_id: str) -> str: + response = self._request( + "POST", + _workspace_path("intake", "/experiment-groups"), + expected=(201,), + json={ + "name": group.name, + "description": group.description, + "summary": group.summary, + "insight_id": insight_id, + "default_sort": group.default_sort, + }, + ) + return str(response.json()["id"]) + + def create_evaluation(self, group_id: str, evaluation: EvaluationSpec) -> None: + self._request( + "POST", + _workspace_path("intake", "/evaluations"), + expected=(201,), + json={ + "name": evaluation.name, + "experiment_group_id": group_id, + "dataset_name": "insights-demo-cases", + "source_link": evaluation.source_link, + }, + ) + + def ingest_session(self, evaluation: EvaluationSpec, session: SessionSpec) -> None: + finished_at = session.started_at + timedelta(milliseconds=session.latency_ms) + agent_name, model_name = evaluation.agent + self._request( + "POST", + _workspace_path("intake", "/ingest/atif"), + expected=(201,), + json={ + "schema_version": "ATIF-v1.7", + "session_id": session.session_id, + "evaluation_context": { + "evaluation_id": evaluation.name, + "test_case_id": session.test_case_id, + }, + "extra": { + "verifier": { + "started_at": _iso(session.started_at), + "finished_at": _iso(finished_at), + }, + "verifier_result": { + "rewards": { + "quality": session.quality, + "correctness": session.correctness, + } + }, + }, + "agent": { + "name": agent_name, + "version": "1.0.0", + "model_name": model_name, + }, + "steps": [ + { + "step_id": 1, + "timestamp": _iso(session.started_at), + "source": "user", + "message": f"Investigate support request {session.test_case_id}.", + }, + { + "step_id": 2, + "timestamp": _iso(finished_at), + "source": "agent", + "model_name": model_name, + "message": "Retrieved current evidence and returned a grounded answer.", + "metrics": { + "prompt_tokens": 320, + "completion_tokens": 96, + "cost_usd": session.cost_usd, + }, + }, + ], + }, + ) + + def evaluation_is_ready(self, evaluation: EvaluationSpec) -> bool: + response = self._request( + "GET", + _workspace_path("intake", f"/evaluations/{evaluation.name}"), + ).json() + return ( + response.get("run_count") == len(evaluation.sessions) + and bool(response.get("aggregate_scores")) + and response.get("cost_usd") is not None + and response.get("latency_ms") is not None + ) + + +def clean( + api: DemoAPI, + *, + sleep: Callable[[float], None] = time.sleep, + quiet: bool = False, +) -> None: + """Delete only the dedicated demo workspace.""" + api.delete_workspace(sleep=sleep) + if not quiet: + print(f"Deleted workspace '{DEMO_WORKSPACE}'.") + print("ClickHouse telemetry remains physically stored; public Intake APIs expose no delete operation.") + + +def seed( + api: DemoAPI, + *, + sleep: Callable[[float], None] = time.sleep, +) -> None: + """Recreate and seed the deterministic demo workspace.""" + api.preflight() + clean(api, sleep=sleep, quiet=True) + api.create_workspace(sleep=sleep) + + fixture = build_fixture() + insight_ids = {insight.key: api.create_insight(insight) for insight in fixture.insights} + evaluations: list[EvaluationSpec] = [] + for group in fixture.groups: + group_id = api.create_group(group, insight_ids[group.insight_key]) + for evaluation in group.evaluations: + api.create_evaluation(group_id, evaluation) + evaluations.append(evaluation) + for session in evaluation.sessions: + api.ingest_session(evaluation, session) + + for insight in fixture.insights: + api.update_insight_traces(insight_ids[insight.key], insight.trace_refs) + + pending = {evaluation.name: evaluation for evaluation in evaluations} + for _ in range(30): + pending = {name: evaluation for name, evaluation in pending.items() if not api.evaluation_is_ready(evaluation)} + if not pending: + break + sleep(1.0) + if pending: + raise DemoError(f"Timed out waiting for evaluation rollups: {', '.join(sorted(pending))}") + + print( + f"Seeded workspace '{DEMO_WORKSPACE}' with {len(fixture.insights)} insights, " + f"{len(fixture.groups)} groups, {len(evaluations)} evaluations, and 11 traces." + ) + print(f"Open {api.base_url}/studio/workspaces/{DEMO_WORKSPACE}/optimizer") + + +def _iso(value: datetime) -> str: + return value.isoformat().replace("+00:00", "Z") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("seed", "clean")) + parser.add_argument( + "--base-url", + default=os.getenv("NMP_BASE_URL", DEFAULT_BASE_URL), + help=f"Platform base URL (default: NMP_BASE_URL or {DEFAULT_BASE_URL}).", + ) + args = parser.parse_args(argv) + + try: + with DemoAPI(args.base_url) as api: + if args.command == "seed": + seed(api) + else: + clean(api) + except DemoError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/_perms.py b/plugins/nemo-insights/src/nemo_insights_plugin/_perms.py index 9691133348..2f1a3c1a41 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/_perms.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/_perms.py @@ -14,6 +14,13 @@ class InsightPerms(PermissionSet, namespace="insights.insights"): DELETE = perm("Delete insights") +class EvalAuthorRunPerms(PermissionSet, namespace="insights.eval-author-runs"): + CREATE = perm("Create Eval Author runs") + LIST = perm("List Eval Author runs") + READ = perm("Read Eval Author runs") + UPDATE = perm("Update Eval Author runs") + + class AnalysisConfigPerms(PermissionSet, namespace="insights.analysis-configs"): ENABLE = perm("Enable periodic analysis") DISABLE = perm("Disable periodic analysis") diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/entities.py b/plugins/nemo-insights/src/nemo_insights_plugin/entities.py index aa4765f6a6..537960d110 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/entities.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/entities.py @@ -7,7 +7,7 @@ from enum import StrEnum from nemo_platform_plugin.entity import NemoEntity -from pydantic import Field +from pydantic import BaseModel, ConfigDict, Field class InsightStatus(StrEnum): @@ -24,6 +24,112 @@ class AnalysisConfigStatus(StrEnum): ERROR = "error" +class EvalAuthorRunStatus(StrEnum): + """Lifecycle state for an externally executed Eval Author run.""" + + CREATED = "created" + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + CANCELLED = "cancelled" + + +class EvalAuthorRunStage(StrEnum): + """Current producer stage for an Eval Author run.""" + + INITIALIZING = "initializing" + MATERIALIZING_TRACES = "materializing_traces" + ANALYZING_TRACES = "analyzing_traces" + DISCOVERING_RUNNER = "discovering_runner" + AUTHORING_VERIFIER = "authoring_verifier" + VALIDATING = "validating" + PUBLISHING = "publishing" + COMPLETED = "completed" + + +class EvalAuthorCaptureStatus(StrEnum): + """Completeness of one captured artifact family.""" + + COMPLETE = "complete" + PARTIAL = "partial" + UNAVAILABLE = "unavailable" + + +class EvalAuthorConfigDetails(BaseModel): + """Resolved Eval Author tuning parameters.""" + + model_config = ConfigDict(extra="forbid") + + max_traces: int = Field(default=10, ge=1) + max_summary_tokens: int = Field(default=80_000, ge=1) + max_validation_repair_attempts: int = Field(default=5, ge=0, le=10) + + +class EvalAuthorInputs(BaseModel): + """Small references to the inputs consumed by a run.""" + + model_config = ConfigDict(extra="forbid") + + agent: str + task_template: str + train_dataset: str + validation_dataset: str + trace_refs: list[str] = Field(default_factory=list) + + +class EvalAuthorModels(BaseModel): + """Resolved model configuration used by the producer.""" + + model_config = ConfigDict(extra="forbid") + + smart: str + fast: str + + +class EvalAuthorProvenance(BaseModel): + """Source and runner provenance for a run.""" + + model_config = ConfigDict(extra="forbid") + + optimizer_branch: str + optimizer_commit: str + runner: str + + +class EvalAuthorOutputs(BaseModel): + """Fileset-backed outputs and compact result metadata.""" + + model_config = ConfigDict(extra="forbid") + + artifact_fileset: str | None = None + insight_suite: str | None = None + train_dataset: str | None = None + validation_dataset: str | None = None + metric_names: list[str] = Field(default_factory=list) + train_task_count: int = Field(default=0, ge=0) + validation_task_count: int = Field(default=0, ge=0) + + +class EvalAuthorCapture(BaseModel): + """Capture completeness and redaction metadata.""" + + model_config = ConfigDict(extra="forbid") + + prompt: EvalAuthorCaptureStatus = EvalAuthorCaptureStatus.UNAVAILABLE + trajectory: EvalAuthorCaptureStatus = EvalAuthorCaptureStatus.UNAVAILABLE + redactions: bool = False + redacted_fields: list[str] = Field(default_factory=list) + + +class EvalAuthorValidation(BaseModel): + """Verifier validation result and attempt count.""" + + model_config = ConfigDict(extra="forbid") + + status: str = "not_run" + attempt_count: int = Field(default=0, ge=0) + + class Insight(NemoEntity, entity_type="insights_insight"): """A persistent problem, theme, or category of issues in the agent under test.""" @@ -69,6 +175,26 @@ class Insight(NemoEntity, entity_type="insights_insight"): ) +class EvalAuthorRun(NemoEntity, entity_type="insights_eval_author_run"): + """Durable lifecycle and artifact index for one Eval Author attempt.""" + + insight_id: str = Field(description="Insight that originated this run.") + status: EvalAuthorRunStatus = Field(default=EvalAuthorRunStatus.CREATED) + stage: EvalAuthorRunStage = Field(default=EvalAuthorRunStage.INITIALIZING) + evaluator_type: str = Field(default="harbor") + config: EvalAuthorConfigDetails + inputs: EvalAuthorInputs + models: EvalAuthorModels + provenance: EvalAuthorProvenance + outputs: EvalAuthorOutputs = Field(default_factory=EvalAuthorOutputs) + capture: EvalAuthorCapture = Field(default_factory=EvalAuthorCapture) + validation: EvalAuthorValidation = Field(default_factory=EvalAuthorValidation) + summary: str = "" + error: str | None = None + started_at: datetime | None = None + completed_at: datetime | None = None + + class AnalysisConfig(NemoEntity, entity_type="insights_analysis_config"): """Per-agent opt-in state for framework-managed periodic analysis. diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/schema.py b/plugins/nemo-insights/src/nemo_insights_plugin/schema.py index 0a304718fb..129c552f72 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/schema.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/schema.py @@ -9,11 +9,21 @@ AnalysisConfig, AnalysisConfigStatus, AnalysisRunStatus, + EvalAuthorCapture, + EvalAuthorConfigDetails, + EvalAuthorInputs, + EvalAuthorModels, + EvalAuthorOutputs, + EvalAuthorProvenance, + EvalAuthorRun, + EvalAuthorRunStage, + EvalAuthorRunStatus, + EvalAuthorValidation, Insight, InsightStatus, ) from nemo_platform_plugin.schema import NemoListResponse -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field class CreateInsightRequest(BaseModel): @@ -50,7 +60,61 @@ class UpdateInsightRequest(BaseModel): trace_refs: list[str] | None = None -InsightPage = NemoListResponse[Insight] +class InsightListItem(Insight, entity_type="insights_insight"): + """Insight representation used only in the paginated list response.""" + + experiment_group_count: int | None = Field( + default=None, + description="Number of live experiment groups linked to this insight.", + ) + last_seen_at: datetime | None = Field( + default=None, + description="Newest start timestamp among the insight's currently referenced traces.", + ) + + +InsightPage = NemoListResponse[InsightListItem] + + +class CreateEvalAuthorRunRequest(BaseModel): + """Body for ``POST /eval-author-runs``.""" + + model_config = ConfigDict(extra="forbid") + + name: str | None = Field( + default=None, + description="Optional producer name. The service generates a short unique name when omitted.", + ) + insight_id: str + status: EvalAuthorRunStatus = EvalAuthorRunStatus.CREATED + stage: EvalAuthorRunStage = EvalAuthorRunStage.INITIALIZING + evaluator_type: str = "harbor" + config: EvalAuthorConfigDetails + inputs: EvalAuthorInputs + models: EvalAuthorModels + provenance: EvalAuthorProvenance + outputs: EvalAuthorOutputs = Field(default_factory=EvalAuthorOutputs) + capture: EvalAuthorCapture = Field(default_factory=EvalAuthorCapture) + validation: EvalAuthorValidation = Field(default_factory=EvalAuthorValidation) + summary: str = "" + error: str | None = None + + +class UpdateEvalAuthorRunRequest(BaseModel): + """Body for ``PATCH /eval-author-runs/{run_id}``; omitted fields are unchanged.""" + + model_config = ConfigDict(extra="forbid") + + status: EvalAuthorRunStatus | None = None + stage: EvalAuthorRunStage | None = None + outputs: EvalAuthorOutputs | None = None + capture: EvalAuthorCapture | None = None + validation: EvalAuthorValidation | None = None + summary: str | None = None + error: str | None = None + + +EvalAuthorRunPage = NemoListResponse[EvalAuthorRun] class UpdateAnalysisConfigRequest(BaseModel): diff --git a/plugins/nemo-insights/src/nemo_insights_plugin/service.py b/plugins/nemo-insights/src/nemo_insights_plugin/service.py index 583ff6b18a..c9e97647e7 100644 --- a/plugins/nemo-insights/src/nemo_insights_plugin/service.py +++ b/plugins/nemo-insights/src/nemo_insights_plugin/service.py @@ -8,14 +8,24 @@ """ import logging +from datetime import datetime, timezone from typing import ClassVar +from uuid import uuid4 from fastapi import APIRouter, Depends, HTTPException, Query -from nemo_insights_plugin._perms import AnalysisConfigPerms, AnalysisRunStatusPerms, InsightPerms +from nemo_insights_plugin._perms import ( + AnalysisConfigPerms, + AnalysisRunStatusPerms, + EvalAuthorRunPerms, + InsightPerms, +) from nemo_insights_plugin.authz import scope from nemo_insights_plugin.entities import ( AnalysisConfig, AnalysisRunStatus, + EvalAuthorRun, + EvalAuthorRunStage, + EvalAuthorRunStatus, Insight, InsightStatus, ) @@ -23,10 +33,14 @@ from nemo_insights_plugin.schema import ( AnalysisConfigPage, AnalysisRunStatusPage, + CreateEvalAuthorRunRequest, CreateInsightRequest, + EvalAuthorRunPage, + InsightListItem, InsightPage, UpdateAnalysisConfigRequest, UpdateAnalysisRunStatusRequest, + UpdateEvalAuthorRunRequest, UpdateInsightRequest, ) from nemo_platform_plugin.authz import CallerKind, path_rule @@ -42,9 +56,41 @@ from nemo_platform_plugin.jobs.routes import add_job_routes from nemo_platform_plugin.schema import PaginationData from nemo_platform_plugin.service import NemoService, RouterSpec +from nmp.intake.entities.experiments import ExperimentGroup +from nmp.intake.spans.api.dependencies import SpansServiceDep logger = logging.getLogger(__name__) +_TERMINAL_EVAL_AUTHOR_STATUSES = frozenset( + { + EvalAuthorRunStatus.SUCCEEDED, + EvalAuthorRunStatus.FAILED, + EvalAuthorRunStatus.CANCELLED, + } +) +_EVAL_AUTHOR_STATUS_TRANSITIONS = { + EvalAuthorRunStatus.CREATED: frozenset( + { + EvalAuthorRunStatus.RUNNING, + EvalAuthorRunStatus.SUCCEEDED, + EvalAuthorRunStatus.FAILED, + EvalAuthorRunStatus.CANCELLED, + } + ), + EvalAuthorRunStatus.RUNNING: _TERMINAL_EVAL_AUTHOR_STATUSES, + EvalAuthorRunStatus.SUCCEEDED: frozenset(), + EvalAuthorRunStatus.FAILED: frozenset(), + EvalAuthorRunStatus.CANCELLED: frozenset(), +} +_EVAL_AUTHOR_STAGE_INDEX = {stage: index for index, stage in enumerate(EvalAuthorRunStage)} + + +def _to_list_item(insight: Insight) -> InsightListItem: + item = InsightListItem.model_validate(insight.model_dump(exclude_computed_fields=True)) + if insight.__pydantic_private__ is not None: + item.__pydantic_private__ = insight.__pydantic_private__.copy() + return item + class InsightsService(NemoService): """NeMo Insights plugin service. @@ -55,7 +101,7 @@ class InsightsService(NemoService): """ name: ClassVar[str] = "insights" - dependencies: ClassVar[list[str]] = ["entities", "jobs"] + dependencies: ClassVar[list[str]] = ["entities", "jobs", "intake"] def get_routers(self) -> list[RouterSpec]: return [ @@ -71,6 +117,12 @@ def get_routers(self) -> list[RouterSpec]: description="Per-agent opt-in state for periodic insights analysis.", prefix="/v2/workspaces/{workspace}", ), + RouterSpec( + _build_eval_author_runs_router(), + tag="Insights Eval Author Runs", + description="Lifecycle and artifact indexes for externally executed Eval Author runs.", + prefix="/v2/workspaces/{workspace}", + ), RouterSpec( _build_analysis_run_statuses_router(), tag="Insights Analysis Run Statuses", @@ -129,6 +181,7 @@ async def create_insight( @path_rule(callers=[CallerKind.PRINCIPAL], permissions=[InsightPerms.LIST]) async def list_insights( workspace: str, + spans_service: SpansServiceDep, page: int = Query(default=1, ge=1, description="Page number (1-indexed)."), page_size: int = Query(default=20, ge=1, le=100, description="Items per page."), sort: str = Query( @@ -159,9 +212,39 @@ async def list_insights( logger.exception("Failed to list insights") raise HTTPException(status_code=500, detail="Failed to list insights.") from exc + items = [_to_list_item(insight) for insight in result.data] + if items: + insight_ids = [item.id for item in items] + try: + counts = await entity_client.count_by( + ExperimentGroup, + "insight_id", + workspace=workspace, + filter_obj={ + "insight_id": {"$in": insight_ids}, + "is_deleted": False, + }, + ) + except Exception: + logger.exception("Failed to count experiment groups for insights") + else: + for item in items: + item.experiment_group_count = counts.get(item.id, 0) + + try: + last_seen_at = await spans_service.latest_trace_started_at_by_group( + workspace=workspace, + trace_refs_by_group={item.id: item.trace_refs for item in items}, + ) + except Exception: + logger.exception("Failed to find latest traces for insights") + else: + for item in items: + item.last_seen_at = last_seen_at.get(item.id) + pagination = PaginationData.model_validate(result.pagination.model_dump()) if result.pagination else None return InsightPage( - data=result.data, + data=items, pagination=pagination, sort=sort, filter=filter_obj or None, @@ -269,6 +352,234 @@ async def delete_insight( return router +async def _get_workspace_insight( + entity_client: NemoEntitiesClient, + *, + workspace: str, + insight_id: str, +) -> Insight: + try: + insight = await entity_client.get_by_id(Insight, entity_id=insight_id) + except NemoEntityNotFoundError as exc: + raise HTTPException( + status_code=422, + detail=f"Insight '{insight_id}' does not exist in workspace '{workspace}'.", + ) from exc + if insight.workspace != workspace: + raise HTTPException( + status_code=422, + detail=f"Insight '{insight_id}' does not exist in workspace '{workspace}'.", + ) + return insight + + +async def _get_workspace_eval_author_run( + entity_client: NemoEntitiesClient, + *, + workspace: str, + run_id: str, +) -> EvalAuthorRun: + try: + run = await entity_client.get_by_id(EvalAuthorRun, entity_id=run_id) + except NemoEntityNotFoundError as exc: + raise HTTPException( + status_code=404, + detail=f"Eval Author run '{run_id}' not found in workspace '{workspace}'.", + ) from exc + if run.workspace != workspace: + raise HTTPException( + status_code=404, + detail=f"Eval Author run '{run_id}' not found in workspace '{workspace}'.", + ) + return run + + +def _validate_eval_author_update( + run: EvalAuthorRun, + body: UpdateEvalAuthorRunRequest, +) -> None: + if body.status is not None and body.status != run.status: + allowed = _EVAL_AUTHOR_STATUS_TRANSITIONS[run.status] + if body.status not in allowed: + raise HTTPException( + status_code=409, + detail=f"Illegal Eval Author run status transition: {run.status.value} -> {body.status.value}.", + ) + if body.stage is not None and body.stage != run.stage: + if run.status in _TERMINAL_EVAL_AUTHOR_STATUSES: + raise HTTPException(status_code=409, detail="A terminal Eval Author run cannot change stage.") + if _EVAL_AUTHOR_STAGE_INDEX[body.stage] < _EVAL_AUTHOR_STAGE_INDEX[run.stage]: + raise HTTPException( + status_code=409, + detail=f"Eval Author run stage cannot regress: {run.stage.value} -> {body.stage.value}.", + ) + next_status = body.status or run.status + next_stage = body.stage or run.stage + if next_status == EvalAuthorRunStatus.SUCCEEDED and next_stage != EvalAuthorRunStage.COMPLETED: + raise HTTPException( + status_code=422, + detail="A succeeded Eval Author run must be in the completed stage.", + ) + + +def _build_eval_author_runs_router() -> APIRouter: + router = APIRouter() + + @router.post( + "/eval-author-runs", + response_model=EvalAuthorRun, + status_code=201, + tags=["Insights Eval Author Runs"], + ) + @scope.write + @path_rule(callers=[CallerKind.PRINCIPAL], permissions=[EvalAuthorRunPerms.CREATE]) + async def create_eval_author_run( + workspace: str, + body: CreateEvalAuthorRunRequest, + entity_client: NemoEntitiesClient = Depends(get_entity_client), + ) -> EvalAuthorRun: + await _get_workspace_insight(entity_client, workspace=workspace, insight_id=body.insight_id) + if body.status == EvalAuthorRunStatus.SUCCEEDED and body.stage != EvalAuthorRunStage.COMPLETED: + raise HTTPException( + status_code=422, + detail="A succeeded Eval Author run must be in the completed stage.", + ) + now = datetime.now(timezone.utc) + run = EvalAuthorRun( + name=body.name or f"eval-author-run-{uuid4().hex[:12]}", + workspace=workspace, + insight_id=body.insight_id, + status=body.status, + stage=body.stage, + evaluator_type=body.evaluator_type, + config=body.config, + inputs=body.inputs, + models=body.models, + provenance=body.provenance, + outputs=body.outputs, + capture=body.capture, + validation=body.validation, + summary=body.summary, + error=body.error, + started_at=now if body.status != EvalAuthorRunStatus.CREATED else None, + completed_at=now if body.status in _TERMINAL_EVAL_AUTHOR_STATUSES else None, + ) + try: + return await entity_client.create(run) + except NemoEntityValidationError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + except Exception as exc: + logger.exception("Failed to create Eval Author run") + raise HTTPException(status_code=500, detail="Failed to create Eval Author run.") from exc + + @router.get( + "/eval-author-runs", + response_model=EvalAuthorRunPage, + tags=["Insights Eval Author Runs"], + ) + @scope.read + @path_rule(callers=[CallerKind.PRINCIPAL], permissions=[EvalAuthorRunPerms.LIST]) + async def list_eval_author_runs( + workspace: str, + page: int = Query(default=1, ge=1), + page_size: int = Query(default=20, ge=1, le=100), + sort: str = Query(default="-created_at"), + insight_id: str | None = Query(default=None), + status: EvalAuthorRunStatus | None = Query(default=None), + created_at: datetime | None = Query(default=None, description="Return runs created at or after this time."), + entity_client: NemoEntitiesClient = Depends(get_entity_client), + ) -> EvalAuthorRunPage: + filter_obj: dict[str, object] = {} + if insight_id is not None: + filter_obj["insight_id"] = insight_id + if status is not None: + filter_obj["status"] = status.value + if created_at is not None: + filter_obj["created_at"] = {"$gte": created_at} + try: + result = await entity_client.list( + EvalAuthorRun, + workspace=workspace, + page=page, + page_size=page_size, + sort=sort, + filter_obj=filter_obj or None, + ) + except Exception as exc: + logger.exception("Failed to list Eval Author runs") + raise HTTPException(status_code=500, detail="Failed to list Eval Author runs.") from exc + pagination = PaginationData.model_validate(result.pagination.model_dump()) if result.pagination else None + return EvalAuthorRunPage( + data=result.data, + pagination=pagination, + sort=sort, + filter=filter_obj or None, + ) + + @router.get( + "/eval-author-runs/{run_id}", + response_model=EvalAuthorRun, + tags=["Insights Eval Author Runs"], + ) + @scope.read + @path_rule(callers=[CallerKind.PRINCIPAL], permissions=[EvalAuthorRunPerms.READ]) + async def get_eval_author_run( + workspace: str, + run_id: str, + entity_client: NemoEntitiesClient = Depends(get_entity_client), + ) -> EvalAuthorRun: + return await _get_workspace_eval_author_run(entity_client, workspace=workspace, run_id=run_id) + + @router.patch( + "/eval-author-runs/{run_id}", + response_model=EvalAuthorRun, + tags=["Insights Eval Author Runs"], + ) + @scope.write + @path_rule(callers=[CallerKind.PRINCIPAL], permissions=[EvalAuthorRunPerms.UPDATE]) + async def update_eval_author_run( + workspace: str, + run_id: str, + body: UpdateEvalAuthorRunRequest, + entity_client: NemoEntitiesClient = Depends(get_entity_client), + ) -> EvalAuthorRun: + run = await _get_workspace_eval_author_run(entity_client, workspace=workspace, run_id=run_id) + _validate_eval_author_update(run, body) + now = datetime.now(timezone.utc) + if body.status is not None: + if body.status == EvalAuthorRunStatus.RUNNING and run.started_at is None: + run.started_at = now + if body.status in _TERMINAL_EVAL_AUTHOR_STATUSES and run.completed_at is None: + run.completed_at = now + run.status = body.status + if body.stage is not None: + run.stage = body.stage + if body.outputs is not None: + run.outputs = body.outputs + if body.capture is not None: + run.capture = body.capture + if body.validation is not None: + run.validation = body.validation + if body.summary is not None: + run.summary = body.summary + if "error" in body.model_fields_set: + run.error = body.error + try: + return await entity_client.update(run) + except NemoEntityNotFoundError as exc: + raise HTTPException( + status_code=404, + detail=f"Eval Author run '{run_id}' not found in workspace '{workspace}'.", + ) from exc + except NemoEntityValidationError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + except Exception as exc: + logger.exception("Failed to update Eval Author run") + raise HTTPException(status_code=500, detail="Failed to update Eval Author run.") from exc + + return router + + def _config_not_found(agent: str, workspace: str) -> str: """Standard 404 detail for a missing analysis config.""" return f"Analysis config for agent '{agent}' not found in workspace '{workspace}'." diff --git a/plugins/nemo-insights/testbed/ingest.py b/plugins/nemo-insights/testbed/ingest.py index cfaa2a7171..9291001981 100644 --- a/plugins/nemo-insights/testbed/ingest.py +++ b/plugins/nemo-insights/testbed/ingest.py @@ -133,7 +133,7 @@ def create_experiment( "experiment_group_id": experiment_group_id, "dataset_name": dataset_name, "dataset_version": dataset_version, - "metadata": metadata, + "metadata": {key: str(value) for key, value in metadata.items() if value is not None}, } owns_client = client is None client = client or httpx.Client(timeout=30.0) diff --git a/plugins/nemo-insights/tests/test_eval_author_runs.py b/plugins/nemo-insights/tests/test_eval_author_runs.py new file mode 100644 index 0000000000..c185f4122c --- /dev/null +++ b/plugins/nemo-insights/tests/test_eval_author_runs.py @@ -0,0 +1,308 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock + +from fastapi import FastAPI +from fastapi.testclient import TestClient +from nemo_insights_plugin.entities import ( + EvalAuthorCapture, + EvalAuthorCaptureStatus, + EvalAuthorConfigDetails, + EvalAuthorInputs, + EvalAuthorModels, + EvalAuthorOutputs, + EvalAuthorProvenance, + EvalAuthorRun, + EvalAuthorRunStage, + EvalAuthorRunStatus, + EvalAuthorValidation, + Insight, +) +from nemo_insights_plugin.service import InsightsService +from nemo_platform_plugin.entity_client import NemoPaginationInfo, get_entity_client + + +def _insight(*, workspace: str = "workspace-a") -> Insight: + insight = Insight( + name="unsafe-transfer", + workspace=workspace, + title="Unsafe transfer", + agent="airline-agent", + description="The agent escalates an in-scope request.", + trace_refs=["trace-1"], + ) + insight._id = "insight-1" + return insight + + +def _run( + *, + workspace: str = "workspace-a", + status: EvalAuthorRunStatus = EvalAuthorRunStatus.RUNNING, + stage: EvalAuthorRunStage = EvalAuthorRunStage.AUTHORING_VERIFIER, +) -> EvalAuthorRun: + run = EvalAuthorRun( + name="eval-author-run-123", + workspace=workspace, + insight_id="insight-1", + status=status, + stage=stage, + evaluator_type="harbor", + config=EvalAuthorConfigDetails(), + inputs=EvalAuthorInputs( + agent="airline-agent", + task_template="fileset://workspace-a/template", + train_dataset="fileset://workspace-a/train-source", + validation_dataset="fileset://workspace-a/validation-source", + trace_refs=["trace-1"], + ), + models=EvalAuthorModels(smart="gpt-smart", fast="gpt-fast"), + provenance=EvalAuthorProvenance( + optimizer_branch="codex/eval-author", + optimizer_commit="abcdef", + runner="run_eval_author", + ), + started_at=datetime(2026, 7, 22, 18, 0, tzinfo=timezone.utc), + ) + run._id = "run-1" + return run + + +def _payload() -> dict[str, object]: + return { + "insight_id": "insight-1", + "config": { + "max_traces": 1, + "max_summary_tokens": 80_000, + "max_validation_repair_attempts": 5, + }, + "inputs": { + "agent": "airline-agent", + "task_template": "fileset://workspace-a/template", + "train_dataset": "fileset://workspace-a/train-source", + "validation_dataset": "fileset://workspace-a/validation-source", + "trace_refs": ["trace-1"], + }, + "models": {"smart": "gpt-smart", "fast": "gpt-fast"}, + "provenance": { + "optimizer_branch": "codex/eval-author", + "optimizer_commit": "abcdef", + "runner": "run_eval_author", + }, + } + + +def _app(entity_client: AsyncMock) -> TestClient: + app = FastAPI() + eval_author_router = next( + spec for spec in InsightsService().get_routers() if spec.tag == "Insights Eval Author Runs" + ) + app.include_router(eval_author_router.router, prefix=eval_author_router.prefix) + app.dependency_overrides[get_entity_client] = lambda: entity_client + return TestClient(app) + + +def test_create_run_validates_insight_and_sets_started_at() -> None: + entity_client = AsyncMock() + entity_client.get_by_id.return_value = _insight() + + async def create(entity: EvalAuthorRun) -> EvalAuthorRun: + entity._id = "run-created" + return entity + + entity_client.create.side_effect = create + + response = _app(entity_client).post( + "/v2/workspaces/workspace-a/eval-author-runs", + json={**_payload(), "status": "running", "stage": "analyzing_traces"}, + ) + + assert response.status_code == 201 + body = response.json() + assert body["id"] == "run-created" + assert body["name"].startswith("eval-author-run-") + assert body["started_at"] is not None + assert body["completed_at"] is None + entity_client.get_by_id.assert_awaited_once_with(Insight, entity_id="insight-1") + + +def test_create_run_rejects_insight_from_another_workspace() -> None: + entity_client = AsyncMock() + entity_client.get_by_id.return_value = _insight(workspace="workspace-b") + + response = _app(entity_client).post( + "/v2/workspaces/workspace-a/eval-author-runs", + json=_payload(), + ) + + assert response.status_code == 422 + assert "does not exist in workspace 'workspace-a'" in response.json()["detail"] + entity_client.create.assert_not_awaited() + + +def test_list_runs_forwards_filters_pagination_and_sort() -> None: + entity_client = AsyncMock() + run = _run() + entity_client.list.return_value = SimpleNamespace( + data=[run], + pagination=NemoPaginationInfo( + page=2, + page_size=10, + current_page_size=1, + total_pages=2, + total_results=11, + ), + ) + + response = _app(entity_client).get( + "/v2/workspaces/workspace-a/eval-author-runs", + params={ + "page": 2, + "page_size": 10, + "sort": "-created_at", + "insight_id": "insight-1", + "status": "running", + "created_at": "2026-07-01T00:00:00Z", + }, + ) + + assert response.status_code == 200 + assert response.json()["pagination"]["total_results"] == 11 + entity_client.list.assert_awaited_once_with( + EvalAuthorRun, + workspace="workspace-a", + page=2, + page_size=10, + sort="-created_at", + filter_obj={ + "insight_id": "insight-1", + "status": "running", + "created_at": {"$gte": datetime(2026, 7, 1, tzinfo=timezone.utc)}, + }, + ) + + +def test_get_run_hides_cross_workspace_entity() -> None: + entity_client = AsyncMock() + entity_client.get_by_id.return_value = _run(workspace="workspace-b") + + response = _app(entity_client).get("/v2/workspaces/workspace-a/eval-author-runs/run-1") + + assert response.status_code == 404 + + +def test_update_run_rejects_stage_regression() -> None: + entity_client = AsyncMock() + entity_client.get_by_id.return_value = _run(stage=EvalAuthorRunStage.VALIDATING) + + response = _app(entity_client).patch( + "/v2/workspaces/workspace-a/eval-author-runs/run-1", + json={"stage": "analyzing_traces"}, + ) + + assert response.status_code == 409 + assert "cannot regress" in response.json()["detail"] + entity_client.update.assert_not_awaited() + + +def test_update_run_succeeds_and_sets_completion_timestamp() -> None: + entity_client = AsyncMock() + run = _run() + entity_client.get_by_id.return_value = run + entity_client.update.side_effect = lambda entity: entity + outputs = EvalAuthorOutputs( + artifact_fileset="fileset://workspace-a/opt-ea-123-artifacts", + insight_suite="fileset://workspace-a/insight-suite", + train_dataset="fileset://workspace-a/opt-ea-123-train", + validation_dataset="fileset://workspace-a/opt-ea-123-validation", + metric_names=["safe_transfer"], + train_task_count=50, + validation_task_count=30, + ) + + response = _app(entity_client).patch( + "/v2/workspaces/workspace-a/eval-author-runs/run-1", + json={ + "status": "succeeded", + "stage": "completed", + "outputs": outputs.model_dump(mode="json"), + "capture": EvalAuthorCapture( + prompt=EvalAuthorCaptureStatus.COMPLETE, + trajectory=EvalAuthorCaptureStatus.COMPLETE, + redactions=True, + redacted_fields=["authorization"], + ).model_dump(mode="json"), + "validation": EvalAuthorValidation(status="passed", attempt_count=1).model_dump( + mode="json" + ), + "summary": "Authored one verifier.", + }, + ) + + assert response.status_code == 200 + body = response.json() + assert body["status"] == "succeeded" + assert body["completed_at"] is not None + assert body["outputs"]["metric_names"] == ["safe_transfer"] + assert body["capture"]["redacted_fields"] == ["authorization"] + + +def test_failed_run_retains_partial_artifacts_and_error() -> None: + entity_client = AsyncMock() + run = _run() + entity_client.get_by_id.return_value = run + entity_client.update.side_effect = lambda entity: entity + + response = _app(entity_client).patch( + "/v2/workspaces/workspace-a/eval-author-runs/run-1", + json={ + "status": "failed", + "outputs": { + "artifact_fileset": "fileset://workspace-a/opt-ea-123-artifacts", + "metric_names": [], + "train_task_count": 0, + "validation_task_count": 0, + }, + "capture": { + "prompt": "partial", + "trajectory": "partial", + "redactions": True, + "redacted_fields": ["api_key"], + }, + "error": "Validation did not converge.", + }, + ) + + assert response.status_code == 200 + body = response.json() + assert body["status"] == "failed" + assert body["stage"] == "authoring_verifier" + assert body["outputs"]["artifact_fileset"].endswith("-artifacts") + assert body["error"] == "Validation did not converge." + assert body["completed_at"] is not None + + +def test_terminal_run_cannot_transition_or_change_stage() -> None: + entity_client = AsyncMock() + run = _run( + status=EvalAuthorRunStatus.FAILED, + stage=EvalAuthorRunStage.AUTHORING_VERIFIER, + ) + entity_client.get_by_id.return_value = run + + status_response = _app(entity_client).patch( + "/v2/workspaces/workspace-a/eval-author-runs/run-1", + json={"status": "running"}, + ) + stage_response = _app(entity_client).patch( + "/v2/workspaces/workspace-a/eval-author-runs/run-1", + json={"stage": "validating"}, + ) + + assert status_response.status_code == 409 + assert stage_response.status_code == 409 diff --git a/plugins/nemo-insights/tests/test_insights_list_contract.py b/plugins/nemo-insights/tests/test_insights_list_contract.py new file mode 100644 index 0000000000..e2162cbbcb --- /dev/null +++ b/plugins/nemo-insights/tests/test_insights_list_contract.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock + +from fastapi import FastAPI +from fastapi.testclient import TestClient +from nemo_insights_plugin.entities import Insight +from nemo_insights_plugin.service import InsightsService +from nemo_platform_plugin.entity_client import NemoPaginationInfo, get_entity_client +from nmp.intake.entities.experiments import ExperimentGroup +from nmp.intake.spans.api.dependencies import get_spans_service + + +def _insight(name: str, entity_id: str) -> Insight: + insight = Insight( + name=name, + workspace="default", + title=f"Title for {name}", + agent="test-agent", + description=f"Description for {name}", + ) + insight._id = entity_id + return insight + + +def _app(entity_client: AsyncMock, spans_service: AsyncMock) -> FastAPI: + app = FastAPI() + for spec in InsightsService().get_routers(): + app.include_router(spec.router, prefix=spec.prefix) + app.dependency_overrides[get_entity_client] = lambda: entity_client + app.dependency_overrides[get_spans_service] = lambda: spans_service + return app + + +def test_list_insights_enriches_the_page_with_counts_and_last_seen_at() -> None: + entity_client = AsyncMock() + spans_service = AsyncMock() + insights = [ + _insight("first", "insight-a"), + _insight("second", "insight-b"), + _insight("third", "insight-c"), + ] + insights[0].trace_refs = ["trace-old", "trace-new"] + insights[1].trace_refs = ["trace-missing"] + entity_client.list.return_value = SimpleNamespace( + data=insights, + pagination=NemoPaginationInfo( + page=1, + page_size=20, + current_page_size=len(insights), + total_pages=1, + total_results=len(insights), + ), + ) + entity_client.count_by.return_value = {"insight-a": 3} + latest = datetime(2026, 1, 2, tzinfo=timezone.utc) + spans_service.latest_trace_started_at_by_group.return_value = {"insight-a": latest} + + response = TestClient(_app(entity_client, spans_service)).get("/v2/workspaces/default/insights") + + assert response.status_code == 200 + assert [(item["id"], item["experiment_group_count"], item["last_seen_at"]) for item in response.json()["data"]] == [ + ("insight-a", 3, "2026-01-02T00:00:00Z"), + ("insight-b", 0, None), + ("insight-c", 0, None), + ] + entity_client.count_by.assert_awaited_once_with( + ExperimentGroup, + "insight_id", + workspace="default", + filter_obj={ + "insight_id": {"$in": ["insight-a", "insight-b", "insight-c"]}, + "is_deleted": False, + }, + ) + spans_service.latest_trace_started_at_by_group.assert_awaited_once_with( + workspace="default", + trace_refs_by_group={ + "insight-a": ["trace-old", "trace-new"], + "insight-b": ["trace-missing"], + "insight-c": [], + }, + ) diff --git a/plugins/nemo-insights/tests/testbed/test_ingest.py b/plugins/nemo-insights/tests/testbed/test_ingest.py index 8ad4bb2c41..ffa0e757de 100644 --- a/plugins/nemo-insights/tests/testbed/test_ingest.py +++ b/plugins/nemo-insights/tests/testbed/test_ingest.py @@ -179,7 +179,7 @@ def test_create_experiment_posts_full_body(): experiment_group_id="grp-1", dataset_name="tau2:airline", dataset_version="v1", - metadata={"seed": 300}, + metadata={"seed": 300, "num_tasks": None}, client=stub, ) assert stub.calls == [ @@ -191,7 +191,7 @@ def test_create_experiment_posts_full_body(): "experiment_group_id": "grp-1", "dataset_name": "tau2:airline", "dataset_version": "v1", - "metadata": {"seed": 300}, + "metadata": {"seed": "300"}, }, ) ] diff --git a/services/core/entities/src/nmp/core/entities/api/v2/entities/endpoints.py b/services/core/entities/src/nmp/core/entities/api/v2/entities/endpoints.py index 80d8842e3a..d0fe28e5c3 100644 --- a/services/core/entities/src/nmp/core/entities/api/v2/entities/endpoints.py +++ b/services/core/entities/src/nmp/core/entities/api/v2/entities/endpoints.py @@ -47,7 +47,8 @@ from sqlalchemy.exc import IntegrityError -class EntitiesPage(Page[Entity]): ... +class EntitiesPage(Page[Entity]): + group_counts: dict[str, int] | None = None router = APIRouter() @@ -336,23 +337,18 @@ async def list_entities( description="Sort field", examples=["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"], ), + count_by: str | None = Query( + default=None, + description="Optional direct string data field whose matching values should be counted.", + ), ) -> EntitiesPage: """List entities with filtering, supporting cross-workspace queries.""" accessible_workspaces = await get_accessible_workspaces(repository) # Handle cross-workspace query (workspace = "*") if workspace == ALL_WORKSPACES: # Build combined filter for workspace access and user's filter - combined_filter = add_workspace_filtering(accessible_workspaces, filter, field="workspace") - - entities, total = await repository.list_entities( - workspace=ALL_WORKSPACES, # Don't filter by single workspace - entity_type=entity_type, - page=page, - page_size=page_size, - sort=sort, - filter_op=combined_filter, - relationship_child_workspaces=accessible_workspaces, - ) + query_workspace = ALL_WORKSPACES + effective_filter = add_workspace_filtering(accessible_workspaces, filter, field="workspace") else: raise_if_workspace_inaccessible( accessible_workspaces, @@ -362,16 +358,30 @@ async def list_entities( # Check if workspace is being deleted (404 for user requests) await validate_workspace_not_deleting(workspace_repository, auth_client, workspace) - # Standard single-workspace query - entities, total = await repository.list_entities( - workspace=workspace, - entity_type=entity_type, - page=page, - page_size=page_size, - sort=sort, - filter_op=filter, - relationship_child_workspaces=accessible_workspaces, - ) + query_workspace = workspace + effective_filter = filter + + entities, total = await repository.list_entities( + workspace=query_workspace, + entity_type=entity_type, + page=page, + page_size=page_size, + sort=sort, + filter_op=effective_filter, + relationship_child_workspaces=accessible_workspaces, + ) + group_counts = None + if count_by is not None: + try: + group_counts = await repository.count_entities_by( + workspace=query_workspace, + entity_type=entity_type, + group_by=count_by, + filter_op=effective_filter, + relationship_child_workspaces=accessible_workspaces, + ) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) from e return EntitiesPage( data=entities, @@ -384,6 +394,7 @@ async def list_entities( ), sort=sort, filter=filter.to_dict() if filter else None, + group_counts=group_counts, ) diff --git a/services/core/entities/src/nmp/core/entities/app/repository/entity.py b/services/core/entities/src/nmp/core/entities/app/repository/entity.py index bd0386498a..ef8e168a2d 100644 --- a/services/core/entities/src/nmp/core/entities/app/repository/entity.py +++ b/services/core/entities/src/nmp/core/entities/app/repository/entity.py @@ -111,6 +111,20 @@ async def list_entities( """ pass + @abstractmethod + async def count_entities_by( + self, + *, + workspace: str, + entity_type: str, + group_by: str, + filter_op: FilterOperation | None = None, + relationship_child_workspaces: set[str] | None = None, + session: AsyncSession | None = None, + ) -> dict[str, int]: + """Count filtered entities grouped by a direct string data field.""" + pass + @abstractmethod async def update_entity( self, diff --git a/services/core/entities/src/nmp/core/entities/app/repository/sqlalchemy/entity.py b/services/core/entities/src/nmp/core/entities/app/repository/sqlalchemy/entity.py index ff617ed380..f0b948c247 100644 --- a/services/core/entities/src/nmp/core/entities/app/repository/sqlalchemy/entity.py +++ b/services/core/entities/src/nmp/core/entities/app/repository/sqlalchemy/entity.py @@ -15,10 +15,12 @@ from nmp.core.entities.app.repository.sqlalchemy.models import DBEntity from nmp.core.entities.entities import Entity from nmp.core.entities.utils.identifiers import generate_entity_id -from sqlalchemy import func, select +from sqlalchemy import String, cast, func, select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from sqlalchemy.orm.exc import StaleDataError +MAX_GROUP_COUNT_ROWS = 1000 + class SQLAlchemyEntityRepository(EntityRepositoryInterface): """SQLAlchemy implementation of Entity repository.""" @@ -197,6 +199,53 @@ async def list_entities( return entities, total + async def count_entities_by( + self, + *, + workspace: str, + entity_type: str, + group_by: str, + filter_op: FilterOperation | None = None, + relationship_child_workspaces: set[str] | None = None, + session: AsyncSession | None = None, + ) -> dict[str, int]: + """Count filtered entities grouped by a direct string data field.""" + async with self._get_session(session) as sess: + parts = group_by.split(".") + if len(parts) != 2 or parts[0] != "data" or not parts[1].isidentifier(): + raise ValueError(f"Field '{group_by}' is not a direct string data field") + + field = parts[1] + raw_group_column = DBEntity.data[field] + group_column = func.trim(cast(raw_group_column, String), '"') + if self._is_sqlite(sess): + json_type = func.json_type(DBEntity.data, f"$.{field}") + string_type = "text" + else: + json_type = func.json_typeof(raw_group_column) + string_type = "string" + + filter_repo = SQLAlchemyFilterRepository( + DBEntity, relationship_child_workspaces=relationship_child_workspaces + ) + query = select(group_column, func.count()).select_from(DBEntity).where(DBEntity.entity_type == entity_type) + + if workspace != ALL_WORKSPACES: + query = query.where(DBEntity.workspace == workspace) + + if filter_op is not None: + query = query.where(filter_op.apply(filter_repo)) + + query = query.where(json_type == string_type) + + rows = (await sess.execute(query.group_by(group_column).limit(MAX_GROUP_COUNT_ROWS + 1))).all() + if len(rows) > MAX_GROUP_COUNT_ROWS: + raise ValueError( + f"Grouped count has more than {MAX_GROUP_COUNT_ROWS} distinct values; " + "narrow the filter or choose a lower-cardinality field." + ) + return {str(key): int(count) for key, count in rows} + async def update_entity( self, *, diff --git a/services/core/entities/tests/integration/test_generic_entities.py b/services/core/entities/tests/integration/test_generic_entities.py index 6175090916..d71a199b9a 100644 --- a/services/core/entities/tests/integration/test_generic_entities.py +++ b/services/core/entities/tests/integration/test_generic_entities.py @@ -3,6 +3,8 @@ """Integration tests for generic entity API v2 endpoints.""" +import json + import pytest from httpx import AsyncClient @@ -152,6 +154,50 @@ async def test_list_entities_with_pagination(self, client: AsyncClient, ctx): assert result["pagination"]["total_results"] == 5 assert result["pagination"]["total_pages"] == 3 + async def test_list_entities_returns_group_counts_for_filtered_field(self, client: AsyncClient, ctx): + """Test list results include counts for a requested filtered data field.""" + entities = [ + {"name": "group-count-a-1", "data": {"insight_id": "insight-a", "is_deleted": False}}, + {"name": "group-count-a-2", "data": {"insight_id": "insight-a", "is_deleted": False}}, + {"name": "group-count-b-1", "data": {"insight_id": "insight-b", "is_deleted": False}}, + {"name": "group-count-deleted", "data": {"insight_id": "insight-a", "is_deleted": True}}, + ] + for entity in entities: + response = await client.post( + "/apis/entities/v2/workspaces/default/entities/experiment_group", + json=entity, + ) + assert response.status_code == 201 + + response = await client.get( + "/apis/entities/v2/workspaces/default/entities/experiment_group", + params={ + "count_by": "data.insight_id", + "filter": json.dumps( + { + "data.insight_id": {"$in": ["insight-a", "insight-b"]}, + "data.is_deleted": False, + } + ), + "page_size": 1, + }, + ) + + assert response.status_code == 200 + result = response.json() + assert result["group_counts"] == {"insight-a": 2, "insight-b": 1} + assert len(result["data"]) == 1 + + @pytest.mark.parametrize("count_by", ["name", ""]) + async def test_list_entities_rejects_unsupported_count_field(self, client: AsyncClient, ctx, count_by: str): + response = await client.get( + "/apis/entities/v2/workspaces/default/entities/experiment_group", + params={"count_by": count_by}, + ) + + assert response.status_code == 400 + assert "direct string data field" in response.json()["detail"] + async def test_update_entity_by_name(self, client: AsyncClient, ctx): """Test updating an entity by name.""" await client.post( diff --git a/services/core/entities/tests/repository/test_entity_group_counts.py b/services/core/entities/tests/repository/test_entity_group_counts.py new file mode 100644 index 0000000000..b63abd8519 --- /dev/null +++ b/services/core/entities/tests/repository/test_entity_group_counts.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for grouped entity counts.""" + +import pytest +from nmp.common.api.filter import ComparisonOperation, FilterOperator +from nmp.core.entities.app.repository import SQLAlchemyEntityRepository +from nmp.core.entities.app.repository.sqlalchemy import entity as entity_repository + +pytestmark = pytest.mark.asyncio + + +async def test_counts_filtered_entities_grouped_by_direct_string_data_field( + entity_repo: SQLAlchemyEntityRepository, setup_workspaces +): + """Count live experiment groups grouped by string insight_id values.""" + entities = ( + ("live-a-1", {"insight_id": "insight-a", "is_deleted": False}), + ("live-a-2", {"insight_id": "insight-a", "is_deleted": False}), + ("live-b", {"insight_id": "insight-b", "is_deleted": False}), + ("deleted", {"insight_id": "insight-a", "is_deleted": True}), + ("missing", {"is_deleted": False}), + ("null", {"insight_id": None, "is_deleted": False}), + ("boolean", {"insight_id": True, "is_deleted": False}), + ("numeric", {"insight_id": 1, "is_deleted": False}), + ) + for name, data in entities: + await entity_repo.create_entity( + workspace="workspace-1", + entity_type="experiment_group", + name=name, + data=data, + ) + + filter_op = ComparisonOperation(field="data.is_deleted", operator=FilterOperator.EQ, value=False) + + counts = await entity_repo.count_entities_by( + workspace="workspace-1", + entity_type="experiment_group", + group_by="data.insight_id", + filter_op=filter_op, + ) + + assert counts == {"insight-a": 2, "insight-b": 1} + + +@pytest.mark.parametrize("field", ["name", "data.nested.value", "data.", "data.not-valid"]) +async def test_rejects_unsupported_group_fields(entity_repo: SQLAlchemyEntityRepository, setup_workspaces, field: str): + with pytest.raises(ValueError, match="direct string data field"): + await entity_repo.count_entities_by( + workspace="workspace-1", + entity_type="experiment_group", + group_by=field, + ) + + +async def test_rejects_group_counts_over_limit( + entity_repo: SQLAlchemyEntityRepository, setup_workspaces, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(entity_repository, "MAX_GROUP_COUNT_ROWS", 2) + for index in range(3): + await entity_repo.create_entity( + workspace="workspace-1", + entity_type="experiment_group", + name=f"group-{index}", + data={"value": f"group-{index}"}, + ) + + with pytest.raises(ValueError, match="more than 2 distinct values"): + await entity_repo.count_entities_by( + workspace="workspace-1", + entity_type="experiment_group", + group_by="data.value", + ) diff --git a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py index 8a0e7afe7b..a2d72339e6 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/endpoints.py @@ -166,7 +166,9 @@ async def create_experiment_group( openapi_extra=generate_openapi_extra_params( filter_schema=ExperimentGroupFilter, filter_description=( - "Filter experiment groups by name, or by a metadata key/value: filter[metadata.]=." + "Filter experiment groups by name, insight_id, is_deleted, or a metadata key/value " + "(filter[metadata.]=). " + "Pass is_deleted=true to return only soft-deleted groups; omit to see only live ones." ), ), ) diff --git a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py index 3067e8bf2a..3f0daf19eb 100644 --- a/services/intake/src/nmp/intake/api/v2/experiments/schemas.py +++ b/services/intake/src/nmp/intake/api/v2/experiments/schemas.py @@ -253,6 +253,10 @@ class ExperimentGroupFilter(Filter): """Filter for listing ExperimentGroups.""" name: str | None = Field(default=None, description="Filter groups by name.") + insight_id: str | None = Field( + default=None, + description="Filter groups by the id of the insight that seeded them.", + ) is_deleted: bool | None = Field( default=None, description="When true, returns only soft-deleted groups. Omit (or false) to see only live groups.", diff --git a/services/intake/src/nmp/intake/spans/clickhouse_client.py b/services/intake/src/nmp/intake/spans/clickhouse_client.py index f16fa6e612..43635e484f 100644 --- a/services/intake/src/nmp/intake/spans/clickhouse_client.py +++ b/services/intake/src/nmp/intake/spans/clickhouse_client.py @@ -11,6 +11,7 @@ from dataclasses import dataclass from typing import Any +from clickhouse_connect.driver.external import ExternalData from fastapi import HTTPException, Request from nmp.intake.config import IntakeConfig from nmp.intake.spans.clickhouse_migrations import ( @@ -83,10 +84,16 @@ async def query( *, parameters: Sequence[Any] | dict[str, Any] | None = None, settings: dict[str, Any] | None = None, + external_data: ExternalData | None = None, ) -> Any: await self.bootstrap_schema() raw_client = await self._get_raw_client() - return await raw_client.query(query, parameters=parameters, settings=settings) + return await raw_client.query( + query, + parameters=parameters, + settings=settings, + external_data=external_data, + ) async def insert( self, diff --git a/services/intake/src/nmp/intake/spans/service.py b/services/intake/src/nmp/intake/spans/service.py index c8b3e51708..2340b029e8 100644 --- a/services/intake/src/nmp/intake/spans/service.py +++ b/services/intake/src/nmp/intake/spans/service.py @@ -5,6 +5,8 @@ from __future__ import annotations +from datetime import datetime + from nmp.common.api.common import PaginatedResult from nmp.intake.spans.annotations_repository import AnnotationsRepository from nmp.intake.spans.domain import ( @@ -140,6 +142,17 @@ async def get_trace(self, *, workspace: str, trace_id: str, mode: TraceMode) -> raise TraceNotFoundError(workspace, trace_id) return trace + async def latest_trace_started_at_by_group( + self, + *, + workspace: str, + trace_refs_by_group: dict[str, list[str]], + ) -> dict[str, datetime]: + return await self._traces.latest_trace_started_at_by_group( + workspace=workspace, + trace_refs_by_group=trace_refs_by_group, + ) + async def get_session(self, *, workspace: str, session_id: str) -> IntakeSession: session = await self._sessions.get_session(workspace=workspace, session_id=session_id) if session is None: diff --git a/services/intake/src/nmp/intake/spans/trace_repository.py b/services/intake/src/nmp/intake/spans/trace_repository.py index f324c78390..03572c790e 100644 --- a/services/intake/src/nmp/intake/spans/trace_repository.py +++ b/services/intake/src/nmp/intake/spans/trace_repository.py @@ -5,9 +5,11 @@ from __future__ import annotations +import json from datetime import datetime, timezone from typing import Any +from clickhouse_connect.driver.external import ExternalData from nmp.common.api.common import PaginatedResult from nmp.intake.spans.clickhouse_client import ClickHouseSpanClient from nmp.intake.spans.domain import IntakeTrace, TraceListFilter, TraceMode @@ -148,6 +150,56 @@ async def get_trace(self, *, workspace: str, trace_id: str, mode: TraceMode) -> ) return result.data[0] if result.data else None + async def latest_trace_started_at_by_group( + self, + *, + workspace: str, + trace_refs_by_group: dict[str, list[str]], + ) -> dict[str, datetime]: + pairs = [ + (group_id, trace_id) + for group_id, trace_ids in trace_refs_by_group.items() + for trace_id in dict.fromkeys(trace_ids) + ] + if not pairs: + return {} + + trace_index_table = self._client.table("trace_index") + result = await self._client.query( + f""" + WITH + refs AS ( + SELECT group_id, trace_id + FROM trace_refs + ), + traces AS ( + SELECT + trace_roots.trace_id AS id, + trace_roots.root_started_at AS started_at + FROM {trace_index_table} AS trace_roots FINAL + WHERE trace_roots.workspace = %(workspace)s + AND trace_roots.is_deleted = 0 + AND trace_roots.trace_id IN (SELECT trace_id FROM refs) + ORDER BY trace_roots.root_started_at ASC, trace_roots.root_span_id ASC + LIMIT 1 BY trace_roots.workspace, trace_roots.source_format, trace_roots.trace_id + ) + SELECT refs.group_id, max(traces.started_at) AS started_at + FROM refs + INNER JOIN traces ON traces.id = refs.trace_id + GROUP BY refs.group_id + """, + parameters={"workspace": workspace}, + external_data=ExternalData( + file_name="trace_refs.jsonl", + data=b"\n".join( + json.dumps({"group_id": group_id, "trace_id": trace_id}).encode() for group_id, trace_id in pairs + ), + fmt="JSONEachRow", + structure="group_id String, trace_id String", + ), + ) + return {str(group_id): started_at for group_id, started_at in result.result_rows} + def _trace_rows_sql( *, trace_index_sql: str, spans_table: str, mode: TraceMode, sort: str diff --git a/services/intake/tests/integration/test_experiments_crud.py b/services/intake/tests/integration/test_experiments_crud.py index a60215513f..93fb7e8f7d 100644 --- a/services/intake/tests/integration/test_experiments_crud.py +++ b/services/intake/tests/integration/test_experiments_crud.py @@ -65,6 +65,19 @@ def test_experiment_group_crud(client: TestClient) -> None: assert missing.status_code == 404 +def test_filter_groups_by_insight_id(client: TestClient) -> None: + """The groups list can be filtered server-side by the seeding insight id.""" + seeded = client.post(GROUPS, json={"name": "seeded-group", "insight_id": "insight-abc"}) + assert seeded.status_code == 201, seeded.text + other = client.post(GROUPS, json={"name": "unseeded-group"}) + assert other.status_code == 201, other.text + + listed = client.get(GROUPS, params={"filter[insight_id]": "insight-abc"}) + assert listed.status_code == 200, listed.text + names = {g["name"] for g in listed.json()["data"]} + assert names == {"seeded-group"} + + def test_experiment_group_update_description(client: TestClient) -> None: client.post(GROUPS, json={"name": "grp", "description": "old"}) updated = client.put(f"{GROUPS}/grp", json={"name": "grp", "description": "new"}) diff --git a/services/intake/tests/test_traces_clickhouse_repository.py b/services/intake/tests/test_traces_clickhouse_repository.py index 23fb2bfdaf..f89f707601 100644 --- a/services/intake/tests/test_traces_clickhouse_repository.py +++ b/services/intake/tests/test_traces_clickhouse_repository.py @@ -3,6 +3,7 @@ """Trace repository tests.""" +import json from datetime import datetime, timedelta, timezone from typing import cast @@ -22,14 +23,22 @@ class _Client: def __init__(self, query_results: list[_QueryResult] | None = None) -> None: self.queries: list[str] = [] self.parameters: list[dict[str, object]] = [] + self.external_data: list[object | None] = [] self.query_results = query_results or [] def table(self, name: str) -> str: return name - async def query(self, query: str, *, parameters: dict[str, object]) -> _QueryResult: + async def query( + self, + query: str, + *, + parameters: dict[str, object], + external_data: object | None = None, + ) -> _QueryResult: self.queries.append(query) self.parameters.append(parameters) + self.external_data.append(external_data) if self.query_results: return self.query_results.pop(0) if query.lstrip().startswith("SELECT count()"): @@ -81,6 +90,40 @@ async def test_summary_mode_reads_root_spans_without_metric_aggregates(): assert "payload_char_limit" not in client.parameters[1] +@pytest.mark.asyncio +async def test_latest_trace_started_at_by_group_aggregates_all_references_in_one_query(): + latest = datetime(2026, 1, 2, tzinfo=timezone.utc) + client = _Client(query_results=[_QueryResult([("insight-a", latest)])]) + repository = _repository(client) + + result = await repository.latest_trace_started_at_by_group( + workspace="workspace-a", + trace_refs_by_group={ + "insight-a": ["trace-old", "trace-new"], + "insight-empty": [], + "insight-missing": ["trace-missing"], + }, + ) + + assert result == {"insight-a": latest} + assert len(client.queries) == 1 + assert "FROM trace_refs" in client.queries[0] + assert "max(traces.started_at) AS started_at" in client.queries[0] + assert "GROUP BY refs.group_id" in client.queries[0] + assert client.parameters[0] == {"workspace": "workspace-a"} + external_data = client.external_data[0] + assert external_data is not None + assert external_data.query_params == { + "trace_refs_format": "JSONEachRow", + "trace_refs_structure": "group_id String, trace_id String", + } + assert [json.loads(line) for line in external_data.form_data["trace_refs"][1].splitlines()] == [ + {"group_id": "insight-a", "trace_id": "trace-old"}, + {"group_id": "insight-a", "trace_id": "trace-new"}, + {"group_id": "insight-missing", "trace_id": "trace-missing"}, + ] + + @pytest.mark.asyncio async def test_preview_mode_bounds_payloads_and_adds_trace_aggregate_block(): client = _Client() diff --git a/services/studio/src/nmp/studio/env_mappings.py b/services/studio/src/nmp/studio/env_mappings.py index 5a9503d3d4..9462fddf79 100644 --- a/services/studio/src/nmp/studio/env_mappings.py +++ b/services/studio/src/nmp/studio/env_mappings.py @@ -138,6 +138,11 @@ class EnvMapping: config_path="studio.feature_flags.model_compare_enabled", default="false", ), + EnvMapping( + marker="STUDIO_UI_VITE_FF_OPTIMIZER_ENABLED", + config_path="studio.feature_flags.optimizer_enabled", + default="false", + ), EnvMapping( marker="STUDIO_UI_VITE_FF_SAFE_SYNTHESIZER_ENABLED", config_path="studio.feature_flags.safe_synthesizer_enabled", diff --git a/web/packages/studio/env/.env.dev.local.sample b/web/packages/studio/env/.env.dev.local.sample index 2d6d1f8a17..fc4ccdefec 100644 --- a/web/packages/studio/env/.env.dev.local.sample +++ b/web/packages/studio/env/.env.dev.local.sample @@ -31,6 +31,7 @@ VITE_FF_INTAKE_ENABLED='true' VITE_FF_JOBS_ENABLED='true' VITE_FF_MEMBERS_ENABLED='preview' VITE_FF_MODEL_COMPARE_ENABLED='true' +VITE_FF_OPTIMIZER_ENABLED='false' VITE_FF_SAFE_SYNTHESIZER_ENABLED='false' VITE_FF_SECRETS_ENABLED='true' VITE_FF_SETTINGS_ENABLED='true' diff --git a/web/packages/studio/env/.env.fastapi b/web/packages/studio/env/.env.fastapi index 886cdd4565..8cd0b411d0 100644 --- a/web/packages/studio/env/.env.fastapi +++ b/web/packages/studio/env/.env.fastapi @@ -32,5 +32,6 @@ VITE_FF_INFERENCE_PROVIDER_ENABLED=STUDIO_UI_VITE_FF_INFERENCE_PROVIDER_ENABLED VITE_FF_INTAKE_ENABLED=STUDIO_UI_VITE_FF_INTAKE_ENABLED VITE_FF_MEMBERS_ENABLED=STUDIO_UI_VITE_FF_MEMBERS_ENABLED VITE_FF_MODEL_COMPARE_ENABLED=STUDIO_UI_VITE_FF_MODEL_COMPARE_ENABLED +VITE_FF_OPTIMIZER_ENABLED=STUDIO_UI_VITE_FF_OPTIMIZER_ENABLED VITE_FF_SAFE_SYNTHESIZER_ENABLED=STUDIO_UI_VITE_FF_SAFE_SYNTHESIZER_ENABLED VITE_FF_SECRETS_ENABLED=STUDIO_UI_VITE_FF_SECRETS_ENABLED diff --git a/web/packages/studio/src/api/optimizer.ts b/web/packages/studio/src/api/optimizer.ts new file mode 100644 index 0000000000..3378dbb37e --- /dev/null +++ b/web/packages/studio/src/api/optimizer.ts @@ -0,0 +1,325 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { customFetch, type ErrorType } from '@nemo/sdk/generated/fetchers/platform'; +import type { HTTPValidationError, PaginationData } from '@nemo/sdk/generated/platform/schema'; +import { + type QueryClient, + type UseMutationOptions, + type UseMutationResult, + type UseQueryOptions, + type UseQueryResult, + useMutation, + useQuery, +} from '@tanstack/react-query'; + +/** + * The Insights plugin is a NeMo Platform backend plugin whose routes are NOT + * part of the generated SDK. We hand-write typed hooks against its endpoints + * using `customFetch`, which applies the same base URL + OIDC auth handling as + * the generated clients. + * + * Routes mount at `/apis/insights/v2/workspaces/{workspace}/...`. + */ + +export type InsightStatus = 'open' | 'resolved' | 'deleted'; + +export interface Insight { + /** Store-assigned id — used to fetch a single insight (`GET /insights/{id}`). */ + id: string; + /** Entity name (unique slug within the workspace). */ + name: string; + /** Short, human-readable sentence naming the core issue. */ + title: string; + /** The actionable problem statement. */ + description: string; + /** Registered agent name or local path this insight is about. */ + agent: string; + /** Lifecycle state. Defaults to `open`. */ + status: InsightStatus; + /** Intake trace ids identified as evidence for this insight. */ + trace_refs: string[]; + created_at?: string; + updated_at?: string; + [key: string]: unknown; +} + +export interface InsightListItem extends Insight { + /** Number of experiment groups linked to this insight, or null when unknown. */ + experiment_group_count: number | null; + /** Newest start timestamp among the insight's currently referenced traces. */ + last_seen_at?: string | null; +} + +export type OptimizerListInsightsParams = Record & { + page?: number; + page_size?: number; + sort?: string; + filter?: Record; +}; + +export interface InsightPage { + data?: InsightListItem[]; + pagination?: PaginationData; + [key: string]: unknown; +} + +export type EvalAuthorRunStatus = 'created' | 'running' | 'succeeded' | 'failed' | 'cancelled'; + +export type EvalAuthorRunStage = + | 'initializing' + | 'materializing_traces' + | 'analyzing_traces' + | 'discovering_runner' + | 'authoring_verifier' + | 'validating' + | 'publishing' + | 'completed'; + +export type EvalAuthorCaptureStatus = 'complete' | 'partial' | 'unavailable'; + +export interface EvalAuthorRun { + id: string; + name: string; + workspace: string; + insight_id: string; + status: EvalAuthorRunStatus; + stage: EvalAuthorRunStage; + evaluator_type: string; + config: { + max_traces: number; + max_summary_tokens: number; + max_validation_repair_attempts: number; + }; + inputs: { + agent: string; + task_template: string; + train_dataset: string; + validation_dataset: string; + trace_refs: string[]; + }; + models: { + smart: string; + fast: string; + }; + provenance: { + optimizer_branch: string; + optimizer_commit: string; + runner: string; + }; + outputs: { + artifact_fileset?: string | null; + insight_suite?: string | null; + train_dataset?: string | null; + validation_dataset?: string | null; + metric_names: string[]; + train_task_count: number; + validation_task_count: number; + }; + capture: { + prompt: EvalAuthorCaptureStatus; + trajectory: EvalAuthorCaptureStatus; + redactions: boolean; + redacted_fields: string[]; + }; + validation: { + status: string; + attempt_count: number; + }; + summary: string; + error?: string | null; + started_at?: string | null; + completed_at?: string | null; + created_at?: string; + updated_at?: string; +} + +export interface EvalAuthorRunPage { + data?: EvalAuthorRun[]; + pagination?: PaginationData; +} + +export interface OptimizerListEvalAuthorRunsParams extends Record { + page?: number; + page_size?: number; + sort?: string; + insight_id?: string; + status?: EvalAuthorRunStatus; + created_at?: string; +} + +const optimizerInsightsPath = (workspace: string, path = '') => + `/apis/insights/v2/workspaces/${encodeURIComponent(String(workspace))}/insights${path}`; + +const optimizerEvalAuthorRunsPath = (workspace: string, path = '') => + `/apis/insights/v2/workspaces/${encodeURIComponent(String(workspace))}/eval-author-runs${path}`; + +type QueryOptions = { + query?: Partial>; +}; + +type MutationOptions = { + mutation?: UseMutationOptions; +}; + +export interface UpdateInsightRequest { + title?: string; + agent?: string; + description?: string; + status?: InsightStatus; + trace_refs?: string[]; +} + +export const optimizerListInsights = ( + workspace: string, + params?: OptimizerListInsightsParams, + signal?: AbortSignal +) => + customFetch({ + url: optimizerInsightsPath(workspace), + method: 'GET', + params, + signal, + }); + +export const getOptimizerListInsightsQueryKey = ( + workspace: string, + params?: OptimizerListInsightsParams +) => [optimizerInsightsPath(workspace), ...(params ? [params] : [])] as const; + +export const useOptimizerListInsights = >( + workspace: string, + params?: OptimizerListInsightsParams, + options?: QueryOptions>, TError>, + queryClient?: QueryClient +): UseQueryResult>, TError> => + useQuery( + { + queryKey: getOptimizerListInsightsQueryKey(workspace, params), + queryFn: ({ signal }) => optimizerListInsights(workspace, params, signal), + ...options?.query, + }, + queryClient + ); + +export const optimizerGetInsight = (workspace: string, insightId: string, signal?: AbortSignal) => + customFetch({ + url: optimizerInsightsPath(workspace, `/${encodeURIComponent(insightId)}`), + method: 'GET', + signal, + }); + +export const getOptimizerGetInsightQueryKey = (workspace: string, insightId: string) => + [optimizerInsightsPath(workspace, `/${encodeURIComponent(insightId)}`)] as const; + +export const useOptimizerGetInsight = >( + workspace: string, + insightId: string, + options?: QueryOptions>, TError>, + queryClient?: QueryClient +): UseQueryResult>, TError> => + useQuery( + { + queryKey: getOptimizerGetInsightQueryKey(workspace, insightId), + queryFn: ({ signal }) => optimizerGetInsight(workspace, insightId, signal), + enabled: !!insightId, + ...options?.query, + }, + queryClient + ); + +export const optimizerUpdateInsight = ( + workspace: string, + insightId: string, + data: UpdateInsightRequest +) => + customFetch({ + url: optimizerInsightsPath(workspace, `/${encodeURIComponent(insightId)}`), + method: 'PATCH', + data, + }); + +export const useOptimizerUpdateInsight = < + TError = ErrorType, + TContext = unknown, +>( + options?: MutationOptions< + Awaited>, + { workspace: string; insightId: string; data: UpdateInsightRequest }, + TError, + TContext + >, + queryClient?: QueryClient +): UseMutationResult< + Awaited>, + TError, + { workspace: string; insightId: string; data: UpdateInsightRequest }, + TContext +> => + useMutation( + { + mutationKey: ['optimizerUpdateInsight'], + mutationFn: ({ workspace, insightId, data }) => + optimizerUpdateInsight(workspace, insightId, data), + ...options?.mutation, + }, + queryClient + ); + +export const optimizerListEvalAuthorRuns = ( + workspace: string, + params?: OptimizerListEvalAuthorRunsParams, + signal?: AbortSignal +) => + customFetch({ + url: optimizerEvalAuthorRunsPath(workspace), + method: 'GET', + params, + signal, + }); + +export const getOptimizerListEvalAuthorRunsQueryKey = ( + workspace: string, + params?: OptimizerListEvalAuthorRunsParams +) => [optimizerEvalAuthorRunsPath(workspace), ...(params ? [params] : [])] as const; + +export const useOptimizerListEvalAuthorRuns = >( + workspace: string, + params?: OptimizerListEvalAuthorRunsParams, + options?: QueryOptions>, TError>, + queryClient?: QueryClient +): UseQueryResult>, TError> => + useQuery( + { + queryKey: getOptimizerListEvalAuthorRunsQueryKey(workspace, params), + queryFn: ({ signal }) => optimizerListEvalAuthorRuns(workspace, params, signal), + ...options?.query, + }, + queryClient + ); + +export const optimizerGetEvalAuthorRun = (workspace: string, runId: string, signal?: AbortSignal) => + customFetch({ + url: optimizerEvalAuthorRunsPath(workspace, `/${encodeURIComponent(runId)}`), + method: 'GET', + signal, + }); + +export const getOptimizerGetEvalAuthorRunQueryKey = (workspace: string, runId: string) => + [optimizerEvalAuthorRunsPath(workspace, `/${encodeURIComponent(runId)}`)] as const; + +export const useOptimizerGetEvalAuthorRun = >( + workspace: string, + runId: string, + options?: QueryOptions>, TError>, + queryClient?: QueryClient +): UseQueryResult>, TError> => + useQuery( + { + queryKey: getOptimizerGetEvalAuthorRunQueryKey(workspace, runId), + queryFn: ({ signal }) => optimizerGetEvalAuthorRun(workspace, runId, signal), + enabled: !!runId, + ...options?.query, + }, + queryClient + ); diff --git a/web/packages/studio/src/assets/voyager.svg b/web/packages/studio/src/assets/voyager.svg new file mode 100644 index 0000000000..cc7b82fd25 --- /dev/null +++ b/web/packages/studio/src/assets/voyager.svg @@ -0,0 +1,162 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/web/packages/studio/src/components/ChangesetBadge/index.tsx b/web/packages/studio/src/components/ChangesetBadge/index.tsx new file mode 100644 index 0000000000..6d76c4116e --- /dev/null +++ b/web/packages/studio/src/components/ChangesetBadge/index.tsx @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Anchor, Badge } from '@nvidia/foundations-react-core'; +import { SquareArrowOutUpRight } from 'lucide-react'; +import { type FC } from 'react'; + +interface ChangesetBadgeProps { + /** The experiment's `source_link` URL. */ + href: string; +} + +/** + * Blue "Changeset" badge linking to an experiment's source changeset. Used on both the experiment + * group table and the experiment detail header. The trailing external-link icon signals the link + * opens the source in a new tab. `stopPropagation` keeps the link from also triggering a clickable + * parent (e.g. a table row's row-click navigation). + */ +export const ChangesetBadge: FC = ({ href }) => ( + event.stopPropagation()}> + + Changeset + + +); diff --git a/web/packages/studio/src/components/IntakeLists/IntakeTracesTable.tsx b/web/packages/studio/src/components/IntakeLists/IntakeTracesTable.tsx index e44abbdb2b..1dea190457 100644 --- a/web/packages/studio/src/components/IntakeLists/IntakeTracesTable.tsx +++ b/web/packages/studio/src/components/IntakeLists/IntakeTracesTable.tsx @@ -1,16 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { dateTimeFilter } from '@nemo/common/src/components/DataView/dateTimeFilter'; import { EditColumnsMenu } from '@nemo/common/src/components/DataView/internal'; import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; -import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; import { getSortParamWithWhitelist } from '@nemo/common/src/utils/query'; import { useListTraces } from '@nemo/sdk/generated/platform/api'; import type { Trace, TraceFilter, TraceSortField } from '@nemo/sdk/generated/platform/schema'; -import { Badge, Button } from '@nvidia/foundations-react-core'; +import { Button } from '@nvidia/foundations-react-core'; import { getErrorMessage } from '@studio/api/common/utils'; import { isDefaultStartedAtFilter, @@ -18,19 +16,13 @@ import { type StartedAtFilterEntry, useSeededStartedAtFilter, } from '@studio/components/IntakeLists/defaultStartedAtFilter'; -import { IntakePayloadPreviewCell } from '@studio/components/IntakeLists/IntakePayloadPreviewCell'; import { IntakeTelemetryDataView } from '@studio/components/IntakeLists/IntakeTelemetryDataView'; +import { makeIntakeTraceColumns } from '@studio/components/IntakeLists/intakeTraceColumns'; import { useWorkspaceFromPathIfExists } from '@studio/hooks/useWorkspaceFromPath'; import { getIntakeTraceRoute } from '@studio/routes/utils'; -import { - formatCost, - formatDurationMs, - formatInteger, - getTraceDisplayName, -} from '@studio/util/intakeTelemetry'; import { keepPreviousData } from '@tanstack/react-query'; import { Columns3 } from 'lucide-react'; -import { type ComponentProps, type FC, type ReactNode, useState } from 'react'; +import { type FC, type ReactNode, useState } from 'react'; import { useNavigate } from 'react-router-dom'; export interface IntakeTracesTableProps { @@ -101,97 +93,6 @@ const SeededIntakeTracesTable: FC< } ); - const makeColumns: ComponentProps>['makeColumns'] = ({ - accessor, - }) => [ - accessor('id', { - id: 'id', - header: 'Trace', - size: 280, - enableSorting: false, - meta: { - filter: { - type: 'text' as const, - label: 'Trace ID', - placeholder: 'Filter by trace ID', - }, - }, - cell: ({ row }) => { - const trace = row.original; - const label = getTraceDisplayName(trace); - return label; - }, - }), - accessor('input', { - id: 'input', - header: 'Input', - size: 360, - enableSorting: false, - cell: ({ row }) => , - }), - accessor('output', { - id: 'output', - header: 'Output', - size: 360, - enableSorting: false, - cell: ({ row }) => , - }), - { - id: 'duration_ms', - header: 'Duration', - size: 120, - enableSorting: false, - cell: ({ row }) => formatDurationMs(row.original.duration_ms), - }, - { - id: 'span_count', - header: 'Spans', - size: 90, - enableSorting: false, - cell: ({ row }) => formatInteger(row.original.span_count), - }, - { - id: 'error_count', - header: 'Errors', - size: 90, - enableSorting: false, - cell: ({ row }) => { - const errorCount = row.original.error_count ?? 0; - return errorCount > 0 ? ( - - {formatInteger(errorCount)} - - ) : ( - formatInteger(errorCount) - ); - }, - }, - { - id: 'total_tokens', - header: 'Tokens', - size: 120, - enableSorting: false, - cell: ({ row }) => formatInteger(row.original.total_tokens), - }, - { - id: 'cost_usd', - header: 'Cost', - size: 110, - enableSorting: false, - cell: ({ row }) => formatCost(row.original.cost_usd), - }, - accessor('started_at', { - id: 'started_at', - header: 'Started', - size: 150, - enableSorting: true, - meta: { - filter: dateTimeFilter('Started At'), - }, - cell: ({ row }) => , - }), - ]; - if (error) { return ; } @@ -203,7 +104,11 @@ const SeededIntakeTracesTable: FC< return ( dataViewState={dataViewState} - makeColumns={makeColumns} + makeColumns={makeIntakeTraceColumns({ + traceIdFilter: true, + startedAtSort: true, + startedAtFilter: true, + })} slotEndPortalTargetId={slotEndPortalTargetId} toolbarSlotEnd={ >['makeColumns']; + +export interface IntakeTraceColumnOptions { + /** Expose a Trace ID text filter on the first column (workspace browse table). */ + traceIdFilter?: boolean; + /** Allow sorting by started_at (server-backed list only). */ + startedAtSort?: boolean; + /** Expose a Started At datetime filter (workspace browse table). */ + startedAtFilter?: boolean; +} + +/** + * Shared trace table columns for Intake browse (`IntakeTracesTable`) and insight evidence + * (`InsightTracesTable`). Keeps headers, sizes, and formatters in one place. + */ +export const makeIntakeTraceColumns = + ({ + traceIdFilter = false, + startedAtSort = false, + startedAtFilter = false, + }: IntakeTraceColumnOptions = {}): MakeIntakeTraceColumns => + ({ accessor }) => [ + accessor('id', { + id: 'id', + header: 'Trace', + size: 280, + enableSorting: false, + meta: traceIdFilter + ? { + filter: { + type: 'text' as const, + label: 'Trace ID', + placeholder: 'Filter by trace ID', + }, + } + : undefined, + cell: ({ row }) => getTraceDisplayName(row.original), + }), + accessor('input', { + id: 'input', + header: 'Input', + size: 360, + enableSorting: false, + cell: ({ row }) => , + }), + accessor('output', { + id: 'output', + header: 'Output', + size: 360, + enableSorting: false, + cell: ({ row }) => , + }), + { + id: 'duration_ms', + header: 'Duration', + size: 120, + enableSorting: false, + cell: ({ row }) => formatDurationMs(row.original.duration_ms), + }, + { + id: 'span_count', + header: 'Spans', + size: 90, + enableSorting: false, + cell: ({ row }) => formatInteger(row.original.span_count), + }, + { + id: 'error_count', + header: 'Errors', + size: 90, + enableSorting: false, + cell: ({ row }) => { + const errorCount = row.original.error_count ?? 0; + return errorCount > 0 ? ( + + {formatInteger(errorCount)} + + ) : ( + formatInteger(errorCount) + ); + }, + }, + { + id: 'total_tokens', + header: 'Tokens', + size: 120, + enableSorting: false, + cell: ({ row }) => formatInteger(row.original.total_tokens), + }, + { + id: 'cost_usd', + header: 'Cost', + size: 110, + enableSorting: false, + cell: ({ row }) => formatCost(row.original.cost_usd), + }, + accessor('started_at', { + id: 'started_at', + header: 'Started', + size: 150, + enableSorting: startedAtSort, + meta: startedAtFilter ? { filter: dateTimeFilter('Started At') } : undefined, + cell: ({ row }) => , + }), + ]; diff --git a/web/packages/studio/src/components/OriginatingInsightLink/index.tsx b/web/packages/studio/src/components/OriginatingInsightLink/index.tsx new file mode 100644 index 0000000000..ed3dfe3313 --- /dev/null +++ b/web/packages/studio/src/components/OriginatingInsightLink/index.tsx @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Anchor, Text } from '@nvidia/foundations-react-core'; +import voyagerArt from '@studio/assets/voyager.svg'; +import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; +import { getOptimizerInsightRoute } from '@studio/routes/utils'; +import { type FC } from 'react'; +import { Link } from 'react-router-dom'; + +interface OriginatingInsightLinkProps { + insightId: string; +} + +/** + * The Voyager artwork above an "Originating insight" label, both linking to the insight detail page. + * Shared by the experiment group detail page and the experiment detail root-cause card. + */ +export const OriginatingInsightLink: FC = ({ insightId }) => { + const workspace = useWorkspaceFromPath(); + return ( + + + + + Originating +
+ insight + + + + ); +}; diff --git a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx index 86cdd6adf6..514987dc2e 100644 --- a/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/ExperimentGroupDataView/index.tsx @@ -19,6 +19,7 @@ import type { ExperimentGroupResponse, } from '@nemo/sdk/generated/platform/schema'; import { Button, Text, Tooltip } from '@nvidia/foundations-react-core'; +import { ChangesetBadge } from '@studio/components/ChangesetBadge'; import { Empty } from '@studio/components/dataViews/ExperimentGroupDataView/Empty'; import { MeanValueTooltipCell } from '@studio/components/dataViews/ExperimentGroupDataView/MeanValueTooltipCell'; import { @@ -293,6 +294,16 @@ export const ExperimentGroupDataView: FC = ({ grou ); }, }), + accessor('source_link', { + header: 'Source', + enableSorting: false, + size: 140, + cell: ({ row }) => { + const { source_link } = row.original; + if (!source_link) return -; + return ; + }, + }), accessor((original) => original.agent_names?.join(', '), { id: 'agent_names', header: 'Agent Names', diff --git a/web/packages/studio/src/constants/environment.ts b/web/packages/studio/src/constants/environment.ts index 4c48e76a05..afb976b737 100644 --- a/web/packages/studio/src/constants/environment.ts +++ b/web/packages/studio/src/constants/environment.ts @@ -47,6 +47,7 @@ export const INTAKE_ENABLED = featureFlags.intakeEnabled !== false; export const JOBS_ENABLED = featureFlags.jobsEnabled !== false; export const MEMBERS_ENABLED = featureFlags.membersEnabled !== false; export const MODEL_COMPARE_ENABLED = featureFlags.modelCompareEnabled !== false; +export const OPTIMIZER_ENABLED = featureFlags.optimizerEnabled !== false; export const SAFE_SYNTHESIZER_ENABLED = featureFlags.safeSynthesizerEnabled !== false; export const SECRETS_ENABLED = featureFlags.secretsEnabled !== false; export const SETTINGS_ENABLED = featureFlags.settingsEnabled !== false; diff --git a/web/packages/studio/src/constants/featureFlags/featureFlags.ts b/web/packages/studio/src/constants/featureFlags/featureFlags.ts index ea0a36a5a6..c55b51f528 100644 --- a/web/packages/studio/src/constants/featureFlags/featureFlags.ts +++ b/web/packages/studio/src/constants/featureFlags/featureFlags.ts @@ -72,6 +72,7 @@ export const flagDefinitions = { jobsEnabled: previewFlag('VITE_FF_JOBS_ENABLED', true), membersEnabled: previewFlag('VITE_FF_MEMBERS_ENABLED'), modelCompareEnabled: previewFlag('VITE_FF_MODEL_COMPARE_ENABLED'), + optimizerEnabled: previewFlag('VITE_FF_OPTIMIZER_ENABLED', false), safeSynthesizerEnabled: previewFlag('VITE_FF_SAFE_SYNTHESIZER_ENABLED', false), secretsEnabled: previewFlag('VITE_FF_SECRETS_ENABLED', true), settingsEnabled: previewFlag('VITE_FF_SETTINGS_ENABLED', true), diff --git a/web/packages/studio/src/constants/routes.ts b/web/packages/studio/src/constants/routes.ts index 530be044c8..6821eedb5f 100644 --- a/web/packages/studio/src/constants/routes.ts +++ b/web/packages/studio/src/constants/routes.ts @@ -38,6 +38,8 @@ export const ROUTE_PARAMS = { /** Benchmark entity name segment under evaluation/benchmarks/:name */ benchmarkName: 'benchmarkName', experimentGroupName: 'experimentGroupName', + insightId: 'insightId', + evalAuthorRunId: 'evalAuthorRunId', evaluationName: 'evaluationName', guardrailConfigName: 'guardrailConfigName', } as const; @@ -110,6 +112,9 @@ export const ROUTES = { dataDesignerJobNewLegacy: `/workspaces/:${P.workspace}/data-designer/new/legacy`, secrets: `/workspaces/:${P.workspace}/secrets`, guardrails: `/workspaces/:${P.workspace}/guardrails`, + optimizer: `/workspaces/:${P.workspace}/optimizer`, + optimizerInsight: `/workspaces/:${P.workspace}/optimizer/:${P.insightId}`, + optimizerEvalAuthorRun: `/workspaces/:${P.workspace}/optimizer/:${P.insightId}/eval-author-runs/:${P.evalAuthorRunId}`, guardrailDetail: `/workspaces/:${P.workspace}/guardrails/:${P.guardrailConfigName}`, settings: `/workspaces/:${P.workspace}/settings`, /** Workspace members and role-based access (Entities role bindings) */ diff --git a/web/packages/studio/src/routes/EvaluationDetailRoute/EvaluationDetailMetrics.tsx b/web/packages/studio/src/routes/EvaluationDetailRoute/EvaluationDetailMetrics.tsx index 6789c960aa..11f718b7b1 100644 --- a/web/packages/studio/src/routes/EvaluationDetailRoute/EvaluationDetailMetrics.tsx +++ b/web/packages/studio/src/routes/EvaluationDetailRoute/EvaluationDetailMetrics.tsx @@ -6,6 +6,7 @@ import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; import { formatDurationMs } from '@nemo/common/src/utils/date'; import { useGetEvaluation } from '@nemo/sdk/generated/platform/api'; import { Divider, Flex, Text, Tooltip } from '@nvidia/foundations-react-core'; +import { ChangesetBadge } from '@studio/components/ChangesetBadge'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { tooltipClassName } from '@studio/styles/common'; import { type FC, type ReactNode } from 'react'; @@ -40,6 +41,17 @@ export const EvaluationDetailMetrics: FC = ({ eval return ( + {experiment?.source_link ? ( + <> + } + loading={isLoading} + orientation="vertical" + /> + + + ) : null} { + vi.stubEnv('VITE_FF_OPTIMIZER_ENABLED', 'true'); +}); + +const WORKSPACE = 'test-workspace'; +const GROUP_NAME = 'test-group'; +const EVALUATION_NAME = 'test-evaluation'; + +const evaluation = { + id: 'evaluation-id', + name: EVALUATION_NAME, + workspace: WORKSPACE, + experiment_group_id: 'group-id', + dataset_name: 'dataset', + description: 'Evaluation description', +} satisfies Partial; + +const group = { + id: 'group-id', + name: GROUP_NAME, + workspace: WORKSPACE, + insight_id: 'insight-id', + default_sort: '-created_at', + evaluation_count: 0, +} satisfies Partial; + +describe('EvaluationDetailRoute with Optimizer enabled', () => { + it('renders the originating insight description instead of relabeling the evaluation description', async () => { + server.use( + http.get('*/apis/intake/v2/workspaces/:workspace/evaluations/:name', () => + HttpResponse.json(evaluation) + ), + http.get('*/apis/intake/v2/workspaces/:workspace/experiment-groups/:name', () => + HttpResponse.json(group) + ), + http.get('*/apis/intake/v2/workspaces/:workspace/evaluations/:name/sessions', () => + HttpResponse.json({ + data: [], + pagination: { + page: 1, + page_size: 25, + current_page_size: 0, + total_pages: 0, + total_results: 0, + }, + }) + ), + http.get('*/apis/insights/v2/workspaces/:workspace/insights/:insightId', () => + HttpResponse.json({ + id: 'insight-id', + name: 'insight', + title: 'Insight', + description: 'Actual insight description', + agent: 'agent', + status: 'open', + trace_refs: [], + }) + ) + ); + + renderRoute(, { + history: `/workspaces/${WORKSPACE}/experiment/${GROUP_NAME}/${EVALUATION_NAME}`, + routes: [ + { + path: ROUTES.workspace.evaluationDetail, + element: , + }, + ], + }); + + expect(await screen.findByText('Actual insight description')).toBeInTheDocument(); + expect(screen.getByText('Insight description')).toBeInTheDocument(); + expect(screen.queryByText('Evaluation description')).not.toBeInTheDocument(); + expect(screen.getByRole('link', { name: /originating insight/i })).toHaveAttribute( + 'href', + `/workspaces/${WORKSPACE}/optimizer/insight-id` + ); + }); +}); diff --git a/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx b/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx index 2df2ca6d3b..6db07615b2 100644 --- a/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx +++ b/web/packages/studio/src/routes/EvaluationDetailRoute/index.tsx @@ -1,10 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { useGetEvaluation } from '@nemo/sdk/generated/platform/api'; -import { Badge, PageHeader, Stack, Text } from '@nvidia/foundations-react-core'; +import { useGetEvaluation, useGetExperimentGroup } from '@nemo/sdk/generated/platform/api'; +import { Badge, Card, Flex, PageHeader, Stack, Text } from '@nvidia/foundations-react-core'; +import { useOptimizerGetInsight } from '@studio/api/optimizer'; import { AccessibleTitle } from '@studio/components/AccessibleTitle'; import { EvaluationSessionsDataView } from '@studio/components/dataViews/EvaluationSessionsDataView'; +import { OriginatingInsightLink } from '@studio/components/OriginatingInsightLink'; +import { OPTIMIZER_ENABLED } from '@studio/constants/environment'; import { ROUTE_PARAMS } from '@studio/constants/routes'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; @@ -19,7 +22,14 @@ export const EvaluationDetailRoute: FC = () => { ROUTE_PARAMS.experimentGroupName, ROUTE_PARAMS.evaluationName, ]); - const { data: experiment } = useGetEvaluation(workspace, evaluationName); + const { data: evaluation } = useGetEvaluation(workspace, evaluationName); + // Evaluations reach their originating insight through the owning group's insight_id. + const { data: experimentGroup } = useGetExperimentGroup(workspace, experimentGroupName); + const insightId = experimentGroup?.insight_id ?? ''; + const { data: insight } = useOptimizerGetInsight(workspace, insightId, { + query: { enabled: OPTIMIZER_ENABLED && Boolean(insightId) }, + }); + const showInsightCard = Boolean(insight?.description); useBreadcrumbs({ items: [ @@ -38,15 +48,26 @@ export const EvaluationDetailRoute: FC = () => { + {showInsightCard ? ( + + + + + Insight description + {insight?.description} + + + + ) : null}
Test cases - {experiment?.run_count !== undefined && ( + {evaluation?.run_count !== undefined && ( - {experiment.run_count} + {evaluation.run_count} )}
diff --git a/web/packages/studio/src/routes/EvaluationDetailRoute/optimizerDisabled.test.tsx b/web/packages/studio/src/routes/EvaluationDetailRoute/optimizerDisabled.test.tsx new file mode 100644 index 0000000000..6f6125a36a --- /dev/null +++ b/web/packages/studio/src/routes/EvaluationDetailRoute/optimizerDisabled.test.tsx @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + EvaluationResponse, + ExperimentGroupResponse, +} from '@nemo/sdk/generated/platform/schema'; +import { ROUTES } from '@studio/constants/routes'; +import { server } from '@studio/mocks/node'; +import { EvaluationDetailRoute } from '@studio/routes/EvaluationDetailRoute'; +import { renderRoute, screen } from '@studio/tests/util/render'; +import { http, HttpResponse } from 'msw'; + +vi.hoisted(() => { + vi.stubEnv('VITE_FF_OPTIMIZER_ENABLED', 'false'); +}); + +const WORKSPACE = 'test-workspace'; +const GROUP_NAME = 'test-group'; +const EVALUATION_NAME = 'test-evaluation'; + +const evaluation = { + id: 'evaluation-id', + name: EVALUATION_NAME, + workspace: WORKSPACE, + experiment_group_id: 'group-id', + dataset_name: 'dataset', + description: 'Evaluation description', +} satisfies Partial; + +const group = { + id: 'group-id', + name: GROUP_NAME, + workspace: WORKSPACE, + insight_id: 'insight-id', + default_sort: '-created_at', + evaluation_count: 0, +} satisfies Partial; + +describe('EvaluationDetailRoute with Optimizer disabled', () => { + it('renders the evaluation description without requesting or linking to the insight', async () => { + const insightRequest = vi.fn(); + server.use( + http.get('*/apis/intake/v2/workspaces/:workspace/evaluations/:name', () => + HttpResponse.json(evaluation) + ), + http.get('*/apis/intake/v2/workspaces/:workspace/experiment-groups/:name', () => + HttpResponse.json(group) + ), + http.get('*/apis/intake/v2/workspaces/:workspace/evaluations/:name/sessions', () => + HttpResponse.json({ + data: [], + pagination: { + page: 1, + page_size: 25, + current_page_size: 0, + total_pages: 0, + total_results: 0, + }, + }) + ), + http.get('*/apis/insights/v2/workspaces/:workspace/insights/:insightId', () => { + insightRequest(); + return HttpResponse.json({}); + }) + ); + + renderRoute(, { + history: `/workspaces/${WORKSPACE}/experiment/${GROUP_NAME}/${EVALUATION_NAME}`, + routes: [ + { + path: ROUTES.workspace.evaluationDetail, + element: , + }, + ], + }); + + expect(await screen.findByText('Evaluation description')).toBeInTheDocument(); + expect(insightRequest).not.toHaveBeenCalled(); + expect(screen.queryByText('Insight description')).not.toBeInTheDocument(); + expect(screen.queryByRole('link', { name: /originating insight/i })).not.toBeInTheDocument(); + }); +}); diff --git a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.test.tsx b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.test.tsx new file mode 100644 index 0000000000..a538763633 --- /dev/null +++ b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.test.tsx @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import { ROUTES } from '@studio/constants/routes'; +import { server } from '@studio/mocks/node'; +import { ExperimentGroupDetailRoute } from '@studio/routes/ExperimentGroupDetailRoute'; +import { renderRoute, screen } from '@studio/tests/util/render'; +import { http, HttpResponse } from 'msw'; + +vi.hoisted(() => { + vi.stubEnv('VITE_FF_OPTIMIZER_ENABLED', 'false'); +}); + +const WORKSPACE = 'test-workspace'; +const GROUP_NAME = 'test-group'; + +const group = { + id: 'group-id', + name: GROUP_NAME, + workspace: WORKSPACE, + description: 'Editable group description', + summary: 'Generated group summary', + insight_id: 'insight-id', + default_sort: '-created_at', + evaluation_count: 0, +} satisfies Partial; + +describe('ExperimentGroupDetailRoute', () => { + it('renders the summary without requesting Optimizer when disabled', async () => { + const insightRequest = vi.fn(); + server.use( + http.get('*/apis/intake/v2/workspaces/:workspace/experiment-groups/:name', () => + HttpResponse.json(group) + ), + http.get('*/apis/intake/v2/workspaces/:workspace/evaluations', () => + HttpResponse.json({ + data: [], + pagination: { + page: 1, + page_size: 25, + current_page_size: 0, + total_pages: 0, + total_results: 0, + }, + }) + ), + http.get('*/apis/insights/v2/workspaces/:workspace/insights/:insightId', () => { + insightRequest(); + return HttpResponse.json({}); + }) + ); + + renderRoute(, { + history: `/workspaces/${WORKSPACE}/experiment/${GROUP_NAME}`, + routes: [ + { + path: ROUTES.workspace.experimentGroupDetail, + element: , + }, + ], + }); + + expect(await screen.findByText('Generated group summary')).toBeInTheDocument(); + expect(screen.queryByText('Editable group description')).not.toBeInTheDocument(); + expect(insightRequest).not.toHaveBeenCalled(); + expect(screen.queryByRole('link', { name: /originating insight/i })).not.toBeInTheDocument(); + }); +}); diff --git a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx index a5d69429e8..bec1a1b2b3 100644 --- a/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx +++ b/web/packages/studio/src/routes/ExperimentGroupDetailRoute/index.tsx @@ -3,10 +3,23 @@ import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; import { useGetExperimentGroup } from '@nemo/sdk/generated/platform/api'; -import { Badge, Button, PageHeader, Stack, Text } from '@nvidia/foundations-react-core'; +import { + Anchor, + Badge, + Button, + Card, + Flex, + PageHeader, + Stack, + Text, +} from '@nvidia/foundations-react-core'; +import { useOptimizerGetInsight } from '@studio/api/optimizer'; import { AccessibleTitle } from '@studio/components/AccessibleTitle'; import { ExperimentGroupDataView } from '@studio/components/dataViews/ExperimentGroupDataView'; import { ExperimentGroupEditModal } from '@studio/components/ExperimentGroupEditModal'; +import { OriginatingInsightLink } from '@studio/components/OriginatingInsightLink'; +import { OPTIMIZER_ENABLED } from '@studio/constants/environment'; +import { LINK_DOCS_STUDIO_EVALUATION } from '@studio/constants/links'; import { ROUTE_PARAMS } from '@studio/constants/routes'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; @@ -20,6 +33,10 @@ export const ExperimentGroupDetailRoute: FC = () => { const workspace = useWorkspaceFromPath(); const { experimentGroupName } = useRequiredPathParams([ROUTE_PARAMS.experimentGroupName]); const { data: group, error } = useGetExperimentGroup(workspace, experimentGroupName); + // The insight is a group-level concept, reached via the group's insight_id. + const { data: insight } = useOptimizerGetInsight(workspace, group?.insight_id ?? '', { + query: { enabled: OPTIMIZER_ENABLED && Boolean(group?.insight_id) }, + }); const [editOpen, setEditOpen] = useState(false); useBreadcrumbs({ @@ -35,7 +52,14 @@ export const ExperimentGroupDetailRoute: FC = () => { + An experiment is a group of evaluation runs aligned toward a common objective.{' '} + + Learn more + + + } slotActions={ + } + > + + + Run the following CLI command to start experiments for this insight. + + + + + + ); +}; diff --git a/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.test.tsx b/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.test.tsx new file mode 100644 index 0000000000..979300e94a --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.test.tsx @@ -0,0 +1,111 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { DEFAULT_WORKSPACE } from '@nemo/common/src/models/constants'; +import type { Trace } from '@nemo/sdk/generated/platform/schema'; +import { server } from '@studio/mocks/node'; +import { InsightTracesTable } from '@studio/routes/optimizer/InsightTracesTable'; +import { renderRoute, screen, waitFor } from '@studio/tests/util/render'; +import userEvent from '@testing-library/user-event'; +import { http, HttpResponse } from 'msw'; + +const makeTrace = (sequence: number): Trace => ({ + id: `trace-${String(sequence).padStart(2, '0')}`, + session_id: `session-${sequence}`, + workspace: DEFAULT_WORKSPACE, + name: `Trace ${String(sequence).padStart(2, '0')}`, + started_at: `2026-07-20T12:${String(sequence).padStart(2, '0')}:00Z`, + status: 'success', +}); + +const installTraceHandler = ({ + traces, + failedIds = [], +}: { + traces: Trace[]; + failedIds?: string[]; +}) => { + const requests: Array<{ id: string; mode: string | null }> = []; + const tracesById = new Map(traces.map((trace) => [trace.id, trace])); + + server.use( + http.get('*/apis/intake/v2/workspaces/:workspace/traces/:traceId', ({ params, request }) => { + const id = String(params['traceId']); + requests.push({ id, mode: new URL(request.url).searchParams.get('mode') }); + + if (failedIds.includes(id)) { + return HttpResponse.json({ detail: `Could not load ${id}` }, { status: 500 }); + } + + return HttpResponse.json(tracesById.get(id)); + }) + ); + + return requests; +}; + +describe('InsightTracesTable', () => { + it('requests and displays only the current page in reference order using preview mode', async () => { + const user = userEvent.setup(); + const traces = Array.from({ length: 11 }, (_, index) => makeTrace(index + 1)); + const traceIds = traces.map((trace) => trace.id); + const requests = installTraceHandler({ traces }); + + renderRoute(, { + history: '/optimizer?page_size=10', + }); + + await screen.findByText('Trace 01'); + await waitFor(() => + expect(requests).toEqual( + traceIds.slice(0, 10).map((id) => ({ + id, + mode: 'preview', + })) + ) + ); + expect(screen.queryByText('Trace 11')).not.toBeInTheDocument(); + const rows = screen.getAllByRole('row'); + expect(rows[1]).toHaveTextContent('Trace 01'); + expect(rows[2]).toHaveTextContent('Trace 02'); + expect(screen.getByText('1-10 of 11 items')).toBeInTheDocument(); + + await user.click(screen.getByRole('button', { name: /next page/i })); + + expect(await screen.findByText('Trace 11')).toBeInTheDocument(); + await waitFor(() => expect(requests.map(({ id }) => id)).toEqual(traceIds)); + expect(screen.queryByText('Trace 01')).not.toBeInTheDocument(); + expect(screen.getByText('11-11 of 11 items')).toBeInTheDocument(); + }); + + it('keeps successful rows visible when part of the current page fails', async () => { + const traces = [makeTrace(1), makeTrace(2)]; + installTraceHandler({ traces, failedIds: [traces[1].id] }); + + renderRoute( + trace.id)} + /> + ); + + expect(await screen.findByText('Trace 01')).toBeInTheDocument(); + expect(screen.getByText("1 of 2 traces couldn't be loaded.")).toBeInTheDocument(); + expect(screen.getByText('1-2 of 2 items')).toBeInTheDocument(); + }); + + it('shows an error instead of an empty state when every current-page request fails', async () => { + const traces = [makeTrace(1), makeTrace(2)]; + installTraceHandler({ traces, failedIds: traces.map((trace) => trace.id) }); + + renderRoute( + trace.id)} + /> + ); + + expect(await screen.findByText('Error')).toBeInTheDocument(); + expect(screen.queryByText('This insight has no linked traces.')).not.toBeInTheDocument(); + }); +}); diff --git a/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.tsx b/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.tsx new file mode 100644 index 0000000000..0aaaa45e86 --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/InsightTracesTable/index.tsx @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { EditColumnsMenu } from '@nemo/common/src/components/DataView/internal'; +import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; +import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; +import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; +import { getGetTraceQueryKey, getTrace } from '@nemo/sdk/generated/platform/api'; +import type { Trace } from '@nemo/sdk/generated/platform/schema'; +import { Flex, Stack, Text } from '@nvidia/foundations-react-core'; +import { getErrorMessage } from '@studio/api/common/utils'; +import { IntakeTelemetryDataView } from '@studio/components/IntakeLists/IntakeTelemetryDataView'; +import { makeIntakeTraceColumns } from '@studio/components/IntakeLists/intakeTraceColumns'; +import { getIntakeTraceRoute } from '@studio/routes/utils'; +import { useQueries } from '@tanstack/react-query'; +import { Columns3, TriangleAlert } from 'lucide-react'; +import { type FC } from 'react'; +import { useNavigate } from 'react-router-dom'; + +const TRACE_PREVIEW_PARAMS = { mode: 'preview' } as const; + +export interface InsightTracesTableProps { + workspace: string; + /** Intake trace ids (the insight's `trace_refs`). */ + traceIds: string[]; +} + +/** + * Renders an insight's evidence traces using the same columns and DataView shell as + * `IntakeTracesTable`. Unlike the workspace browse table, this fetches only the referenced + * traces by id and preserves `traceIds` order (no server sort/filter). + */ +export const InsightTracesTable: FC = ({ workspace, traceIds }) => { + const navigate = useNavigate(); + const dataViewState = useStudioDataViewState(); + const { pageIndex, pageSize } = dataViewState.pagination.state; + const firstVisibleIndex = pageIndex * pageSize; + const visibleTraceIds = traceIds.slice(firstVisibleIndex, firstVisibleIndex + pageSize); + + const results = useQueries({ + queries: visibleTraceIds.map((id) => ({ + queryKey: getGetTraceQueryKey(workspace, id, TRACE_PREVIEW_PARAMS), + queryFn: ({ signal }) => getTrace(workspace, id, TRACE_PREVIEW_PARAMS, signal), + enabled: Boolean(workspace) && Boolean(id), + })), + }); + + const traces = results.map((r) => r.data).filter((t): t is Trace => Boolean(t)); + const isFetching = results.some((r) => r.isFetching); + const failedCount = results.filter((r) => r.isError).length; + const allFailed = + visibleTraceIds.length > 0 && failedCount === visibleTraceIds.length && !isFetching; + const firstError = results.find((r) => r.error)?.error; + + return ( + + {failedCount > 0 && !allFailed ? ( + + + + {failedCount} of {visibleTraceIds.length} traces couldn't be loaded. + + + ) : null} + + dataViewState={dataViewState} + makeColumns={makeIntakeTraceColumns()} + onRowClick={(trace) => navigate(getIntakeTraceRoute(workspace, trace.id))} + toolbarSlotEnd={ + } + > + <> + + Columns + + + } + attributes={{ + DataViewRoot: { + data: traces, + totalCount: traceIds.length, + requestStatus: allFailed ? 'error' : isFetching ? 'loading' : undefined, + }, + DataViewTableContent: { + renderEmptyState: () => ( + + ), + renderErrorState: () => ( + + ), + }, + }} + /> + + ); +}; diff --git a/web/packages/studio/src/routes/optimizer/OptimizerEvalAuthorRunRoute/artifactUtils.ts b/web/packages/studio/src/routes/optimizer/OptimizerEvalAuthorRunRoute/artifactUtils.ts new file mode 100644 index 0000000000..5a261ebad5 --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/OptimizerEvalAuthorRunRoute/artifactUtils.ts @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { useDatasetFileContent } from '@studio/api/datasets/useDatasetFileContent'; + +export interface FilesetRef { + workspace: string; + name: string; +} + +export const parseFilesetRef = (ref: string | null | undefined): FilesetRef | null => { + if (!ref) return null; + try { + const url = new URL(ref); + if (url.protocol !== 'fileset:' || !url.hostname) return null; + const name = url.pathname.split('/').filter(Boolean)[0]; + return name ? { workspace: url.hostname, name } : null; + } catch { + return null; + } +}; + +export const useArtifactText = ( + fileset: FilesetRef | null, + path: string | null | undefined, + enabled = true +) => + useDatasetFileContent({ + workspace: fileset?.workspace ?? '', + name: fileset?.name ?? '', + path: path ?? '', + enabled: enabled && !!fileset && !!path, + }); + +export const parseArtifactJson = (content: string | undefined): T | null => { + if (!content) return null; + try { + return JSON.parse(content) as T; + } catch { + return null; + } +}; diff --git a/web/packages/studio/src/routes/optimizer/OptimizerEvalAuthorRunRoute/artifacts.tsx b/web/packages/studio/src/routes/optimizer/OptimizerEvalAuthorRunRoute/artifacts.tsx new file mode 100644 index 0000000000..ae273fe10b --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/OptimizerEvalAuthorRunRoute/artifacts.tsx @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CodeEditor } from '@nemo/common/src/components/CodeEditor'; +import { ContentType } from '@nemo/common/src/components/CodeEditor/constants'; +import { filesDownloadFile } from '@nemo/sdk/generated/platform/api'; +import { Button, Spinner } from '@nvidia/foundations-react-core'; +import { type FilesetRef } from '@studio/routes/optimizer/OptimizerEvalAuthorRunRoute/artifactUtils'; +import { Download } from 'lucide-react'; +import { type FC } from 'react'; + +interface ArtifactCodeProps { + content: string | undefined; + loading: boolean; + contentType?: ContentType; + emptyMessage: string; +} + +export const ArtifactCode: FC = ({ + content, + loading, + contentType = ContentType.JSON, + emptyMessage, +}) => { + if (loading) { + return ; + } + if (content === undefined) { + return
{emptyMessage}
; + } + return ( + + ); +}; + +interface DownloadArtifactButtonProps { + fileset: FilesetRef; + path: string; +} + +export const DownloadArtifactButton: FC = ({ fileset, path }) => { + const download = async () => { + const blob = await filesDownloadFile(fileset.workspace, fileset.name, path); + const href = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = href; + link.download = path.split('/').at(-1) ?? 'artifact'; + link.click(); + URL.revokeObjectURL(href); + }; + + return ( + + ); +}; diff --git a/web/packages/studio/src/routes/optimizer/OptimizerEvalAuthorRunRoute/index.tsx b/web/packages/studio/src/routes/optimizer/OptimizerEvalAuthorRunRoute/index.tsx new file mode 100644 index 0000000000..01386a8c97 --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/OptimizerEvalAuthorRunRoute/index.tsx @@ -0,0 +1,614 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CodeEditor } from '@nemo/common/src/components/CodeEditor'; +import { ContentType } from '@nemo/common/src/components/CodeEditor/constants'; +import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; +import { KVPair } from '@nemo/common/src/components/KVPair'; +import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; +import { useFilesListFilesetFiles } from '@nemo/sdk/generated/platform/api'; +import { + Anchor, + Banner, + Button, + Flex, + Grid, + PageHeader, + Panel, + SelectContent, + SelectItem, + SelectListbox, + SelectRoot, + SelectTrigger, + Stack, + TabsContent, + TabsList, + TabsRoot, + TabsTrigger, + Text, +} from '@nvidia/foundations-react-core'; +import { type EvalAuthorRun, useOptimizerGetEvalAuthorRun } from '@studio/api/optimizer'; +import { AccessibleTitle } from '@studio/components/AccessibleTitle'; +import { FilesetFilePreviewContent } from '@studio/components/FilesetFilePreviewPanel/FilesetFilePreviewContent'; +import { Loading } from '@studio/components/Layouts/Loading'; +import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; +import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; +import { EvalAuthorRunStatusBadge } from '@studio/routes/optimizer/EvalAuthorRunStatusBadge'; +import { + ArtifactCode, + DownloadArtifactButton, +} from '@studio/routes/optimizer/OptimizerEvalAuthorRunRoute/artifacts'; +import { + parseArtifactJson, + parseFilesetRef, + useArtifactText, +} from '@studio/routes/optimizer/OptimizerEvalAuthorRunRoute/artifactUtils'; +import { + getFilesetDetailRoute, + getIntakeTraceRoute, + getOptimizerInsightRoute, + getOptimizerRoute, +} from '@studio/routes/utils'; +import { useEffect, useMemo, useState, type FC, type ReactNode } from 'react'; +import { Link, useParams } from 'react-router-dom'; + +interface ArtifactManifest { + schema_version: number; + prompt_requests?: Array<{ attempt: number; turn: number; path: string }>; +} + +interface CapturedRequest { + schema_version?: number; + captured_at?: string; + attempt?: number; + turn?: number; + model?: string; + messages?: Array<{ role?: string; content?: unknown }>; + params?: Record; + redactions?: { applied?: boolean; fields?: string[] }; +} + +interface VerifierManifest { + schema_version: number; + entrypoint?: string; + metrics?: Array<{ + name: string; + reward_keys?: string[]; + entrypoint?: string; + files?: Array<{ + path: string; + artifact_path?: string; + language?: string; + sha256?: string; + applied_to_train_tasks?: number; + applied_to_validation_tasks?: number; + }>; + }>; +} + +const TERMINAL_STATUSES = new Set(['succeeded', 'failed', 'cancelled']); + +const isTerminal = (run: EvalAuthorRun | undefined) => !!run && TERMINAL_STATUSES.has(run.status); + +const duration = (run: EvalAuthorRun): string => { + if (!run.started_at) return '—'; + const end = run.completed_at ? Date.parse(run.completed_at) : Date.now(); + const seconds = Math.max(0, Math.round((end - Date.parse(run.started_at)) / 1_000)); + if (seconds < 60) return `${seconds}s`; + return `${Math.floor(seconds / 60)}m ${seconds % 60}s`; +}; + +const filesetLink = (ref: string | null | undefined): ReactNode => { + const fileset = parseFilesetRef(ref); + if (!fileset || !ref) return '—'; + return ( + + {ref} + + ); +}; + +const OverviewTab: FC<{ run: EvalAuthorRun }> = ({ run }) => ( + + {run.error && ( + + {run.error} + + )} + + + + } /> + + + + : '—'} + /> + : '—'} + /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + } + /> + + + + + + + + + + + + +); + +const PromptTab: FC<{ run: EvalAuthorRun }> = ({ run }) => { + const fileset = parseFilesetRef(run.outputs.artifact_fileset); + const manifestQuery = useArtifactText(fileset, 'manifest.json', !!fileset); + const manifest = parseArtifactJson(manifestQuery.data); + const requests = useMemo(() => manifest?.prompt_requests ?? [], [manifest?.prompt_requests]); + const [selectedPath, setSelectedPath] = useState(''); + + useEffect(() => { + if (!selectedPath && requests[0]?.path) setSelectedPath(requests[0].path); + }, [requests, selectedPath]); + + const requestQuery = useArtifactText(fileset, selectedPath, !!selectedPath); + const request = parseArtifactJson(requestQuery.data); + const trajectoryQuery = useArtifactText( + fileset, + 'trajectory/atif.json', + run.capture.trajectory !== 'unavailable' + ); + + if (run.capture.prompt === 'unavailable') { + return ( + + Not captured for this legacy run. The request cannot be reconstructed after execution. + + ); + } + + return ( + + {(run.capture.redactions || request?.redactions?.applied) && ( + + The effective request is exact except for named secret redactions:{' '} + {[...run.capture.redacted_fields, ...(request?.redactions?.fields ?? [])].join(', ') || + 'sensitive values'} + . + + )} + + setSelectedPath(value)}> + { + const selected = requests.find((item) => item.path === selectedPath); + return selected ? `Attempt ${selected.attempt}, turn ${selected.turn}` : null; + }} + /> + + + {requests.map((item) => ( + + Attempt {item.attempt}, turn {item.turn} + + ))} + + + + {fileset && selectedPath && ( + + )} + + + + + + + } + /> + + {(request?.messages ?? []).map((message, index) => ( + + + {typeof message.content === 'string' + ? message.content + : JSON.stringify(message.content, null, 2)} + + + ))} + + + + + + + + + ); +}; + +const VerifierTab: FC<{ run: EvalAuthorRun }> = ({ run }) => { + const fileset = parseFilesetRef(run.outputs.artifact_fileset); + const manifestQuery = useArtifactText(fileset, 'verifier/manifest.json', !!fileset); + const manifest = parseArtifactJson(manifestQuery.data); + const metrics = useMemo(() => manifest?.metrics ?? [], [manifest?.metrics]); + const [metricName, setMetricName] = useState(''); + const [filePath, setFilePath] = useState(''); + const metric = metrics.find((item) => item.name === metricName) ?? metrics[0]; + const files = useMemo(() => metric?.files ?? [], [metric?.files]); + + useEffect(() => { + if (!metricName && metrics[0]?.name) setMetricName(metrics[0].name); + }, [metricName, metrics]); + useEffect(() => { + const next = + files[0]?.artifact_path ?? (files[0]?.path ? `verifier/files/${files[0].path}` : ''); + if ( + !filePath || + !files.some((file) => (file.artifact_path ?? `verifier/files/${file.path}`) === filePath) + ) { + setFilePath(next); + } + }, [filePath, files]); + + const sourceQuery = useArtifactText(fileset, filePath, !!filePath); + const selectedFile = files.find( + (file) => (file.artifact_path ?? `verifier/files/${file.path}`) === filePath + ); + + return ( + + + setMetricName(value)} + > + + + + {metrics.map((item) => ( + + {item.name} + + ))} + + + + setFilePath(value)}> + + + + {files.map((file) => { + const path = file.artifact_path ?? `verifier/files/${file.path}`; + return ( + + {file.path} + + ); + })} + + + + + + + + + + + + + + + + + +
+ + + +
+ + + {run.outputs.train_dataset && filesetLink(run.outputs.train_dataset)} + {run.outputs.validation_dataset && filesetLink(run.outputs.validation_dataset)} + + + ); +}; + +const DataTab: FC<{ run: EvalAuthorRun }> = ({ run }) => { + const fileset = parseFilesetRef(run.outputs.artifact_fileset); + const insightQuery = useArtifactText(fileset, 'analysis/insight.json', !!fileset); + const conventionsQuery = useArtifactText(fileset, 'analysis/runner-conventions.md', !!fileset); + const validationQuery = useArtifactText(fileset, 'validation/attempts.json', !!fileset); + + return ( + + + + {run.inputs.trace_refs.map((traceId) => ( + + {traceId} + + ))} + + + + + + + + + + + + + + + + + + + + + + + + + + ); +}; + +const ArtifactsTab: FC<{ run: EvalAuthorRun }> = ({ run }) => { + const fileset = parseFilesetRef(run.outputs.artifact_fileset); + const { + data: response, + isLoading, + isError, + } = useFilesListFilesetFiles(fileset?.workspace ?? '', fileset?.name ?? '', undefined, { + query: { enabled: !!fileset }, + }); + const files = useMemo(() => response?.data ?? [], [response?.data]); + const [selectedPath, setSelectedPath] = useState(''); + + if (!fileset) { + return ( + + This run has no artifact Fileset. + + ); + } + + return ( + + + + + Browse full Fileset + + + {selectedPath && } + + {isError ? ( + + ) : ( + + + + {isLoading && } + {files.map((file) => ( + + ))} + + +
+ {selectedPath ? ( + + ) : ( + + Select an artifact to preview. + + )} +
+ + )} + + ); +}; + +export const OptimizerEvalAuthorRunRoute: FC = () => { + const workspace = useWorkspaceFromPath(); + const { insightId = '', evalAuthorRunId = '' } = useParams<{ + insightId: string; + evalAuthorRunId: string; + }>(); + const { + data: run, + isLoading, + isError, + refetch, + } = useOptimizerGetEvalAuthorRun(workspace, evalAuthorRunId, { + query: { + refetchInterval: (query) => (isTerminal(query.state.data) ? false : 5_000), + }, + }); + + useBreadcrumbs({ + items: [ + { href: getOptimizerRoute(workspace), slotLabel: 'Insights' }, + { href: getOptimizerInsightRoute(workspace, insightId), slotLabel: 'Insight' }, + { slotLabel: run?.name ?? evalAuthorRunId }, + ], + }); + + if (isLoading && !run) return ; + if (isError || !run) { + return ( + + void refetch()}> + Retry + + } + /> + + ); + } + + return ( + + + } + /> + + + Overview + Prompt + Verifier + Data + Artifacts + + + + + + + + + + + + + + + + + + + + ); +}; diff --git a/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/InsightEvalAuthorRuns.tsx b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/InsightEvalAuthorRuns.tsx new file mode 100644 index 0000000000..875f259db1 --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/InsightEvalAuthorRuns.tsx @@ -0,0 +1,177 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import * as DataView from '@nemo/common/src/components/DataView/internal'; +import { useRowClick } from '@nemo/common/src/components/DataView/useRowClick'; +import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; +import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; +import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; +import { DEFAULT_PAGE_SIZE_OPTIONS } from '@nemo/common/src/constants/pagination'; +import { Button, Text } from '@nvidia/foundations-react-core'; +import { type EvalAuthorRun, useOptimizerListEvalAuthorRuns } from '@studio/api/optimizer'; +import { EvalAuthorRunStatusBadge } from '@studio/routes/optimizer/EvalAuthorRunStatusBadge'; +import { getOptimizerEvalAuthorRunRoute } from '@studio/routes/utils'; +import { keepPreviousData } from '@tanstack/react-query'; +import { FileCode2 } from 'lucide-react'; +import { type ComponentProps, type FC } from 'react'; +import { useNavigate } from 'react-router-dom'; + +const TERMINAL_STATUSES = new Set(['succeeded', 'failed', 'cancelled']); + +const isEvalAuthorRun = (value: unknown): value is EvalAuthorRun => { + if (!value || typeof value !== 'object') return false; + const candidate = value as Partial; + return ( + typeof candidate.id === 'string' && + typeof candidate.status === 'string' && + typeof candidate.stage === 'string' && + !!candidate.outputs && + Array.isArray(candidate.outputs.metric_names) + ); +}; + +const duration = (run: EvalAuthorRun): string => { + if (!run.started_at) return '—'; + const end = run.completed_at ? Date.parse(run.completed_at) : Date.now(); + const seconds = Math.max(0, Math.round((end - Date.parse(run.started_at)) / 1_000)); + if (seconds < 60) return `${seconds}s`; + const minutes = Math.floor(seconds / 60); + const remainder = seconds % 60; + return `${minutes}m ${remainder}s`; +}; + +const makeColumns: ComponentProps>['makeColumns'] = ({ + accessor, +}) => [ + accessor('name', { + header: 'Run', + enableSorting: false, + size: 220, + cell: ({ getValue }) => {getValue()}, + }), + accessor('status', { + header: 'Status', + enableSorting: false, + size: 110, + cell: ({ getValue }) => , + }), + accessor('stage', { + header: 'Stage', + enableSorting: false, + size: 150, + cell: ({ getValue }) => {getValue().replaceAll('_', ' ')}, + }), + accessor('outputs', { + id: 'metrics', + header: 'Metrics', + enableSorting: false, + size: 220, + cell: ({ getValue }) => {getValue()?.metric_names.join(', ') || '—'}, + }), + accessor('outputs', { + id: 'train_task_count', + header: 'Train', + enableSorting: false, + size: 72, + cell: ({ getValue }) => {getValue()?.train_task_count ?? '—'}, + }), + accessor('outputs', { + id: 'validation_task_count', + header: 'Validation', + enableSorting: false, + size: 88, + cell: ({ getValue }) => {getValue()?.validation_task_count ?? '—'}, + }), + accessor('started_at', { + header: 'Started', + enableSorting: false, + size: 120, + cell: ({ getValue }) => (getValue() ? : —), + }), + accessor('completed_at', { + id: 'duration', + header: 'Duration', + enableSorting: false, + size: 100, + cell: ({ row }) => {duration(row.original)}, + }), +]; + +interface InsightEvalAuthorRunsProps { + workspace: string; + insightId: string; +} + +export const InsightEvalAuthorRuns: FC = ({ workspace, insightId }) => { + const navigate = useNavigate(); + const dataViewState = DataView.useDataViewState({ + pagination: { paginationOptions: DEFAULT_PAGE_SIZE_OPTIONS }, + }); + const { pageIndex, pageSize } = dataViewState.pagination.state; + const params = { + page: pageIndex + 1, + page_size: pageSize, + sort: '-created_at', + insight_id: insightId, + }; + const { + data: response, + isError, + isFetching, + refetch, + } = useOptimizerListEvalAuthorRuns(workspace, params, { + query: { + placeholderData: keepPreviousData, + refetchInterval: (query) => + query.state.data?.data?.some((run) => !TERMINAL_STATUSES.has(run.status)) ? 5_000 : false, + }, + }); + const runs = (response?.data ?? []).filter(isEvalAuthorRun); + const { wrapColumns, onClick, className } = useRowClick( + (run: EvalAuthorRun) => navigate(getOptimizerEvalAuthorRunRoute(workspace, insightId, run.id)), + runs + ); + + return ( + +
+ ( + } + /> + )} + renderErrorState={() => ( + void refetch()}> + Retry + + } + /> + )} + /> + +
+ + ); +}; diff --git a/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/InsightExperimentGroups.tsx b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/InsightExperimentGroups.tsx new file mode 100644 index 0000000000..d2b3917b3c --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/InsightExperimentGroups.tsx @@ -0,0 +1,139 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import * as DataView from '@nemo/common/src/components/DataView/internal'; +import { useRowClick } from '@nemo/common/src/components/DataView/useRowClick'; +import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; +import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; +import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; +import { DEFAULT_PAGE_SIZE_OPTIONS } from '@nemo/common/src/constants/pagination'; +import { useListExperimentGroups } from '@nemo/sdk/generated/platform/api'; +import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import { Button, Text } from '@nvidia/foundations-react-core'; +import { getExperimentGroupDetailRoute } from '@studio/routes/utils'; +import { keepPreviousData } from '@tanstack/react-query'; +import { FlaskConical } from 'lucide-react'; +import { type ComponentProps, type FC } from 'react'; +import { useNavigate } from 'react-router-dom'; + +const makeColumns: ComponentProps>['makeColumns'] = ({ + accessor, +}) => [ + accessor('name', { + header: 'Experiments', + enableSorting: false, + size: 280, + cell: ({ getValue }) => {getValue()}, + }), + accessor('evaluation_count', { + header: 'Evaluations', + enableSorting: false, + size: 100, + cell: ({ getValue }) => {getValue() ?? 0}, + }), + accessor('updated_at', { + header: 'Updated', + enableSorting: false, + size: 120, + cell: ({ row }) => + row.original.updated_at ? ( + + ) : ( + — + ), + }), +]; + +interface InsightExperimentGroupsProps { + workspace: string; + insightId: string; + onRunExperiment: () => void; + runExperimentDisabled: boolean; +} + +export const InsightExperimentGroups: FC = ({ + workspace, + insightId, + onRunExperiment, + runExperimentDisabled, +}) => { + const navigate = useNavigate(); + const dataViewState = DataView.useDataViewState({ + pagination: { paginationOptions: DEFAULT_PAGE_SIZE_OPTIONS }, + }); + const { pageIndex, pageSize } = dataViewState.pagination.state; + const { + data: response, + isError, + isFetching, + refetch, + } = useListExperimentGroups( + workspace, + { + page: pageIndex + 1, + page_size: pageSize, + sort: '-created_at', + filter: { insight_id: insightId }, + }, + { query: { placeholderData: keepPreviousData } } + ); + const groups = response?.data ?? []; + const { wrapColumns, onClick, className } = useRowClick( + (group: ExperimentGroupResponse) => + navigate(getExperimentGroupDetailRoute(workspace, group.name)), + groups + ); + + return ( + +
+ ( + } + actions={ + + } + /> + )} + renderErrorState={() => ( + void refetch()}> + Retry + + } + /> + )} + /> + +
+ + ); +}; diff --git a/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.test.tsx b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.test.tsx new file mode 100644 index 0000000000..19952edb37 --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.test.tsx @@ -0,0 +1,218 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + getListEvaluationsQueryKey, + getListExperimentGroupsQueryKey, +} from '@nemo/sdk/generated/platform/api'; +import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema'; +import type { Insight } from '@studio/api/optimizer'; +import { PLATFORM_BASE_URL } from '@studio/constants/environment'; +import { ROUTES } from '@studio/constants/routes'; +import { server } from '@studio/mocks/node'; +import { OptimizerInsightRoute } from '@studio/routes/optimizer/OptimizerInsightRoute'; +import { getOptimizerInsightRoute } from '@studio/routes/utils'; +import { renderRoute, screen, waitFor, within } from '@studio/tests/util/render'; +import userEvent from '@testing-library/user-event'; +import { http, HttpResponse } from 'msw'; +import { useParams } from 'react-router-dom'; + +const WORKSPACE = 'workspace-a'; +const INSIGHT_ID = 'insight-a'; +const INSIGHT_URL = `${PLATFORM_BASE_URL}/apis/insights/v2/workspaces/:workspace/insights/:insightId`; +const RUNS_URL = `${PLATFORM_BASE_URL}/apis/insights/v2/workspaces/:workspace/eval-author-runs`; +const GROUPS_URL = `${PLATFORM_BASE_URL}${getListExperimentGroupsQueryKey(':workspace')[0]}`; +const EVALUATIONS_URL = `${PLATFORM_BASE_URL}${getListEvaluationsQueryKey(':workspace')[0]}`; + +const insight: Insight = { + id: INSIGHT_ID, + name: INSIGHT_ID, + title: 'Slow responses', + description: 'The agent responds too slowly.', + agent: 'research-agent', + status: 'open', + trace_refs: [], +}; + +const pagination = ({ + page = 1, + pageSize = 10, + currentPageSize = 0, + totalResults = 0, +}: { + page?: number; + pageSize?: number; + currentPageSize?: number; + totalResults?: number; +} = {}) => ({ + page, + page_size: pageSize, + current_page_size: currentPageSize, + total_pages: Math.ceil(totalResults / pageSize), + total_results: totalResults, +}); + +const makeGroup = ( + id: string, + overrides: Partial = {} +): ExperimentGroupResponse => + ({ + id, + name: `${id}-name`, + workspace: WORKSPACE, + insight_id: insight.id, + default_sort: '-created_at', + summary: `${id} summary`, + description: `${id} description`, + evaluation_count: 3, + created_at: '2026-07-19T12:00:00Z', + updated_at: '2026-07-20T12:00:00Z', + ...overrides, + }) as ExperimentGroupResponse; + +const ExperimentGroupDestination = () => { + const { experimentGroupName } = useParams(); + return
{`Opened ${experimentGroupName}`}
; +}; + +const renderInsight = (history = getOptimizerInsightRoute(WORKSPACE, INSIGHT_ID)) => + renderRoute(undefined, { + history, + routes: [ + { path: ROUTES.workspace.optimizerInsight, element: }, + { + path: ROUTES.workspace.experimentGroupDetail, + element: , + }, + ], + }); + +describe('OptimizerInsightRoute experiments', () => { + beforeEach(() => { + server.use( + http.get(INSIGHT_URL, () => HttpResponse.json(insight)), + http.get(RUNS_URL, () => HttpResponse.json({ data: [], pagination: pagination() })), + http.get(GROUPS_URL, () => HttpResponse.json({ data: [], pagination: pagination() })) + ); + }); + + it('renders a compact Experiments list without requesting Evaluations', async () => { + const group = makeGroup('latency-experiment'); + const evaluationRequest = vi.fn(() => HttpResponse.json({})); + server.use( + http.get(GROUPS_URL, ({ request }) => { + const params = new URL(request.url).searchParams; + expect(params.get('filter[insight_id]')).toBe(INSIGHT_ID); + return HttpResponse.json({ + data: [group], + pagination: pagination({ currentPageSize: 1, totalResults: 1 }), + }); + }), + http.get(EVALUATIONS_URL, evaluationRequest) + ); + + renderInsight(); + + const row = await screen.findByRole('row', { name: new RegExp(group.name) }); + expect(within(row).getByText(String(group.evaluation_count))).toBeInTheDocument(); + expect(evaluationRequest).not.toHaveBeenCalled(); + }); + + it('distinguishes a group-list failure from a successful empty page', async () => { + server.use(http.get(GROUPS_URL, () => new HttpResponse(null, { status: 500 }))); + const { unmount } = renderInsight(); + + expect(await screen.findByText('Failed to load experiments')).toBeInTheDocument(); + expect(screen.queryByText('No experiments for this insight.')).not.toBeInTheDocument(); + unmount(); + + server.use( + http.get(GROUPS_URL, () => HttpResponse.json({ data: [], pagination: pagination() })) + ); + renderInsight(); + + expect(await screen.findByText('No experiments for this insight.')).toBeInTheDocument(); + expect(screen.queryByText('Failed to load experiments')).not.toBeInTheDocument(); + }); + + it('requests and displays only the selected ExperimentGroup page', async () => { + const user = userEvent.setup(); + const firstPageGroups = Array.from({ length: 10 }, (_unused, index) => + makeGroup(`page-one-${index + 1}`) + ); + const secondPageGroup = makeGroup('page-two-1'); + const requests: Array<{ page: string | null; pageSize: string | null }> = []; + + server.use( + http.get(GROUPS_URL, ({ request }) => { + const params = new URL(request.url).searchParams; + const page = params.get('page'); + requests.push({ page, pageSize: params.get('page_size') }); + return HttpResponse.json({ + data: page === '2' ? [secondPageGroup] : firstPageGroups, + pagination: pagination({ + page: Number(page ?? 1), + currentPageSize: page === '2' ? 1 : 10, + totalResults: 11, + }), + }); + }) + ); + + renderInsight(); + + expect(await screen.findByText(firstPageGroups[0].name)).toBeInTheDocument(); + expect(screen.queryByText(secondPageGroup.name)).not.toBeInTheDocument(); + + const nextPageButton = screen + .getAllByRole('button', { name: /next page/i }) + .find((button) => !button.hasAttribute('disabled')); + if (!nextPageButton) throw new Error('Enabled Experiments next-page button not found'); + await user.click(nextPageButton); + + expect(await screen.findByText(secondPageGroup.name)).toBeInTheDocument(); + expect(screen.queryByText(firstPageGroups[0].name)).not.toBeInTheDocument(); + await waitFor(() => + expect(requests).toEqual([ + { page: '1', pageSize: '10' }, + { page: '2', pageSize: '10' }, + ]) + ); + + await user.click(screen.getByText(secondPageGroup.name)); + expect(await screen.findByText(`Opened ${secondPageGroup.name}`)).toBeInTheDocument(); + }); + + it('reports status mutation failures through the Studio toast', async () => { + server.use(http.patch(INSIGHT_URL, () => new HttpResponse(null, { status: 500 }))); + renderInsight(); + + await userEvent.click(await screen.findByRole('button', { name: 'Delete' })); + + expect(await screen.findByText('Failed to update insight.')).toBeInTheDocument(); + }); + + it('renders Eval Author lifecycle statuses with their domain labels', async () => { + server.use( + http.get(RUNS_URL, () => + HttpResponse.json({ + data: [ + { + id: 'run-a', + name: 'run-a', + status: 'succeeded', + stage: 'completed', + outputs: { metric_names: ['groundedness'] }, + }, + ], + pagination: pagination({ currentPageSize: 1, totalResults: 1 }), + }) + ) + ); + + renderInsight(); + + expect(await screen.findByText('Succeeded')).toBeInTheDocument(); + expect(screen.queryByText('Unknown')).not.toBeInTheDocument(); + }); +}); diff --git a/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.tsx b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.tsx new file mode 100644 index 0000000000..49a0d8ff4d --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/OptimizerInsightRoute/index.tsx @@ -0,0 +1,223 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; +import { KVPair } from '@nemo/common/src/components/KVPair'; +import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; +import { useToast } from '@nemo/common/src/providers/toast/useToast'; +import { + Anchor, + Button, + Card, + Divider, + Flex, + PageHeader, + Stack, + Tag, + Text, +} from '@nvidia/foundations-react-core'; +import { + getOptimizerGetInsightQueryKey, + getOptimizerListInsightsQueryKey, + useOptimizerGetInsight, + useOptimizerUpdateInsight, + type InsightStatus, +} from '@studio/api/optimizer'; +import { AccessibleTitle } from '@studio/components/AccessibleTitle'; +import { ExpandableMessage } from '@studio/components/ExpandableMessage'; +import { FeatureFlagBadge } from '@studio/components/FeatureFlagBadge'; +import { Loading } from '@studio/components/Layouts/Loading'; +import { LINK_DOCS_STUDIO_EVALUATION } from '@studio/constants/links'; +import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; +import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; +import { InsightOpenModal } from '@studio/routes/optimizer/InsightOpenModal'; +import { insightActions, insightStatusColor } from '@studio/routes/optimizer/insightStatus'; +import { InsightTracesTable } from '@studio/routes/optimizer/InsightTracesTable'; +import { InsightEvalAuthorRuns } from '@studio/routes/optimizer/OptimizerInsightRoute/InsightEvalAuthorRuns'; +import { InsightExperimentGroups } from '@studio/routes/optimizer/OptimizerInsightRoute/InsightExperimentGroups'; +import { getOptimizerRoute } from '@studio/routes/utils'; +import { useQueryClient } from '@tanstack/react-query'; +import { type FC, useState } from 'react'; +import { Link, useParams } from 'react-router-dom'; + +export const OptimizerInsightRoute: FC = () => { + const workspace = useWorkspaceFromPath(); + const { insightId = '' } = useParams<{ insightId: string }>(); + const queryClient = useQueryClient(); + const toast = useToast(); + + const { + data: insight, + isLoading, + isError, + refetch, + } = useOptimizerGetInsight(workspace, insightId); + + const { mutate: updateInsight, isPending: isUpdating } = useOptimizerUpdateInsight({ + mutation: { + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: getOptimizerGetInsightQueryKey(workspace, insightId), + }); + queryClient.invalidateQueries({ + queryKey: getOptimizerListInsightsQueryKey(workspace), + }); + }, + onError: () => toast.error('Failed to update insight.'), + }, + }); + + const [openModalOpen, setOpenModalOpen] = useState(false); + + // The external agent changes the status after it creates the experiment. + const handleAction = (target: InsightStatus) => { + if (target === 'open') { + setOpenModalOpen(true); + return; + } + updateInsight({ workspace, insightId, data: { status: target } }); + }; + + useBreadcrumbs({ + items: [ + { href: getOptimizerRoute(workspace), slotLabel: 'Insights' }, + { slotLabel: insight?.title ?? insightId }, + ], + }); + + if (isLoading && !insight) { + return ; + } + + if (isError || !insight) { + return ( + + + + + + + + + } + /> + + + ); + } + + const traceRefs = insight.trace_refs ?? []; + + return ( + + + + {insight.title} + + + } + slotDescription={ + <> + Insight generated from observed sessions by the analyst agent.{' '} + + Learn more + + + } + slotActions={ + + {insightActions(insight.status).map((action) => ( + + ))} + + } + /> + +
+ + {insight.status} + + } + /> + + + + : '—'} + /> + + : '—'} + /> +
+ +
+ + + Description + {insight.description ? ( + + ) : ( + — + )} + + + + + handleAction('open')} + runExperimentDisabled={isUpdating} + /> + +
+ + + Eval Author runs + + + + + Observed Sessions ({traceRefs.length}) + + + + + setOpenModalOpen(false)} + /> + + ); +}; diff --git a/web/packages/studio/src/routes/optimizer/OptimizerRoute/index.test.tsx b/web/packages/studio/src/routes/optimizer/OptimizerRoute/index.test.tsx new file mode 100644 index 0000000000..565a0ef767 --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/OptimizerRoute/index.test.tsx @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { DEFAULT_WORKSPACE } from '@nemo/common/src/models/constants'; +import type { InsightListItem } from '@studio/api/optimizer'; +import { PLATFORM_BASE_URL } from '@studio/constants/environment'; +import { ROUTES } from '@studio/constants/routes'; +import { server } from '@studio/mocks/node'; +import { OptimizerRoute } from '@studio/routes/optimizer/OptimizerRoute'; +import { getOptimizerRoute } from '@studio/routes/utils'; +import { renderRoute, screen, within } from '@studio/tests/util/render'; +import { http, HttpResponse } from 'msw'; + +const INSIGHTS_URL = `${PLATFORM_BASE_URL}/apis/insights/v2/workspaces/:workspace/insights`; +const EXPERIMENT_GROUPS_URL = '*/apis/intake/v2/workspaces/:workspace/experiment-groups'; + +const makeInsight = (id: string, title: string): InsightListItem => ({ + id, + name: id, + title, + description: `${title} description`, + agent: 'research-agent', + status: 'open', + trace_refs: ['trace-1'], + experiment_group_count: null, + created_at: '2026-07-20T12:00:00Z', + updated_at: '2026-07-20T12:00:00Z', +}); + +const insightsPage = (data: InsightListItem[]) => ({ + data, + pagination: { + page: 1, + page_size: 50, + current_page_size: data.length, + total_pages: 1, + total_results: data.length, + }, +}); + +const renderList = () => + renderRoute(undefined, { + history: getOptimizerRoute(DEFAULT_WORKSPACE), + routes: [{ path: ROUTES.workspace.optimizer, element: }], + }); + +const findCell = async (insightTitle: string, columnName: string): Promise => { + const row = await screen.findByRole('row', { name: new RegExp(insightTitle) }); + + const headers = screen.getAllByRole('columnheader'); + const column = headers.findIndex((header) => header.textContent?.includes(columnName)); + if (column < 0) throw new Error(`${columnName} column not found`); + + return within(row).getAllByRole('cell')[column]; +}; + +describe('OptimizerRoute', () => { + it('renders server-provided list metadata without per-row requests', async () => { + const experimentGroupRequest = vi.fn(() => HttpResponse.json({})); + const insights = [ + { + ...makeInsight('positive', 'Positive count'), + experiment_group_count: 7, + last_seen_at: '2026-07-21T12:00:00Z', + }, + { ...makeInsight('zero', 'Zero count'), experiment_group_count: 0 }, + makeInsight('null', 'Null count'), + ]; + server.use( + http.get(INSIGHTS_URL, () => HttpResponse.json(insightsPage(insights))), + http.get(EXPERIMENT_GROUPS_URL, experimentGroupRequest) + ); + + renderList(); + + expect(await findCell('Positive count', 'Experiments')).toHaveTextContent('7'); + expect(await findCell('Zero count', 'Experiments')).toHaveTextContent('0'); + expect(await findCell('Null count', 'Experiments')).toHaveTextContent('—'); + expect((await findCell('Positive count', 'Last Seen')).querySelector('time')).toHaveAttribute( + 'datetime', + '2026-07-21T12:00:00Z' + ); + expect(await findCell('Zero count', 'Last Seen')).toHaveTextContent('—'); + expect(experimentGroupRequest).not.toHaveBeenCalled(); + }); +}); diff --git a/web/packages/studio/src/routes/optimizer/OptimizerRoute/index.tsx b/web/packages/studio/src/routes/optimizer/OptimizerRoute/index.tsx new file mode 100644 index 0000000000..7c50bcc2c2 --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/OptimizerRoute/index.tsx @@ -0,0 +1,175 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + ROW_ACTIONS_COLUMN_SIZE, + StudioDataView, +} from '@nemo/common/src/components/DataView/StudioDataView'; +import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; +import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; +import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; +import { Flex, PageHeader, Stack, Tag, Text } from '@nvidia/foundations-react-core'; +import { getErrorMessage } from '@studio/api/common/utils'; +import { type InsightListItem, useOptimizerListInsights } from '@studio/api/optimizer'; +import { AccessibleTitle } from '@studio/components/AccessibleTitle'; +import { ErrorPanel } from '@studio/components/ErrorPanel'; +import { FeatureFlagBadge } from '@studio/components/FeatureFlagBadge'; +import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; +import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; +import { insightStatusColor } from '@studio/routes/optimizer/insightStatus'; +import { getOptimizerInsightRoute, getOptimizerRoute } from '@studio/routes/utils'; +import { keepPreviousData } from '@tanstack/react-query'; +import { Lightbulb } from 'lucide-react'; +import { type ComponentProps, type FC } from 'react'; +import { useNavigate } from 'react-router-dom'; + +export const OptimizerRoute: FC = () => { + const workspace = useWorkspaceFromPath(); + + useBreadcrumbs({ + items: [{ href: getOptimizerRoute(workspace), slotLabel: 'Insights' }], + }); + + const navigate = useNavigate(); + + const dataViewState = useStudioDataViewState({ + defaultSort: [{ id: 'created_at', desc: true }], + }); + + const sortState = dataViewState.sorting.state[0]; + const sortParam = sortState ? `${sortState.desc ? '-' : ''}${sortState.id}` : '-created_at'; + + const { data, isFetching, error } = useOptimizerListInsights( + workspace, + { + page: dataViewState.pagination.state.pageIndex + 1, + page_size: dataViewState.pagination.state.pageSize, + sort: sortParam, + }, + { query: { placeholderData: keepPreviousData } } + ); + + const makeColumns: ComponentProps>['makeColumns'] = ( + { accessor }, + { rowActionsColumn } + ) => [ + accessor('status', { + header: 'Status', + enableSorting: false, + size: 110, + cell({ row }) { + const status = row.original.status; + return ( + + {status} + + ); + }, + }), + accessor('title', { + header: 'Insight', + enableSorting: false, + size: 240, + cell({ row }) { + return {row.original.title}; + }, + }), + accessor('agent', { + header: 'Agent', + enableSorting: false, + size: 160, + cell({ row }) { + return {row.original.agent || '—'}; + }, + }), + accessor('trace_refs', { + id: 'traces', + header: 'Traces', + enableSorting: false, + size: 80, + cell({ row }) { + return {row.original.trace_refs?.length ?? 0}; + }, + }), + accessor('experiment_group_count', { + header: 'Experiments', + enableSorting: false, + size: 110, + cell({ row }) { + return {row.original.experiment_group_count ?? '—'}; + }, + }), + accessor('created_at', { + header: 'Created', + enableSorting: true, + size: 140, + cell({ row }) { + return row.original.created_at ? ( + + ) : ( + — + ); + }, + }), + accessor('last_seen_at', { + header: 'Last Seen', + enableSorting: false, + size: 140, + cell({ row }) { + return row.original.last_seen_at ? ( + + ) : ( + — + ); + }, + }), + rowActionsColumn({ + size: ROW_ACTIONS_COLUMN_SIZE, + enableResizing: false, + rowActions: () => [], + }), + ]; + + return ( + + + + Insights + + + } + slotDescription="Leverage the optimizer agent to review your code and traces and suggest insights." + /> + navigate(getOptimizerInsightRoute(workspace, row.id))} + attributes={{ + DataViewRoot: { + data: data?.data ?? [], + totalCount: data?.pagination?.total_results, + requestStatus: error ? 'error' : isFetching ? 'loading' : undefined, + }, + DataViewTableContent: { + renderEmptyState: () => ( + } + header="No insights yet" + emptyMessage="Run an optimizer analysis on an agent to surface insights here." + /> + ), + renderErrorState: () => ( + + ), + }, + }} + /> + + + ); +}; diff --git a/web/packages/studio/src/routes/optimizer/insightStatus.test.ts b/web/packages/studio/src/routes/optimizer/insightStatus.test.ts new file mode 100644 index 0000000000..8d7e8867d7 --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/insightStatus.test.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { insightActions } from '@studio/routes/optimizer/insightStatus'; + +describe('insightStatus', () => { + it('only offers transitions accepted by the optimizer API', () => { + expect(insightActions('open')).toEqual([ + { label: 'Delete', target: 'deleted', kind: 'secondary' }, + { label: 'Resolve', target: 'resolved', kind: 'primary' }, + ]); + expect(insightActions('resolved')).toEqual([ + { label: 'Run experiment', target: 'open', kind: 'primary', color: 'brand' }, + ]); + expect(insightActions('deleted')).toEqual([ + { label: 'Run experiment', target: 'open', kind: 'primary', color: 'brand' }, + ]); + }); +}); diff --git a/web/packages/studio/src/routes/optimizer/insightStatus.ts b/web/packages/studio/src/routes/optimizer/insightStatus.ts new file mode 100644 index 0000000000..6bb7072272 --- /dev/null +++ b/web/packages/studio/src/routes/optimizer/insightStatus.ts @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { Button, Tag } from '@nvidia/foundations-react-core'; +import type { InsightStatus } from '@studio/api/optimizer'; +import type { ComponentProps } from 'react'; + +type TagColor = ComponentProps['color']; + +/** Tag color per insight status. Exhaustive over InsightStatus. */ +export const INSIGHT_STATUS_COLOR: Record = { + open: 'blue', + resolved: 'green', + deleted: 'gray', +}; + +export const insightStatusColor = (status: InsightStatus): TagColor => + INSIGHT_STATUS_COLOR[status] ?? 'gray'; + +/** A status-change action rendered as a button on the insight page. */ +export interface InsightAction { + label: string; + /** Status the button transitions the insight to. */ + target: InsightStatus; + kind: 'primary' | 'secondary'; + /** Optional button color; 'brand' renders the green primary CTA. */ + color?: ComponentProps['color']; +} + +// Canonical order: secondary actions first, Run experiment (green brand CTA) on the right. +const OPEN: InsightAction = { + label: 'Run experiment', + target: 'open', + kind: 'primary', + color: 'brand', +}; +const DELETE: InsightAction = { label: 'Delete', target: 'deleted', kind: 'secondary' }; +const RESOLVE: InsightAction = { label: 'Resolve', target: 'resolved', kind: 'primary' }; + +export const insightActions = (status: InsightStatus): InsightAction[] => { + return status === 'open' ? [DELETE, RESOLVE] : [OPEN]; +}; diff --git a/web/packages/studio/src/routes/utils.ts b/web/packages/studio/src/routes/utils.ts index a734049c64..fa56a5f309 100644 --- a/web/packages/studio/src/routes/utils.ts +++ b/web/packages/studio/src/routes/utils.ts @@ -21,6 +21,7 @@ import { JOBS_ENABLED, MEMBERS_ENABLED, MODEL_COMPARE_ENABLED, + OPTIMIZER_ENABLED, SAFE_SYNTHESIZER_ENABLED, SECRETS_ENABLED, SETTINGS_ENABLED, @@ -98,6 +99,9 @@ export const gateDeploymentsRoutes = (routes: RouteObject | RouteObject[]) => export const gateModelCompareRoutes = (routes: RouteObject | RouteObject[]) => gateRoutes(MODEL_COMPARE_ENABLED, routes); +export const gateOptimizerRoutes = (routes: RouteObject | RouteObject[]) => + gateRoutes(OPTIMIZER_ENABLED, routes); + type WorkspacePathParams = { workspace: string; }; @@ -367,6 +371,26 @@ export const getGuardrailsRoute = (workspace: string) => { return generatePath(ROUTES.workspace.guardrails, { workspace }); }; +export const getOptimizerRoute = (workspace: string) => { + return generatePath(ROUTES.workspace.optimizer, { workspace }); +}; + +export const getOptimizerInsightRoute = (workspace: string, insightId: string) => { + return generatePath(ROUTES.workspace.optimizerInsight, { workspace, insightId }); +}; + +export const getOptimizerEvalAuthorRunRoute = ( + workspace: string, + insightId: string, + evalAuthorRunId: string +) => { + return generatePath(ROUTES.workspace.optimizerEvalAuthorRun, { + workspace, + insightId, + evalAuthorRunId, + }); +}; + export const getGuardrailDetailRoute = (workspace: string, guardrailConfigName: string) => { return generatePath(ROUTES.workspace.guardrailDetail, { workspace, diff --git a/web/packages/studio/src/tests/title-change.test.tsx b/web/packages/studio/src/tests/title-change.test.tsx index 1b8f61709b..45e3f604a3 100644 --- a/web/packages/studio/src/tests/title-change.test.tsx +++ b/web/packages/studio/src/tests/title-change.test.tsx @@ -35,6 +35,8 @@ const pathParams = { [RP.benchmarkName]: 'test-benchmark', [RP.experimentGroupName]: 'test-experiment-group', [RP.evaluationName]: 'test-experiment', + [RP.insightId]: 'test-insight', + [RP.evalAuthorRunId]: 'test-eval-author-run', [RP.guardrailConfigName]: 'test-guardrail-config', };