From eda26440c1b337a94eada1cb4a3f3bea25d084c6 Mon Sep 17 00:00:00 2001 From: Octavian Drulea Date: Mon, 3 Aug 2026 20:08:20 -0700 Subject: [PATCH 1/4] feat(studio): email-security-analyst is lone option in modal; agents list cleanup Signed-off-by: Octavian Drulea --- web/packages/studio/AGENTS.md | 41 ++ .../dataViews/AgentsDataView/index.test.tsx | 12 +- .../dataViews/AgentsDataView/index.tsx | 44 +- .../ComparisonDeltaCell.tsx | 2 +- .../EvalComparisonTable.tsx | 2 +- .../studio/src/constants/sampleAgents.test.ts | 83 ---- .../studio/src/constants/sampleAgents.ts | 57 +-- web/packages/studio/src/mocks/handlers.ts | 2 - .../studio/src/mocks/handlers/sampleAgents.ts | 77 --- .../studio/src/routes/agents/AGENTS.md | 466 ++++++++++++++++++ .../CreateExampleAgentModal/index.tsx | 65 +-- .../agents/AgentsListRoute/index.test.tsx | 354 ------------- .../src/util/buildSuggestedModelOptions.ts | 10 + web/packages/studio/src/util/sampleAgents.ts | 23 + 14 files changed, 612 insertions(+), 626 deletions(-) delete mode 100644 web/packages/studio/src/constants/sampleAgents.test.ts delete mode 100644 web/packages/studio/src/mocks/handlers/sampleAgents.ts create mode 100644 web/packages/studio/src/routes/agents/AGENTS.md delete mode 100644 web/packages/studio/src/routes/agents/AgentsListRoute/index.test.tsx create mode 100644 web/packages/studio/src/util/sampleAgents.ts diff --git a/web/packages/studio/AGENTS.md b/web/packages/studio/AGENTS.md index e90d88e242..f9a6bd41bf 100644 --- a/web/packages/studio/AGENTS.md +++ b/web/packages/studio/AGENTS.md @@ -54,3 +54,44 @@ Wrong — transparent dropdown: Exception: call sites that fill SelectContent with custom children containing their own background (e.g. a sticky Block with bg-surface) are fine as-is. + +### KUI form gotchas — the silent-failure set + +Each compiles, lints and typechecks clean, then renders nothing wrong — the element is +simply inert or invisible. + +**`slotError` needs `status="error"` to render.** Without a status `FormField` shows +`slotHelp` instead and drops the message. `Controlled*` wrappers set status from +react-hook-form only, so any other error source must set it too (`formFieldProps` spreads +last, so it wins): + +```tsx +formFieldProps={{ slotError: fieldError, status: fieldError ? 'error' : undefined }} +``` + +**`FormModal.disabled` blocks closing.** It means "busy": it kills Cancel and stops +`handleUserClose`, trapping the user. For validation use `submitDisabled`. + +```tsx +disabled={isPending} // busy → intentionally locked +submitDisabled={!isValid} // invalid → submit blocked, dismiss still works +``` + +**`Text` has no `color="danger"`, and `text-danger` is not an emitted utility.** `danger` +exists only on `Button`; the token is `--text-color-feedback-danger`. Devtools showing +**"Inherited from"** means no rule matched at all, not that yours lost a specificity fight. + +```tsx +… +``` + +**Modals seeded from a prop must re-seed on `open`.** `useForm({ defaultValues })` reads +once at mount; a persistently-rendered modal mounts before the prop exists and keeps the +empty default forever. Callers passing a constant work by accident, so the bug arrives +with the second caller. + +```tsx +useEffect(() => { + resetForm(makeDefaultValues(agentProp)); // NOT `if (!open)` — that only resets on close +}, [open, agentProp, resetForm]); +``` diff --git a/web/packages/studio/src/components/dataViews/AgentsDataView/index.test.tsx b/web/packages/studio/src/components/dataViews/AgentsDataView/index.test.tsx index 9c586bb7aa..ba52a80785 100644 --- a/web/packages/studio/src/components/dataViews/AgentsDataView/index.test.tsx +++ b/web/packages/studio/src/components/dataViews/AgentsDataView/index.test.tsx @@ -139,7 +139,7 @@ describe('CombinedAgentsTable', () => { }); describe('row actions', () => { - it('shows Deploy, Test models, Clone, and Delete actions for agent rows', async () => { + it('shows Deploy, Compare Models, Clone, and Delete actions for agent rows', async () => { const user = userEvent.setup(); renderTable(); @@ -149,7 +149,7 @@ describe('CombinedAgentsTable', () => { await user.click(menuButtons[0]); const deployItems = await screen.findAllByRole('menuitem', { name: 'Deploy' }); - const testModelItems = screen.getAllByRole('menuitem', { name: 'Test models' }); + const testModelItems = screen.getAllByRole('menuitem', { name: 'Compare Models' }); const cloneItems = screen.getAllByRole('menuitem', { name: 'Clone' }); const deleteItems = screen.getAllByRole('menuitem', { name: 'Delete' }); expect(deployItems.length).toBeGreaterThan(0); @@ -158,7 +158,7 @@ describe('CombinedAgentsTable', () => { expect(deleteItems.length).toBeGreaterThan(0); }); - it('opens Playground with the row model selected when Test models is selected', async () => { + it('opens Playground with the row model selected when Compare Models is selected', async () => { const user = userEvent.setup(); renderRoute(undefined, { history: getAgentsListRoute(WORKSPACE), @@ -177,7 +177,7 @@ describe('CombinedAgentsTable', () => { await screen.findByText(MOCK_AGENTS[0].name); await user.click(screen.getAllByRole('button', { name: /actions/i })[0]); - await user.click((await screen.findAllByRole('menuitem', { name: 'Test models' }))[0]); + await user.click((await screen.findAllByRole('menuitem', { name: 'Compare Models' }))[0]); expect(await screen.findByTestId('model-compare-location')).toHaveTextContent( `${getModelCompareRoute(WORKSPACE)}?model=${encodeURIComponent( @@ -186,7 +186,7 @@ describe('CombinedAgentsTable', () => { ); }); - it('hides Test models when Playground is disabled', async () => { + it('hides Compare Models when Playground is disabled', async () => { const user = userEvent.setup(); renderRoute(undefined, { history: getAgentsListRoute(WORKSPACE), @@ -205,7 +205,7 @@ describe('CombinedAgentsTable', () => { expect((await screen.findAllByRole('menuitem', { name: 'Deploy' })).length).toBeGreaterThan( 0 ); - expect(screen.queryByRole('menuitem', { name: 'Test models' })).not.toBeInTheDocument(); + expect(screen.queryByRole('menuitem', { name: 'Compare Models' })).not.toBeInTheDocument(); }); it('calls onCloneAgent with the row when Clone is selected', async () => { diff --git a/web/packages/studio/src/components/dataViews/AgentsDataView/index.tsx b/web/packages/studio/src/components/dataViews/AgentsDataView/index.tsx index f80b72de4d..c0c1723064 100644 --- a/web/packages/studio/src/components/dataViews/AgentsDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/AgentsDataView/index.tsx @@ -25,7 +25,7 @@ import { } from '@nemo/sdk/generated/agents/api'; import type { Agent } from '@nemo/sdk/generated/agents/schema/Agent'; import type { AgentDeployment } from '@nemo/sdk/generated/agents/schema/AgentDeployment'; -import { Button, Divider, Flex, Text } from '@nvidia/foundations-react-core'; +import { Button, Text } from '@nvidia/foundations-react-core'; import { getAgentModelNames } from '@studio/components/dataViews/AgentsDataView/utils'; import { DeleteConfirmationModal } from '@studio/components/DeleteConfirmationModal'; import { DocumentationButton } from '@studio/components/DocumentationButton'; @@ -34,7 +34,7 @@ import { LINK_DOCS_STUDIO } from '@studio/constants/links'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { getModelCompareRoute } from '@studio/routes/utils'; import { keepPreviousData, useQueryClient } from '@tanstack/react-query'; -import { HatGlasses, Trash, X } from 'lucide-react'; +import { HatGlasses, Trash } from 'lucide-react'; import { ComponentProps, FC, useEffect, useMemo, useState } from 'react'; import { useNavigate } from 'react-router'; @@ -177,12 +177,6 @@ export const AgentsTable: FC = ({ }); }, [agentsData, deploymentsData]); - const rowSelection = dataViewState.rowSelection.state; - const selectedAgents = useMemo( - () => tableData.filter((row) => rowSelection[row.id]), - [tableData, rowSelection] - ); - const deleteAgentMutation = useAgentsDeleteAgent(); const deleteDeploymentMutation = useAgentsDeleteDeployment(); @@ -291,7 +285,7 @@ export const AgentsTable: FC = ({ ...(canTestModels ? [ { - children: 'Test models', + children: 'Compare Models', onSelect: () => { const target = getModelCompareRoute(workspace); const model = row.models[0]; @@ -321,32 +315,18 @@ export const AgentsTable: FC = ({ return ( <> - {selectedAgents.length > 0 && ( - - - {selectedAgents.length} {selectedAgents.length === 1 ? 'row' : 'rows'} selected - - - - - - - - )} ( + + )} onRowClick={(row: AgentTableRow) => { onAgentRowClick?.(row); }} diff --git a/web/packages/studio/src/components/dataViews/EvalComparisonTable/ComparisonDeltaCell.tsx b/web/packages/studio/src/components/dataViews/EvalComparisonTable/ComparisonDeltaCell.tsx index a4840cbe3e..b52c5dcf0f 100644 --- a/web/packages/studio/src/components/dataViews/EvalComparisonTable/ComparisonDeltaCell.tsx +++ b/web/packages/studio/src/components/dataViews/EvalComparisonTable/ComparisonDeltaCell.tsx @@ -3,7 +3,7 @@ import { Badge, Flex, Text } from '@nvidia/foundations-react-core'; import type { ComparisonMetricDelta } from '@studio/components/dataViews/EvalComparisonTable/types'; -import { formatScore } from '@studio/routes/agents/AgentEvaluationsRoute/evalScores'; +import { formatScore } from '@studio/components/evaluation/utils'; import { Equal, Minus, Plus } from 'lucide-react'; import type { FC } from 'react'; diff --git a/web/packages/studio/src/components/dataViews/EvalComparisonTable/EvalComparisonTable.tsx b/web/packages/studio/src/components/dataViews/EvalComparisonTable/EvalComparisonTable.tsx index eff9031560..b9a6eec24f 100644 --- a/web/packages/studio/src/components/dataViews/EvalComparisonTable/EvalComparisonTable.tsx +++ b/web/packages/studio/src/components/dataViews/EvalComparisonTable/EvalComparisonTable.tsx @@ -17,7 +17,7 @@ import { metricNamesForComparisons, scoreForMetric, } from '@studio/components/dataViews/EvalComparisonTable/utils'; -import { formatScore } from '@studio/routes/agents/AgentEvaluationsRoute/evalScores'; +import { formatScore } from '@studio/components/evaluation/utils'; import { useMemo, type ComponentProps, type FC } from 'react'; const METRIC_COLUMN_ID = 'metric'; diff --git a/web/packages/studio/src/constants/sampleAgents.test.ts b/web/packages/studio/src/constants/sampleAgents.test.ts deleted file mode 100644 index 27c410f921..0000000000 --- a/web/packages/studio/src/constants/sampleAgents.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { - EVALUATION_SAMPLE_AGENTS, - evaluationSampleAgentKeyForAgentName, - isSampleAgentName, - SAMPLE_AGENTS, - sampleAgentKeyForAgentName, -} from '@studio/constants/sampleAgents'; - -describe('sampleAgentKeyForAgentName', () => { - it('matches a generated example agent name to its key', () => { - expect(sampleAgentKeyForAgentName('email-phishing-demo-agent-9lhh53')).toBe( - 'email_phishing_analyzer' - ); - expect(sampleAgentKeyForAgentName('calculator-demo-agent-abc123')).toBe('calculator'); - }); - - it('returns undefined for non-example agents and empty input', () => { - expect(sampleAgentKeyForAgentName('my-custom-agent')).toBeUndefined(); - expect(sampleAgentKeyForAgentName(undefined)).toBeUndefined(); - expect(sampleAgentKeyForAgentName('')).toBeUndefined(); - }); - - it('requires the prefix separator (no partial-token match)', () => { - // 'calculator-demo-agentx-...' is not a real 'calculator-demo-agent-' name. - expect(sampleAgentKeyForAgentName('calculator-demo-agentxyz')).toBeUndefined(); - }); - - it('picks the longest matching prefix when one is a substring of another', () => { - const registry = [ - { namePrefix: 'test', key: 'short' }, - { namePrefix: 'test-agent', key: 'long' }, - ]; - const match = (name: string) => - registry - .filter((a) => name.startsWith(`${a.namePrefix}-`)) - .sort((a, b) => b.namePrefix.length - a.namePrefix.length)[0]?.key; - expect(match('test-agent-abc123')).toBe('long'); - expect(match('test-abc123')).toBe('short'); - }); - - it('every registry prefix resolves to its own key', () => { - for (const agent of SAMPLE_AGENTS) { - expect(sampleAgentKeyForAgentName(`${agent.namePrefix}-zzzz99`)).toBe(agent.key); - } - }); -}); - -describe('isSampleAgentName', () => { - it('agrees with sampleAgentKeyForAgentName (same boundary rule)', () => { - const names = [ - 'email-phishing-demo-agent-9lhh53', - 'calculator-demo-agent-abc123', - 'calculator-demo-agentxyz', // partial token — no separator - 'my-custom-agent', - '', - ]; - for (const name of names) { - expect(isSampleAgentName(name)).toBe(sampleAgentKeyForAgentName(name) !== undefined); - } - }); - - it('requires the prefix separator', () => { - expect(isSampleAgentName('calculator-demo-agent-abc123')).toBe(true); - expect(isSampleAgentName('calculator-demo-agentxyz')).toBe(false); - }); -}); - -describe('evaluation samples', () => { - it('keeps creation-only samples out of the evaluation picker', () => { - expect(SAMPLE_AGENTS.some((agent) => agent.key === 'calculator')).toBe(true); - expect(EVALUATION_SAMPLE_AGENTS.map((agent) => agent.key)).toEqual([ - 'calculator', - 'email_phishing_analyzer', - ]); - expect(evaluationSampleAgentKeyForAgentName('calculator-demo-agent-abc123')).toBe('calculator'); - expect(evaluationSampleAgentKeyForAgentName('email-phishing-demo-agent-abc123')).toBe( - 'email_phishing_analyzer' - ); - }); -}); diff --git a/web/packages/studio/src/constants/sampleAgents.ts b/web/packages/studio/src/constants/sampleAgents.ts index e898afbad4..053262accd 100644 --- a/web/packages/studio/src/constants/sampleAgents.ts +++ b/web/packages/studio/src/constants/sampleAgents.ts @@ -5,27 +5,27 @@ import { z } from 'zod'; // Registry of canned example agents. Each entry references curated static assets // under public/sample-agents// by path (fetched on demand, never bundled) — -// mirroring src/constants/sampleDatasets.ts. Used by both the Create Example Agent -// modal (fetch + parse agent.yml, inject model, POST). Samples with an -// evalConfigPath also appear in the Run Evaluation modal. +// mirroring src/constants/sampleDatasets.ts. Used by the Create Example Agent +// modal (fetch + parse agent.yml, inject model, POST). +// +// Eval configs are a SEPARATE registry (EVAL_CONFIG_SAMPLES) on purpose: either +// paradigm can target any agent, so a config is not owned by an agent. // // INVARIANT: an entry whose agent.yml uses a custom NAT `_type` requires that // tool's Python package to be installed in the deploy venv, or the deployment // fails at startup. Current mappings: // _type: calculator -> plugins/nemo-agents/examples/calculator-agent // _type: email_phishing_analyzer -> plugins/nemo-agents/examples/email-phishing-analyzer +// _type: analyze_email -> plugins/nemo-agents/examples/email-security-analyst +// _type: extract_iocs -> plugins/nemo-agents/examples/email-security-analyst export interface SampleAgent { - /** Stable key; also the dropdown value and label. */ key: string; - label: string; + displayName: string; description: string; /** Prefix for generated agent names; drives onboarding detection. */ namePrefix: string; /** Public path to the NAT workflow config (parsed + model-injected at create). */ agentConfigPath: string; - /** Public path to a reusable nemo-evaluator eval-config.json. Samples without - * one remain available for agent creation but not evaluation seeding. */ - evalConfigPath?: string; /** Config format identifier sent to the create API. Defaults to * `nat-workflow-v1` server-side when omitted; set to `nemo-agents-spec-v1` * for Fabric-backed samples so the API validates them as Fabric, not NAT. */ @@ -34,25 +34,15 @@ export interface SampleAgent { export const SAMPLE_AGENTS: SampleAgent[] = [ { - key: 'calculator', - label: 'calculator', - description: 'A ReAct agent with a calculator and datetime tool.', - namePrefix: 'calculator-demo-agent', - agentConfigPath: 'sample-agents/calculator/agent.yml', - evalConfigPath: 'sample-agents/calculator/eval-config.json', - }, - { - key: 'email_phishing_analyzer', - label: 'email_phishing_analyzer', - description: 'A ReAct agent that inspects an email body for phishing signals.', - namePrefix: 'email-phishing-demo-agent', - agentConfigPath: 'sample-agents/email-phishing-analyzer/agent.yml', - evalConfigPath: 'sample-agents/email-phishing-analyzer/eval-config.json', + key: 'email_security_analyst', + displayName: 'Email Security Analyst', + description: + 'An analyst-facing email security assistant: select one or more messages, optionally ask a question, and it routes to the capability that answers it.', + namePrefix: 'email-security-analyst', + agentConfigPath: 'sample-agents/email-security-analyst/agent.yml', }, ]; -// Eval configs are a SEPARATE registry (EVAL_CONFIG_SAMPLES) on purpose: either -// paradigm can target any agent, so a config is not owned by an agent. export interface EvalConfigSample { key: string; displayName: string; @@ -92,27 +82,11 @@ export const DEFAULT_EVAL_CONFIG_KEY = EVAL_CONFIG_SAMPLES[0].key; export const getEvalConfigSample = (key: string): EvalConfigSample => EVAL_CONFIG_SAMPLES.find((sample) => sample.key === key) ?? EVAL_CONFIG_SAMPLES[0]; -export type EvaluationSampleAgent = SampleAgent & { evalConfigPath: string }; - -export const EVALUATION_SAMPLE_AGENTS = SAMPLE_AGENTS.filter( - (agent): agent is EvaluationSampleAgent => typeof agent.evalConfigPath === 'string' -); - export const DEFAULT_SAMPLE_AGENT_KEY = SAMPLE_AGENTS[0].key; export const getSampleAgent = (key: string): SampleAgent => SAMPLE_AGENTS.find((agent) => agent.key === key) ?? SAMPLE_AGENTS[0]; -export const getEvaluationSampleAgent = (key: string): EvaluationSampleAgent => - EVALUATION_SAMPLE_AGENTS.find((agent) => agent.key === key) ?? EVALUATION_SAMPLE_AGENTS[0]; - -export const evaluationSampleAgentKeyForAgentName = ( - name: string | undefined -): string | undefined => { - const key = sampleAgentKeyForAgentName(name); - return EVALUATION_SAMPLE_AGENTS.some((agent) => agent.key === key) ? key : undefined; -}; - export const buildSampleAgentName = (namePrefix: string): string => `${namePrefix}-${Math.random().toString(36).slice(2, 8)}`; @@ -122,8 +96,7 @@ export const isSampleAgentName = (name: string): boolean => /** * Infer which sample-agent example a deployed agent came from by matching its * generated name (`${namePrefix}-`). Returns the example key, or - * undefined for agents not created from an example. Used to auto-select the - * matching eval config. + * undefined for agents not created from an example. * * Robustness: requires the `${namePrefix}-` separator (so a prefix only matches * a real name boundary, not a partial token) and picks the LONGEST matching diff --git a/web/packages/studio/src/mocks/handlers.ts b/web/packages/studio/src/mocks/handlers.ts index 9d3833565d..22c4d7d776 100644 --- a/web/packages/studio/src/mocks/handlers.ts +++ b/web/packages/studio/src/mocks/handlers.ts @@ -17,7 +17,6 @@ import { evaluatorHandlers } from '@studio/mocks/handlers/evaluator'; import { filesetsHandlers } from '@studio/mocks/handlers/filesets'; import { guardrailsHandlers } from '@studio/mocks/handlers/guardrails'; import { modelsHandlers } from '@studio/mocks/handlers/models'; -import { sampleAgentsHandlers } from '@studio/mocks/handlers/sampleAgents'; import { sampleDatasetsHandlers } from '@studio/mocks/handlers/sampleDatasets'; import { secretsHandlers } from '@studio/mocks/handlers/secrets'; import { workspacesHandlers } from '@studio/mocks/handlers/workspaces'; @@ -73,7 +72,6 @@ export interface HypermodelParams { * but tests can override these with `server.use`. */ export const handlers = [ - ...sampleAgentsHandlers, ...sampleDatasetsHandlers, // Evaluator V2 — fixtures loaded on first use to keep initial handler graph smaller diff --git a/web/packages/studio/src/mocks/handlers/sampleAgents.ts b/web/packages/studio/src/mocks/handlers/sampleAgents.ts deleted file mode 100644 index a4b06f2368..0000000000 --- a/web/packages/studio/src/mocks/handlers/sampleAgents.ts +++ /dev/null @@ -1,77 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { http, HttpResponse } from 'msw'; - -// Realistic fixtures for the public/sample-agents/* static assets. The create -// flow parses the returned agent.yml, so these must be valid NAT config YAML — -// not a '{}' stub. -const PHISHING_AGENT_YAML = `functions: - email_phishing_analyzer: - _type: email_phishing_analyzer - llm: llm -llms: - llm: - _type: openai - api_key: not-used - model_name: \${NEMO_DEFAULT_MODEL} - temperature: 0.0 -workflow: - _type: tool_calling_agent - tool_names: [email_phishing_analyzer] - llm_name: llm -`; - -const CALCULATOR_AGENT_YAML = `function_groups: - calculator: - _type: calculator -functions: - current_datetime: - _type: current_datetime -llms: - llm: - _type: openai - api_key: not-used - model_name: \${NEMO_DEFAULT_MODEL} - temperature: 0.0 -workflow: - _type: react_agent - tool_names: [calculator, current_datetime] - llm_name: llm - use_native_tool_calling: true -`; - -const EVAL_YAML = `llms: - judge_llm: - _type: openai - model_name: nvidia-nemotron-3-super-120b-a12b -eval: - general: - dataset: - _type: csv - file_path: smaller_test.csv - evaluators: - accuracy: - _type: tunable_rag_evaluator - llm_name: judge_llm -`; - -/** Handlers for sample-agent static asset requests (paths relative to BASE_URL). */ -export const sampleAgentsHandlers = [ - http.get(/\/sample-agents\/.+/, ({ request }) => { - const path = new URL(request.url).pathname; - if (path.endsWith('/agent.yml')) { - const body = path.includes('/calculator/') ? CALCULATOR_AGENT_YAML : PHISHING_AGENT_YAML; - return HttpResponse.text(body, { headers: { 'Content-Type': 'application/yaml' } }); - } - if (path.endsWith('.yml')) { - return HttpResponse.text(EVAL_YAML, { headers: { 'Content-Type': 'application/yaml' } }); - } - if (path.endsWith('.csv')) { - return HttpResponse.text('subject,body,label\nHi,benign body,benign\n', { - headers: { 'Content-Type': 'text/csv' }, - }); - } - return HttpResponse.text('[]', { headers: { 'Content-Type': 'application/json' } }); - }), -]; diff --git a/web/packages/studio/src/routes/agents/AGENTS.md b/web/packages/studio/src/routes/agents/AGENTS.md new file mode 100644 index 0000000000..8967d2649a --- /dev/null +++ b/web/packages/studio/src/routes/agents/AGENTS.md @@ -0,0 +1,466 @@ +# Agent Routes + +Everything under `src/routes/agents/` — the agents list, detail, deployments, monitor, +suggestions, and evaluations routes — plus the sample-agent registry those routes read from. + +Contents: + +- [Adding a New Example Agent](#adding-a-new-example-agent) — end-to-end checklist +- [Agent Evaluation](#agent-evaluation) — how Studio runs evaluations + +--- + +# Adding a New Example Agent + +Example agent = entry in `src/constants/sampleAgents.ts`. Create-Example modal deploys it; Run-Evaluation modal seeds a config from it. + +Touches: Python package, 2 ymls, 1 constants file, 1 Studio rebuild. + +## Checklist + +1. Plugin package `plugins/nemo-agents/examples//` — mirror `email-phishing-analyzer/`. `pyproject.toml` with `[project.entry-points."nat.components"] = ".register"`; `src//register.py` = one `FunctionBaseConfig` + `@register_function` per tool; `tests/`. +2. Register in 4 places (below). +3. `uv sync`, then **restart services**. `nat start fastapi` discovers `_type`s only at process start. +4. Studio asset `public/sample-agents//agent.yml` — second independent copy, hand-synced. +5. `SAMPLE_AGENTS` entry. `evalConfigPath` optional; omit → absent from Run Evaluation. +6. `pnpm --filter nemo-studio-ui build:fastapi`. +7. Verify (below). + +## Registration: 4 places + +`{ workspace = true }` is a pointer into root `members`, not standalone. Missing root member → `references a workspace in tool.uv.sources, but is not a workspace member`. + +| File | Section | Entry | +| ------------------------------------ | ----------------------------- | ------------------------------------------------ | +| root `pyproject.toml` | `[tool.uv.sources]` | `nemo-agents-example- = { workspace = true }` | +| root `pyproject.toml` | `[tool.uv.workspace] members` | `"plugins/nemo-agents/examples/"` | +| `plugins/nemo-agents/pyproject.toml` | `[project] dependencies` | `"nemo-agents-example-"` | +| `plugins/nemo-agents/pyproject.toml` | `[tool.uv.sources]` | `nemo-agents-example- = { workspace = true }` | + +Tried, rejected: + +- members glob `examples/*` — only 3 of 11 subdirs are packages; uv errors on first without `pyproject.toml`. +- path source `{ path = "...", editable = true }` in plugin only — resolves without root membership, still rewrites `uv.lock`. +- `uv pip install -e `, no toml entries — zero repo diff, but `uv sync` **prunes it**; `_type`s vanish, deploy fails at startup. + +Wheel bundle (`packages/nemo_platform/pyproject.toml` `[tool.bundle-package.*]`) lists only `calculator`. Others resolve under workspace `uv sync`, **not** from a wheel. + +**Re-lock needs `uv lock --python 3.12`.** `nooa` (git dep of `nemo-experimentalist-plugin`) requires py `>=3.12,<3.14`; uv builds git-dep metadata with the active interpreter (venv is 3.11). Committed lock caches that metadata, so plain `uv sync` works and this only bites on a real re-resolve. Resulting diff is large but purely additive (0 deletions). + +## Two agent ymls, on purpose + +| Copy | Workflow | Consumer | +| --------------------------------------- | -------------------- | --------- | +| `plugins/.../-agent.yml` | `react_agent` | CLI / NAT | +| `public/sample-agents//agent.yml` | `tool_calling_agent` | Studio | + +Independent files, hand-synced. Studio serves only `public/`; it cannot read `plugins/`. + +Studio copy constraints: + +- LLM key must be literally `llm` — `loadSampleAgentConfig.ts` hardcodes `config.llms.llm`, throws otherwise. +- `model_name: ${NEMO_DEFAULT_MODEL}` inert — Studio overwrites with dropdown pick before POST. +- Prefer `tool_calling_agent`; ReAct text prompt makes small models mis-parse a scratchpad and run away. + +## max_tokens + +Phishing's `1024` truncates any agent emitting a JSON report or batch triage. Reasoning tokens exhaust the budget: + +``` +LLM output truncated (finish_reason='length'). output_tokens=1024, ... +Truncated output: 'We need to list...' +``` + +`4096` sufficed for ~6-item batch + 5-field JSON report. Model-size independent — same truncation on 30B and 120B. + +Capping reasoning tokens instead is unavailable: NAT's knob is `thinking: bool`, gated in `nat/data_models/thinking_mixin.py` by regex `^nvidia/(llama|nvidia).*nemotron`. Platform names are workspace-qualified (`default/...`) → never match → deploy fails `thinking is not supported for model_name`. (`max_thinking_tokens` in phishing's eval config is on the llm-judge metric, different layer.) + +## Rebuild + +`public/` copies into `dist/` at build; platform serves **built** `dist/`. Stale `dist/` → new asset 404s at runtime though the file exists and tests pass. + +```bash +pnpm --filter nemo-studio-ui build:fastapi +curl -o /dev/null -w '%{http_code}\n' http://127.0.0.1:8080/studio/sample-agents//agent.yml +``` + +`404` — including for existing samples — means `dist/` predates `public/sample-agents/`. + +## Verify + +Green tests are insufficient; they mock the asset fetch. + +```bash +# 1. _types registered +uv run python -c " +import .register +from nat.cli.type_registry import GlobalTypeRegistry +r = GlobalTypeRegistry.get() +print(sorted(c.local_name for c in r.get_registered_functions() if c.module_name==''))" + +# 2. asset served, not just on disk +curl -sf http://127.0.0.1:8080/studio/sample-agents//agent.yml >/dev/null && echo served + +# 3. real Create-Example path: fetch served yml -> inject model -> +# POST /apis/agents/v2/workspaces/default/agents -> deploy -> invoke /-/generate +``` + +**Read `nemo agents logs --agent ` first.** Workflow error messages mislead: `ReActAgentParsingFailedError ... LLM output: ''` and `No response received from agent` were the same root cause, visible only in logs as `502 Bad Gateway - Backend returned 404: {"error":{"message":"Model not found"}}`. + +Confirm the model is served before blaming the agent. Names differing by transposition (`nemotron-nano-3-30b` vs `nemotron-3-nano-30b`) can both exist as entities with only one reachable: + +```bash +curl -s -o /dev/null -w '%{http_code}\n' -X POST \ + http://127.0.0.1:8080/apis/inference-gateway/v2/workspaces/default/openai/-/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"model":"","messages":[{"role":"user","content":"say ok"}],"max_tokens":16}' +``` + +## Naming + +`namePrefix` drives `sampleAgentKeyForAgentName` (matches `${namePrefix}-`, longest wins). Changing an existing `namePrefix` is breaking: deployed agents stop matching, silently fall back to `EVALUATION_SAMPLE_AGENTS[0]` — wrong eval config, no error. + +--- + +# Agent Evaluation + +How Studio runs agent evaluations. + +## Overview + +- **Runner:** nemo-evaluator's **`agent-evaluate/jobs`** endpoint (task-based, `AgentEvaluateJob`), + with an **agent** target. This is the canonical path for Studio agent evaluation, per the + nemo-evaluator team — it is purpose-built for agent tasks with extensive artifact + trace + collection. (A legacy row-based `evaluate/jobs` endpoint also exists — see the bottom.) +- **Sample agents:** `public/sample-agents//` is the seed repository (agent config + + an `eval-config.json`). Studio reads these for the "use example" flow. +- **Metrics:** Studio creates metrics **only** as `InlineMetricPayload` — a built-in metric + serialized to JSON, reconstructed at runtime. No `CloudPickleMetricPayload` (no Python, no + pickled code). + +## Tasks, metrics, scores + +Three levels, don't conflate them: + +- **task** — one input case = one agent call (the "unit of work being evaluated"). The sample + has 5 tasks (5 emails). +- **metric** — a scorer applied to a task's output; a task can carry more than one. +- **score** — a value a metric emits; one metric can emit many (a judge can rate several + dimensions from one call). + +The evaluation loop is `task × trial × metric → score`. + +## Reusable eval configs (Filesets) + +nemo-evaluator has no "eval config" entity, so a reusable config is stored as an +**`eval-config.json`** file in a named **Fileset**. The tasks are stored **inline** in the +config (the `agent-evaluate` spec takes inline `tasks[]`; a Task/Taskset _reference_ is planned +but not yet implemented, so large datasets are inlined for now). + +Two shapes are involved: + +- **Example template** (`public/sample-agents/*/eval-config.json`): a shared-metric layout — + `tasks[]` (no per-task metrics) + one top-level `metric` (no judge_model). Read only when + creating a config from an example; never submitted directly. +- **Persisted spec** (what a Fileset actually stores = the yardstick): an `AgentEvalInputSpec` + **minus `target`**. Every task carries its own `metrics[]` with the **judge model baked in**; + `max_concurrent_tasks` is baked. No agent target (that is per-run). + +Persisted `eval-config.json` shape (the yardstick): + +```json +{ + "tasks": [ + { + "id": "Claim Your Free iPhone Now!", + "intent": "Classify this email as phishing or benign", + "inputs": { "instruction": "" }, + "reference": { "label": "phishing" }, + "metrics": [ + { + /* inline metric bundle — see Payloads; judge model baked into payload.metric.model */ + } + ] + } + ], + "max_concurrent_tasks": 1 +} +``` + +**Create ("Use Example"):** Studio reads the example template, **fans the shared `metric` onto +each task** and **injects the chosen judge model** (`buildPersistedSpec`), then writes that +persisted spec into a new Fileset. The judge is now part of the stored yardstick. + +**Reuse ("Choose Fileset"):** the user selects a Fileset **by name**. Studio reads its +`eval-config.json` **as-is** (`parsePersistedSpec`) — no re-fan, no judge re-pick — so the +yardstick (tasks + metric + judge) is identical across every run. Only the agent **target** is +injected at submit; the whole is wrapped as `{ spec }` for the job request. The saved spec is +also a valid `nemo evaluator agent-evaluate submit --spec-file` input once a `target` is added. + +Injected by Studio **at submit** (never stored in the config): the agent **target** only. The +judge model and `max_concurrent_tasks` are part of the persisted yardstick. + +**`max_concurrent_tasks` defaults to `1` (serial).** This is a conservative default for local NAT +deployments. NAT currently maps workflow failures, including output truncation, to `422`; a single +failure aborts the job. Concurrency was ruled out as the root cause, but serial execution avoids +adding load while this upstream error classification remains. To tolerate failures instead, set +`target.params.ignore_request_failure: true` (failed trials score `NaN`). See Gotchas. + +## Endpoints + +| Purpose | Method + path (`/apis/evaluator/v2/workspaces/{ws}`) | +| ----------- | ---------------------------------------------------- | +| List jobs | `GET .../agent-evaluate/jobs` | +| Submit | `POST .../agent-evaluate/jobs` | +| Get job | `GET .../agent-evaluate/jobs/{name}` | +| Poll status | `GET .../agent-evaluate/jobs/{name}/status` | +| Cancel | `POST .../agent-evaluate/jobs/{name}/cancel` | +| Logs | `GET .../agent-evaluate/jobs/{name}/logs` | +| Results | `GET .../agent-eval-results/{name}` | + +Fileset create/upload (Files service), for seeding a config into a new Fileset: + +| Purpose | Method + path | +| -------------- | ------------------------------------------------------------- | +| Create fileset | `POST /apis/files/v2/workspaces/{ws}/filesets` | +| Upload file | `PUT /apis/files/v2/workspaces/{ws}/filesets/{name}/-/{path}` | + +Submit body is wrapped: `{"spec": { ...AgentEvalInputSpec }}`. + +## Payloads + +### Job spec + +```json +{ + "spec": { + "tasks": [ + /* each carries metrics[] from the persisted spec (judge already baked) */ + ], + "target": { + "kind": "agent", + "agent": { + /* see below */ + } + }, + "max_concurrent_tasks": 1 + } +} +``` + +### Target (generic agent) + +```json +{ + "kind": "agent", + "agent": { + "format": "generic", + "url": ".../agents//-/generate", + "name": "", + "body": { "input_message": "{{ instruction }}" }, + "response_path": "$.value", + "stream": false + } +} +``` + +Use the non-streaming `/generate` endpoint. Do **not** use `/generate/full` — its per-token +SSE stream leaves only the last token in the captured output and every score collapses to 0. + +**`body` renders against the task inputs directly.** A generic agent's request is a passthrough +of the task row, so `body` references task input fields by name — `{{ instruction }}` — not a +chat wrapper. `instruction` is the single canonical task input. + +### Task + +```json +{ + "id": "Claim Your Free iPhone Now!", + "intent": "Classify this email as phishing or benign", + "inputs": { "instruction": "" }, + "reference": { "label": "phishing" }, + "metrics": [ + /* inline metric bundle, fanned on at config-create time (judge baked in) */ + ] +} +``` + +`inputs.instruction` is the prompt (falls back to `intent`). `reference` is grader-only ground +truth, never shown to the agent. + +### Inline metric (llm-judge) + +```json +{ + "bundle_kind": "metric-bundle", + "bundle_format_version": "v1", + "metric_type": "llm-judge", + "metadata": { "description": null, "labels": {} }, + "outputs": [ + { + "name": "accuracy", + "description": null, + "value_json_schema": { + "description": "Continuous numeric metric value.", + "title": "ContinuousScore", + "type": "number" + } + } + ], + "secrets": {}, + "payload": { + "kind": "inline", + "metric": { + "type": "llm-judge", + "model": "workspace/name", + /* judge ModelRef string — baked into the persisted spec at config-create time */ + "prompt_template": { + "messages": [ + { + "role": "user", + "content": "... Expected label: \"{{ item.reference.label }}\" ... {{ sample.output_text }} ... respond {\"accuracy\": 0|1}" + } + ] + }, + "scores": [{ "name": "accuracy", "minimum": 0, "maximum": 1 }], + "inference": { + "max_tokens": 1024, + "extra_body": { "nvext": { "max_thinking_tokens": 256 } } + }, + "reasoning": { "end_token": "" } + } + } +} +``` + +Notes on `llm-judge`: + +- `type` is `"llm-judge"` (hyphen). Enum values are inconsistent — some metrics use + underscores (`answer_accuracy`), llm-judge uses a hyphen. +- `prompt_template` must be the **messages-object** form. A bare string routes to the dead + `/completions` endpoint (502). The object form routes to `/chat/completions`. +- `reasoning.end_token: ""` strips a reasoning model's (e.g. Nemotron) thinking trace + before the JSON parser runs; without it the score is `NaN`. +- Bound `inference.extra_body.nvext.max_thinking_tokens` for NIM reasoning models. The total + `max_tokens` budget includes reasoning; without a thinking cap the judge can consume the entire + budget before emitting its structured JSON, producing a `NaN` score. +- In `agent-evaluate` the judge prompt references `{{ item.reference.label }}` / + `{{ item.inputs.instruction }}` and the agent output as `{{ sample.output_text }}`. + +### Multiple scores / multiple metrics + +A metric can emit **many scores** from one judge call — `scores` is a list. Use it when the +dimensions are genuinely distinct (e.g. `coverage`, `correctness`, `relevance`): + +```json +"scores": [ + { "name": "coverage", "minimum": 0, "maximum": 1 }, + { "name": "correctness", "minimum": 0, "maximum": 1 }, + { "name": "relevance", "minimum": 0, "maximum": 1 } +] +``` + +A task can also carry **multiple metrics** — `metrics` is a list; each runs independently. The +phishing sample uses one metric with one `accuracy` score because that is what a binary +classifier measures. + +## Result + +`GET .../agent-eval-results/{name}` returns aggregate scores per metric-score: + +```json +{ + "scores": { + "scores": [ + { + "name": "llm-judge.accuracy", + "count": 5, + "nan_count": 0, + "mean": 0.8, + "min": 0.0, + "max": 1.0, + "std_dev": 0.4, + "score_type": "range" + } + ] + } +} +``` + +`score_type` is `range` (numeric aggregate) or `rubric` (category distribution). The full +per-task bundle (trials, evidence, traces) lives in the fileset referenced by `bundle_ref`. + +## Gotchas + +- **Dataset is inline.** The `agent-evaluate` spec takes inline `tasks[]`; there is no + dataset/fileset/taskset reference yet (planned). Large datasets must be inlined for now. +- **Agent must be deployed and running before submit** — a not-yet-ready agent connection + fails the job. +- **Use `/generate`, not `/generate/full`** (per-token SSE zeroes the score). +- **Run tasks serially (`max_concurrent_tasks: 1`) by default.** NAT currently reports workflow + failures such as output truncation as **422**; `422` is not retried, so one failure kills the + whole job. Serial execution is conservative but does not fix truncation. Configure an adequate + agent output budget, or set `target.params.ignore_request_failure: true` to accept `NaN` trials. +- **`body` uses `{{ instruction }}`, not a `messages` wrapper** — a generic agent's request is a + task-row passthrough with no `messages` key to index. + +--- + +## `evaluate/jobs` (row-based, dataset-driven) + +Both endpoints are live and Studio submits to either. The modal picks by CONFIG SHAPE +(`isDatasetEvalSpec`, submitEvaluationSpec.ts) — no registry metadata: + +| Config keys | Endpoint | Sample | +| ------------------------------------------- | --------------------- | ----------------------- | +| `tasks[]`, each with its own `metrics[]` | `agent-evaluate/jobs` | email-security-analyst | +| `dataset` + `metrics[]` + `prompt_template` | `evaluate/jobs` | email-phishing-analyzer | + +Dataset-driven applies ONE metric set to every row of a dataset; task-driven attaches metrics +per task. Use dataset-driven for a uniform classifier over a labeled corpus, task-driven when +tasks differ in shape and need different scorers. + +Differences that bite: + +- `target` is the BARE agent object, NOT `{kind, agent}` — `EvaluateInputSpec` is `extra="forbid"`. +- `body` renders `{{ prompt }}` (row-based), not `{{ instruction }}` (task-based). +- `prompt_template` is REQUIRED, and `params` must be exactly `RunConfigOnline`; the job 500s + otherwise. A bare `{parallelism: N}` parses as plain `RunConfig` and fails. +- The dataset is a `FilesetRef` (`workspace/fileset#file.jsonl`) or inline rows. FilesetRef gives + users dataset/config separation, which is what most expect. +- Results come from `eval-results/{name}` (not `agent-eval-results`) and the aggregate-scores + artifact writes `scores` as a LIST of score objects, each containing `percentiles` and + `histogram` OBJECTS. Rendering those as React children throws — see ResultsPanel's + `toScoreEntries`. +- Dataset-driven jobs do NOT appear in the Studio agent-evaluations list, which reads + `agent-evaluate/jobs` only. +- `nemo evaluator evaluate run --spec-file` crashes on a FilesetRef dataset (Event loop is + closed); `submit` is unaffected. Inline rows work locally. + +Shape (reference only): + +```json +{ + "spec": { + "dataset": "default/#data.csv", + "metrics": [ + /* one shared inline metric */ + ], + "target": { + "format": "generic", + "url": ".../-/generate", + "response_path": "$.value", + "stream": false + }, + "prompt_template": { "messages": [{ "role": "user", "content": "{{ item. }}" }] }, + "params": { + /* full RunConfigOnline: parallelism, limit_samples, ignore_request_failure, request_timeout, max_retries */ + } + } +} +``` + +Results come from `GET .../eval-results/{name}` (note: `eval-results`, not `agent-eval-results`) +and include percentiles + histogram. Gotcha: an agent target requires `params` typed exactly as +`RunConfigOnline` (full shape) — a bare `{ "parallelism": N }` parses as plain `RunConfig` and +the job 500s. diff --git a/web/packages/studio/src/routes/agents/AgentsListRoute/CreateExampleAgentModal/index.tsx b/web/packages/studio/src/routes/agents/AgentsListRoute/CreateExampleAgentModal/index.tsx index aaf4bf1db7..fe126a1368 100644 --- a/web/packages/studio/src/routes/agents/AgentsListRoute/CreateExampleAgentModal/index.tsx +++ b/web/packages/studio/src/routes/agents/AgentsListRoute/CreateExampleAgentModal/index.tsx @@ -3,6 +3,7 @@ import { zodResolver } from '@hookform/resolvers/zod'; import { ControlledSearchableSelect } from '@nemo/common/src/components/form/ControlledSearchableSelect'; +import { ControlledSelect } from '@nemo/common/src/components/form/ControlledSelect'; import { FormModal } from '@nemo/common/src/components/FormModal'; import { useToast } from '@nemo/common/src/providers/toast/useToast'; import { getAgentsListAgentsQueryKey, useAgentsCreateAgent } from '@nemo/sdk/generated/agents/api'; @@ -30,15 +31,22 @@ import type { import { getAgentDetailRoute, getAgentsListRoute } from '@studio/routes/utils'; import { buildSuggestedModelOptions, - pickDefaultModelName, + pickModelNameForExample, SUGGESTED_MODEL_GROUP_LABELS, } from '@studio/util/buildSuggestedModelOptions'; -import { useQueryClient } from '@tanstack/react-query'; -import { type FC, useEffect, useRef, useState } from 'react'; -import { type SubmitHandler, useForm } from 'react-hook-form'; +import { loadSampleAgentModelName } from '@studio/util/sampleAgents'; +import { useQuery, useQueryClient } from '@tanstack/react-query'; +import { type FC, useEffect, useMemo, useState } from 'react'; +import { type SubmitHandler, useForm, useWatch } from 'react-hook-form'; import { useNavigate } from 'react-router'; -export const CreateExampleAgentModal: FC = ({ +// Since useForm is called in the component itself, key-based remount needs a thin outer wrapper +// otherwise there's nothing to put the key on +export const CreateExampleAgentModal: FC = (props) => ( + +); + +const CreateExampleAgentModalInner: FC = ({ open, onClose, workspace, @@ -53,11 +61,11 @@ export const CreateExampleAgentModal: FC = ({ { page_size: DEFAULT_LARGE_PAGE_SIZE }, { query: { enabled: open && !!workspace } } ); - const models = modelsPage?.data ?? []; + const models = useMemo(() => modelsPage?.data ?? [], [modelsPage?.data]); const modelOptions = buildSuggestedModelOptions(models); - const exampleOptions = SAMPLE_AGENTS.map((example) => ({ + const exampleItems = SAMPLE_AGENTS.map((example) => ({ value: example.key, - label: example.label, + children: example.displayName, })); const { @@ -91,7 +99,7 @@ export const CreateExampleAgentModal: FC = ({ const { control, - reset: resetForm, + setValue, handleSubmit, formState: { errors }, } = useForm({ @@ -101,28 +109,30 @@ export const CreateExampleAgentModal: FC = ({ mode: 'onChange', }); - const seededRef = useRef(false); + const exampleKey = useWatch({ control, name: 'exampleKey' }); + + const { data: preferredModel } = useQuery({ + queryKey: ['sample-agent-model', exampleKey], + queryFn: () => loadSampleAgentModelName(getSampleAgent(exampleKey).agentConfigPath), + enabled: open && !!exampleKey, + staleTime: Infinity, + }); + + const defaultModel = useMemo( + () => pickModelNameForExample(models, preferredModel), + [models, preferredModel] + ); + useEffect(() => { - if (!open) { - seededRef.current = false; - resetForm({ exampleKey: DEFAULT_SAMPLE_AGENT_KEY, modelName: '' }); - return; - } - if (seededRef.current) return; - const defaultModel = pickDefaultModelName(models); - if (defaultModel) { - resetForm({ exampleKey: DEFAULT_SAMPLE_AGENT_KEY, modelName: defaultModel }); - seededRef.current = true; - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [open, modelsPage, resetForm]); + if (!defaultModel) return; + setValue('modelName', defaultModel, { shouldValidate: true }); + }, [defaultModel, setValue]); const [loadError, setLoadError] = useState(undefined); const reset = () => { resetMutation(); setLoadError(undefined); - resetForm({ exampleKey: DEFAULT_SAMPLE_AGENT_KEY, modelName: '' }); }; const resetAndClose = () => { @@ -174,11 +184,10 @@ export const CreateExampleAgentModal: FC = ({ loading={isPending} errorText={errorMessage} > - exampleItems.find((item) => item.value === v)?.children} formFieldProps={{ slotLabel: 'Example', slotError: errors.exampleKey?.message, diff --git a/web/packages/studio/src/routes/agents/AgentsListRoute/index.test.tsx b/web/packages/studio/src/routes/agents/AgentsListRoute/index.test.tsx deleted file mode 100644 index 12274a1135..0000000000 --- a/web/packages/studio/src/routes/agents/AgentsListRoute/index.test.tsx +++ /dev/null @@ -1,354 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { getAgentsListAgentsQueryKey } from '@nemo/sdk/generated/agents/api'; -import { getModelsListModelsQueryKey } from '@nemo/sdk/generated/platform/api'; -import { markExampleAgentIntroShown } from '@studio/components/sidePanels/AgentPanels/AgentPanel/walkthroughStorage'; -import { PLATFORM_BASE_URL } from '@studio/constants/environment'; -import { ROUTES } from '@studio/constants/routes'; -import { workspace1 } from '@studio/mocks/entity-store/projects'; -import { server } from '@studio/mocks/node'; -import { AgentsListRoute } from '@studio/routes/agents/AgentsListRoute'; -import { getAgentsListRoute } from '@studio/routes/utils'; -import { renderRoute, screen, waitFor } from '@studio/tests/util/render'; -import { within } from '@testing-library/react'; -import userEvent from '@testing-library/user-event'; -import { http, HttpResponse } from 'msw'; - -vi.mock('@studio/plugins/PluginContext', async (importOriginal) => ({ - ...(await importOriginal()), - usePluginsLoaded: () => true, - usePluginsError: () => false, - usePluginInstalled: () => true, -})); - -const workspace = workspace1.workspace; -const MODELS_URL = `${PLATFORM_BASE_URL}${getModelsListModelsQueryKey(':workspace')[0]}`; -const CREATE_AGENT_URL = `${PLATFORM_BASE_URL}${getAgentsListAgentsQueryKey(':workspace')[0]}`; - -const mockModels = (names: string[]) => { - server.use( - http.get(MODELS_URL, () => - HttpResponse.json({ - data: names.map((name) => ({ name, workspace })), - pagination: { - page: 1, - page_size: 50, - current_page_size: names.length, - total_pages: 1, - total_results: names.length, - }, - sort: '-created_at', - filter: null, - }) - ) - ); -}; - -const renderList = () => - renderRoute(undefined, { - history: getAgentsListRoute(workspace), - routes: [ - { path: ROUTES.workspace.agentsList, element: }, - { path: ROUTES.workspace.agentDetail, element:
Agent detail page
}, - ], - }); - -const openModal = async (user: ReturnType): Promise => { - await user.click(await screen.findByRole('button', { name: 'Create Example Agent' })); - const dialog = await screen.findByRole('dialog'); - await within(dialog).findByRole('combobox', { name: 'Model' }); - return dialog; -}; - -describe('AgentsListRoute', () => { - beforeEach(() => sessionStorage.clear()); - - it('renders the page shell', async () => { - renderList(); - expect(await screen.findByText('Agents')).toBeInTheDocument(); - expect( - screen.getByText('View and manage AI agents and their deployments.') - ).toBeInTheDocument(); - }); - - it('opens the modal with the suggested model preselected', async () => { - const user = userEvent.setup(); - mockModels(['nvidia-nemotron-nano-9b-v2']); - renderList(); - - const dialog = await openModal(user); - await waitFor(() => - expect(within(dialog).getByRole('combobox', { name: 'Model' })).toHaveTextContent( - 'nvidia-nemotron-nano-9b-v2' - ) - ); - }); - - it('creates the example agent with the default suggested model and onboards (navigates)', async () => { - const user = userEvent.setup(); - mockModels(['meta-llama-3-1-70b-instruct', 'nvidia-nemotron-super-49b']); - - let captured: { name?: string; description?: string; config?: Record } = {}; - server.use( - http.post(CREATE_AGENT_URL, async ({ request, params }) => { - captured = (await request.json()) as typeof captured; - return HttpResponse.json({ ...captured, workspace: params['workspace'] }); - }) - ); - - renderList(); - const dialog = await openModal(user); - await waitFor(() => - expect(within(dialog).getByRole('combobox', { name: 'Model' })).toHaveTextContent( - 'nvidia-nemotron-super-49b' - ) - ); - await user.click(within(dialog).getByRole('button', { name: 'Create' })); - - expect(await screen.findByText('Agent detail page')).toBeInTheDocument(); - - expect(captured.name).toMatch(/^calculator-demo-agent-[a-z0-9]{6}$/); - expect(captured.description).toBeTruthy(); - const config = captured.config as { - workflow: { _type: string; tool_names: string[]; use_native_tool_calling: boolean }; - function_groups: Record; - functions: Record; - llms: { llm: { model_name: string } }; - }; - expect(config.workflow._type).toBe('react_agent'); - expect(config.workflow.tool_names).toEqual(['calculator', 'current_datetime']); - expect(config.workflow.use_native_tool_calling).toBe(true); - expect(config.function_groups.calculator._type).toBe('calculator'); - expect(config.functions.current_datetime._type).toBe('current_datetime'); - expect(config.llms.llm.model_name).toBe('nvidia-nemotron-super-49b'); - }); - - it('lets the user pick a different model', async () => { - const user = userEvent.setup(); - mockModels(['nvidia-nemotron-super-49b', 'meta-llama-3-1-70b-instruct']); - - let modelName: string | undefined; - server.use( - http.post(CREATE_AGENT_URL, async ({ request, params }) => { - const body = (await request.json()) as { - config: { llms: { llm: { model_name: string } } }; - }; - modelName = body.config.llms.llm.model_name; - return HttpResponse.json({ - name: 'calculator-demo-agent-abc123', - workspace: params['workspace'], - }); - }) - ); - - renderList(); - const dialog = await openModal(user); - await waitFor(() => - expect(within(dialog).getByRole('combobox', { name: 'Model' })).toHaveTextContent( - 'nvidia-nemotron-super-49b' - ) - ); - - await user.click(within(dialog).getByRole('combobox', { name: 'Model' })); - await user.click(await screen.findByRole('option', { name: 'meta-llama-3-1-70b-instruct' })); - await user.click(within(dialog).getByRole('button', { name: 'Create' })); - - await waitFor(() => expect(modelName).toBe('meta-llama-3-1-70b-instruct')); - }); - - it('creates the email phishing example when that example is selected', async () => { - const user = userEvent.setup(); - mockModels(['nvidia-nemotron-super-49b']); - - let captured: { name?: string; description?: string; config?: Record } = {}; - server.use( - http.post(CREATE_AGENT_URL, async ({ request, params }) => { - captured = (await request.json()) as typeof captured; - return HttpResponse.json({ ...captured, workspace: params['workspace'] }); - }) - ); - - renderList(); - const dialog = await openModal(user); - await waitFor(() => - expect(within(dialog).getByRole('combobox', { name: 'Model' })).toHaveTextContent( - 'nvidia-nemotron-super-49b' - ) - ); - - await user.click(within(dialog).getByRole('combobox', { name: 'Example' })); - await user.click(await screen.findByRole('option', { name: 'email_phishing_analyzer' })); - await user.click(within(dialog).getByRole('button', { name: 'Create' })); - - await waitFor(() => expect(captured.name).toMatch(/^email-phishing-demo-agent-[a-z0-9]{6}$/)); - const config = captured.config as { - workflow: { tool_names: string[] }; - functions: Record; - llms: { llm: { model_name: string } }; - }; - expect(config.workflow.tool_names).toEqual(['email_phishing_analyzer']); - expect(config.functions.email_phishing_analyzer._type).toBe('email_phishing_analyzer'); - expect(config.llms.llm.model_name).toBe('nvidia-nemotron-super-49b'); - }); - - it('excludes non-chat models from the picker', async () => { - const user = userEvent.setup(); - mockModels(['nvidia-nv-embedqa-e5-v5', 'nvidia-nemotron-nano-9b-v2']); - renderList(); - - const dialog = await openModal(user); - await waitFor(() => - expect(within(dialog).getByRole('combobox', { name: 'Model' })).toHaveTextContent( - 'nvidia-nemotron-nano-9b-v2' - ) - ); - await user.click(within(dialog).getByRole('combobox', { name: 'Model' })); - - expect( - await screen.findByRole('option', { name: 'nvidia-nemotron-nano-9b-v2' }) - ).toBeInTheDocument(); - expect( - screen.queryByRole('option', { name: 'nvidia-nv-embedqa-e5-v5' }) - ).not.toBeInTheDocument(); - }); - - it('refetches the agents list after creating so the new agent appears immediately', async () => { - const user = userEvent.setup(); - mockModels(['nvidia-nemotron-nano-9b-v2']); - // Returning user → stays on the list, so the table query is still mounted. - markExampleAgentIntroShown(); - - let agentListFetches = 0; - server.use( - http.get(CREATE_AGENT_URL, () => { - agentListFetches += 1; - return HttpResponse.json({ - data: [], - pagination: { - page: 1, - page_size: 50, - current_page_size: 0, - total_pages: 1, - total_results: 0, - }, - sort: '-created_at', - filter: null, - }); - }), - http.post(CREATE_AGENT_URL, async ({ request, params }) => { - const body = (await request.json()) as { name?: string }; - return HttpResponse.json({ ...body, workspace: params['workspace'] }); - }) - ); - - renderList(); - await waitFor(() => expect(agentListFetches).toBeGreaterThan(0)); - const before = agentListFetches; - - const dialog = await openModal(user); - await waitFor(() => - expect(within(dialog).getByRole('combobox', { name: 'Model' })).toHaveTextContent( - 'nvidia-nemotron-nano-9b-v2' - ) - ); - await user.click(within(dialog).getByRole('button', { name: 'Create' })); - - // Create invalidates the list, triggering an immediate refetch (not the 15s poll). - await waitFor(() => expect(agentListFetches).toBeGreaterThan(before)); - }); - - it('does not onboard for a later example agent in the same session', async () => { - const user = userEvent.setup(); - mockModels(['nvidia-nemotron-nano-9b-v2']); - markExampleAgentIntroShown(); - - let created = false; - server.use( - http.post(CREATE_AGENT_URL, async ({ request, params }) => { - created = true; - const body = (await request.json()) as { name?: string }; - return HttpResponse.json({ ...body, workspace: params['workspace'] }); - }) - ); - - renderList(); - const dialog = await openModal(user); - await waitFor(() => - expect(within(dialog).getByRole('combobox', { name: 'Model' })).toHaveTextContent( - 'nvidia-nemotron-nano-9b-v2' - ) - ); - await user.click(within(dialog).getByRole('button', { name: 'Create' })); - - await waitFor(() => expect(created).toBe(true)); - expect(screen.queryByText('Agent detail page')).not.toBeInTheDocument(); - expect(screen.getByRole('button', { name: 'Create Example Agent' })).toBeInTheDocument(); - }); - - it('does not onboard when an example agent already exists in the workspace', async () => { - const user = userEvent.setup(); - mockModels(['nvidia-nemotron-nano-9b-v2']); - server.use( - http.get(CREATE_AGENT_URL, () => - HttpResponse.json({ - data: [{ name: 'calculator-demo-agent-abc123', workspace }], - pagination: { - page: 1, - page_size: 50, - current_page_size: 1, - total_pages: 1, - total_results: 1, - }, - sort: '-created_at', - filter: null, - }) - ) - ); - - let created = false; - server.use( - http.post(CREATE_AGENT_URL, async ({ request, params }) => { - created = true; - const body = (await request.json()) as { name?: string }; - return HttpResponse.json({ ...body, workspace: params['workspace'] }); - }) - ); - - renderList(); - await screen.findByText('calculator-demo-agent-abc123'); - const dialog = await openModal(user); - await waitFor(() => - expect(within(dialog).getByRole('combobox', { name: 'Model' })).toHaveTextContent( - 'nvidia-nemotron-nano-9b-v2' - ) - ); - await user.click(within(dialog).getByRole('button', { name: 'Create' })); - - await waitFor(() => expect(created).toBe(true)); - expect(screen.queryByText('Agent detail page')).not.toBeInTheDocument(); - expect(screen.getByRole('button', { name: 'Create Example Agent' })).toBeInTheDocument(); - }); - - it('does not create when the workspace has no models', async () => { - const user = userEvent.setup(); - mockModels([]); - - let createCalled = false; - server.use( - http.post(CREATE_AGENT_URL, () => { - createCalled = true; - return HttpResponse.json({ name: 'unexpected', workspace }); - }) - ); - - renderList(); - const dialog = await openModal(user); - await user.click(within(dialog).getByRole('button', { name: 'Create' })); - - await waitFor(() => - expect(within(dialog).getByRole('combobox', { name: 'Model' })).toBeInTheDocument() - ); - expect(createCalled).toBe(false); - }); -}); diff --git a/web/packages/studio/src/util/buildSuggestedModelOptions.ts b/web/packages/studio/src/util/buildSuggestedModelOptions.ts index d109cddd25..e21522e2ff 100644 --- a/web/packages/studio/src/util/buildSuggestedModelOptions.ts +++ b/web/packages/studio/src/util/buildSuggestedModelOptions.ts @@ -58,3 +58,13 @@ export const pickDefaultModelName = (models: ModelListEntry[]): string | undefin const names = models.map((m) => m.name); return names.find(isSuggested) ?? names.find(isLlmCandidate); }; + +export const pickModelNameForExample = ( + models: ModelListEntry[], + preferred: string | null | undefined +): string | undefined => { + if (preferred && models.some((m) => m.name === preferred && isLlmCandidate(m.name))) { + return preferred; + } + return pickDefaultModelName(models); +}; diff --git a/web/packages/studio/src/util/sampleAgents.ts b/web/packages/studio/src/util/sampleAgents.ts new file mode 100644 index 0000000000..d4f42bb9da --- /dev/null +++ b/web/packages/studio/src/util/sampleAgents.ts @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { fetchSampleText } from '@studio/api/agents/fetchSampleText'; +import YAML from 'yaml'; + +export const loadSampleAgentModelName = async (agentConfigPath: string): Promise => { + const text = await fetchSampleText(agentConfigPath); + let config: Record | undefined; + try { + config = YAML.parse(text) as Record | undefined; + } catch { + return null; + } + const llm = (config?.llms as { llm?: unknown } | undefined)?.llm; + if (!llm || typeof llm !== 'object' || Array.isArray(llm)) return null; + + const modelName = (llm as Record).model_name; + if (typeof modelName !== 'string' || modelName.includes('${')) return null; + + const bare = modelName.includes('/') ? (modelName.split('/').pop() ?? modelName) : modelName; + return bare.trim() || null; +}; From caee7a361fa9ef6268d4c342a329570eb33f0645 Mon Sep 17 00:00:00 2001 From: Octavian Drulea Date: Wed, 5 Aug 2026 15:36:39 -0700 Subject: [PATCH 2/4] chore(studio): drop unneeded files from PR Signed-off-by: Octavian Drulea --- web/packages/studio/AGENTS.md | 41 -- .../dataViews/AgentsDataView/index.test.tsx | 12 +- .../dataViews/AgentsDataView/index.tsx | 2 +- .../studio/src/routes/agents/AGENTS.md | 466 ------------------ 4 files changed, 7 insertions(+), 514 deletions(-) delete mode 100644 web/packages/studio/src/routes/agents/AGENTS.md diff --git a/web/packages/studio/AGENTS.md b/web/packages/studio/AGENTS.md index f9a6bd41bf..e90d88e242 100644 --- a/web/packages/studio/AGENTS.md +++ b/web/packages/studio/AGENTS.md @@ -54,44 +54,3 @@ Wrong — transparent dropdown: Exception: call sites that fill SelectContent with custom children containing their own background (e.g. a sticky Block with bg-surface) are fine as-is. - -### KUI form gotchas — the silent-failure set - -Each compiles, lints and typechecks clean, then renders nothing wrong — the element is -simply inert or invisible. - -**`slotError` needs `status="error"` to render.** Without a status `FormField` shows -`slotHelp` instead and drops the message. `Controlled*` wrappers set status from -react-hook-form only, so any other error source must set it too (`formFieldProps` spreads -last, so it wins): - -```tsx -formFieldProps={{ slotError: fieldError, status: fieldError ? 'error' : undefined }} -``` - -**`FormModal.disabled` blocks closing.** It means "busy": it kills Cancel and stops -`handleUserClose`, trapping the user. For validation use `submitDisabled`. - -```tsx -disabled={isPending} // busy → intentionally locked -submitDisabled={!isValid} // invalid → submit blocked, dismiss still works -``` - -**`Text` has no `color="danger"`, and `text-danger` is not an emitted utility.** `danger` -exists only on `Button`; the token is `--text-color-feedback-danger`. Devtools showing -**"Inherited from"** means no rule matched at all, not that yours lost a specificity fight. - -```tsx -… -``` - -**Modals seeded from a prop must re-seed on `open`.** `useForm({ defaultValues })` reads -once at mount; a persistently-rendered modal mounts before the prop exists and keeps the -empty default forever. Callers passing a constant work by accident, so the bug arrives -with the second caller. - -```tsx -useEffect(() => { - resetForm(makeDefaultValues(agentProp)); // NOT `if (!open)` — that only resets on close -}, [open, agentProp, resetForm]); -``` diff --git a/web/packages/studio/src/components/dataViews/AgentsDataView/index.test.tsx b/web/packages/studio/src/components/dataViews/AgentsDataView/index.test.tsx index ba52a80785..9c586bb7aa 100644 --- a/web/packages/studio/src/components/dataViews/AgentsDataView/index.test.tsx +++ b/web/packages/studio/src/components/dataViews/AgentsDataView/index.test.tsx @@ -139,7 +139,7 @@ describe('CombinedAgentsTable', () => { }); describe('row actions', () => { - it('shows Deploy, Compare Models, Clone, and Delete actions for agent rows', async () => { + it('shows Deploy, Test models, Clone, and Delete actions for agent rows', async () => { const user = userEvent.setup(); renderTable(); @@ -149,7 +149,7 @@ describe('CombinedAgentsTable', () => { await user.click(menuButtons[0]); const deployItems = await screen.findAllByRole('menuitem', { name: 'Deploy' }); - const testModelItems = screen.getAllByRole('menuitem', { name: 'Compare Models' }); + const testModelItems = screen.getAllByRole('menuitem', { name: 'Test models' }); const cloneItems = screen.getAllByRole('menuitem', { name: 'Clone' }); const deleteItems = screen.getAllByRole('menuitem', { name: 'Delete' }); expect(deployItems.length).toBeGreaterThan(0); @@ -158,7 +158,7 @@ describe('CombinedAgentsTable', () => { expect(deleteItems.length).toBeGreaterThan(0); }); - it('opens Playground with the row model selected when Compare Models is selected', async () => { + it('opens Playground with the row model selected when Test models is selected', async () => { const user = userEvent.setup(); renderRoute(undefined, { history: getAgentsListRoute(WORKSPACE), @@ -177,7 +177,7 @@ describe('CombinedAgentsTable', () => { await screen.findByText(MOCK_AGENTS[0].name); await user.click(screen.getAllByRole('button', { name: /actions/i })[0]); - await user.click((await screen.findAllByRole('menuitem', { name: 'Compare Models' }))[0]); + await user.click((await screen.findAllByRole('menuitem', { name: 'Test models' }))[0]); expect(await screen.findByTestId('model-compare-location')).toHaveTextContent( `${getModelCompareRoute(WORKSPACE)}?model=${encodeURIComponent( @@ -186,7 +186,7 @@ describe('CombinedAgentsTable', () => { ); }); - it('hides Compare Models when Playground is disabled', async () => { + it('hides Test models when Playground is disabled', async () => { const user = userEvent.setup(); renderRoute(undefined, { history: getAgentsListRoute(WORKSPACE), @@ -205,7 +205,7 @@ describe('CombinedAgentsTable', () => { expect((await screen.findAllByRole('menuitem', { name: 'Deploy' })).length).toBeGreaterThan( 0 ); - expect(screen.queryByRole('menuitem', { name: 'Compare Models' })).not.toBeInTheDocument(); + expect(screen.queryByRole('menuitem', { name: 'Test models' })).not.toBeInTheDocument(); }); it('calls onCloneAgent with the row when Clone is selected', async () => { diff --git a/web/packages/studio/src/components/dataViews/AgentsDataView/index.tsx b/web/packages/studio/src/components/dataViews/AgentsDataView/index.tsx index c0c1723064..c2391cbe08 100644 --- a/web/packages/studio/src/components/dataViews/AgentsDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/AgentsDataView/index.tsx @@ -285,7 +285,7 @@ export const AgentsTable: FC = ({ ...(canTestModels ? [ { - children: 'Compare Models', + children: 'Test models', onSelect: () => { const target = getModelCompareRoute(workspace); const model = row.models[0]; diff --git a/web/packages/studio/src/routes/agents/AGENTS.md b/web/packages/studio/src/routes/agents/AGENTS.md deleted file mode 100644 index 8967d2649a..0000000000 --- a/web/packages/studio/src/routes/agents/AGENTS.md +++ /dev/null @@ -1,466 +0,0 @@ -# Agent Routes - -Everything under `src/routes/agents/` — the agents list, detail, deployments, monitor, -suggestions, and evaluations routes — plus the sample-agent registry those routes read from. - -Contents: - -- [Adding a New Example Agent](#adding-a-new-example-agent) — end-to-end checklist -- [Agent Evaluation](#agent-evaluation) — how Studio runs evaluations - ---- - -# Adding a New Example Agent - -Example agent = entry in `src/constants/sampleAgents.ts`. Create-Example modal deploys it; Run-Evaluation modal seeds a config from it. - -Touches: Python package, 2 ymls, 1 constants file, 1 Studio rebuild. - -## Checklist - -1. Plugin package `plugins/nemo-agents/examples//` — mirror `email-phishing-analyzer/`. `pyproject.toml` with `[project.entry-points."nat.components"] = ".register"`; `src//register.py` = one `FunctionBaseConfig` + `@register_function` per tool; `tests/`. -2. Register in 4 places (below). -3. `uv sync`, then **restart services**. `nat start fastapi` discovers `_type`s only at process start. -4. Studio asset `public/sample-agents//agent.yml` — second independent copy, hand-synced. -5. `SAMPLE_AGENTS` entry. `evalConfigPath` optional; omit → absent from Run Evaluation. -6. `pnpm --filter nemo-studio-ui build:fastapi`. -7. Verify (below). - -## Registration: 4 places - -`{ workspace = true }` is a pointer into root `members`, not standalone. Missing root member → `references a workspace in tool.uv.sources, but is not a workspace member`. - -| File | Section | Entry | -| ------------------------------------ | ----------------------------- | ------------------------------------------------ | -| root `pyproject.toml` | `[tool.uv.sources]` | `nemo-agents-example- = { workspace = true }` | -| root `pyproject.toml` | `[tool.uv.workspace] members` | `"plugins/nemo-agents/examples/"` | -| `plugins/nemo-agents/pyproject.toml` | `[project] dependencies` | `"nemo-agents-example-"` | -| `plugins/nemo-agents/pyproject.toml` | `[tool.uv.sources]` | `nemo-agents-example- = { workspace = true }` | - -Tried, rejected: - -- members glob `examples/*` — only 3 of 11 subdirs are packages; uv errors on first without `pyproject.toml`. -- path source `{ path = "...", editable = true }` in plugin only — resolves without root membership, still rewrites `uv.lock`. -- `uv pip install -e `, no toml entries — zero repo diff, but `uv sync` **prunes it**; `_type`s vanish, deploy fails at startup. - -Wheel bundle (`packages/nemo_platform/pyproject.toml` `[tool.bundle-package.*]`) lists only `calculator`. Others resolve under workspace `uv sync`, **not** from a wheel. - -**Re-lock needs `uv lock --python 3.12`.** `nooa` (git dep of `nemo-experimentalist-plugin`) requires py `>=3.12,<3.14`; uv builds git-dep metadata with the active interpreter (venv is 3.11). Committed lock caches that metadata, so plain `uv sync` works and this only bites on a real re-resolve. Resulting diff is large but purely additive (0 deletions). - -## Two agent ymls, on purpose - -| Copy | Workflow | Consumer | -| --------------------------------------- | -------------------- | --------- | -| `plugins/.../-agent.yml` | `react_agent` | CLI / NAT | -| `public/sample-agents//agent.yml` | `tool_calling_agent` | Studio | - -Independent files, hand-synced. Studio serves only `public/`; it cannot read `plugins/`. - -Studio copy constraints: - -- LLM key must be literally `llm` — `loadSampleAgentConfig.ts` hardcodes `config.llms.llm`, throws otherwise. -- `model_name: ${NEMO_DEFAULT_MODEL}` inert — Studio overwrites with dropdown pick before POST. -- Prefer `tool_calling_agent`; ReAct text prompt makes small models mis-parse a scratchpad and run away. - -## max_tokens - -Phishing's `1024` truncates any agent emitting a JSON report or batch triage. Reasoning tokens exhaust the budget: - -``` -LLM output truncated (finish_reason='length'). output_tokens=1024, ... -Truncated output: 'We need to list...' -``` - -`4096` sufficed for ~6-item batch + 5-field JSON report. Model-size independent — same truncation on 30B and 120B. - -Capping reasoning tokens instead is unavailable: NAT's knob is `thinking: bool`, gated in `nat/data_models/thinking_mixin.py` by regex `^nvidia/(llama|nvidia).*nemotron`. Platform names are workspace-qualified (`default/...`) → never match → deploy fails `thinking is not supported for model_name`. (`max_thinking_tokens` in phishing's eval config is on the llm-judge metric, different layer.) - -## Rebuild - -`public/` copies into `dist/` at build; platform serves **built** `dist/`. Stale `dist/` → new asset 404s at runtime though the file exists and tests pass. - -```bash -pnpm --filter nemo-studio-ui build:fastapi -curl -o /dev/null -w '%{http_code}\n' http://127.0.0.1:8080/studio/sample-agents//agent.yml -``` - -`404` — including for existing samples — means `dist/` predates `public/sample-agents/`. - -## Verify - -Green tests are insufficient; they mock the asset fetch. - -```bash -# 1. _types registered -uv run python -c " -import .register -from nat.cli.type_registry import GlobalTypeRegistry -r = GlobalTypeRegistry.get() -print(sorted(c.local_name for c in r.get_registered_functions() if c.module_name==''))" - -# 2. asset served, not just on disk -curl -sf http://127.0.0.1:8080/studio/sample-agents//agent.yml >/dev/null && echo served - -# 3. real Create-Example path: fetch served yml -> inject model -> -# POST /apis/agents/v2/workspaces/default/agents -> deploy -> invoke /-/generate -``` - -**Read `nemo agents logs --agent ` first.** Workflow error messages mislead: `ReActAgentParsingFailedError ... LLM output: ''` and `No response received from agent` were the same root cause, visible only in logs as `502 Bad Gateway - Backend returned 404: {"error":{"message":"Model not found"}}`. - -Confirm the model is served before blaming the agent. Names differing by transposition (`nemotron-nano-3-30b` vs `nemotron-3-nano-30b`) can both exist as entities with only one reachable: - -```bash -curl -s -o /dev/null -w '%{http_code}\n' -X POST \ - http://127.0.0.1:8080/apis/inference-gateway/v2/workspaces/default/openai/-/v1/chat/completions \ - -H 'Content-Type: application/json' \ - -d '{"model":"","messages":[{"role":"user","content":"say ok"}],"max_tokens":16}' -``` - -## Naming - -`namePrefix` drives `sampleAgentKeyForAgentName` (matches `${namePrefix}-`, longest wins). Changing an existing `namePrefix` is breaking: deployed agents stop matching, silently fall back to `EVALUATION_SAMPLE_AGENTS[0]` — wrong eval config, no error. - ---- - -# Agent Evaluation - -How Studio runs agent evaluations. - -## Overview - -- **Runner:** nemo-evaluator's **`agent-evaluate/jobs`** endpoint (task-based, `AgentEvaluateJob`), - with an **agent** target. This is the canonical path for Studio agent evaluation, per the - nemo-evaluator team — it is purpose-built for agent tasks with extensive artifact + trace - collection. (A legacy row-based `evaluate/jobs` endpoint also exists — see the bottom.) -- **Sample agents:** `public/sample-agents//` is the seed repository (agent config + - an `eval-config.json`). Studio reads these for the "use example" flow. -- **Metrics:** Studio creates metrics **only** as `InlineMetricPayload` — a built-in metric - serialized to JSON, reconstructed at runtime. No `CloudPickleMetricPayload` (no Python, no - pickled code). - -## Tasks, metrics, scores - -Three levels, don't conflate them: - -- **task** — one input case = one agent call (the "unit of work being evaluated"). The sample - has 5 tasks (5 emails). -- **metric** — a scorer applied to a task's output; a task can carry more than one. -- **score** — a value a metric emits; one metric can emit many (a judge can rate several - dimensions from one call). - -The evaluation loop is `task × trial × metric → score`. - -## Reusable eval configs (Filesets) - -nemo-evaluator has no "eval config" entity, so a reusable config is stored as an -**`eval-config.json`** file in a named **Fileset**. The tasks are stored **inline** in the -config (the `agent-evaluate` spec takes inline `tasks[]`; a Task/Taskset _reference_ is planned -but not yet implemented, so large datasets are inlined for now). - -Two shapes are involved: - -- **Example template** (`public/sample-agents/*/eval-config.json`): a shared-metric layout — - `tasks[]` (no per-task metrics) + one top-level `metric` (no judge_model). Read only when - creating a config from an example; never submitted directly. -- **Persisted spec** (what a Fileset actually stores = the yardstick): an `AgentEvalInputSpec` - **minus `target`**. Every task carries its own `metrics[]` with the **judge model baked in**; - `max_concurrent_tasks` is baked. No agent target (that is per-run). - -Persisted `eval-config.json` shape (the yardstick): - -```json -{ - "tasks": [ - { - "id": "Claim Your Free iPhone Now!", - "intent": "Classify this email as phishing or benign", - "inputs": { "instruction": "" }, - "reference": { "label": "phishing" }, - "metrics": [ - { - /* inline metric bundle — see Payloads; judge model baked into payload.metric.model */ - } - ] - } - ], - "max_concurrent_tasks": 1 -} -``` - -**Create ("Use Example"):** Studio reads the example template, **fans the shared `metric` onto -each task** and **injects the chosen judge model** (`buildPersistedSpec`), then writes that -persisted spec into a new Fileset. The judge is now part of the stored yardstick. - -**Reuse ("Choose Fileset"):** the user selects a Fileset **by name**. Studio reads its -`eval-config.json` **as-is** (`parsePersistedSpec`) — no re-fan, no judge re-pick — so the -yardstick (tasks + metric + judge) is identical across every run. Only the agent **target** is -injected at submit; the whole is wrapped as `{ spec }` for the job request. The saved spec is -also a valid `nemo evaluator agent-evaluate submit --spec-file` input once a `target` is added. - -Injected by Studio **at submit** (never stored in the config): the agent **target** only. The -judge model and `max_concurrent_tasks` are part of the persisted yardstick. - -**`max_concurrent_tasks` defaults to `1` (serial).** This is a conservative default for local NAT -deployments. NAT currently maps workflow failures, including output truncation, to `422`; a single -failure aborts the job. Concurrency was ruled out as the root cause, but serial execution avoids -adding load while this upstream error classification remains. To tolerate failures instead, set -`target.params.ignore_request_failure: true` (failed trials score `NaN`). See Gotchas. - -## Endpoints - -| Purpose | Method + path (`/apis/evaluator/v2/workspaces/{ws}`) | -| ----------- | ---------------------------------------------------- | -| List jobs | `GET .../agent-evaluate/jobs` | -| Submit | `POST .../agent-evaluate/jobs` | -| Get job | `GET .../agent-evaluate/jobs/{name}` | -| Poll status | `GET .../agent-evaluate/jobs/{name}/status` | -| Cancel | `POST .../agent-evaluate/jobs/{name}/cancel` | -| Logs | `GET .../agent-evaluate/jobs/{name}/logs` | -| Results | `GET .../agent-eval-results/{name}` | - -Fileset create/upload (Files service), for seeding a config into a new Fileset: - -| Purpose | Method + path | -| -------------- | ------------------------------------------------------------- | -| Create fileset | `POST /apis/files/v2/workspaces/{ws}/filesets` | -| Upload file | `PUT /apis/files/v2/workspaces/{ws}/filesets/{name}/-/{path}` | - -Submit body is wrapped: `{"spec": { ...AgentEvalInputSpec }}`. - -## Payloads - -### Job spec - -```json -{ - "spec": { - "tasks": [ - /* each carries metrics[] from the persisted spec (judge already baked) */ - ], - "target": { - "kind": "agent", - "agent": { - /* see below */ - } - }, - "max_concurrent_tasks": 1 - } -} -``` - -### Target (generic agent) - -```json -{ - "kind": "agent", - "agent": { - "format": "generic", - "url": ".../agents//-/generate", - "name": "", - "body": { "input_message": "{{ instruction }}" }, - "response_path": "$.value", - "stream": false - } -} -``` - -Use the non-streaming `/generate` endpoint. Do **not** use `/generate/full` — its per-token -SSE stream leaves only the last token in the captured output and every score collapses to 0. - -**`body` renders against the task inputs directly.** A generic agent's request is a passthrough -of the task row, so `body` references task input fields by name — `{{ instruction }}` — not a -chat wrapper. `instruction` is the single canonical task input. - -### Task - -```json -{ - "id": "Claim Your Free iPhone Now!", - "intent": "Classify this email as phishing or benign", - "inputs": { "instruction": "" }, - "reference": { "label": "phishing" }, - "metrics": [ - /* inline metric bundle, fanned on at config-create time (judge baked in) */ - ] -} -``` - -`inputs.instruction` is the prompt (falls back to `intent`). `reference` is grader-only ground -truth, never shown to the agent. - -### Inline metric (llm-judge) - -```json -{ - "bundle_kind": "metric-bundle", - "bundle_format_version": "v1", - "metric_type": "llm-judge", - "metadata": { "description": null, "labels": {} }, - "outputs": [ - { - "name": "accuracy", - "description": null, - "value_json_schema": { - "description": "Continuous numeric metric value.", - "title": "ContinuousScore", - "type": "number" - } - } - ], - "secrets": {}, - "payload": { - "kind": "inline", - "metric": { - "type": "llm-judge", - "model": "workspace/name", - /* judge ModelRef string — baked into the persisted spec at config-create time */ - "prompt_template": { - "messages": [ - { - "role": "user", - "content": "... Expected label: \"{{ item.reference.label }}\" ... {{ sample.output_text }} ... respond {\"accuracy\": 0|1}" - } - ] - }, - "scores": [{ "name": "accuracy", "minimum": 0, "maximum": 1 }], - "inference": { - "max_tokens": 1024, - "extra_body": { "nvext": { "max_thinking_tokens": 256 } } - }, - "reasoning": { "end_token": "" } - } - } -} -``` - -Notes on `llm-judge`: - -- `type` is `"llm-judge"` (hyphen). Enum values are inconsistent — some metrics use - underscores (`answer_accuracy`), llm-judge uses a hyphen. -- `prompt_template` must be the **messages-object** form. A bare string routes to the dead - `/completions` endpoint (502). The object form routes to `/chat/completions`. -- `reasoning.end_token: ""` strips a reasoning model's (e.g. Nemotron) thinking trace - before the JSON parser runs; without it the score is `NaN`. -- Bound `inference.extra_body.nvext.max_thinking_tokens` for NIM reasoning models. The total - `max_tokens` budget includes reasoning; without a thinking cap the judge can consume the entire - budget before emitting its structured JSON, producing a `NaN` score. -- In `agent-evaluate` the judge prompt references `{{ item.reference.label }}` / - `{{ item.inputs.instruction }}` and the agent output as `{{ sample.output_text }}`. - -### Multiple scores / multiple metrics - -A metric can emit **many scores** from one judge call — `scores` is a list. Use it when the -dimensions are genuinely distinct (e.g. `coverage`, `correctness`, `relevance`): - -```json -"scores": [ - { "name": "coverage", "minimum": 0, "maximum": 1 }, - { "name": "correctness", "minimum": 0, "maximum": 1 }, - { "name": "relevance", "minimum": 0, "maximum": 1 } -] -``` - -A task can also carry **multiple metrics** — `metrics` is a list; each runs independently. The -phishing sample uses one metric with one `accuracy` score because that is what a binary -classifier measures. - -## Result - -`GET .../agent-eval-results/{name}` returns aggregate scores per metric-score: - -```json -{ - "scores": { - "scores": [ - { - "name": "llm-judge.accuracy", - "count": 5, - "nan_count": 0, - "mean": 0.8, - "min": 0.0, - "max": 1.0, - "std_dev": 0.4, - "score_type": "range" - } - ] - } -} -``` - -`score_type` is `range` (numeric aggregate) or `rubric` (category distribution). The full -per-task bundle (trials, evidence, traces) lives in the fileset referenced by `bundle_ref`. - -## Gotchas - -- **Dataset is inline.** The `agent-evaluate` spec takes inline `tasks[]`; there is no - dataset/fileset/taskset reference yet (planned). Large datasets must be inlined for now. -- **Agent must be deployed and running before submit** — a not-yet-ready agent connection - fails the job. -- **Use `/generate`, not `/generate/full`** (per-token SSE zeroes the score). -- **Run tasks serially (`max_concurrent_tasks: 1`) by default.** NAT currently reports workflow - failures such as output truncation as **422**; `422` is not retried, so one failure kills the - whole job. Serial execution is conservative but does not fix truncation. Configure an adequate - agent output budget, or set `target.params.ignore_request_failure: true` to accept `NaN` trials. -- **`body` uses `{{ instruction }}`, not a `messages` wrapper** — a generic agent's request is a - task-row passthrough with no `messages` key to index. - ---- - -## `evaluate/jobs` (row-based, dataset-driven) - -Both endpoints are live and Studio submits to either. The modal picks by CONFIG SHAPE -(`isDatasetEvalSpec`, submitEvaluationSpec.ts) — no registry metadata: - -| Config keys | Endpoint | Sample | -| ------------------------------------------- | --------------------- | ----------------------- | -| `tasks[]`, each with its own `metrics[]` | `agent-evaluate/jobs` | email-security-analyst | -| `dataset` + `metrics[]` + `prompt_template` | `evaluate/jobs` | email-phishing-analyzer | - -Dataset-driven applies ONE metric set to every row of a dataset; task-driven attaches metrics -per task. Use dataset-driven for a uniform classifier over a labeled corpus, task-driven when -tasks differ in shape and need different scorers. - -Differences that bite: - -- `target` is the BARE agent object, NOT `{kind, agent}` — `EvaluateInputSpec` is `extra="forbid"`. -- `body` renders `{{ prompt }}` (row-based), not `{{ instruction }}` (task-based). -- `prompt_template` is REQUIRED, and `params` must be exactly `RunConfigOnline`; the job 500s - otherwise. A bare `{parallelism: N}` parses as plain `RunConfig` and fails. -- The dataset is a `FilesetRef` (`workspace/fileset#file.jsonl`) or inline rows. FilesetRef gives - users dataset/config separation, which is what most expect. -- Results come from `eval-results/{name}` (not `agent-eval-results`) and the aggregate-scores - artifact writes `scores` as a LIST of score objects, each containing `percentiles` and - `histogram` OBJECTS. Rendering those as React children throws — see ResultsPanel's - `toScoreEntries`. -- Dataset-driven jobs do NOT appear in the Studio agent-evaluations list, which reads - `agent-evaluate/jobs` only. -- `nemo evaluator evaluate run --spec-file` crashes on a FilesetRef dataset (Event loop is - closed); `submit` is unaffected. Inline rows work locally. - -Shape (reference only): - -```json -{ - "spec": { - "dataset": "default/#data.csv", - "metrics": [ - /* one shared inline metric */ - ], - "target": { - "format": "generic", - "url": ".../-/generate", - "response_path": "$.value", - "stream": false - }, - "prompt_template": { "messages": [{ "role": "user", "content": "{{ item. }}" }] }, - "params": { - /* full RunConfigOnline: parallelism, limit_samples, ignore_request_failure, request_timeout, max_retries */ - } - } -} -``` - -Results come from `GET .../eval-results/{name}` (note: `eval-results`, not `agent-eval-results`) -and include percentiles + histogram. Gotcha: an agent target requires `params` typed exactly as -`RunConfigOnline` (full shape) — a bare `{ "parallelism": N }` parses as plain `RunConfig` and -the job 500s. From ed1e04d4e5f51697107bb96c79e5682673b233c4 Mon Sep 17 00:00:00 2001 From: Octavian Drulea Date: Wed, 5 Aug 2026 16:55:05 -0700 Subject: [PATCH 3/4] fix(studio): clear bulk selecitons when workspace changes Signed-off-by: Octavian Drulea --- .../src/components/dataViews/AgentsDataView/index.tsx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/web/packages/studio/src/components/dataViews/AgentsDataView/index.tsx b/web/packages/studio/src/components/dataViews/AgentsDataView/index.tsx index c2391cbe08..ca2e5a1d85 100644 --- a/web/packages/studio/src/components/dataViews/AgentsDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/AgentsDataView/index.tsx @@ -115,6 +115,14 @@ export const AgentsTable: FC = ({ }, }); + // `keepPreviousData` keeps the previous workspace's rows on screen after a switch, + // so a selection made there would still resolve — and delete by name against the new + // workspace. Drop it as soon as the workspace changes. + const clearRowSelection = dataViewState.rowSelection.set; + useEffect(() => { + clearRowSelection({}); + }, [workspace, clearRowSelection]); + const page = dataViewState.pagination.state.pageIndex + 1; const pageSize = dataViewState.pagination.state.pageSize; const sortParam = getSortParamWithWhitelist( From 60bc78e4eaff60acec83affc4a1865332e6eb163 Mon Sep 17 00:00:00 2001 From: Octavian Drulea Date: Wed, 5 Aug 2026 19:23:20 -0700 Subject: [PATCH 4/4] fix(studio): clear bulk selection if workspace changes Signed-off-by: Octavian Drulea --- .../src/components/dataViews/AgentsDataView/index.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/web/packages/studio/src/components/dataViews/AgentsDataView/index.tsx b/web/packages/studio/src/components/dataViews/AgentsDataView/index.tsx index ca2e5a1d85..d4a7680cd9 100644 --- a/web/packages/studio/src/components/dataViews/AgentsDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/AgentsDataView/index.tsx @@ -115,12 +115,13 @@ export const AgentsTable: FC = ({ }, }); - // `keepPreviousData` keeps the previous workspace's rows on screen after a switch, - // so a selection made there would still resolve — and delete by name against the new - // workspace. Drop it as soon as the workspace changes. + // `keepPreviousData` keeps the previous workspace's rows on screen after a switch, so a + // selection or a pending delete made there would still resolve — and delete by name + // against the new workspace. Drop both as soon as the workspace changes. const clearRowSelection = dataViewState.rowSelection.set; useEffect(() => { clearRowSelection({}); + setDeleteState(null); }, [workspace, clearRowSelection]); const page = dataViewState.pagination.state.pageIndex + 1;