diff --git a/plugins/nemo-agents/examples/email-phishing-analyzer/src/nat_email_phishing_analyzer/email-phishing-agent.yml b/plugins/nemo-agents/examples/email-phishing-analyzer/src/nat_email_phishing_analyzer/email-phishing-agent.yml index c8929da4a1..0c55302a8f 100644 --- a/plugins/nemo-agents/examples/email-phishing-analyzer/src/nat_email_phishing_analyzer/email-phishing-agent.yml +++ b/plugins/nemo-agents/examples/email-phishing-analyzer/src/nat_email_phishing_analyzer/email-phishing-agent.yml @@ -28,7 +28,7 @@ llms: api_key: not-used model_name: ${NEMO_DEFAULT_MODEL} temperature: 0.0 - max_tokens: 512 + max_tokens: 1024 workflow: _type: react_agent diff --git a/web/packages/common/src/components/DatasetFileSelect/ControlledDatasetFileSelect.tsx b/web/packages/common/src/components/DatasetFileSelect/ControlledDatasetFileSelect.tsx index fe673d7222..23584a2693 100644 --- a/web/packages/common/src/components/DatasetFileSelect/ControlledDatasetFileSelect.tsx +++ b/web/packages/common/src/components/DatasetFileSelect/ControlledDatasetFileSelect.tsx @@ -39,6 +39,7 @@ interface ControlledDatasetFileSelectProps extends UseControllerComponentProps { datasetLabel?: string; /** Auto-select the first root-level accepted file on fileset selection. */ autoSelectFirstAcceptable?: boolean; + showUpdatedAt?: boolean; /** * Callback fired when a file is selected. Useful for custom validation or processing. * Called with the selected file info, or null when file is cleared. @@ -83,6 +84,7 @@ export const ControlledDatasetFileSelect: FC = filesetPurpose, datasetLabel, autoSelectFirstAcceptable, + showUpdatedAt, }) => { const { field: { onChange, value }, @@ -144,6 +146,7 @@ export const ControlledDatasetFileSelect: FC = filesetPurpose={filesetPurpose} datasetLabel={datasetLabel} autoSelectFirstAcceptable={autoSelectFirstAcceptable} + showUpdatedAt={showUpdatedAt} /> ); diff --git a/web/packages/common/src/components/DatasetFileSelect/DatasetFileSelect.tsx b/web/packages/common/src/components/DatasetFileSelect/DatasetFileSelect.tsx index 3d7a154dd1..4e9cc15ab9 100644 --- a/web/packages/common/src/components/DatasetFileSelect/DatasetFileSelect.tsx +++ b/web/packages/common/src/components/DatasetFileSelect/DatasetFileSelect.tsx @@ -56,6 +56,7 @@ interface DatasetFileSelectProps { datasetLabel?: string; /** Auto-select the first root-level accepted file on fileset selection. */ autoSelectFirstAcceptable?: boolean; + showUpdatedAt?: boolean; } /** @@ -87,6 +88,7 @@ export const DatasetFileSelect: FC = ({ filesetPurpose, datasetLabel, autoSelectFirstAcceptable, + showUpdatedAt, }) => { const [isModalOpen, setIsModalOpen] = useState(false); @@ -211,6 +213,7 @@ export const DatasetFileSelect: FC = ({ filesetPurpose={filesetPurpose} datasetLabel={datasetLabel} autoSelectFirstAcceptable={autoSelectFirstAcceptable} + showUpdatedAt={showUpdatedAt} /> ) : ( ; }; @@ -106,6 +107,7 @@ export const uploadModalReducer = ( allowMultipleFileSelection: state.allowMultipleFileSelection, invalidFileMode: state.invalidFileMode, allowNewDataset: state.allowNewDataset, + showUpdatedAt: state.showUpdatedAt, }; case 'UPDATE_DATASET': diff --git a/web/packages/common/src/components/UploadModal/DatasetUploader/Select.tsx b/web/packages/common/src/components/UploadModal/DatasetUploader/Select.tsx index 3a74460794..a2f653610b 100644 --- a/web/packages/common/src/components/UploadModal/DatasetUploader/Select.tsx +++ b/web/packages/common/src/components/UploadModal/DatasetUploader/Select.tsx @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { getFileExtension } from '@nemo/common/src/components/DatasetFileSelect/utils'; +import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; import { useUploadModalContext } from '@nemo/common/src/components/UploadModal/Context/useUploadModalContext'; import { getExistingFileId } from '@nemo/common/src/components/UploadModal/utils'; import { getEntityReference } from '@nemo/common/src/namedEntity'; @@ -20,14 +21,29 @@ interface Props { error?: string; } -const filesetToOption = (fileset: FilesetOutput) => ({ +const filesetToOption = (fileset: FilesetOutput, showUpdatedAt?: boolean) => ({ children: fileset.name ?? '', value: getEntityReference(fileset), + ...(showUpdatedAt && fileset.updated_at + ? { + slotEnd: ( + + + + ), + } + : {}), }); export const DatasetSelect: FC = ({ project, disabled, error }) => { const [state, dispatch] = useUploadModalContext(); - const { dataset, allowNewDataset, acceptableFileTypes, autoSelectFirstAcceptable } = state; + const { + dataset, + allowNewDataset, + acceptableFileTypes, + autoSelectFirstAcceptable, + showUpdatedAt, + } = state; const purpose = state.filesetPurpose ?? 'dataset'; const label = state.datasetLabel ?? 'Dataset'; @@ -105,9 +121,9 @@ export const DatasetSelect: FC = ({ project, disabled, error }) => { return ( filesets ?.sort((filesetA, filesetB) => (filesetA?.name || '').localeCompare(filesetB.name || '')) - .map(filesetToOption) || [] + .map((fileset) => filesetToOption(fileset, showUpdatedAt)) || [] ); - }, [filesets, isLoading, isError]); + }, [filesets, isLoading, isError, showUpdatedAt]); return ( & { /** Called once the picked / uploaded file is committed. */ onSubmit: (data: SubmitUploadType) => void; @@ -154,6 +155,7 @@ export const InlineUploadPicker: FC = ({ filesetPurpose, datasetLabel, autoSelectFirstAcceptable, + showUpdatedAt, onSubmit, addButtonText = 'Add file', autoCommit = false, @@ -178,6 +180,7 @@ export const InlineUploadPicker: FC = ({ datasetLabel: datasetLabel ?? uploadModalInitialState.datasetLabel, autoSelectFirstAcceptable: autoSelectFirstAcceptable ?? uploadModalInitialState.autoSelectFirstAcceptable, + showUpdatedAt: showUpdatedAt ?? uploadModalInitialState.showUpdatedAt, }), [ allowMultipleFileSelection, @@ -188,6 +191,7 @@ export const InlineUploadPicker: FC = ({ filesetPurpose, datasetLabel, autoSelectFirstAcceptable, + showUpdatedAt, ] ); return ( diff --git a/web/packages/common/src/components/UploadModal/types.ts b/web/packages/common/src/components/UploadModal/types.ts index c5474028dd..645aa60842 100644 --- a/web/packages/common/src/components/UploadModal/types.ts +++ b/web/packages/common/src/components/UploadModal/types.ts @@ -72,6 +72,8 @@ export interface UploadModalProps { datasetLabel?: string; /** Auto-select the first root-level accepted file on fileset selection. */ autoSelectFirstAcceptable?: boolean; + /** Show each fileset's updated-at date in the picker options (opt-in). */ + showUpdatedAt?: boolean; attributes?: { ModalRoot?: React.ComponentProps; ModalContent?: React.ComponentProps; diff --git a/web/packages/studio/public/sample-agents/calculator/eval-config.json b/web/packages/studio/public/sample-agents/calculator/eval-config.json new file mode 100644 index 0000000000..5413edf2ce --- /dev/null +++ b/web/packages/studio/public/sample-agents/calculator/eval-config.json @@ -0,0 +1,89 @@ +{ + "tasks": [ + { + "id": "calc-0", + "intent": "Compute the product of 3 and 7 and compare it to the current hour", + "inputs": { + "instruction": "What is the product of 3 and 7, and is it greater than the current hour?" + }, + "reference": { + "product": 21 + } + }, + { + "id": "calc-1", + "intent": "Compute the sum of 12 and 8 and compare it to the current day of the week", + "inputs": { + "instruction": "Is the sum of 12 and 8 greater than the current day of the week in number form?" + }, + "reference": { + "sum": 20 + } + }, + { + "id": "calc-2", + "intent": "Compute the difference between 50 and 35 and compare it to the current minute", + "inputs": { + "instruction": "What is the difference between 50 and 35, and is it smaller than the current minute of the hour?" + }, + "reference": { + "difference": 15 + } + } + ], + "metric": { + "bundle_kind": "metric-bundle", + "bundle_format_version": "v1", + "metric_type": "llm-judge", + "metadata": { + "description": null, + "labels": {} + }, + "outputs": [ + { + "name": "accuracy", + "description": null, + "value_json_schema": { + "description": "Continuous numeric metric value.", + "title": "ContinuousScore", + "type": "number" + } + } + ], + "secrets": {}, + "payload": { + "kind": "inline", + "metric": { + "type": "llm-judge", + "model": "default/nvidia-nemotron-3-super-120b-a12b", + "prompt_template": { + "messages": [ + { + "role": "user", + "content": "You are scoring a calculator agent that performs arithmetic and datetime comparisons.\n\nThe expected numeric result for this task is: {{ item.reference }}.\n\nAgent response:\n{{ sample.output_text }}\n\nScore whether the agent's response contains the correct numeric result and a valid datetime comparison.\n- accuracy = 1 if the response contains the correct number (e.g. 21 for 3*7) and makes a valid comparison to the current time value.\n- accuracy = 0 if the numeric result is wrong or the comparison is missing or nonsensical.\n- accuracy = 0.5 if the numeric result is correct but the datetime comparison is absent or unclear.\n\nRespond with a JSON object {\"accuracy\": } where accuracy is 0, 0.5, or 1." + } + ] + }, + "scores": [ + { + "name": "accuracy", + "minimum": 0, + "maximum": 1 + } + ], + "inference": { + "max_tokens": 1024, + "extra_body": { + "nvext": { + "max_thinking_tokens": 256 + } + } + }, + "reasoning": { + "end_token": "" + } + } + } + }, + "max_concurrent_tasks": 1 +} diff --git a/web/packages/studio/public/sample-agents/email-phishing-analyzer/agent.yml b/web/packages/studio/public/sample-agents/email-phishing-analyzer/agent.yml index f963686ddf..688723ced1 100644 --- a/web/packages/studio/public/sample-agents/email-phishing-analyzer/agent.yml +++ b/web/packages/studio/public/sample-agents/email-phishing-analyzer/agent.yml @@ -20,7 +20,7 @@ llms: api_key: not-used model_name: ${NEMO_DEFAULT_MODEL} temperature: 0.0 - max_tokens: 512 + max_tokens: 1024 workflow: _type: tool_calling_agent tool_names: [email_phishing_analyzer] diff --git a/web/packages/studio/public/sample-agents/email-phishing-analyzer/eval-config.json b/web/packages/studio/public/sample-agents/email-phishing-analyzer/eval-config.json new file mode 100644 index 0000000000..ebfd2e08bb --- /dev/null +++ b/web/packages/studio/public/sample-agents/email-phishing-analyzer/eval-config.json @@ -0,0 +1,199 @@ +{ + "tasks": [ + { + "id": "email-0", + "intent": "Classify this email as phishing or benign", + "inputs": { + "instruction": "Subject: Claim Your Free iPhone Now!\nFrom: prize@example.com\n\nDear valued customer,\nCongratulations! You have been selected to receive a brand new iPhone absolutely free. To claim your prize, simply click the link below and provide your shipping address.\nhttp://malicious-link.example.com/claim\nThis offer is limited, so act fast!" + }, + "reference": { + "label": "phishing" + }, + "metadata": [ + { + "key": "arrival_time", + "value": "2023-05-14 10:15:30" + }, + { + "key": "intents", + "value": "{'money': {'label': 'Money', 'id': 0, 'score': 0.9998}, 'banking': {'label': 'Personal', 'id': 1, 'score': 0.9997}, 'crypto': {'label': 'NonCrypto', 'id': 1, 'score': 0.9996}}" + }, + { + "key": "source", + "value": "gift" + }, + { + "key": "extra_info", + "value": "unverified" + } + ] + }, + { + "id": "email-1", + "intent": "Classify this email as phishing or benign", + "inputs": { + "instruction": "Subject: Urgent: Your Account Has Been Suspended\nFrom: security-alerts@bank.com\n\nHello,\nWe have detected unusual activity on your account. To prevent suspension, please verify your identity by clicking the link below and entering your credentials.\nhttp://verify-account.example.com\nIf you do not verify within 24 hours, your account will be disabled.\nThank you,\nSupport Team" + }, + "reference": { + "label": "phishing" + }, + "metadata": [ + { + "key": "arrival_time", + "value": "2023-06-22 14:07:12" + }, + { + "key": "intents", + "value": "{'money': {'label': 'Money', 'id': 0, 'score': 0.9999}, 'banking': {'label': 'Personal', 'id': 1, 'score': 0.9999}, 'crypto': {'label': 'NonCrypto', 'id': 1, 'score': 0.9995}}" + }, + { + "key": "source", + "value": "password" + }, + { + "key": "extra_info", + "value": "suspicious" + } + ] + }, + { + "id": "email-2", + "intent": "Classify this email as phishing or benign", + "inputs": { + "instruction": "Subject: Important: Invoice Attached\nFrom: accounts@shop-example.com\n\nHi there,\nPlease find the invoice attached for your recent purchase. Click here to view the details.\nhttp://invoice-example.com/view?invoice=12345\nIf you have any questions, feel free to contact us.\nBest regards,\nCustomer Service" + }, + "reference": { + "label": "phishing" + }, + "metadata": [ + { + "key": "arrival_time", + "value": "2023-07-01 09:30:45" + }, + { + "key": "intents", + "value": "{'money': {'label': 'Money', 'id': 0, 'score': 0.9997}, 'banking': {'label': 'Personal', 'id': 1, 'score': 0.9998}, 'crypto': {'label': 'NonCrypto', 'id': 1, 'score': 0.9994}}" + }, + { + "key": "source", + "value": "money" + }, + { + "key": "extra_info", + "value": "pending" + } + ] + }, + { + "id": "email-3", + "intent": "Classify this email as phishing or benign", + "inputs": { + "instruction": "Subject: Project Meeting Reminder\nFrom: bob@example.com\n\nHi Team,\nJust wanted to remind you about our project update meeting on Friday at 2pm. Please let me know if you can attend.\nThanks!\n-Bob" + }, + "reference": { + "label": "benign" + }, + "metadata": [ + { + "key": "arrival_time", + "value": "2023-08-10 15:30:00" + }, + { + "key": "intents", + "value": "{'money': {'label': 'NonMoney', 'id': 1, 'score': 0.9995}, 'banking': {'label': 'NonPersonal', 'id': 1, 'score': 0.9995}, 'crypto': {'label': 'NonCrypto', 'id': 1, 'score': 0.9994}}" + }, + { + "key": "source", + "value": "meeting" + }, + { + "key": "extra_info", + "value": "trusted" + } + ] + }, + { + "id": "email-4", + "intent": "Classify this email as phishing or benign", + "inputs": { + "instruction": "Subject: Invoice Follow-up\nFrom: alice@company.com\n\nHi John,\nPlease find the invoice #1234 attached for your recent purchase. Let me know if you have any questions.\nBest regards,\nAlice" + }, + "reference": { + "label": "benign" + }, + "metadata": [ + { + "key": "arrival_time", + "value": "2023-09-01 09:15:22" + }, + { + "key": "intents", + "value": "{'money': {'label': 'Money', 'id': 0, 'score': 0.9996}, 'banking': {'label': 'Personal', 'id': 1, 'score': 0.9996}, 'crypto': {'label': 'NonCrypto', 'id': 1, 'score': 0.9993}}" + }, + { + "key": "source", + "value": "finance" + }, + { + "key": "extra_info", + "value": "trusted" + } + ] + } + ], + "metric": { + "bundle_kind": "metric-bundle", + "bundle_format_version": "v1", + "metric_type": "llm-judge", + "metadata": { + "description": null, + "labels": {} + }, + "outputs": [ + { + "name": "accuracy", + "description": null, + "value_json_schema": { + "description": "Continuous numeric metric value.", + "title": "ContinuousScore", + "type": "number" + } + } + ], + "secrets": {}, + "payload": { + "kind": "inline", + "metric": { + "type": "llm-judge", + "model": "default/nvidia-nemotron-3-super-120b-a12b", + "prompt_template": { + "messages": [ + { + "role": "user", + "content": "You are scoring an email phishing classifier. The expected answer is the ground-truth label: \"{{ item.reference.label }}\". The agent's response may contain reasoning; extract ONLY its final verdict (the last clear \"phishing\" or \"benign\").\n\nAgent response:\n{{ sample.output_text }}\n\nRespond with a JSON object {\"accuracy\": } where accuracy is 1 if the extracted verdict matches the expected label, else 0." + } + ] + }, + "scores": [ + { + "name": "accuracy", + "minimum": 0, + "maximum": 1 + } + ], + "inference": { + "max_tokens": 1024, + "extra_body": { + "nvext": { + "max_thinking_tokens": 256 + } + } + }, + "reasoning": { + "end_token": "" + } + } + } + }, + "max_concurrent_tasks": 1 +} diff --git a/web/packages/studio/src/api/evaluation/agent-evaluations.test.ts b/web/packages/studio/src/api/evaluation/agent-evaluations.test.ts new file mode 100644 index 0000000000..85015a8b6e --- /dev/null +++ b/web/packages/studio/src/api/evaluation/agent-evaluations.test.ts @@ -0,0 +1,210 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AgentEvaluateJob } from '@nemo/sdk/generated/evaluator/schema'; +import { + type AgentEvalBundle, + type AgentEvalResult, + agentNameForJob, + aggregateScoresOf, + fetchAgentEvalJob, + fetchAgentEvalJobs, + joinBundleByTask, + parseBundleRef, +} from '@studio/api/evaluation/agent-evaluations'; + +const customFetchMock = vi.fn(); +vi.mock('@nemo/sdk/generated/fetchers/evaluator', () => ({ + customFetch: (...args: unknown[]) => customFetchMock(...args), +})); + +const filesDownloadFileMock = vi.fn(); +vi.mock('@nemo/sdk/generated/platform/api', () => ({ + filesDownloadFile: (...args: unknown[]) => filesDownloadFileMock(...args), +})); + +beforeEach(() => { + customFetchMock.mockReset(); + filesDownloadFileMock.mockReset(); +}); + +const baseJob = (overrides: Partial = {}): AgentEvaluateJob => + ({ + name: 'eval-1', + workspace: 'ws-a', + status: 'completed', + created_at: '2026-05-05T00:00:00Z', + updated_at: '2026-05-05T00:01:00Z', + spec: { target: { kind: 'agent', agent: { name: 'support-bot-mini' } }, tasks: [{}, {}] }, + ...overrides, + }) as AgentEvaluateJob; + +describe('fetchAgentEvalJobs', () => { + it('walks all pages until a short page ends pagination', async () => { + const page1 = Array.from({ length: 50 }, (_, i) => baseJob({ name: `j-${i}` })); + const page2 = Array.from({ length: 50 }, (_, i) => baseJob({ name: `j-${50 + i}` })); + const page3 = [baseJob({ name: 'j-100' })]; + customFetchMock + .mockResolvedValueOnce({ data: page1 }) + .mockResolvedValueOnce({ data: page2 }) + .mockResolvedValueOnce({ data: page3 }); + const all = await fetchAgentEvalJobs('ws-a', new AbortController().signal); + expect(all).toHaveLength(101); + expect(customFetchMock).toHaveBeenCalledTimes(3); + }); + + it('targets the agent-evaluate endpoint', async () => { + customFetchMock.mockResolvedValueOnce({ data: [] }); + await fetchAgentEvalJobs('ws-a', new AbortController().signal); + expect(customFetchMock).toHaveBeenCalledWith( + expect.objectContaining({ + url: '/apis/evaluator/v2/workspaces/ws-a/agent-evaluate/jobs', + }) + ); + }); +}); + +describe('fetchAgentEvalJob', () => { + it('returns the job when the platform responds with one', async () => { + customFetchMock.mockResolvedValueOnce(baseJob({ name: 'eval-42' })); + const job = await fetchAgentEvalJob('ws-a', 'eval-42', new AbortController().signal); + expect(job?.name).toBe('eval-42'); + }); + + it('returns null on 404', async () => { + customFetchMock.mockRejectedValueOnce({ response: { status: 404 } }); + const job = await fetchAgentEvalJob('ws-a', 'missing', new AbortController().signal); + expect(job).toBeNull(); + }); +}); + +describe('agentNameForJob', () => { + it('reads the agent name from the target', () => { + expect(agentNameForJob(baseJob())).toBe('support-bot-mini'); + }); + + it('returns null when no target agent is set', () => { + expect(agentNameForJob(baseJob({ spec: {} as AgentEvaluateJob['spec'] }))).toBeNull(); + }); +}); + +describe('aggregateScoresOf', () => { + it('flattens the nested scores shape', () => { + const result = { + scores: { + scores: [{ name: 'llm-judge.accuracy', count: 5, nan_count: 0, score_type: 'range' }], + }, + } as AgentEvalResult; + expect(aggregateScoresOf(result)).toHaveLength(1); + expect(aggregateScoresOf(null)).toEqual([]); + }); +}); + +describe('parseBundleRef', () => { + it('splits "workspace/fileset#inner/path"', () => { + expect(parseBundleRef('default/job-fileset-x#results/attempt-1/agent-eval-results')).toEqual({ + fileset: 'job-fileset-x', + innerPath: 'results/attempt-1/agent-eval-results', + }); + }); + + it('returns null without a fragment', () => { + expect(parseBundleRef('default/job-fileset-x')).toBeNull(); + }); +}); + +describe('joinBundleByTask', () => { + it('joins tasks, trials, and scores by task id', () => { + const bundle: AgentEvalBundle = { + tasks: [ + { + id: 'A', + intent: 'classify', + inputs: { instruction: 'email a' }, + reference: { label: 'phishing' }, + }, + ], + trials: [ + { id: 't1', task_id: 'A', status: 'completed', output: { output_text: 'phishing' } }, + ], + scores: [ + { + id: 's1', + task_id: 'A', + trial_id: 't1', + metric_type: 'llm-judge', + status: 'completed', + outputs: [{ name: 'accuracy', value: 1 }], + diagnostics: [], + }, + ], + }; + const [row] = joinBundleByTask(bundle); + expect(row.taskId).toBe('A'); + expect(row.responseText).toBe('phishing'); + expect(row.instruction).toBe('email a'); + expect(row.reference).toEqual({ label: 'phishing' }); + expect(row.scores).toEqual([{ name: 'llm-judge.accuracy', value: 1 }]); + }); + + it('returns [] for a null bundle', () => { + expect(joinBundleByTask(null)).toEqual([]); + }); + + it("attaches only the selected trial's scores when a task has multiple trials", () => { + const bundle: AgentEvalBundle = { + tasks: [{ id: 'A', inputs: { instruction: 'email a' } }], + // Two trials for task A; the join keeps the last one (t2). + trials: [ + { id: 't1', task_id: 'A', status: 'completed', output: { output_text: 'benign' } }, + { id: 't2', task_id: 'A', status: 'completed', output: { output_text: 'phishing' } }, + ], + scores: [ + { + id: 's1', + task_id: 'A', + trial_id: 't1', + metric_type: 'llm-judge', + status: 'completed', + outputs: [{ name: 'accuracy', value: 0 }], + diagnostics: ['t1-diag'], + }, + { + id: 's2', + task_id: 'A', + trial_id: 't2', + metric_type: 'llm-judge', + status: 'completed', + outputs: [{ name: 'accuracy', value: 1 }], + diagnostics: ['t2-diag'], + }, + ], + }; + const [row] = joinBundleByTask(bundle); + // responseText and scores/diagnostics must come from the same trial (t2). + expect(row.responseText).toBe('phishing'); + expect(row.scores).toEqual([{ name: 'llm-judge.accuracy', value: 1 }]); + expect(row.diagnostics).toEqual(['t2-diag']); + }); + + it('normalizes serialized NaN score values for display', () => { + const bundle: AgentEvalBundle = { + tasks: [{ id: 'A' }], + trials: [{ id: 't1', task_id: 'A', status: 'completed' }], + scores: [ + { + id: 's1', + task_id: 'A', + trial_id: 't1', + metric_type: 'llm-judge', + status: 'completed', + outputs: [{ name: 'accuracy', value: 'NaN' }], + }, + ], + }; + + expect(joinBundleByTask(bundle)[0].scores).toEqual([ + { name: 'llm-judge.accuracy', value: null }, + ]); + }); +}); diff --git a/web/packages/studio/src/api/evaluation/agent-evaluations.ts b/web/packages/studio/src/api/evaluation/agent-evaluations.ts new file mode 100644 index 0000000000..9eb9efbf05 --- /dev/null +++ b/web/packages/studio/src/api/evaluation/agent-evaluations.ts @@ -0,0 +1,248 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + evaluatorCancelAgentEvaluateJob, + evaluatorCreateAgentEvaluateJob, + evaluatorGetAgentEvalResult, + evaluatorGetAgentEvaluateJob, + evaluatorListAgentEvaluateJobs, +} from '@nemo/sdk/generated/evaluator/api'; +import type { + AggregateRangeScore, + AggregateRubricScore, + AgentEvaluateJob, + AgentEvaluateJobRequest, + AgentEvalResult, + AgentEvaluateJobsSortField, +} from '@nemo/sdk/generated/evaluator/schema'; +import { filesDownloadFile } from '@nemo/sdk/generated/platform/api'; + +const PAGE_SIZE = 50; + +/** Aggregate score — numeric range or rubric category distribution. */ +export type AgentEvalAggregateScore = AggregateRangeScore | AggregateRubricScore; + +/** Re-export so callers continue to import AgentEvalResult from this module. */ +export type { AgentEvalResult }; + +/** The agent name a job evaluated, read from its target (spec.target.agent.name). + * Strips only this job's own ``workspace/`` prefix so the result compares equal to a + * bare agent name; any other ``/`` in the name is left intact. */ +export const agentNameForJob = (job: AgentEvaluateJob): string | null => { + const target = job.spec.target as { kind?: string; agent?: { name?: string } } | null | undefined; + if (!target || target.kind !== 'agent') return null; + const name = target.agent?.name; + if (typeof name !== 'string' || name.length === 0) return null; + const prefix = job.workspace ? `${job.workspace}/` : ''; + return prefix && name.startsWith(prefix) ? name.slice(prefix.length) : name; +}; + +export const fetchAgentEvalJobs = async ( + workspace: string, + signal: AbortSignal +): Promise => { + const all: AgentEvaluateJob[] = []; + let page = 1; + while (true) { + const res = await evaluatorListAgentEvaluateJobs( + workspace, + { page, page_size: PAGE_SIZE, sort: '-created_at' as AgentEvaluateJobsSortField }, + signal + ); + const batch = res?.data ?? []; + all.push(...batch); + if (batch.length < PAGE_SIZE) break; + page++; + } + return all; +}; + +export const fetchAgentEvalJob = async ( + workspace: string, + name: string, + signal: AbortSignal +): Promise => { + try { + return await evaluatorGetAgentEvaluateJob(workspace, name, signal); + } catch (err) { + const e = err as { response?: { status?: number }; status?: number }; + if (e?.response?.status === 404 || e?.status === 404) return null; + throw err; + } +}; + +export const cancelAgentEvalJob = async ( + workspace: string, + name: string, + signal: AbortSignal +): Promise => { + await evaluatorCancelAgentEvaluateJob(workspace, name, signal); +}; + +export const submitAgentEvalJob = async ( + workspace: string, + request: AgentEvaluateJobRequest, + signal?: AbortSignal +): Promise => evaluatorCreateAgentEvaluateJob(workspace, request, signal); + +// --------------------------------------------------------------------------- +// Structured results (agent-eval-results record) +// --------------------------------------------------------------------------- + +/** Flatten a result to its aggregate score rows (empty when none/absent). */ +export const aggregateScoresOf = (result: AgentEvalResult | null): AgentEvalAggregateScore[] => + result?.scores?.scores ?? []; + +export const fetchAgentEvalResult = async ( + workspace: string, + name: string, + signal: AbortSignal +): Promise => { + try { + return await evaluatorGetAgentEvalResult(workspace, name, signal); + } catch (err) { + const e = err as { response?: { status?: number }; status?: number }; + if (e?.response?.status === 404 || e?.status === 404) return null; + throw err; + } +}; + +/** One trial row from trials.jsonl — the agent's response to a task. */ +export interface AgentEvalTrialRow { + id: string; + task_id: string; + status: string; + output?: { output_text?: string | null } | null; + evidence?: unknown; + metadata?: Record; +} + +/** One score row from scores.jsonl — a task's metric outputs + diagnostics. */ +export interface AgentEvalScoreRow { + id: string; + task_id: string; + trial_id: string; + metric_type: string; + status: string; + outputs?: Array<{ name: string; value: number | 'NaN' | null }>; + diagnostics?: unknown[]; + metadata?: Record; +} + +/** One task row from tasks.jsonl — the evaluated input + ground truth. */ +export interface AgentEvalTaskRow { + id: string; + intent?: string; + inputs?: { instruction?: string | null }; + reference?: Record; + metadata?: Record; +} + +export interface AgentEvalBundle { + tasks: AgentEvalTaskRow[]; + trials: AgentEvalTrialRow[]; + scores: AgentEvalScoreRow[]; +} + +/** Split a bundle_ref ("workspace/fileset#inner/path") into its parts. */ +export const parseBundleRef = ( + bundleRef: string +): { fileset: string; innerPath: string } | null => { + const [location, innerPath] = bundleRef.split('#'); + if (!location || !innerPath) return null; + const fileset = location.includes('/') + ? (location.split('/').slice(1).join('/') ?? '') + : location; + if (!fileset) return null; + return { fileset, innerPath: innerPath.replace(/^\/+/, '') }; +}; + +const downloadJsonl = async ( + workspace: string, + fileset: string, + remotePath: string, + signal: AbortSignal +): Promise => { + const blob = await filesDownloadFile(workspace, fileset, remotePath, signal); + if (!blob) return []; + const text = await blob.text(); + return text + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as T); +}; + +/** Loads the per-task bundle (tasks/trials/scores) for a completed job. Returns + * null when the bundle is not referenced or cannot be read (job not finished). */ +export const fetchAgentEvalBundle = async ( + workspace: string, + bundleRef: string | undefined, + signal: AbortSignal +): Promise => { + if (!bundleRef) return null; + const parsed = parseBundleRef(bundleRef); + if (!parsed) return null; + const { fileset, innerPath } = parsed; + const at = (name: string): string => `${innerPath}/${name}`; + try { + const [tasks, trials, scores] = await Promise.all([ + downloadJsonl(workspace, fileset, at('tasks.jsonl'), signal), + downloadJsonl(workspace, fileset, at('trials.jsonl'), signal), + downloadJsonl(workspace, fileset, at('scores.jsonl'), signal), + ]); + return { tasks, trials, scores }; + } catch (err) { + const e = err as { response?: { status?: number }; status?: number }; + if (e?.response?.status === 404 || e?.status === 404) return null; + throw err; + } +}; + +/** A per-task row joining the task, its trial (response), and its score(s). */ +export interface AgentEvalTaskDetail { + taskId: string; + intent?: string; + instruction?: string | null; + reference?: Record; + metadata?: Record; + status: string; + responseText?: string | null; + scores: Array<{ name: string; value: number | null }>; + diagnostics: unknown[]; +} + +/** Join a bundle's tasks/trials/scores into one per-task row list. */ +export const joinBundleByTask = (bundle: AgentEvalBundle | null): AgentEvalTaskDetail[] => { + if (!bundle) return []; + const trialByTask = new Map(bundle.trials.map((t) => [t.task_id, t])); + const scoresByTask = new Map(); + for (const s of bundle.scores) { + const list = scoresByTask.get(s.task_id) ?? []; + list.push(s); + scoresByTask.set(s.task_id, list); + } + return bundle.tasks.map((task) => { + const trial = trialByTask.get(task.id); + // Keep only the selected trial's scores: with multiple trials per task, scores/diagnostics + // must come from the same trial as the displayed responseText, not every trial's. + const taskScores = (scoresByTask.get(task.id) ?? []).filter((s) => s.trial_id === trial?.id); + return { + taskId: task.id, + intent: task.intent, + instruction: task.inputs?.instruction ?? null, + reference: task.reference, + metadata: task.metadata, + status: trial?.status ?? 'unknown', + responseText: trial?.output?.output_text ?? null, + scores: taskScores.flatMap((s) => + (s.outputs ?? []).map((o) => ({ + name: `${s.metric_type}.${o.name}`, + value: typeof o.value === 'number' && Number.isFinite(o.value) ? o.value : null, + })) + ), + diagnostics: taskScores.flatMap((s) => s.diagnostics ?? []), + }; + }); +}; diff --git a/web/packages/studio/src/api/evaluation/eval-config-fileset.ts b/web/packages/studio/src/api/evaluation/eval-config-fileset.ts new file mode 100644 index 0000000000..2184ba1971 --- /dev/null +++ b/web/packages/studio/src/api/evaluation/eval-config-fileset.ts @@ -0,0 +1,141 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + filesCreateFileset, + filesListFilesetFiles, + filesUploadFile, +} from '@nemo/sdk/generated/platform/api'; + +// Idempotent seeding of an eval-config fileset, shared by the agent-evaluation +// submit flow (which supplies its own `eval-config.json`) and the optimizer +// apply flow (which relies on the bundled react-eval default below). + +const isNotFoundError = (err: unknown): boolean => { + const e = err as { response?: { status?: number }; status?: number }; + return e?.response?.status === 404 || e?.status === 404; +}; + +const isConflictError = (err: unknown): boolean => { + const e = err as { response?: { status?: number }; status?: number }; + return e?.response?.status === 409 || e?.status === 409; +}; + +const isCanceledError = (err: unknown): boolean => { + const e = err as { name?: string; code?: string }; + return e?.name === 'AbortError' || e?.name === 'CanceledError' || e?.code === 'ERR_CANCELED'; +}; + +export const SAMPLE_EVAL_CONFIG_PATH = 'react-eval.yml'; +export const SAMPLE_EVAL_DATA_PATH = 'react-eval-data.json'; + +export const SAMPLE_EVAL_YAML = `# react-eval.yml — bundled sample seeded by the optimizer apply flow. +# +# Evaluates against the deployed agent endpoint. The judge LLM scores answers +# and must be available in the workspace. + +llms: + llm: + _type: openai + model_name: nvidia-nemotron-3-nano-30b-a3b + temperature: 0.0 + max_tokens: 1024 + + judge_llm: + _type: openai + model_name: nvidia-nemotron-3-super-120b-a12b + temperature: 0.0 + max_tokens: 1024 + +eval: + general: + max_concurrency: 4 + output_dir: eval/agent + dataset: + _type: json + file_path: ${SAMPLE_EVAL_DATA_PATH} + evaluators: + accuracy: + _type: tunable_rag_evaluator + llm_name: judge_llm + default_scoring: true + default_score_weights: + coverage: 0.5 + correctness: 0.3 + relevance: 0.2 + judge_llm_prompt: > + You are an evaluator. Score whether the generated answer correctly + addresses the question compared to the expected answer description. + Rules: + - Score is a float between 0.0 and 1.0. + - 1.0 means the answer fully satisfies the expected answer criteria. + - Provide a 1-2 sentence reasoning. +`; + +export const SAMPLE_EVAL_DATA_JSON = JSON.stringify( + [ + { + id: 1, + question: 'Who invented the telephone, and what is the current time?', + answer: + 'Answer must mention Alexander Graham Bell as the inventor of the telephone and include the current time', + }, + { + id: 2, + question: 'What is the capital of France, and what day of the week is it today?', + answer: + 'Answer must state that the capital of France is Paris and include the current day of the week', + }, + { + id: 3, + question: "When was the theory of general relativity published, and what is today's date?", + answer: + "Answer must mention 1915 as the year general relativity was published and include today's date", + }, + ], + null, + 2 +); + +export interface EvalSeedFile { + path: string; + content: string; + type: string; +} + +/** Default seed files: the bundled react sample. Used by the optimizer apply + * flow and by the eval modal's fallback. */ +const defaultEvalSeedFiles = (): EvalSeedFile[] => [ + { path: SAMPLE_EVAL_CONFIG_PATH, content: SAMPLE_EVAL_YAML, type: 'application/yaml' }, + { path: SAMPLE_EVAL_DATA_PATH, content: SAMPLE_EVAL_DATA_JSON, type: 'application/json' }, +]; + +export const ensureEvalConfigFileset = async ( + workspace: string, + fileset: string, + signal: AbortSignal, + files: EvalSeedFile[] = defaultEvalSeedFiles(), + description?: string +): Promise => { + let existingPaths = new Set(); + try { + const listing = await filesListFilesetFiles(workspace, fileset, undefined, signal); + existingPaths = new Set((listing?.data ?? []).map((f) => f.path)); + } catch (err) { + if (isCanceledError(err)) throw err; + if (!isNotFoundError(err)) throw err; + try { + await filesCreateFileset(workspace, { name: fileset, description }, signal); + } catch (createErr) { + if (isCanceledError(createErr)) throw createErr; + // Ignore only 409 (parallel apply already created it); surface everything else. + if (!isConflictError(createErr)) throw createErr; + } + } + // Idempotent: never overwrite files already present in the fileset. + const uploads = files.filter((f) => !existingPaths.has(f.path)); + for (const u of uploads) { + const blob = new Blob([u.content], { type: u.type }); + await filesUploadFile(workspace, fileset, u.path, blob, signal); + } +}; diff --git a/web/packages/studio/src/components/agents/AgentBlockingInput/EvalConfigBlockingInput.test.tsx b/web/packages/studio/src/components/agents/AgentBlockingInput/EvalConfigBlockingInput.test.tsx index fa5d4a01d1..2789d2bc8a 100644 --- a/web/packages/studio/src/components/agents/AgentBlockingInput/EvalConfigBlockingInput.test.tsx +++ b/web/packages/studio/src/components/agents/AgentBlockingInput/EvalConfigBlockingInput.test.tsx @@ -1,13 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { SAMPLE_EVAL_CONFIG_PATH } from '@studio/api/evaluation/eval-config-fileset'; import { EvalConfigBlockingInput } from '@studio/components/agents/AgentBlockingInput/EvalConfigBlockingInput'; import type { AgentBlockingInputRequest, AgentBlockingInputSecondaryAction, AgentBlockingInputSubmission, } from '@studio/components/agents/AgentBlockingInput/types'; -import { SAMPLE_EVAL_CONFIG_PATH } from '@studio/routes/agents/AgentSuggestionsRoute/constants'; import { render } from '@testing-library/react'; interface CapturedFilesetProps { diff --git a/web/packages/studio/src/components/agents/AgentBlockingInput/EvalConfigBlockingInput.tsx b/web/packages/studio/src/components/agents/AgentBlockingInput/EvalConfigBlockingInput.tsx index aa20454ed5..be44bb9159 100644 --- a/web/packages/studio/src/components/agents/AgentBlockingInput/EvalConfigBlockingInput.tsx +++ b/web/packages/studio/src/components/agents/AgentBlockingInput/EvalConfigBlockingInput.tsx @@ -1,10 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { SAMPLE_EVAL_CONFIG_PATH } from '@studio/api/evaluation/eval-config-fileset'; import { FilesetFileBlockingInput } from '@studio/components/agents/AgentBlockingInput/FilesetFileBlockingInput'; import type { FilesetFileBlockingInputProps } from '@studio/components/agents/AgentBlockingInput/types'; import { getStringValue } from '@studio/components/agents/AgentBlockingInput/utils'; -import { SAMPLE_EVAL_CONFIG_PATH } from '@studio/routes/agents/AgentSuggestionsRoute/constants'; import { evalFilesetForAgent } from '@studio/routes/agents/AgentSuggestionsRoute/utils'; import type { FC } from 'react'; diff --git a/web/packages/studio/src/components/evaluation/JudgeModelSelect.tsx b/web/packages/studio/src/components/evaluation/JudgeModelSelect.tsx index 4880f9f2b7..c7f808b430 100644 --- a/web/packages/studio/src/components/evaluation/JudgeModelSelect.tsx +++ b/web/packages/studio/src/components/evaluation/JudgeModelSelect.tsx @@ -66,7 +66,12 @@ export const JudgeModelSelect = }; return ( - + = ({ {job.name} - + diff --git a/web/packages/studio/src/components/sidePanels/AgentPanels/AgentPanel/index.test.tsx b/web/packages/studio/src/components/sidePanels/AgentPanels/AgentPanel/index.test.tsx index b3c25646ec..5351eb8cbd 100644 --- a/web/packages/studio/src/components/sidePanels/AgentPanels/AgentPanel/index.test.tsx +++ b/web/packages/studio/src/components/sidePanels/AgentPanels/AgentPanel/index.test.tsx @@ -8,22 +8,32 @@ import { TestProviders } from '@studio/tests/util/TestProviders'; import { render, screen } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { http, HttpResponse } from 'msw'; -import { MemoryRouter } from 'react-router-dom'; +import { type ReactNode } from 'react'; +import { MemoryRouter, Route, Routes } from 'react-router-dom'; // These match mock agent names in handlers.ts const MOCK_AGENT_WITH_DEPLOYMENTS = 'react-agent'; // has rag-agent-prod (running) + sql-agent-dev (stopped) const MOCK_AGENT_WITH_ERROR_DEPLOYMENT = 'react-agent2'; // has chat-agent-staging (error) const MOCK_AGENT_UNKNOWN = 'unknown-agent'; -const renderPanel = (agentName?: string, open = true) => +// Render under a workspace route so route-aware hooks (e.g. useWorkspaceFromPath +// in the modal's JudgeModelSelect) resolve the ":workspace" param. +const renderInWorkspace = (node: ReactNode) => render( - - + + + + ); +const renderPanel = (agentName?: string, open = true) => + renderInWorkspace( + + ); + describe('AgentPanel', () => { describe('when closed', () => { it('renders nothing visible when open is false', () => { @@ -145,18 +155,14 @@ describe('AgentPanel', () => { describe('defaultTab prop', () => { it('opens on the Chat Playground tab when defaultTab is chat-playground', () => { - render( - - - - - + renderInWorkspace( + ); expect(screen.getByRole('radio', { name: 'Chat Playground' })).toBeChecked(); @@ -167,18 +173,14 @@ describe('AgentPanel', () => { it('shows a Deploy this Agent action when the agent has no healthy deployments', async () => { const user = userEvent.setup(); // react-agent2 has only chat-agent-staging (status=error) → no healthy deployments - render( - - - - - + renderInWorkspace( + ); expect( @@ -213,18 +215,14 @@ describe('AgentPanel', () => { ) ); - render( - - - - - + renderInWorkspace( + ); expect( @@ -239,17 +237,13 @@ describe('AgentPanel', () => { const user = userEvent.setup(); const onOpenChange = vi.fn(); - render( - - - - - + renderInWorkspace( + ); const closeButton = screen.getByRole('button', { name: /close/i }); diff --git a/web/packages/studio/src/components/sidePanels/AgentPanels/AgentPanel/useAgentPanel.ts b/web/packages/studio/src/components/sidePanels/AgentPanels/AgentPanel/useAgentPanel.ts index 11b0d8f241..5abc1f3dc4 100644 --- a/web/packages/studio/src/components/sidePanels/AgentPanels/AgentPanel/useAgentPanel.ts +++ b/web/packages/studio/src/components/sidePanels/AgentPanels/AgentPanel/useAgentPanel.ts @@ -9,8 +9,8 @@ import { useAgentsListAgents, useAgentsListDeployments, } from '@nemo/sdk/generated/agents/api'; +import { agentNameForJob, fetchAgentEvalJobs } from '@studio/api/evaluation/agent-evaluations'; import { RECENT_EVAL_LIMIT } from '@studio/components/sidePanels/AgentPanels/AgentPanel/constants'; -import { fetchAgentEvalJobs } from '@studio/routes/agents/AgentEvaluationsRoute/api'; import { useQuery, useQueryClient } from '@tanstack/react-query'; import { useMemo } from 'react'; @@ -90,12 +90,7 @@ export const useAgentPanel = ({ if (!agentName) return []; const all = agentEvalsData ?? []; // Match either the bare agent name or a workspace-prefixed ref. - const matches = all.filter((job) => { - const a = job.spec.agent; - if (typeof a !== 'string') return false; - const bare = a.includes('/') ? a.split('/').pop() : a; - return a === agentName || bare === agentName; - }); + const matches = all.filter((job) => agentNameForJob(job) === agentName); return matches.slice(0, RECENT_EVAL_LIMIT); }, [agentEvalsData, agentName]); diff --git a/web/packages/studio/src/constants/sampleAgents.test.ts b/web/packages/studio/src/constants/sampleAgents.test.ts index e2e236ed16..27c410f921 100644 --- a/web/packages/studio/src/constants/sampleAgents.test.ts +++ b/web/packages/studio/src/constants/sampleAgents.test.ts @@ -2,6 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 import { + EVALUATION_SAMPLE_AGENTS, + evaluationSampleAgentKeyForAgentName, isSampleAgentName, SAMPLE_AGENTS, sampleAgentKeyForAgentName, @@ -65,3 +67,17 @@ describe('isSampleAgentName', () => { expect(isSampleAgentName('calculator-demo-agentxyz')).toBe(false); }); }); + +describe('evaluation samples', () => { + it('keeps creation-only samples out of the evaluation picker', () => { + expect(SAMPLE_AGENTS.some((agent) => agent.key === 'calculator')).toBe(true); + expect(EVALUATION_SAMPLE_AGENTS.map((agent) => agent.key)).toEqual([ + 'calculator', + 'email_phishing_analyzer', + ]); + expect(evaluationSampleAgentKeyForAgentName('calculator-demo-agent-abc123')).toBe('calculator'); + expect(evaluationSampleAgentKeyForAgentName('email-phishing-demo-agent-abc123')).toBe( + 'email_phishing_analyzer' + ); + }); +}); diff --git a/web/packages/studio/src/constants/sampleAgents.ts b/web/packages/studio/src/constants/sampleAgents.ts index 1a1bd79908..575514e43d 100644 --- a/web/packages/studio/src/constants/sampleAgents.ts +++ b/web/packages/studio/src/constants/sampleAgents.ts @@ -6,8 +6,8 @@ import { z } from 'zod'; // Registry of canned example agents. Each entry references curated static assets // under public/sample-agents// by path (fetched on demand, never bundled) — // mirroring src/constants/sampleDatasets.ts. Used by both the Create Example Agent -// modal (fetch + parse agent.yml, inject model, POST) and the Run Evaluation modal -// (seed eval.yml + dataset into the {agent}-eval fileset). +// modal (fetch + parse agent.yml, inject model, POST). Samples with an +// evalConfigPath also appear in the Run Evaluation modal. // // INVARIANT: an entry whose agent.yml uses a custom NAT `_type` requires that // tool's Python package to be installed in the deploy venv, or the deployment @@ -23,12 +23,9 @@ export interface SampleAgent { namePrefix: string; /** Public path to the NAT workflow config (parsed + model-injected at create). */ agentConfigPath: string; - /** Public path to the NAT eval config (seeded verbatim into the eval fileset - * under its basename). */ - evalConfigPath: string; - /** Public path to the eval dataset, seeded alongside the eval config under its - * basename. That basename MUST equal the eval config's dataset file_path. */ - evalDataPath: string; + /** Public path to a reusable nemo-evaluator eval-config.json. Samples without + * one remain available for agent creation but not evaluation seeding. */ + evalConfigPath?: string; } export const SAMPLE_AGENTS: SampleAgent[] = [ @@ -38,8 +35,7 @@ export const SAMPLE_AGENTS: SampleAgent[] = [ description: 'A ReAct agent with a calculator and datetime tool.', namePrefix: 'calculator-demo-agent', agentConfigPath: 'sample-agents/calculator/agent.yml', - evalConfigPath: 'sample-agents/calculator/eval.yml', - evalDataPath: 'sample-agents/calculator/calculator-eval-data.json', + evalConfigPath: 'sample-agents/calculator/eval-config.json', }, { key: 'email_phishing_analyzer', @@ -47,16 +43,31 @@ export const SAMPLE_AGENTS: SampleAgent[] = [ description: 'A ReAct agent that inspects an email body for phishing signals.', namePrefix: 'email-phishing-demo-agent', agentConfigPath: 'sample-agents/email-phishing-analyzer/agent.yml', - evalConfigPath: 'sample-agents/email-phishing-analyzer/eval.yml', - evalDataPath: 'sample-agents/email-phishing-analyzer/smaller_test.csv', + evalConfigPath: 'sample-agents/email-phishing-analyzer/eval-config.json', }, ]; +export type EvaluationSampleAgent = SampleAgent & { evalConfigPath: string }; + +export const EVALUATION_SAMPLE_AGENTS = SAMPLE_AGENTS.filter( + (agent): agent is EvaluationSampleAgent => typeof agent.evalConfigPath === 'string' +); + export const DEFAULT_SAMPLE_AGENT_KEY = SAMPLE_AGENTS[0].key; export const getSampleAgent = (key: string): SampleAgent => SAMPLE_AGENTS.find((agent) => agent.key === key) ?? SAMPLE_AGENTS[0]; +export const getEvaluationSampleAgent = (key: string): EvaluationSampleAgent => + EVALUATION_SAMPLE_AGENTS.find((agent) => agent.key === key) ?? EVALUATION_SAMPLE_AGENTS[0]; + +export const evaluationSampleAgentKeyForAgentName = ( + name: string | undefined +): string | undefined => { + const key = sampleAgentKeyForAgentName(name); + return EVALUATION_SAMPLE_AGENTS.some((agent) => agent.key === key) ? key : undefined; +}; + export const buildSampleAgentName = (namePrefix: string): string => `${namePrefix}-${Math.random().toString(36).slice(2, 8)}`; diff --git a/web/packages/studio/src/mocks/handlers.ts b/web/packages/studio/src/mocks/handlers.ts index 88b4d9dc9a..c93766e05d 100644 --- a/web/packages/studio/src/mocks/handlers.ts +++ b/web/packages/studio/src/mocks/handlers.ts @@ -13,6 +13,7 @@ import { import { PLATFORM_BASE_URL } from '@studio/constants/environment'; import { customizerHandlers } from '@studio/mocks/handlers/customizer'; import { deploymentsHandlers } from '@studio/mocks/handlers/deployments'; +import { evaluatorHandlers } from '@studio/mocks/handlers/evaluator'; import { filesetsHandlers } from '@studio/mocks/handlers/filesets'; import { guardrailsHandlers } from '@studio/mocks/handlers/guardrails'; import { modelsHandlers } from '@studio/mocks/handlers/models'; @@ -700,6 +701,7 @@ export const handlers = [ ...workspacesHandlers, ...customizerHandlers, ...deploymentsHandlers, + ...evaluatorHandlers, ...modelsHandlers, ...secretsHandlers, ...filesetsHandlers, diff --git a/web/packages/studio/src/mocks/handlers/evaluator.ts b/web/packages/studio/src/mocks/handlers/evaluator.ts new file mode 100644 index 0000000000..46d580058c --- /dev/null +++ b/web/packages/studio/src/mocks/handlers/evaluator.ts @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { PLATFORM_BASE_URL } from '@studio/constants/environment'; +import { http, HttpResponse } from 'msw'; + +export const evaluatorHandlers = [ + // nemo-evaluator agent-evaluate jobs — Studio's agent-evaluation feature runs + // here. Default to an empty list / 404 so the list + detail routes render + // their empty/not-found states in tests without per-test overrides. + http.get(`${PLATFORM_BASE_URL}/apis/evaluator/v2/workspaces/:workspace/agent-evaluate/jobs`, () => + HttpResponse.json({ data: [], pagination: { total: 0, page: 1, page_size: 50 } }) + ), + http.get( + `${PLATFORM_BASE_URL}/apis/evaluator/v2/workspaces/:workspace/agent-evaluate/jobs/:name`, + () => HttpResponse.json({ detail: 'Not found' }, { status: 404 }) + ), + http.get( + `${PLATFORM_BASE_URL}/apis/evaluator/v2/workspaces/:workspace/agent-evaluate/jobs/:name/status`, + () => HttpResponse.json({ name: '', status: 'unknown' }) + ), + http.get( + `${PLATFORM_BASE_URL}/apis/evaluator/v2/workspaces/:workspace/agent-eval-results/:name`, + () => HttpResponse.json({ detail: 'Not found' }, { status: 404 }) + ), +]; diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AGENTS.md b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AGENTS.md new file mode 100644 index 0000000000..8df57f69d6 --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AGENTS.md @@ -0,0 +1,320 @@ +# Agent Evaluation + +How Studio runs agent evaluations. + +## Overview + +- **Runner:** nemo-evaluator's **`agent-evaluate/jobs`** endpoint (task-based, `AgentEvaluateJob`), + with an **agent** target. This is the canonical path for Studio agent evaluation, per the + nemo-evaluator team — it is purpose-built for agent tasks with extensive artifact + trace + collection. (A legacy row-based `evaluate/jobs` endpoint also exists — see the bottom.) +- **Sample agents:** `public/sample-agents//` is the seed repository (agent config + + an `eval-config.json`). Studio reads these for the "use example" flow. +- **Metrics:** Studio creates metrics **only** as `InlineMetricPayload` — a built-in metric + serialized to JSON, reconstructed at runtime. No `CloudPickleMetricPayload` (no Python, no + pickled code). + +## Tasks, metrics, scores + +Three levels, don't conflate them: + +- **task** — one input case = one agent call (the "unit of work being evaluated"). The sample + has 5 tasks (5 emails). +- **metric** — a scorer applied to a task's output; a task can carry more than one. +- **score** — a value a metric emits; one metric can emit many (a judge can rate several + dimensions from one call). + +The evaluation loop is `task × trial × metric → score`. + +## Reusable eval configs (Filesets) + +nemo-evaluator has no "eval config" entity, so a reusable config is stored as an +**`eval-config.json`** file in a named **Fileset**. The tasks are stored **inline** in the +config (the `agent-evaluate` spec takes inline `tasks[]`; a Task/Taskset _reference_ is planned +but not yet implemented, so large datasets are inlined for now). + +Two shapes are involved: + +- **Example template** (`public/sample-agents/*/eval-config.json`): a shared-metric layout — + `tasks[]` (no per-task metrics) + one top-level `metric` (no judge_model). Read only when + creating a config from an example; never submitted directly. +- **Persisted spec** (what a Fileset actually stores = the yardstick): an `AgentEvalInputSpec` + **minus `target`**. Every task carries its own `metrics[]` with the **judge model baked in**; + `max_concurrent_tasks` is baked. No agent target (that is per-run). + +Persisted `eval-config.json` shape (the yardstick): + +```json +{ + "tasks": [ + { + "id": "Claim Your Free iPhone Now!", + "intent": "Classify this email as phishing or benign", + "inputs": { "instruction": "" }, + "reference": { "label": "phishing" }, + "metrics": [ + { + /* inline metric bundle — see Payloads; judge model baked into payload.metric.model */ + } + ] + } + ], + "max_concurrent_tasks": 1 +} +``` + +**Create ("Use Example"):** Studio reads the example template, **fans the shared `metric` onto +each task** and **injects the chosen judge model** (`buildPersistedSpec`), then writes that +persisted spec into a new Fileset. The judge is now part of the stored yardstick. + +**Reuse ("Choose Fileset"):** the user selects a Fileset **by name**. Studio reads its +`eval-config.json` **as-is** (`parsePersistedSpec`) — no re-fan, no judge re-pick — so the +yardstick (tasks + metric + judge) is identical across every run. Only the agent **target** is +injected at submit; the whole is wrapped as `{ spec }` for the job request. The saved spec is +also a valid `nemo evaluator agent-evaluate submit --spec-file` input once a `target` is added. + +Injected by Studio **at submit** (never stored in the config): the agent **target** only. The +judge model and `max_concurrent_tasks` are part of the persisted yardstick. + +**`max_concurrent_tasks` defaults to `1` (serial).** This is a conservative default for local NAT +deployments. NAT currently maps workflow failures, including output truncation, to `422`; a single +failure aborts the job. Concurrency was ruled out as the root cause, but serial execution avoids +adding load while this upstream error classification remains. To tolerate failures instead, set +`target.params.ignore_request_failure: true` (failed trials score `NaN`). See Gotchas. + +## Endpoints + +| Purpose | Method + path (`/apis/evaluator/v2/workspaces/{ws}`) | +| ----------- | ---------------------------------------------------- | +| List jobs | `GET .../agent-evaluate/jobs` | +| Submit | `POST .../agent-evaluate/jobs` | +| Get job | `GET .../agent-evaluate/jobs/{name}` | +| Poll status | `GET .../agent-evaluate/jobs/{name}/status` | +| Cancel | `POST .../agent-evaluate/jobs/{name}/cancel` | +| Logs | `GET .../agent-evaluate/jobs/{name}/logs` | +| Results | `GET .../agent-eval-results/{name}` | + +Fileset create/upload (Files service), for seeding a config into a new Fileset: + +| Purpose | Method + path | +| -------------- | ------------------------------------------------------------- | +| Create fileset | `POST /apis/files/v2/workspaces/{ws}/filesets` | +| Upload file | `PUT /apis/files/v2/workspaces/{ws}/filesets/{name}/-/{path}` | + +Submit body is wrapped: `{"spec": { ...AgentEvalInputSpec }}`. + +## Payloads + +### Job spec + +```json +{ + "spec": { + "tasks": [ + /* each carries metrics[] from the persisted spec (judge already baked) */ + ], + "target": { + "kind": "agent", + "agent": { + /* see below */ + } + }, + "max_concurrent_tasks": 1 + } +} +``` + +### Target (generic agent) + +```json +{ + "kind": "agent", + "agent": { + "format": "generic", + "url": ".../agents//-/generate", + "name": "", + "body": { "input_message": "{{ instruction }}" }, + "response_path": "$.value", + "stream": false + } +} +``` + +Use the non-streaming `/generate` endpoint. Do **not** use `/generate/full` — its per-token +SSE stream leaves only the last token in the captured output and every score collapses to 0. + +**`body` renders against the task inputs directly.** A generic agent's request is a passthrough +of the task row, so `body` references task input fields by name — `{{ instruction }}` — not a +chat wrapper. `instruction` is the single canonical task input. + +### Task + +```json +{ + "id": "Claim Your Free iPhone Now!", + "intent": "Classify this email as phishing or benign", + "inputs": { "instruction": "" }, + "reference": { "label": "phishing" }, + "metrics": [ + /* inline metric bundle, fanned on at config-create time (judge baked in) */ + ] +} +``` + +`inputs.instruction` is the prompt (falls back to `intent`). `reference` is grader-only ground +truth, never shown to the agent. + +### Inline metric (llm-judge) + +```json +{ + "bundle_kind": "metric-bundle", + "bundle_format_version": "v1", + "metric_type": "llm-judge", + "metadata": { "description": null, "labels": {} }, + "outputs": [ + { + "name": "accuracy", + "description": null, + "value_json_schema": { + "description": "Continuous numeric metric value.", + "title": "ContinuousScore", + "type": "number" + } + } + ], + "secrets": {}, + "payload": { + "kind": "inline", + "metric": { + "type": "llm-judge", + "model": "workspace/name", + /* judge ModelRef string — baked into the persisted spec at config-create time */ + "prompt_template": { + "messages": [ + { + "role": "user", + "content": "... Expected label: \"{{ item.reference.label }}\" ... {{ sample.output_text }} ... respond {\"accuracy\": 0|1}" + } + ] + }, + "scores": [{ "name": "accuracy", "minimum": 0, "maximum": 1 }], + "inference": { + "max_tokens": 1024, + "extra_body": { "nvext": { "max_thinking_tokens": 256 } } + }, + "reasoning": { "end_token": "" } + } + } +} +``` + +Notes on `llm-judge`: + +- `type` is `"llm-judge"` (hyphen). Enum values are inconsistent — some metrics use + underscores (`answer_accuracy`), llm-judge uses a hyphen. +- `prompt_template` must be the **messages-object** form. A bare string routes to the dead + `/completions` endpoint (502). The object form routes to `/chat/completions`. +- `reasoning.end_token: ""` strips a reasoning model's (e.g. Nemotron) thinking trace + before the JSON parser runs; without it the score is `NaN`. +- Bound `inference.extra_body.nvext.max_thinking_tokens` for NIM reasoning models. The total + `max_tokens` budget includes reasoning; without a thinking cap the judge can consume the entire + budget before emitting its structured JSON, producing a `NaN` score. +- In `agent-evaluate` the judge prompt references `{{ item.reference.label }}` / + `{{ item.inputs.instruction }}` and the agent output as `{{ sample.output_text }}`. + +### Multiple scores / multiple metrics + +A metric can emit **many scores** from one judge call — `scores` is a list. Use it when the +dimensions are genuinely distinct (e.g. `coverage`, `correctness`, `relevance`): + +```json +"scores": [ + { "name": "coverage", "minimum": 0, "maximum": 1 }, + { "name": "correctness", "minimum": 0, "maximum": 1 }, + { "name": "relevance", "minimum": 0, "maximum": 1 } +] +``` + +A task can also carry **multiple metrics** — `metrics` is a list; each runs independently. The +phishing sample uses one metric with one `accuracy` score because that is what a binary +classifier measures. + +## Result + +`GET .../agent-eval-results/{name}` returns aggregate scores per metric-score: + +```json +{ + "scores": { + "scores": [ + { + "name": "llm-judge.accuracy", + "count": 5, + "nan_count": 0, + "mean": 0.8, + "min": 0.0, + "max": 1.0, + "std_dev": 0.4, + "score_type": "range" + } + ] + } +} +``` + +`score_type` is `range` (numeric aggregate) or `rubric` (category distribution). The full +per-task bundle (trials, evidence, traces) lives in the fileset referenced by `bundle_ref`. + +## Gotchas + +- **Dataset is inline.** The `agent-evaluate` spec takes inline `tasks[]`; there is no + dataset/fileset/taskset reference yet (planned). Large datasets must be inlined for now. +- **Agent must be deployed and running before submit** — a not-yet-ready agent connection + fails the job. +- **Use `/generate`, not `/generate/full`** (per-token SSE zeroes the score). +- **Run tasks serially (`max_concurrent_tasks: 1`) by default.** NAT currently reports workflow + failures such as output truncation as **422**; `422` is not retried, so one failure kills the + whole job. Serial execution is conservative but does not fix truncation. Configure an adequate + agent output budget, or set `target.params.ignore_request_failure: true` to accept `NaN` trials. +- **`body` uses `{{ instruction }}`, not a `messages` wrapper** — a generic agent's request is a + task-row passthrough with no `messages` key to index. + +--- + +## Legacy: `evaluate/jobs` (row-based, not for agent eval) + +nemo-evaluator also exposes a row-based `evaluate/jobs` endpoint (`EvaluateJob`). It predates +the agent path and serves **prompt/completion-style datasets**. It _can_ take an agent target +and a `dataset` that is a **`FilesetRef`** (a CSV/JSONL file in a Fileset, no row inlining) — +attractive for large datasets — but its agent-eval functionality is limited compared to +`agent-evaluate` (no per-task structure, traces, or artifact aggregation). **Do not use it for +Studio agent evaluation.** Recorded here only so the two endpoints are not confused. + +Shape (reference only): + +```json +{ + "spec": { + "dataset": "default/#data.csv", + "metrics": [ + /* one shared inline metric */ + ], + "target": { + "format": "generic", + "url": ".../-/generate", + "response_path": "$.value", + "stream": false + }, + "prompt_template": { "messages": [{ "role": "user", "content": "{{ item. }}" }] }, + "params": { + /* full RunConfigOnline: parallelism, limit_samples, ignore_request_failure, request_timeout, max_retries */ + } + } +} +``` + +Results come from `GET .../eval-results/{name}` (note: `eval-results`, not `agent-eval-results`) +and include percentiles + histogram. Gotcha: an agent target requires `params` typed exactly as +`RunConfigOnline` (full shape) — a bare `{ "parallelism": N }` parses as plain `RunConfig` and +the job 500s. diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationDetailRoute.tsx b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationDetailRoute.tsx index 78174b4491..254a3b5a98 100644 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationDetailRoute.tsx +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationDetailRoute.tsx @@ -9,7 +9,6 @@ import { StatusBadge } from '@nemo/common/src/components/StatusBadge'; import { useToast } from '@nemo/common/src/providers/toast/useToast'; import type { PlatformJobStatus } from '@nemo/sdk/generated/platform/schema'; import { - Badge, Block, Button, Flex, @@ -20,29 +19,32 @@ import { Stack, Text, } from '@nvidia/foundations-react-core'; +import { + aggregateScoresOf, + agentNameForJob, + cancelAgentEvalJob, + fetchAgentEvalBundle, + fetchAgentEvalJob, + fetchAgentEvalResult, + joinBundleByTask, +} from '@studio/api/evaluation/agent-evaluations'; import { AccessibleTitle } from '@studio/components/AccessibleTitle'; import { StatusLogsContent } from '@studio/components/evaluation/Jobs/StatusLogsContent'; import { ROUTE_PARAMS } from '@studio/constants/routes'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; +import { AgentEvalScoresPanel } from '@studio/routes/agents/AgentEvaluationsRoute/components/AgentEvalScoresPanel'; +import { AgentEvalTaskResultsPanel } from '@studio/routes/agents/AgentEvaluationsRoute/components/AgentEvalTaskResultsPanel'; import { - cancelAgentEvalJob, - fetchAgentEvalJob, - fetchEvalConfigFiles, - fetchEvaluatorOutputs, - fetchWorkflowOutput, - outputFilesetForJob, -} from '@studio/routes/agents/AgentEvaluationsRoute/api'; -import { EvalConfigFilesPanel } from '@studio/routes/agents/AgentEvaluationsRoute/components/EvalConfigFilesPanel'; -import { EvaluatorOutputPanel } from '@studio/routes/agents/AgentEvaluationsRoute/components/EvaluatorOutputPanel'; -import { WorkflowOutputPanel } from '@studio/routes/agents/AgentEvaluationsRoute/components/WorkflowOutputPanel'; -import { formatScore, scoreColor } from '@studio/routes/agents/AgentEvaluationsRoute/evalScores'; -import { fetchEvalAverageScores } from '@studio/routes/agents/AgentSuggestionsRoute/api'; -import { getAgentEvaluationsListRoute, getAgentsListRoute } from '@studio/routes/utils'; + getAgentEvaluationsListRoute, + getAgentsListRoute, + getFilesetDetailRoute, +} from '@studio/routes/utils'; import { useRequiredPathParams } from '@studio/util/hooks/useRequiredPathParams'; import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; -import { ClipboardList, FlaskConical, FolderOpen, ScrollText } from 'lucide-react'; +import { ClipboardList, FlaskConical, ScrollText } from 'lucide-react'; import { type FC } from 'react'; +import { Link } from 'react-router-dom'; const TERMINAL_STATUSES = new Set([ 'completed', @@ -71,8 +73,8 @@ export const AgentEvaluationDetailRoute: FC = () => { ], }); - // Job + status — refetched while the job is non-terminal so the badge - // stays live without forcing a page reload. + // Job + status — refetched while the job is non-terminal so the badge stays + // live without forcing a page reload. const { data: job, isLoading: isLoadingJob } = useQuery({ queryKey: ['agent-eval-job', workspace, jobName] as const, queryFn: ({ signal }) => fetchAgentEvalJob(workspace, jobName, signal), @@ -80,44 +82,26 @@ export const AgentEvaluationDetailRoute: FC = () => { refetchInterval: (query) => (isTerminal(query.state.data?.status) ? false : 5_000), }); - const outputFileset = job ? outputFilesetForJob(job) : null; const isJobTerminal = isTerminal(job?.status); - // Evaluator scores from the output fileset — only fetched once the job is - // terminal; otherwise the fileset doesn't exist yet (or is partial). - const { data: scores, isLoading: isLoadingScores } = useQuery({ - queryKey: ['agent-eval-scores', workspace, outputFileset] as const, - queryFn: ({ signal }) => - outputFileset - ? fetchEvalAverageScores(workspace, outputFileset, signal) - : Promise.resolve([]), - enabled: !!outputFileset && isJobTerminal, - }); - - // Per-evaluator output (items + judge reasoning) for inline rendering. - // Same terminal-only gating as scores. - const { data: evaluatorOutputs, isLoading: isLoadingEvaluatorOutputs } = useQuery({ - queryKey: ['agent-eval-evaluator-outputs', workspace, outputFileset] as const, - queryFn: ({ signal }) => - outputFileset ? fetchEvaluatorOutputs(workspace, outputFileset, signal) : Promise.resolve([]), - enabled: !!outputFileset && isJobTerminal, - }); - - // workflow_output.json — the agent's responses to the dataset. - const { data: workflowOutput, isLoading: isLoadingWorkflow } = useQuery({ - queryKey: ['agent-eval-workflow-output', workspace, outputFileset] as const, - queryFn: ({ signal }) => - outputFileset ? fetchWorkflowOutput(workspace, outputFileset, signal) : Promise.resolve(null), - enabled: !!outputFileset && isJobTerminal, + // Aggregate scores (mean/min/max per metric) from the queryable result record. + // Only meaningful once the job is terminal. The record is persisted best-effort and + // may lag the terminal status, so poll while it is still absent and stop once it loads. + const { data: result, isLoading: isLoadingResult } = useQuery({ + queryKey: ['agent-eval-result', workspace, jobName] as const, + queryFn: ({ signal }) => fetchAgentEvalResult(workspace, jobName, signal), + enabled: !!workspace && !!jobName && isJobTerminal, + refetchInterval: (query) => (query.state.data == null ? 5_000 : false), }); - // config_original.yml / config_effective.yml / config_metadata.json so - // the user can audit what actually ran without leaving the page. - const { data: configFiles, isLoading: isLoadingConfigFiles } = useQuery({ - queryKey: ['agent-eval-config-files', workspace, outputFileset] as const, - queryFn: ({ signal }) => - outputFileset ? fetchEvalConfigFiles(workspace, outputFileset, signal) : Promise.resolve([]), - enabled: !!outputFileset && isJobTerminal, + // Per-task detail (agent response + per-task score + diagnostics) from the + // result bundle referenced by the record. Gated on the result being loaded; polls + // while the bundle is still absent so late-written artifacts are picked up. + const { data: bundle, isLoading: isLoadingBundle } = useQuery({ + queryKey: ['agent-eval-bundle', workspace, jobName, result?.bundle_ref] as const, + queryFn: ({ signal }) => fetchAgentEvalBundle(workspace, result?.bundle_ref, signal), + enabled: !!workspace && isJobTerminal && !!result?.bundle_ref, + refetchInterval: (query) => (query.state.data == null ? 5_000 : false), }); const cancelMutation = useMutation({ @@ -152,14 +136,19 @@ export const AgentEvaluationDetailRoute: FC = () => { ); } - const statusError = job.error_details?.message ?? job.status_details?.message; + const statusMessage = + typeof job.status_details?.message === 'string' ? job.status_details.message : null; + const errorMessage = + typeof job.error_details?.message === 'string' ? job.error_details.message : null; + const scores = aggregateScoresOf(result ?? null); + const taskDetails = joinBundleByTask(bundle ?? null); return ( { value={} loading={isLoadingJob} /> - - + - + {job.description && ( + + {job.description} + + } + /> + )} } + value={job.created_at ? : ''} loading={isLoadingJob} /> } + value={job.updated_at ? : ''} loading={isLoadingJob} /> - {statusError && ( + {(errorMessage ?? statusMessage) && ( - {statusError} + + {errorMessage ?? statusMessage} } /> @@ -223,7 +219,7 @@ export const AgentEvaluationDetailRoute: FC = () => { } elevation="high" density="compact" @@ -233,63 +229,23 @@ export const AgentEvaluationDetailRoute: FC = () => { Scores are computed once the job reaches a terminal state. )} - {isJobTerminal && isLoadingScores && ( + {isJobTerminal && isLoadingResult && ( )} - {isJobTerminal && !isLoadingScores && (scores?.length ?? 0) === 0 && ( - - No evaluator scores parsed from the output fileset. - - )} - {isJobTerminal && !isLoadingScores && (scores?.length ?? 0) > 0 && ( - - {scores!.map((s) => ( - - - {s.evaluator} - - - {formatScore(s.averageScore)} - - - ))} - - )} + {isJobTerminal && !isLoadingResult && } - {!isJobTerminal && ( - } - elevation="high" - density="compact" - > - - Per-item scores, agent responses, and the run config appear once the job completes. - - - )} - - {isJobTerminal && - (isLoadingEvaluatorOutputs || isLoadingWorkflow || isLoadingConfigFiles) && ( - - - - )} - - {isJobTerminal && !isLoadingEvaluatorOutputs && (evaluatorOutputs ?? []).length > 0 && ( - - {evaluatorOutputs!.map((output) => ( - - ))} - + {isJobTerminal && isLoadingBundle && ( + + + )} - {isJobTerminal && !isLoadingWorkflow && workflowOutput && workflowOutput.length > 0 && ( - + {isJobTerminal && !isLoadingBundle && taskDetails.length > 0 && ( + )} }> @@ -299,10 +255,6 @@ export const AgentEvaluationDetailRoute: FC = () => { jobStatus={job.status as PlatformJobStatus} /> - - {isJobTerminal && !isLoadingConfigFiles && (configFiles ?? []).length > 0 && ( - - )} ); diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationsListRoute.tsx b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationsListRoute.tsx index ed2519aecb..a1f19c7fff 100644 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationsListRoute.tsx +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationsListRoute.tsx @@ -17,10 +17,10 @@ import { Text, TextInput, } from '@nvidia/foundations-react-core'; +import { agentNameForJob, fetchAgentEvalJobs } from '@studio/api/evaluation/agent-evaluations'; import { AccessibleTitle } from '@studio/components/AccessibleTitle'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; import { useBreadcrumbs } from '@studio/providers/breadcrumbs/useBreadcrumbs'; -import { fetchAgentEvalJobs } from '@studio/routes/agents/AgentEvaluationsRoute/api'; import { SubmitEvaluationModal } from '@studio/routes/agents/AgentEvaluationsRoute/components/SubmitEvaluationModal'; import { getAgentEvaluationDetailRoute, getAgentsListRoute } from '@studio/routes/utils'; import { useQuery } from '@tanstack/react-query'; @@ -88,9 +88,9 @@ export const AgentEvaluationsListRoute: FC = () => { const all = data ?? []; const search = agentSearch.trim().toLowerCase(); const filtered = all.filter((job) => { - if (!matchesStatus(job.status, statusFilter)) return false; + if (!matchesStatus(job.status ?? '', statusFilter)) return false; if (search) { - const agent = (job.spec.agent ?? '').toLowerCase(); + const agent = (agentNameForJob(job) ?? '').toLowerCase(); const name = job.name.toLowerCase(); if (!agent.includes(search) && !name.includes(search)) return false; } @@ -99,14 +99,14 @@ export const AgentEvaluationsListRoute: FC = () => { const sorted = [...filtered].sort((a, b) => { switch (sortKey) { case 'created_asc': - return a.created_at.localeCompare(b.created_at); + return (a.created_at ?? '').localeCompare(b.created_at ?? ''); case 'name_asc': return a.name.localeCompare(b.name); case 'name_desc': return b.name.localeCompare(a.name); case 'created_desc': default: - return b.created_at.localeCompare(a.created_at); + return (b.created_at ?? '').localeCompare(a.created_at ?? ''); } }); return sorted; @@ -198,14 +198,9 @@ export const AgentEvaluationsListRoute: FC = () => { {job.name} - {job.spec.agent && ( + {agentNameForJob(job) && ( - Agent: {job.spec.agent} - - )} - {job.spec.eval_config && ( - - Config: {job.spec.eval_config} + Agent: {agentNameForJob(job)} )} @@ -215,7 +210,7 @@ export const AgentEvaluationsListRoute: FC = () => { - Created + Created {job.created_at ? : '—'} diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/api.test.ts b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/api.test.ts deleted file mode 100644 index 6909656eff..0000000000 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/api.test.ts +++ /dev/null @@ -1,117 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { - fetchAgentEvalJob, - fetchAgentEvalJobs, - fetchAgentEvalOutputFiles, - outputFilesetForJob, - type AgentEvalJob, -} from '@studio/routes/agents/AgentEvaluationsRoute/api'; - -const customFetchMock = vi.fn(); -vi.mock('@nemo/sdk/generated/fetchers/platform', () => ({ - customFetch: (...args: unknown[]) => customFetchMock(...args), -})); - -const filesListFilesetFilesMock = vi.fn(); -const filesDownloadFileMock = vi.fn(); -vi.mock('@nemo/sdk/generated/platform/api', () => ({ - filesListFilesetFiles: (...args: unknown[]) => filesListFilesetFilesMock(...args), - filesDownloadFile: (...args: unknown[]) => filesDownloadFileMock(...args), -})); - -beforeEach(() => { - customFetchMock.mockReset(); - filesListFilesetFilesMock.mockReset(); - filesDownloadFileMock.mockReset(); -}); - -const baseJob = (overrides: Partial = {}): AgentEvalJob => ({ - name: 'eval-1', - workspace: 'ws-a', - status: 'completed', - created_at: '2026-05-05T00:00:00Z', - updated_at: '2026-05-05T00:01:00Z', - spec: { agent: 'support-bot-mini', eval_config: 'eval.yaml' }, - ...overrides, -}); - -describe('fetchAgentEvalJobs', () => { - it('paginates until a short page is returned', async () => { - // Two full pages of 50 + a 1-item tail page; the helper must walk all - // three to return the full list. - const page1 = Array.from({ length: 50 }, (_, i) => baseJob({ name: `j-${i}` })); - const page2 = Array.from({ length: 50 }, (_, i) => baseJob({ name: `j-${50 + i}` })); - const page3 = [baseJob({ name: 'j-100' })]; - customFetchMock - .mockResolvedValueOnce({ data: page1 }) - .mockResolvedValueOnce({ data: page2 }) - .mockResolvedValueOnce({ data: page3 }); - const all = await fetchAgentEvalJobs('ws-a', new AbortController().signal); - expect(all).toHaveLength(101); - expect(customFetchMock).toHaveBeenCalledTimes(3); - }); - - it('returns an empty array when the first page is empty', async () => { - customFetchMock.mockResolvedValueOnce({ data: [] }); - const all = await fetchAgentEvalJobs('ws-a', new AbortController().signal); - expect(all).toEqual([]); - expect(customFetchMock).toHaveBeenCalledTimes(1); - }); -}); - -describe('fetchAgentEvalJob', () => { - it('returns the job when the platform responds with one', async () => { - customFetchMock.mockResolvedValueOnce(baseJob({ name: 'eval-42' })); - const job = await fetchAgentEvalJob('ws-a', 'eval-42', new AbortController().signal); - expect(job?.name).toBe('eval-42'); - }); - - it('returns null when the platform returns no body', async () => { - customFetchMock.mockResolvedValueOnce(undefined); - const job = await fetchAgentEvalJob('ws-a', 'missing', new AbortController().signal); - expect(job).toBeNull(); - }); -}); - -describe('fetchAgentEvalOutputFiles', () => { - it('treats 404 as an empty fileset (job ran but never wrote outputs)', async () => { - filesListFilesetFilesMock.mockRejectedValueOnce({ response: { status: 404 } }); - const files = await fetchAgentEvalOutputFiles( - 'ws-a', - 'support-bot-mini-eval-out', - new AbortController().signal - ); - expect(files).toEqual([]); - }); - - it('rethrows non-404 errors so the caller can surface them', async () => { - filesListFilesetFilesMock.mockRejectedValueOnce({ response: { status: 500 } }); - await expect( - fetchAgentEvalOutputFiles('ws-a', 'fileset', new AbortController().signal) - ).rejects.toMatchObject({ response: { status: 500 } }); - }); -}); - -describe('outputFilesetForJob', () => { - it('prefers spec.output verbatim when set as a bare name', () => { - expect(outputFilesetForJob(baseJob({ spec: { output: 'custom-out' } }))).toBe('custom-out'); - }); - - it('strips the workspace prefix from spec.output when given as workspace/name', () => { - expect(outputFilesetForJob(baseJob({ spec: { output: 'ws-a/custom-out' } }))).toBe( - 'custom-out' - ); - }); - - it('falls back to -eval-out when spec.output is unset', () => { - expect(outputFilesetForJob(baseJob({ spec: { agent: 'support-bot-mini' } }))).toBe( - 'support-bot-mini-eval-out' - ); - }); - - it('returns null when neither spec.output nor spec.agent is set', () => { - expect(outputFilesetForJob(baseJob({ spec: {} }))).toBeNull(); - }); -}); diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/api.ts b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/api.ts deleted file mode 100644 index 2bd6357dfa..0000000000 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/api.ts +++ /dev/null @@ -1,325 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { customFetch } from '@nemo/sdk/generated/fetchers/platform'; -import { filesDownloadFile, filesListFilesetFiles } from '@nemo/sdk/generated/platform/api'; - -const PAGE_SIZE = 50; - -// --------------------------------------------------------------------------- -// Job listing + retrieval -// --------------------------------------------------------------------------- - -export interface AgentEvalJobSpec { - agent?: string | null; - eval_config?: string; - eval_config_fileset?: string | null; - output?: string | null; - workspace?: string; -} - -export interface AgentEvalJob { - name: string; - description?: string | null; - workspace: string; - status: string; - created_at: string; - updated_at: string; - spec: AgentEvalJobSpec; - status_details?: { message?: string } | null; - error_details?: { message?: string } | null; -} - -interface PaginatedJobsResponse { - data?: AgentEvalJob[]; - pagination?: { total?: number; page?: number; page_size?: number }; -} - -const evalJobsBasePath = (workspace: string): string => - `/apis/agents/v2/workspaces/${encodeURIComponent(workspace)}/jobs/evaluate`; - -const evalJobPath = (workspace: string, name: string): string => - `${evalJobsBasePath(workspace)}/${encodeURIComponent(name)}`; - -export const fetchAgentEvalJobs = async ( - workspace: string, - signal: AbortSignal -): Promise => { - const all: AgentEvalJob[] = []; - let page = 1; - while (true) { - const res = await customFetch({ - url: evalJobsBasePath(workspace), - method: 'GET', - params: { page, page_size: PAGE_SIZE, sort: '-created_at' }, - signal, - }); - const batch = res?.data ?? []; - all.push(...batch); - if (batch.length < PAGE_SIZE) break; - page++; - } - return all; -}; - -export const fetchAgentEvalJob = async ( - workspace: string, - name: string, - signal: AbortSignal -): Promise => { - try { - const res = await customFetch({ - url: evalJobPath(workspace, name), - method: 'GET', - signal, - }); - return res ?? null; - } catch (err) { - const e = err as { response?: { status?: number }; status?: number }; - if (e?.response?.status === 404 || e?.status === 404) return null; - throw err; - } -}; - -export const cancelAgentEvalJob = async ( - workspace: string, - name: string, - signal: AbortSignal -): Promise => { - await customFetch({ - url: `${evalJobPath(workspace, name)}/cancel`, - method: 'POST', - signal, - }); -}; - -// --------------------------------------------------------------------------- -// Output fileset (eval results) -// --------------------------------------------------------------------------- - -export interface AgentEvalOutputFile { - path: string; - size?: number; -} - -export const fetchAgentEvalOutputFiles = async ( - workspace: string, - outputFileset: string, - signal: AbortSignal -): Promise => { - try { - const listing = await filesListFilesetFiles(workspace, outputFileset, undefined, signal); - return (listing?.data ?? []).map((f) => ({ path: f.path, size: f.size ?? undefined })); - } catch (err) { - const e = err as { response?: { status?: number }; status?: number }; - if (e?.response?.status === 404 || e?.status === 404) return []; - throw err; - } -}; - -export const downloadAgentEvalOutputFile = async ( - workspace: string, - outputFileset: string, - remotePath: string, - signal: AbortSignal -): Promise => { - const blob = await filesDownloadFile(workspace, outputFileset, remotePath, signal); - return blob ?? null; -}; - -// --------------------------------------------------------------------------- -// Parsed eval result shapes -// --------------------------------------------------------------------------- - -export interface EvalScoreBreakdown { - /** ``coverage_score`` / ``correctness_score`` / ``relevance_score`` etc. */ - [key: string]: number; -} - -export interface EvaluatorOutputItem { - id: string | number; - score: number | null; - /** nat-eval ``tunable_rag_evaluator`` writes a structured object; other - * evaluators write a plain string. Render whichever is present. */ - reasoning: - | string - | { - question?: string; - answer_description?: string; - generated_answer?: string; - score_breakdown?: EvalScoreBreakdown; - reasoning?: string; - }; - error: string | null; -} - -export interface EvaluatorOutput { - evaluator: string; - averageScore: number | null; - items: EvaluatorOutputItem[]; -} - -export interface WorkflowOutputItem { - id: string | number; - question: string; - answer: string; - generated_answer: string | null; - intermediate_steps?: unknown[]; - expected_intermediate_steps?: unknown[]; -} - -const downloadJson = async ( - workspace: string, - fileset: string, - remotePath: string, - signal: AbortSignal -): Promise => { - const blob = await filesDownloadFile(workspace, fileset, remotePath, signal); - if (!blob) return null; - return JSON.parse(await blob.text()) as T; -}; - -const EVALUATOR_OUTPUT_BASENAME_RE = /^([^/]+)_output\.json$/; -const NON_EVALUATOR_BASENAMES = new Set(['workflow_output.json', 'workflow_output_atif.json']); -const WORKFLOW_OUTPUT_BASENAMES = new Set(['workflow_output.json']); -const CONFIG_BASENAMES = new Set([ - 'config_original.yml', - 'config_effective.yml', - 'config_metadata.json', -]); - -const basenameOf = (path: string): string => path.split('/').pop() ?? path; - -/** Reads every ``_output.json`` in the output fileset and parses - * the full per-item payload so the detail page can render a table without - * forcing the user to download files. Downloads run in parallel via - * ``Promise.all`` — they target independent files. */ -export const fetchEvaluatorOutputs = async ( - workspace: string, - outputFileset: string, - signal: AbortSignal -): Promise => { - const files = await fetchAgentEvalOutputFiles(workspace, outputFileset, signal); - const candidates = files.flatMap((f) => { - const base = basenameOf(f.path); - if (NON_EVALUATOR_BASENAMES.has(base)) return []; - const m = EVALUATOR_OUTPUT_BASENAME_RE.exec(base); - return m ? [{ file: f, evaluator: m[1] }] : []; - }); - const settled = await Promise.all( - candidates.map(async ({ file, evaluator }) => { - try { - const parsed = await downloadJson<{ - average_score?: unknown; - eval_output_items?: EvaluatorOutputItem[]; - }>(workspace, outputFileset, file.path, signal); - if (!parsed) return null; - const avg = parsed.average_score; - return { - evaluator, - averageScore: typeof avg === 'number' && Number.isFinite(avg) ? avg : null, - items: Array.isArray(parsed.eval_output_items) ? parsed.eval_output_items : [], - } satisfies EvaluatorOutput; - } catch { - // Skip malformed evaluator files so one bad one doesn't drop the others. - return null; - } - }) - ); - return settled.filter((s): s is EvaluatorOutput => s !== null); -}; - -/** Loads ``workflow_output.json`` (the agent's responses to the dataset) - * if present in the output fileset. Returns null when not yet written. */ -export const fetchWorkflowOutput = async ( - workspace: string, - outputFileset: string, - signal: AbortSignal -): Promise => { - const files = await fetchAgentEvalOutputFiles(workspace, outputFileset, signal); - const wf = files.find((f) => WORKFLOW_OUTPUT_BASENAMES.has(basenameOf(f.path))); - if (!wf) return null; - try { - const parsed = await downloadJson(workspace, outputFileset, wf.path, signal); - if (Array.isArray(parsed)) return parsed as WorkflowOutputItem[]; - return null; - } catch { - return null; - } -}; - -export interface EvalConfigFile { - /** File basename (e.g. ``config_original.yml``). */ - name: string; - /** Full path inside the fileset, used for downloads. */ - path: string; - /** Decoded UTF-8 content of the file. */ - content: string; - /** ``yaml`` for ``.yml``/``.yaml``, ``json`` otherwise. */ - language: 'yaml' | 'json' | 'text'; -} - -const detectLanguage = (name: string): EvalConfigFile['language'] => { - if (name.endsWith('.yml') || name.endsWith('.yaml')) return 'yaml'; - if (name.endsWith('.json')) return 'json'; - return 'text'; -}; - -/** Loads the eval-run config snapshots (``config_original.yml``, - * ``config_effective.yml``, ``config_metadata.json``) so the detail page - * can render them inline rather than as download-only file rows. - * Downloads run in parallel via ``Promise.all`` — independent files. */ -export const fetchEvalConfigFiles = async ( - workspace: string, - outputFileset: string, - signal: AbortSignal -): Promise => { - const files = await fetchAgentEvalOutputFiles(workspace, outputFileset, signal); - const candidates = files.flatMap((f) => { - const base = basenameOf(f.path); - return CONFIG_BASENAMES.has(base) ? [{ file: f, name: base }] : []; - }); - const settled = await Promise.all( - candidates.map(async ({ file, name }) => { - try { - const blob = await filesDownloadFile(workspace, outputFileset, file.path, signal); - if (!blob) return null; - return { - name, - path: file.path, - content: await blob.text(), - language: detectLanguage(name), - } satisfies EvalConfigFile; - } catch { - // Skip unreadable config files individually. - return null; - } - }) - ); - return settled - .filter((s): s is EvalConfigFile => s !== null) - .sort((a, b) => a.name.localeCompare(b.name)); -}; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -/** - * Sibling output fileset name follows the same convention as the optimizer - * apply path: ``-eval-out``. The ``output`` field on the - * spec carries the literal fileset name when set; older / hand-submitted - * jobs without ``output`` fall back to a derived guess. - */ -export const outputFilesetForJob = (job: AgentEvalJob): string | null => { - const explicit = job.spec.output; - if (typeof explicit === 'string' && explicit.length > 0) { - return explicit.includes('/') ? (explicit.split('/').pop() ?? null) : explicit; - } - const agent = job.spec.agent; - if (typeof agent === 'string' && agent.length > 0) { - const bare = agent.includes('/') ? agent.split('/').pop()! : agent; - return `${bare}-eval-out`; - } - return null; -}; diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/AgentEvalScoresPanel.test.tsx b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/AgentEvalScoresPanel.test.tsx new file mode 100644 index 0000000000..b676325006 --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/AgentEvalScoresPanel.test.tsx @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { AgentEvalScoresPanel } from '@studio/routes/agents/AgentEvaluationsRoute/components/AgentEvalScoresPanel'; +import { render, screen } from '@studio/tests/util/render'; + +describe('AgentEvalScoresPanel', () => { + it('reports scored and NaN rows without subtracting NaNs twice', () => { + render( + + ); + + expect(screen.getByText('4/5 scored · range 0.000–1.000')).toBeInTheDocument(); + expect(screen.getByText('0.750')).toBeInTheDocument(); + }); + + it('renders the rubric distribution and mode category for rubric scores', () => { + render( + + ); + + expect(screen.getByText('Most frequent: partially_helpful')).toBeInTheDocument(); + expect(screen.getByText('unhelpful: 1')).toBeInTheDocument(); + expect(screen.getByText('partially_helpful: 2')).toBeInTheDocument(); + expect(screen.getByText('helpful: 0')).toBeInTheDocument(); + expect(screen.getByText('0.667')).toBeInTheDocument(); + }); +}); diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/AgentEvalScoresPanel.tsx b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/AgentEvalScoresPanel.tsx new file mode 100644 index 0000000000..afdd9e0e4f --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/AgentEvalScoresPanel.tsx @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Badge, Block, Flex, Stack, Text } from '@nvidia/foundations-react-core'; +import type { AgentEvalAggregateScore } from '@studio/api/evaluation/agent-evaluations'; +import { formatScore, scoreColor } from '@studio/routes/agents/AgentEvaluationsRoute/evalScores'; +import { type FC } from 'react'; + +interface AgentEvalScoresPanelProps { + scores: AgentEvalAggregateScore[]; +} + +/** Aggregate score summary per metric: the mean as a colored badge plus the + * min/max range and sample count. Sourced from the `agent-eval-results` + * record (see this route's AGENTS.md). */ +export const AgentEvalScoresPanel: FC = ({ scores }) => { + if (scores.length === 0) { + return No scores recorded for this evaluation.; + } + + return ( + + {scores.map((s) => { + const total = s.count + s.nan_count; + return ( + + + + {s.name} + + + {s.count}/{total} scored + {typeof s.min === 'number' && typeof s.max === 'number' + ? ` · range ${formatScore(s.min)}–${formatScore(s.max)}` + : ''} + + {s.score_type === 'rubric' && ( + + {s.mode_category ? ( + + Most frequent: {s.mode_category} + + ) : null} + + {s.rubric_distribution.map((r) => ( + + {r.label}: {r.count ?? 0} + + ))} + + + )} + + + {formatScore(s.mean)} + + + ); + })} + + ); +}; diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/AgentEvalTaskResultsPanel.tsx b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/AgentEvalTaskResultsPanel.tsx new file mode 100644 index 0000000000..1cbef011cc --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/AgentEvalTaskResultsPanel.tsx @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { AccordionPanel } from '@nemo/common/src/components/AccordionPanel'; +import { StatusBadge } from '@nemo/common/src/components/StatusBadge'; +import { Badge, Block, Card, Flex, Stack, Text } from '@nvidia/foundations-react-core'; +import type { AgentEvalTaskDetail } from '@studio/api/evaluation/agent-evaluations'; +import { formatScore, scoreColor } from '@studio/routes/agents/AgentEvaluationsRoute/evalScores'; +import { ListChecks } from 'lucide-react'; +import { type FC } from 'react'; + +interface AgentEvalTaskResultsPanelProps { + tasks: AgentEvalTaskDetail[]; +} + +const headingFor = (task: AgentEvalTaskDetail): string => { + const firstLine = (task.instruction ?? '').split('\n', 1)[0] ?? ''; + const match = /^\s*subject:\s*(.+)$/i.exec(firstLine); + return match ? match[1].trim() : task.taskId; +}; + +const referenceText = (reference?: Record): string | null => { + if (!reference || Object.keys(reference).length === 0) return null; + const values = Object.values(reference); + if (values.length === 1 && typeof values[0] === 'string') return values[0]; + return JSON.stringify(reference); +}; + +const metadataEntries = (metadata?: Record): Array<[string, string]> => + Object.entries(metadata ?? {}).map(([k, v]) => [ + k, + typeof v === 'string' ? v : JSON.stringify(v), + ]); + +export const AgentEvalTaskResultsPanel: FC = ({ tasks }) => { + if (tasks.length === 0) { + return No per-task results recorded for this evaluation.; + } + + return ( + }> + + {tasks.map((task) => { + const expected = referenceText(task.reference); + const metadata = metadataEntries(task.metadata); + return ( + + + {task.scores.map((s) => ( + + {s.name}: {formatScore(s.value)} + + ))} + + + + + + {headingFor(task)} + + + {expected && ( + + Expected: {expected} + + )} + + + + + + + Agent response + + + {task.responseText ?? '—'} + + + + {task.instruction && ( + + + Input + + + {task.instruction} + + + )} + + {metadata.length > 0 && ( + + + Metadata + + + {metadata.map(([key, value]) => ( + + + {key}: + + + {value} + + + ))} + + + )} + + {task.diagnostics.length > 0 && ( + + + Diagnostics + + + {task.diagnostics.map((d) => JSON.stringify(d)).join('\n')} + + + )} + + + ); + })} + + + ); +}; diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/EvalConfigFilesPanel.tsx b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/EvalConfigFilesPanel.tsx deleted file mode 100644 index 700e8c78ce..0000000000 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/EvalConfigFilesPanel.tsx +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { Accordion, Block, Panel, Text } from '@nvidia/foundations-react-core'; -import type { EvalConfigFile } from '@studio/routes/agents/AgentEvaluationsRoute/api'; -import { FileCode } from 'lucide-react'; -import type { FC } from 'react'; - -const PRETTY_NAME: Record = { - 'config_original.yml': 'Original eval config', - 'config_effective.yml': 'Effective eval config', - 'config_metadata.json': 'Run metadata', -}; - -interface EvalConfigFilesPanelProps { - files: EvalConfigFile[]; -} - -export const EvalConfigFilesPanel: FC = ({ files }) => { - return ( - } - elevation="high" - density="compact" - > - {files.length === 0 ? ( - No config snapshots in the output fileset. - ) : ( - ({ - chevronPosition: 'start', - value: f.name, - slotTrigger: {PRETTY_NAME[f.name] ?? f.name}, - slotContent: ( - - 
-                  {f.content}
-
- - ), - }))} - /> - )} - - ); -}; diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/EvaluatorOutputPanel.tsx b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/EvaluatorOutputPanel.tsx deleted file mode 100644 index 01b09b1a17..0000000000 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/EvaluatorOutputPanel.tsx +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { Badge, Block, Card, Flex, Panel, Stack, Text } from '@nvidia/foundations-react-core'; -import type { EvaluatorOutput } from '@studio/routes/agents/AgentEvaluationsRoute/api'; -import { EvaluatorReasoning } from '@studio/routes/agents/AgentEvaluationsRoute/components/EvaluatorReasoning'; -import { formatScore, scoreColor } from '@studio/routes/agents/AgentEvaluationsRoute/evalScores'; -import { FlaskConical } from 'lucide-react'; -import type { FC } from 'react'; - -interface EvaluatorOutputPanelProps { - output: EvaluatorOutput; -} - -export const EvaluatorOutputPanel: FC = ({ output }) => { - return ( - - {output.evaluator} ({output.items.length} item - {output.items.length === 1 ? '' : 's'}) - - } - slotIcon={} - elevation="high" - density="compact" - > - - - Average score - - {formatScore(output.averageScore)} - - - {output.items.length === 0 ? ( - No per-item results recorded. - ) : ( - // One card per item — the tunable_rag_evaluator's reasoning has - // 4–5 substructures (question / expected / generated / breakdown / - // judge), which crammed into a table cell becomes unreadable. - - {output.items.map((item, idx) => ( - - {/* Pin the score chip to the card's top-right corner so it - reads as a status indicator and doesn't push the - reasoning content down. ``pr-density-3xl`` on the - content stack reserves room so long Question/Expected - text doesn't run under the badge. */} -
- - {formatScore(item.score)} - -
- - {item.error ? ( - - {item.error} - - ) : ( - - )} - - - ))} - - )} - - - ); -}; diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/EvaluatorReasoning.tsx b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/EvaluatorReasoning.tsx deleted file mode 100644 index e58ebc83cf..0000000000 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/EvaluatorReasoning.tsx +++ /dev/null @@ -1,67 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { Badge, Flex, Stack, Text } from '@nvidia/foundations-react-core'; -import type { - EvalScoreBreakdown, - EvaluatorOutputItem, -} from '@studio/routes/agents/AgentEvaluationsRoute/api'; -import { LabeledSection } from '@studio/routes/agents/AgentEvaluationsRoute/components/LabeledSection'; -import { formatScore, scoreColor } from '@studio/routes/agents/AgentEvaluationsRoute/evalScores'; -import type { FC } from 'react'; - -interface EvaluatorReasoningProps { - reasoning: EvaluatorOutputItem['reasoning']; -} - -/** Renders the per-item ``reasoning`` payload from a nat-eval evaluator - * output. The ``tunable_rag_evaluator`` writes a structured object with - * question / expected / generated / per-component score breakdown / judge - * reasoning; other evaluators may write a plain string — fall through to a - * single ``Text`` render in that case. */ -export const EvaluatorReasoning: FC = ({ reasoning }) => { - if (typeof reasoning === 'string') { - return ( - - {reasoning} - - ); - } - - const breakdown = reasoning.score_breakdown; - return ( - - {reasoning.question && ( - - {reasoning.question} - - )} - {reasoning.answer_description && ( - - {reasoning.answer_description} - - )} - {reasoning.generated_answer !== undefined && ( - - {reasoning.generated_answer || '(empty)'} - - )} - {breakdown && Object.keys(breakdown).length > 0 && ( - - - {Object.entries(breakdown as EvalScoreBreakdown).map(([k, v]) => ( - - {k.replace(/_score$/, '').replace(/_/g, ' ')}: {formatScore(v)} - - ))} - - - )} - {reasoning.reasoning && ( - - {reasoning.reasoning} - - )} - - ); -}; diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/SubmitEvaluationModal.tsx b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/SubmitEvaluationModal.tsx index d818e49fa0..4c57ef2ff0 100644 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/SubmitEvaluationModal.tsx +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/SubmitEvaluationModal.tsx @@ -7,40 +7,39 @@ import { parseFilesetLocation } from '@nemo/common/src/components/DatasetFileSel import { ControlledSelect } from '@nemo/common/src/components/form/ControlledSelect'; import { ControlledTextInput } from '@nemo/common/src/components/form/ControlledTextInput'; import { FormModal, type FormModalProps } from '@nemo/common/src/components/FormModal'; +import { getURNFromNamedEntityRef } from '@nemo/common/src/namedEntity'; import { useToast } from '@nemo/common/src/providers/toast/useToast'; -import { customFetch } from '@nemo/sdk/generated/fetchers/platform'; -import { filesCreateFileset } from '@nemo/sdk/generated/platform/api'; +import { useAgentsListAgents } from '@nemo/sdk/generated/agents/api'; +import type { AgentEvaluateJobRequest } from '@nemo/sdk/generated/evaluator/schema'; +import { filesDownloadFile, filesListFilesetFiles } from '@nemo/sdk/generated/platform/api'; import { SegmentedControl, Stack, Text } from '@nvidia/foundations-react-core'; import { fetchSampleText } from '@studio/api/agents/fetchSampleText'; -import { type Agent } from '@studio/components/dataViews/AgentsDataView'; -import { PLATFORM_BASE_URL } from '@studio/constants/environment'; +import { submitAgentEvalJob } from '@studio/api/evaluation/agent-evaluations'; import { - DEFAULT_SAMPLE_AGENT_KEY, - SAMPLE_AGENTS, - sampleAgentKeyForAgentName, -} from '@studio/constants/sampleAgents'; + ensureEvalConfigFileset, + type EvalSeedFile, +} from '@studio/api/evaluation/eval-config-fileset'; +import { JudgeModelSelect } from '@studio/components/evaluation/JudgeModelSelect'; import { - fetchAgentEvalJobs, - type AgentEvalJob, -} from '@studio/routes/agents/AgentEvaluationsRoute/api'; + EVALUATION_SAMPLE_AGENTS, + evaluationSampleAgentKeyForAgentName, + getEvaluationSampleAgent, +} from '@studio/constants/sampleAgents'; +import { useJudgeModels } from '@studio/hooks/evaluation/useJudgeModels'; import { - buildSubmitSpec, - CREATE_NEW, - evalOutputDescription, - evaluateRequestBody, + bareName, + buildAgentEvalRequestBody, + buildPersistedSpec, generateEvalConfigName, - generateOutputFilesetName, MODE_DEFAULT, MODE_FILESET, - type SubmitSpec, + parseEvalConfig, + parsePersistedSpec, + type PersistedEvalSpec, } from '@studio/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec'; -import { - ensureEvalConfigFileset, - type EvalSeedFile, -} from '@studio/routes/agents/AgentSuggestionsRoute/api'; -import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; -import { type FC, useEffect, useMemo, useRef } from 'react'; -import { type SubmitHandler, useForm, useWatch } from 'react-hook-form'; +import { keepPreviousData, useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { type FC, useEffect, useRef, useState } from 'react'; +import { FormProvider, type SubmitHandler, useForm, useWatch } from 'react-hook-form'; import { z } from 'zod'; const EVAL_CONFIG_MODE_ITEMS = [ @@ -48,27 +47,30 @@ const EVAL_CONFIG_MODE_ITEMS = [ { value: MODE_FILESET, children: 'Choose Fileset' }, ]; -/** Strip an optional ``workspace/`` prefix so agent references compare by name. */ -const bareName = (value?: string | null): string | null => { - if (typeof value !== 'string' || value.length === 0) return null; - return value.includes('/') ? (value.split('/').pop() ?? null) : value; -}; +/** Flat filename the reusable config is stored as inside its fileset. */ +const EVAL_CONFIG_FILENAME = 'eval-config.json'; + +const submitEvaluationBaseSchema = z.object({ + agent: z.string().min(1, 'Agent is required'), + judgeModel: z.string(), + mode: z.enum([MODE_DEFAULT, MODE_FILESET]), + exampleKey: z.string(), + newName: z.string(), + configFile: z.string().nullable(), +}); + +type SubmitEvaluationFormData = z.infer; -const submitEvaluationSchema = z - .object({ - agent: z.string().min(1, 'Agent is required'), - // Existing eval-config fileset to reuse, or CREATE_NEW to make one. - evalConfig: z.string().min(1, 'Select or create an eval config'), - // Create-mode fields (only enforced when evalConfig === CREATE_NEW). - newName: z.string(), - mode: z.enum([MODE_DEFAULT, MODE_FILESET]), - exampleKey: z.string(), - datasetFile: z.string().nullable(), - }) - .superRefine((data, ctx) => { - if (data.evalConfig !== CREATE_NEW) return; +const makeSubmitEvaluationSchema = (requiresJudgeModel: () => boolean) => + submitEvaluationBaseSchema.superRefine((data, ctx) => { + if (requiresJudgeModel() && !data.judgeModel) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Judge model is required', + path: ['judgeModel'], + }); + } if (data.mode === MODE_DEFAULT) { - // newName becomes the fileset name — enforce the platform naming rules. const name = data.newName.trim(); if (!name) { ctx.addIssue({ @@ -84,33 +86,25 @@ const submitEvaluationSchema = z }); } } - if (data.mode === MODE_FILESET && !parseFilesetLocation(data.datasetFile ?? '')?.objectPath) { + if (data.mode === MODE_FILESET && !parseFilesetLocation(data.configFile ?? '')?.objectPath) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: 'Pick an eval YAML inside an existing fileset', - path: ['datasetFile'], + message: 'Pick an eval-config.json inside an existing fileset', + path: ['configFile'], }); } }); -type SubmitEvaluationFormData = z.infer; - const makeDefaultValues = (agent?: string): SubmitEvaluationFormData => ({ agent: agent ?? '', - // Default to create; existing configs are one click away in the dropdown. - evalConfig: CREATE_NEW, - newName: generateEvalConfigName(), + judgeModel: '', mode: MODE_DEFAULT, - // Auto-match the example to the agent it was created from (by name prefix), - // falling back to the first example for non-example agents. - exampleKey: sampleAgentKeyForAgentName(agent) ?? DEFAULT_SAMPLE_AGENT_KEY, - datasetFile: null, + exampleKey: evaluationSampleAgentKeyForAgentName(agent) ?? EVALUATION_SAMPLE_AGENTS[0]?.key ?? '', + newName: generateEvalConfigName(), + configFile: null, }); interface SubmitEvaluationModalProps extends Pick { - /** Workspace to submit the eval into. Passed in (rather than resolved - * from the URL) so the modal can render outside a workspace route — e.g. - * inside an open ``AgentPanel`` test. */ workspace: string; /** When provided, pre-fills + locks the agent selector. */ agent?: string; @@ -118,6 +112,65 @@ interface SubmitEvaluationModalProps extends Pick void; } +/** True when a fileset of this name already holds an eval-config.json. A "Create new" + * submission must use a free name: ensureEvalConfigFileset never overwrites an existing + * file, so a name collision would leave the fileset holding the old config while we submit + * the new spec — the persisted yardstick and the evaluated spec would diverge. */ +const evalConfigFilesetExists = async ( + workspace: string, + fileset: string, + signal: AbortSignal +): Promise => { + try { + const listing = await filesListFilesetFiles(workspace, fileset, undefined, signal); + return (listing?.data ?? []).some((f) => f.path === EVAL_CONFIG_FILENAME); + } catch (err) { + const e = err as { response?: { status?: number }; status?: number }; + if (e?.response?.status === 404 || e?.status === 404) return false; + throw err; + } +}; + +/** Resolves the persisted yardstick spec for this submission. In "Use Example" mode + * it builds the spec from the sample template (fanning the metric onto every task with + * the picked judge baked in) and seeds it into a new fileset; in "Choose Fileset" mode + * it reads the saved spec back verbatim (no re-fan, no judge re-pick). */ +const loadPersistedSpec = async ( + workspace: string, + formData: SubmitEvaluationFormData +): Promise => { + if (formData.mode === MODE_DEFAULT) { + const signal = new AbortController().signal; + const name = formData.newName.trim(); + if (await evalConfigFilesetExists(workspace, name, signal)) { + throw new Error(`A fileset named "${name}" already exists — choose a different name`); + } + const example = getEvaluationSampleAgent(formData.exampleKey); + const template = parseEvalConfig(await fetchSampleText(example.evalConfigPath)); + const spec = buildPersistedSpec(template, formData.judgeModel || null); + const files: EvalSeedFile[] = [ + { + path: EVAL_CONFIG_FILENAME, + content: JSON.stringify(spec, null, 2), + type: 'application/json', + }, + ]; + await ensureEvalConfigFileset(workspace, name, signal, files, 'Agent Evaluation Config'); + return spec; + } + // Choose-fileset mode: read the saved yardstick spec out of its fileset, as-is. + const parsed = parseFilesetLocation(formData.configFile ?? ''); + if (!parsed?.objectPath) throw new Error('No eval-config.json selected'); + const blob = await filesDownloadFile( + workspace, + parsed.name, + parsed.objectPath, + new AbortController().signal + ); + if (!blob) throw new Error('Failed to read the selected eval config'); + return parsePersistedSpec(await blob.text()); +}; + export const SubmitEvaluationModal: FC = ({ open, onClose, @@ -128,25 +181,78 @@ export const SubmitEvaluationModal: FC = ({ const toast = useToast(); const queryClient = useQueryClient(); - const { data: agents = [], isLoading: isAgentsLoading } = useQuery({ - queryKey: ['agents', workspace], + // Ref keeps isLlmJudge current for the zod schema getter at validation time. + const isLlmJudgeRef = useRef(false); + const [schema] = useState(() => makeSubmitEvaluationSchema(() => isLlmJudgeRef.current)); + + const { data: agentsResponse, isLoading: isAgentsLoading } = useAgentsListAgents( + workspace, + undefined, + { query: { enabled: open && !agentProp } } + ); + const agents = agentsResponse?.data ?? []; + + const methods = useForm({ + resolver: zodResolver(schema), + defaultValues: makeDefaultValues(agentProp), + mode: 'onSubmit', + reValidateMode: 'onChange', + }); + const { + control, + reset: resetForm, + setValue, + getValues, + handleSubmit, + setError, + clearErrors, + formState, + } = methods; + const { errors } = formState; + + const mode = useWatch({ control, name: 'mode' }); + const selectedAgent = useWatch({ control, name: 'agent' }); + const exampleKey = useWatch({ control, name: 'exampleKey' }); + + // Fetch and parse the selected example config early to detect metric type and default model. + const { data: exampleConfig } = useQuery({ + queryKey: ['eval-config-preview', exampleKey], queryFn: async () => { - const response = await fetch( - `${PLATFORM_BASE_URL}/apis/agents/v2/workspaces/${workspace}/agents` - ); - if (!response.ok) throw new Error('Failed to fetch agents'); - const json = (await response.json()) as { data: Agent[] }; - return json.data; + const example = getEvaluationSampleAgent(exampleKey); + if (!example) return null; + const text = await fetchSampleText(example.evalConfigPath); + return parseEvalConfig(text); }, - enabled: open && !agentProp, + enabled: open && mode === MODE_DEFAULT && !!exampleKey, + staleTime: Infinity, + // Retain the prior example's parsed config while the next one loads so the + // judge picker stays mounted (all examples are llm-judge) — no flicker. + placeholderData: keepPreviousData, }); - // Prior eval jobs — the source for the "existing eval config" dropdown. - const { data: jobs = [], isLoading: isJobsLoading } = useQuery({ - queryKey: ['agent-eval-jobs', workspace], - queryFn: ({ signal }) => fetchAgentEvalJobs(workspace, signal), - enabled: open, - }); + const isLlmJudge = mode === MODE_DEFAULT && exampleConfig?.metric.metric_type === 'llm-judge'; + isLlmJudgeRef.current = isLlmJudge; + + const defaultModelRef = + isLlmJudge && typeof exampleConfig?.metric.payload.metric.model === 'string' + ? exampleConfig.metric.payload.metric.model + : undefined; + + // Fetch judge models eagerly so they're ready when isLlmJudge resolves. + const { data: judgeModels } = useJudgeModels({ enabled: open }); + + // Pre-populate judge model from the config's ModelRef when modal opens or data arrives. + // Uses getValues (not a reactive watch) to avoid re-running on every model change. + useEffect(() => { + if (!open || !isLlmJudge || !defaultModelRef || !judgeModels?.length) return; + if (getValues('judgeModel')) return; + const target = bareName(defaultModelRef); + const match = judgeModels.find((m) => m.name === target); + if (match) { + const urn = getURNFromNamedEntityRef(match); + if (urn) setValue('judgeModel', urn); + } + }, [open, isLlmJudge, defaultModelRef, judgeModels, getValues, setValue]); const { mutateAsync: submitEvaluation, @@ -154,47 +260,20 @@ export const SubmitEvaluationModal: FC = ({ isPending, reset: resetMutation, } = useMutation({ - mutationFn: async (spec: SubmitSpec) => { - if (spec.seedSources) { - // Seed the selected example's eval assets into the new config fileset. - const files: EvalSeedFile[] = await Promise.all( - spec.seedSources.map(async (source) => ({ - path: source.path, - content: await fetchSampleText(source.assetPath), - type: source.type, - })) - ); - await ensureEvalConfigFileset( - workspace, - spec.evalConfigFileset, - new AbortController().signal, - files, - 'Agent Evaluation Config' - ); - } - // Pre-create the output fileset so it carries a description; the job's - // auto-create no-ops once it exists. Best-effort — never block submission. - const outputFileset = generateOutputFilesetName(spec.agent); - try { - await filesCreateFileset(workspace, { - name: outputFileset, - description: evalOutputDescription(spec), - purpose: 'generic', - }); - } catch { - // Job still auto-creates the fileset (without a description). - } - const body = evaluateRequestBody(spec, outputFileset); - const res = await customFetch<{ name?: string }>({ - url: `/apis/agents/v2/workspaces/${encodeURIComponent(workspace)}/jobs/evaluate`, - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - data: body, + mutationFn: async (formData: SubmitEvaluationFormData) => { + const spec = await loadPersistedSpec(workspace, formData); + const filesetName = + formData.mode === MODE_DEFAULT + ? formData.newName.trim() + : (parseFilesetLocation(formData.configFile ?? '')?.name ?? undefined); + const body = buildAgentEvalRequestBody(spec, { + workspace, + agent: formData.agent, + filesetName, }); - if (!res?.name) { - throw new Error('Submission did not return a job name'); - } - return res.name; + const created = await submitAgentEvalJob(workspace, body as AgentEvaluateJobRequest); + if (!created?.name) throw new Error('Submission did not return a job name'); + return created.name; }, onSuccess: (jobName) => { toast.success(`Evaluation "${jobName}" submitted`); @@ -204,97 +283,25 @@ export const SubmitEvaluationModal: FC = ({ }, }); - const { - control, - reset: resetForm, - setValue, - handleSubmit, - setError, - clearErrors, - formState: { errors }, - } = useForm({ - resolver: zodResolver(submitEvaluationSchema), - defaultValues: makeDefaultValues(agentProp), - disabled: isPending, - mode: 'onSubmit', - reValidateMode: 'onChange', - }); - - const evalConfig = useWatch({ control, name: 'evalConfig' }); - const mode = useWatch({ control, name: 'mode' }); - const selectedAgent = useWatch({ control, name: 'agent' }); - - // Existing eval configs for the agent: distinct config filesets from prior - // jobs, mapped to the YAML each ran. - const existingConfigs = useMemo(() => { - const map = new Map(); - const agentKey = bareName(selectedAgent); - if (!agentKey) return map; - for (const job of jobs as AgentEvalJob[]) { - if (bareName(job.spec.agent) !== agentKey) continue; - const fileset = job.spec.eval_config_fileset; - if (typeof fileset === 'string' && fileset.length > 0 && !map.has(fileset)) { - map.set(fileset, job.spec.eval_config ?? ''); - } - } - return map; - }, [jobs, selectedAgent]); - - const evalConfigItems = useMemo( - () => [ - ...Array.from(existingConfigs.keys()).map((fileset) => ({ - value: fileset, - children: fileset, - })), - { value: CREATE_NEW, children: '+ Create new eval config' }, - ], - [existingConfigs] - ); - - // When the chosen agent maps to a known example, auto-select its eval config. + // Keep the example matched to the agent it was created from. useEffect(() => { - const matchedKey = sampleAgentKeyForAgentName(selectedAgent); + const matchedKey = evaluationSampleAgentKeyForAgentName(selectedAgent); if (matchedKey) setValue('exampleKey', matchedKey); }, [selectedAgent, setValue]); - // Preselect the latest existing config for the agent (else create). Ref-guarded - // to run once per agent so it doesn't override the user's later manual pick. - const autoSelectedAgentRef = useRef(null); - useEffect(() => { - if (!open) { - autoSelectedAgentRef.current = null; - return; - } - if (isJobsLoading || !selectedAgent) return; - if (autoSelectedAgentRef.current === selectedAgent) return; - autoSelectedAgentRef.current = selectedAgent; - const latest = existingConfigs.keys().next().value; - setValue('evalConfig', latest ?? CREATE_NEW); - }, [open, isJobsLoading, selectedAgent, existingConfigs, setValue]); - - useEffect(() => { - resetForm(makeDefaultValues(agentProp)); - }, [agentProp, resetForm]); - useEffect(() => { - if (!open) { - resetForm(makeDefaultValues(agentProp)); - } + if (!open) resetForm(makeDefaultValues(agentProp)); }, [open, agentProp, resetForm]); - const reset = () => { + const resetAndClose = () => { resetMutation(); resetForm(makeDefaultValues(agentProp)); - }; - - const resetAndClose = () => { - reset(); onClose(); }; const onSubmit: SubmitHandler = async (formData) => { try { - await submitEvaluation(buildSubmitSpec(formData, existingConfigs)); + await submitEvaluation(formData); } catch { // Error rendered via errorText prop. } @@ -307,8 +314,6 @@ export const SubmitEvaluationModal: FC = ({ ? 'An error occurred' : undefined; - const isCreating = evalConfig === CREATE_NEW; - return ( = ({ errorText={errorMessage} className="w-[690px]! max-w-[95vw]!" > - - {agentProp ? ( - - Evaluating agent {agentProp} - - ) : ( - - agent.name ? [{ value: agent.name, children: agent.name }] : [] - )} - formFieldProps={{ - slotLabel: 'Agent', - slotError: errors.agent?.message, - }} - /> - )} - {selectedAgent ? ( - - - Eval Config + + + {agentProp ? ( + + Evaluating agent {agentProp} + ) : ( + agent.name ? [{ value: agent.name, children: agent.name }] : [] + )} + formFieldProps={{ slotLabel: 'Agent', slotError: errors.agent?.message }} /> - {isCreating && ( - <> - { - setValue('mode', v as typeof MODE_DEFAULT | typeof MODE_FILESET, { - shouldValidate: false, - }); - clearErrors('datasetFile'); - }} - items={EVAL_CONFIG_MODE_ITEMS} - /> - {mode === MODE_DEFAULT ? ( - <> - ({ - value: example.key, - children: example.label, - }))} - formFieldProps={{ - slotLabel: 'Example', - slotError: errors.exampleKey?.message, - }} - /> - + + Eval Config + + { + setValue('mode', v as typeof MODE_DEFAULT | typeof MODE_FILESET, { + shouldValidate: false, + }); + clearErrors('configFile'); + }} + items={EVAL_CONFIG_MODE_ITEMS} + /> + + {mode === MODE_DEFAULT ? ( + <> + ({ + value: example.key, + children: example.label, + }))} + formFieldProps={{ slotLabel: 'Example', slotError: errors.exampleKey?.message }} + /> + {isLlmJudge && ( + + formFieldName="judgeModel" + slotLabel="Judge Model" /> - - ) : ( - setError('datasetFile', error)} - clearError={() => clearErrors('datasetFile')} - workspace={workspace} - inline - autoCommit - autoSelectFirstAcceptable - filesetPurpose="generic" - datasetLabel="Fileset" + )} + - )} - - )} - - ) : null} - + + ) : ( + setError('configFile', error)} + clearError={() => clearErrors('configFile')} + workspace={workspace} + inline + autoCommit + autoSelectFirstAcceptable + showUpdatedAt + filesetPurpose="generic" + datasetLabel="Fileset" + formFieldProps={{ slotError: errors.configFile?.message }} + /> + )} + + ) : null} + + ); }; diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/WorkflowOutputPanel.tsx b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/WorkflowOutputPanel.tsx deleted file mode 100644 index 443ee74173..0000000000 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/WorkflowOutputPanel.tsx +++ /dev/null @@ -1,137 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { Badge, Block, Flex, Panel, Stack, Text } from '@nvidia/foundations-react-core'; -import type { - EvaluatorOutput, - WorkflowOutputItem, -} from '@studio/routes/agents/AgentEvaluationsRoute/api'; -import { formatScore, scoreColor } from '@studio/routes/agents/AgentEvaluationsRoute/evalScores'; -import { MessagesSquare } from 'lucide-react'; -import { useMemo, type FC } from 'react'; - -interface WorkflowOutputPanelProps { - items: WorkflowOutputItem[]; - evaluatorOutputs?: EvaluatorOutput[]; -} - -interface ScoreLookup { - /** Per-evaluator score for one workflow item, looked up by ``id``. */ - byEvaluator: Map; -} - -const buildScoreLookups = (evaluatorOutputs: EvaluatorOutput[]): Map => { - // Outer key is the workflow item's ``id`` stringified — nat-eval ids can be - // numeric or string, and JSON serialises them differently across files. - const out = new Map(); - for (const evalOut of evaluatorOutputs) { - for (const item of evalOut.items) { - const key = String(item.id); - let entry = out.get(key); - if (!entry) { - entry = { byEvaluator: new Map() }; - out.set(key, entry); - } - entry.byEvaluator.set(evalOut.evaluator, item.score); - } - } - return out; -}; - -export const WorkflowOutputPanel: FC = ({ - items, - evaluatorOutputs = [], -}) => { - const scoreLookups = useMemo(() => buildScoreLookups(evaluatorOutputs), [evaluatorOutputs]); - const evaluatorNames = useMemo( - () => evaluatorOutputs.map((e) => e.evaluator), - [evaluatorOutputs] - ); - const showScores = evaluatorNames.length > 0; - - return ( - } - elevation="high" - density="compact" - > - - {items.length === 0 ? ( - No workflow output recorded. - ) : ( - - - - - - - - {showScores && ( - - )} - - - - {items.map((item, idx) => { - const lookup = scoreLookups.get(String(item.id)); - return ( - - - - - - {showScores && ( - - )} - - ); - })} - -
- ID - - Question - - Expected - - Generated - - Scores -
- {String(item.id)} - - {item.question} - - - {item.answer} - - - {item.generated_answer ? ( - {item.generated_answer} - ) : ( - - (empty) - - )} - - - {evaluatorNames.map((name) => { - const score = lookup?.byEvaluator.get(name); - return ( - - {name}:{' '} - {formatScore(score ?? null)} - - ); - })} - -
- )} - - - ); -}; diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.test.ts b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.test.ts index d61e761170..7b5ceefde8 100644 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.test.ts +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.test.ts @@ -1,98 +1,196 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { SAMPLE_AGENTS } from '@studio/constants/sampleAgents'; import { - buildSubmitSpec, - CREATE_NEW, - evalOutputDescription, - evaluateRequestBody, - generateOutputFilesetName, + bareName, + buildAgentEvalRequestBody, + buildAgentTarget, + buildPersistedSpec, + type EvalConfig, + fanMetricOntoTasks, + injectJudgeModel, + type InlineMetricBundle, + parseEvalConfig, + parsePersistedSpec, + type PersistedEvalSpec, } from '@studio/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec'; -const baseForm = { - agent: 'my-agent', - evalConfig: CREATE_NEW, - newName: '', - mode: 'default' as const, - exampleKey: SAMPLE_AGENTS[0].key, - datasetFile: null as string | null, +/** A config whose metric already carries a baked-in judge ModelRef. */ +const configWithBakedJudge = (): EvalConfig => ({ + ...config, + metric: { + ...metric, + payload: { kind: 'inline', metric: { ...metric.payload.metric, model: 'ws-a/baked-in' } }, + }, +}); + +const metric: InlineMetricBundle = { + bundle_kind: 'metric-bundle', + bundle_format_version: 'v1', + metric_type: 'llm-judge', + payload: { kind: 'inline', metric: { type: 'llm-judge', scores: [{ name: 'accuracy' }] } }, }; -describe('buildSubmitSpec', () => { - it('reuses an existing eval config untouched (no seed sources)', () => { - const existing = new Map([['wise-pretzel', 'analyzer-eval.yml']]); - const spec = buildSubmitSpec({ ...baseForm, evalConfig: 'wise-pretzel' }, existing); +const config: EvalConfig = { + tasks: [ + { + id: 'A', + intent: 'classify', + inputs: { instruction: 'email a' }, + reference: { label: 'phishing' }, + }, + { + id: 'B', + intent: 'classify', + inputs: { instruction: 'email b' }, + reference: { label: 'benign' }, + }, + ], + metric, + max_concurrent_tasks: 2, +}; - expect(spec).toEqual({ - agent: 'my-agent', - evalConfig: 'analyzer-eval.yml', - evalConfigFileset: 'wise-pretzel', - }); - expect(spec.seedSources).toBeUndefined(); +describe('bareName', () => { + it('strips a workspace prefix', () => { + expect(bareName('default/my-model')).toBe('my-model'); + expect(bareName('my-model')).toBe('my-model'); }); +}); - it('creates a new slug fileset and seeds the example config + dataset', () => { - const spec = buildSubmitSpec( - { ...baseForm, evalConfig: CREATE_NEW, mode: 'default', newName: ' wise-pretzel ' }, - new Map() - ); +describe('buildAgentTarget', () => { + it('targets the non-streaming /generate endpoint of the agent', () => { + const target = buildAgentTarget('ws-a', 'support-bot'); + expect(target.kind).toBe('agent'); + expect(target.agent.format).toBe('generic'); + expect(target.agent.stream).toBe(false); + expect(target.agent.response_path).toBe('$.value'); + expect(target.agent.body).toEqual({ input_message: '{{ instruction }}' }); + expect(target.agent.url).toContain('/agents/support-bot/-/generate'); + expect(target.agent.url).not.toContain('/generate/full'); + }); +}); - // Trimmed slug becomes the eval-config fileset (also the output target). - expect(spec.evalConfigFileset).toBe('wise-pretzel'); - expect(spec.evalConfig.startsWith(`${SAMPLE_AGENTS[0].key}-`)).toBe(true); - expect(spec.seedSources).toHaveLength(2); +describe('injectJudgeModel', () => { + it('sets the judge ModelRef string without mutating the input', () => { + const out = injectJudgeModel(metric, 'ws-a/nemotron-super'); + expect(out.payload.metric.model).toBe('ws-a/nemotron-super'); + expect(metric.payload.metric.model).toBeUndefined(); }); +}); - it("reuses the picked file's own fileset when creating from a fileset YAML", () => { - const spec = buildSubmitSpec( - { - ...baseForm, - evalConfig: CREATE_NEW, - mode: 'fileset', - datasetFile: 'default/my-fs#eval.yml', - }, - new Map() - ); +describe('fanMetricOntoTasks', () => { + it('attaches the judge-injected metric to every task', () => { + const tasks = fanMetricOntoTasks(config, 'ws-a/j'); + expect(tasks).toHaveLength(2); + for (const t of tasks) { + expect(t.metrics).toHaveLength(1); + expect(t.metrics[0].payload.metric.model).toBe('ws-a/j'); + } + }); - expect(spec.evalConfigFileset).toBe('my-fs'); - expect(spec.evalConfig).toBe('eval.yml'); - expect(spec.seedSources).toBeUndefined(); + it("keeps the config's own judge when none is supplied", () => { + const tasks = fanMetricOntoTasks(configWithBakedJudge(), null); + expect(tasks[0].metrics[0].payload.metric.model).toBe('ws-a/baked-in'); }); }); -describe('generateOutputFilesetName', () => { - it('mints a fresh per-run -eval-out- name', () => { - const a = generateOutputFilesetName('my-agent'); - const b = generateOutputFilesetName('my-agent'); - expect(a).toMatch(/^my-agent-eval-out-[a-z0-9]{5}$/); - expect(a).not.toBe(b); // random suffix differs per call +describe('buildPersistedSpec', () => { + it('fans the judge-injected metric onto every task', () => { + const spec = buildPersistedSpec(config, 'ws-a/judge'); + expect(spec.tasks).toHaveLength(2); + for (const t of spec.tasks) { + expect(t.metrics).toHaveLength(1); + expect(t.metrics[0].payload.metric.model).toBe('ws-a/judge'); + } + expect(spec.max_concurrent_tasks).toBe(2); + }); + + it("keeps the template metric's own model when no judge is supplied", () => { + const spec = buildPersistedSpec(configWithBakedJudge(), null); + expect(spec.tasks[0].metrics[0].payload.metric.model).toBe('ws-a/baked-in'); + }); + + it('defaults max_concurrent_tasks when the template omits it', () => { + const spec = buildPersistedSpec({ tasks: config.tasks, metric }, 'ws-a/j'); + expect(spec.max_concurrent_tasks).toBe(1); + }); +}); + +describe('buildAgentEvalRequestBody', () => { + const persisted = (): PersistedEvalSpec => buildPersistedSpec(config, 'ws-a/judge'); + + it('assembles a {spec:{tasks,target,max_concurrent_tasks}} body and injects the target', () => { + const body = buildAgentEvalRequestBody(persisted(), { + workspace: 'ws-a', + agent: 'ws-a/support-bot', + }); + expect(body.spec.tasks).toHaveLength(2); + expect(body.spec.max_concurrent_tasks).toBe(2); + expect(body.spec.target.agent.name).toBe('support-bot'); + expect(body.spec.tasks[0].metrics[0].metric_type).toBe('llm-judge'); + // Judge is already baked into the persisted spec; submit does not touch it. + expect(body.spec.tasks[0].metrics[0].payload.metric.model).toBe('ws-a/judge'); + }); + + it('sets the fileset name as the job description when provided', () => { + const body = buildAgentEvalRequestBody(persisted(), { + workspace: 'ws-a', + agent: 'a', + filesetName: 'wise-blue', + }); + expect(body.description).toBe('wise-blue'); }); }); -describe('evalOutputDescription', () => { - it('describes the agent and eval-config fileset', () => { - expect( - evalOutputDescription({ - agent: 'my-agent', - evalConfig: 'eval.yml', - evalConfigFileset: 'wise-pretzel', - }) - ).toBe('Agent Evaluation output, agent: my-agent, config: wise-pretzel'); +describe('parseEvalConfig', () => { + it('parses a valid template', () => { + const parsed = parseEvalConfig(JSON.stringify(config)); + expect(parsed.tasks).toHaveLength(2); + expect(parsed.metric.metric_type).toBe('llm-judge'); + }); + + it('rejects a template with no tasks', () => { + expect(() => parseEvalConfig(JSON.stringify({ tasks: [], metric }))).toThrow(/tasks/); + }); + + it('rejects a template with no metric', () => { + expect(() => parseEvalConfig(JSON.stringify({ tasks: config.tasks }))).toThrow(/metric/); + }); + + it('rejects a metric missing payload.metric', () => { + expect(() => + parseEvalConfig(JSON.stringify({ tasks: config.tasks, metric: { metric_type: 'llm-judge' } })) + ).toThrow(/payload\.metric/); }); }); -describe('evaluateRequestBody', () => { - it('sends the chosen eval-config fileset and the given per-run output fileset', () => { - const body = evaluateRequestBody( - { agent: 'my-agent', evalConfig: 'eval.yml', evalConfigFileset: 'wise-pretzel' }, - 'my-agent-eval-out-ab3d9' - ); - - expect(body.spec.eval_config).toBe('eval.yml'); - expect(body.spec.eval_config_fileset).toBe('wise-pretzel'); - // Output is a distinct per-run fileset, not the config fileset. - expect(body.spec.output).toBe('my-agent-eval-out-ab3d9'); - expect(body.spec.output).not.toBe(body.spec.eval_config_fileset); +describe('parsePersistedSpec', () => { + const spec = (): PersistedEvalSpec => buildPersistedSpec(config, 'ws-a/judge'); + + it('parses a valid persisted spec round-tripped through JSON', () => { + const parsed = parsePersistedSpec(JSON.stringify(spec())); + expect(parsed.tasks).toHaveLength(2); + expect(parsed.tasks[0].metrics[0].payload.metric.model).toBe('ws-a/judge'); + expect(parsed.max_concurrent_tasks).toBe(2); + }); + + it('rejects a spec with no tasks', () => { + expect(() => parsePersistedSpec(JSON.stringify({ tasks: [] }))).toThrow(/tasks/); + }); + + it('rejects a task with no metrics', () => { + expect(() => + parsePersistedSpec(JSON.stringify({ tasks: [{ id: 'A', intent: 'x', metrics: [] }] })) + ).toThrow(/metrics/); + }); + + it('rejects a task metric missing payload.metric', () => { + expect(() => + parsePersistedSpec( + JSON.stringify({ + tasks: [{ id: 'A', intent: 'x', metrics: [{ metric_type: 'llm-judge' }] }], + }) + ) + ).toThrow(/payload\.metric/); }); }); diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.ts b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.ts index 767d300297..00b4abbd0c 100644 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.ts +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.ts @@ -1,128 +1,186 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { parseFilesetLocation } from '@nemo/common/src/components/DatasetFileSelect/parseFilesetLocation'; import { generateDefaultName } from '@nemo/common/src/utils/generateDefaultName'; -import { getSampleAgent } from '@studio/constants/sampleAgents'; -import { evalOutputFilesetFor } from '@studio/routes/agents/AgentSuggestionsRoute/utils'; - -export const MODE_DEFAULT = 'default'; -export const MODE_FILESET = 'fileset'; +import { PLATFORM_BASE_URL } from '@studio/constants/environment'; /** Sentinel ``evalConfig`` value that switches the form into create mode. */ export const CREATE_NEW = '__create_new__'; -/** Suggested name for a new eval config (e.g. "wise-blue"). */ +export const MODE_DEFAULT = 'default'; +export const MODE_FILESET = 'fileset'; + +/** Suggested name for a new eval-config fileset (e.g. "wise-blue"). */ export const generateEvalConfigName = (): string => generateDefaultName({ length: 2 }); -/** Form values the eval-submit modal collects. */ -export interface SubmitEvaluationFormValues { - agent: string; - /** Existing eval-config fileset to reuse, or CREATE_NEW to make one. */ - evalConfig: string; - newName: string; - mode: typeof MODE_DEFAULT | typeof MODE_FILESET; - exampleKey: string; - datasetFile: string | null; +/** Default parallelism for a submitted eval (Studio default; the config value is a hint). */ +export const DEFAULT_MAX_CONCURRENT_TASKS = 1; + +// --------------------------------------------------------------------------- +// eval-config.json shape (stored in a fileset, read at submit) +// --------------------------------------------------------------------------- + +/** One inline metric bundle as stored in eval-config.json (no judge_model — + * it is injected at submit). Kept loose: Studio does not re-validate the + * built-in metric shape, it only injects the model and fans it onto tasks. */ +export interface InlineMetricBundle { + bundle_kind: string; + bundle_format_version: string; + metric_type: string; + metadata?: Record; + outputs?: unknown[]; + secrets?: Record; + payload: { + kind: 'inline'; + metric: Record & { model?: unknown }; + }; } -export interface EvalSeedSource { - /** Flat filename seeded into the fileset. */ - path: string; - /** Public asset path fetched on demand for the file's content. */ - assetPath: string; - type: string; +export interface EvalConfigTask { + id: string; + intent: string; + inputs?: { instruction?: string | null }; + reference?: Record; } -export interface SubmitSpec { +/** The example template: inline tasks + one shared metric (metric not yet fanned). */ +export interface EvalConfig { + tasks: EvalConfigTask[]; + metric: InlineMetricBundle; + max_concurrent_tasks?: number; +} + +/** A task with the shared metric fanned onto it (judge baked in). */ +export type EvalSpecTask = EvalConfigTask & { metrics: InlineMetricBundle[] }; + +/** The persisted yardstick stored in a fileset: tasks-with-metrics, no target. + * An `AgentEvalInputSpec` minus `target` — submit injects the per-run agent. */ +export interface PersistedEvalSpec { + tasks: EvalSpecTask[]; + max_concurrent_tasks?: number; +} + +// --------------------------------------------------------------------------- +// Submit-time selections + request assembly +// --------------------------------------------------------------------------- + +export interface SubmitSelections { + workspace: string; + /** Agent (bare name) to evaluate; used to build the generic target. */ agent: string; - evalConfig: string; - evalConfigFileset: string; - /** Files to seed into ``evalConfigFileset`` before submitting. Omitted when - * reusing an existing config. */ - seedSources?: EvalSeedSource[]; + /** Eval-config fileset name, stored as the job description for display in the detail view. */ + filesetName?: string; } -export const contentTypeForFile = (name: string): string => { - if (name.endsWith('.json')) return 'application/json'; - if (name.endsWith('.csv')) return 'text/csv'; - return 'application/yaml'; -}; +/** Strip an optional ``workspace/`` prefix, returning the bare model/agent name. */ +export const bareName = (value: string): string => + value.includes('/') ? (value.split('/').pop() ?? value) : value; -/** Basename of a public asset path — the flat name it's seeded as in the fileset. */ -export const fileNameOf = (path: string): string => path.slice(path.lastIndexOf('/') + 1); - -/** Builds the eval-job spec from the form (reuse existing config, pick a - * fileset YAML, or seed an example into a new fileset). */ -export const buildSubmitSpec = ( - formData: SubmitEvaluationFormValues, - existingConfigs: Map -): SubmitSpec => { - if (formData.evalConfig !== CREATE_NEW) { - return { - agent: formData.agent, - evalConfig: existingConfigs.get(formData.evalConfig) ?? '', - evalConfigFileset: formData.evalConfig, - }; - } - if (formData.mode === MODE_FILESET) { - // datasetFile is validated by the schema refine before we get here. - const parsed = parseFilesetLocation(formData.datasetFile!)!; - return { - agent: formData.agent, - evalConfig: parsed.objectPath, - evalConfigFileset: parsed.name, - }; - } - const example = getSampleAgent(formData.exampleKey); - // Namespace the config per example so switching examples doesn't reuse the - // first-seeded config. - const evalConfigName = `${example.key}-${fileNameOf(example.evalConfigPath)}`; - return { - agent: formData.agent, - evalConfig: evalConfigName, - evalConfigFileset: formData.newName.trim(), - seedSources: [ - { - path: evalConfigName, - assetPath: example.evalConfigPath, - type: contentTypeForFile(example.evalConfigPath), - }, - { - path: fileNameOf(example.evalDataPath), - assetPath: example.evalDataPath, - type: contentTypeForFile(example.evalDataPath), - }, - ], - }; -}; +/** The generic agent target: the deployed agent's non-streaming ``/generate``. */ +export const buildAgentTarget = (workspace: string, agent: string) => ({ + kind: 'agent' as const, + agent: { + format: 'generic' as const, + url: `${PLATFORM_BASE_URL}/apis/agents/v2/workspaces/${encodeURIComponent(workspace)}/agents/${encodeURIComponent(bareName(agent))}/-/generate`, + name: bareName(agent), + body: { input_message: '{{ instruction }}' }, + response_path: '$.value', + stream: false, + }, +}); -const OUTPUT_SUFFIX_LENGTH = 5; -const OUTPUT_SUFFIX_ALPHABET = 'abcdefghijklmnopqrstuvwxyz0123456789'; +/** Set the metric's judge model to a ``workspace/name`` ModelRef (resolved to a + * reachable Model server-side). Does not mutate input. */ +export const injectJudgeModel = ( + metric: InlineMetricBundle, + judgeModel: string +): InlineMetricBundle => ({ + ...metric, + payload: { + ...metric.payload, + metric: { ...metric.payload.metric, model: judgeModel }, + }, +}); -/** Random 5-char suffix so re-runs don't 409 on an existing output fileset. */ -const randomSuffix = (): string => { - const bytes = new Uint8Array(OUTPUT_SUFFIX_LENGTH); - crypto.getRandomValues(bytes); - let out = ''; - for (const b of bytes) out += OUTPUT_SUFFIX_ALPHABET[b % OUTPUT_SUFFIX_ALPHABET.length]; - return out; +/** Fan the shared metric onto every task. A judge model is injected only when + * one is supplied; otherwise the template metric's own model is kept as-is. */ +export const fanMetricOntoTasks = ( + config: EvalConfig, + judgeModel: string | null +): EvalSpecTask[] => { + const metric = judgeModel ? injectJudgeModel(config.metric, judgeModel) : config.metric; + return config.tasks.map((task) => ({ ...task, metrics: [metric] })); }; -/** Fresh per-run output fileset name (``-eval-out-``). */ -export const generateOutputFilesetName = (agent: string): string => - `${evalOutputFilesetFor(agent)}-${randomSuffix()}`; - -/** Description stamped on the eval output fileset. */ -export const evalOutputDescription = (spec: SubmitSpec): string => - `Agent Evaluation output, agent: ${spec.agent}, config: ${spec.evalConfigFileset}`; +/** Build the persisted yardstick from an example template: fan the shared metric + * (judge baked in) onto every task. This is what gets stored in the fileset. */ +export const buildPersistedSpec = ( + config: EvalConfig, + judgeModel: string | null +): PersistedEvalSpec => ({ + tasks: fanMetricOntoTasks(config, judgeModel), + max_concurrent_tasks: config.max_concurrent_tasks ?? DEFAULT_MAX_CONCURRENT_TASKS, +}); -/** POST body for ``/jobs/evaluate``. */ -export const evaluateRequestBody = (spec: SubmitSpec, output: string) => ({ +/** Build the ``agent-evaluate/jobs`` POST body from a persisted spec + selections. */ +export const buildAgentEvalRequestBody = ( + spec: PersistedEvalSpec, + selections: SubmitSelections +) => ({ + ...(selections.filesetName ? { description: selections.filesetName } : {}), spec: { - agent: spec.agent, - eval_config: spec.evalConfig, - eval_config_fileset: spec.evalConfigFileset, - output, + tasks: spec.tasks, + target: buildAgentTarget(selections.workspace, selections.agent), + max_concurrent_tasks: spec.max_concurrent_tasks ?? DEFAULT_MAX_CONCURRENT_TASKS, }, }); + +/** Parse an example template blob, validating the minimal required shape. */ +export const parseEvalConfig = (text: string): EvalConfig => { + const parsed = JSON.parse(text) as Partial; + if (!Array.isArray(parsed.tasks) || parsed.tasks.length === 0) { + throw new Error('eval-config.json must contain a non-empty "tasks" array'); + } + if (!parsed.metric || typeof parsed.metric !== 'object') { + throw new Error('eval-config.json must contain a "metric"'); + } + + const { payload } = parsed.metric; + if ( + !payload || + typeof payload !== 'object' || + !payload.metric || + typeof payload.metric !== 'object' + ) { + throw new Error('eval-config.json "metric" must contain a "payload.metric" object'); + } + return { + tasks: parsed.tasks, + metric: parsed.metric, + max_concurrent_tasks: parsed.max_concurrent_tasks, + }; +}; + +/** Parse a persisted yardstick spec (the reuse path): tasks each carry their own + * metrics, no top-level ``metric``. Submitted as-is with only a target injected. */ +export const parsePersistedSpec = (text: string): PersistedEvalSpec => { + const parsed = JSON.parse(text) as Partial; + if (!Array.isArray(parsed.tasks) || parsed.tasks.length === 0) { + throw new Error('eval-config.json must contain a non-empty "tasks" array'); + } + for (const task of parsed.tasks) { + if (!Array.isArray(task.metrics) || task.metrics.length === 0) { + throw new Error('eval-config.json every task must contain a non-empty "metrics" array'); + } + const payload = task.metrics[0]?.payload; + if ( + !payload || + typeof payload !== 'object' || + !payload.metric || + typeof payload.metric !== 'object' + ) { + throw new Error('eval-config.json task metric must contain a "payload.metric" object'); + } + } + return { tasks: parsed.tasks, max_concurrent_tasks: parsed.max_concurrent_tasks }; +}; diff --git a/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/api.ts b/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/api.ts index 3e13634349..a24036af6d 100644 --- a/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/api.ts +++ b/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/api.ts @@ -11,12 +11,6 @@ import { } from '@nemo/sdk/generated/platform/api'; import type { ModelEntity } from '@nemo/sdk/generated/platform/schema/ModelEntity'; import { PLATFORM_BASE_URL } from '@studio/constants/environment'; -import { - SAMPLE_EVAL_CONFIG_PATH, - SAMPLE_EVAL_DATA_JSON, - SAMPLE_EVAL_DATA_PATH, - SAMPLE_EVAL_YAML, -} from '@studio/routes/agents/AgentSuggestionsRoute/constants'; import type { AgentListing, ApplyResult, @@ -222,52 +216,6 @@ export const uploadToFileset = async ( await filesUploadFile(workspace, OPTIMIZER_FILESET, path, blob, signal); }; -// Ensure the named fileset exists and contains the bundled sample eval config. -// Idempotent: existing files are left untouched (the fileset may have been -// customized by the user). Failures here surface as the apply step failing — -// the agent + deployment are already in place at that point. -export interface EvalSeedFile { - path: string; - content: string; - type: string; -} - -/** Default seed files: the bundled react sample. Used by the optimizer apply - * flow and by the eval modal's fallback. */ -const defaultEvalSeedFiles = (): EvalSeedFile[] => [ - { path: SAMPLE_EVAL_CONFIG_PATH, content: SAMPLE_EVAL_YAML, type: 'application/yaml' }, - { path: SAMPLE_EVAL_DATA_PATH, content: SAMPLE_EVAL_DATA_JSON, type: 'application/json' }, -]; - -export const ensureEvalConfigFileset = async ( - workspace: string, - fileset: string, - signal: AbortSignal, - files: EvalSeedFile[] = defaultEvalSeedFiles(), - description?: string -): Promise => { - let existingPaths = new Set(); - try { - const listing = await filesListFilesetFiles(workspace, fileset, undefined, signal); - existingPaths = new Set((listing?.data ?? []).map((f) => f.path)); - } catch (err) { - if (isCanceledError(err)) throw err; - if (!isNotFoundError(err)) throw err; - try { - await filesCreateFileset(workspace, { name: fileset, description }, signal); - } catch (createErr) { - if (isCanceledError(createErr)) throw createErr; - // 409 is fine — a parallel apply already created it. - } - } - // Idempotent: never overwrite files already present in the fileset. - const uploads = files.filter((f) => !existingPaths.has(f.path)); - for (const u of uploads) { - const blob = new Blob([u.content], { type: u.type }); - await filesUploadFile(workspace, fileset, u.path, blob, signal); - } -}; - const ALLOWED_APPLY_METHODS = new Set(['POST', 'PUT', 'PATCH', 'DELETE']); interface ApplyContext { diff --git a/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/constants.ts b/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/constants.ts index cfcf91891f..4a500e0ecf 100644 --- a/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/constants.ts +++ b/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/constants.ts @@ -39,81 +39,3 @@ export const TYPE_OPTIONS = [ export const SEVERITY_ORDER: Record = { high: 0, medium: 1, low: 2 }; export const STALE_SUGGESTION_MS = 7 * 24 * 60 * 60 * 1000; - -// Bundled sample eval config + dataset, vendored from -// plugins/nemo-agents/examples/react-agent/ so the optimizer's apply flow can -// stand up an eval pipeline without a pre-uploaded one. The eval invokes -// `nat eval` against the agent's running endpoint, so llms.llm is mostly -// unused — the worker LLM is the deployed agent's. llms.judge_llm IS used -// for scoring and must be available in the workspace's inference gateway; -// missing judge fails the eval loudly, which is the correct signal. -export const SAMPLE_EVAL_CONFIG_PATH = 'react-eval.yml'; -export const SAMPLE_EVAL_DATA_PATH = 'react-eval-data.json'; - -export const SAMPLE_EVAL_YAML = `# react-eval.yml — bundled sample seeded by the optimizer apply flow. -# -# Evaluates against the deployed agent endpoint. The judge LLM scores answers -# and must be available in the workspace. - -llms: - llm: - _type: openai - model_name: nvidia-nemotron-3-nano-30b-a3b - temperature: 0.0 - max_tokens: 1024 - - judge_llm: - _type: openai - model_name: nvidia-nemotron-3-super-120b-a12b - temperature: 0.0 - max_tokens: 1024 - -eval: - general: - max_concurrency: 4 - output_dir: eval/agent - dataset: - _type: json - file_path: ${SAMPLE_EVAL_DATA_PATH} - evaluators: - accuracy: - _type: tunable_rag_evaluator - llm_name: judge_llm - default_scoring: true - default_score_weights: - coverage: 0.5 - correctness: 0.3 - relevance: 0.2 - judge_llm_prompt: > - You are an evaluator. Score whether the generated answer correctly - addresses the question compared to the expected answer description. - Rules: - - Score is a float between 0.0 and 1.0. - - 1.0 means the answer fully satisfies the expected answer criteria. - - Provide a 1-2 sentence reasoning. -`; - -export const SAMPLE_EVAL_DATA_JSON = JSON.stringify( - [ - { - id: 1, - question: 'Who invented the telephone, and what is the current time?', - answer: - 'Answer must mention Alexander Graham Bell as the inventor of the telephone and include the current time', - }, - { - id: 2, - question: 'What is the capital of France, and what day of the week is it today?', - answer: - 'Answer must state that the capital of France is Paris and include the current day of the week', - }, - { - id: 3, - question: "When was the theory of general relativity published, and what is today's date?", - answer: - "Answer must mention 1915 as the year general relativity was published and include today's date", - }, - ], - null, - 2 -); diff --git a/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/useOptimizerSuggestions.test.tsx b/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/useOptimizerSuggestions.test.tsx index 2cf3475ca1..177538faa3 100644 --- a/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/useOptimizerSuggestions.test.tsx +++ b/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/useOptimizerSuggestions.test.tsx @@ -24,12 +24,15 @@ const mocks = vi.hoisted(() => ({ waitForEvalJob: vi.fn(), })); +vi.mock('@studio/api/evaluation/eval-config-fileset', () => ({ + ensureEvalConfigFileset: (...args: unknown[]) => mocks.ensureEvalConfigFileset(...args), +})); + vi.mock('@studio/routes/agents/AgentSuggestionsRoute/api', () => ({ applySuggestion: (...args: unknown[]) => mocks.applySuggestion(...args), archivePreviousRun: (...args: unknown[]) => mocks.archivePreviousRun(...args), CONTENT_SAFETY_MODEL_RE: /content[.-]?safety|safety[.-]?guard|gliner/i, checkContentSafety: (...args: unknown[]) => mocks.checkContentSafety(...args), - ensureEvalConfigFileset: (...args: unknown[]) => mocks.ensureEvalConfigFileset(...args), fetchAgents: (...args: unknown[]) => mocks.fetchAgents(...args), fetchEvalAverageScores: (...args: unknown[]) => mocks.fetchEvalAverageScores(...args), fetchModels: (...args: unknown[]) => mocks.fetchModels(...args), diff --git a/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/useOptimizerSuggestions.ts b/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/useOptimizerSuggestions.ts index f9caff84bd..863ce26c0d 100644 --- a/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/useOptimizerSuggestions.ts +++ b/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/useOptimizerSuggestions.ts @@ -1,12 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { ensureEvalConfigFileset } from '@studio/api/evaluation/eval-config-fileset'; import { applySuggestion, archivePreviousRun, CONTENT_SAFETY_MODEL_RE, checkContentSafety, - ensureEvalConfigFileset, fetchAgents, fetchEvalAverageScores, fetchModels, diff --git a/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/utils.ts b/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/utils.ts index 7288f4b797..4f64e01f9c 100644 --- a/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/utils.ts +++ b/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/utils.ts @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 import type { ModelEntity } from '@nemo/sdk/generated/platform/schema/ModelEntity'; +import { SAMPLE_EVAL_CONFIG_PATH } from '@studio/api/evaluation/eval-config-fileset'; import type { AgentConfig } from '@studio/components/dataViews/AgentsDataView'; -import { SAMPLE_EVAL_CONFIG_PATH } from '@studio/routes/agents/AgentSuggestionsRoute/constants'; import type { AgentListing, AnalyzeInput,