diff --git a/web/packages/common/src/components/DatasetFileSelect/ControlledDatasetFileSelect.tsx b/web/packages/common/src/components/DatasetFileSelect/ControlledDatasetFileSelect.tsx index 43e31dd848..fe673d7222 100644 --- a/web/packages/common/src/components/DatasetFileSelect/ControlledDatasetFileSelect.tsx +++ b/web/packages/common/src/components/DatasetFileSelect/ControlledDatasetFileSelect.tsx @@ -9,6 +9,7 @@ import { parseFilesetLocation } from '@nemo/common/src/components/DatasetFileSel import { parseFilesetUrl } from '@nemo/common/src/components/DatasetFileSelect/utils'; import type { FileListItem } from '@nemo/common/src/components/FileList'; import type { UseControllerComponentProps } from '@nemo/common/src/utils/types'; +import type { FilesetPurpose } from '@nemo/sdk/generated/platform/schema'; import { FormField } from '@nvidia/foundations-react-core'; import { FC, useMemo } from 'react'; import { useController } from 'react-hook-form'; @@ -32,6 +33,12 @@ interface ControlledDatasetFileSelectProps extends UseControllerComponentProps { /** Inline-only: skip the "Add" button and commit on selection; also hides * the file list rendered below the picker. */ autoCommit?: boolean; + /** Fileset ``purpose`` the picker lists. Defaults to ``'dataset'``. */ + filesetPurpose?: FilesetPurpose; + /** Label for the fileset picker. Defaults to ``'Dataset'``. */ + datasetLabel?: string; + /** Auto-select the first root-level accepted file on fileset selection. */ + autoSelectFirstAcceptable?: boolean; /** * Callback fired when a file is selected. Useful for custom validation or processing. * Called with the selected file info, or null when file is cleared. @@ -73,6 +80,9 @@ export const ControlledDatasetFileSelect: FC = listLabel, inline, autoCommit, + filesetPurpose, + datasetLabel, + autoSelectFirstAcceptable, }) => { const { field: { onChange, value }, @@ -131,6 +141,9 @@ export const ControlledDatasetFileSelect: FC = listLabel={listLabel} inline={inline} autoCommit={autoCommit} + filesetPurpose={filesetPurpose} + datasetLabel={datasetLabel} + autoSelectFirstAcceptable={autoSelectFirstAcceptable} /> ); diff --git a/web/packages/common/src/components/DatasetFileSelect/DatasetFileSelect.tsx b/web/packages/common/src/components/DatasetFileSelect/DatasetFileSelect.tsx index 08593e5775..3d7a154dd1 100644 --- a/web/packages/common/src/components/DatasetFileSelect/DatasetFileSelect.tsx +++ b/web/packages/common/src/components/DatasetFileSelect/DatasetFileSelect.tsx @@ -10,6 +10,7 @@ import { FileList, FileListItem } from '@nemo/common/src/components/FileList'; import { UploadModal } from '@nemo/common/src/components/UploadModal/index'; import { InlineUploadPicker } from '@nemo/common/src/components/UploadModal/InlineUploadPicker'; import type { SubmitUploadType } from '@nemo/common/src/components/UploadModal/types'; +import type { FilesetPurpose } from '@nemo/sdk/generated/platform/schema'; import { SidePanel, Stack, Text } from '@nvidia/foundations-react-core'; import { FolderOpen } from 'lucide-react'; import { FC, useEffect, useMemo, useRef, useState } from 'react'; @@ -49,6 +50,12 @@ interface DatasetFileSelectProps { * selects a file. Also hides the post-commit file list since the parent * form already reflects the selection. */ autoCommit?: boolean; + /** Fileset ``purpose`` the picker lists. Defaults to ``'dataset'``. */ + filesetPurpose?: FilesetPurpose; + /** Label for the fileset picker. Defaults to ``'Dataset'``. */ + datasetLabel?: string; + /** Auto-select the first root-level accepted file on fileset selection. */ + autoSelectFirstAcceptable?: boolean; } /** @@ -77,6 +84,9 @@ export const DatasetFileSelect: FC = ({ listLabel, inline = false, autoCommit = false, + filesetPurpose, + datasetLabel, + autoSelectFirstAcceptable, }) => { const [isModalOpen, setIsModalOpen] = useState(false); @@ -198,6 +208,9 @@ export const DatasetFileSelect: FC = ({ invalidFileMode={invalidFileMode} onSubmit={handleModalSubmit} autoCommit={autoCommit} + filesetPurpose={filesetPurpose} + datasetLabel={datasetLabel} + autoSelectFirstAcceptable={autoSelectFirstAcceptable} /> ) : ( ; }; diff --git a/web/packages/common/src/components/UploadModal/DatasetUploader/Select.test.tsx b/web/packages/common/src/components/UploadModal/DatasetUploader/Select.test.tsx index d13124900d..458215eb87 100644 --- a/web/packages/common/src/components/UploadModal/DatasetUploader/Select.test.tsx +++ b/web/packages/common/src/components/UploadModal/DatasetUploader/Select.test.tsx @@ -3,7 +3,10 @@ import { UploadModalProvider } from '@nemo/common/src/components/UploadModal/Context/UploadModalProvider'; import { useUploadModalContext } from '@nemo/common/src/components/UploadModal/Context/useUploadModalContext'; -import { UploadModalState } from '@nemo/common/src/components/UploadModal/Context/useUploadModalReducer'; +import { + uploadModalInitialState, + UploadModalState, +} from '@nemo/common/src/components/UploadModal/Context/useUploadModalReducer'; import { DatasetSelect } from '@nemo/common/src/components/UploadModal/DatasetUploader/Select'; import { filesListFilesetFiles, useFilesListFilesets } from '@nemo/sdk/generated/platform/api'; import { FilesetOutput } from '@nemo/sdk/generated/platform/schema'; @@ -57,7 +60,7 @@ const ContextReader = ({ return null; }; -const createWrapper = () => { +const createWrapper = (initialState?: Partial) => { const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false }, @@ -65,11 +68,15 @@ const createWrapper = () => { }); return ({ children }: { children: React.ReactNode }) => ( - {children} + + {children} + ); }; +const filesetFile = (path: string) => ({ path, file_ref: `ref-${path}` }); + describe('DatasetSelect', () => { const user = userEvent.setup(); @@ -182,6 +189,54 @@ describe('DatasetSelect', () => { ); }); + it('auto-selects the first root-level accepted file when autoSelectFirstAcceptable is set', async () => { + vi.mocked(filesListFilesetFiles).mockResolvedValueOnce({ + data: [filesetFile('smaller_test.csv'), filesetFile('email_phishing_analyzer-eval.yml')], + } as Awaited>); + + let contextState: UploadModalState | undefined; + render( + <> + + (contextState = state)} /> + , + { wrapper: createWrapper({ autoSelectFirstAcceptable: true, acceptableFileTypes: ['.yml'] }) } + ); + + await user.click(screen.getByRole('combobox')); + await user.click(await screen.findByRole('option', { name: 'dataset1' })); + + await waitFor(() => { + expect(contextState?.selectedFiles).toHaveLength(1); + }); + expect((contextState?.selectedFiles[0]?.file as { path?: string }).path).toBe( + 'email_phishing_analyzer-eval.yml' + ); + }); + + it('selects nothing when no root-level accepted file exists', async () => { + vi.mocked(filesListFilesetFiles).mockResolvedValueOnce({ + data: [filesetFile('smaller_test.csv'), filesetFile('nested/config.yml')], + } as Awaited>); + + let contextState: UploadModalState | undefined; + render( + <> + + (contextState = state)} /> + , + { wrapper: createWrapper({ autoSelectFirstAcceptable: true, acceptableFileTypes: ['.yml'] }) } + ); + + await user.click(screen.getByRole('combobox')); + await user.click(await screen.findByRole('option', { name: 'dataset1' })); + + await waitFor(() => { + expect(contextState?.dataset?.type).toBe('existing'); + }); + expect(contextState?.selectedFiles).toHaveLength(0); + }); + it('includes "New Dataset" option', async () => { render(, { wrapper: createWrapper(), diff --git a/web/packages/common/src/components/UploadModal/DatasetUploader/Select.tsx b/web/packages/common/src/components/UploadModal/DatasetUploader/Select.tsx index 45de88a018..3a74460794 100644 --- a/web/packages/common/src/components/UploadModal/DatasetUploader/Select.tsx +++ b/web/packages/common/src/components/UploadModal/DatasetUploader/Select.tsx @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { getFileExtension } from '@nemo/common/src/components/DatasetFileSelect/utils'; import { useUploadModalContext } from '@nemo/common/src/components/UploadModal/Context/useUploadModalContext'; import { getExistingFileId } from '@nemo/common/src/components/UploadModal/utils'; import { getEntityReference } from '@nemo/common/src/namedEntity'; @@ -26,7 +27,9 @@ const filesetToOption = (fileset: FilesetOutput) => ({ export const DatasetSelect: FC = ({ project, disabled, error }) => { const [state, dispatch] = useUploadModalContext(); - const { dataset, allowNewDataset } = state; + const { dataset, allowNewDataset, acceptableFileTypes, autoSelectFirstAcceptable } = state; + const purpose = state.filesetPurpose ?? 'dataset'; + const label = state.datasetLabel ?? 'Dataset'; // Extract workspace from project (project format is "workspace/name" or just "workspace") const workspace = project.includes('/') ? project.split('/')[0] : project; @@ -43,7 +46,7 @@ export const DatasetSelect: FC = ({ project, disabled, error }) => { */ page_size: 100, // v2 API max is 100 sort: 'created_at', - filter: { purpose: 'dataset' }, + filter: { purpose }, }); const filesets = useMemo(() => filesetsResponse?.data ?? [], [filesetsResponse]); @@ -67,14 +70,22 @@ export const DatasetSelect: FC = ({ project, disabled, error }) => { try { const filesResponse = await filesListFilesetFiles(fileset.workspace, fileset.name); const filesetFiles = filesResponse.data ?? []; - dispatch({ - type: 'SET_FILES', - payload: filesetFiles.map((file) => ({ - id: getExistingFileId(file), - type: 'existing', - file, - })), - }); + const uploadFiles = filesetFiles.map( + (file) => ({ id: getExistingFileId(file), type: 'existing', file }) as const + ); + dispatch({ type: 'SET_FILES', payload: uploadFiles }); + // Auto-select the first root-level accepted file (only when >1, since + // the reducer already auto-selects a lone file). + if (autoSelectFirstAcceptable && uploadFiles.length > 1) { + const allowed = acceptableFileTypes.map((t) => t.toLowerCase()); + const target = uploadFiles.find((f) => { + const path = f.file.path; + if (path.includes('/')) return false; + const ext = getFileExtension(path)?.toLowerCase(); + return !!ext && allowed.includes(ext); + }); + if (target) dispatch({ type: 'TOGGLE_FILE_SELECTION', payload: target }); + } dispatch({ type: 'SET_FETCHING', payload: false }); } catch (error) { console.error('Error fetching dataset files', error); @@ -100,7 +111,7 @@ export const DatasetSelect: FC = ({ project, disabled, error }) => { return ( @@ -135,14 +146,14 @@ export const DatasetSelect: FC = ({ project, disabled, error }) => { ] : []), { - slotHeading: 'Existing Datasets', + slotHeading: `Existing ${label}s`, attributes: { MenuHeading: { className: 'hidden', 'aria-hidden': true } }, items: datasetOptions, }, ]} value={selectedDatasetOption} onValueChange={handleDatasetSelect} - placeholder="Select a dataset" + placeholder={`Select a ${label.toLowerCase()}`} /> )} diff --git a/web/packages/common/src/components/UploadModal/InlineUploadPicker.tsx b/web/packages/common/src/components/UploadModal/InlineUploadPicker.tsx index 10862867ff..09b539cb23 100644 --- a/web/packages/common/src/components/UploadModal/InlineUploadPicker.tsx +++ b/web/packages/common/src/components/UploadModal/InlineUploadPicker.tsx @@ -22,6 +22,9 @@ type InlineUploadPickerProps = Pick< | 'acceptableFileSize' | 'invalidFileMode' | 'allowNewDataset' + | 'filesetPurpose' + | 'datasetLabel' + | 'autoSelectFirstAcceptable' > & { /** Called once the picked / uploaded file is committed. */ onSubmit: (data: SubmitUploadType) => void; @@ -148,6 +151,9 @@ export const InlineUploadPicker: FC = ({ acceptableFileSize, invalidFileMode, allowNewDataset, + filesetPurpose, + datasetLabel, + autoSelectFirstAcceptable, onSubmit, addButtonText = 'Add file', autoCommit = false, @@ -168,6 +174,10 @@ export const InlineUploadPicker: FC = ({ acceptableFileSize: acceptableFileSize ?? uploadModalInitialState.acceptableFileSize, invalidFileMode: invalidFileMode ?? uploadModalInitialState.invalidFileMode, allowNewDataset: effectiveAllowNewDataset, + filesetPurpose: filesetPurpose ?? uploadModalInitialState.filesetPurpose, + datasetLabel: datasetLabel ?? uploadModalInitialState.datasetLabel, + autoSelectFirstAcceptable: + autoSelectFirstAcceptable ?? uploadModalInitialState.autoSelectFirstAcceptable, }), [ allowMultipleFileSelection, @@ -175,6 +185,9 @@ export const InlineUploadPicker: FC = ({ acceptableFileSize, invalidFileMode, effectiveAllowNewDataset, + filesetPurpose, + datasetLabel, + autoSelectFirstAcceptable, ] ); return ( diff --git a/web/packages/common/src/components/UploadModal/SimpleFilesTable.tsx b/web/packages/common/src/components/UploadModal/SimpleFilesTable.tsx index 5178dc68b4..03788ff2ec 100644 --- a/web/packages/common/src/components/UploadModal/SimpleFilesTable.tsx +++ b/web/packages/common/src/components/UploadModal/SimpleFilesTable.tsx @@ -62,9 +62,11 @@ export const SimpleFilesTable = () => { // eslint-disable-next-line react-hooks/exhaustive-deps }, [files, allowedExtensions, invalidFileMode]); + const hasValidSelection = selectedFiles.some((file) => isFileAllowed(file)); const disabledFilesMessage = invalidFileMode === 'disable' && allowedExtensions.size > 0 && + !hasValidSelection && visibleFiles.some((file) => !isFileAllowed(file)) ? `Only ${acceptableFileTypes.join(', ')} files can be selected. Upload a supported file or choose a different fileset.` : null; @@ -111,6 +113,7 @@ export const SimpleFilesTable = () => { col.accessor('name', { header: 'Name' }), col.accessor('size', { header: 'Size', + size: 120, cell: (ctx) => formatFileSize(ctx.getValue()), }), ], @@ -129,7 +132,8 @@ export const SimpleFilesTable = () => { return ( -
+ {/* Name column fills the row; Size (col 3) is pinned to 120px. */} +
; ModalContent?: React.ComponentProps; diff --git a/web/packages/studio/public/sample-agents/email-phishing-analyzer/eval.yml b/web/packages/studio/public/sample-agents/email-phishing-analyzer/eval.yml index 7b9923913f..078e53f86c 100644 --- a/web/packages/studio/public/sample-agents/email-phishing-analyzer/eval.yml +++ b/web/packages/studio/public/sample-agents/email-phishing-analyzer/eval.yml @@ -20,7 +20,7 @@ llms: eval: general: max_concurrency: 4 - output_dir: eval/agent + output_dir: . dataset: _type: csv file_path: smaller_test.csv diff --git a/web/packages/studio/src/components/DatasetsTable/columns.tsx b/web/packages/studio/src/components/DatasetsTable/columns.tsx index e3776a6ce5..b1f3dc4686 100644 --- a/web/packages/studio/src/components/DatasetsTable/columns.tsx +++ b/web/packages/studio/src/components/DatasetsTable/columns.tsx @@ -67,6 +67,10 @@ export function makeDatasetsTableColumns({ header: 'Name', enableSorting: enableFilters, size: 175, + cell({ row }) { + const name = row.original?.name; + return name ? {name} : null; + }, }), accessor((row) => getStorageBackend(row.storage), { id: 'storage_type', @@ -130,7 +134,7 @@ export function makeDatasetsTableColumns({ cell({ row }) { const path = getStoragePath(row.original?.storage); return path ? ( - + {path} ) : null; diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationsListRoute.test.tsx b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationsListRoute.test.tsx index ee73a80317..d0413648ed 100644 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationsListRoute.test.tsx +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationsListRoute.test.tsx @@ -21,7 +21,7 @@ describe('AgentEvaluationsListRoute', () => { it('renders the page header and submit button', async () => { renderList(); expect(await screen.findByText('Agent Evaluations')).toBeInTheDocument(); - expect(screen.getByRole('button', { name: 'New evaluation' })).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Run Evaluation' })).toBeInTheDocument(); }); it('shows the empty state when no eval jobs are returned (default mock)', async () => { diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationsListRoute.tsx b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationsListRoute.tsx index 21e88e4dbc..ed2519aecb 100644 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationsListRoute.tsx +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationsListRoute.tsx @@ -120,8 +120,8 @@ export const AgentEvaluationsListRoute: FC = () => { slotHeading="Agent Evaluations" slotDescription="Evaluation jobs run against deployed agents — submitted by the optimizer apply flow or directly via the evaluate-agent job API." slotActions={ - } /> diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/SubmitEvaluationModal.tsx b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/SubmitEvaluationModal.tsx index 699f603c43..d818e49fa0 100644 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/SubmitEvaluationModal.tsx +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/SubmitEvaluationModal.tsx @@ -5,84 +5,101 @@ import { zodResolver } from '@hookform/resolvers/zod'; import { ControlledDatasetFileSelect } from '@nemo/common/src/components/DatasetFileSelect/ControlledDatasetFileSelect'; import { parseFilesetLocation } from '@nemo/common/src/components/DatasetFileSelect/parseFilesetLocation'; import { ControlledSelect } from '@nemo/common/src/components/form/ControlledSelect'; +import { ControlledTextInput } from '@nemo/common/src/components/form/ControlledTextInput'; import { FormModal, type FormModalProps } from '@nemo/common/src/components/FormModal'; import { useToast } from '@nemo/common/src/providers/toast/useToast'; import { customFetch } from '@nemo/sdk/generated/fetchers/platform'; -import { Block, RadioGroup, Stack, Text } from '@nvidia/foundations-react-core'; +import { filesCreateFileset } from '@nemo/sdk/generated/platform/api'; +import { SegmentedControl, Stack, Text } from '@nvidia/foundations-react-core'; import { fetchSampleText } from '@studio/api/agents/fetchSampleText'; import { type Agent } from '@studio/components/dataViews/AgentsDataView'; import { PLATFORM_BASE_URL } from '@studio/constants/environment'; import { DEFAULT_SAMPLE_AGENT_KEY, - getSampleAgent, SAMPLE_AGENTS, sampleAgentKeyForAgentName, } from '@studio/constants/sampleAgents'; +import { + fetchAgentEvalJobs, + type AgentEvalJob, +} from '@studio/routes/agents/AgentEvaluationsRoute/api'; +import { + buildSubmitSpec, + CREATE_NEW, + evalOutputDescription, + evaluateRequestBody, + generateEvalConfigName, + generateOutputFilesetName, + MODE_DEFAULT, + MODE_FILESET, + type SubmitSpec, +} from '@studio/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec'; import { ensureEvalConfigFileset, type EvalSeedFile, } from '@studio/routes/agents/AgentSuggestionsRoute/api'; -import { - evalFilesetForAgent, - evalOutputFilesetFor, -} from '@studio/routes/agents/AgentSuggestionsRoute/utils'; import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; -import { type FC, useEffect } from 'react'; +import { type FC, useEffect, useMemo, useRef } from 'react'; import { type SubmitHandler, useForm, useWatch } from 'react-hook-form'; import { z } from 'zod'; -const MODE_DEFAULT = 'default'; -const MODE_FILESET = 'fileset'; - const EVAL_CONFIG_MODE_ITEMS = [ - { value: MODE_DEFAULT, children: 'Use example evaluation config' }, - { value: MODE_FILESET, children: 'Select or upload a config file from a fileset' }, + { value: MODE_DEFAULT, children: 'Use Example' }, + { value: MODE_FILESET, children: 'Choose Fileset' }, ]; -const contentTypeForFile = (name: string): string => { - if (name.endsWith('.json')) return 'application/json'; - if (name.endsWith('.csv')) return 'text/csv'; - return 'application/yaml'; +/** Strip an optional ``workspace/`` prefix so agent references compare by name. */ +const bareName = (value?: string | null): string | null => { + if (typeof value !== 'string' || value.length === 0) return null; + return value.includes('/') ? (value.split('/').pop() ?? null) : value; }; -/** Basename of a public asset path — the flat name it's seeded as in the fileset. */ -const fileNameOf = (path: string): string => path.slice(path.lastIndexOf('/') + 1); - const submitEvaluationSchema = z .object({ agent: z.string().min(1, 'Agent is required'), + // Existing eval-config fileset to reuse, or CREATE_NEW to make one. + evalConfig: z.string().min(1, 'Select or create an eval config'), + // Create-mode fields (only enforced when evalConfig === CREATE_NEW). + newName: z.string(), mode: z.enum([MODE_DEFAULT, MODE_FILESET]), exampleKey: z.string(), datasetFile: z.string().nullable(), }) - .refine( - (data) => - data.mode !== MODE_FILESET || - (typeof data.datasetFile === 'string' && - !!parseFilesetLocation(data.datasetFile)?.objectPath), - { - message: 'Pick an eval YAML inside an existing fileset', - path: ['datasetFile'], + .superRefine((data, ctx) => { + if (data.evalConfig !== CREATE_NEW) return; + if (data.mode === MODE_DEFAULT) { + // newName becomes the fileset name — enforce the platform naming rules. + const name = data.newName.trim(); + if (!name) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Name is required', + path: ['newName'], + }); + } else if (!/^[a-zA-Z0-9_.-]+$/.test(name)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Use only letters, digits, dots, hyphens, and underscores', + path: ['newName'], + }); + } } - ); + if (data.mode === MODE_FILESET && !parseFilesetLocation(data.datasetFile ?? '')?.objectPath) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Pick an eval YAML inside an existing fileset', + path: ['datasetFile'], + }); + } + }); type SubmitEvaluationFormData = z.infer; -const SUFFIX_LENGTH = 5; -const SUFFIX_ALPHABET = 'abcdefghijklmnopqrstuvwxyz0123456789'; - -/** Mirrors the optimizer's randomSiblingSuffix so this surface looks the - * same; isolated copy so the form doesn't reach into utils.ts internals. */ -const randomSuffix = (): string => { - const bytes = new Uint8Array(SUFFIX_LENGTH); - crypto.getRandomValues(bytes); - let out = ''; - for (const b of bytes) out += SUFFIX_ALPHABET[b % SUFFIX_ALPHABET.length]; - return out; -}; - const makeDefaultValues = (agent?: string): SubmitEvaluationFormData => ({ agent: agent ?? '', + // Default to create; existing configs are one click away in the dropdown. + evalConfig: CREATE_NEW, + newName: generateEvalConfigName(), mode: MODE_DEFAULT, // Auto-match the example to the agent it was created from (by name prefix), // falling back to the first example for non-example agents. @@ -101,24 +118,6 @@ interface SubmitEvaluationModalProps extends Pick void; } -interface EvalSeedSource { - /** Flat filename seeded into the fileset. */ - path: string; - /** Public asset path fetched on demand for the file's content. */ - assetPath: string; - type: string; -} - -interface SubmitSpec { - agent: string; - evalConfig: string; - evalConfigFileset: string; - /** When set, fetch each source and seed it into ``evalConfigFileset`` before - * POSTing to ``/jobs/evaluate``. Omitted when the user picks an existing - * fileset since we shouldn't overwrite their files. */ - seedSources?: EvalSeedSource[]; -} - export const SubmitEvaluationModal: FC = ({ open, onClose, @@ -142,6 +141,13 @@ export const SubmitEvaluationModal: FC = ({ enabled: open && !agentProp, }); + // Prior eval jobs — the source for the "existing eval config" dropdown. + const { data: jobs = [], isLoading: isJobsLoading } = useQuery({ + queryKey: ['agent-eval-jobs', workspace], + queryFn: ({ signal }) => fetchAgentEvalJobs(workspace, signal), + enabled: open, + }); + const { mutateAsync: submitEvaluation, error: submitError, @@ -150,9 +156,7 @@ export const SubmitEvaluationModal: FC = ({ } = useMutation({ mutationFn: async (spec: SubmitSpec) => { if (spec.seedSources) { - // Default mode: fetch the selected example's eval assets on demand and - // seed them into the per-agent eval-config fileset so the user doesn't - // have to pre-upload anything. + // Seed the selected example's eval assets into the new config fileset. const files: EvalSeedFile[] = await Promise.all( spec.seedSources.map(async (source) => ({ path: source.path, @@ -164,19 +168,23 @@ export const SubmitEvaluationModal: FC = ({ workspace, spec.evalConfigFileset, new AbortController().signal, - files + files, + 'Agent Evaluation Config' ); } - const body = { - spec: { - agent: spec.agent, - eval_config: spec.evalConfig, - eval_config_fileset: spec.evalConfigFileset, - // Auto-generated per submission so re-running for the same agent - // doesn't 409 on an existing output fileset. - output: `${evalOutputFilesetFor(spec.agent)}-${randomSuffix()}`, - }, - }; + // Pre-create the output fileset so it carries a description; the job's + // auto-create no-ops once it exists. Best-effort — never block submission. + const outputFileset = generateOutputFilesetName(spec.agent); + try { + await filesCreateFileset(workspace, { + name: outputFileset, + description: evalOutputDescription(spec), + purpose: 'generic', + }); + } catch { + // Job still auto-creates the fileset (without a description). + } + const body = evaluateRequestBody(spec, outputFileset); const res = await customFetch<{ name?: string }>({ url: `/apis/agents/v2/workspaces/${encodeURIComponent(workspace)}/jobs/evaluate`, method: 'POST', @@ -212,15 +220,58 @@ export const SubmitEvaluationModal: FC = ({ reValidateMode: 'onChange', }); + const evalConfig = useWatch({ control, name: 'evalConfig' }); const mode = useWatch({ control, name: 'mode' }); const selectedAgent = useWatch({ control, name: 'agent' }); + // Existing eval configs for the agent: distinct config filesets from prior + // jobs, mapped to the YAML each ran. + const existingConfigs = useMemo(() => { + const map = new Map(); + const agentKey = bareName(selectedAgent); + if (!agentKey) return map; + for (const job of jobs as AgentEvalJob[]) { + if (bareName(job.spec.agent) !== agentKey) continue; + const fileset = job.spec.eval_config_fileset; + if (typeof fileset === 'string' && fileset.length > 0 && !map.has(fileset)) { + map.set(fileset, job.spec.eval_config ?? ''); + } + } + return map; + }, [jobs, selectedAgent]); + + const evalConfigItems = useMemo( + () => [ + ...Array.from(existingConfigs.keys()).map((fileset) => ({ + value: fileset, + children: fileset, + })), + { value: CREATE_NEW, children: '+ Create new eval config' }, + ], + [existingConfigs] + ); + // When the chosen agent maps to a known example, auto-select its eval config. useEffect(() => { const matchedKey = sampleAgentKeyForAgentName(selectedAgent); if (matchedKey) setValue('exampleKey', matchedKey); }, [selectedAgent, setValue]); + // Preselect the latest existing config for the agent (else create). Ref-guarded + // to run once per agent so it doesn't override the user's later manual pick. + const autoSelectedAgentRef = useRef(null); + useEffect(() => { + if (!open) { + autoSelectedAgentRef.current = null; + return; + } + if (isJobsLoading || !selectedAgent) return; + if (autoSelectedAgentRef.current === selectedAgent) return; + autoSelectedAgentRef.current = selectedAgent; + const latest = existingConfigs.keys().next().value; + setValue('evalConfig', latest ?? CREATE_NEW); + }, [open, isJobsLoading, selectedAgent, existingConfigs, setValue]); + useEffect(() => { resetForm(makeDefaultValues(agentProp)); }, [agentProp, resetForm]); @@ -242,44 +293,8 @@ export const SubmitEvaluationModal: FC = ({ }; const onSubmit: SubmitHandler = async (formData) => { - let spec: SubmitSpec; - if (formData.mode === MODE_FILESET) { - // Schema refine guarantees ``datasetFile`` parses to a fileset reference - // with a non-empty ``objectPath`` before reaching this point. - const parsed = parseFilesetLocation(formData.datasetFile!)!; - spec = { - agent: formData.agent, - evalConfig: parsed.objectPath, - evalConfigFileset: parsed.name, - }; - } else { - const example = getSampleAgent(formData.exampleKey); - // Namespace the seeded config per example. The {agent}-eval fileset is - // shared and ensureEvalConfigFileset skips existing files, so seeding every - // example as a bare "eval.yml" would make the first-seeded config stick when - // switching examples on the same agent. (Datasets already have distinct - // basenames.) - const evalConfigName = `${example.key}-${fileNameOf(example.evalConfigPath)}`; - spec = { - agent: formData.agent, - evalConfig: evalConfigName, - evalConfigFileset: evalFilesetForAgent(formData.agent), - seedSources: [ - { - path: evalConfigName, - assetPath: example.evalConfigPath, - type: contentTypeForFile(example.evalConfigPath), - }, - { - path: fileNameOf(example.evalDataPath), - assetPath: example.evalDataPath, - type: contentTypeForFile(example.evalDataPath), - }, - ], - }; - } try { - await submitEvaluation(spec); + await submitEvaluation(buildSubmitSpec(formData, existingConfigs)); } catch { // Error rendered via errorText prop. } @@ -292,16 +307,19 @@ export const SubmitEvaluationModal: FC = ({ ? 'An error occurred' : undefined; + const isCreating = evalConfig === CREATE_NEW; + return ( {agentProp ? ( @@ -321,53 +339,79 @@ export const SubmitEvaluationModal: FC = ({ }} /> )} - - - Evaluation config - - { - setValue('mode', v as typeof MODE_DEFAULT | typeof MODE_FILESET, { - shouldValidate: false, - }); - clearErrors('datasetFile'); - }} - items={EVAL_CONFIG_MODE_ITEMS} - /> - - {mode === MODE_DEFAULT ? ( - ({ - value: example.key, - children: example.label, - }))} - formFieldProps={{ - slotLabel: 'Example', - slotError: errors.exampleKey?.message, - }} - /> - ) : null} - {mode === MODE_FILESET ? ( - setError('datasetFile', error)} - clearError={() => clearErrors('datasetFile')} - workspace={workspace} - inline - autoCommit - formFieldProps={{ - slotError: errors.datasetFile?.message, - }} - /> + {selectedAgent ? ( + + + Eval Config + + + {isCreating && ( + <> + { + setValue('mode', v as typeof MODE_DEFAULT | typeof MODE_FILESET, { + shouldValidate: false, + }); + clearErrors('datasetFile'); + }} + items={EVAL_CONFIG_MODE_ITEMS} + /> + {mode === MODE_DEFAULT ? ( + <> + ({ + value: example.key, + children: example.label, + }))} + formFieldProps={{ + slotLabel: 'Example', + slotError: errors.exampleKey?.message, + }} + /> + + + ) : ( + setError('datasetFile', error)} + clearError={() => clearErrors('datasetFile')} + workspace={workspace} + inline + autoCommit + autoSelectFirstAcceptable + filesetPurpose="generic" + datasetLabel="Fileset" + formFieldProps={{ + slotError: errors.datasetFile?.message, + }} + /> + )} + + )} + ) : null} diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.test.ts b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.test.ts new file mode 100644 index 0000000000..d61e761170 --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.test.ts @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { SAMPLE_AGENTS } from '@studio/constants/sampleAgents'; +import { + buildSubmitSpec, + CREATE_NEW, + evalOutputDescription, + evaluateRequestBody, + generateOutputFilesetName, +} from '@studio/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec'; + +const baseForm = { + agent: 'my-agent', + evalConfig: CREATE_NEW, + newName: '', + mode: 'default' as const, + exampleKey: SAMPLE_AGENTS[0].key, + datasetFile: null as string | null, +}; + +describe('buildSubmitSpec', () => { + it('reuses an existing eval config untouched (no seed sources)', () => { + const existing = new Map([['wise-pretzel', 'analyzer-eval.yml']]); + const spec = buildSubmitSpec({ ...baseForm, evalConfig: 'wise-pretzel' }, existing); + + expect(spec).toEqual({ + agent: 'my-agent', + evalConfig: 'analyzer-eval.yml', + evalConfigFileset: 'wise-pretzel', + }); + expect(spec.seedSources).toBeUndefined(); + }); + + it('creates a new slug fileset and seeds the example config + dataset', () => { + const spec = buildSubmitSpec( + { ...baseForm, evalConfig: CREATE_NEW, mode: 'default', newName: ' wise-pretzel ' }, + new Map() + ); + + // Trimmed slug becomes the eval-config fileset (also the output target). + expect(spec.evalConfigFileset).toBe('wise-pretzel'); + expect(spec.evalConfig.startsWith(`${SAMPLE_AGENTS[0].key}-`)).toBe(true); + expect(spec.seedSources).toHaveLength(2); + }); + + it("reuses the picked file's own fileset when creating from a fileset YAML", () => { + const spec = buildSubmitSpec( + { + ...baseForm, + evalConfig: CREATE_NEW, + mode: 'fileset', + datasetFile: 'default/my-fs#eval.yml', + }, + new Map() + ); + + expect(spec.evalConfigFileset).toBe('my-fs'); + expect(spec.evalConfig).toBe('eval.yml'); + expect(spec.seedSources).toBeUndefined(); + }); +}); + +describe('generateOutputFilesetName', () => { + it('mints a fresh per-run -eval-out- name', () => { + const a = generateOutputFilesetName('my-agent'); + const b = generateOutputFilesetName('my-agent'); + expect(a).toMatch(/^my-agent-eval-out-[a-z0-9]{5}$/); + expect(a).not.toBe(b); // random suffix differs per call + }); +}); + +describe('evalOutputDescription', () => { + it('describes the agent and eval-config fileset', () => { + expect( + evalOutputDescription({ + agent: 'my-agent', + evalConfig: 'eval.yml', + evalConfigFileset: 'wise-pretzel', + }) + ).toBe('Agent Evaluation output, agent: my-agent, config: wise-pretzel'); + }); +}); + +describe('evaluateRequestBody', () => { + it('sends the chosen eval-config fileset and the given per-run output fileset', () => { + const body = evaluateRequestBody( + { agent: 'my-agent', evalConfig: 'eval.yml', evalConfigFileset: 'wise-pretzel' }, + 'my-agent-eval-out-ab3d9' + ); + + expect(body.spec.eval_config).toBe('eval.yml'); + expect(body.spec.eval_config_fileset).toBe('wise-pretzel'); + // Output is a distinct per-run fileset, not the config fileset. + expect(body.spec.output).toBe('my-agent-eval-out-ab3d9'); + expect(body.spec.output).not.toBe(body.spec.eval_config_fileset); + }); +}); diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.ts b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.ts new file mode 100644 index 0000000000..767d300297 --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.ts @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { parseFilesetLocation } from '@nemo/common/src/components/DatasetFileSelect/parseFilesetLocation'; +import { generateDefaultName } from '@nemo/common/src/utils/generateDefaultName'; +import { getSampleAgent } from '@studio/constants/sampleAgents'; +import { evalOutputFilesetFor } from '@studio/routes/agents/AgentSuggestionsRoute/utils'; + +export const MODE_DEFAULT = 'default'; +export const MODE_FILESET = 'fileset'; + +/** Sentinel ``evalConfig`` value that switches the form into create mode. */ +export const CREATE_NEW = '__create_new__'; + +/** Suggested name for a new eval config (e.g. "wise-blue"). */ +export const generateEvalConfigName = (): string => generateDefaultName({ length: 2 }); + +/** Form values the eval-submit modal collects. */ +export interface SubmitEvaluationFormValues { + agent: string; + /** Existing eval-config fileset to reuse, or CREATE_NEW to make one. */ + evalConfig: string; + newName: string; + mode: typeof MODE_DEFAULT | typeof MODE_FILESET; + exampleKey: string; + datasetFile: string | null; +} + +export interface EvalSeedSource { + /** Flat filename seeded into the fileset. */ + path: string; + /** Public asset path fetched on demand for the file's content. */ + assetPath: string; + type: string; +} + +export interface SubmitSpec { + agent: string; + evalConfig: string; + evalConfigFileset: string; + /** Files to seed into ``evalConfigFileset`` before submitting. Omitted when + * reusing an existing config. */ + seedSources?: EvalSeedSource[]; +} + +export const contentTypeForFile = (name: string): string => { + if (name.endsWith('.json')) return 'application/json'; + if (name.endsWith('.csv')) return 'text/csv'; + return 'application/yaml'; +}; + +/** Basename of a public asset path — the flat name it's seeded as in the fileset. */ +export const fileNameOf = (path: string): string => path.slice(path.lastIndexOf('/') + 1); + +/** Builds the eval-job spec from the form (reuse existing config, pick a + * fileset YAML, or seed an example into a new fileset). */ +export const buildSubmitSpec = ( + formData: SubmitEvaluationFormValues, + existingConfigs: Map +): SubmitSpec => { + if (formData.evalConfig !== CREATE_NEW) { + return { + agent: formData.agent, + evalConfig: existingConfigs.get(formData.evalConfig) ?? '', + evalConfigFileset: formData.evalConfig, + }; + } + if (formData.mode === MODE_FILESET) { + // datasetFile is validated by the schema refine before we get here. + const parsed = parseFilesetLocation(formData.datasetFile!)!; + return { + agent: formData.agent, + evalConfig: parsed.objectPath, + evalConfigFileset: parsed.name, + }; + } + const example = getSampleAgent(formData.exampleKey); + // Namespace the config per example so switching examples doesn't reuse the + // first-seeded config. + const evalConfigName = `${example.key}-${fileNameOf(example.evalConfigPath)}`; + return { + agent: formData.agent, + evalConfig: evalConfigName, + evalConfigFileset: formData.newName.trim(), + seedSources: [ + { + path: evalConfigName, + assetPath: example.evalConfigPath, + type: contentTypeForFile(example.evalConfigPath), + }, + { + path: fileNameOf(example.evalDataPath), + assetPath: example.evalDataPath, + type: contentTypeForFile(example.evalDataPath), + }, + ], + }; +}; + +const OUTPUT_SUFFIX_LENGTH = 5; +const OUTPUT_SUFFIX_ALPHABET = 'abcdefghijklmnopqrstuvwxyz0123456789'; + +/** Random 5-char suffix so re-runs don't 409 on an existing output fileset. */ +const randomSuffix = (): string => { + const bytes = new Uint8Array(OUTPUT_SUFFIX_LENGTH); + crypto.getRandomValues(bytes); + let out = ''; + for (const b of bytes) out += OUTPUT_SUFFIX_ALPHABET[b % OUTPUT_SUFFIX_ALPHABET.length]; + return out; +}; + +/** Fresh per-run output fileset name (``-eval-out-``). */ +export const generateOutputFilesetName = (agent: string): string => + `${evalOutputFilesetFor(agent)}-${randomSuffix()}`; + +/** Description stamped on the eval output fileset. */ +export const evalOutputDescription = (spec: SubmitSpec): string => + `Agent Evaluation output, agent: ${spec.agent}, config: ${spec.evalConfigFileset}`; + +/** POST body for ``/jobs/evaluate``. */ +export const evaluateRequestBody = (spec: SubmitSpec, output: string) => ({ + spec: { + agent: spec.agent, + eval_config: spec.evalConfig, + eval_config_fileset: spec.evalConfigFileset, + output, + }, +}); diff --git a/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/api.ts b/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/api.ts index f54587ea57..3e13634349 100644 --- a/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/api.ts +++ b/web/packages/studio/src/routes/agents/AgentSuggestionsRoute/api.ts @@ -243,7 +243,8 @@ export const ensureEvalConfigFileset = async ( workspace: string, fileset: string, signal: AbortSignal, - files: EvalSeedFile[] = defaultEvalSeedFiles() + files: EvalSeedFile[] = defaultEvalSeedFiles(), + description?: string ): Promise => { let existingPaths = new Set(); try { @@ -253,7 +254,7 @@ export const ensureEvalConfigFileset = async ( if (isCanceledError(err)) throw err; if (!isNotFoundError(err)) throw err; try { - await filesCreateFileset(workspace, { name: fileset }, signal); + await filesCreateFileset(workspace, { name: fileset, description }, signal); } catch (createErr) { if (isCanceledError(createErr)) throw createErr; // 409 is fine — a parallel apply already created it.