diff --git a/web/packages/studio/src/components/ModelComparePrompts/ExpandableCell.tsx b/web/packages/common/src/components/DataView/TableExpandableCell/index.tsx similarity index 82% rename from web/packages/studio/src/components/ModelComparePrompts/ExpandableCell.tsx rename to web/packages/common/src/components/DataView/TableExpandableCell/index.tsx index 486f8047eb..9a35865916 100644 --- a/web/packages/studio/src/components/ModelComparePrompts/ExpandableCell.tsx +++ b/web/packages/common/src/components/DataView/TableExpandableCell/index.tsx @@ -2,15 +2,18 @@ // SPDX-License-Identifier: Apache-2.0 import { Text } from '@nvidia/foundations-react-core'; -import type { ExpandedCellState } from '@studio/components/ModelComparePrompts/types'; import { Maximize2 } from 'lucide-react'; import type { FC, ReactNode } from 'react'; -/** Table cell with vertical scroll and an expand-to-modal button */ -export const ExpandableCell: FC<{ +export interface TableExpandableCellState { + title: string; + content: string; +} + +export const TableExpandableCell: FC<{ content: string; title: string; - onExpand: (state: ExpandedCellState) => void; + onExpand: (state: TableExpandableCellState) => void; footer?: ReactNode; boldContent?: boolean; }> = ({ content, title, onExpand, footer, boldContent }) => { diff --git a/web/packages/studio/src/api/evaluation/agent-evaluations.test.ts b/web/packages/studio/src/api/evaluation/agent-evaluations.test.ts index 85015a8b6e..0a60c030ba 100644 --- a/web/packages/studio/src/api/evaluation/agent-evaluations.test.ts +++ b/web/packages/studio/src/api/evaluation/agent-evaluations.test.ts @@ -7,8 +7,10 @@ import { type AgentEvalResult, agentNameForJob, aggregateScoresOf, + evalConfigName, fetchAgentEvalJob, fetchAgentEvalJobs, + fetchAgentEvalResultsForJobs, joinBundleByTask, parseBundleRef, } from '@studio/api/evaluation/agent-evaluations'; @@ -88,6 +90,48 @@ describe('agentNameForJob', () => { }); }); +describe('evalConfigName', () => { + it('reads the fileset name from spec.benchmark.eval_config', () => { + const job = baseJob({ + spec: { + target: { kind: 'agent', agent: { name: 'a' } }, + tasks: [{}], + benchmark: { eval_config_fileset: 'wise-blue' }, + } as unknown as AgentEvaluateJob['spec'], + }); + expect(evalConfigName(job)).toBe('wise-blue'); + }); + + it('returns null when benchmark is absent, ignoring any description', () => { + expect(evalConfigName(baseJob({ description: 'legacy-desc' }))).toBeNull(); + }); +}); + +describe('fetchAgentEvalResultsForJobs', () => { + it('returns an empty map without fetching when there are no job names', async () => { + const map = await fetchAgentEvalResultsForJobs('ws-a', [], new AbortController().signal); + expect(map.size).toBe(0); + expect(customFetchMock).not.toHaveBeenCalled(); + }); + + it('keys results by name for the requested jobs', async () => { + customFetchMock.mockResolvedValue({ + data: [ + { name: 'eval-1', scores: { scores: [] } }, + { name: 'eval-2', scores: { scores: [] } }, + ], + }); + const map = await fetchAgentEvalResultsForJobs( + 'ws-a', + ['eval-1', 'eval-2'], + new AbortController().signal + ); + expect(map.get('eval-1')?.name).toBe('eval-1'); + expect(map.get('eval-2')?.name).toBe('eval-2'); + expect(map.size).toBe(2); + }); +}); + describe('aggregateScoresOf', () => { it('flattens the nested scores shape', () => { const result = { diff --git a/web/packages/studio/src/api/evaluation/agent-evaluations.ts b/web/packages/studio/src/api/evaluation/agent-evaluations.ts index 9eb9efbf05..16f7c9b6a8 100644 --- a/web/packages/studio/src/api/evaluation/agent-evaluations.ts +++ b/web/packages/studio/src/api/evaluation/agent-evaluations.ts @@ -1,11 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { withOperators } from '@nemo/common/src/api/filterOperators'; import { evaluatorCancelAgentEvaluateJob, evaluatorCreateAgentEvaluateJob, evaluatorGetAgentEvalResult, evaluatorGetAgentEvaluateJob, + evaluatorListAgentEvalResults, evaluatorListAgentEvaluateJobs, } from '@nemo/sdk/generated/evaluator/api'; import type { @@ -15,6 +17,7 @@ import type { AgentEvaluateJobRequest, AgentEvalResult, AgentEvaluateJobsSortField, + ResultFilter, } from '@nemo/sdk/generated/evaluator/schema'; import { filesDownloadFile } from '@nemo/sdk/generated/platform/api'; @@ -30,7 +33,10 @@ export type { AgentEvalResult }; * Strips only this job's own ``workspace/`` prefix so the result compares equal to a * bare agent name; any other ``/`` in the name is left intact. */ export const agentNameForJob = (job: AgentEvaluateJob): string | null => { - const target = job.spec.target as { kind?: string; agent?: { name?: string } } | null | undefined; + const target = job.spec?.target as + | { kind?: string; agent?: { name?: string } } + | null + | undefined; if (!target || target.kind !== 'agent') return null; const name = target.agent?.name; if (typeof name !== 'string' || name.length === 0) return null; @@ -38,6 +44,11 @@ export const agentNameForJob = (job: AgentEvaluateJob): string | null => { return prefix && name.startsWith(prefix) ? name.slice(prefix.length) : name; }; +export const evalConfigName = (job: AgentEvaluateJob): string | null => { + const name = job.spec?.benchmark?.eval_config_fileset; + return typeof name === 'string' && name.length > 0 ? name : null; +}; + export const fetchAgentEvalJobs = async ( workspace: string, signal: AbortSignal @@ -108,6 +119,26 @@ export const fetchAgentEvalResult = async ( } }; +const RESULTS_PAGE_MAX = 100; + +export const fetchAgentEvalResultsForJobs = async ( + workspace: string, + jobNames: string[], + signal: AbortSignal +): Promise> => { + if (jobNames.length === 0) return new Map(); + const page = await evaluatorListAgentEvalResults( + workspace, + { + page: 1, + page_size: Math.min(jobNames.length, RESULTS_PAGE_MAX), + filter: withOperators({ name: { $in: jobNames } }), + }, + signal + ); + return new Map((page?.data ?? []).map((result) => [result.name, result])); +}; + /** One trial row from trials.jsonl — the agent's response to a task. */ export interface AgentEvalTrialRow { id: string; diff --git a/web/packages/studio/src/components/ModelComparePrompts/ModelCompareTable.tsx b/web/packages/studio/src/components/ModelComparePrompts/ModelCompareTable.tsx index 25acc0c5b6..e5a642d63d 100644 --- a/web/packages/studio/src/components/ModelComparePrompts/ModelCompareTable.tsx +++ b/web/packages/studio/src/components/ModelComparePrompts/ModelCompareTable.tsx @@ -2,13 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 import type { ModelWorkspaceGroup } from '@nemo/common/src/api/models/useModels'; +import { TableExpandableCell } from '@nemo/common/src/components/DataView/TableExpandableCell'; import { getPartsFromReference } from '@nemo/common/src/namedEntity'; import type { FileSampleMethod } from '@nemo/common/src/utils/sampleTextLines'; import { Button, Flex, Select, Text } from '@nvidia/foundations-react-core'; import { StatsBadge } from '@studio/components/chat/StatsBadge'; import type { DatasetInputFileResult } from '@studio/components/DatasetInputFile'; import { FileSamplingMethodSelect } from '@studio/components/FileSamplingSnippet/FileSamplingMethodSelect'; -import { ExpandableCell } from '@studio/components/ModelComparePrompts/ExpandableCell'; import { ModelColumnSelect } from '@studio/components/ModelComparePrompts/ModelColumnSelect'; import type { ExpandedCellState, @@ -221,7 +221,7 @@ export const ModelCompareTable: FC = ({ return ( - = ({ } return ( - setExpandedCell({ ...state, stats: response.stats })} diff --git a/web/packages/studio/src/components/ModelComparePrompts/types.ts b/web/packages/studio/src/components/ModelComparePrompts/types.ts index f9e58928fa..2644cf9280 100644 --- a/web/packages/studio/src/components/ModelComparePrompts/types.ts +++ b/web/packages/studio/src/components/ModelComparePrompts/types.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import type { ModelWorkspaceGroup } from '@nemo/common/src/api/models/useModels'; +import type { TableExpandableCellState } from '@nemo/common/src/components/DataView/TableExpandableCell'; import type { SharedModelEntry } from '@studio/routes/ModelCompareRoute/types'; export interface ResponseStats { @@ -27,9 +28,7 @@ export interface PromptRow { responses: Record; } -export interface ExpandedCellState { - title: string; - content: string; +export interface ExpandedCellState extends TableExpandableCellState { stats?: ResponseStats; } diff --git a/web/packages/studio/src/components/dataViews/AgentEvaluationsDataView/index.tsx b/web/packages/studio/src/components/dataViews/AgentEvaluationsDataView/index.tsx new file mode 100644 index 0000000000..02728f7d3a --- /dev/null +++ b/web/packages/studio/src/components/dataViews/AgentEvaluationsDataView/index.tsx @@ -0,0 +1,296 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { withOperators } from '@nemo/common/src/api/filterOperators'; +import { + ROW_ACTIONS_COLUMN_SIZE, + ROW_SELECTION_COLUMN_SIZE, + StudioDataView, +} from '@nemo/common/src/components/DataView/StudioDataView'; +import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; +import { StatusBadge } from '@nemo/common/src/components/StatusBadge'; +import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; +import { JOB_POLLING_INTERVAL_MS } from '@nemo/common/src/constants'; +import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; +import { getSortParamWithWhitelist } from '@nemo/common/src/utils/query'; +import { + getEvaluatorListAgentEvaluateJobsQueryKey, + useEvaluatorDeleteAgentEvaluateJob, + useEvaluatorListAgentEvaluateJobs, +} from '@nemo/sdk/generated/evaluator/api'; +import { + type AgentEvaluateJob, + type AgentEvaluateJobsListFilter, + AgentEvaluateJobsSortField, +} from '@nemo/sdk/generated/evaluator/schema'; +import { Button, Stack, Text } from '@nvidia/foundations-react-core'; +import { + aggregateScoresOf, + agentNameForJob, + evalConfigName, + fetchAgentEvalResultsForJobs, +} from '@studio/api/evaluation/agent-evaluations'; +import { BulkDeleteModal } from '@studio/components/BulkDeleteModal'; +import { QuickActionsMenuRoot } from '@studio/components/QuickActionsMenu/QuickActionsMenuRoot'; +import { STATUS_FILTER_OPTIONS } from '@studio/constants/platformJobs'; +import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; +import { formatScore } from '@studio/routes/agents/AgentEvaluationsRoute/evalScores'; +import { getAgentEvaluationDetailRoute, getFilesetDetailRoute } from '@studio/routes/utils'; +import { getTextWithCount } from '@studio/util/strings'; +import { keepPreviousData, useQuery, useQueryClient } from '@tanstack/react-query'; +import { Trash } from 'lucide-react'; +import { ComponentProps, useMemo, useState } from 'react'; +import { Link, useNavigate } from 'react-router-dom'; + +type AgentEvalJobRow = AgentEvaluateJob & { id: string }; + +const STATUS_OPTIONS_WITH_ALL = [{ value: '', label: 'All' }, ...STATUS_FILTER_OPTIONS]; + +const SORTABLE_FIELDS = Object.values(AgentEvaluateJobsSortField).filter((v) => !v.startsWith('-')); +const DEFAULT_SORT = AgentEvaluateJobsSortField['-created_at']; + +export const AgentEvaluationsDataView = () => { + const workspace = useWorkspaceFromPath(); + const navigate = useNavigate(); + const queryClient = useQueryClient(); + + const [deleteJobs, setDeleteJobs] = useState([]); + + const dataViewState = useStudioDataViewState({ + defaultSort: [{ id: 'created_at', desc: true }], + }); + + const deleteJobMutation = useEvaluatorDeleteAgentEvaluateJob({ + mutation: { + onSuccess: () => + queryClient.resetQueries({ + queryKey: getEvaluatorListAgentEvaluateJobsQueryKey(workspace), + }), + }, + }); + + const handleDeleteJobs = async (jobsToDelete: AgentEvalJobRow[]) => { + const results = await Promise.allSettled( + jobsToDelete.map((job) => deleteJobMutation.mutateAsync({ workspace, name: job.name })) + ); + const failures = results.flatMap((result, i) => + result.status === 'rejected' + ? [ + `"${jobsToDelete[i].name}": ${result.reason instanceof Error ? result.reason.message : 'Unknown error'}`, + ] + : [] + ); + if (failures.length > 0) { + throw new Error( + `Failed to delete ${failures.length} evaluation${failures.length !== 1 ? 's' : ''}: ${failures.join('; ')}` + ); + } + }; + + const { + data: jobsData, + isLoading, + error, + } = useEvaluatorListAgentEvaluateJobs( + workspace, + { + page: dataViewState.pagination.state.pageIndex + 1, + page_size: dataViewState.pagination.state.pageSize, + sort: getSortParamWithWhitelist( + dataViewState.sorting.state, + SORTABLE_FIELDS, + DEFAULT_SORT + ) as AgentEvaluateJobsSortField, + filter: { + ...dataViewState.apiFilter.filter, + ...(dataViewState.apiFilter.searchText + ? withOperators({ + name: { $like: dataViewState.apiFilter.searchText }, + }) + : {}), + }, + }, + { + query: { + placeholderData: keepPreviousData, + staleTime: 0, + refetchOnWindowFocus: true, + refetchInterval: JOB_POLLING_INTERVAL_MS, + }, + } + ); + + const jobs = useMemo( + () => (jobsData?.data ?? []).map((job) => ({ ...job, id: job.id || job.name })), + [jobsData?.data] + ); + + const jobNames = useMemo(() => jobs.map((job) => job.name).filter(Boolean), [jobs]); + + const { data: resultsByName } = useQuery({ + queryKey: ['agent-eval-results-for-jobs', workspace, jobNames] as const, + queryFn: ({ signal }) => fetchAgentEvalResultsForJobs(workspace, jobNames, signal), + enabled: jobNames.length > 0, + placeholderData: keepPreviousData, + refetchInterval: JOB_POLLING_INTERVAL_MS, + }); + + const makeColumns: ComponentProps>['makeColumns'] = ( + { accessor }, + { rowSelectionColumn, rowActionsColumn } + ) => [ + rowSelectionColumn({ size: ROW_SELECTION_COLUMN_SIZE }), + accessor((original) => (original ? (evalConfigName(original) ?? '') : ''), { + id: 'eval_config', + header: 'Eval Config', + size: 200, + enableSorting: false, + cell: ({ row }) => { + const configName = evalConfigName(row.original); + return configName ? ( + e.stopPropagation()} + > + {configName} + + ) : null; + }, + }), + accessor((original) => (original ? (agentNameForJob(original) ?? '') : ''), { + id: 'agent', + header: 'Agent', + size: 200, + enableSorting: false, + }), + accessor((original) => original?.name || '', { + id: 'name', + header: 'Job Name', + }), + accessor((original) => original?.status || '', { + id: 'status', + header: 'Status', + size: 160, + meta: { + filter: { + type: 'single-select' as const, + label: 'Status', + options: STATUS_OPTIONS_WITH_ALL, + }, + }, + cell: ({ row }) => , + }), + accessor(() => '', { + id: 'score', + header: 'Score', + size: 220, + enableSorting: false, + cell: ({ row }) => { + const scores = aggregateScoresOf(resultsByName?.get(row.original.name) ?? null); + if (scores.length === 0) return null; + return ( + + {scores.map((s) => ( + + {s.name}: {formatScore(s.mean)} + + ))} + + ); + }, + }), + accessor((original) => original?.created_at || '', { + id: 'created_at', + header: 'Created', + size: 200, + enableSorting: true, + cell: ({ row }) => , + }), + rowActionsColumn({ + size: ROW_ACTIONS_COLUMN_SIZE, + enableResizing: false, + cell: ({ row }) => ( + setDeleteJobs([row.original]) }]} + /> + ), + }), + ]; + + const hasActiveFilters = + !!dataViewState.debouncedSearchBar || dataViewState.debouncedColumnFilters.length > 0; + const isInitialEmpty = jobs.length === 0 && !isLoading && !error && !hasActiveFilters; + + if (error) { + return ( + + ); + } + + return ( + <> + + dataViewState={dataViewState} + searchField="name" + makeColumns={makeColumns} + onRowClick={(row) => { + if (!row.name) return; + navigate(getAgentEvaluationDetailRoute(workspace, row.name)); + }} + renderBulkActions={({ selectedRows }) => ( + + )} + attributes={{ + DataViewSearchBar: { + placeholder: 'Search by name', + }, + DataViewRoot: { + data: jobs, + totalCount: jobsData?.pagination?.total_results ?? 0, + requestStatus: isLoading && !jobsData ? 'loading' : undefined, + }, + DataViewTableContent: { + renderEmptyState: () => + isInitialEmpty ? ( + + ) : ( + + Clear Filters + + } + /> + ), + }, + }} + /> + + 0} + onDelete={handleDeleteJobs} + title={(count) => `Delete ${getTextWithCount('Evaluation', count)}`} + onClose={() => { + setDeleteJobs([]); + dataViewState.rowSelection.set({}); + }} + /> + + ); +}; diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationDetailRoute.tsx b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationDetailRoute.tsx index 254a3b5a98..72b45b2eb3 100644 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationDetailRoute.tsx +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/AgentEvaluationDetailRoute.tsx @@ -23,10 +23,12 @@ import { aggregateScoresOf, agentNameForJob, cancelAgentEvalJob, + evalConfigName, fetchAgentEvalBundle, fetchAgentEvalJob, fetchAgentEvalResult, joinBundleByTask, + parseBundleRef, } from '@studio/api/evaluation/agent-evaluations'; import { AccessibleTitle } from '@studio/components/AccessibleTitle'; import { StatusLogsContent } from '@studio/components/evaluation/Jobs/StatusLogsContent'; @@ -142,6 +144,9 @@ export const AgentEvaluationDetailRoute: FC = () => { typeof job.error_details?.message === 'string' ? job.error_details.message : null; const scores = aggregateScoresOf(result ?? null); const taskDetails = joinBundleByTask(bundle ?? null); + const artifactsFileset = result?.bundle_ref + ? (parseBundleRef(result.bundle_ref)?.fileset ?? null) + : null; return ( @@ -182,15 +187,15 @@ export const AgentEvaluationDetailRoute: FC = () => { value={String(job.spec.tasks?.length ?? '-')} loading={isLoadingJob} /> - {job.description && ( + {evalConfigName(job) && ( - {job.description} + {evalConfigName(job)} } /> @@ -205,6 +210,19 @@ export const AgentEvaluationDetailRoute: FC = () => { value={job.updated_at ? : ''} loading={isLoadingJob} /> + {artifactsFileset && ( + + View files + + } + /> + )} {(errorMessage ?? statusMessage) && ( ['agent-eval-jobs', workspace] as const; - -const STATUS_FILTER_OPTIONS = [ - { value: '', children: 'All statuses' }, - { value: 'running', children: 'Running' }, - { value: 'queued', children: 'Queued' }, - { value: 'completed', children: 'Completed' }, - { value: 'failed', children: 'Failed' }, - { value: 'cancelled', children: 'Cancelled' }, -]; - -const SORT_OPTIONS = [ - { value: 'created_desc', children: 'Newest first' }, - { value: 'created_asc', children: 'Oldest first' }, - { value: 'name_asc', children: 'Name (A→Z)' }, - { value: 'name_desc', children: 'Name (Z→A)' }, -]; - -type SortKey = 'created_desc' | 'created_asc' | 'name_asc' | 'name_desc'; - -const STATUS_BUCKETS: Record = { - running: ['running'], - queued: ['queued', 'pending', 'created'], - completed: ['completed', 'succeeded', 'success'], - failed: ['failed', 'error'], - cancelled: ['cancelled', 'canceled'], -}; - -const matchesStatus = (status: string, filter: string): boolean => { - if (!filter) return true; - const bucket = STATUS_BUCKETS[filter] ?? [filter]; - return bucket.includes((status ?? '').toLowerCase()); -}; +import { getAgentsListRoute } from '@studio/routes/utils'; +import { useState, type FC } from 'react'; export const AgentEvaluationsListRoute: FC = () => { const workspace = useWorkspaceFromPath(); const [submitOpen, setSubmitOpen] = useState(false); - const [statusFilter, setStatusFilter] = useState(''); - const [agentSearch, setAgentSearch] = useState(''); - const [sortKey, setSortKey] = useState('created_desc'); useBreadcrumbs({ items: [ { slotLabel: 'Agents', href: getAgentsListRoute(workspace) }, @@ -74,47 +20,9 @@ export const AgentEvaluationsListRoute: FC = () => { ], }); - const { data, isLoading, error, refetch } = useQuery({ - queryKey: AGENT_EVAL_JOBS_QUERY_KEY(workspace), - queryFn: ({ signal }) => fetchAgentEvalJobs(workspace, signal), - enabled: !!workspace, - // Eval jobs transition through queued → running → completed; refetch - // every 10s while the user is on the page so the list reflects state - // changes without a manual reload. - refetchInterval: 10_000, - }); - - const jobs = useMemo(() => { - const all = data ?? []; - const search = agentSearch.trim().toLowerCase(); - const filtered = all.filter((job) => { - if (!matchesStatus(job.status ?? '', statusFilter)) return false; - if (search) { - const agent = (agentNameForJob(job) ?? '').toLowerCase(); - const name = job.name.toLowerCase(); - if (!agent.includes(search) && !name.includes(search)) return false; - } - return true; - }); - const sorted = [...filtered].sort((a, b) => { - switch (sortKey) { - case 'created_asc': - return (a.created_at ?? '').localeCompare(b.created_at ?? ''); - case 'name_asc': - return a.name.localeCompare(b.name); - case 'name_desc': - return b.name.localeCompare(a.name); - case 'created_desc': - default: - return (b.created_at ?? '').localeCompare(a.created_at ?? ''); - } - }); - return sorted; - }, [data, statusFilter, agentSearch, sortKey]); - return ( - + { } /> - - {isLoading && ( - - - - )} - - {error && !isLoading && ( - - void refetch()}> - Retry - - } - /> - - )} - - {!isLoading && !error && (data ?? []).length > 0 && ( - - - setAgentSearch(e.target.value)} - /> - - - setSortKey(v as SortKey)} - /> - - - )} - - {!isLoading && !error && (data ?? []).length === 0 && ( - - )} - - {!isLoading && !error && (data ?? []).length > 0 && jobs.length === 0 && ( - No evaluations match the current filters. - )} - - {!isLoading && !error && jobs.length > 0 && ( - - {jobs.map((job) => ( - - - - - - {job.name} - - - {agentNameForJob(job) && ( - - Agent: {agentNameForJob(job)} - - )} - - - - - - - - Created {job.created_at ? : '—'} - - - - - - ))} - - )} + { + it('shows the empty state when there are no tasks', () => { + render(); + expect( + screen.getByText('No per-task results recorded for this evaluation.') + ).toBeInTheDocument(); + }); + + it('renders a task row and opens a cell in a modal', async () => { + render(); + + fireEvent.click(screen.getByRole('button', { name: /Task Results \(1\)/ })); + + expect(await screen.findByText('llm-judge.accuracy: 1.000')).toBeInTheDocument(); + + const expandButtons = screen.getAllByLabelText('Expand cell'); + fireEvent.click(expandButtons[expandButtons.length - 1]); + + expect(await screen.findByText('Task 1 — Agent Response')).toBeInTheDocument(); + }); +}); diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/AgentEvalTaskResultsPanel.tsx b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/AgentEvalTaskResultsPanel.tsx index 1cbef011cc..3755bd6e4d 100644 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/AgentEvalTaskResultsPanel.tsx +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/AgentEvalTaskResultsPanel.tsx @@ -2,23 +2,23 @@ // SPDX-License-Identifier: Apache-2.0 import { AccordionPanel } from '@nemo/common/src/components/AccordionPanel'; +import { StudioDataView } from '@nemo/common/src/components/DataView/StudioDataView'; +import { + TableExpandableCell, + type TableExpandableCellState, +} from '@nemo/common/src/components/DataView/TableExpandableCell'; import { StatusBadge } from '@nemo/common/src/components/StatusBadge'; -import { Badge, Block, Card, Flex, Stack, Text } from '@nvidia/foundations-react-core'; +import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; +import { Badge, Block, Button, Flex, Modal, Stack, Text } from '@nvidia/foundations-react-core'; import type { AgentEvalTaskDetail } from '@studio/api/evaluation/agent-evaluations'; -import { formatScore, scoreColor } from '@studio/routes/agents/AgentEvaluationsRoute/evalScores'; +import { formatScore } from '@studio/routes/agents/AgentEvaluationsRoute/evalScores'; import { ListChecks } from 'lucide-react'; -import { type FC } from 'react'; +import { type ComponentProps, type FC, useCallback, useMemo, useState } from 'react'; interface AgentEvalTaskResultsPanelProps { tasks: AgentEvalTaskDetail[]; } -const headingFor = (task: AgentEvalTaskDetail): string => { - const firstLine = (task.instruction ?? '').split('\n', 1)[0] ?? ''; - const match = /^\s*subject:\s*(.+)$/i.exec(firstLine); - return match ? match[1].trim() : task.taskId; -}; - const referenceText = (reference?: Record): string | null => { if (!reference || Object.keys(reference).length === 0) return null; const values = Object.values(reference); @@ -26,107 +26,162 @@ const referenceText = (reference?: Record): string | null => { return JSON.stringify(reference); }; -const metadataEntries = (metadata?: Record): Array<[string, string]> => - Object.entries(metadata ?? {}).map(([k, v]) => [ - k, - typeof v === 'string' ? v : JSON.stringify(v), - ]); +const diagnosticsText = (diagnostics: unknown[]): string => + diagnostics.map((d) => JSON.stringify(d)).join('\n'); + +const LongCell: FC<{ + content: string; + title: string; + onExpand: (state: TableExpandableCellState) => void; +}> = ({ content, title, onExpand }) => + content ? ( + + ) : ( + + — + + ); export const AgentEvalTaskResultsPanel: FC = ({ tasks }) => { + const [expandedCell, setExpandedCell] = useState(null); + const dataViewState = useStudioDataViewState(); + + const { pageIndex, pageSize } = dataViewState.pagination.state; + const pageRows = useMemo( + () => tasks.slice(pageIndex * pageSize, (pageIndex + 1) * pageSize), + [tasks, pageIndex, pageSize] + ); + + const makeColumns = useCallback< + ComponentProps>['makeColumns'] + >( + (col) => [ + col.display({ + id: 'task', + header: 'Task', + size: 80, + cell: ({ row }) => ( + {pageIndex * pageSize + row.index + 1} + ), + }), + col.display({ + id: 'score', + header: 'Score', + size: 150, + cell: ({ row }) => ( + + {row.original.scores.map((s) => ( + + {s.name}: {formatScore(s.value)} + + ))} + + ), + }), + col.display({ + id: 'input', + header: 'Input', + size: 280, + cell: ({ row }) => ( + + ), + }), + col.display({ + id: 'expected', + header: 'Expected', + size: 140, + cell: ({ row }) => { + const expected = referenceText(row.original.reference); + return expected ? ( + + {expected} + + ) : ( + + — + + ); + }, + }), + col.display({ + id: 'response', + header: 'Agent Response', + size: 280, + cell: ({ row }) => ( + + ), + }), + col.display({ + id: 'status', + header: 'Status', + size: 110, + cell: ({ row }) => , + }), + col.display({ + id: 'diagnostics', + header: 'Diagnostics', + size: 220, + cell: ({ row }) => ( + + ), + }), + ], + [pageIndex, pageSize] + ); + if (tasks.length === 0) { return No per-task results recorded for this evaluation.; } return ( }> - - {tasks.map((task) => { - const expected = referenceText(task.reference); - const metadata = metadataEntries(task.metadata); - return ( - - - {task.scores.map((s) => ( - - {s.name}: {formatScore(s.value)} - - ))} - - - - - - {headingFor(task)} - - - {expected && ( - - Expected: {expected} - - )} - - - - - - - Agent response - - - {task.responseText ?? '—'} - - - - {task.instruction && ( - - - Input - - - {task.instruction} - - - )} - - {metadata.length > 0 && ( - - - Metadata - - - {metadata.map(([key, value]) => ( - - - {key}: - - - {value} - - - ))} - - - )} - - {task.diagnostics.length > 0 && ( - - - Diagnostics - - - {task.diagnostics.map((d) => JSON.stringify(d)).join('\n')} - - - )} - - - ); - })} - +
+ row.taskId }, + }, + }} + /> +
+ { + if (!open) setExpandedCell(null); + }} + slotHeading={expandedCell?.title ?? 'Cell Content'} + className="w-[90vw] max-w-[1000px]" + slotFooter={ + + + + } + > +
+ + {expandedCell?.content} + +
+ ); }; diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/SubmitEvaluationModal.tsx b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/SubmitEvaluationModal.tsx index 4c57ef2ff0..2e674eca1e 100644 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/SubmitEvaluationModal.tsx +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/SubmitEvaluationModal.tsx @@ -9,6 +9,7 @@ import { ControlledTextInput } from '@nemo/common/src/components/form/Controlled import { FormModal, type FormModalProps } from '@nemo/common/src/components/FormModal'; import { getURNFromNamedEntityRef } from '@nemo/common/src/namedEntity'; import { useToast } from '@nemo/common/src/providers/toast/useToast'; +import { FILESET_NAME_REGEXP } from '@nemo/common/src/utils/filesetName'; import { useAgentsListAgents } from '@nemo/sdk/generated/agents/api'; import type { AgentEvaluateJobRequest } from '@nemo/sdk/generated/evaluator/schema'; import { filesDownloadFile, filesListFilesetFiles } from '@nemo/sdk/generated/platform/api'; @@ -37,6 +38,7 @@ import { parsePersistedSpec, type PersistedEvalSpec, } from '@studio/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec'; +import { DATASET_NAME_PATTERN_MESSAGE } from '@studio/routes/FilesetNewRoute/constants'; import { keepPreviousData, useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; import { type FC, useEffect, useRef, useState } from 'react'; import { FormProvider, type SubmitHandler, useForm, useWatch } from 'react-hook-form'; @@ -78,10 +80,10 @@ const makeSubmitEvaluationSchema = (requiresJudgeModel: () => boolean) => message: 'Name is required', path: ['newName'], }); - } else if (!/^[a-zA-Z0-9_.-]+$/.test(name)) { + } else if (!FILESET_NAME_REGEXP.test(name)) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: 'Use only letters, digits, dots, hyphens, and underscores', + message: DATASET_NAME_PATTERN_MESSAGE, path: ['newName'], }); } diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.test.ts b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.test.ts index 7b5ceefde8..b884c98e2b 100644 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.test.ts +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.test.ts @@ -5,6 +5,7 @@ import { bareName, buildAgentEvalRequestBody, buildAgentTarget, + buildEvalJobName, buildPersistedSpec, type EvalConfig, fanMetricOntoTasks, @@ -132,13 +133,42 @@ describe('buildAgentEvalRequestBody', () => { expect(body.spec.tasks[0].metrics[0].payload.metric.model).toBe('ws-a/judge'); }); - it('sets the fileset name as the job description when provided', () => { + it('sets benchmark and a fileset-prefixed job name when provided', () => { const body = buildAgentEvalRequestBody(persisted(), { workspace: 'ws-a', agent: 'a', filesetName: 'wise-blue', }); - expect(body.description).toBe('wise-blue'); + expect(body.spec.benchmark).toEqual({ eval_config_fileset: 'wise-blue' }); + expect(body.name).toMatch(/^wise-blue-[a-z0-9]{8}$/); + }); + + it('omits benchmark and name when no fileset name is provided', () => { + const body = buildAgentEvalRequestBody(persisted(), { workspace: 'ws-a', agent: 'a' }); + expect(body.spec.benchmark).toBeUndefined(); + expect(body.name).toBeUndefined(); + }); +}); + +describe('buildEvalJobName', () => { + it('prefixes with the fileset name and appends a random suffix', () => { + expect(buildEvalJobName('wise-blue')).toMatch(/^wise-blue-[a-z0-9]{8}$/); + }); + + it('lowercases and preserves valid characters', () => { + expect(buildEvalJobName('My_Config.v2')).toMatch(/^my_config\.v2-[a-z0-9]{8}$/); + }); + + it('strips leading non-letter characters so the name starts with a letter', () => { + expect(buildEvalJobName('123-config')).toMatch(/^config-[a-z0-9]{8}$/); + }); + + it('collapses double hyphens and trims edges', () => { + expect(buildEvalJobName('--a--b--')).toMatch(/^a-b-[a-z0-9]{8}$/); + }); + + it('stays within the 63-character limit', () => { + expect(buildEvalJobName('a'.repeat(200)).length).toBeLessThanOrEqual(63); }); }); diff --git a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.ts b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.ts index 00b4abbd0c..149846654a 100644 --- a/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.ts +++ b/web/packages/studio/src/routes/agents/AgentEvaluationsRoute/components/submitEvaluationSpec.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { FILESET_NAME_MAX_LENGTH, toValidFilesetName } from '@nemo/common/src/utils/filesetName'; import { generateDefaultName } from '@nemo/common/src/utils/generateDefaultName'; import { PLATFORM_BASE_URL } from '@studio/constants/environment'; @@ -16,6 +17,14 @@ export const generateEvalConfigName = (): string => generateDefaultName({ length /** Default parallelism for a submitted eval (Studio default; the config value is a hint). */ export const DEFAULT_MAX_CONCURRENT_TASKS = 1; +export const buildEvalJobName = (filesetName: string): string => { + const suffix = Math.random().toString(36).slice(2, 10).padEnd(8, '0'); + const base = toValidFilesetName(filesetName) + .slice(0, FILESET_NAME_MAX_LENGTH - suffix.length - 1) + .replace(/-+$/, ''); + return `${base}-${suffix}`; +}; + // --------------------------------------------------------------------------- // eval-config.json shape (stored in a fileset, read at submit) // --------------------------------------------------------------------------- @@ -68,7 +77,7 @@ export interface SubmitSelections { workspace: string; /** Agent (bare name) to evaluate; used to build the generic target. */ agent: string; - /** Eval-config fileset name, stored as the job description for display in the detail view. */ + /** Eval-config fileset name, stored under spec.benchmark.eval_config_fileset for display. */ filesetName?: string; } @@ -127,11 +136,14 @@ export const buildAgentEvalRequestBody = ( spec: PersistedEvalSpec, selections: SubmitSelections ) => ({ - ...(selections.filesetName ? { description: selections.filesetName } : {}), + ...(selections.filesetName ? { name: buildEvalJobName(selections.filesetName) } : {}), spec: { tasks: spec.tasks, target: buildAgentTarget(selections.workspace, selections.agent), max_concurrent_tasks: spec.max_concurrent_tasks ?? DEFAULT_MAX_CONCURRENT_TASKS, + ...(selections.filesetName + ? { benchmark: { eval_config_fileset: selections.filesetName } } + : {}), }, });