Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,18 @@
// SPDX-License-Identifier: Apache-2.0

import { Text } from '@nvidia/foundations-react-core';
import type { ExpandedCellState } from '@studio/components/ModelComparePrompts/types';
import { Maximize2 } from 'lucide-react';
import type { FC, ReactNode } from 'react';

/** Table cell with vertical scroll and an expand-to-modal button */
export const ExpandableCell: FC<{
export interface TableExpandableCellState {
title: string;
content: string;
}

export const TableExpandableCell: FC<{
content: string;
title: string;
onExpand: (state: ExpandedCellState) => void;
onExpand: (state: TableExpandableCellState) => void;
footer?: ReactNode;
boldContent?: boolean;
}> = ({ content, title, onExpand, footer, boldContent }) => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ import {
type AgentEvalResult,
agentNameForJob,
aggregateScoresOf,
evalConfigName,
fetchAgentEvalJob,
fetchAgentEvalJobs,
fetchAgentEvalResultsForJobs,
joinBundleByTask,
parseBundleRef,
} from '@studio/api/evaluation/agent-evaluations';
Expand Down Expand Up @@ -88,6 +90,48 @@ describe('agentNameForJob', () => {
});
});

describe('evalConfigName', () => {
it('reads the fileset name from spec.benchmark.eval_config', () => {
const job = baseJob({
spec: {
target: { kind: 'agent', agent: { name: 'a' } },
tasks: [{}],
benchmark: { eval_config_fileset: 'wise-blue' },
} as unknown as AgentEvaluateJob['spec'],
});
expect(evalConfigName(job)).toBe('wise-blue');
});

it('returns null when benchmark is absent, ignoring any description', () => {
expect(evalConfigName(baseJob({ description: 'legacy-desc' }))).toBeNull();
});
});

describe('fetchAgentEvalResultsForJobs', () => {
it('returns an empty map without fetching when there are no job names', async () => {
const map = await fetchAgentEvalResultsForJobs('ws-a', [], new AbortController().signal);
expect(map.size).toBe(0);
expect(customFetchMock).not.toHaveBeenCalled();
});

it('keys results by name for the requested jobs', async () => {
customFetchMock.mockResolvedValue({
data: [
{ name: 'eval-1', scores: { scores: [] } },
{ name: 'eval-2', scores: { scores: [] } },
],
});
const map = await fetchAgentEvalResultsForJobs(
'ws-a',
['eval-1', 'eval-2'],
new AbortController().signal
);
expect(map.get('eval-1')?.name).toBe('eval-1');
expect(map.get('eval-2')?.name).toBe('eval-2');
expect(map.size).toBe(2);
});
});

describe('aggregateScoresOf', () => {
it('flattens the nested scores shape', () => {
const result = {
Expand Down
33 changes: 32 additions & 1 deletion web/packages/studio/src/api/evaluation/agent-evaluations.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { withOperators } from '@nemo/common/src/api/filterOperators';
import {
evaluatorCancelAgentEvaluateJob,
evaluatorCreateAgentEvaluateJob,
evaluatorGetAgentEvalResult,
evaluatorGetAgentEvaluateJob,
evaluatorListAgentEvalResults,
evaluatorListAgentEvaluateJobs,
} from '@nemo/sdk/generated/evaluator/api';
import type {
Expand All @@ -15,6 +17,7 @@ import type {
AgentEvaluateJobRequest,
AgentEvalResult,
AgentEvaluateJobsSortField,
ResultFilter,
} from '@nemo/sdk/generated/evaluator/schema';
import { filesDownloadFile } from '@nemo/sdk/generated/platform/api';

Expand All @@ -30,14 +33,22 @@ export type { AgentEvalResult };
* Strips only this job's own ``workspace/`` prefix so the result compares equal to a
* bare agent name; any other ``/`` in the name is left intact. */
export const agentNameForJob = (job: AgentEvaluateJob): string | null => {
const target = job.spec.target as { kind?: string; agent?: { name?: string } } | null | undefined;
const target = job.spec?.target as
| { kind?: string; agent?: { name?: string } }
| null
| undefined;
if (!target || target.kind !== 'agent') return null;
const name = target.agent?.name;
if (typeof name !== 'string' || name.length === 0) return null;
const prefix = job.workspace ? `${job.workspace}/` : '';
return prefix && name.startsWith(prefix) ? name.slice(prefix.length) : name;
};

export const evalConfigName = (job: AgentEvaluateJob): string | null => {
const name = job.spec?.benchmark?.eval_config_fileset;
return typeof name === 'string' && name.length > 0 ? name : null;
};

export const fetchAgentEvalJobs = async (
workspace: string,
signal: AbortSignal
Expand Down Expand Up @@ -108,6 +119,26 @@ export const fetchAgentEvalResult = async (
}
};

const RESULTS_PAGE_MAX = 100;

export const fetchAgentEvalResultsForJobs = async (
workspace: string,
jobNames: string[],
signal: AbortSignal
): Promise<Map<string, AgentEvalResult>> => {
if (jobNames.length === 0) return new Map();
const page = await evaluatorListAgentEvalResults(
workspace,
{
page: 1,
page_size: Math.min(jobNames.length, RESULTS_PAGE_MAX),
filter: withOperators<ResultFilter>({ name: { $in: jobNames } }),
},
signal
);
return new Map((page?.data ?? []).map((result) => [result.name, result]));
};

/** One trial row from trials.jsonl — the agent's response to a task. */
export interface AgentEvalTrialRow {
id: string;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
// SPDX-License-Identifier: Apache-2.0

import type { ModelWorkspaceGroup } from '@nemo/common/src/api/models/useModels';
import { TableExpandableCell } from '@nemo/common/src/components/DataView/TableExpandableCell';
import { getPartsFromReference } from '@nemo/common/src/namedEntity';
import type { FileSampleMethod } from '@nemo/common/src/utils/sampleTextLines';
import { Button, Flex, Select, Text } from '@nvidia/foundations-react-core';
import { StatsBadge } from '@studio/components/chat/StatsBadge';
import type { DatasetInputFileResult } from '@studio/components/DatasetInputFile';
import { FileSamplingMethodSelect } from '@studio/components/FileSamplingSnippet/FileSamplingMethodSelect';
import { ExpandableCell } from '@studio/components/ModelComparePrompts/ExpandableCell';
import { ModelColumnSelect } from '@studio/components/ModelComparePrompts/ModelColumnSelect';
import type {
ExpandedCellState,
Expand Down Expand Up @@ -221,7 +221,7 @@ export const ModelCompareTable: FC<ModelCompareTableProps> = ({
return (
<tr key={row.sourceIndex} className="bg-surface-raised">
<td className={`${rowBottom}border-r border-base p-0 align-top`}>
<ExpandableCell
<TableExpandableCell
content={row.prompt}
title={`Prompt (dataset row ${row.sourceIndex})`}
onExpand={setExpandedCell}
Expand Down Expand Up @@ -258,7 +258,7 @@ export const ModelCompareTable: FC<ModelCompareTableProps> = ({
}
return (
<td key={m.id} className={`${rowBottom}${colBorder}border-base p-0 align-top`}>
<ExpandableCell
<TableExpandableCell
content={response.text}
title={`${modelName} response (dataset row ${row.sourceIndex})`}
onExpand={(state) => setExpandedCell({ ...state, stats: response.stats })}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// SPDX-License-Identifier: Apache-2.0

import type { ModelWorkspaceGroup } from '@nemo/common/src/api/models/useModels';
import type { TableExpandableCellState } from '@nemo/common/src/components/DataView/TableExpandableCell';
import type { SharedModelEntry } from '@studio/routes/ModelCompareRoute/types';

export interface ResponseStats {
Expand All @@ -27,9 +28,7 @@ export interface PromptRow {
responses: Record<number, ResponseResult | null | undefined>;
}

export interface ExpandedCellState {
title: string;
content: string;
export interface ExpandedCellState extends TableExpandableCellState {
stats?: ResponseStats;
}

Expand Down
Loading