Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,446 changes: 1,446 additions & 0 deletions plugins/nemo-evaluator/openapi/openapi.yaml

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions plugins/nemo-evaluator/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -51,5 +51,8 @@ asyncio_mode = "auto"
pythonpath = ["src"]
testpaths = ["tests"]

# Opt this plugin into OpenAPI spec generation
[tool.nemo.openapi]

[tool.pyright]
extraPaths = ["src"]
6 changes: 6 additions & 0 deletions web/packages/sdk/orval/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,12 @@ export const serviceConfigs: Record<string, ServiceConfig> = {
apiEnvKeys: ['VITE_PLATFORM_BASE_URL'],
zod: true,
},
evaluator: {
path: 'evaluator',
url: `../../../../plugins/nemo-evaluator/openapi/openapi.yaml`,
apiEnvKeys: ['VITE_PLATFORM_BASE_URL'],
zod: true,
},
};

export const serviceToConfig = {
Expand Down
7 changes: 5 additions & 2 deletions web/packages/sdk/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,10 @@
"gen:customizer-zod": "ORVAL_CLIENT=zod tsx ./orval/generate.ts customizer",
"gen:data-designer": "tsx ./orval/generate.ts data-designer",
"gen:data-designer-zod": "ORVAL_CLIENT=zod tsx ./orval/generate.ts data-designer",
"gen:evaluator": "tsx ./orval/generate.ts evaluator",
"gen:evaluator-zod": "ORVAL_CLIENT=zod tsx ./orval/generate.ts evaluator",
"gen:deployment-management": "tsx ./orval/generate.ts deployment-management",
"gen:entity-store": "tsx ./orval/generate.ts entity-store",
"gen:evaluator": "tsx ./orval/generate.ts evaluator",
"gen:guardrails": "tsx ./orval/generate.ts guardrails",
"gen:inference-gateway": "tsx ./orval/generate.ts inference-gateway",
"gen:jobs": "tsx ./orval/generate.ts jobs",
Expand All @@ -25,7 +26,9 @@
"gen:platform-zod": "ORVAL_CLIENT=zod tsx ./orval/generate.ts platform",
"gen:safe-synthesizer": "tsx ./orval/generate.ts safe-synthesizer",
"gen:safe-synthesizer-zod": "ORVAL_CLIENT=zod tsx ./orval/generate.ts safe-synthesizer",
"test": "vitest run"
"test": "vitest run",
"test:ci": "vitest run",
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@tanstack/react-query": "catalog:",
Expand Down
20 changes: 5 additions & 15 deletions web/packages/studio/src/api/evaluation/EvaluationApiError.ts
Original file line number Diff line number Diff line change
@@ -1,23 +1,13 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { ErrorResponse, HTTPValidationError } from '@nemo/sdk/generated/platform/schema';

const isHttpValidationError = (
response: HTTPValidationError | ErrorResponse
): response is HTTPValidationError => {
return 'detail' in response && Array.isArray(response.detail);
};
import type { HTTPValidationError } from '@nemo/sdk/generated/evaluator/schema';

export class EvaluationApiError extends Error {
constructor(public response: ErrorResponse | HTTPValidationError) {
let message;
if (isHttpValidationError(response)) {
message = response.detail?.map((err: { msg?: string }) => err.msg).join('');
} else {
// ErrorResponse has {detail: string}
message = response.detail;
}
constructor(public response: HTTPValidationError) {
const message = Array.isArray(response.detail)
? response.detail.map((err: { msg?: string }) => err.msg).join('')
: response.detail;

super(message || 'Unknown error');
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,15 @@

import { NamedEntity } from '@nemo/common/src/namedEntity';
import {
evaluationGetMetricJob,
evaluationListMetricJobsResults,
evaluationListMetricJobs,
} from '@nemo/sdk/generated/platform/api';
evaluatorGetEvaluateJob,
evaluatorListEvaluateJobResults,
evaluatorListEvaluateJobs,
} from '@nemo/sdk/generated/evaluator/api';
import {
MetricEvaluationJob,
MetricEvaluationJobsPage,
EvaluateJob,
EvaluateJobsPage,
PlatformJobListResultResponse,
} from '@nemo/sdk/generated/platform/schema';
} from '@nemo/sdk/generated/evaluator/schema';
import { useDetailsChartsStore } from '@studio/api/evaluation/details/useDetailsChartsStore';
import { getEvaluationJobModel, isEvaluationJobSucceeded } from '@studio/selectors/evaluationJob';
import { useQuery } from '@tanstack/react-query';
Expand All @@ -23,24 +23,24 @@ import { useQuery } from '@tanstack/react-query';
* Example usage in a component:
* fetchAllEvaluationsByConfig(workspace);
*
* @returns {Object} Promise<MetricEvaluationJobsPage>
* @returns {Object} Promise<EvaluateJobsPage>
*/
const fetchAllEvaluationsByConfig = async (
workspace: string | undefined
): Promise<MetricEvaluationJobsPage> => {
): Promise<EvaluateJobsPage> => {
if (!workspace) {
return {
data: [],
};
}

let allEvaluations: MetricEvaluationJobsPage['data'] = [];
let allEvaluations: EvaluateJobsPage['data'] = [];
let currentPage = 1;
let totalPages = 1; // initialize to avoid infinite loop
const pageSize = 100;

while (currentPage <= totalPages) {
const response = await evaluationListMetricJobs(workspace, {
const response = await evaluatorListEvaluateJobs(workspace, {
page_size: pageSize,
page: currentPage,
sort: '-created_at',
Expand Down Expand Up @@ -77,11 +77,11 @@ const fetchAllEvaluationsByConfig = async (
* Example usage in a component:
* useUniqueModelsByConfig(workspace, id);
*
* @returns {Object} <Record<string, MetricEvaluationJob[]>
* @returns {Object} <Record<string, EvaluateJob[]>
*/

export const useUniqueModelsByConfig = (workspace: string | undefined, evaluationId?: string) => {
return useQuery<Record<string, MetricEvaluationJob[]>, Error>({
return useQuery<Record<string, EvaluateJob[]>, Error>({
queryKey: ['uniqueModelsByConfig', workspace, evaluationId],
queryFn: async () => {
const evaluationsByConfig = await fetchAllEvaluationsByConfig(workspace);
Expand All @@ -97,13 +97,13 @@ export const useUniqueModelsByConfig = (workspace: string | undefined, evaluatio

if (modelName) {
if (!acc[modelName]) {
acc[modelName] = [] as MetricEvaluationJob[];
acc[modelName] = [] as EvaluateJob[];
}
acc[modelName].push(job);
}
return acc;
},
{} as Record<string, MetricEvaluationJob[]>
{} as Record<string, EvaluateJob[]>
);
},
placeholderData: (prevData) => prevData ?? {},
Expand All @@ -130,7 +130,7 @@ export const useAggregatedEvaluationResults = (config: NamedEntity) => {
}
return await Promise.all(
selectedEvaluations.map((evaluationId) =>
evaluationListMetricJobsResults(config.workspace!, evaluationId)
evaluatorListEvaluateJobResults(config.workspace!, evaluationId)
)
);
},
Expand All @@ -146,20 +146,20 @@ export const useAggregatedEvaluationResults = (config: NamedEntity) => {
* Example usage in a component:
* useAggregatedEvaluations(config);
*
* @returns {Array} MetricEvaluationJob[]
* @returns {Array} EvaluateJob[]
*/
export const useAggregatedEvaluations = (config: NamedEntity) => {
const { selectedEvaluations } = useDetailsChartsStore();

return useQuery<MetricEvaluationJob[], Error>({
return useQuery<EvaluateJob[], Error>({
queryKey: ['aggregatedEvaluations', config, selectedEvaluations],
queryFn: async () => {
if (!config.workspace) {
return [];
}
return await Promise.all(
selectedEvaluations.map((evaluationId) =>
evaluationGetMetricJob(config.workspace!, evaluationId)
evaluatorGetEvaluateJob(config.workspace!, evaluationId)
)
);
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// SPDX-License-Identifier: Apache-2.0

import type { ExcludedChatCompletionMessageParam } from '@nemo/common/src/types/chat';
import { evaluationDownloadMetricJobResultRowScores } from '@nemo/sdk/generated/platform/api';
import { evaluatorDownloadEvaluateJobResult } from '@nemo/sdk/generated/evaluator/api';
import { useQuery } from '@tanstack/react-query';

export interface EvaluationMetrics {
Expand Down Expand Up @@ -34,16 +34,14 @@ export interface EvaluationResultItem {
metrics: EvaluationMetrics;
}

// V2 row score shape from the platform API
interface V2RowScore {
interface RowScore {
index: number;
row: Record<string, unknown>;
scores?: Record<string, number>;
error?: string;
}

// Map V2 row scores to the EvaluationResultItem format used by the UI
function mapV2RowScores(rows: V2RowScore[]): EvaluationResultItem[] {
function mapRowScores(rows: RowScore[]): EvaluationResultItem[] {
return rows.map((row) => ({
item: {
prompt: String(row.row?.prompt ?? row.row?.input ?? ''),
Expand All @@ -55,19 +53,17 @@ function mapV2RowScores(rows: V2RowScore[]): EvaluationResultItem[] {
sample: {
output_text: String(row.row?.output ?? row.row?.output_text ?? ''),
},
metrics: mapV2Scores(row.scores),
metrics: mapScores(row.scores),
}));
}

function mapV2Scores(scores?: Record<string, number>): EvaluationMetrics {
function mapScores(scores?: Record<string, number>): EvaluationMetrics {
if (!scores) return {};

const result: EvaluationMetrics = {};

// Map V2 flat scores to the nested metrics structure the UI expects
for (const [key, value] of Object.entries(scores)) {
if (key === 'sentence' || key === 'corpus') {
// BLEU sub-scores
if (!result.bleu) {
result.bleu = { scores: {} };
}
Expand All @@ -83,27 +79,31 @@ function mapV2Scores(scores?: Record<string, number>): EvaluationMetrics {
return result;
}

async function fetchEvaluationResultsV2(
async function fetchEvaluationResults(
workspace: string,
jobName: string
jobName: string,
name: string
): Promise<EvaluationResultItem[]> {
const blob = await evaluationDownloadMetricJobResultRowScores(workspace, jobName);
const blob = await evaluatorDownloadEvaluateJobResult(workspace, jobName, name);

const text = await blob.text();
const parsed = JSON.parse(text);

// The response is either an array of row scores or wrapped in a container
const rows: V2RowScore[] = Array.isArray(parsed)
const rows: RowScore[] = Array.isArray(parsed)
? parsed
: (parsed.data ?? parsed.row_scores ?? []);

return mapV2RowScores(rows);
return mapRowScores(rows);
}

export const useEvaluationDownloadResultsById = (workspace: string, jobName: string) => {
export const useEvaluationDownloadResultsById = (
workspace: string,
jobName: string,
name = 'row_scores'
) => {
return useQuery<EvaluationResultItem[], Error>({
queryKey: ['evaluationDownloadResults', workspace, jobName],
queryFn: () => fetchEvaluationResultsV2(workspace, jobName),
queryKey: ['evaluationDownloadResults', workspace, jobName, name],
queryFn: () => fetchEvaluationResults(workspace, jobName, name),
enabled: !!workspace && !!jobName,
});
};
21 changes: 5 additions & 16 deletions web/packages/studio/src/api/evaluation/index.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
import { evaluationListMetricJobs } from '@nemo/sdk/generated/platform/api';
import { EvaluationListMetricJobsParams } from '@nemo/sdk/generated/platform/schema';
import { EvaluationJobWithTaskMetrics } from '@studio/api/evaluation/useEvaluationsWithMetrics';
import { evaluatorListEvaluateJobs } from '@nemo/sdk/generated/evaluator/api';
import type { EvaluatorListEvaluateJobsParams } from '@nemo/sdk/generated/evaluator/schema';
import type { EvaluationJobWithTaskMetrics } from '@studio/api/evaluation/useEvaluationsWithMetrics';

export interface FetchEvaluationsWithMetricsOptions {
workspace: string;
query?: EvaluationListMetricJobsParams;
query?: EvaluatorListEvaluateJobsParams;
signal?: AbortSignal;
}

Expand All @@ -15,7 +15,7 @@ export const fetchEvaluationsWithMetrics = async ({
query,
signal,
}: FetchEvaluationsWithMetricsOptions) => {
const evaluations = await evaluationListMetricJobs(workspace, query, signal);
const evaluations = await evaluatorListEvaluateJobs(workspace, query, signal);
if (evaluations.data?.length > 0) {
const evaluationsWithMetrics = await Promise.all(
evaluations.data.map(async (evaluation) => {
Expand All @@ -28,19 +28,8 @@ export const fetchEvaluationsWithMetrics = async ({
}

const evaluationWithMetrics: EvaluationJobWithTaskMetrics = { ...evaluation, tasks: {} };
// TODO: Download results
// const evaluationResult = await getEvaluationResult(
// evaluation.workspace,
// evaluation.id!,
// evaluation.name,
// signal
// );
// if (evaluationResult.download_url) {
// }
return evaluationWithMetrics;
} catch {
// if an eval job failed, there will be a 404 returned by fetchEvaluationResults
// return the original evaluation if there's an error
return evaluation;
}
})
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { NamedEntity } from '@nemo/common/src/namedEntity';
import { useEvaluationListMetricJobs } from '@nemo/sdk/generated/platform/api';
import { EvaluationListMetricJobsParams } from '@nemo/sdk/generated/platform/schema';
import { useEvaluatorListEvaluateJobs } from '@nemo/sdk/generated/evaluator/api';
import type { EvaluatorListEvaluateJobsParams } from '@nemo/sdk/generated/evaluator/schema';

/**
* Custom hook to fetch evaluation jobs filtered by configuration.
Expand All @@ -19,15 +18,13 @@ import { EvaluationListMetricJobsParams } from '@nemo/sdk/generated/platform/sch
*/
export const useEvaluationsByConfig = (
workspace: string,
config: NamedEntity & Partial<EvaluationListMetricJobsParams>
config: Partial<EvaluatorListEvaluateJobsParams>
) => {
const params: EvaluationListMetricJobsParams = {
const params: EvaluatorListEvaluateJobsParams = {
page: config.page,
page_size: config.page_size,
sort: 'created_at',
// TODO: Support filtering by config namespace and name
// config: config.namespace && config.name ? `${config.namespace}/${config.name}` : undefined,
};

return useEvaluationListMetricJobs(workspace, params);
return useEvaluatorListEvaluateJobs(workspace, params);
};
Loading
Loading