Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import {
SelectContent,
SelectItem,
SelectListbox,
SelectRoot,
SelectTrigger,
Text,
} from '@nvidia/foundations-react-core';
import {
metricLabel,
type ParetoMetric,
} from '@studio/components/charts/ExperimentGroupParetoChart/utils';
import type { FC } from 'react';

interface MetricSelectProps {
label: string;
value: string;
metrics: readonly ParetoMetric[];
onChange: (id: string) => void;
}

export const MetricSelect: FC<MetricSelectProps> = ({ label, value, metrics, onChange }) => (
<label className="flex items-center gap-2">
<Text kind="body/regular/sm" color="subtle">
{label}
</Text>
<SelectRoot value={value} onValueChange={onChange} size="small">
<SelectTrigger
className="w-26"
size="small"
aria-label={label}
// The trigger shows the raw value by default; map it back to the metric's label.
renderValue={(v) => (typeof v === 'string' && v ? metricLabel(v) : undefined)}
/>
{/* Keep the dropdown readable even when the trigger is compact/narrow. */}
<SelectContent className="min-w-48">
<SelectListbox>
{metrics.map((metric) => (
<SelectItem key={metric.id} value={metric.id}>
{metric.label}
</SelectItem>
))}
</SelectListbox>
</SelectContent>
</SelectRoot>
</label>
);
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { Text } from '@nvidia/foundations-react-core';
import type {
ParetoMetric,
ParetoPlotPoint,
} from '@studio/components/charts/ExperimentGroupParetoChart/utils';
import type { FC } from 'react';

/** Format a metric value for tooltips: cost as USD, latency in ms, evaluator scores as-is. */
const formatMetricValue = (metric: ParetoMetric, value: number): string => {
if (metric.id === 'cost_usd') {
return `$${value.toLocaleString(undefined, { maximumFractionDigits: 4 })}`;
}
if (metric.id === 'latency_ms') {
return `${Math.round(value).toLocaleString()} ms`;
}
return value.toLocaleString(undefined, { maximumFractionDigits: 3 });
};

interface ParetoTooltipProps {
active?: boolean;
payload?: ReadonlyArray<{ payload: ParetoPlotPoint }>;
xMetric: ParetoMetric;
yMetric: ParetoMetric;
}

export const ParetoTooltip: FC<ParetoTooltipProps> = ({ active, payload, xMetric, yMetric }) => {
const point = payload?.[0]?.payload;
if (!active || !point) return null;
return (
<div className="rounded border border-base bg-surface p-2 shadow-md">
<Text kind="body/semibold/sm">{point.name}</Text>
<div className="mt-1 flex flex-col gap-0.5">
<Text kind="body/regular/xs">
{xMetric.label}: {formatMetricValue(xMetric, point.x)}
</Text>
<Text kind="body/regular/xs">
{yMetric.label}: {formatMetricValue(yMetric, point.y)}
</Text>
{point.onFrontier && (
<Text kind="body/regular/xs" color="brand">
On the Pareto frontier
</Text>
)}
</div>
</div>
);
};
Original file line number Diff line number Diff line change
Expand Up @@ -7,23 +7,15 @@ import {
useUpdateExperimentGroup,
} from '@nemo/sdk/generated/platform/api';
import type { ExperimentGroupResponse } from '@nemo/sdk/generated/platform/schema';
import {
Button,
SelectContent,
SelectItem,
SelectListbox,
SelectRoot,
SelectTrigger,
Text,
} from '@nvidia/foundations-react-core';
import { Button, Text } from '@nvidia/foundations-react-core';
import { MetricSelect } from '@studio/components/charts/ExperimentGroupParetoChart/MetricSelect';
import { ParetoTooltip } from '@studio/components/charts/ExperimentGroupParetoChart/ParetoTooltip';
import { useParetoEvaluations } from '@studio/components/charts/ExperimentGroupParetoChart/useParetoEvaluations';
import {
buildParetoPoints,
deriveParetoMetrics,
metricLabel,
type ParetoMetric,
type ParetoPlotPoint,
} from '@studio/components/charts/ExperimentGroupParetoChart/paretoMetrics';
import { useParetoEvaluations } from '@studio/components/charts/ExperimentGroupParetoChart/useParetoEvaluations';
} from '@studio/components/charts/ExperimentGroupParetoChart/utils';
import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations';
import { useQueryClient } from '@tanstack/react-query';
import { Loader2, Save } from 'lucide-react';
Expand All @@ -41,114 +33,28 @@ import {
interface ExperimentGroupParetoChartProps {
workspace: string;
group: ExperimentGroupResponse;
/**
* The group's full evaluation set, when the caller already has it loaded (a small group that fit on
* the leaderboard's first page). Supplying it lets the chart render from the shared rows and skip its
* own all-evaluations fetch — which re-runs the same server-side rollup.
*/
/** The group's full evaluation set when the caller already has it loaded (fits one page), so the
* chart renders from these instead of refetching. */
preloadedEvaluations?: EvaluationRow[];
/**
* True while the caller is still loading rows it will supply via `preloadedEvaluations`. The chart
* shows a loading state and holds off its own fetch, rather than briefly rendering empty. Only set
* for groups known (from `evaluation_count`) to fit one page — larger groups fetch immediately.
*/
/** True while the caller is still loading rows it will pass via `preloadedEvaluations`; the chart
* shows a loading state and holds off its own fetch. */
preloadPending?: boolean;
}

const CHART_HEIGHT = 360;
const DEFAULT_X_METRIC = 'cost_usd';
const DEFAULT_Y_METRIC = 'latency_ms';

/** Unit-less axis tick labels (units live on the axis label + chart title). Big values are compacted
* so they don't wrap and collide with the rotated axis title (16000 -> "16K"); small values keep full
* precision so close cost/score ticks don't all round to the same number (0.05, 0.11 stay distinct). */
/** Compact big tick values so they don't collide with the axis title (16000 -> "16K"); keep small
* values precise so close cost/score ticks stay distinct. */
const formatAxisTick = (value: number): string =>
Math.abs(value) >= 1000
? value.toLocaleString(undefined, { notation: 'compact', maximumFractionDigits: 1 })
: value.toLocaleString(undefined, { maximumFractionDigits: 3 });

/** Format a metric value for tooltips: cost as USD, latency in ms, evaluator scores as-is. */
const formatMetricValue = (metric: ParetoMetric, value: number): string => {
if (metric.id === 'cost_usd') {
return `$${value.toLocaleString(undefined, { maximumFractionDigits: 4 })}`;
}
if (metric.id === 'latency_ms') {
return `${Math.round(value).toLocaleString()} ms`;
}
return value.toLocaleString(undefined, { maximumFractionDigits: 3 });
};

interface ParetoTooltipProps {
active?: boolean;
payload?: ReadonlyArray<{ payload: ParetoPlotPoint }>;
xMetric: ParetoMetric;
yMetric: ParetoMetric;
}

const ParetoTooltip: FC<ParetoTooltipProps> = ({ active, payload, xMetric, yMetric }) => {
const point = payload?.[0]?.payload;
if (!active || !point) return null;
return (
<div className="rounded border border-base bg-surface p-2 shadow-md">
<Text kind="body/semibold/sm">{point.name}</Text>
<div className="mt-1 flex flex-col gap-0.5">
<Text kind="body/regular/xs">
{xMetric.label}: {formatMetricValue(xMetric, point.x)}
</Text>
<Text kind="body/regular/xs">
{yMetric.label}: {formatMetricValue(yMetric, point.y)}
</Text>
{point.onFrontier && (
<Text kind="body/regular/xs" color="brand">
On the Pareto frontier
</Text>
)}
</div>
</div>
);
};

interface MetricSelectProps {
label: string;
value: string;
metrics: readonly ParetoMetric[];
onChange: (id: string) => void;
}

const MetricSelect: FC<MetricSelectProps> = ({ label, value, metrics, onChange }) => (
<label className="flex items-center gap-2">
<Text kind="body/regular/sm" color="subtle">
{label}
</Text>
<SelectRoot value={value} onValueChange={onChange} size="small">
<SelectTrigger
className="w-26"
size="small"
aria-label={label}
// The trigger shows the raw value by default; map it back to the metric's label.
renderValue={(v) => (typeof v === 'string' && v ? metricLabel(v) : undefined)}
/>
{/* Keep the dropdown readable even when the trigger is compact/narrow. */}
<SelectContent className="min-w-48">
<SelectListbox>
{metrics.map((metric) => (
<SelectItem key={metric.id} value={metric.id}>
{metric.label}
</SelectItem>
))}
</SelectListbox>
</SelectContent>
</SelectRoot>
</label>
);

/**
* Cost-vs-accuracy Pareto view for an experiment group: one point per evaluation with the Pareto
* frontier highlighted. Points come from the group's evaluations (the existing list endpoint, which
* already carries each evaluation's cost/latency/evaluator rollup means). The two axes are chosen from
* the group's available metrics (cost, latency, and each evaluator) and are **persisted on the group**
* — changing a picker saves the selection so it survives reloads and is shared across viewers. Seeds
* from the group's saved axes, defaulting to cost vs. latency (present for every group).
* Pareto view for an experiment group: one point per evaluation, frontier highlighted. The two axes
* are picked from the group's metrics and persisted on the group (shared across viewers).
*/
export const ExperimentGroupParetoChart: FC<ExperimentGroupParetoChartProps> = ({
workspace,
Expand All @@ -159,9 +65,6 @@ export const ExperimentGroupParetoChart: FC<ExperimentGroupParetoChartProps> = (
const queryClient = useQueryClient();
const toast = useToast();

// Reuse the caller's rows when it already has the whole group loaded. While it's still loading and
// about to supply them (`preloadPending`), hold off our own fetch and show a loading state instead of
// rendering empty. Otherwise fetch the evaluations ourselves (in parallel — see the `enabled` flag).
const hasPreloaded = preloadedEvaluations !== undefined;
const {
rows: fetchedRows,
Expand All @@ -172,8 +75,6 @@ export const ExperimentGroupParetoChart: FC<ExperimentGroupParetoChartProps> = (
const isLoading = hasPreloaded ? false : preloadPending || isFetching;
const metrics = useMemo(() => deriveParetoMetrics(points), [points]);

// Selected axes are optimistic local state seeded from the group's saved config (available
// synchronously from the `group` prop). Changes update the chart immediately and persist below.
const [xMetricId, setXMetricId] = useState(group.pareto?.x_metric ?? DEFAULT_X_METRIC);
const [yMetricId, setYMetricId] = useState(group.pareto?.y_metric ?? DEFAULT_Y_METRIC);

Expand All @@ -189,8 +90,7 @@ export const ExperimentGroupParetoChart: FC<ExperimentGroupParetoChartProps> = (
},
});

// Persist the axes on the group. PUT is a full replace, so send every current field — only `pareto`
// changes.
// PUT is a full replace, so send every field — only `pareto` changes.
const persistAxes = (xMetric: string, yMetric: string) => {
saveGroup({
workspace,
Expand All @@ -207,17 +107,14 @@ export const ExperimentGroupParetoChart: FC<ExperimentGroupParetoChartProps> = (
});
};

// Picking a metric only updates the local view; persisting the group-wide default is an explicit,
// clearly-labeled action (Save button) so users know a save is shared with everyone.
const handleXChange = (id: string) => setXMetricId(id);
const handleYChange = (id: string) => setYMetricId(id);

const savedX = group.pareto?.x_metric ?? DEFAULT_X_METRIC;
const savedY = group.pareto?.y_metric ?? DEFAULT_Y_METRIC;
const hasUnsavedAxes = xMetricId !== savedX || yMetricId !== savedY;

// Fall back to the first/second metric if a saved id isn't in the current data (e.g. an evaluator
// that dropped out). Cost and latency are always present, so a fallback always exists.
// Fall back to the first/second metric when a saved id isn't in the data; cost/latency always exist.
const xMetric = metrics.find((m) => m.id === xMetricId) ?? metrics[0];
const yMetric = metrics.find((m) => m.id === yMetricId) ?? metrics[1] ?? metrics[0];

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,37 +4,29 @@
import { useListEvaluations } from '@nemo/sdk/generated/platform/api';
import type { EvaluationFilter } from '@nemo/sdk/generated/platform/schema';
import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations';
import { DEFAULT_LARGE_PAGE_SIZE } from '@studio/constants/constants';
import { useMemo } from 'react';

/** The list endpoint caps page_size at 1000; a group's evaluation set is far smaller, so one page
* covers every point the Pareto chart needs (the leaderboard, by contrast, is paginated). */
const MAX_EVALUATIONS = 1000;

export interface ParetoEvaluations {
rows: EvaluationRow[];
isLoading: boolean;
isError: boolean;
}

/**
* Loads every evaluation in a group in one unpaginated request for the Pareto chart. Reuses the
* existing list endpoint — each evaluation already carries the cost/latency/evaluator rollup means the
* chart plots — so no dedicated endpoint is needed.
*/
/** Loads every evaluation in a group in one request for the Pareto chart, reusing the list endpoint
* (each evaluation already carries the rollup means the chart plots). */
export function useParetoEvaluations(
workspace: string,
experimentGroupId: string,
options?: { enabled?: boolean }
): ParetoEvaluations {
// Callers disable the fetch when they already have the whole group loaded (a small group that fit on
// the leaderboard's first page), so this all-evaluations request — which re-runs the same server-side
// rollup — is skipped entirely.
// Disabled by callers that already hold the whole group, to avoid a redundant all-evaluations fetch.
const enabled = (options?.enabled ?? true) && !!experimentGroupId;
const { data, isLoading, isError } = useListEvaluations(
workspace,
{
page: 1,
page_size: MAX_EVALUATIONS,
page_size: DEFAULT_LARGE_PAGE_SIZE,
filter: { experiment_group_id: experimentGroupId } as EvaluationFilter,
},
{ query: { enabled } }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import {
buildParetoPoints,
deriveParetoMetrics,
type ParetoMetric,
} from '@studio/components/charts/ExperimentGroupParetoChart/paretoMetrics';
} from '@studio/components/charts/ExperimentGroupParetoChart/utils';
import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroupDataView/useExperimentGroupEvaluations';

// Only the fields the Pareto accessors read (name + cost/latency/evaluator rollup means) are set;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,7 @@ import type { EvaluationRow } from '@studio/components/dataViews/ExperimentGroup
export type MetricDirection = 'min' | 'max';

export interface ParetoMetric {
/**
* Stable id, using the same metric vocabulary the API stores in `group.pareto` and the list
* sort/filter fields: `cost_usd`, `latency_ms`, or `evaluators.<name>`.
*/
/** Stable id in the API's metric vocabulary: `cost_usd`, `latency_ms`, or `evaluators.<name>`. */
readonly id: string;
readonly label: string;
readonly direction: MetricDirection;
Expand All @@ -20,12 +17,8 @@ export interface ParetoMetric {
const capitalize = (value: string): string =>
value ? value.charAt(0).toUpperCase() + value.slice(1) : value;

/**
* The display label for a metric id, resolvable synchronously from the id alone (no loaded points
* needed): `cost_usd` -> "Cost (USD)", `latency_ms` -> "Latency (ms)", `evaluators.<name>` -> the
* capitalized evaluator name. Used so a saved evaluator axis renders its real label immediately
* instead of flashing the cost/latency fallback while the chart data loads.
*/
/** Display label for a metric id, resolved from the id alone: `cost_usd` -> "Cost (USD)",
* `latency_ms` -> "Latency (ms)", `evaluators.<name>` -> the capitalized name. */
export function metricLabel(id: string): string {
if (id === 'cost_usd') return 'Cost (USD)';
if (id === 'latency_ms') return 'Latency (ms)';
Expand All @@ -46,11 +39,8 @@ const LATENCY_METRIC: ParetoMetric = {
accessor: (row) => row.latency_ms?.mean,
};

/**
* The metrics a user may plot on either axis: cost and latency (always present, minimized), plus one
* option per evaluator seen across the group's evaluations (maximized). Evaluator names are dynamic —
* they differ per customer — so they're derived from the data rather than hardcoded.
*/
/** Metrics selectable on either axis: cost and latency (minimized) plus one per evaluator seen in the
* data (maximized). Evaluator names are dynamic, so they're derived from the rows. */
export function deriveParetoMetrics(rows: readonly EvaluationRow[]): ParetoMetric[] {
const evaluatorNames = [
...new Set(rows.flatMap((row) => Object.keys(row.aggregate_scores ?? {}))),
Expand Down
Loading