Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
219 changes: 219 additions & 0 deletions src/lib/inference/nim.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,83 @@ describe("nim", () => {
}
});

// Regression #2669: the previous fix added `name` only for homogeneous
// hosts, so mixed-GPU machines (e.g. RTX PRO 6000 + GB300 on the QA
// verification host) lost the model info entirely. We now also surface
// the per-GPU breakdown via `gpus` so the preflight line can render it.
it("populates the gpus breakdown on mixed-model hosts (regression #2669)", () => {
const runCapture = vi.fn((cmd: string | string[]) => {
if (!Array.isArray(cmd)) throw new Error("expected argv array");
if (
cmd[0] === "nvidia-smi" &&
cmd.some((a: string) => a.includes("name,memory.total"))
) {
return "NVIDIA RTX PRO 6000 Blackwell Max-Q, 97887\nNVIDIA GB300, 256703\n";
}
return "";
});
const { nimModule, restore } = loadNimWithMockedRunner(runCapture);

try {
const result = nimModule.detectGpu();
expect(result?.name).toBeUndefined();
expect(result?.gpus).toEqual([
{ name: "NVIDIA RTX PRO 6000 Blackwell Max-Q", memoryMB: 97887 },
{ name: "NVIDIA GB300", memoryMB: 256703 },
]);
} finally {
restore();
}
});

it("populates the gpus breakdown on homogeneous hosts too", () => {
const runCapture = vi.fn((cmd: string | string[]) => {
if (!Array.isArray(cmd)) throw new Error("expected argv array");
if (
cmd[0] === "nvidia-smi" &&
cmd.some((a: string) => a.includes("name,memory.total"))
) {
return "NVIDIA H100 80GB HBM3, 81920\nNVIDIA H100 80GB HBM3, 81920\n";
}
return "";
});
const { nimModule, restore } = loadNimWithMockedRunner(runCapture);

try {
const result = nimModule.detectGpu();
expect(result?.gpus).toEqual([
{ name: "NVIDIA H100 80GB HBM3", memoryMB: 81920 },
{ name: "NVIDIA H100 80GB HBM3", memoryMB: 81920 },
]);
} finally {
restore();
}
});

it("populates the gpus breakdown on the unified-memory fallback path (#2669 GB10)", () => {
// Spark / Jetson don't have memory.total per GPU — host RAM is split
// evenly across the named devices. Used by the original GB10 reporter.
const runCapture = vi.fn((cmd: string | string[]) => {
if (!Array.isArray(cmd)) throw new Error("expected argv array");
if (cmd.some((a: string) => a.includes("memory.total"))) return "";
if (cmd.some((a: string) => a.includes("query-gpu=name"))) {
return "NVIDIA GB10\n";
}
if (cmd[0] === "free" && cmd[1] === "-m") {
return " total used free\nMem: 131072 10240 90000\nSwap: 0 0 0";
}
return "";
});
const { nimModule, restore } = loadNimWithMockedRunner(runCapture);

try {
const result = nimModule.detectGpu();
expect(result?.gpus).toEqual([{ name: "NVIDIA GB10", memoryMB: 131072 }]);
} finally {
restore();
}
});

it("detects GB10 unified-memory GPUs as Spark-capable NVIDIA devices", () => {
const runCapture = vi.fn((cmd: string | string[]) => {
if (!Array.isArray(cmd)) throw new Error("expected argv array");
Expand Down Expand Up @@ -388,6 +465,148 @@ describe("nim", () => {
});
});

describe("groupGpusByName", () => {
it("groups identical names and sums their memory", () => {
expect(
nim.groupGpusByName([
{ name: "NVIDIA H100 80GB HBM3", memoryMB: 81920 },
{ name: "NVIDIA H100 80GB HBM3", memoryMB: 81920 },
]),
).toEqual([{ name: "NVIDIA H100 80GB HBM3", count: 2, memoryMB: 163840 }]);
});

it("preserves first-appearance order across distinct names", () => {
expect(
nim.groupGpusByName([
{ name: "NVIDIA RTX PRO 6000 Blackwell Max-Q", memoryMB: 97887 },
{ name: "NVIDIA GB300", memoryMB: 256703 },
]).map((g: { name: string }) => g.name),
).toEqual(["NVIDIA RTX PRO 6000 Blackwell Max-Q", "NVIDIA GB300"]);
});

it("groups duplicates and singletons together (2x H100 + 1x A100)", () => {
expect(
nim.groupGpusByName([
{ name: "NVIDIA H100 80GB HBM3", memoryMB: 81920 },
{ name: "NVIDIA H100 80GB HBM3", memoryMB: 81920 },
{ name: "NVIDIA A100 40GB", memoryMB: 40960 },
]),
).toEqual([
{ name: "NVIDIA H100 80GB HBM3", count: 2, memoryMB: 163840 },
{ name: "NVIDIA A100 40GB", count: 1, memoryMB: 40960 },
]);
});

it("normalizes internal whitespace before comparing names", () => {
// Defensive: nvidia-smi shouldn't return double spaces, but if a driver
// ever does, we shouldn't split what is logically the same model.
expect(
nim.groupGpusByName([
{ name: "NVIDIA H100 80GB HBM3", memoryMB: 81920 },
{ name: "NVIDIA H100 80GB HBM3", memoryMB: 81920 },
]),
).toEqual([{ name: "NVIDIA H100 80GB HBM3", count: 2, memoryMB: 163840 }]);
});

it("drops rows with blank names", () => {
expect(
nim.groupGpusByName([
{ name: "", memoryMB: 81920 },
{ name: " ", memoryMB: 81920 },
{ name: "NVIDIA GB300", memoryMB: 256703 },
]),
).toEqual([{ name: "NVIDIA GB300", count: 1, memoryMB: 256703 }]);
});
});

describe("formatNvidiaGpuPreflightLines", () => {
it("renders single GPU as a compact one-liner", () => {
const lines = nim.formatNvidiaGpuPreflightLines({
type: "nvidia",
name: "NVIDIA GB300",
gpus: [{ name: "NVIDIA GB300", memoryMB: 284208 }],
count: 1,
totalMemoryMB: 284208,
perGpuMB: 284208,
nimCapable: true,
});
expect(lines).toEqual(["NVIDIA GPU detected (NVIDIA GB300, 284208 MB)"]);
});

it("renders N homogeneous GPUs as `Nx <model>` in the compact form", () => {
const lines = nim.formatNvidiaGpuPreflightLines({
type: "nvidia",
name: "NVIDIA H100 80GB HBM3",
gpus: [
{ name: "NVIDIA H100 80GB HBM3", memoryMB: 81920 },
{ name: "NVIDIA H100 80GB HBM3", memoryMB: 81920 },
],
count: 2,
totalMemoryMB: 163840,
perGpuMB: 81920,
nimCapable: true,
});
expect(lines).toEqual([
"NVIDIA GPU detected (2x NVIDIA H100 80GB HBM3, 163840 MB)",
]);
});

// Regression #2669: this is the case the previous fix missed entirely.
it("renders mixed-model 1+1 with breakdown and no `Nx ` prefix", () => {
const lines = nim.formatNvidiaGpuPreflightLines({
type: "nvidia",
gpus: [
{ name: "NVIDIA RTX PRO 6000 Blackwell Max-Q", memoryMB: 97887 },
{ name: "NVIDIA GB300", memoryMB: 256703 },
],
count: 2,
totalMemoryMB: 354590,
perGpuMB: 97887,
nimCapable: true,
});
expect(lines).toEqual([
"NVIDIA GPU detected: 2 GPUs, 354590 MB VRAM",
" - NVIDIA RTX PRO 6000 Blackwell Max-Q (97887 MB)",
" - NVIDIA GB300 (256703 MB)",
]);
});

it("renders mixed-model with duplicates using `Nx ` prefix across all groups", () => {
const lines = nim.formatNvidiaGpuPreflightLines({
type: "nvidia",
gpus: [
{ name: "NVIDIA H100 80GB HBM3", memoryMB: 81920 },
{ name: "NVIDIA H100 80GB HBM3", memoryMB: 81920 },
{ name: "NVIDIA A100 40GB", memoryMB: 40960 },
],
count: 3,
totalMemoryMB: 204800,
perGpuMB: 81920,
nimCapable: true,
});
expect(lines).toEqual([
"NVIDIA GPU detected: 3 GPUs, 204800 MB VRAM",
" - 2x NVIDIA H100 80GB HBM3 (163840 MB)",
" - 1x NVIDIA A100 40GB (40960 MB)",
]);
});

it("falls back to count-only when every parsed row had a blank name", () => {
const lines = nim.formatNvidiaGpuPreflightLines({
type: "nvidia",
gpus: [
{ name: "", memoryMB: 81920 },
{ name: "", memoryMB: 81920 },
],
count: 2,
totalMemoryMB: 163840,
perGpuMB: 81920,
nimCapable: true,
});
expect(lines).toEqual(["NVIDIA GPU detected: 2 GPU(s), 163840 MB VRAM"]);
});
});

describe("nimStatus", () => {
it("returns not running for nonexistent container", () => {
const st = nim.nimStatus("nonexistent-test-xyz");
Expand Down
79 changes: 79 additions & 0 deletions src/lib/inference/nim.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,24 @@ export interface NimModel {

export type NvidiaPlatform = "spark" | "station" | "linux";

export interface NimGpu {
name: string;
memoryMB: number;
}

export interface GpuGroup {
name: string;
count: number;
memoryMB: number;
}

export interface GpuDetection {
type: string;
name?: string;
// Per-GPU breakdown when available (primary nvidia-smi --query-gpu path).
// Always populated alongside `name` for NVIDIA; absent on the count-only
// fallback when every parsed row had a blank name. See #2669.
gpus?: NimGpu[];
count: number;
totalMemoryMB: number;
perGpuMB: number;
Expand All @@ -44,6 +59,65 @@ export interface GpuDetection {
platform?: NvidiaPlatform;
}

// Group GPUs by their nvidia-smi model name, preserving first-appearance order.
// Names are whitespace-normalized; rows with blank names are dropped (the caller
// falls through to the count-only display in that case). We deliberately do not
// include memoryMB in the group key — within a single host, nvidia-smi reports
// stable name strings that already disambiguate memory variants (e.g.
// "H100 80GB HBM3" vs "H100 40GB"). The only theoretical collision is ECC-mode
// reporting variance on otherwise-identical cards, which is rare enough that
// splitting the group would create more confusion than it solves.
export function groupGpusByName(gpus: readonly NimGpu[]): GpuGroup[] {
const groups: GpuGroup[] = [];
for (const g of gpus) {
const name = g.name.replace(/\s+/g, " ").trim();
if (!name) continue;
const existing = groups.find((grp) => grp.name === name);
if (existing) {
existing.count += 1;
existing.memoryMB += g.memoryMB;
} else {
groups.push({ name, count: 1, memoryMB: g.memoryMB });
}
}
return groups;
}

// Render the preflight summary for an NVIDIA GPU detection. Returns one
// or more lines that the caller prefixes with ` ✓ ` / prints directly.
//
// - Homogeneous (1 GPU or N of the same model) → single compact line:
// NVIDIA GPU detected (<model>, <vram> MB)
// NVIDIA GPU detected (Nx <model>, <vram> MB)
// - Mixed model → aggregate header + indented per-group breakdown:
// NVIDIA GPU detected: 2 GPUs, 354590 MB VRAM
// - NVIDIA RTX PRO 6000 Blackwell Max-Q (97887 MB)
// - NVIDIA GB300 (256703 MB)
// Within one breakdown block, `Nx ` is added to every group when any
// group has count > 1 (preserves column alignment); otherwise dropped.
// - No usable names → last-resort count-only fallback.
//
// See #2669 for the multi-GPU case the previous fix missed.
export function formatNvidiaGpuPreflightLines(gpu: GpuDetection): string[] {
if (gpu.name) {
const detail = gpu.count > 1 ? `${gpu.count}x ${gpu.name}` : gpu.name;
return [`NVIDIA GPU detected (${detail}, ${gpu.totalMemoryMB} MB)`];
}
if (gpu.gpus && gpu.gpus.length > 0) {
const groups = groupGpusByName(gpu.gpus);
if (groups.length > 0) {
const lines = [`NVIDIA GPU detected: ${gpu.count} GPUs, ${gpu.totalMemoryMB} MB VRAM`];
const anyDuplicate = groups.some((grp) => grp.count > 1);
for (const grp of groups) {
const prefix = anyDuplicate ? `${grp.count}x ` : "";
lines.push(` - ${prefix}${grp.name} (${grp.memoryMB} MB)`);
}
return lines;
}
}
return [`NVIDIA GPU detected: ${gpu.count} GPU(s), ${gpu.totalMemoryMB} MB VRAM`];
}

// Read the platform model name from firmware. Try DMI first (covers Spark
// and Station, observed empirically), fall back to devicetree on systems
// without DMI tables. Returns "" if neither is readable.
Expand Down Expand Up @@ -165,6 +239,7 @@ export function detectGpu(): GpuDetection | null {
return {
type: "nvidia",
...(allSameName ? { name: firstName } : {}),
gpus: parsed.map((p) => ({ name: p.name, memoryMB: p.memoryMB })),
count: parsed.length,
totalMemoryMB,
perGpuMB: parsed[0].memoryMB,
Expand Down Expand Up @@ -217,9 +292,13 @@ export function detectGpu(): GpuDetection | null {
: firmwarePlatform === "station"
? "station"
: "linux";
// Memory.total is not available on unified-memory devices, so we split
// the host RAM evenly across the named GPUs for the per-GPU breakdown.
// Approximation, but the only number nvidia-smi gives us in this path.
return {
type: "nvidia",
name: unifiedGpuNames[0],
gpus: unifiedGpuNames.map((name: string) => ({ name, memoryMB: perGpuMB })),
count,
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
totalMemoryMB,
perGpuMB: perGpuMB || totalMemoryMB,
Expand Down
14 changes: 4 additions & 10 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4118,16 +4118,10 @@ async function preflight(
// GPU
const gpu = nim.detectGpu();
if (gpu && gpu.type === "nvidia") {
if (gpu.name) {
// Match DevTest case 517913 cross-check format when the GPU model is
// known: `NVIDIA GPU detected (<model>, <vram> MB)`. Multi-GPU hosts
// with the same model render as `Nx <model>` inside the parens.
const detail = gpu.count > 1 ? `${gpu.count}x ${gpu.name}` : gpu.name;
console.log(` ✓ NVIDIA GPU detected (${detail}, ${gpu.totalMemoryMB} MB)`);
} else {
// Mixed-model or unnamed devices fall back to the count-only form so
// we never falsely attribute one GPU's name to the others.
console.log(` ✓ NVIDIA GPU detected: ${gpu.count} GPU(s), ${gpu.totalMemoryMB} MB VRAM`);
const lines = nim.formatNvidiaGpuPreflightLines(gpu);
console.log(` ✓ ${lines[0]}`);
for (const extra of lines.slice(1)) {
console.log(` ${extra}`);
}
if (!gpu.nimCapable) {
console.log(" ⓘ Local NIM unavailable — GPU VRAM too small");
Expand Down
Loading