diff --git a/src/lib/adapters/docker/image.ts b/src/lib/adapters/docker/image.ts index 44f052e1cd9..67c55870199 100644 --- a/src/lib/adapters/docker/image.ts +++ b/src/lib/adapters/docker/image.ts @@ -43,6 +43,14 @@ export function dockerRmi(imageRef: string, opts: DockerRunOptions = {}): Docker return dockerRun(["rmi", imageRef], opts); } +export function dockerTag( + source: string, + target: string, + opts: DockerRunOptions = {}, +): DockerRunResult { + return dockerRun(["tag", source, target], opts); +} + export function dockerListImagesFormat( reference: string, format: string, diff --git a/src/lib/adapters/docker/index.test.ts b/src/lib/adapters/docker/index.test.ts index 74f0ebf1f7e..924c778b82e 100644 --- a/src/lib/adapters/docker/index.test.ts +++ b/src/lib/adapters/docker/index.test.ts @@ -17,11 +17,13 @@ import { dockerContainerInspectFormat, dockerInfoFormat, dockerListVolumesByPrefix, + dockerManifestInspect, dockerPull, - dockerRename, dockerRemoveVolumesByPrefix, + dockerRename, dockerRmi, dockerRunDetached, + dockerTag, } from "./index"; describe("docker helpers", () => { @@ -130,6 +132,22 @@ describe("docker helpers", () => { ]); }); + it("prefixes docker argv for manifest inspect and tag helpers (#3885)", () => { + runCaptureMock.mockReturnValue('{"manifests":[]}'); + + dockerManifestInspect("nvcr.io/nim/nvidia/x:latest", { ignoreError: true }); + dockerTag("nvcr.io/nim/nvidia/x@sha256:abc", "nvcr.io/nim/nvidia/x:latest"); + + expect(runCaptureMock).toHaveBeenCalledWith( + ["docker", "manifest", "inspect", "nvcr.io/nim/nvidia/x:latest"], + { ignoreError: true }, + ); + expect(runMock).toHaveBeenCalledWith( + ["docker", "tag", "nvcr.io/nim/nvidia/x@sha256:abc", "nvcr.io/nim/nvidia/x:latest"], + {}, + ); + }); + it("filters docker volume names by exact prefix", () => { runCaptureMock.mockReturnValue( [ diff --git a/src/lib/adapters/docker/inspect.ts b/src/lib/adapters/docker/inspect.ts index 8fed1735e12..b542118be95 100644 --- a/src/lib/adapters/docker/inspect.ts +++ b/src/lib/adapters/docker/inspect.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { dockerCapture, dockerRun, type DockerCaptureOptions, type DockerRunOptions } from "./run"; +import { type DockerCaptureOptions, type DockerRunOptions, dockerCapture, dockerRun } from "./run"; export function dockerInspect(args: readonly string[], opts: DockerRunOptions = {}) { return dockerRun(["inspect", ...args], opts); @@ -26,3 +26,10 @@ export function dockerContainerInspectFormat( ): string { return dockerCapture(["inspect", "--type", "container", "--format", format, containerName], opts); } + +// Capture `docker manifest inspect ` (registry manifest, not a local image). +// For a multi-arch tag this is the OCI image index JSON; callers parse it to +// resolve a per-arch digest. Returns "" on failure when ignoreError is set. +export function dockerManifestInspect(imageRef: string, opts: DockerCaptureOptions = {}): string { + return dockerCapture(["manifest", "inspect", imageRef], opts); +} diff --git a/src/lib/inference/nim.test.ts b/src/lib/inference/nim.test.ts index d36e8062b1c..7d4fad5abd9 100644 --- a/src/lib/inference/nim.test.ts +++ b/src/lib/inference/nim.test.ts @@ -106,6 +106,271 @@ describe("nim", () => { }); }); + // NIM-style OCI index: per-arch linux images + buildkit attestation manifests + // (unknown/unknown) that trip the NGC pull. See #3885. + const NIM_INDEX_JSON = JSON.stringify({ + schemaVersion: 2, + mediaType: "application/vnd.oci.image.index.v1+json", + manifests: [ + { + mediaType: "application/vnd.oci.image.manifest.v1+json", + size: 10203, + digest: "sha256:amd64image", + platform: { architecture: "amd64", os: "linux" }, + }, + { + mediaType: "application/vnd.oci.image.manifest.v1+json", + size: 566, + digest: "sha256:amd64attestation", + platform: { architecture: "unknown", os: "unknown" }, + }, + { + mediaType: "application/vnd.oci.image.manifest.v1+json", + size: 10201, + digest: "sha256:arm64image", + platform: { architecture: "arm64", os: "linux" }, + }, + { + mediaType: "application/vnd.oci.image.manifest.v1+json", + size: 566, + digest: "sha256:arm64attestation", + platform: { architecture: "unknown", os: "unknown" }, + }, + ], + }); + describe("nodeArchToOci", () => { + it("maps x64 to amd64 and passes other arches through", () => { + expect(nim.nodeArchToOci("x64")).toBe("amd64"); + expect(nim.nodeArchToOci("arm64")).toBe("arm64"); + expect(nim.nodeArchToOci("ppc64le")).toBe("ppc64le"); + }); + }); + + describe("imageRepository", () => { + it("drops :tag and @digest, preserving the registry path", () => { + expect(nim.imageRepository("nvcr.io/nim/nvidia/nemotron-3-nano:latest")).toBe( + "nvcr.io/nim/nvidia/nemotron-3-nano", + ); + expect(nim.imageRepository("nvcr.io/nim/nvidia/nemotron-3-nano@sha256:abc")).toBe( + "nvcr.io/nim/nvidia/nemotron-3-nano", + ); + expect(nim.imageRepository("repo/image")).toBe("repo/image"); + }); + + it("treats a colon only in the final path segment as a tag (registry port)", () => { + expect(nim.imageRepository("localhost:5000/team/image:1.0")).toBe( + "localhost:5000/team/image", + ); + }); + }); + + describe("selectPlatformManifestDigest", () => { + it("returns the linux digest for the requested arch, excluding attestation manifests", () => { + expect(nim.selectPlatformManifestDigest(NIM_INDEX_JSON, "arm64")).toBe("sha256:arm64image"); + expect(nim.selectPlatformManifestDigest(NIM_INDEX_JSON, "amd64")).toBe("sha256:amd64image"); + }); + + it("returns null when no entry matches the arch", () => { + expect(nim.selectPlatformManifestDigest(NIM_INDEX_JSON, "ppc64le")).toBeNull(); + }); + + it("returns null for a single-image manifest (no manifests array)", () => { + const single = JSON.stringify({ + schemaVersion: 2, + mediaType: "application/vnd.oci.image.manifest.v1+json", + config: {}, + layers: [], + }); + expect(nim.selectPlatformManifestDigest(single, "amd64")).toBeNull(); + }); + + it("returns null for malformed or empty JSON", () => { + expect(nim.selectPlatformManifestDigest("not json", "amd64")).toBeNull(); + expect(nim.selectPlatformManifestDigest("", "amd64")).toBeNull(); + }); + }); + + describe("pullNimImage", () => { + function findCall(run: Mock, verb: string): string[] | undefined { + const found = run.mock.calls.find((c) => { + const argv = c[0] as string[]; + return Array.isArray(argv) && argv[0] === "docker" && argv[1] === verb; + }); + return found ? (found[0] as string[]) : undefined; + } + + it("resolves the host-arch manifest digest, pulls by digest, then tags back (#3885)", () => { + const origArch = Object.getOwnPropertyDescriptor(process, "arch"); + const run = vi.fn(); + const runCapture = vi.fn((cmd: string | string[]) => { + if (Array.isArray(cmd) && cmd.includes("manifest") && cmd.includes("inspect")) { + return NIM_INDEX_JSON; + } + return ""; + }); + const { nimModule, restore } = loadNimWithMockedRunner(runCapture, run); + try { + Object.defineProperty(process, "arch", { value: "x64", configurable: true }); + const image = nimModule.pullNimImage("nvidia/nemotron-3-nano-30b-a3b"); + expect(image).toBe("nvcr.io/nim/nvidia/nemotron-3-nano:latest"); + + const expectedRef = "nvcr.io/nim/nvidia/nemotron-3-nano@sha256:amd64image"; + + // Pull the per-arch digest, never the bare :latest index (the index pull + // is what triggers the attestation-manifest fetch). + expect(findCall(run, "pull")).toEqual(["docker", "pull", expectedRef]); + expect( + run.mock.calls.some( + (c) => + Array.isArray(c[0]) && + (c[0] as string[])[2] === "nvcr.io/nim/nvidia/nemotron-3-nano:latest", + ), + ).toBe(false); + // Re-tag so the run path can start the container by its :latest ref. + expect(findCall(run, "tag")).toEqual([ + "docker", + "tag", + expectedRef, + "nvcr.io/nim/nvidia/nemotron-3-nano:latest", + ]); + } finally { + if (origArch) Object.defineProperty(process, "arch", origArch); + restore(); + } + }); + + it("falls back to a plain tag pull when manifest inspect yields no index (#3885)", () => { + const run = vi.fn(); + const runCapture = vi.fn(() => ""); // manifest inspect unavailable / not an index + const { nimModule, restore } = loadNimWithMockedRunner(runCapture, run); + try { + nimModule.pullNimImage("nvidia/nemotron-3-nano-30b-a3b"); + expect(findCall(run, "pull")).toEqual([ + "docker", + "pull", + "nvcr.io/nim/nvidia/nemotron-3-nano:latest", + ]); + expect(findCall(run, "tag")).toBeUndefined(); + } finally { + restore(); + } + }); + + it("falls back to a plain tag pull when dockerManifestInspect throws (#3885)", () => { + const run = vi.fn(); + const runCapture = vi.fn(() => { + throw new Error("docker manifest unavailable"); + }); + const { nimModule, restore } = loadNimWithMockedRunner(runCapture, run); + try { + nimModule.pullNimImage("nvidia/nemotron-3-nano-30b-a3b"); + expect(findCall(run, "pull")).toEqual([ + "docker", + "pull", + "nvcr.io/nim/nvidia/nemotron-3-nano:latest", + ]); + expect(findCall(run, "tag")).toBeUndefined(); + } finally { + restore(); + } + }); + }); + + describe("parseServedModelId", () => { + it("returns the first data[].id from a /v1/models body", () => { + const body = JSON.stringify({ + object: "list", + data: [{ id: "nvidia/nemotron-3-nano", object: "model", owned_by: "vllm" }], + }); + expect(nim.parseServedModelId(body)).toBe("nvidia/nemotron-3-nano"); + }); + + it("returns null for empty data, missing data, or malformed JSON", () => { + expect(nim.parseServedModelId(JSON.stringify({ object: "list", data: [] }))).toBeNull(); + expect(nim.parseServedModelId(JSON.stringify({ object: "list" }))).toBeNull(); + expect(nim.parseServedModelId("not json")).toBeNull(); + expect(nim.parseServedModelId("")).toBeNull(); + }); + }); + + describe("getServedModelId", () => { + it("curls /v1/models and returns the served id", () => { + const runCapture = vi.fn((cmd: string | string[]) => { + if (Array.isArray(cmd) && cmd.some((a) => a.includes("/v1/models"))) { + return JSON.stringify({ data: [{ id: "nvidia/nemotron-3-nano" }] }); + } + return ""; + }); + const { nimModule, restore } = loadNimWithMockedRunner(runCapture); + try { + expect(nimModule.getServedModelId(8000)).toBe("nvidia/nemotron-3-nano"); + const call = runCapture.mock.calls.find( + (c) => Array.isArray(c[0]) && (c[0] as string[]).some((a) => a.includes("/v1/models")), + ); + expect(call?.[0]).toContain("http://127.0.0.1:8000/v1/models"); + } finally { + restore(); + } + }); + + it("returns null when the endpoint is unreachable", () => { + const { nimModule, restore } = loadNimWithMockedRunner(vi.fn(() => "")); + try { + expect(nimModule.getServedModelId(8000)).toBeNull(); + } finally { + restore(); + } + }); + }); + + describe("adoptServedModelId", () => { + it("returns the served id when it differs from the catalog name (#3885)", () => { + const runCapture = vi.fn(() => JSON.stringify({ data: [{ id: "nvidia/nemotron-3-nano" }] })); + const { nimModule, restore } = loadNimWithMockedRunner(runCapture); + try { + expect(nimModule.adoptServedModelId("nvidia/nemotron-3-nano-30b-a3b", 8000)).toBe( + "nvidia/nemotron-3-nano", + ); + } finally { + restore(); + } + }); + + it("keeps the catalog value when the served id matches or is unavailable", () => { + const match = loadNimWithMockedRunner( + vi.fn(() => JSON.stringify({ data: [{ id: "meta/llama-3.1-8b-instruct" }] })), + ); + try { + expect(match.nimModule.adoptServedModelId("meta/llama-3.1-8b-instruct", 8000)).toBe( + "meta/llama-3.1-8b-instruct", + ); + } finally { + match.restore(); + } + const down = loadNimWithMockedRunner(vi.fn(() => "")); + try { + expect(down.nimModule.adoptServedModelId("nvidia/nemotron-3-nano-30b-a3b", 8000)).toBe( + "nvidia/nemotron-3-nano-30b-a3b", + ); + } finally { + down.restore(); + } + }); + + it("ignores an unsafe served id and keeps the catalog value (#3885)", () => { + const m = loadNimWithMockedRunner( + vi.fn(() => JSON.stringify({ data: [{ id: "bad id; rm -rf /" }] })), + ); + try { + expect(m.nimModule.adoptServedModelId("nvidia/nemotron-3-nano-30b-a3b", 8000)).toBe( + "nvidia/nemotron-3-nano-30b-a3b", + ); + } finally { + m.restore(); + } + }); + }); + describe("containerName", () => { it("prefixes with nemoclaw-nim-", () => { expect(nim.containerName("my-sandbox")).toBe("nemoclaw-nim-my-sandbox"); @@ -1339,9 +1604,9 @@ describe("nim", () => { const { nimModule, restore } = loadNimWithMockedRunner(runCapture); try { - expect(nimModule.DEFAULT_NIM_HEALTH_TIMEOUT_SECONDS).toBe(900); + expect(nimModule.DEFAULT_NIM_HEALTH_TIMEOUT_SECONDS).toBe(1200); expect(nimModule.waitForNimHealth(9000)).toBe(true); - expect(consoleLogs.some((line) => line.includes("timeout: 900s"))).toBe(true); + expect(consoleLogs.some((line) => line.includes("timeout: 1200s"))).toBe(true); } finally { console.log = origLog; restore(); diff --git a/src/lib/inference/nim.ts b/src/lib/inference/nim.ts index 822950b4230..500a1f3216e 100644 --- a/src/lib/inference/nim.ts +++ b/src/lib/inference/nim.ts @@ -10,16 +10,19 @@ const { dockerForceRm, dockerLoginPasswordStdin, dockerLogs, + dockerManifestInspect, dockerPort, dockerPull, dockerRm, dockerRunDetached, dockerStop, + dockerTag, } = require("../adapters/docker"); const { sleepSeconds } = require("../core/wait"); const nimImages = require("../../../bin/lib/nim-images.json"); import { VLLM_PORT } from "../core/ports"; +import { isSafeModelId } from "../validation"; import { type Arm64WslDockerDesktopGpuProver, isDenylistedNvidiaGpuName, @@ -29,7 +32,7 @@ import { const UNIFIED_MEMORY_GPU_TAGS = ["GB10", "Thor", "Orin", "Xavier", "Jetson", "Tegra"]; const NIM_STATUS_PROBE_TIMEOUT_MS = 5000; -export const DEFAULT_NIM_HEALTH_TIMEOUT_SECONDS = 900; +export const DEFAULT_NIM_HEALTH_TIMEOUT_SECONDS = 1200; export interface NimModel { name: string; @@ -354,6 +357,51 @@ export function canRunNimWithMemory(totalMemoryMB: number): boolean { return nimImages.models.some((m: NimModel) => m.minGpuMemoryMB <= totalMemoryMB); } +// First model id from a NIM `/v1/models` body, or null if absent/unparseable. +export function parseServedModelId(modelsJson: string): string | null { + try { + const doc = JSON.parse(modelsJson); + const data = Array.isArray(doc?.data) ? doc.data : []; + for (const entry of data) { + if (typeof entry?.id === "string" && entry.id.length > 0) return entry.id; + } + } catch { + /* not JSON */ + } + return null; +} + +// Model id a running local NIM actually serves; null if unreachable/empty. +export function getServedModelId(port = VLLM_PORT): string | null { + const out = runCapture( + [ + "curl", + "-sf", + "--connect-timeout", + "5", + "--max-time", + "5", + `http://127.0.0.1:${Number(port)}/v1/models`, + ], + { ignoreError: true }, + ); + return out ? parseServedModelId(out) : null; +} + +// Adopt the id NIM serves (from /v1/models) when it differs from the catalog +// name and is safe; the catalog id otherwise 404s on validation. See #3885. +export function adoptServedModelId(catalogModel: string | null, port = VLLM_PORT): string | null { + const served = getServedModelId(port); + if (!served || served === catalogModel) return catalogModel; + // /v1/models is local-controlled — refuse an unsafe id; don't echo it (log-injection). + if (!isSafeModelId(served)) { + console.error(` NIM reported an invalid model id; keeping "${catalogModel}".`); + return catalogModel; + } + console.log(` NIM serves "${served}" (catalog "${catalogModel}"); using served id.`); + return served; +} + export function detectGpu(deps: DetectGpuDeps = {}): GpuDetection | null { // Try NVIDIA first — query name, total, and free VRAM in a single call so // the preflight line can show the GPU model alongside the memory size and @@ -665,6 +713,90 @@ export function dockerLoginNgc(apiKey: string): boolean { return result.status === 0; } +// Node's process.arch → OCI manifest "architecture" (x64 → amd64; others match). +export function nodeArchToOci(arch: string): string { + if (arch === "x64") return "amd64"; + return arch; +} + +interface ManifestPlatform { + architecture?: string; + os?: string; +} +interface ManifestIndexEntry { + digest?: string; + platform?: ManifestPlatform; +} +interface ManifestIndexDoc { + manifests?: ManifestIndexEntry[]; +} + +// Linux image-manifest digest for `ociArch` from `docker manifest inspect` JSON, +// or null if not a multi-arch index / no match. Arch+os match skips attestations. +export function selectPlatformManifestDigest( + manifestJson: string, + ociArch: string, +): string | null { + let doc: ManifestIndexDoc; + try { + doc = JSON.parse(manifestJson); + } catch { + return null; + } + const manifests = Array.isArray(doc?.manifests) ? doc.manifests : []; + for (const entry of manifests) { + if ( + entry?.platform?.architecture === ociArch && + entry?.platform?.os === "linux" && + typeof entry.digest === "string" && + entry.digest.length > 0 + ) { + return entry.digest; + } + } + return null; +} + +// Repository portion of an image ref, dropping `:tag`/`@digest` (port-safe). +export function imageRepository(imageRef: string): string { + const lastSlash = imageRef.lastIndexOf("/"); + const prefix = lastSlash === -1 ? "" : imageRef.slice(0, lastSlash + 1); + const lastSegment = lastSlash === -1 ? imageRef : imageRef.slice(lastSlash + 1); + const atIdx = lastSegment.indexOf("@"); + if (atIdx !== -1) return prefix + lastSegment.slice(0, atIdx); + const colonIdx = lastSegment.indexOf(":"); + if (colonIdx !== -1) return prefix + lastSegment.slice(0, colonIdx); + return imageRef; +} + +// Pull `image` avoiding the NIM-on-NGC break: docker's containerd store fetches +// the index's buildkit attestation manifest, which nvcr.io rejects ("Incorrect +// Repository Format") after pulling all layers. Pull the host-arch manifest by +// digest instead (no index walk); plain pull when not a resolvable index. #3885. +function pullImageResolvingPlatform(image: string): void { + let manifestJson = ""; + try { + manifestJson = dockerManifestInspect(image, { ignoreError: true }) || ""; + } catch { + manifestJson = ""; + } + const digest = manifestJson + ? selectPlatformManifestDigest(manifestJson, nodeArchToOci(process.arch)) + : null; + if (!digest) { + // No resolvable multi-arch index — plain tag pull. On Docker 29.x this can + // re-hit the NGC attestation failure (#3885); surface the path taken. + console.log(` No platform manifest resolved; pulling ${image} by tag.`); + dockerPull(image); + return; + } + const digestRef = `${imageRepository(image)}@${digest}`; + console.log(` Resolved ${nodeArchToOci(process.arch)} manifest: ${digestRef}`); + dockerPull(digestRef); + // Tag back to the friendly ref so the run path starts the container by `image`. + dockerTag(digestRef, image); +} + export function pullNimImage(model: string): string { const image = getImageForModel(model); if (!image) { @@ -672,7 +804,7 @@ export function pullNimImage(model: string): string { process.exit(1); } console.log(` Pulling NIM image: ${image}`); - dockerPull(image); + pullImageResolvingPlatform(image); return image; } @@ -771,7 +903,7 @@ export function waitForNimHealth( } // Short-circuit if the container has already exited — typically NGC auth // failure or OOM during model load. Without this, the wizard polls the - // full timeout (default 900s) against a dead container. See #3333. + // full timeout (default 1200s) against a dead container. See #3333. if (container) { const state = dockerContainerInspectFormat("{{.State.Status}}", container, { ignoreError: true, diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 46bd4c70e31..6363ad1508d 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -4788,6 +4788,7 @@ async function setupNim( console.error(" Local NVIDIA NIM base URL could not be determined."); process.exit(1); } + model = nim.adoptServedModelId(model); const nimValidationUrl = getLocalProviderValidationBaseUrl(provider) || endpointUrl; const validation = await validateOpenAiLikeSelection( "Local NVIDIA NIM", @@ -4801,10 +4802,8 @@ async function setupNim( if (!validation.ok) { continue selectionLoop; } - preferredInferenceApi = validation.api; - // NIM uses vLLM internally — same tool-call-parser limitation - // applies to /v1/responses. Force chat completions. - if (preferredInferenceApi !== "openai-completions") { + // NIM (vLLM) mishandles the /v1/responses developer role; force chat completions. + if (validation.api !== "openai-completions") { console.log( " ℹ Using chat completions API (tool-call-parser requires /v1/chat/completions)", );