fix(inference): pull NIM by platform digest, use served model id

src/lib/adapters/docker/image.ts

@@ -43,6 +43,14 @@ export function dockerRmi(imageRef: string, opts: DockerRunOptions = {}): Docker
   return dockerRun(["rmi", imageRef], opts);
 }
 
+export function dockerTag(
+  source: string,
+  target: string,
+  opts: DockerRunOptions = {},
+): DockerRunResult {
+  return dockerRun(["tag", source, target], opts);
+}
+
 export function dockerListImagesFormat(
   reference: string,
   format: string,

src/lib/adapters/docker/index.test.ts

@@ -17,11 +17,13 @@ import {
   dockerContainerInspectFormat,
   dockerInfoFormat,
   dockerListVolumesByPrefix,
+  dockerManifestInspect,
   dockerPull,
-  dockerRename,
   dockerRemoveVolumesByPrefix,
+  dockerRename,
   dockerRmi,
   dockerRunDetached,
+  dockerTag,
 } from "./index";
 
 describe("docker helpers", () => {
@@ -130,6 +132,22 @@ describe("docker helpers", () => {
     ]);
   });
 
+  it("prefixes docker argv for manifest inspect and tag helpers (#3885)", () => {
+    runCaptureMock.mockReturnValue('{"manifests":[]}');
+
+    dockerManifestInspect("nvcr.io/nim/nvidia/x:latest", { ignoreError: true });
+    dockerTag("nvcr.io/nim/nvidia/x@sha256:abc", "nvcr.io/nim/nvidia/x:latest");
+
+    expect(runCaptureMock).toHaveBeenCalledWith(
+      ["docker", "manifest", "inspect", "nvcr.io/nim/nvidia/x:latest"],
+      { ignoreError: true },
+    );
+    expect(runMock).toHaveBeenCalledWith(
+      ["docker", "tag", "nvcr.io/nim/nvidia/x@sha256:abc", "nvcr.io/nim/nvidia/x:latest"],
+      {},
+    );
+  });
+
   it("filters docker volume names by exact prefix", () => {
     runCaptureMock.mockReturnValue(
       [

src/lib/adapters/docker/inspect.ts

@@ -1,7 +1,7 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { dockerCapture, dockerRun, type DockerCaptureOptions, type DockerRunOptions } from "./run";
+import { type DockerCaptureOptions, type DockerRunOptions, dockerCapture, dockerRun } from "./run";
 
 export function dockerInspect(args: readonly string[], opts: DockerRunOptions = {}) {
   return dockerRun(["inspect", ...args], opts);
@@ -26,3 +26,10 @@ export function dockerContainerInspectFormat(
 ): string {
   return dockerCapture(["inspect", "--type", "container", "--format", format, containerName], opts);
 }
+
+// Capture `docker manifest inspect <ref>` (registry manifest, not a local image).
+// For a multi-arch tag this is the OCI image index JSON; callers parse it to
+// resolve a per-arch digest. Returns "" on failure when ignoreError is set.
+export function dockerManifestInspect(imageRef: string, opts: DockerCaptureOptions = {}): string {
+  return dockerCapture(["manifest", "inspect", imageRef], opts);
+}

src/lib/inference/nim.test.ts

@@ -106,6 +106,271 @@ describe("nim", () => {
     });
   });
 
+  // NIM-style OCI index: per-arch linux images + buildkit attestation manifests
+  // (unknown/unknown) that trip the NGC pull. See #3885.
+  const NIM_INDEX_JSON = JSON.stringify({
+    schemaVersion: 2,
+    mediaType: "application/vnd.oci.image.index.v1+json",
+    manifests: [
+      {
+        mediaType: "application/vnd.oci.image.manifest.v1+json",
+        size: 10203,
+        digest: "sha256:amd64image",
+        platform: { architecture: "amd64", os: "linux" },
+      },
+      {
+        mediaType: "application/vnd.oci.image.manifest.v1+json",
+        size: 566,
+        digest: "sha256:amd64attestation",
+        platform: { architecture: "unknown", os: "unknown" },
+      },
+      {
+        mediaType: "application/vnd.oci.image.manifest.v1+json",
+        size: 10201,
+        digest: "sha256:arm64image",
+        platform: { architecture: "arm64", os: "linux" },
+      },
+      {
+        mediaType: "application/vnd.oci.image.manifest.v1+json",
+        size: 566,
+        digest: "sha256:arm64attestation",
+        platform: { architecture: "unknown", os: "unknown" },
+      },
+    ],
+  });
+  describe("nodeArchToOci", () => {
+    it("maps x64 to amd64 and passes other arches through", () => {
+      expect(nim.nodeArchToOci("x64")).toBe("amd64");
+      expect(nim.nodeArchToOci("arm64")).toBe("arm64");
+      expect(nim.nodeArchToOci("ppc64le")).toBe("ppc64le");
+    });
+  });
+
+  describe("imageRepository", () => {
+    it("drops :tag and @digest, preserving the registry path", () => {
+      expect(nim.imageRepository("nvcr.io/nim/nvidia/nemotron-3-nano:latest")).toBe(
+        "nvcr.io/nim/nvidia/nemotron-3-nano",
+      );
+      expect(nim.imageRepository("nvcr.io/nim/nvidia/nemotron-3-nano@sha256:abc")).toBe(
+        "nvcr.io/nim/nvidia/nemotron-3-nano",
+      );
+      expect(nim.imageRepository("repo/image")).toBe("repo/image");
+    });
+
+    it("treats a colon only in the final path segment as a tag (registry port)", () => {
+      expect(nim.imageRepository("localhost:5000/team/image:1.0")).toBe(
+        "localhost:5000/team/image",
+      );
+    });
+  });
+
+  describe("selectPlatformManifestDigest", () => {
+    it("returns the linux digest for the requested arch, excluding attestation manifests", () => {
+      expect(nim.selectPlatformManifestDigest(NIM_INDEX_JSON, "arm64")).toBe("sha256:arm64image");
+      expect(nim.selectPlatformManifestDigest(NIM_INDEX_JSON, "amd64")).toBe("sha256:amd64image");
+    });
+
+    it("returns null when no entry matches the arch", () => {
+      expect(nim.selectPlatformManifestDigest(NIM_INDEX_JSON, "ppc64le")).toBeNull();
+    });
+
+    it("returns null for a single-image manifest (no manifests array)", () => {
+      const single = JSON.stringify({
+        schemaVersion: 2,
+        mediaType: "application/vnd.oci.image.manifest.v1+json",
+        config: {},
+        layers: [],
+      });
+      expect(nim.selectPlatformManifestDigest(single, "amd64")).toBeNull();
+    });
+
+    it("returns null for malformed or empty JSON", () => {
+      expect(nim.selectPlatformManifestDigest("not json", "amd64")).toBeNull();
+      expect(nim.selectPlatformManifestDigest("", "amd64")).toBeNull();
+    });
+  });
+
+  describe("pullNimImage", () => {
+    function findCall(run: Mock, verb: string): string[] | undefined {
+      const found = run.mock.calls.find((c) => {
+        const argv = c[0] as string[];
+        return Array.isArray(argv) && argv[0] === "docker" && argv[1] === verb;
+      });
+      return found ? (found[0] as string[]) : undefined;
+    }
+
+    it("resolves the host-arch manifest digest, pulls by digest, then tags back (#3885)", () => {
+      const origArch = Object.getOwnPropertyDescriptor(process, "arch");
+      const run = vi.fn();
+      const runCapture = vi.fn((cmd: string | string[]) => {
+        if (Array.isArray(cmd) && cmd.includes("manifest") && cmd.includes("inspect")) {
+          return NIM_INDEX_JSON;
+        }
+        return "";
+      });
+      const { nimModule, restore } = loadNimWithMockedRunner(runCapture, run);
+      try {
+        Object.defineProperty(process, "arch", { value: "x64", configurable: true });
+        const image = nimModule.pullNimImage("nvidia/nemotron-3-nano-30b-a3b");
+        expect(image).toBe("nvcr.io/nim/nvidia/nemotron-3-nano:latest");
+
+        const expectedRef = "nvcr.io/nim/nvidia/nemotron-3-nano@sha256:amd64image";
+
+        // Pull the per-arch digest, never the bare :latest index (the index pull
+        // is what triggers the attestation-manifest fetch).
+        expect(findCall(run, "pull")).toEqual(["docker", "pull", expectedRef]);
+        expect(
+          run.mock.calls.some(
+            (c) =>
+              Array.isArray(c[0]) &&
+              (c[0] as string[])[2] === "nvcr.io/nim/nvidia/nemotron-3-nano:latest",
+          ),
+        ).toBe(false);
+        // Re-tag so the run path can start the container by its :latest ref.
+        expect(findCall(run, "tag")).toEqual([
+          "docker",
+          "tag",
+          expectedRef,
+          "nvcr.io/nim/nvidia/nemotron-3-nano:latest",
+        ]);
+      } finally {
+        if (origArch) Object.defineProperty(process, "arch", origArch);
+        restore();
+      }
+    });
+
+    it("falls back to a plain tag pull when manifest inspect yields no index (#3885)", () => {
+      const run = vi.fn();
+      const runCapture = vi.fn(() => ""); // manifest inspect unavailable / not an index
+      const { nimModule, restore } = loadNimWithMockedRunner(runCapture, run);
+      try {
+        nimModule.pullNimImage("nvidia/nemotron-3-nano-30b-a3b");
+        expect(findCall(run, "pull")).toEqual([
+          "docker",
+          "pull",
+          "nvcr.io/nim/nvidia/nemotron-3-nano:latest",
+        ]);
+        expect(findCall(run, "tag")).toBeUndefined();
+      } finally {
+        restore();
+      }
+    });
+
+    it("falls back to a plain tag pull when dockerManifestInspect throws (#3885)", () => {
+      const run = vi.fn();
+      const runCapture = vi.fn(() => {
+        throw new Error("docker manifest unavailable");
+      });
+      const { nimModule, restore } = loadNimWithMockedRunner(runCapture, run);
+      try {
+        nimModule.pullNimImage("nvidia/nemotron-3-nano-30b-a3b");
+        expect(findCall(run, "pull")).toEqual([
+          "docker",
+          "pull",
+          "nvcr.io/nim/nvidia/nemotron-3-nano:latest",
+        ]);
+        expect(findCall(run, "tag")).toBeUndefined();
+      } finally {
+        restore();
+      }
+    });
+  });
+
+  describe("parseServedModelId", () => {
+    it("returns the first data[].id from a /v1/models body", () => {
+      const body = JSON.stringify({
+        object: "list",
+        data: [{ id: "nvidia/nemotron-3-nano", object: "model", owned_by: "vllm" }],
+      });
+      expect(nim.parseServedModelId(body)).toBe("nvidia/nemotron-3-nano");
+    });
+
+    it("returns null for empty data, missing data, or malformed JSON", () => {
+      expect(nim.parseServedModelId(JSON.stringify({ object: "list", data: [] }))).toBeNull();
+      expect(nim.parseServedModelId(JSON.stringify({ object: "list" }))).toBeNull();
+      expect(nim.parseServedModelId("not json")).toBeNull();
+      expect(nim.parseServedModelId("")).toBeNull();
+    });
+  });
+
+  describe("getServedModelId", () => {
+    it("curls /v1/models and returns the served id", () => {
+      const runCapture = vi.fn((cmd: string | string[]) => {
+        if (Array.isArray(cmd) && cmd.some((a) => a.includes("/v1/models"))) {
+          return JSON.stringify({ data: [{ id: "nvidia/nemotron-3-nano" }] });
+        }
+        return "";
+      });
+      const { nimModule, restore } = loadNimWithMockedRunner(runCapture);
+      try {
+        expect(nimModule.getServedModelId(8000)).toBe("nvidia/nemotron-3-nano");
+        const call = runCapture.mock.calls.find(
+          (c) => Array.isArray(c[0]) && (c[0] as string[]).some((a) => a.includes("/v1/models")),
+        );
+        expect(call?.[0]).toContain("http://127.0.0.1:8000/v1/models");
+      } finally {
+        restore();
+      }
+    });
+
+    it("returns null when the endpoint is unreachable", () => {
+      const { nimModule, restore } = loadNimWithMockedRunner(vi.fn(() => ""));
+      try {
+        expect(nimModule.getServedModelId(8000)).toBeNull();
+      } finally {
+        restore();
+      }
+    });
+  });
+
+  describe("adoptServedModelId", () => {
+    it("returns the served id when it differs from the catalog name (#3885)", () => {
+      const runCapture = vi.fn(() => JSON.stringify({ data: [{ id: "nvidia/nemotron-3-nano" }] }));
+      const { nimModule, restore } = loadNimWithMockedRunner(runCapture);
+      try {
+        expect(nimModule.adoptServedModelId("nvidia/nemotron-3-nano-30b-a3b", 8000)).toBe(
+          "nvidia/nemotron-3-nano",
+        );
+      } finally {
+        restore();
+      }
+    });
+
+    it("keeps the catalog value when the served id matches or is unavailable", () => {
+      const match = loadNimWithMockedRunner(
+        vi.fn(() => JSON.stringify({ data: [{ id: "meta/llama-3.1-8b-instruct" }] })),
+      );
+      try {
+        expect(match.nimModule.adoptServedModelId("meta/llama-3.1-8b-instruct", 8000)).toBe(
+          "meta/llama-3.1-8b-instruct",
+        );
+      } finally {
+        match.restore();
+      }
+      const down = loadNimWithMockedRunner(vi.fn(() => ""));
+      try {
+        expect(down.nimModule.adoptServedModelId("nvidia/nemotron-3-nano-30b-a3b", 8000)).toBe(
+          "nvidia/nemotron-3-nano-30b-a3b",
+        );
+      } finally {
+        down.restore();
+      }
+    });
+
+    it("ignores an unsafe served id and keeps the catalog value (#3885)", () => {
+      const m = loadNimWithMockedRunner(
+        vi.fn(() => JSON.stringify({ data: [{ id: "bad id; rm -rf /" }] })),
+      );
+      try {
+        expect(m.nimModule.adoptServedModelId("nvidia/nemotron-3-nano-30b-a3b", 8000)).toBe(
+          "nvidia/nemotron-3-nano-30b-a3b",
+        );
+      } finally {
+        m.restore();
+      }
+    });
+  });
+
   describe("containerName", () => {
     it("prefixes with nemoclaw-nim-", () => {
       expect(nim.containerName("my-sandbox")).toBe("nemoclaw-nim-my-sandbox");
@@ -1339,9 +1604,9 @@ describe("nim", () => {
       const { nimModule, restore } = loadNimWithMockedRunner(runCapture);
 
       try {
-        expect(nimModule.DEFAULT_NIM_HEALTH_TIMEOUT_SECONDS).toBe(900);
+        expect(nimModule.DEFAULT_NIM_HEALTH_TIMEOUT_SECONDS).toBe(1200);
         expect(nimModule.waitForNimHealth(9000)).toBe(true);
-        expect(consoleLogs.some((line) => line.includes("timeout: 900s"))).toBe(true);
+        expect(consoleLogs.some((line) => line.includes("timeout: 1200s"))).toBe(true);
       } finally {
         console.log = origLog;
         restore();