fix(preflight): warn before gateway start when nvidia.com/gpu CDI spec is missing

docs/reference/troubleshooting.md

@@ -925,6 +925,22 @@ GPU passthrough is not CI-tested on DGX Spark.
 It is expected to work when you pass `--gpu` and the NVIDIA Container Toolkit is configured.
 Verify the toolkit is configured by running `docker run --rm --runtime=nvidia --gpus all nvidia/cuda:12.8.0-base-ubuntu24.04 nvidia-smi` from the host.
 
+### `unresolvable CDI devices nvidia.com/gpu=all` during gateway start
+
+Recent NVIDIA Container Toolkit installs configure the Docker daemon for Container Device Interface (CDI) device injection, which OpenShell's `gateway start --gpu` then auto-selects.
+If no `nvidia.com/gpu` CDI spec has been generated on the host yet, gateway start fails with `Docker responded with status code 500: CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all`.
+`nemoclaw onboard` now detects this gap during preflight and prints the remediation up front, but the underlying fix is the same on any Docker host whose `docker info` advertises a non-empty `CDISpecDirs`.
+
+Generate the spec, verify it lists `nvidia.com/gpu` entries, then rerun onboarding:
+
+```console
+$ sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
+$ nvidia-ctk cdi list
+$ nemoclaw onboard
+```
+
+If GPU passthrough is not required on this host, rerun onboarding with `--no-gpu` instead.
+
 ### `pip install` fails with a system-packages error
 
 Recent Ubuntu releases (including DGX Spark's Ubuntu 24.04) mark the system Python install as externally managed, so `pip install` without a virtual environment fails.

src/lib/onboard.ts

@@ -3245,7 +3245,36 @@ function waitForSandboxReady(sandboxName: string, attempts = 10, delaySeconds =
 
 // ── Step 1: Preflight ────────────────────────────────────────────
 
-async function preflight(): Promise<ReturnType<typeof nim.detectGpu>> {
+// CDI spec gap (#3152). When Docker is configured for CDI device injection
+// (CDISpecDirs is set) but no nvidia.com/gpu spec is present, OpenShell's
+// `gateway start --gpu` fails minutes later with `unresolvable CDI devices
+// nvidia.com/gpu=all`. Block now and surface `nvidia-ctk cdi generate`. The
+// check is a no-op when the user opts out of GPU passthrough (--no-gpu),
+// since the legacy nvidia runtime does not need a CDI spec.
+//
+// Extracted so the same guard runs on the `--resume` branch, where preflight()
+// itself is skipped via the cached session.
+function assertCdiNvidiaGpuSpecPresent(
+  host: ReturnType<typeof assessHost>,
+  optedOutGpuPassthrough: boolean,
+): void {
+  if (!host.cdiNvidiaGpuSpecMissing || optedOutGpuPassthrough) return;
+  console.error(
+    "  Docker is configured for CDI device injection (CDISpecDirs is set), but no",
+  );
+  console.error(
+    "  nvidia.com/gpu CDI spec was found on the host. OpenShell's gateway start will",
+  );
+  console.error(
+    "  fail with `unresolvable CDI devices nvidia.com/gpu=all` (issue #3152).",
+  );
+  printRemediationActions(planHostRemediation(host));
+  process.exit(1);
+}
+
+async function preflight(
+  preflightOpts: { optedOutGpuPassthrough?: boolean } = {},
+): Promise<ReturnType<typeof nim.detectGpu>> {
   step(1, 8, "Preflight checks");
 
   const host = assessHost();
@@ -3258,6 +3287,8 @@ async function preflight(): Promise<ReturnType<typeof nim.detectGpu>> {
   }
   console.log("  ✓ Docker is running");
 
+  assertCdiNvidiaGpuSpecPresent(host, preflightOpts.optedOutGpuPassthrough === true);
+
   // DNS resolution from inside containers (#2101). A corp firewall that
   // blocks outbound UDP:53 to public resolvers leaves the sandbox build
   // unable to resolve registry.npmjs.org; npm then retries for ~15 min and
@@ -9389,9 +9420,20 @@ async function onboard(opts: OnboardOptions = {}): Promise<void> {
     if (resumePreflight) {
       skippedStepMessage("preflight", "cached");
       gpu = nim.detectGpu();
+      // Re-check the CDI spec gap on resume (#3152). The cached preflight
+      // result does not capture host CDI state, and the original onboard
+      // attempt that wrote the cache likely aborted at gateway-start with
+      // exactly this CDI failure — so resuming without re-checking would
+      // walk into the same wall. Honour persisted `gpuPassthrough: false`
+      // from the prior session as an opt-out, since the resume invocation
+      // does not need to re-pass `--no-gpu` to keep that intent (the same
+      // resolution is replayed a few lines below for `gpuPassthrough`).
+      const resumeOptedOutGpuPassthrough =
+        opts.noGpu === true || (opts.gpu !== true && session?.gpuPassthrough === false);
+      assertCdiNvidiaGpuSpecPresent(assessHost(), resumeOptedOutGpuPassthrough);
     } else {
       startRecordedStep("preflight");
-      gpu = await preflight();
+      gpu = await preflight({ optedOutGpuPassthrough: opts.noGpu === true });
       onboardSession.markStepComplete("preflight");
     }
 

src/lib/preflight.test.ts

@@ -14,6 +14,7 @@ import {
   isDockerUnderProvisioned,
   MIN_RECOMMENDED_DOCKER_CPUS,
   MIN_RECOMMENDED_DOCKER_MEM_GIB,
+  parseDockerCdiSpecDirs,
   parseDockerInfoCpus,
   parseDockerInfoMemTotalBytes,
   parseDockerStorageDriver,
@@ -499,6 +500,200 @@ describe("parseDockerUsesContainerdSnapshotter", () => {
   });
 });
 
+describe("parseDockerCdiSpecDirs", () => {
+  it("extracts the dirs from `docker info --format '{{json .}}'` output", () => {
+    const fixture = JSON.stringify({ CDISpecDirs: ["/etc/cdi", "/var/run/cdi"] });
+    expect(parseDockerCdiSpecDirs(fixture)).toEqual(["/etc/cdi", "/var/run/cdi"]);
+  });
+
+  it("returns an empty array when CDISpecDirs is absent", () => {
+    expect(parseDockerCdiSpecDirs(JSON.stringify({ ServerVersion: "27.0" }))).toEqual([]);
+  });
+
+  it("returns an empty array when CDISpecDirs is the empty list", () => {
+    expect(parseDockerCdiSpecDirs(JSON.stringify({ CDISpecDirs: [] }))).toEqual([]);
+  });
+
+  it("returns an empty array on empty input", () => {
+    expect(parseDockerCdiSpecDirs("")).toEqual([]);
+  });
+});
+
+describe("assessHost — CDI device-spec gap (#3152)", () => {
+  it("flags missing nvidia.com/gpu specs on an NVIDIA Linux host with CDI dirs configured", () => {
+    const result = assessHost({
+      platform: "linux",
+      env: {},
+      release: "6.8.0-58-generic",
+      readFileImpl: () => "Linux version 6.8.0-58-generic",
+      readdirImpl: () => [],
+      dockerInfoOutput: JSON.stringify({
+        ServerVersion: "27.0",
+        OperatingSystem: "Ubuntu 24.04",
+        CDISpecDirs: ["/etc/cdi", "/var/run/cdi"],
+      }),
+      commandExistsImpl: (name: string) => name === "docker",
+      gpuProbeImpl: () => true,
+    });
+
+    expect(result.dockerCdiSpecDirs).toEqual(["/etc/cdi", "/var/run/cdi"]);
+    expect(result.cdiNvidiaGpuSpecMissing).toBe(true);
+  });
+
+  it("does not flag the host when an nvidia.com/gpu YAML spec is present", () => {
+    const result = assessHost({
+      platform: "linux",
+      env: {},
+      release: "6.8.0-58-generic",
+      readFileImpl: (filePath: string) =>
+        filePath.endsWith("nvidia.yaml")
+          ? "cdiVersion: 0.5.0\nkind: nvidia.com/gpu\ndevices: []\n"
+          : "Linux version 6.8.0-58-generic",
+      readdirImpl: (dir: string) => (dir === "/etc/cdi" ? ["nvidia.yaml"] : []),
+      dockerInfoOutput: JSON.stringify({
+        ServerVersion: "27.0",
+        CDISpecDirs: ["/etc/cdi", "/var/run/cdi"],
+      }),
+      commandExistsImpl: (name: string) => name === "docker",
+      gpuProbeImpl: () => true,
+    });
+
+    expect(result.cdiNvidiaGpuSpecMissing).toBe(false);
+  });
+
+  it("accepts a JSON-serialised CDI spec as well", () => {
+    const result = assessHost({
+      platform: "linux",
+      env: {},
+      release: "6.8.0-58-generic",
+      readFileImpl: (filePath: string) =>
+        filePath.endsWith("nvidia.json")
+          ? '{"cdiVersion":"0.5.0","kind":"nvidia.com/gpu","devices":[]}'
+          : "Linux version 6.8.0-58-generic",
+      readdirImpl: (dir: string) => (dir === "/etc/cdi" ? ["nvidia.json"] : []),
+      dockerInfoOutput: JSON.stringify({
+        ServerVersion: "27.0",
+        CDISpecDirs: ["/etc/cdi"],
+      }),
+      commandExistsImpl: (name: string) => name === "docker",
+      gpuProbeImpl: () => true,
+    });
+
+    expect(result.cdiNvidiaGpuSpecMissing).toBe(false);
+  });
+
+  it("does not flag a non-NVIDIA Linux host even with CDI dirs configured", () => {
+    const result = assessHost({
+      platform: "linux",
+      env: {},
+      release: "6.8.0-58-generic",
+      readFileImpl: () => "Linux version 6.8.0-58-generic",
+      readdirImpl: () => [],
+      dockerInfoOutput: JSON.stringify({
+        ServerVersion: "27.0",
+        CDISpecDirs: ["/etc/cdi"],
+      }),
+      commandExistsImpl: (name: string) => name === "docker",
+      gpuProbeImpl: () => false,
+    });
+
+    expect(result.cdiNvidiaGpuSpecMissing).toBe(false);
+  });
+
+  it("does not flag a host that does not advertise CDISpecDirs", () => {
+    const result = assessHost({
+      platform: "linux",
+      env: {},
+      release: "6.8.0-58-generic",
+      readFileImpl: () => "Linux version 6.8.0-58-generic",
+      readdirImpl: () => [],
+      dockerInfoOutput: JSON.stringify({ ServerVersion: "24.0" }),
+      commandExistsImpl: (name: string) => name === "docker",
+      gpuProbeImpl: () => true,
+    });
+
+    expect(result.dockerCdiSpecDirs).toEqual([]);
+    expect(result.cdiNvidiaGpuSpecMissing).toBe(false);
+  });
+
+  it("does not flag macOS even when the docker info shape would otherwise match", () => {
+    const result = assessHost({
+      platform: "darwin",
+      env: {},
+      readFileImpl: () => "",
+      readdirImpl: () => [],
+      dockerInfoOutput: JSON.stringify({ CDISpecDirs: ["/etc/cdi"] }),
+      commandExistsImpl: (name: string) => name === "docker",
+      gpuProbeImpl: () => true,
+    });
+
+    expect(result.cdiNvidiaGpuSpecMissing).toBe(false);
+  });
+
+  it("does not accept a sibling device class such as nvidia.com/gpu-extra as a satisfying spec", () => {
+    const result = assessHost({
+      platform: "linux",
+      env: {},
+      release: "6.8.0-58-generic",
+      readFileImpl: (filePath: string) =>
+        filePath.endsWith("nvidia-extra.yaml")
+          ? "cdiVersion: 0.5.0\nkind: nvidia.com/gpu-extra\ndevices: []\n"
+          : "Linux version 6.8.0-58-generic",
+      readdirImpl: (dir: string) => (dir === "/etc/cdi" ? ["nvidia-extra.yaml"] : []),
+      dockerInfoOutput: JSON.stringify({
+        ServerVersion: "27.0",
+        CDISpecDirs: ["/etc/cdi"],
+      }),
+      commandExistsImpl: (name: string) => name === "docker",
+      gpuProbeImpl: () => true,
+    });
+
+    expect(result.cdiNvidiaGpuSpecMissing).toBe(true);
+  });
+
+  it("does not accept a sibling device class in JSON form either", () => {
+    const result = assessHost({
+      platform: "linux",
+      env: {},
+      release: "6.8.0-58-generic",
+      readFileImpl: (filePath: string) =>
+        filePath.endsWith("nvidia-extra.json")
+          ? '{"cdiVersion":"0.5.0","kind":"nvidia.com/gpu-extra","devices":[]}'
+          : "Linux version 6.8.0-58-generic",
+      readdirImpl: (dir: string) => (dir === "/etc/cdi" ? ["nvidia-extra.json"] : []),
+      dockerInfoOutput: JSON.stringify({
+        ServerVersion: "27.0",
+        CDISpecDirs: ["/etc/cdi"],
+      }),
+      commandExistsImpl: (name: string) => name === "docker",
+      gpuProbeImpl: () => true,
+    });
+
+    expect(result.cdiNvidiaGpuSpecMissing).toBe(true);
+  });
+
+  it("ignores spec files whose `kind` only mentions nvidia.com/gpu in a comment", () => {
+    const result = assessHost({
+      platform: "linux",
+      env: {},
+      release: "6.8.0-58-generic",
+      readFileImpl: (filePath: string) =>
+        filePath.endsWith("notes.yaml")
+          ? "# this used to declare nvidia.com/gpu; now stripped\nkind: example.com/cpu\n"
+          : "Linux version 6.8.0-58-generic",
+      readdirImpl: (dir: string) => (dir === "/etc/cdi" ? ["notes.yaml"] : []),
+      dockerInfoOutput: JSON.stringify({
+        ServerVersion: "27.0",
+        CDISpecDirs: ["/etc/cdi"],
+      }),
+      commandExistsImpl: (name: string) => name === "docker",
+      gpuProbeImpl: () => true,
+    });
+
+    expect(result.cdiNvidiaGpuSpecMissing).toBe(true);
+  });
+});
+
 describe("planHostRemediation", () => {
   it("recommends starting docker when installed but unreachable and service inactive", () => {
     const actions = planHostRemediation({
@@ -522,6 +717,8 @@ describe("planHostRemediation", () => {
       isUnsupportedRuntime: false,
       isHeadlessLikely: false,
       hasNvidiaGpu: false,
+      dockerCdiSpecDirs: [],
+      cdiNvidiaGpuSpecMissing: false,
       notes: [],
     });
 
@@ -552,6 +749,8 @@ describe("planHostRemediation", () => {
       isUnsupportedRuntime: false,
       isHeadlessLikely: false,
       hasNvidiaGpu: false,
+      dockerCdiSpecDirs: [],
+      cdiNvidiaGpuSpecMissing: false,
       notes: [],
     });
 
@@ -586,6 +785,8 @@ describe("planHostRemediation", () => {
       isUnsupportedRuntime: true,
       isHeadlessLikely: false,
       hasNvidiaGpu: false,
+      dockerCdiSpecDirs: [],
+      cdiNvidiaGpuSpecMissing: false,
       notes: [],
     });
 
@@ -618,6 +819,8 @@ describe("planHostRemediation", () => {
       isUnsupportedRuntime: false,
       isHeadlessLikely: false,
       hasNvidiaGpu: false,
+      dockerCdiSpecDirs: [],
+      cdiNvidiaGpuSpecMissing: false,
       notes: [],
     });
 
@@ -647,11 +850,54 @@ describe("planHostRemediation", () => {
       isUnsupportedRuntime: false,
       isHeadlessLikely: false,
       hasNvidiaGpu: false,
+      dockerCdiSpecDirs: [],
+      cdiNvidiaGpuSpecMissing: false,
       notes: [],
     });
 
     expect(actions.some((action: { id: string }) => action.id === "install_openshell")).toBe(true);
   });
+
+  it("emits a blocking generate_nvidia_cdi_spec action when CDI dirs are configured but no nvidia.com/gpu spec exists", () => {
+    const actions = planHostRemediation({
+      platform: "linux",
+      isWsl: false,
+      runtime: "docker",
+      packageManager: "apt",
+      systemctlAvailable: true,
+      dockerServiceActive: true,
+      dockerServiceEnabled: true,
+      dockerInstalled: true,
+      dockerRunning: true,
+      dockerReachable: true,
+      nodeInstalled: true,
+      openshellInstalled: true,
+      dockerCgroupVersion: "v2",
+      dockerDefaultCgroupnsMode: "unknown",
+      isContainerRuntimeUnderProvisioned: false,
+      hasNestedOverlayConflict: false,
+      requiresHostCgroupnsFix: false,
+      isUnsupportedRuntime: false,
+      isHeadlessLikely: false,
+      hasNvidiaGpu: true,
+      dockerCdiSpecDirs: ["/etc/cdi", "/var/run/cdi"],
+      cdiNvidiaGpuSpecMissing: true,
+      notes: [],
+    });
+
+    const action = actions.find(
+      (entry: { id: string }) => entry.id === "generate_nvidia_cdi_spec",
+    );
+    expect(action).toBeTruthy();
+    expect(action?.kind).toBe("sudo");
+    expect(action?.blocking).toBe(true);
+    expect(action?.commands[0]).toBe(
+      "sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml",
+    );
+    expect(action?.commands[1]).toContain("nvidia-ctk cdi list");
+    expect(action?.commands[2]).toContain("nemoclaw onboard");
+    expect(action?.reason).toContain("nvidia.com/gpu");
+  });
 });
 
 describe("ensureSwap", () => {