fix: address issue #5489

src/lib/onboard.ts

@@ -1577,23 +1577,6 @@ function waitForSandboxReady(sandboxName: string, attempts = 10, delaySeconds =
 
 // ── Step 1: Preflight ────────────────────────────────────────────
 
-// Keep the Docker CDI guard near preflight so resume hits the same early failure path.
-// Jetson/Tegra uses Docker's NVIDIA runtime backend and is exempt from CDI.
-function assertCdiNvidiaGpuSpecPresent(
-  host: ReturnType<typeof assessHost>,
-  optedOutGpuPassthrough: boolean,
-  hostGpuPlatform: string | null | undefined = null,
-): void {
-  if (hostGpuPlatform === "jetson" || preflightUtils.isWslDockerDesktopRuntime(host)) return;
-  if (!(host.cdiNvidiaGpuSpecNeedsRepair || host.cdiNvidiaGpuSpecMissing) || optedOutGpuPassthrough)
-    return;
-  console.error(
-    "  Docker is configured for CDI device injection (CDISpecDirs is set), but the NVIDIA GPU CDI spec is missing or stale. OpenShell GPU startup can fail until the CDI spec is refreshed.",
-  );
-  printRemediationActions(planHostRemediation(host));
-  process.exit(1);
-}
-
 type PreflightOptions = Pick<
   OnboardOptions,
   "sandboxGpu" | "sandboxGpuDevice" | "gpu" | "noGpu"
@@ -1640,11 +1623,13 @@ async function preflight(
     device: preflightOpts.sandboxGpuDevice ?? null,
   });
   exitOnSandboxGpuConfigErrors(sandboxGpuConfig);
-  const optedOutGpuPassthrough =
-    preflightOpts.optedOutGpuPassthrough === true ||
-    preflightOpts.noGpu === true ||
-    !sandboxGpuConfig.sandboxGpuEnabled;
-  assertCdiNvidiaGpuSpecPresent(host, optedOutGpuPassthrough, sandboxGpuConfig.hostGpuPlatform);
+  const explicitlyOptedOutGpuPassthrough =
+    preflightOpts.optedOutGpuPassthrough === true || preflightOpts.noGpu === true;
+  preflightUtils.assertCdiNvidiaGpuSpecPresent(
+    host,
+    explicitlyOptedOutGpuPassthrough,
+    sandboxGpuConfig.hostGpuPlatform,
+  );
 
   assertDockerBridgeAndContainerDnsHealthy(host, isNonInteractive());
 
@@ -4956,7 +4941,7 @@ async function onboard(opts: OnboardOptions = {}): Promise<void> {
         detectGpu: nim.detectGpu,
         runPreflight: (preflightOptions) => preflight({ ...opts, ...preflightOptions }),
         assessHost,
-        assertCdiNvidiaGpuSpecPresent,
+        assertCdiNvidiaGpuSpecPresent: preflightUtils.assertCdiNvidiaGpuSpecPresent,
         rejectUnsupportedContainerRuntime,
         assertDockerBridgeAndContainerDnsHealthy,
         resolveSandboxGpuConfig,

src/lib/onboard/preflight-cdi.test.ts

@@ -4,7 +4,11 @@
 import { describe, expect, it } from "vitest";
 // Import through the compiled dist/ output so coverage is attributed to the
 // CLI build output that the ratchet measures.
-import { assessHost, planHostRemediation } from "../../../dist/lib/onboard/preflight";
+import {
+  assessHost,
+  planHostRemediation,
+  shouldEnforceCdiNvidiaGpuSpec,
+} from "../../../dist/lib/onboard/preflight";
 
 type HostAssessment = Parameters<typeof planHostRemediation>[0];
 
@@ -38,6 +42,18 @@ function baseAssessment(overrides: Partial<HostAssessment> = {}): HostAssessment
   };
 }
 
+function runCaptureWithLspci(lspciOutput: string): (command: readonly string[]) => string {
+  const resultByCmd: Record<string, string> = { "nvidia-smi": "", lspci: lspciOutput };
+  return (command) => {
+    const last = command[command.length - 1];
+    return command[0] === "sh" && command[1] === "-c"
+      ? last === "lspci" || last === "apt-get"
+        ? `/usr/bin/${last}`
+        : ""
+      : (resultByCmd[command[0]] ?? "");
+  };
+}
+
 function healthySystemctlAndStat(command: readonly string[]) {
   if (command[0] === "systemctl" && command[1] === "is-enabled") return "enabled";
   if (command[0] === "systemctl" && command[1] === "is-active") return "active";
@@ -319,6 +335,82 @@ describe("planHostRemediation — CDI", () => {
     expect(action?.reason).toContain("path disabled");
   });
 
+  it("blocks with toolkit/CDI remediation when NVIDIA hardware is present but nvidia-smi is unavailable (#5489)", () => {
+    // Repro: NVIDIA GPU hardware present (lspci) but the driver is not loaded,
+    // so nvidia-smi is unavailable. Docker CDI dirs are configured and the
+    // nvidia-container-toolkit is absent. Onboard must still flag the missing
+    // CDI spec and emit the install_nvidia_container_toolkit remediation block.
+    // Drive detection through runCaptureImpl (the real detectNvidiaGpu path)
+    // rather than gpuProbeImpl so the red->green transition exercises the fix.
+    const result = assessHost({
+      platform: "linux",
+      env: {},
+      release: "6.8.0-58-generic",
+      readFileImpl: () => "Linux version 6.8.0-58-generic",
+      readdirImpl: () => [],
+      dockerInfoOutput: JSON.stringify({
+        ServerVersion: "29.5.3",
+        OperatingSystem: "Ubuntu 24.04",
+        CDISpecDirs: ["/etc/cdi", "/var/run/cdi"],
+      }),
+      commandExistsImpl: (name: string) => name === "docker",
+      runCaptureImpl: runCaptureWithLspci(
+        "01:00.0 VGA compatible controller: NVIDIA Corporation GK104 [GeForce GTX 660 Ti] (rev a1)",
+      ),
+    });
+
+    expect(result.hasNvidiaGpu).toBe(true);
+    expect(result.cdiNvidiaGpuSpecMissing).toBe(true);
+
+    const actions = planHostRemediation(result);
+    const action = actions.find((entry) => entry.id === "install_nvidia_container_toolkit");
+    expect(action).toBeTruthy();
+    expect(action?.blocking).toBe(true);
+    expect(
+      action?.commands.some(
+        (command) => command === "sudo apt-get install -y nvidia-container-toolkit",
+      ),
+    ).toBe(true);
+    expect(
+      action?.commands.some((command) =>
+        command.startsWith("sudo nvidia-ctk cdi generate --output="),
+      ),
+    ).toBe(true);
+  });
+
+  it("does not treat non-GPU NVIDIA PCI devices as a GPU in the lspci fallback (#5489)", () => {
+    // Hosts with NVIDIA/Mellanox NICs (or other non-GPU NVIDIA PCI devices)
+    // expose "nvidia" in lspci output without any display-class GPU. The
+    // hardware fallback must restrict matching to display PCI classes so these
+    // hosts are not falsely flagged and forced through CDI/toolkit remediation.
+    const result = assessHost({
+      platform: "linux",
+      env: {},
+      release: "6.8.0-58-generic",
+      readFileImpl: () => "Linux version 6.8.0-58-generic",
+      readdirImpl: () => [],
+      dockerInfoOutput: JSON.stringify({
+        ServerVersion: "29.5.3",
+        OperatingSystem: "Ubuntu 24.04",
+        CDISpecDirs: ["/etc/cdi", "/var/run/cdi"],
+      }),
+      commandExistsImpl: (name: string) => name === "docker",
+      runCaptureImpl: runCaptureWithLspci(
+        [
+          "01:00.0 Ethernet controller: Mellanox Technologies MT27800 Family [ConnectX-5]",
+          "02:00.0 Infiniband controller: NVIDIA Corporation MT28908 Family [ConnectX-6]",
+        ].join("\n"),
+      ),
+    });
+
+    expect(result.hasNvidiaGpu).toBe(false);
+
+    const action = planHostRemediation(result).find(
+      (entry) => entry.id === "install_nvidia_container_toolkit",
+    );
+    expect(action).toBeFalsy();
+  });
+
   it("bootstraps nvidia-container-toolkit before missing-spec generation", () => {
     const actions = planHostRemediation(
       baseAssessment({
@@ -341,3 +433,49 @@ describe("planHostRemediation — CDI", () => {
     ).toBe(true);
   });
 });
+
+describe("shouldEnforceCdiNvidiaGpuSpec (#5489 enforcement gate)", () => {
+  it("enforces when the spec is missing and the operator did not explicitly opt out", () => {
+    // The #5489 scenario: GPU hardware present (so cdiNvidiaGpuSpecMissing is
+    // true) with sandbox GPU AUTO-disabled (nvidia-smi unavailable). Auto-disable
+    // must NOT be treated as an opt-out, so the gate enforces.
+    expect(
+      shouldEnforceCdiNvidiaGpuSpec({
+        cdiNvidiaGpuSpecMissing: true,
+        cdiNvidiaGpuSpecNeedsRepair: false,
+        explicitlyOptedOutGpuPassthrough: false,
+      }),
+    ).toBe(true);
+  });
+
+  it("enforces when the spec needs repair (stale) and not explicitly opted out", () => {
+    expect(
+      shouldEnforceCdiNvidiaGpuSpec({
+        cdiNvidiaGpuSpecMissing: false,
+        cdiNvidiaGpuSpecNeedsRepair: true,
+        explicitlyOptedOutGpuPassthrough: false,
+      }),
+    ).toBe(true);
+  });
+
+  it("does NOT enforce when the operator explicitly opted out of GPU passthrough (--no-gpu)", () => {
+    // Escape hatch: a host with an unusable GPU can still onboard CPU-only.
+    expect(
+      shouldEnforceCdiNvidiaGpuSpec({
+        cdiNvidiaGpuSpecMissing: true,
+        cdiNvidiaGpuSpecNeedsRepair: true,
+        explicitlyOptedOutGpuPassthrough: true,
+      }),
+    ).toBe(false);
+  });
+
+  it("does NOT enforce when the CDI spec is present and healthy", () => {
+    expect(
+      shouldEnforceCdiNvidiaGpuSpec({
+        cdiNvidiaGpuSpecMissing: false,
+        cdiNvidiaGpuSpecNeedsRepair: false,
+        explicitlyOptedOutGpuPassthrough: false,
+      }),
+    ).toBe(false);
+  });
+});

src/lib/onboard/preflight.ts

@@ -16,6 +16,7 @@ import os from "node:os";
 import path from "node:path";
 
 import { DASHBOARD_PORT } from "../core/ports";
+import { printRemediationActions } from "./remediation";
 import {
   assessNvidiaCdiHost,
   buildNvidiaCdiRefreshCommands,
@@ -385,11 +386,46 @@ function isHeadlessLikely(env: NodeJS.ProcessEnv): boolean {
   return !env.DISPLAY && !env.WAYLAND_DISPLAY && !env.TERM_PROGRAM;
 }
 
-function detectNvidiaGpu(runCaptureImpl: RunCaptureFn): boolean {
-  if (!commandExists("nvidia-smi", runCaptureImpl)) {
+// lspci line shape: "<slot> <class label>: <vendor> <device> ...".
+// The slot token contains colons (e.g. "01:00.0"), so anchor on the class
+// label that follows it and ends at the first ": ".
+const LSPCI_LINE = /^\S+\s+([^:]+):\s*(.*)$/;
+// NVIDIA GPUs surface as display-class devices: "VGA compatible controller"
+// (graphics cards), "3D controller" (datacenter/Tesla parts), or the generic
+// "Display controller". Restricting to these classes prevents NVIDIA/Mellanox
+// NICs and other non-GPU NVIDIA PCI devices from being mistaken for a GPU.
+const PCI_DISPLAY_CLASS = /\b(?:vga compatible controller|3d controller|display controller)\b/i;
+
+function lspciLineIsNvidiaGpu(line: string): boolean {
+  const match = LSPCI_LINE.exec(line.trim());
+  if (!match) return false;
+  const [, classLabel, deviceDescription] = match;
+  return PCI_DISPLAY_CLASS.test(classLabel) && /nvidia/i.test(deviceDescription);
+}
+
+function detectNvidiaGpuHardware(runCaptureImpl: RunCaptureFn): boolean {
+  // PCI bus probe so a physically present NVIDIA GPU is still detected when the
+  // driver is not loaded (nvidia-smi unavailable). Mirrors the lspci hint used
+  // by the onboarding GPU-passthrough note.
+  if (!commandExists("lspci", runCaptureImpl)) {
     return false;
   }
-  return Boolean(String(runCaptureImpl(["nvidia-smi", "-L"], { ignoreError: true }) || "").trim());
+  const output = String(runCaptureImpl(["lspci"], { ignoreError: true }) || "");
+  return output.split("\n").some(lspciLineIsNvidiaGpu);
+}
+
+function detectNvidiaGpu(runCaptureImpl: RunCaptureFn): boolean {
+  if (
+    commandExists("nvidia-smi", runCaptureImpl) &&
+    Boolean(String(runCaptureImpl(["nvidia-smi", "-L"], { ignoreError: true }) || "").trim())
+  ) {
+    return true;
+  }
+  // The driver may be missing or unloaded (nvidia-smi absent/empty) even when
+  // NVIDIA GPU hardware is present. Fall back to a hardware probe so CDI/toolkit
+  // remediation still fires when the toolkit is missing and Docker CDI dirs are
+  // configured (#5489); otherwise preflight silently skips toolkit enforcement.
+  return detectNvidiaGpuHardware(runCaptureImpl);
 }
 
 function detectPackageManager(runCaptureImpl: RunCaptureFn): PackageManager {
@@ -604,6 +640,47 @@ export function assessHost(opts: AssessHostOpts = {}): HostAssessment {
   return assessment;
 }
 
+/**
+ * Decide whether onboarding must enforce a present-and-configured NVIDIA CDI
+ * spec (i.e. block on a missing/stale spec). The fix for #5489 makes
+ * `assessHost().hasNvidiaGpu` true via an lspci hardware probe when the driver
+ * is unloaded, which is what flags `cdiNvidiaGpuSpecMissing`. The onboard gate
+ * must enforce based on whether the operator *explicitly* opted out of GPU
+ * passthrough — NOT on whether sandbox GPU was *auto*-disabled because
+ * `nvidia-smi` is unavailable. Auto-disable was the bypass that let onboard skip
+ * the toolkit/CDI remediation in #5489; an explicit `--no-gpu` still skips it so
+ * a host with an unusable GPU can still onboard CPU-only.
+ */
+export function shouldEnforceCdiNvidiaGpuSpec(opts: {
+  cdiNvidiaGpuSpecMissing: boolean;
+  cdiNvidiaGpuSpecNeedsRepair: boolean;
+  explicitlyOptedOutGpuPassthrough: boolean;
+}): boolean {
+  if (opts.explicitlyOptedOutGpuPassthrough) return false;
+  return opts.cdiNvidiaGpuSpecNeedsRepair || opts.cdiNvidiaGpuSpecMissing;
+}
+
+export function assertCdiNvidiaGpuSpecPresent(
+  host: HostAssessment,
+  explicitlyOptedOutGpuPassthrough: boolean,
+  hostGpuPlatform: string | null | undefined = null,
+): void {
+  if (hostGpuPlatform === "jetson" || isWslDockerDesktopRuntime(host)) return;
+  if (
+    !shouldEnforceCdiNvidiaGpuSpec({
+      cdiNvidiaGpuSpecMissing: host.cdiNvidiaGpuSpecMissing,
+      cdiNvidiaGpuSpecNeedsRepair: host.cdiNvidiaGpuSpecNeedsRepair ?? false,
+      explicitlyOptedOutGpuPassthrough,
+    })
+  )
+    return;
+  console.error(
+    "  Docker is configured for CDI device injection (CDISpecDirs is set), but the NVIDIA GPU CDI spec is missing or stale. OpenShell GPU startup can fail until the CDI spec is refreshed.",
+  );
+  printRemediationActions(planHostRemediation(host));
+  process.exit(1);
+}
+
 export function planHostRemediation(assessment: HostAssessment): RemediationAction[] {
   const actions: RemediationAction[] = [];