diff --git a/docs/reference/troubleshooting.md b/docs/reference/troubleshooting.md index 65a150609d0..384b89d1aea 100644 --- a/docs/reference/troubleshooting.md +++ b/docs/reference/troubleshooting.md @@ -925,6 +925,22 @@ GPU passthrough is not CI-tested on DGX Spark. It is expected to work when you pass `--gpu` and the NVIDIA Container Toolkit is configured. Verify the toolkit is configured by running `docker run --rm --runtime=nvidia --gpus all nvidia/cuda:12.8.0-base-ubuntu24.04 nvidia-smi` from the host. +### `unresolvable CDI devices nvidia.com/gpu=all` during gateway start + +Recent NVIDIA Container Toolkit installs configure the Docker daemon for Container Device Interface (CDI) device injection, which OpenShell's `gateway start --gpu` then auto-selects. +If no `nvidia.com/gpu` CDI spec has been generated on the host yet, gateway start fails with `Docker responded with status code 500: CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all`. +`nemoclaw onboard` now detects this gap during preflight and prints the remediation up front, but the underlying fix is the same on any Docker host whose `docker info` advertises a non-empty `CDISpecDirs`. + +Generate the spec, verify it lists `nvidia.com/gpu` entries, then rerun onboarding: + +```console +$ sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml +$ nvidia-ctk cdi list +$ nemoclaw onboard +``` + +If GPU passthrough is not required on this host, rerun onboarding with `--no-gpu` instead. + ### `pip install` fails with a system-packages error Recent Ubuntu releases (including DGX Spark's Ubuntu 24.04) mark the system Python install as externally managed, so `pip install` without a virtual environment fails. diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index a41a3ca9370..75cfcd2f6d6 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -3245,7 +3245,36 @@ function waitForSandboxReady(sandboxName: string, attempts = 10, delaySeconds = // ── Step 1: Preflight ──────────────────────────────────────────── -async function preflight(): Promise> { +// CDI spec gap (#3152). When Docker is configured for CDI device injection +// (CDISpecDirs is set) but no nvidia.com/gpu spec is present, OpenShell's +// `gateway start --gpu` fails minutes later with `unresolvable CDI devices +// nvidia.com/gpu=all`. Block now and surface `nvidia-ctk cdi generate`. The +// check is a no-op when the user opts out of GPU passthrough (--no-gpu), +// since the legacy nvidia runtime does not need a CDI spec. +// +// Extracted so the same guard runs on the `--resume` branch, where preflight() +// itself is skipped via the cached session. +function assertCdiNvidiaGpuSpecPresent( + host: ReturnType, + optedOutGpuPassthrough: boolean, +): void { + if (!host.cdiNvidiaGpuSpecMissing || optedOutGpuPassthrough) return; + console.error( + " Docker is configured for CDI device injection (CDISpecDirs is set), but no", + ); + console.error( + " nvidia.com/gpu CDI spec was found on the host. OpenShell's gateway start will", + ); + console.error( + " fail with `unresolvable CDI devices nvidia.com/gpu=all` (issue #3152).", + ); + printRemediationActions(planHostRemediation(host)); + process.exit(1); +} + +async function preflight( + preflightOpts: { optedOutGpuPassthrough?: boolean } = {}, +): Promise> { step(1, 8, "Preflight checks"); const host = assessHost(); @@ -3258,6 +3287,8 @@ async function preflight(): Promise> { } console.log(" ✓ Docker is running"); + assertCdiNvidiaGpuSpecPresent(host, preflightOpts.optedOutGpuPassthrough === true); + // DNS resolution from inside containers (#2101). A corp firewall that // blocks outbound UDP:53 to public resolvers leaves the sandbox build // unable to resolve registry.npmjs.org; npm then retries for ~15 min and @@ -9389,9 +9420,20 @@ async function onboard(opts: OnboardOptions = {}): Promise { if (resumePreflight) { skippedStepMessage("preflight", "cached"); gpu = nim.detectGpu(); + // Re-check the CDI spec gap on resume (#3152). The cached preflight + // result does not capture host CDI state, and the original onboard + // attempt that wrote the cache likely aborted at gateway-start with + // exactly this CDI failure — so resuming without re-checking would + // walk into the same wall. Honour persisted `gpuPassthrough: false` + // from the prior session as an opt-out, since the resume invocation + // does not need to re-pass `--no-gpu` to keep that intent (the same + // resolution is replayed a few lines below for `gpuPassthrough`). + const resumeOptedOutGpuPassthrough = + opts.noGpu === true || (opts.gpu !== true && session?.gpuPassthrough === false); + assertCdiNvidiaGpuSpecPresent(assessHost(), resumeOptedOutGpuPassthrough); } else { startRecordedStep("preflight"); - gpu = await preflight(); + gpu = await preflight({ optedOutGpuPassthrough: opts.noGpu === true }); onboardSession.markStepComplete("preflight"); } diff --git a/src/lib/preflight.test.ts b/src/lib/preflight.test.ts index 5125a5aa93c..c632859aa8c 100644 --- a/src/lib/preflight.test.ts +++ b/src/lib/preflight.test.ts @@ -14,6 +14,7 @@ import { isDockerUnderProvisioned, MIN_RECOMMENDED_DOCKER_CPUS, MIN_RECOMMENDED_DOCKER_MEM_GIB, + parseDockerCdiSpecDirs, parseDockerInfoCpus, parseDockerInfoMemTotalBytes, parseDockerStorageDriver, @@ -499,6 +500,200 @@ describe("parseDockerUsesContainerdSnapshotter", () => { }); }); +describe("parseDockerCdiSpecDirs", () => { + it("extracts the dirs from `docker info --format '{{json .}}'` output", () => { + const fixture = JSON.stringify({ CDISpecDirs: ["/etc/cdi", "/var/run/cdi"] }); + expect(parseDockerCdiSpecDirs(fixture)).toEqual(["/etc/cdi", "/var/run/cdi"]); + }); + + it("returns an empty array when CDISpecDirs is absent", () => { + expect(parseDockerCdiSpecDirs(JSON.stringify({ ServerVersion: "27.0" }))).toEqual([]); + }); + + it("returns an empty array when CDISpecDirs is the empty list", () => { + expect(parseDockerCdiSpecDirs(JSON.stringify({ CDISpecDirs: [] }))).toEqual([]); + }); + + it("returns an empty array on empty input", () => { + expect(parseDockerCdiSpecDirs("")).toEqual([]); + }); +}); + +describe("assessHost — CDI device-spec gap (#3152)", () => { + it("flags missing nvidia.com/gpu specs on an NVIDIA Linux host with CDI dirs configured", () => { + const result = assessHost({ + platform: "linux", + env: {}, + release: "6.8.0-58-generic", + readFileImpl: () => "Linux version 6.8.0-58-generic", + readdirImpl: () => [], + dockerInfoOutput: JSON.stringify({ + ServerVersion: "27.0", + OperatingSystem: "Ubuntu 24.04", + CDISpecDirs: ["/etc/cdi", "/var/run/cdi"], + }), + commandExistsImpl: (name: string) => name === "docker", + gpuProbeImpl: () => true, + }); + + expect(result.dockerCdiSpecDirs).toEqual(["/etc/cdi", "/var/run/cdi"]); + expect(result.cdiNvidiaGpuSpecMissing).toBe(true); + }); + + it("does not flag the host when an nvidia.com/gpu YAML spec is present", () => { + const result = assessHost({ + platform: "linux", + env: {}, + release: "6.8.0-58-generic", + readFileImpl: (filePath: string) => + filePath.endsWith("nvidia.yaml") + ? "cdiVersion: 0.5.0\nkind: nvidia.com/gpu\ndevices: []\n" + : "Linux version 6.8.0-58-generic", + readdirImpl: (dir: string) => (dir === "/etc/cdi" ? ["nvidia.yaml"] : []), + dockerInfoOutput: JSON.stringify({ + ServerVersion: "27.0", + CDISpecDirs: ["/etc/cdi", "/var/run/cdi"], + }), + commandExistsImpl: (name: string) => name === "docker", + gpuProbeImpl: () => true, + }); + + expect(result.cdiNvidiaGpuSpecMissing).toBe(false); + }); + + it("accepts a JSON-serialised CDI spec as well", () => { + const result = assessHost({ + platform: "linux", + env: {}, + release: "6.8.0-58-generic", + readFileImpl: (filePath: string) => + filePath.endsWith("nvidia.json") + ? '{"cdiVersion":"0.5.0","kind":"nvidia.com/gpu","devices":[]}' + : "Linux version 6.8.0-58-generic", + readdirImpl: (dir: string) => (dir === "/etc/cdi" ? ["nvidia.json"] : []), + dockerInfoOutput: JSON.stringify({ + ServerVersion: "27.0", + CDISpecDirs: ["/etc/cdi"], + }), + commandExistsImpl: (name: string) => name === "docker", + gpuProbeImpl: () => true, + }); + + expect(result.cdiNvidiaGpuSpecMissing).toBe(false); + }); + + it("does not flag a non-NVIDIA Linux host even with CDI dirs configured", () => { + const result = assessHost({ + platform: "linux", + env: {}, + release: "6.8.0-58-generic", + readFileImpl: () => "Linux version 6.8.0-58-generic", + readdirImpl: () => [], + dockerInfoOutput: JSON.stringify({ + ServerVersion: "27.0", + CDISpecDirs: ["/etc/cdi"], + }), + commandExistsImpl: (name: string) => name === "docker", + gpuProbeImpl: () => false, + }); + + expect(result.cdiNvidiaGpuSpecMissing).toBe(false); + }); + + it("does not flag a host that does not advertise CDISpecDirs", () => { + const result = assessHost({ + platform: "linux", + env: {}, + release: "6.8.0-58-generic", + readFileImpl: () => "Linux version 6.8.0-58-generic", + readdirImpl: () => [], + dockerInfoOutput: JSON.stringify({ ServerVersion: "24.0" }), + commandExistsImpl: (name: string) => name === "docker", + gpuProbeImpl: () => true, + }); + + expect(result.dockerCdiSpecDirs).toEqual([]); + expect(result.cdiNvidiaGpuSpecMissing).toBe(false); + }); + + it("does not flag macOS even when the docker info shape would otherwise match", () => { + const result = assessHost({ + platform: "darwin", + env: {}, + readFileImpl: () => "", + readdirImpl: () => [], + dockerInfoOutput: JSON.stringify({ CDISpecDirs: ["/etc/cdi"] }), + commandExistsImpl: (name: string) => name === "docker", + gpuProbeImpl: () => true, + }); + + expect(result.cdiNvidiaGpuSpecMissing).toBe(false); + }); + + it("does not accept a sibling device class such as nvidia.com/gpu-extra as a satisfying spec", () => { + const result = assessHost({ + platform: "linux", + env: {}, + release: "6.8.0-58-generic", + readFileImpl: (filePath: string) => + filePath.endsWith("nvidia-extra.yaml") + ? "cdiVersion: 0.5.0\nkind: nvidia.com/gpu-extra\ndevices: []\n" + : "Linux version 6.8.0-58-generic", + readdirImpl: (dir: string) => (dir === "/etc/cdi" ? ["nvidia-extra.yaml"] : []), + dockerInfoOutput: JSON.stringify({ + ServerVersion: "27.0", + CDISpecDirs: ["/etc/cdi"], + }), + commandExistsImpl: (name: string) => name === "docker", + gpuProbeImpl: () => true, + }); + + expect(result.cdiNvidiaGpuSpecMissing).toBe(true); + }); + + it("does not accept a sibling device class in JSON form either", () => { + const result = assessHost({ + platform: "linux", + env: {}, + release: "6.8.0-58-generic", + readFileImpl: (filePath: string) => + filePath.endsWith("nvidia-extra.json") + ? '{"cdiVersion":"0.5.0","kind":"nvidia.com/gpu-extra","devices":[]}' + : "Linux version 6.8.0-58-generic", + readdirImpl: (dir: string) => (dir === "/etc/cdi" ? ["nvidia-extra.json"] : []), + dockerInfoOutput: JSON.stringify({ + ServerVersion: "27.0", + CDISpecDirs: ["/etc/cdi"], + }), + commandExistsImpl: (name: string) => name === "docker", + gpuProbeImpl: () => true, + }); + + expect(result.cdiNvidiaGpuSpecMissing).toBe(true); + }); + + it("ignores spec files whose `kind` only mentions nvidia.com/gpu in a comment", () => { + const result = assessHost({ + platform: "linux", + env: {}, + release: "6.8.0-58-generic", + readFileImpl: (filePath: string) => + filePath.endsWith("notes.yaml") + ? "# this used to declare nvidia.com/gpu; now stripped\nkind: example.com/cpu\n" + : "Linux version 6.8.0-58-generic", + readdirImpl: (dir: string) => (dir === "/etc/cdi" ? ["notes.yaml"] : []), + dockerInfoOutput: JSON.stringify({ + ServerVersion: "27.0", + CDISpecDirs: ["/etc/cdi"], + }), + commandExistsImpl: (name: string) => name === "docker", + gpuProbeImpl: () => true, + }); + + expect(result.cdiNvidiaGpuSpecMissing).toBe(true); + }); +}); + describe("planHostRemediation", () => { it("recommends starting docker when installed but unreachable and service inactive", () => { const actions = planHostRemediation({ @@ -522,6 +717,8 @@ describe("planHostRemediation", () => { isUnsupportedRuntime: false, isHeadlessLikely: false, hasNvidiaGpu: false, + dockerCdiSpecDirs: [], + cdiNvidiaGpuSpecMissing: false, notes: [], }); @@ -552,6 +749,8 @@ describe("planHostRemediation", () => { isUnsupportedRuntime: false, isHeadlessLikely: false, hasNvidiaGpu: false, + dockerCdiSpecDirs: [], + cdiNvidiaGpuSpecMissing: false, notes: [], }); @@ -586,6 +785,8 @@ describe("planHostRemediation", () => { isUnsupportedRuntime: true, isHeadlessLikely: false, hasNvidiaGpu: false, + dockerCdiSpecDirs: [], + cdiNvidiaGpuSpecMissing: false, notes: [], }); @@ -618,6 +819,8 @@ describe("planHostRemediation", () => { isUnsupportedRuntime: false, isHeadlessLikely: false, hasNvidiaGpu: false, + dockerCdiSpecDirs: [], + cdiNvidiaGpuSpecMissing: false, notes: [], }); @@ -647,11 +850,54 @@ describe("planHostRemediation", () => { isUnsupportedRuntime: false, isHeadlessLikely: false, hasNvidiaGpu: false, + dockerCdiSpecDirs: [], + cdiNvidiaGpuSpecMissing: false, notes: [], }); expect(actions.some((action: { id: string }) => action.id === "install_openshell")).toBe(true); }); + + it("emits a blocking generate_nvidia_cdi_spec action when CDI dirs are configured but no nvidia.com/gpu spec exists", () => { + const actions = planHostRemediation({ + platform: "linux", + isWsl: false, + runtime: "docker", + packageManager: "apt", + systemctlAvailable: true, + dockerServiceActive: true, + dockerServiceEnabled: true, + dockerInstalled: true, + dockerRunning: true, + dockerReachable: true, + nodeInstalled: true, + openshellInstalled: true, + dockerCgroupVersion: "v2", + dockerDefaultCgroupnsMode: "unknown", + isContainerRuntimeUnderProvisioned: false, + hasNestedOverlayConflict: false, + requiresHostCgroupnsFix: false, + isUnsupportedRuntime: false, + isHeadlessLikely: false, + hasNvidiaGpu: true, + dockerCdiSpecDirs: ["/etc/cdi", "/var/run/cdi"], + cdiNvidiaGpuSpecMissing: true, + notes: [], + }); + + const action = actions.find( + (entry: { id: string }) => entry.id === "generate_nvidia_cdi_spec", + ); + expect(action).toBeTruthy(); + expect(action?.kind).toBe("sudo"); + expect(action?.blocking).toBe(true); + expect(action?.commands[0]).toBe( + "sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml", + ); + expect(action?.commands[1]).toContain("nvidia-ctk cdi list"); + expect(action?.commands[2]).toContain("nemoclaw onboard"); + expect(action?.reason).toContain("nvidia.com/gpu"); + }); }); describe("ensureSwap", () => { diff --git a/src/lib/preflight.ts b/src/lib/preflight.ts index 8832d10ebe5..220075d5f7c 100644 --- a/src/lib/preflight.ts +++ b/src/lib/preflight.ts @@ -112,6 +112,8 @@ export interface HostAssessment { isUnsupportedRuntime: boolean; isHeadlessLikely: boolean; hasNvidiaGpu: boolean; + dockerCdiSpecDirs: string[]; + cdiNvidiaGpuSpecMissing: boolean; notes: string[]; } @@ -132,6 +134,7 @@ export interface AssessHostOpts { dockerInfoOutput?: string; dockerInfoError?: string; readFileImpl?: (filePath: string, encoding: BufferEncoding) => string; + readdirImpl?: (dir: string) => string[]; runCaptureImpl?: RunCaptureFn; commandExistsImpl?: (commandName: string) => boolean; gpuProbeImpl?: () => boolean; @@ -212,6 +215,61 @@ export function parseDockerUsesContainerdSnapshotter(info = ""): boolean { return /io\.containerd\.snapshotter\.v1/.test(info); } +// Parses the Docker daemon's configured CDI spec directories from `docker +// info --format '{{json .}}'` output. Docker 25+ surfaces these as +// `"CDISpecDirs": ["/etc/cdi", "/var/run/cdi"]` whenever the daemon is built +// with CDI support and `features.cdi=true` (the default on recent installs). +// An empty list means CDI device injection is not enabled, so OpenShell will +// fall back to the legacy `nvidia` runtime path and there is no spec gap to +// worry about. +export function parseDockerCdiSpecDirs(info = ""): string[] { + const match = info.match(/"CDISpecDirs"\s*:\s*\[([^\]]*)\]/); + if (!match) return []; + return Array.from(match[1].matchAll(/"([^"]+)"/g), (m) => m[1]).filter(Boolean); +} + +// True when at least one CDI spec under the configured directories declares +// `kind: nvidia.com/gpu` (the device class OpenShell injects with `--gpu`). +// Specs are typically YAML, but the JSON shape is also accepted because +// `nvidia-ctk cdi generate --format=json` is supported. Errors reading any +// individual file or directory are tolerated — a missing dir is the same +// shape as "no spec found there". +function hasNvidiaCdiSpec( + specDirs: readonly string[], + readdirImpl: (dir: string) => string[], + readFileImpl: (filePath: string, encoding: BufferEncoding) => string, +): boolean { + // YAML keys are unquoted; JSON quotes the kind value. Anchor both patterns + // to the *exact* device-class string `nvidia.com/gpu` and require a value + // terminator (end of line, whitespace + comment, or whitespace + EOL) so a + // sibling spec like `nvidia.com/gpu-extra` does not silently satisfy the + // check and suppress the preflight warning. A comment that merely mentions + // `nvidia.com/gpu` is also rejected because `kindRe` only matches when the + // *whole* scalar value is the device class. + const kindRe = + /^[ \t]*kind[ \t]*:[ \t]*(?:"nvidia\.com\/gpu"|'nvidia\.com\/gpu'|nvidia\.com\/gpu)[ \t]*(?:#.*)?$/im; + const jsonRe = /"kind"\s*:\s*"nvidia\.com\/gpu"/; + for (const dir of specDirs) { + let entries: string[]; + try { + entries = readdirImpl(dir); + } catch { + continue; + } + for (const entry of entries) { + if (!/\.(ya?ml|json)$/i.test(entry)) continue; + let raw: string; + try { + raw = readFileImpl(path.join(dir, entry), "utf-8"); + } catch { + continue; + } + if (kindRe.test(raw) || jsonRe.test(raw)) return true; + } + } + return false; +} + export function parseDockerInfoCpus(info = ""): number | undefined { const jsonMatch = info.match(/"NCPU"\s*:\s*(\d+)/); if (jsonMatch) { @@ -334,6 +392,7 @@ export function assessHost(opts: AssessHostOpts = {}): HostAssessment { ((command: readonly string[], options?: { ignoreError?: boolean }) => runCapture(command, { ignoreError: options?.ignoreError ?? false })); const readFileImpl = opts.readFileImpl ?? fs.readFileSync; + const readdirImpl = opts.readdirImpl ?? ((dir: string) => fs.readdirSync(dir)); const dockerInstalled = opts.commandExistsImpl?.("docker") ?? commandExists("docker", runCaptureImpl); const nodeInstalled = opts.commandExistsImpl?.("node") ?? commandExists("node", runCaptureImpl); @@ -383,6 +442,20 @@ export function assessHost(opts: AssessHostOpts = {}): HostAssessment { const dockerMemTotalBytes = dockerReachable ? parseDockerInfoMemTotalBytes(dockerInfoOutput) : undefined; + // CDI spec gap: Docker 25+ on hosts with `nvidia-container-toolkit` installed + // typically advertises `"CDISpecDirs": ["/etc/cdi", "/var/run/cdi"]` in its + // info output. OpenShell's `gateway start --gpu` then opportunistically + // selects CDI mode and tries to inject `nvidia.com/gpu=all`. If no spec has + // been generated yet (`/etc/cdi/nvidia.yaml` is missing), the gateway start + // fails with `unresolvable CDI devices nvidia.com/gpu=all`. Detect this up + // front so preflight can point the user at `nvidia-ctk cdi generate` before + // we waste minutes downloading the gateway image. See issue #3152. + const dockerCdiSpecDirs = dockerReachable ? parseDockerCdiSpecDirs(dockerInfoOutput) : []; + const cdiNvidiaGpuSpecMissing = + platform === "linux" && + hasNvidiaGpu && + dockerCdiSpecDirs.length > 0 && + !hasNvidiaCdiSpec(dockerCdiSpecDirs, readdirImpl, readFileImpl); const isContainerRuntimeUnderProvisioned = isDockerUnderProvisioned( dockerCpus, dockerMemTotalBytes, @@ -448,6 +521,8 @@ export function assessHost(opts: AssessHostOpts = {}): HostAssessment { isUnsupportedRuntime: runtime === "podman", isHeadlessLikely: isHeadlessLikely(env), hasNvidiaGpu, + dockerCdiSpecDirs, + cdiNvidiaGpuSpecMissing, notes: [], }; @@ -616,6 +691,25 @@ export function planHostRemediation(assessment: HostAssessment): RemediationActi }); } + if (assessment.cdiNvidiaGpuSpecMissing) { + const specDir = assessment.dockerCdiSpecDirs[0] ?? "/etc/cdi"; + actions.push({ + id: "generate_nvidia_cdi_spec", + title: "Generate NVIDIA CDI device specs", + kind: "sudo", + reason: + "Docker is configured for CDI device injection (CDISpecDirs is set) but no " + + "nvidia.com/gpu CDI spec is present on the host. OpenShell's `gateway start --gpu` " + + "will fail with `unresolvable CDI devices nvidia.com/gpu=all` until a spec is generated.", + commands: [ + `sudo nvidia-ctk cdi generate --output=${specDir.replace(/\/+$/, "")}/nvidia.yaml`, + "nvidia-ctk cdi list # verify nvidia.com/gpu entries appear", + "nemoclaw onboard # or rerun with --no-gpu to skip GPU passthrough", + ], + blocking: true, + }); + } + return actions; }