Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion docs/reference/troubleshooting.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -1172,7 +1172,9 @@ Recent NVIDIA Container Toolkit installs configure the Docker daemon for Contain
If no `nvidia.com/gpu` CDI spec has been generated on the host yet, gateway start fails with `Docker responded with status code 500: CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all`.
The standard NemoClaw installer detects this gap before onboarding, first tries to enable the NVIDIA CDI refresh systemd units, and falls back to generating the spec directly with `nvidia-ctk`.
If you run `nemoclaw onboard` directly, preflight prints the manual remediation instead.
The underlying fix is the same on any Docker host whose `docker info` advertises a non-empty `CDISpecDirs`.
The native Linux fix is the same on Docker hosts whose `docker info` advertises a non-empty `CDISpecDirs`.
On WSL with Docker Desktop, Docker may advertise CDI directories even though `--device nvidia.com/gpu=all` is not usable from the WSL distro.
For that runtime, NemoClaw skips Linux CDI repair and uses Docker's `--gpus` compatibility path for sandbox GPU access.

Enable the refresh units, verify they list `nvidia.com/gpu` entries, then rerun onboarding:

Expand All @@ -1190,6 +1192,12 @@ $ sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
$ nvidia-ctk cdi list
```

On WSL with Docker Desktop, confirm Docker Desktop WSL integration is enabled for your distro and verify Docker GPU access from WSL:

```console
$ docker run --rm --gpus all nvcr.io/nvidia/k8s/cuda-sample:nbody nbody -gpu -benchmark
```

If GPU passthrough is not required on this host, rerun onboarding with `--no-gpu` instead.

### Docker GPU patch failed during sandbox create
Expand Down
8 changes: 6 additions & 2 deletions scripts/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1823,9 +1823,13 @@ repair_installer_nvidia_cdi_spec() {
node -e '
const preflightPath = process.argv[1];
try {
const { assessHost, getNvidiaCdiSpecPath } = require(preflightPath);
const { assessHost, getNvidiaCdiSpecPath, isWslDockerDesktopRuntime } = require(preflightPath);
const host = assessHost();
if (host && host.cdiNvidiaGpuSpecMissing) {
if (
host &&
host.cdiNvidiaGpuSpecMissing &&
!isWslDockerDesktopRuntime(host)
) {
process.stdout.write(getNvidiaCdiSpecPath(host));
}
} catch {
Expand Down
1 change: 1 addition & 0 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1788,6 +1788,7 @@ function assertCdiNvidiaGpuSpecPresent(
hostGpuPlatform: string | null | undefined = null,
): void {
if (hostGpuPlatform === "jetson") return;
if (preflightUtils.isWslDockerDesktopRuntime(host)) return;
if (!host.cdiNvidiaGpuSpecMissing || optedOutGpuPassthrough) return;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
console.error(
" Docker is configured for CDI device injection (CDISpecDirs is set), but no",
Expand Down
80 changes: 80 additions & 0 deletions src/lib/onboard/preflight.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
getNvidiaCdiSpecPath,
ensureSwap,
isDockerUnderProvisioned,
isWslDockerDesktopRuntime,
MIN_RECOMMENDED_DOCKER_CPUS,
MIN_RECOMMENDED_DOCKER_MEM_GIB,
parseDockerCdiSpecDirs,
Expand Down Expand Up @@ -839,6 +840,14 @@ describe("getNvidiaCdiSpecPath", () => {
});
});

describe("isWslDockerDesktopRuntime", () => {
it("only matches Docker Desktop-backed WSL hosts", () => {
expect(isWslDockerDesktopRuntime({ isWsl: true, runtime: "docker-desktop" })).toBe(true);
expect(isWslDockerDesktopRuntime({ isWsl: true, runtime: "docker" })).toBe(false);
expect(isWslDockerDesktopRuntime({ isWsl: false, runtime: "docker-desktop" })).toBe(false);
});
});

describe("planHostRemediation", () => {
it("recommends starting docker when installed but unreachable and service inactive", () => {
const actions = planHostRemediation({
Expand Down Expand Up @@ -1105,6 +1114,77 @@ describe("planHostRemediation", () => {
expect(ctkGenerateIndex).toBeGreaterThan(ctkInstallIndex);
});

it("uses non-blocking Docker Desktop GPU compatibility guidance on WSL Docker Desktop", () => {
const actions = planHostRemediation({
platform: "linux",
isWsl: true,
runtime: "docker-desktop",
packageManager: "apt",
systemctlAvailable: true,
dockerServiceActive: null,
dockerServiceEnabled: null,
dockerInstalled: true,
dockerRunning: true,
dockerReachable: true,
nodeInstalled: true,
openshellInstalled: true,
dockerCgroupVersion: "v2",
dockerDefaultCgroupnsMode: "unknown",
isContainerRuntimeUnderProvisioned: false,
hasNestedOverlayConflict: false,
requiresHostCgroupnsFix: false,
isUnsupportedRuntime: false,
isHeadlessLikely: false,
hasNvidiaGpu: true,
dockerCdiSpecDirs: ["/etc/cdi", "/var/run/cdi"],
cdiNvidiaGpuSpecMissing: true,
nvidiaContainerToolkitInstalled: false,
notes: ["Running under WSL"],
});

expect(actions.find((entry) => entry.id === "install_nvidia_container_toolkit")).toBeUndefined();
const action = actions.find((entry) => entry.id === "wsl_docker_desktop_gpu_compatibility");
expect(action).toBeTruthy();
expect(action?.kind).toBe("info");
expect(action?.blocking).toBe(false);
expect(action?.title).toContain("Docker Desktop");
expect(action?.reason).toContain("--gpus");
expect(action?.commands.join("\n")).not.toContain("nvidia-ctk");
expect(action?.commands.join("\n")).not.toContain("nvidia-container-toolkit");
});

it("keeps NVIDIA Container Toolkit remediation for native Docker Engine inside WSL", () => {
const actions = planHostRemediation({
platform: "linux",
isWsl: true,
runtime: "docker",
packageManager: "apt",
systemctlAvailable: true,
dockerServiceActive: true,
dockerServiceEnabled: true,
dockerInstalled: true,
dockerRunning: true,
dockerReachable: true,
nodeInstalled: true,
openshellInstalled: true,
dockerCgroupVersion: "v2",
dockerDefaultCgroupnsMode: "unknown",
isContainerRuntimeUnderProvisioned: false,
hasNestedOverlayConflict: false,
requiresHostCgroupnsFix: false,
isUnsupportedRuntime: false,
isHeadlessLikely: false,
hasNvidiaGpu: true,
dockerCdiSpecDirs: ["/etc/cdi", "/var/run/cdi"],
cdiNvidiaGpuSpecMissing: true,
nvidiaContainerToolkitInstalled: false,
notes: ["Running under WSL"],
});

expect(actions.find((entry) => entry.id === "wsl_docker_desktop_gpu_compatibility")).toBeUndefined();
expect(actions.find((entry) => entry.id === "install_nvidia_container_toolkit")).toBeTruthy();
});

it("emits an install_nvidia_container_toolkit action with a docs pointer when nvidia-ctk is missing on unknown package managers", () => {
const actions = planHostRemediation({
platform: "linux",
Expand Down
24 changes: 23 additions & 1 deletion src/lib/onboard/preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,12 @@ export function getNvidiaCdiSpecPath(
return path.join(normalizeCdiSpecDir(assessment.dockerCdiSpecDirs[0]), "nvidia.yaml");
}

export function isWslDockerDesktopRuntime(
assessment: Pick<HostAssessment, "isWsl" | "runtime">,
): boolean {
return assessment.isWsl && assessment.runtime === "docker-desktop";
}

// True when at least one CDI spec under the configured directories declares
// `kind: nvidia.com/gpu` (the device class OpenShell injects with `--gpu`).
// Specs are typically YAML, but the JSON shape is also accepted because
Expand Down Expand Up @@ -809,7 +815,23 @@ export function planHostRemediation(assessment: HostAssessment): RemediationActi
"nvidia-ctk cdi list # verify nvidia.com/gpu entries appear",
"nemoclaw onboard # or rerun with --no-gpu to skip GPU passthrough",
];
if (assessment.nvidiaContainerToolkitInstalled) {
if (isWslDockerDesktopRuntime(assessment)) {
actions.push({
id: "wsl_docker_desktop_gpu_compatibility",
title: "Use Docker Desktop WSL GPU compatibility path",
kind: "info",
reason:
"Docker Desktop is configured for CDI device injection (CDISpecDirs is set) but no " +
"nvidia.com/gpu CDI spec is visible from WSL. On Docker Desktop-backed WSL, NemoClaw " +
"uses Docker's `--gpus` compatibility path instead of trying to repair Linux host CDI " +
"from inside the WSL distro.",
commands: [
"If sandbox GPU setup later fails, verify Docker Desktop GPU support from WSL with `docker run --rm --gpus all nvcr.io/nvidia/k8s/cuda-sample:nbody nbody -gpu -benchmark`.",
"Rerun with `--no-gpu` to skip GPU passthrough.",
],
blocking: false,
});
} else if (assessment.nvidiaContainerToolkitInstalled) {
actions.push({
id: "generate_nvidia_cdi_spec",
title: "Generate NVIDIA CDI device specs",
Expand Down
63 changes: 63 additions & 0 deletions src/lib/onboard/sandbox-gpu-preflight.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
dockerNvidiaRuntimeAvailable,
formatSandboxGpuPassthroughNote,
parseDockerRuntimeNames,
sandboxGpuRemediationLines,
validateSandboxGpuPreflight,
} from "./sandbox-gpu-preflight";

Expand Down Expand Up @@ -92,6 +93,68 @@ describe("sandbox GPU preflight", () => {
expect(dockerInfo).not.toHaveBeenCalled();
});

it("skips CDI spec validation on Docker Desktop WSL so Docker --gpus can be used", () => {
const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined);
const getDockerCdiSpecDirs = vi.fn(() => ["/etc/cdi"]);
const findReadableNvidiaCdiSpecFiles = vi.fn(() => []);

try {
expect(() =>
validateSandboxGpuPreflight(sandboxGpuConfig(), {
platform: "linux",
env: { WSL_DISTRO_NAME: "Ubuntu" },
dockerInfoFormat: vi.fn(() => '"Docker Desktop"'),
getDockerCdiSpecDirs,
findReadableNvidiaCdiSpecFiles,
}),
).not.toThrow();
expect(getDockerCdiSpecDirs).not.toHaveBeenCalled();
expect(findReadableNvidiaCdiSpecFiles).not.toHaveBeenCalled();
expect(logSpy.mock.calls.map((call) => call[0]).join("\n")).toContain(
"Docker --gpus compatibility path",
);
} finally {
logSpy.mockRestore();
}
});

it("prints neutral WSL remediation when Docker runtime cannot be determined", () => {
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined);
const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number | string | null) => {
throw new Error(`exit:${code}`);
}) as never);

try {
expect(() =>
validateSandboxGpuPreflight(sandboxGpuConfig(), {
platform: "linux",
env: { WSL_DISTRO_NAME: "Ubuntu" },
dockerInfoFormat: vi.fn(() => ""),
getDockerCdiSpecDirs: vi.fn(() => ["/etc/cdi"]),
findReadableNvidiaCdiSpecFiles: vi.fn(() => []),
}),
).toThrow("exit:1");
const message = errorSpy.mock.calls.map((call) => call[0]).join("\n");
expect(message).toContain("could not determine whether Docker is Docker Desktop");
expect(message).toContain("If using Docker Desktop");
expect(message).toContain("If using native Docker Engine inside WSL");
expect(message).not.toContain("sudo systemctl restart docker");
} finally {
errorSpy.mockRestore();
exitSpy.mockRestore();
}
});

it("keeps generic Linux CDI remediation outside Docker Desktop WSL", () => {
expect(sandboxGpuRemediationLines().join("\n")).toContain("sudo nvidia-ctk");
expect(sandboxGpuRemediationLines({ wslDockerDesktop: true }).join("\n")).toContain(
"Docker Desktop WSL",
);
expect(sandboxGpuRemediationLines({ wslDockerDesktopStatus: "unknown" }).join("\n")).toContain(
"could not determine",
);
});

it("treats optional direct sandbox GPU proof failures as non-fatal", () => {
const runOpenshell = vi.fn(() => ({ status: 1, stdout: "", stderr: "optional proof failed" }));
const verifier = createDirectSandboxGpuVerifier({
Expand Down
95 changes: 93 additions & 2 deletions src/lib/onboard/sandbox-gpu-preflight.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,23 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import fs from "node:fs";
import os from "node:os";

import { dockerInfoFormat } from "../adapters/docker";
import { findReadableNvidiaCdiSpecFiles, getDockerCdiSpecDirs } from "./docker-cdi";
import type { SandboxGpuConfig, SandboxGpuFlag } from "./sandbox-gpu-mode";

const SANDBOX_GPU_PREFLIGHT_TIMEOUT_MS = 30_000;

type WslDockerDesktopStatus = "docker-desktop" | "not-docker-desktop" | "unknown";

export type SandboxGpuPreflightDeps = {
platform?: NodeJS.Platform;
env?: NodeJS.ProcessEnv;
release?: string;
procVersion?: string;
readFileImpl?: (filePath: string, encoding: BufferEncoding) => string;
dockerInfoFormat?: (format: string, opts?: Record<string, unknown>) => string;
getDockerCdiSpecDirs?: () => string[];
findReadableNvidiaCdiSpecFiles?: (dirs: string[]) => string[];
Expand Down Expand Up @@ -41,7 +50,77 @@ export function resolveSandboxGpuFlagFromOptions(opts: SandboxGpuFlagOptions): S
return null;
}

export function sandboxGpuRemediationLines(): string[] {
function detectWsl(
deps: Pick<
SandboxGpuPreflightDeps,
"env" | "platform" | "procVersion" | "readFileImpl" | "release"
>,
): boolean {
const platform = deps.platform ?? process.platform;
if (platform !== "linux") return false;
const env = deps.env ?? process.env;
if (env.WSL_DISTRO_NAME || env.WSL_INTEROP) return true;
const release = deps.release ?? os.release();
if (/microsoft/i.test(release)) return true;
const procVersion =
deps.procVersion ??
(() => {
try {
const readFileImpl =
deps.readFileImpl ??
((filePath: string, encoding: BufferEncoding) => fs.readFileSync(filePath, encoding));
return readFileImpl("/proc/version", "utf-8");
} catch {
return "";
}
})();
return /microsoft/i.test(procVersion);
}

function detectDockerDesktopRuntime(deps: SandboxGpuPreflightDeps): WslDockerDesktopStatus {
const dockerInfo = deps.dockerInfoFormat ?? dockerInfoFormat;
try {
const output = String(
dockerInfo("{{json .OperatingSystem}}", {
ignoreError: true,
timeout: SANDBOX_GPU_PREFLIGHT_TIMEOUT_MS,
}),
).trim();
if (!output || output === "<no value>") return "unknown";
return /^"?docker desktop\b/i.test(output) ? "docker-desktop" : "not-docker-desktop";
} catch {
return "unknown";
}
}

function detectWslDockerDesktopStatus(deps: SandboxGpuPreflightDeps): WslDockerDesktopStatus {
if (!detectWsl(deps)) return "not-docker-desktop";
return detectDockerDesktopRuntime(deps);
}

export function sandboxGpuRemediationLines(
options: { wslDockerDesktop?: boolean; wslDockerDesktopStatus?: WslDockerDesktopStatus } = {},
): string[] {
const status =
options.wslDockerDesktopStatus ??
(options.wslDockerDesktop ? "docker-desktop" : "not-docker-desktop");
if (status === "docker-desktop") {
return [
"Docker Desktop WSL detected; NemoClaw uses Docker --gpus compatibility instead of CDI spec validation.",
"If sandbox GPU setup later fails, verify from WSL:",
" docker run --rm --gpus all nvcr.io/nvidia/k8s/cuda-sample:nbody nbody -gpu -benchmark",
"Or force CPU sandbox behavior with NEMOCLAW_SANDBOX_GPU=0.",
];
}
if (status === "unknown") {
return [
"WSL detected, but NemoClaw could not determine whether Docker is Docker Desktop or native Docker Engine.",
"If using Docker Desktop, confirm Settings > Resources > WSL integration is enabled for this distro, restart Docker Desktop, and verify:",
" docker run --rm --gpus all nvcr.io/nvidia/k8s/cuda-sample:nbody nbody -gpu -benchmark",
"If using native Docker Engine inside WSL, install/configure NVIDIA Container Toolkit CDI, then restart Docker.",
"Or force CPU sandbox behavior with NEMOCLAW_SANDBOX_GPU=0.",
];
}
return [
"Install/configure NVIDIA Container Toolkit CDI, then restart Docker:",
" sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml",
Expand Down Expand Up @@ -184,14 +263,26 @@ export function validateSandboxGpuPreflight(
return;
}

const wslDockerDesktopStatus = detectWslDockerDesktopStatus(deps);
if (wslDockerDesktopStatus === "docker-desktop") {
console.log(
" Docker Desktop WSL detected; using Docker --gpus compatibility path instead of CDI spec validation.",
);
return;
}

const cdiSpecDirs = (deps.getDockerCdiSpecDirs ?? getDockerCdiSpecDirs)();
const cdiSpecFiles = (deps.findReadableNvidiaCdiSpecFiles ?? findReadableNvidiaCdiSpecFiles)(
cdiSpecDirs,
);
if (cdiSpecFiles.length === 0) {
console.error("");
console.error(" ✗ Docker CDI GPU support was not detected.");
for (const line of sandboxGpuRemediationLines()) console.error(` ${line}`);
for (const line of sandboxGpuRemediationLines({
wslDockerDesktopStatus,
})) {
console.error(` ${line}`);
}
process.exit(1);
}
console.log(` ✓ Docker CDI GPU support detected (${cdiSpecFiles.join(", ")})`);
Expand Down
Loading
Loading