Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion docs/reference/troubleshooting.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -1172,7 +1172,10 @@ Recent NVIDIA Container Toolkit installs configure the Docker daemon for Contain
If no `nvidia.com/gpu` CDI spec has been generated on the host yet, gateway start fails with `Docker responded with status code 500: CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all`.
The standard NemoClaw installer detects this gap before onboarding, first tries to enable the NVIDIA CDI refresh systemd units, and falls back to generating the spec directly with `nvidia-ctk`.
If you run `nemoclaw onboard` directly, preflight prints the manual remediation instead.
The underlying fix is the same on any Docker host whose `docker info` advertises a non-empty `CDISpecDirs`.
The native Linux fix is the same on Docker hosts whose `docker info` advertises a non-empty `CDISpecDirs`.
On WSL with Docker Desktop, Docker may advertise CDI directories even though `--device nvidia.com/gpu=all` is not usable from the WSL distro.
For that runtime, NemoClaw skips Linux CDI repair and uses Docker's `--gpus` compatibility path for sandbox GPU access.
This compatibility path can be retired once Docker Desktop exposes usable `nvidia.com/gpu` CDI specs inside WSL, or once OpenShell no longer requires host-visible CDI specs for Docker Desktop WSL GPU passthrough.

Enable the refresh units, verify they list `nvidia.com/gpu` entries, then rerun onboarding:

Expand All @@ -1190,6 +1193,12 @@ $ sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
$ nvidia-ctk cdi list
```

On WSL with Docker Desktop, confirm Docker Desktop WSL integration is enabled for your distro and verify Docker GPU access from WSL:

```console
$ docker run --rm --gpus all nvcr.io/nvidia/k8s/cuda-sample:nbody nbody -gpu -benchmark
```

If GPU passthrough is not required on this host, rerun onboarding with `--no-gpu` instead.

### Docker GPU patch failed during sandbox create
Expand Down
8 changes: 6 additions & 2 deletions scripts/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1823,9 +1823,13 @@ repair_installer_nvidia_cdi_spec() {
node -e '
const preflightPath = process.argv[1];
try {
const { assessHost, getNvidiaCdiSpecPath } = require(preflightPath);
const { assessHost, getNvidiaCdiSpecPath, isWslDockerDesktopRuntime } = require(preflightPath);
const host = assessHost();
if (host && host.cdiNvidiaGpuSpecMissing) {
if (
host &&
host.cdiNvidiaGpuSpecMissing &&
!isWslDockerDesktopRuntime(host)
) {
process.stdout.write(getNvidiaCdiSpecPath(host));
}
} catch {
Expand Down
2 changes: 1 addition & 1 deletion src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1787,7 +1787,7 @@ function assertCdiNvidiaGpuSpecPresent(
optedOutGpuPassthrough: boolean,
hostGpuPlatform: string | null | undefined = null,
): void {
if (hostGpuPlatform === "jetson") return;
if (hostGpuPlatform === "jetson" || preflightUtils.isWslDockerDesktopRuntime(host)) return;
if (!host.cdiNvidiaGpuSpecMissing || optedOutGpuPassthrough) return;
console.error(
" Docker is configured for CDI device injection (CDISpecDirs is set), but no",
Expand Down
9 changes: 8 additions & 1 deletion src/lib/onboard/preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ import os from "node:os";
import path from "node:path";

import { DASHBOARD_PORT } from "../core/ports";
import {
isWslDockerDesktopRuntime,
wslDockerDesktopGpuCompatibilityAction,
} from "./wsl-docker-desktop-gpu";
export { isWslDockerDesktopRuntime } from "./wsl-docker-desktop-gpu";

// runner.ts still uses CommonJS-style exports — use require here.
const { runCapture } = require("../runner");
Expand Down Expand Up @@ -809,7 +814,9 @@ export function planHostRemediation(assessment: HostAssessment): RemediationActi
"nvidia-ctk cdi list # verify nvidia.com/gpu entries appear",
"nemoclaw onboard # or rerun with --no-gpu to skip GPU passthrough",
];
if (assessment.nvidiaContainerToolkitInstalled) {
if (isWslDockerDesktopRuntime(assessment)) {
actions.push(wslDockerDesktopGpuCompatibilityAction());
} else if (assessment.nvidiaContainerToolkitInstalled) {
actions.push({
id: "generate_nvidia_cdi_spec",
title: "Generate NVIDIA CDI device specs",
Expand Down
88 changes: 88 additions & 0 deletions src/lib/onboard/sandbox-gpu-preflight.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
dockerNvidiaRuntimeAvailable,
formatSandboxGpuPassthroughNote,
parseDockerRuntimeNames,
sandboxGpuRemediationLines,
validateSandboxGpuPreflight,
} from "./sandbox-gpu-preflight";

Expand Down Expand Up @@ -92,6 +93,68 @@ describe("sandbox GPU preflight", () => {
expect(dockerInfo).not.toHaveBeenCalled();
});

it("skips CDI spec validation on Docker Desktop WSL so Docker --gpus can be used", () => {
const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined);
const getDockerCdiSpecDirs = vi.fn(() => ["/etc/cdi"]);
const findReadableNvidiaCdiSpecFiles = vi.fn(() => []);

try {
expect(() =>
validateSandboxGpuPreflight(sandboxGpuConfig(), {
platform: "linux",
env: { WSL_DISTRO_NAME: "Ubuntu" },
dockerInfoFormat: vi.fn(() => '"Docker Desktop"'),
getDockerCdiSpecDirs,
findReadableNvidiaCdiSpecFiles,
}),
).not.toThrow();
expect(getDockerCdiSpecDirs).not.toHaveBeenCalled();
expect(findReadableNvidiaCdiSpecFiles).not.toHaveBeenCalled();
expect(logSpy.mock.calls.map((call) => call[0]).join("\n")).toContain(
"Docker --gpus compatibility path",
);
} finally {
logSpy.mockRestore();
}
});

it("prints neutral WSL remediation when Docker runtime cannot be determined", () => {
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined);
const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number | string | null) => {
throw new Error(`exit:${code}`);
}) as never);

try {
expect(() =>
validateSandboxGpuPreflight(sandboxGpuConfig(), {
platform: "linux",
env: { WSL_DISTRO_NAME: "Ubuntu" },
dockerInfoFormat: vi.fn(() => ""),
getDockerCdiSpecDirs: vi.fn(() => ["/etc/cdi"]),
findReadableNvidiaCdiSpecFiles: vi.fn(() => []),
}),
).toThrow("exit:1");
const message = errorSpy.mock.calls.map((call) => call[0]).join("\n");
expect(message).toContain("could not determine whether Docker is Docker Desktop");
expect(message).toContain("If using Docker Desktop");
expect(message).toContain("If using native Docker Engine inside WSL");
expect(message).not.toContain("sudo systemctl restart docker");
} finally {
errorSpy.mockRestore();
exitSpy.mockRestore();
}
});

it("keeps generic Linux CDI remediation outside Docker Desktop WSL", () => {
expect(sandboxGpuRemediationLines().join("\n")).toContain("sudo nvidia-ctk");
expect(sandboxGpuRemediationLines({ wslDockerDesktop: true }).join("\n")).toContain(
"Docker Desktop WSL",
);
expect(sandboxGpuRemediationLines({ wslDockerDesktopStatus: "unknown" }).join("\n")).toContain(
"could not determine",
);
});

it("treats optional direct sandbox GPU proof failures as non-fatal", () => {
const runOpenshell = vi.fn(() => ({ status: 1, stdout: "", stderr: "optional proof failed" }));
const verifier = createDirectSandboxGpuVerifier({
Expand Down Expand Up @@ -121,6 +184,31 @@ describe("sandbox GPU preflight", () => {
expect(() => verifier("demo")).toThrow("GPU proof failed: fatal proof");
});

it("uses Docker Desktop WSL guidance when direct sandbox GPU proof fails there", () => {
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined);
const verifier = createDirectSandboxGpuVerifier({
platform: "linux",
env: { WSL_DISTRO_NAME: "Ubuntu" },
dockerInfoFormat: vi.fn(() => '"Docker Desktop"'),
runOpenshell: vi.fn(() => ({ status: 1, stdout: "", stderr: "required proof failed" })),
buildDirectSandboxGpuProofCommands: vi.fn(() => [
{ args: ["sandbox", "exec", "demo", "--", "false"], label: "fatal proof" },
]),
compactText: (value) => value.trim(),
redact: (value) => String(value),
});

try {
expect(() => verifier("demo")).toThrow("GPU proof failed: fatal proof");
const message = errorSpy.mock.calls.map((call) => call[0]).join("\n");
expect(message).toContain("Docker Desktop WSL");
expect(message).toContain("--gpus");
expect(message).not.toContain("sudo nvidia-ctk");
} finally {
errorSpy.mockRestore();
}
});

it("exits with an explicit Jetson NVIDIA runtime message when runtime support is missing", () => {
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined);
const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number | string | null) => {
Expand Down
39 changes: 32 additions & 7 deletions src/lib/onboard/sandbox-gpu-preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,16 @@
import { dockerInfoFormat } from "../adapters/docker";
import { findReadableNvidiaCdiSpecFiles, getDockerCdiSpecDirs } from "./docker-cdi";
import type { SandboxGpuConfig, SandboxGpuFlag } from "./sandbox-gpu-mode";
import {
detectWslDockerDesktopStatus,
type WslDockerDesktopDetectionDeps,
type WslDockerDesktopStatus,
wslDockerDesktopGpuCompatibilityRemediationLines,
} from "./wsl-docker-desktop-gpu";

const SANDBOX_GPU_PREFLIGHT_TIMEOUT_MS = 30_000;

export type SandboxGpuPreflightDeps = {
platform?: NodeJS.Platform;
dockerInfoFormat?: (format: string, opts?: Record<string, unknown>) => string;
export type SandboxGpuPreflightDeps = WslDockerDesktopDetectionDeps & {
getDockerCdiSpecDirs?: () => string[];
findReadableNvidiaCdiSpecFiles?: (dirs: string[]) => string[];
};
Expand Down Expand Up @@ -41,7 +45,14 @@ export function resolveSandboxGpuFlagFromOptions(opts: SandboxGpuFlagOptions): S
return null;
}

export function sandboxGpuRemediationLines(): string[] {
export function sandboxGpuRemediationLines(
options: { wslDockerDesktop?: boolean; wslDockerDesktopStatus?: WslDockerDesktopStatus } = {},
): string[] {
const status =
options.wslDockerDesktopStatus ??
(options.wslDockerDesktop ? "docker-desktop" : "not-docker-desktop");
const wslRemediationLines = wslDockerDesktopGpuCompatibilityRemediationLines(status);
if (wslRemediationLines) return wslRemediationLines;
return [
"Install/configure NVIDIA Container Toolkit CDI, then restart Docker:",
" sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml",
Expand Down Expand Up @@ -126,7 +137,7 @@ function validateJetsonSandboxGpuPreflight(deps: SandboxGpuPreflightDeps): void
console.log(" ✓ Docker NVIDIA runtime detected for Jetson/Tegra sandbox GPU");
}

export interface DirectSandboxGpuVerifierDeps {
export interface DirectSandboxGpuVerifierDeps extends WslDockerDesktopDetectionDeps {
runOpenshell(
args: string[],
opts?: Record<string, unknown>,
Expand Down Expand Up @@ -160,7 +171,9 @@ export function createDirectSandboxGpuVerifier(deps: DirectSandboxGpuVerifierDep
const diagnostic = deps.compactText(deps.redact(`${result.stderr || ""} ${result.stdout || ""}`));
console.error(` ✗ GPU proof failed: ${proof.label}`);
if (diagnostic) console.error(` ${diagnostic.slice(0, 300)}`);
for (const line of sandboxGpuRemediationLines()) {
for (const line of sandboxGpuRemediationLines({
wslDockerDesktopStatus: detectWslDockerDesktopStatus(deps),
})) {
console.error(` ${line}`);
}
const statusText = String(result.status || 1);
Expand All @@ -184,14 +197,26 @@ export function validateSandboxGpuPreflight(
return;
}

const wslDockerDesktopStatus = detectWslDockerDesktopStatus(deps);
if (wslDockerDesktopStatus === "docker-desktop") {
console.log(
" Docker Desktop WSL detected; using Docker --gpus compatibility path instead of CDI spec validation.",
);
return;
}

const cdiSpecDirs = (deps.getDockerCdiSpecDirs ?? getDockerCdiSpecDirs)();
const cdiSpecFiles = (deps.findReadableNvidiaCdiSpecFiles ?? findReadableNvidiaCdiSpecFiles)(
cdiSpecDirs,
);
if (cdiSpecFiles.length === 0) {
console.error("");
console.error(" ✗ Docker CDI GPU support was not detected.");
for (const line of sandboxGpuRemediationLines()) console.error(` ${line}`);
for (const line of sandboxGpuRemediationLines({
wslDockerDesktopStatus,
})) {
console.error(` ${line}`);
}
process.exit(1);
}
console.log(` ✓ Docker CDI GPU support detected (${cdiSpecFiles.join(", ")})`);
Expand Down
66 changes: 66 additions & 0 deletions src/lib/onboard/wsl-docker-desktop-gpu.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it, vi } from "vitest";

vi.mock("../adapters/docker", () => ({
dockerInfoFormat: vi.fn(),
}));

import {
detectWslDockerDesktopStatus,
isWslDockerDesktopRuntime,
WSL_DOCKER_DESKTOP_GPU_COMPATIBILITY_REMOVAL_CONDITION,
wslDockerDesktopGpuCompatibilityAction,
wslDockerDesktopGpuCompatibilityRemediationLines,
} from "./wsl-docker-desktop-gpu";

describe("WSL Docker Desktop GPU compatibility helpers", () => {
it("only matches Docker Desktop-backed WSL host assessments", () => {
expect(isWslDockerDesktopRuntime({ isWsl: true, runtime: "docker-desktop" })).toBe(true);
expect(isWslDockerDesktopRuntime({ isWsl: true, runtime: "docker" })).toBe(false);
expect(isWslDockerDesktopRuntime({ isWsl: false, runtime: "docker-desktop" })).toBe(false);
});

it("detects Docker Desktop status only after WSL detection succeeds", () => {
const dockerInfoFormat = vi.fn(() => '"Docker Desktop"');
expect(
detectWslDockerDesktopStatus({
platform: "linux",
env: { WSL_DISTRO_NAME: "Ubuntu" },
dockerInfoFormat,
}),
).toBe("docker-desktop");
expect(dockerInfoFormat).toHaveBeenCalledWith(
"{{json .OperatingSystem}}",
expect.objectContaining({ ignoreError: true }),
);

expect(
detectWslDockerDesktopStatus({
platform: "linux",
env: {},
release: "6.8.0-generic",
procVersion: "Linux version 6.8.0-generic",
dockerInfoFormat: vi.fn(() => '"Docker Desktop"'),
}),
).toBe("not-docker-desktop");
});

it("centralizes non-blocking Docker --gpus remediation and its removal condition", () => {
const action = wslDockerDesktopGpuCompatibilityAction();
expect(action.kind).toBe("info");
expect(action.blocking).toBe(false);
expect(action.reason).toContain("--gpus");
expect(action.commands.join("\n")).not.toContain("nvidia-ctk");

expect(wslDockerDesktopGpuCompatibilityRemediationLines("docker-desktop")?.join("\n")).toContain(
"Docker --gpus compatibility",
);
expect(wslDockerDesktopGpuCompatibilityRemediationLines("unknown")?.join("\n")).toContain(
"could not determine whether Docker is Docker Desktop",
);
expect(wslDockerDesktopGpuCompatibilityRemediationLines("not-docker-desktop")).toBeNull();
expect(WSL_DOCKER_DESKTOP_GPU_COMPATIBILITY_REMOVAL_CONDITION).toContain("Remove");
});
});
Loading
Loading