diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 22d7da48265..5aebc1802ce 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1358,6 +1358,12 @@ openshell sandbox delete Fix the NVIDIA Container Toolkit or CDI configuration reported in the diagnostics, clean up the failed sandbox, then rerun onboarding. If you do not need GPU access inside the sandbox, rerun with `--no-sandbox-gpu`. Set `NEMOCLAW_DOCKER_GPU_PATCH=0` only when you need to bypass this compatibility path during troubleshooting. +On Docker Desktop WSL the patch is required for GPU passthrough — `NEMOCLAW_DOCKER_GPU_PATCH=0` is ignored on that runtime, and onboarding logs a warning when it is set there. +To skip GPU passthrough entirely on Docker Desktop WSL, rerun with `--no-gpu` or set `NEMOCLAW_SANDBOX_GPU=0`. + +If sandbox creation fails with `CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all`, the OpenShell gateway tried `docker create --device nvidia.com/gpu=all` and Docker could not resolve the CDI spec. +This injection happens inside the gateway, so `NEMOCLAW_DOCKER_GPU_PATCH=0` does not bypass it. +Rerun with `--no-gpu`, or set `NEMOCLAW_SANDBOX_GPU=0` and resume onboarding. If onboarding reports `OpenShell supervisor did not reconnect to the GPU-enabled container.` even though the diagnostic bundle shows the patched container is running and healthy, the supervisor-reconnect wait is treating a transient Error phase (reported while the OpenShell host re-registers the new container) as fatal. The reconnect wait debounces consecutive Error-phase polls before fast-failing, defaulting to fifteen consecutive polls of about 30 seconds in total. diff --git a/src/lib/build-context.test.ts b/src/lib/build-context.test.ts index 271bc73607c..add0bb94bca 100644 --- a/src/lib/build-context.test.ts +++ b/src/lib/build-context.test.ts @@ -190,6 +190,19 @@ describe("printSandboxCreateRecoveryHints", () => { }); expect(stderr()).toContain(""); }); + + it("prints GPU CDI injection guidance pointing at --no-gpu / NEMOCLAW_SANDBOX_GPU=0", () => { + printSandboxCreateRecoveryHints( + "Error response from daemon: CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + ); + + const out = stderr(); + expect(out).toContain("GPU CDI device injection failed"); + expect(out).toContain("NEMOCLAW_DOCKER_GPU_PATCH=0 does not bypass"); + expect(out).toContain("--no-gpu"); + expect(out).toContain("NEMOCLAW_SANDBOX_GPU=0"); + expect(out).toContain("onboard --resume --no-gpu"); + }); }); describe("reconstructImageRefCreateCommand", () => { diff --git a/src/lib/build-context.ts b/src/lib/build-context.ts index 551d6852707..4e70a771522 100644 --- a/src/lib/build-context.ts +++ b/src/lib/build-context.ts @@ -206,6 +206,18 @@ export function printSandboxCreateRecoveryHints( console.error(` Then: ${CLI_NAME} onboard --resume`); return; } + if (failure.kind === "gpu_cdi_injection_failed") { + console.error(" Hint: GPU CDI device injection failed inside the OpenShell gateway."); + console.error( + " The gateway issues `docker create --device nvidia.com/gpu=all` on its own, so", + ); + console.error(" NEMOCLAW_DOCKER_GPU_PATCH=0 does not bypass this path."); + console.error(" Skip GPU passthrough entirely with either:"); + console.error(` ${CLI_NAME} onboard --no-gpu`); + console.error(" NEMOCLAW_SANDBOX_GPU=0 (env var, applies to subsequent runs)"); + console.error(` Recovery: ${CLI_NAME} onboard --resume --no-gpu`); + return; + } console.error(` Recovery: ${CLI_NAME} onboard --resume`); console.error(` Or: ${CLI_NAME} onboard`); } diff --git a/src/lib/onboard/command-support.ts b/src/lib/onboard/command-support.ts index f4ab86651dd..ea178179115 100644 --- a/src/lib/onboard/command-support.ts +++ b/src/lib/onboard/command-support.ts @@ -66,7 +66,8 @@ export function buildOnboardFlags(): Record { description: "Enable direct NVIDIA GPU access inside the sandbox", }), "no-sandbox-gpu": Flags.boolean({ - description: "Force CPU sandbox behavior", + description: + "Force CPU sandbox behavior (equivalent to NEMOCLAW_SANDBOX_GPU=0; alternative to --no-gpu when Docker Desktop WSL CDI injection fails)", }), "sandbox-gpu-device": Flags.string({ description: diff --git a/src/lib/onboard/docker-gpu-local-inference.ts b/src/lib/onboard/docker-gpu-local-inference.ts index abe4482d862..3d6baf7cdb3 100644 --- a/src/lib/onboard/docker-gpu-local-inference.ts +++ b/src/lib/onboard/docker-gpu-local-inference.ts @@ -10,6 +10,7 @@ import { printDockerGpuProofFailure, shouldApplyDockerGpuPatch, } from "./docker-gpu-patch"; +import { isDockerDesktopWslRuntime } from "./docker-gpu-sandbox-create"; import { executeSandboxCommandForVerification } from "./sandbox-verification-exec"; const { @@ -39,11 +40,16 @@ type DockerGpuLocalInferenceConfig = { type DockerGpuLocalInferenceOptions = { dockerDriverGateway: boolean; + dockerDesktopWsl?: boolean; env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform; log?: (message: string) => void; }; +function resolveDockerDesktopWsl(options: DockerGpuLocalInferenceOptions): boolean { + return options.dockerDesktopWsl ?? isDockerDesktopWslRuntime(); +} + function isLocalInferenceProvider(provider: string | null | undefined): provider is string { return Boolean(provider && LOCAL_INFERENCE_PROVIDERS.includes(provider)); } @@ -60,6 +66,7 @@ export function shouldUseDockerGpuPatchHostNetwork( return ( shouldApplyDockerGpuPatch(config, { dockerDriverGateway: options.dockerDriverGateway, + dockerDesktopWsl: resolveDockerDesktopWsl(options), env: options.env, platform: options.platform, }) && getDockerGpuPatchNetworkMode(options.env ?? process.env) === "host" @@ -263,6 +270,7 @@ export function verifyDockerGpuSandboxLocalInference( if ( !shouldApplyDockerGpuPatch(config, { dockerDriverGateway: options.dockerDriverGateway, + dockerDesktopWsl: resolveDockerDesktopWsl(options), env: options.env, platform: options.platform, }) @@ -401,8 +409,10 @@ export function verifyGpuSandboxAfterReady( throw error; } - // When NEMOCLAW_DOCKER_GPU_PATCH=0, useDockerGpuPatch is false and there is no - // GPU-patched sandbox to gate, so skip the local inference reachability gate. + // When the resolved create plan disabled the Docker GPU patch (e.g. + // NEMOCLAW_DOCKER_GPU_PATCH=0 honoured outside Docker Desktop WSL), there is + // no GPU-patched sandbox to gate, so skip the local inference reachability + // gate. if (!options.useDockerGpuPatch) return; const verification = verifyDockerGpuSandboxLocalInference(config, provider, { sandboxName: options.sandboxName, diff --git a/src/lib/onboard/docker-gpu-patch-wsl.test.ts b/src/lib/onboard/docker-gpu-patch-wsl.test.ts new file mode 100644 index 00000000000..07582f33b67 --- /dev/null +++ b/src/lib/onboard/docker-gpu-patch-wsl.test.ts @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { shouldApplyDockerGpuPatch } from "../../../dist/lib/onboard/docker-gpu-patch"; + +describe("shouldApplyDockerGpuPatch on Docker Desktop WSL", () => { + it("ignores NEMOCLAW_DOCKER_GPU_PATCH=0 on Docker Desktop WSL where the patch is required", () => { + const logs: string[] = []; + expect( + shouldApplyDockerGpuPatch( + { sandboxGpuEnabled: true }, + { + env: { NEMOCLAW_DOCKER_GPU_PATCH: "0" }, + platform: "linux", + dockerDriverGateway: true, + dockerDesktopWsl: true, + log: (message) => { + logs.push(message); + }, + }, + ), + ).toBe(true); + expect(logs.some((line) => /NEMOCLAW_DOCKER_GPU_PATCH=0 ignored/i.test(line))).toBe(true); + expect(logs.some((line) => /NEMOCLAW_SANDBOX_GPU=0/.test(line))).toBe(true); + }); + + it("still honors NEMOCLAW_DOCKER_GPU_PATCH=0 when not on Docker Desktop WSL", () => { + expect( + shouldApplyDockerGpuPatch( + { sandboxGpuEnabled: true }, + { + env: { NEMOCLAW_DOCKER_GPU_PATCH: "0" }, + platform: "linux", + dockerDriverGateway: true, + dockerDesktopWsl: false, + }, + ), + ).toBe(false); + }); +}); diff --git a/src/lib/onboard/docker-gpu-patch.ts b/src/lib/onboard/docker-gpu-patch.ts index c32551004bf..1a635f62e1a 100644 --- a/src/lib/onboard/docker-gpu-patch.ts +++ b/src/lib/onboard/docker-gpu-patch.ts @@ -522,17 +522,26 @@ export function shouldApplyDockerGpuPatch( env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform; dockerDriverGateway?: boolean; + dockerDesktopWsl?: boolean; + log?: (message: string) => void; } = {}, ): boolean { const env = options.env ?? process.env; const platform = options.platform ?? process.platform; const dockerDriverGateway = options.dockerDriverGateway ?? platform === "linux"; - return ( - config.sandboxGpuEnabled && - platform === "linux" && - dockerDriverGateway && - String(env.NEMOCLAW_DOCKER_GPU_PATCH || "").trim() !== "0" - ); + if (!(config.sandboxGpuEnabled && platform === "linux" && dockerDriverGateway)) { + return false; + } + const optedOut = String(env.NEMOCLAW_DOCKER_GPU_PATCH || "").trim() === "0"; + if (optedOut && options.dockerDesktopWsl) { + const log = options.log ?? ((message: string) => console.warn(message)); + log( + " NEMOCLAW_DOCKER_GPU_PATCH=0 ignored on Docker Desktop WSL: GPU passthrough on this runtime requires the patch.", + ); + log(" Skip GPU passthrough entirely with --no-gpu or NEMOCLAW_SANDBOX_GPU=0."); + return true; + } + return !optedOut; } export function buildDockerGpuCloneRunOptions( @@ -1250,7 +1259,13 @@ export function printDockerGpuPatchFailureAndExit( if (diagnostics) { console.error(` Diagnostics saved: ${diagnostics.dir}`); } - console.error(" Escape hatch: set NEMOCLAW_DOCKER_GPU_PATCH=0 to skip this patch."); + console.error(" Escape hatches:"); + console.error( + " NEMOCLAW_DOCKER_GPU_PATCH=0 skip this Docker GPU patch (Linux native Docker only; ignored on Docker Desktop WSL where the patch is required).", + ); + console.error( + " NEMOCLAW_SANDBOX_GPU=0 skip GPU passthrough entirely (or rerun with --no-gpu).", + ); printDockerGpuPatchCleanup(sandboxName); process.exit(1); } diff --git a/src/lib/onboard/docker-gpu-sandbox-create.test.ts b/src/lib/onboard/docker-gpu-sandbox-create.test.ts index 6d78dcc77b1..59236572e4d 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.test.ts @@ -7,7 +7,11 @@ import type { DockerGpuPatchFailureContext, DockerGpuPatchResult, } from "../../../dist/lib/onboard/docker-gpu-patch"; -import { createDockerGpuSandboxCreatePatch } from "../../../dist/lib/onboard/docker-gpu-sandbox-create"; +import { buildSandboxGpuCreateArgs } from "../../../dist/lib/onboard/sandbox-gpu-create"; +import { + createDockerGpuSandboxCreatePatch, + resolveDockerGpuSandboxCreatePlan, +} from "../../../dist/lib/onboard/docker-gpu-sandbox-create"; function deferredCreateResult(): DockerGpuPatchResult { return { @@ -220,3 +224,81 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(finalizeBackup).not.toHaveBeenCalled(); }); }); + +describe("resolveDockerGpuSandboxCreatePlan Docker Desktop WSL handling", () => { + it("keeps useDockerGpuPatch=true on Docker Desktop WSL even when NEMOCLAW_DOCKER_GPU_PATCH=0", () => { + const originalEnv = process.env.NEMOCLAW_DOCKER_GPU_PATCH; + process.env.NEMOCLAW_DOCKER_GPU_PATCH = "0"; + try { + const plan = resolveDockerGpuSandboxCreatePlan( + { sandboxGpuEnabled: true }, + { + dockerDriverGateway: true, + detectDockerDesktopWsl: () => true, + }, + ); + expect(plan.useDockerGpuPatch).toBe(true); + } finally { + if (originalEnv === undefined) delete process.env.NEMOCLAW_DOCKER_GPU_PATCH; + else process.env.NEMOCLAW_DOCKER_GPU_PATCH = originalEnv; + } + }); + + it("honors NEMOCLAW_DOCKER_GPU_PATCH=0 when not on Docker Desktop WSL", () => { + const originalEnv = process.env.NEMOCLAW_DOCKER_GPU_PATCH; + process.env.NEMOCLAW_DOCKER_GPU_PATCH = "0"; + try { + const plan = resolveDockerGpuSandboxCreatePlan( + { sandboxGpuEnabled: true }, + { + dockerDriverGateway: true, + detectDockerDesktopWsl: () => false, + }, + ); + expect(plan.useDockerGpuPatch).toBe(false); + } finally { + if (originalEnv === undefined) delete process.env.NEMOCLAW_DOCKER_GPU_PATCH; + else process.env.NEMOCLAW_DOCKER_GPU_PATCH = originalEnv; + } + }); + + it("suppresses the openshell sandbox create --gpu flag on Docker Desktop WSL when the opt-out is ignored", () => { + const originalEnv = process.env.NEMOCLAW_DOCKER_GPU_PATCH; + process.env.NEMOCLAW_DOCKER_GPU_PATCH = "0"; + try { + const sandboxGpuConfig = { sandboxGpuEnabled: true }; + const plan = resolveDockerGpuSandboxCreatePlan(sandboxGpuConfig, { + dockerDriverGateway: true, + detectDockerDesktopWsl: () => true, + }); + expect(plan.useDockerGpuPatch).toBe(true); + const createArgs = buildSandboxGpuCreateArgs(sandboxGpuConfig, { + suppressGpuFlag: plan.useDockerGpuPatch, + }); + expect(createArgs).toEqual([]); + } finally { + if (originalEnv === undefined) delete process.env.NEMOCLAW_DOCKER_GPU_PATCH; + else process.env.NEMOCLAW_DOCKER_GPU_PATCH = originalEnv; + } + }); + + it("emits --gpu when the patch is disabled outside Docker Desktop WSL", () => { + const originalEnv = process.env.NEMOCLAW_DOCKER_GPU_PATCH; + process.env.NEMOCLAW_DOCKER_GPU_PATCH = "0"; + try { + const sandboxGpuConfig = { sandboxGpuEnabled: true }; + const plan = resolveDockerGpuSandboxCreatePlan(sandboxGpuConfig, { + dockerDriverGateway: true, + detectDockerDesktopWsl: () => false, + }); + expect(plan.useDockerGpuPatch).toBe(false); + const createArgs = buildSandboxGpuCreateArgs(sandboxGpuConfig, { + suppressGpuFlag: plan.useDockerGpuPatch, + }); + expect(createArgs).toEqual(["--gpu"]); + } finally { + if (originalEnv === undefined) delete process.env.NEMOCLAW_DOCKER_GPU_PATCH; + else process.env.NEMOCLAW_DOCKER_GPU_PATCH = originalEnv; + } + }); +}); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index c2c0bc88b1e..51eb4ed0426 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -22,6 +22,20 @@ import { waitForOpenShellSupervisorReconnect, } from "./docker-gpu-patch"; import { finalizeDockerGpuPatchBackup } from "./docker-gpu-patch-finalize"; +import { detectWslDockerDesktopStatus } from "./wsl-docker-desktop-gpu"; + +let cachedDockerDesktopWslRuntime: boolean | null = null; + +export function isDockerDesktopWslRuntime(): boolean { + if (cachedDockerDesktopWslRuntime === null) { + cachedDockerDesktopWslRuntime = detectWslDockerDesktopStatus({}) === "docker-desktop"; + } + return cachedDockerDesktopWslRuntime; +} + +export function resetIsDockerDesktopWslRuntimeCache(): void { + cachedDockerDesktopWslRuntime = null; +} type DockerGpuSandboxCreateDeps = Pick< DockerGpuPatchDeps, @@ -288,10 +302,16 @@ function buildFailureContext( export function shouldUseDockerGpuPatchForCreate( config: DockerGpuSandboxConfig, - options: { dockerDriverGateway: boolean; log?: (message: string) => void }, + options: { + dockerDriverGateway: boolean; + dockerDesktopWsl?: boolean; + log?: (message: string) => void; + }, ): boolean { const enabled = shouldApplyDockerGpuPatch(config, { dockerDriverGateway: options.dockerDriverGateway, + dockerDesktopWsl: options.dockerDesktopWsl, + log: options.log, }); if (enabled) { options.log?.( @@ -305,9 +325,18 @@ export function shouldUseDockerGpuPatchForCreate( export function resolveDockerGpuSandboxCreatePlan( config: DockerGpuSandboxConfig, - options: { dockerDriverGateway: boolean }, + options: { + dockerDriverGateway: boolean; + dockerDesktopWsl?: boolean; + detectDockerDesktopWsl?: () => boolean; + }, ): DockerGpuSandboxCreatePlan { - const useDockerGpuPatch = shouldUseDockerGpuPatchForCreate(config, options); + const dockerDesktopWsl = + options.dockerDesktopWsl ?? (options.detectDockerDesktopWsl ?? isDockerDesktopWslRuntime)(); + const useDockerGpuPatch = shouldUseDockerGpuPatchForCreate(config, { + dockerDriverGateway: options.dockerDriverGateway, + dockerDesktopWsl, + }); const logMessage = config.sandboxGpuEnabled ? useDockerGpuPatch ? config.hostGpuPlatform === "jetson" diff --git a/src/lib/validation-cdi.test.ts b/src/lib/validation-cdi.test.ts new file mode 100644 index 00000000000..f6ebf0ffec6 --- /dev/null +++ b/src/lib/validation-cdi.test.ts @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { classifySandboxCreateFailure } from "../../dist/lib/validation"; + +describe("classifySandboxCreateFailure GPU CDI injection arm", () => { + it("detects GPU CDI injection failure from 'CDI device injection failed'", () => { + const result = classifySandboxCreateFailure( + "Error response from daemon: CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + ); + expect(result.kind).toBe("gpu_cdi_injection_failed"); + }); + + it("detects GPU CDI injection failure from 'unresolvable CDI devices' alone", () => { + const result = classifySandboxCreateFailure("unresolvable CDI devices nvidia.com/gpu=all"); + expect(result.kind).toBe("gpu_cdi_injection_failed"); + }); + + it("does NOT misclassify generic CDI mentions as gpu_cdi_injection_failed", () => { + const result = classifySandboxCreateFailure("CDI spec directories configured"); + expect(result.kind).toBe("unknown"); + }); + + it("does NOT classify non-GPU CDI injection failures as gpu_cdi_injection_failed", () => { + const result = classifySandboxCreateFailure( + "CDI device injection failed: unresolvable CDI devices example.com/widget=all", + ); + expect(result.kind).toBe("unknown"); + }); + + it("classifies gpu_cdi_injection_failed even when 'Created sandbox:' is also present", () => { + const output = + "Created sandbox: test-sandbox\nError response from daemon: CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all"; + const result = classifySandboxCreateFailure(output); + expect(result.kind).toBe("gpu_cdi_injection_failed"); + }); +}); diff --git a/src/lib/validation.ts b/src/lib/validation.ts index 876adda6f2d..bd181f3261d 100644 --- a/src/lib/validation.ts +++ b/src/lib/validation.ts @@ -19,6 +19,7 @@ export interface SandboxCreateFailure { | "image_upload_container_missing" | "sandbox_create_incomplete" | "tls_cert_mismatch" + | "gpu_cdi_injection_failed" | "unknown"; uploadedToGateway: boolean; } @@ -127,6 +128,12 @@ export function classifySandboxCreateFailure(output = ""): SandboxCreateFailure ) { return { kind: "image_upload_container_missing", uploadedToGateway }; } + if ( + /(CDI device injection failed|unresolvable CDI devices?)[^\n]*nvidia\.com\/gpu/i.test(text) || + /nvidia\.com\/gpu[^\n]*(CDI device injection failed|unresolvable CDI devices?)/i.test(text) + ) { + return { kind: "gpu_cdi_injection_failed", uploadedToGateway }; + } if (/Created sandbox:/i.test(text)) { return { kind: "sandbox_create_incomplete", uploadedToGateway: true }; }