Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/reference/troubleshooting.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -1358,6 +1358,12 @@ openshell sandbox delete <sandbox-name>
Fix the NVIDIA Container Toolkit or CDI configuration reported in the diagnostics, clean up the failed sandbox, then rerun onboarding.
If you do not need GPU access inside the sandbox, rerun with `--no-sandbox-gpu`.
Set `NEMOCLAW_DOCKER_GPU_PATCH=0` only when you need to bypass this compatibility path during troubleshooting.
On Docker Desktop WSL the patch is required for GPU passthrough — `NEMOCLAW_DOCKER_GPU_PATCH=0` is ignored on that runtime, and onboarding logs a warning when it is set there.
To skip GPU passthrough entirely on Docker Desktop WSL, rerun with `--no-gpu` or set `NEMOCLAW_SANDBOX_GPU=0`.

If sandbox creation fails with `CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all`, the OpenShell gateway tried `docker create --device nvidia.com/gpu=all` and Docker could not resolve the CDI spec.
This injection happens inside the gateway, so `NEMOCLAW_DOCKER_GPU_PATCH=0` does not bypass it.
Rerun with `--no-gpu`, or set `NEMOCLAW_SANDBOX_GPU=0` and resume onboarding.

If onboarding reports `OpenShell supervisor did not reconnect to the GPU-enabled container.` even though the diagnostic bundle shows the patched container is running and healthy, the supervisor-reconnect wait is treating a transient Error phase (reported while the OpenShell host re-registers the new container) as fatal.
The reconnect wait debounces consecutive Error-phase polls before fast-failing, defaulting to fifteen consecutive polls of about 30 seconds in total.
Expand Down
13 changes: 13 additions & 0 deletions src/lib/build-context.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,19 @@ describe("printSandboxCreateRecoveryHints", () => {
});
expect(stderr()).toContain("<built-image>");
});

it("prints GPU CDI injection guidance pointing at --no-gpu / NEMOCLAW_SANDBOX_GPU=0", () => {
printSandboxCreateRecoveryHints(
"Error response from daemon: CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all",
);

const out = stderr();
expect(out).toContain("GPU CDI device injection failed");
expect(out).toContain("NEMOCLAW_DOCKER_GPU_PATCH=0 does not bypass");
expect(out).toContain("--no-gpu");
expect(out).toContain("NEMOCLAW_SANDBOX_GPU=0");
expect(out).toContain("onboard --resume --no-gpu");
});
});

describe("reconstructImageRefCreateCommand", () => {
Expand Down
12 changes: 12 additions & 0 deletions src/lib/build-context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,18 @@ export function printSandboxCreateRecoveryHints(
console.error(` Then: ${CLI_NAME} onboard --resume`);
return;
}
if (failure.kind === "gpu_cdi_injection_failed") {
console.error(" Hint: GPU CDI device injection failed inside the OpenShell gateway.");
console.error(
" The gateway issues `docker create --device nvidia.com/gpu=all` on its own, so",
);
console.error(" NEMOCLAW_DOCKER_GPU_PATCH=0 does not bypass this path.");
console.error(" Skip GPU passthrough entirely with either:");
console.error(` ${CLI_NAME} onboard --no-gpu`);
console.error(" NEMOCLAW_SANDBOX_GPU=0 (env var, applies to subsequent runs)");
console.error(` Recovery: ${CLI_NAME} onboard --resume --no-gpu`);
return;
}
console.error(` Recovery: ${CLI_NAME} onboard --resume`);
console.error(` Or: ${CLI_NAME} onboard`);
}
3 changes: 2 additions & 1 deletion src/lib/onboard/command-support.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,8 @@ export function buildOnboardFlags(): Record<string, any> {
description: "Enable direct NVIDIA GPU access inside the sandbox",
}),
"no-sandbox-gpu": Flags.boolean({
description: "Force CPU sandbox behavior",
description:
"Force CPU sandbox behavior (equivalent to NEMOCLAW_SANDBOX_GPU=0; alternative to --no-gpu when Docker Desktop WSL CDI injection fails)",
}),
"sandbox-gpu-device": Flags.string({
description:
Expand Down
14 changes: 12 additions & 2 deletions src/lib/onboard/docker-gpu-local-inference.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
printDockerGpuProofFailure,
shouldApplyDockerGpuPatch,
} from "./docker-gpu-patch";
import { isDockerDesktopWslRuntime } from "./docker-gpu-sandbox-create";
import { executeSandboxCommandForVerification } from "./sandbox-verification-exec";

const {
Expand Down Expand Up @@ -39,11 +40,16 @@ type DockerGpuLocalInferenceConfig = {

type DockerGpuLocalInferenceOptions = {
dockerDriverGateway: boolean;
dockerDesktopWsl?: boolean;
env?: NodeJS.ProcessEnv;
platform?: NodeJS.Platform;
log?: (message: string) => void;
};

function resolveDockerDesktopWsl(options: DockerGpuLocalInferenceOptions): boolean {
return options.dockerDesktopWsl ?? isDockerDesktopWslRuntime();
}

function isLocalInferenceProvider(provider: string | null | undefined): provider is string {
return Boolean(provider && LOCAL_INFERENCE_PROVIDERS.includes(provider));
}
Expand All @@ -60,6 +66,7 @@ export function shouldUseDockerGpuPatchHostNetwork(
return (
shouldApplyDockerGpuPatch(config, {
dockerDriverGateway: options.dockerDriverGateway,
dockerDesktopWsl: resolveDockerDesktopWsl(options),
env: options.env,
platform: options.platform,
}) && getDockerGpuPatchNetworkMode(options.env ?? process.env) === "host"
Expand Down Expand Up @@ -263,6 +270,7 @@ export function verifyDockerGpuSandboxLocalInference(
if (
!shouldApplyDockerGpuPatch(config, {
dockerDriverGateway: options.dockerDriverGateway,
dockerDesktopWsl: resolveDockerDesktopWsl(options),
env: options.env,
platform: options.platform,
})
Expand Down Expand Up @@ -401,8 +409,10 @@ export function verifyGpuSandboxAfterReady(
throw error;
}

// When NEMOCLAW_DOCKER_GPU_PATCH=0, useDockerGpuPatch is false and there is no
// GPU-patched sandbox to gate, so skip the local inference reachability gate.
// When the resolved create plan disabled the Docker GPU patch (e.g.
// NEMOCLAW_DOCKER_GPU_PATCH=0 honoured outside Docker Desktop WSL), there is
// no GPU-patched sandbox to gate, so skip the local inference reachability
// gate.
if (!options.useDockerGpuPatch) return;
const verification = verifyDockerGpuSandboxLocalInference(config, provider, {
sandboxName: options.sandboxName,
Expand Down
42 changes: 42 additions & 0 deletions src/lib/onboard/docker-gpu-patch-wsl.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it } from "vitest";

import { shouldApplyDockerGpuPatch } from "../../../dist/lib/onboard/docker-gpu-patch";

describe("shouldApplyDockerGpuPatch on Docker Desktop WSL", () => {
it("ignores NEMOCLAW_DOCKER_GPU_PATCH=0 on Docker Desktop WSL where the patch is required", () => {
const logs: string[] = [];
expect(
shouldApplyDockerGpuPatch(
{ sandboxGpuEnabled: true },
{
env: { NEMOCLAW_DOCKER_GPU_PATCH: "0" },
platform: "linux",
dockerDriverGateway: true,
dockerDesktopWsl: true,
log: (message) => {
logs.push(message);
},
},
),
).toBe(true);
expect(logs.some((line) => /NEMOCLAW_DOCKER_GPU_PATCH=0 ignored/i.test(line))).toBe(true);
expect(logs.some((line) => /NEMOCLAW_SANDBOX_GPU=0/.test(line))).toBe(true);
});

it("still honors NEMOCLAW_DOCKER_GPU_PATCH=0 when not on Docker Desktop WSL", () => {
expect(
shouldApplyDockerGpuPatch(
{ sandboxGpuEnabled: true },
{
env: { NEMOCLAW_DOCKER_GPU_PATCH: "0" },
platform: "linux",
dockerDriverGateway: true,
dockerDesktopWsl: false,
},
),
).toBe(false);
});
});
29 changes: 22 additions & 7 deletions src/lib/onboard/docker-gpu-patch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -522,17 +522,26 @@ export function shouldApplyDockerGpuPatch(
env?: NodeJS.ProcessEnv;
platform?: NodeJS.Platform;
dockerDriverGateway?: boolean;
dockerDesktopWsl?: boolean;
log?: (message: string) => void;
} = {},
): boolean {
const env = options.env ?? process.env;
const platform = options.platform ?? process.platform;
const dockerDriverGateway = options.dockerDriverGateway ?? platform === "linux";
return (
config.sandboxGpuEnabled &&
platform === "linux" &&
dockerDriverGateway &&
String(env.NEMOCLAW_DOCKER_GPU_PATCH || "").trim() !== "0"
);
if (!(config.sandboxGpuEnabled && platform === "linux" && dockerDriverGateway)) {
return false;
}
const optedOut = String(env.NEMOCLAW_DOCKER_GPU_PATCH || "").trim() === "0";
if (optedOut && options.dockerDesktopWsl) {
const log = options.log ?? ((message: string) => console.warn(message));
log(
" NEMOCLAW_DOCKER_GPU_PATCH=0 ignored on Docker Desktop WSL: GPU passthrough on this runtime requires the patch.",
);
log(" Skip GPU passthrough entirely with --no-gpu or NEMOCLAW_SANDBOX_GPU=0.");
return true;
}
return !optedOut;
}

export function buildDockerGpuCloneRunOptions(
Expand Down Expand Up @@ -1250,7 +1259,13 @@ export function printDockerGpuPatchFailureAndExit(
if (diagnostics) {
console.error(` Diagnostics saved: ${diagnostics.dir}`);
}
console.error(" Escape hatch: set NEMOCLAW_DOCKER_GPU_PATCH=0 to skip this patch.");
console.error(" Escape hatches:");
console.error(
" NEMOCLAW_DOCKER_GPU_PATCH=0 skip this Docker GPU patch (Linux native Docker only; ignored on Docker Desktop WSL where the patch is required).",
);
console.error(
" NEMOCLAW_SANDBOX_GPU=0 skip GPU passthrough entirely (or rerun with --no-gpu).",
);
printDockerGpuPatchCleanup(sandboxName);
process.exit(1);
}
Expand Down
84 changes: 83 additions & 1 deletion src/lib/onboard/docker-gpu-sandbox-create.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@ import type {
DockerGpuPatchFailureContext,
DockerGpuPatchResult,
} from "../../../dist/lib/onboard/docker-gpu-patch";
import { createDockerGpuSandboxCreatePatch } from "../../../dist/lib/onboard/docker-gpu-sandbox-create";
import { buildSandboxGpuCreateArgs } from "../../../dist/lib/onboard/sandbox-gpu-create";
import {
createDockerGpuSandboxCreatePatch,
resolveDockerGpuSandboxCreatePlan,
} from "../../../dist/lib/onboard/docker-gpu-sandbox-create";

function deferredCreateResult(): DockerGpuPatchResult {
return {
Expand Down Expand Up @@ -220,3 +224,81 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => {
expect(finalizeBackup).not.toHaveBeenCalled();
});
});

describe("resolveDockerGpuSandboxCreatePlan Docker Desktop WSL handling", () => {
it("keeps useDockerGpuPatch=true on Docker Desktop WSL even when NEMOCLAW_DOCKER_GPU_PATCH=0", () => {
const originalEnv = process.env.NEMOCLAW_DOCKER_GPU_PATCH;
process.env.NEMOCLAW_DOCKER_GPU_PATCH = "0";
try {
const plan = resolveDockerGpuSandboxCreatePlan(
{ sandboxGpuEnabled: true },
{
dockerDriverGateway: true,
detectDockerDesktopWsl: () => true,
},
);
expect(plan.useDockerGpuPatch).toBe(true);
} finally {
if (originalEnv === undefined) delete process.env.NEMOCLAW_DOCKER_GPU_PATCH;
else process.env.NEMOCLAW_DOCKER_GPU_PATCH = originalEnv;
}
});

it("honors NEMOCLAW_DOCKER_GPU_PATCH=0 when not on Docker Desktop WSL", () => {
const originalEnv = process.env.NEMOCLAW_DOCKER_GPU_PATCH;
process.env.NEMOCLAW_DOCKER_GPU_PATCH = "0";
try {
const plan = resolveDockerGpuSandboxCreatePlan(
{ sandboxGpuEnabled: true },
{
dockerDriverGateway: true,
detectDockerDesktopWsl: () => false,
},
);
expect(plan.useDockerGpuPatch).toBe(false);
} finally {
if (originalEnv === undefined) delete process.env.NEMOCLAW_DOCKER_GPU_PATCH;
else process.env.NEMOCLAW_DOCKER_GPU_PATCH = originalEnv;
}
});

it("suppresses the openshell sandbox create --gpu flag on Docker Desktop WSL when the opt-out is ignored", () => {
const originalEnv = process.env.NEMOCLAW_DOCKER_GPU_PATCH;
process.env.NEMOCLAW_DOCKER_GPU_PATCH = "0";
try {
const sandboxGpuConfig = { sandboxGpuEnabled: true };
const plan = resolveDockerGpuSandboxCreatePlan(sandboxGpuConfig, {
dockerDriverGateway: true,
detectDockerDesktopWsl: () => true,
});
expect(plan.useDockerGpuPatch).toBe(true);
const createArgs = buildSandboxGpuCreateArgs(sandboxGpuConfig, {
suppressGpuFlag: plan.useDockerGpuPatch,
});
expect(createArgs).toEqual([]);
} finally {
if (originalEnv === undefined) delete process.env.NEMOCLAW_DOCKER_GPU_PATCH;
else process.env.NEMOCLAW_DOCKER_GPU_PATCH = originalEnv;
}
});

it("emits --gpu when the patch is disabled outside Docker Desktop WSL", () => {
const originalEnv = process.env.NEMOCLAW_DOCKER_GPU_PATCH;
process.env.NEMOCLAW_DOCKER_GPU_PATCH = "0";
try {
const sandboxGpuConfig = { sandboxGpuEnabled: true };
const plan = resolveDockerGpuSandboxCreatePlan(sandboxGpuConfig, {
dockerDriverGateway: true,
detectDockerDesktopWsl: () => false,
});
expect(plan.useDockerGpuPatch).toBe(false);
const createArgs = buildSandboxGpuCreateArgs(sandboxGpuConfig, {
suppressGpuFlag: plan.useDockerGpuPatch,
});
expect(createArgs).toEqual(["--gpu"]);
} finally {
if (originalEnv === undefined) delete process.env.NEMOCLAW_DOCKER_GPU_PATCH;
else process.env.NEMOCLAW_DOCKER_GPU_PATCH = originalEnv;
}
});
});
35 changes: 32 additions & 3 deletions src/lib/onboard/docker-gpu-sandbox-create.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,20 @@ import {
waitForOpenShellSupervisorReconnect,
} from "./docker-gpu-patch";
import { finalizeDockerGpuPatchBackup } from "./docker-gpu-patch-finalize";
import { detectWslDockerDesktopStatus } from "./wsl-docker-desktop-gpu";

let cachedDockerDesktopWslRuntime: boolean | null = null;

export function isDockerDesktopWslRuntime(): boolean {
if (cachedDockerDesktopWslRuntime === null) {
cachedDockerDesktopWslRuntime = detectWslDockerDesktopStatus({}) === "docker-desktop";
}
return cachedDockerDesktopWslRuntime;
}

export function resetIsDockerDesktopWslRuntimeCache(): void {
cachedDockerDesktopWslRuntime = null;
}

type DockerGpuSandboxCreateDeps = Pick<
DockerGpuPatchDeps,
Expand Down Expand Up @@ -288,10 +302,16 @@ function buildFailureContext(

export function shouldUseDockerGpuPatchForCreate(
config: DockerGpuSandboxConfig,
options: { dockerDriverGateway: boolean; log?: (message: string) => void },
options: {
dockerDriverGateway: boolean;
dockerDesktopWsl?: boolean;
log?: (message: string) => void;
},
): boolean {
const enabled = shouldApplyDockerGpuPatch(config, {
dockerDriverGateway: options.dockerDriverGateway,
dockerDesktopWsl: options.dockerDesktopWsl,
log: options.log,
});
if (enabled) {
options.log?.(
Expand All @@ -305,9 +325,18 @@ export function shouldUseDockerGpuPatchForCreate(

export function resolveDockerGpuSandboxCreatePlan(
config: DockerGpuSandboxConfig,
options: { dockerDriverGateway: boolean },
options: {
dockerDriverGateway: boolean;
dockerDesktopWsl?: boolean;
detectDockerDesktopWsl?: () => boolean;
},
): DockerGpuSandboxCreatePlan {
const useDockerGpuPatch = shouldUseDockerGpuPatchForCreate(config, options);
const dockerDesktopWsl =
options.dockerDesktopWsl ?? (options.detectDockerDesktopWsl ?? isDockerDesktopWslRuntime)();
const useDockerGpuPatch = shouldUseDockerGpuPatchForCreate(config, {
dockerDriverGateway: options.dockerDriverGateway,
dockerDesktopWsl,
});
const logMessage = config.sandboxGpuEnabled
? useDockerGpuPatch
? config.hostGpuPlatform === "jetson"
Expand Down
Loading
Loading