Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/reference/troubleshooting.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -1358,6 +1358,12 @@ openshell sandbox delete <sandbox-name>
Fix the NVIDIA Container Toolkit or CDI configuration reported in the diagnostics, clean up the failed sandbox, then rerun onboarding.
If you do not need GPU access inside the sandbox, rerun with `--no-sandbox-gpu`.
Set `NEMOCLAW_DOCKER_GPU_PATCH=0` only when you need to bypass this compatibility path during troubleshooting.
On Docker Desktop WSL the patch is required for GPU passthrough — `NEMOCLAW_DOCKER_GPU_PATCH=0` is ignored on that runtime, and onboarding logs a warning when it is set there.
To skip GPU passthrough entirely on Docker Desktop WSL, rerun with `--no-gpu` or set `NEMOCLAW_SANDBOX_GPU=0`.

If sandbox creation fails with `CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all`, the OpenShell gateway tried `docker create --device nvidia.com/gpu=all` and Docker could not resolve the CDI spec.
This injection happens inside the gateway, so `NEMOCLAW_DOCKER_GPU_PATCH=0` does not bypass it.
Rerun with `--no-gpu`, or set `NEMOCLAW_SANDBOX_GPU=0` and resume onboarding.

If onboarding reports `OpenShell supervisor did not reconnect to the GPU-enabled container.` even though the diagnostic bundle shows the patched container is running and healthy, the supervisor-reconnect wait is treating a transient Error phase (reported while the OpenShell host re-registers the new container) as fatal.
The reconnect wait debounces consecutive Error-phase polls before fast-failing, defaulting to fifteen consecutive polls of about 30 seconds in total.
Expand Down
13 changes: 13 additions & 0 deletions src/lib/build-context.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,19 @@ describe("printSandboxCreateRecoveryHints", () => {
});
expect(stderr()).toContain("<built-image>");
});

it("prints GPU CDI injection guidance pointing at --no-gpu / NEMOCLAW_SANDBOX_GPU=0", () => {
printSandboxCreateRecoveryHints(
"Error response from daemon: CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all",
);

const out = stderr();
expect(out).toContain("GPU CDI device injection failed");
expect(out).toContain("NEMOCLAW_DOCKER_GPU_PATCH=0 does not bypass");
expect(out).toContain("--no-gpu");
expect(out).toContain("NEMOCLAW_SANDBOX_GPU=0");
expect(out).toContain("onboard --resume --no-gpu");
});
});

describe("reconstructImageRefCreateCommand", () => {
Expand Down
16 changes: 16 additions & 0 deletions src/lib/build-context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,22 @@ export function printSandboxCreateRecoveryHints(
console.error(` Then: ${CLI_NAME} onboard --resume`);
return;
}
if (failure.kind === "gpu_cdi_injection_failed") {
console.error(
" Hint: GPU CDI device injection failed inside the OpenShell gateway.",
);
console.error(
" The gateway issues `docker create --device nvidia.com/gpu=all` on its own, so",
);
console.error(
" NEMOCLAW_DOCKER_GPU_PATCH=0 does not bypass this path.",
);
console.error(" Skip GPU passthrough entirely with either:");
console.error(` ${CLI_NAME} onboard --no-gpu`);
console.error(" NEMOCLAW_SANDBOX_GPU=0 (env var, applies to subsequent runs)");
console.error(` Recovery: ${CLI_NAME} onboard --resume --no-gpu`);
return;
}
console.error(` Recovery: ${CLI_NAME} onboard --resume`);
console.error(` Or: ${CLI_NAME} onboard`);
}
3 changes: 3 additions & 0 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3464,9 +3464,12 @@ async function createSandbox(
...qrSelectedChannels,
]),
];
const dockerDesktopWsl =
preflightUtils.detectWslDockerDesktopStatus({}) === "docker-desktop";
const { useDockerGpuPatch, logMessage: sandboxGpuLogMessage } =
dockerGpuSandboxCreate.resolveDockerGpuSandboxCreatePlan(effectiveSandboxGpuConfig, {
dockerDriverGateway: isLinuxDockerDriverGatewayEnabled(),
dockerDesktopWsl,
});
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
const initialSandboxPolicy = prepareInitialSandboxCreatePolicy(
basePolicyPath,
Expand Down
34 changes: 34 additions & 0 deletions src/lib/onboard/docker-gpu-patch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,40 @@ describe("docker-gpu-patch", () => {
).toBe(false);
});

it("ignores NEMOCLAW_DOCKER_GPU_PATCH=0 on Docker Desktop WSL where the patch is required", () => {
const logs: string[] = [];
expect(
shouldApplyDockerGpuPatch(
{ sandboxGpuEnabled: true },
{
env: { NEMOCLAW_DOCKER_GPU_PATCH: "0" },
platform: "linux",
dockerDriverGateway: true,
dockerDesktopWsl: true,
log: (message) => {
logs.push(message);
},
},
),
).toBe(true);
expect(logs.some((line) => /NEMOCLAW_DOCKER_GPU_PATCH=0 ignored/i.test(line))).toBe(true);
expect(logs.some((line) => /NEMOCLAW_SANDBOX_GPU=0/.test(line))).toBe(true);
});

it("still honors NEMOCLAW_DOCKER_GPU_PATCH=0 when not on Docker Desktop WSL", () => {
expect(
shouldApplyDockerGpuPatch(
{ sandboxGpuEnabled: true },
{
env: { NEMOCLAW_DOCKER_GPU_PATCH: "0" },
platform: "linux",
dockerDriverGateway: true,
dockerDesktopWsl: false,
},
),
).toBe(false);
});

it("builds clone args that preserve OpenShell labels and runtime settings", () => {
const args = buildDockerGpuCloneRunArgs(inspectFixture(), buildDockerGpuMode("gpus"));

Expand Down
31 changes: 24 additions & 7 deletions src/lib/onboard/docker-gpu-patch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -522,17 +522,28 @@ export function shouldApplyDockerGpuPatch(
env?: NodeJS.ProcessEnv;
platform?: NodeJS.Platform;
dockerDriverGateway?: boolean;
dockerDesktopWsl?: boolean;
log?: (message: string) => void;
} = {},
): boolean {
const env = options.env ?? process.env;
const platform = options.platform ?? process.platform;
const dockerDriverGateway = options.dockerDriverGateway ?? platform === "linux";
return (
config.sandboxGpuEnabled &&
platform === "linux" &&
dockerDriverGateway &&
String(env.NEMOCLAW_DOCKER_GPU_PATCH || "").trim() !== "0"
);
if (!(config.sandboxGpuEnabled && platform === "linux" && dockerDriverGateway)) {
return false;
}
const optedOut = String(env.NEMOCLAW_DOCKER_GPU_PATCH || "").trim() === "0";
if (optedOut && options.dockerDesktopWsl) {
const log = options.log ?? ((message: string) => console.warn(message));
log(
" NEMOCLAW_DOCKER_GPU_PATCH=0 ignored on Docker Desktop WSL: GPU passthrough on this runtime requires the patch.",
);
log(
" Skip GPU passthrough entirely with --no-gpu or NEMOCLAW_SANDBOX_GPU=0.",
);
return true;
}
return !optedOut;
}

export function buildDockerGpuCloneRunOptions(
Expand Down Expand Up @@ -1250,7 +1261,13 @@ export function printDockerGpuPatchFailureAndExit(
if (diagnostics) {
console.error(` Diagnostics saved: ${diagnostics.dir}`);
}
console.error(" Escape hatch: set NEMOCLAW_DOCKER_GPU_PATCH=0 to skip this patch.");
console.error(" Escape hatches:");
console.error(
" NEMOCLAW_DOCKER_GPU_PATCH=0 skip this Docker GPU patch (Linux native Docker only; ignored on Docker Desktop WSL where the patch is required).",
);
console.error(
" NEMOCLAW_SANDBOX_GPU=0 skip GPU passthrough entirely (or rerun with --no-gpu).",
);
printDockerGpuPatchCleanup(sandboxName);
process.exit(1);
}
Expand Down
10 changes: 8 additions & 2 deletions src/lib/onboard/docker-gpu-sandbox-create.ts
Original file line number Diff line number Diff line change
Expand Up @@ -288,10 +288,16 @@ function buildFailureContext(

export function shouldUseDockerGpuPatchForCreate(
config: DockerGpuSandboxConfig,
options: { dockerDriverGateway: boolean; log?: (message: string) => void },
options: {
dockerDriverGateway: boolean;
dockerDesktopWsl?: boolean;
log?: (message: string) => void;
},
): boolean {
const enabled = shouldApplyDockerGpuPatch(config, {
dockerDriverGateway: options.dockerDriverGateway,
dockerDesktopWsl: options.dockerDesktopWsl,
log: options.log,
});
if (enabled) {
options.log?.(
Expand All @@ -305,7 +311,7 @@ export function shouldUseDockerGpuPatchForCreate(

export function resolveDockerGpuSandboxCreatePlan(
config: DockerGpuSandboxConfig,
options: { dockerDriverGateway: boolean },
options: { dockerDriverGateway: boolean; dockerDesktopWsl?: boolean },
): DockerGpuSandboxCreatePlan {
const useDockerGpuPatch = shouldUseDockerGpuPatchForCreate(config, options);
const logMessage = config.sandboxGpuEnabled
Expand Down
5 changes: 4 additions & 1 deletion src/lib/onboard/preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,10 @@ import {
wslDockerDesktopGpuCompatibilityAction,
} from "./wsl-docker-desktop-gpu";
export { getNvidiaCdiSpecPath, parseDockerCdiSpecDirs } from "./docker-cdi";
export { isWslDockerDesktopRuntime } from "./wsl-docker-desktop-gpu";
export {
detectWslDockerDesktopStatus,
isWslDockerDesktopRuntime,
} from "./wsl-docker-desktop-gpu";

// runner.ts still uses CommonJS-style exports — use require here.
const { run, runCapture } = require("../runner");
Expand Down
19 changes: 19 additions & 0 deletions src/lib/validation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,25 @@ describe("classifySandboxCreateFailure", () => {
expect(result.kind).toBe("tls_cert_mismatch");
});

it("detects GPU CDI injection failure from 'CDI device injection failed'", () => {
const result = classifySandboxCreateFailure(
"Error response from daemon: CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all",
);
expect(result.kind).toBe("gpu_cdi_injection_failed");
});

it("detects GPU CDI injection failure from 'unresolvable CDI devices' alone", () => {
const result = classifySandboxCreateFailure(
"unresolvable CDI devices nvidia.com/gpu=all",
);
expect(result.kind).toBe("gpu_cdi_injection_failed");
});

it("does NOT misclassify generic CDI mentions as gpu_cdi_injection_failed", () => {
const result = classifySandboxCreateFailure("CDI spec directories configured");
expect(result.kind).toBe("unknown");
});

it("does NOT classify generic TLS transport errors as tls_cert_mismatch", () => {
expect(classifySandboxCreateFailure("TLS error: connection refused by proxy").kind).toBe(
"unknown",
Expand Down
4 changes: 4 additions & 0 deletions src/lib/validation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ export interface SandboxCreateFailure {
| "image_upload_container_missing"
| "sandbox_create_incomplete"
| "tls_cert_mismatch"
| "gpu_cdi_injection_failed"
| "unknown";
uploadedToGateway: boolean;
}
Expand Down Expand Up @@ -130,6 +131,9 @@ export function classifySandboxCreateFailure(output = ""): SandboxCreateFailure
if (/Created sandbox:/i.test(text)) {
return { kind: "sandbox_create_incomplete", uploadedToGateway: true };
}
if (/CDI device injection failed|unresolvable CDI devices?/i.test(text)) {
return { kind: "gpu_cdi_injection_failed", uploadedToGateway };
}
return { kind: "unknown", uploadedToGateway };
}

Expand Down
Loading