Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 20 additions & 22 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,8 @@ const { trackChildExit } =
require("./onboard/child-exit-tracker") as typeof import("./onboard/child-exit-tracker");
const { reportDockerDriverGatewayStartFailure } =
require("./onboard/docker-driver-gateway-failure") as typeof import("./onboard/docker-driver-gateway-failure");
const { printDockerDaemonRecovery, reportLegacyGatewayStartResultFailure } =
require("./onboard/gateway-start-failure") as typeof import("./onboard/gateway-start-failure");
const dockerDriverGatewayEnv: typeof import("./onboard/docker-driver-gateway-env") =
require("./onboard/docker-driver-gateway-env");
const { getDockerDriverGatewayEndpoint } = dockerDriverGatewayEnv;
Expand Down Expand Up @@ -1211,6 +1213,7 @@ function destroyGateway(

type FinalGatewayStartFailureOptions = {
retries: number;
dockerUnreachable?: boolean;
collectDiagnostics?: () => string | null | undefined;
cleanupGateway?: () => void;
exitProcess?: (code: number) => never;
Expand All @@ -1219,6 +1222,7 @@ type FinalGatewayStartFailureOptions = {

function handleFinalGatewayStartFailure({
retries,
dockerUnreachable = false,
collectDiagnostics = () =>
runCaptureOpenshell(["doctor", "logs", "--name", GATEWAY_NAME], {
ignoreError: true,
Expand All @@ -1228,6 +1232,11 @@ function handleFinalGatewayStartFailure({
exitProcess = (code) => process.exit(code),
printError = (message = "") => console.error(message),
}: FinalGatewayStartFailureOptions): never {
if (dockerUnreachable) {
printDockerDaemonRecovery(printError);
return exitProcess(1);
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

printError(` Gateway failed to start after ${retries + 1} attempts.`);
printError(" Gateway state preserved until diagnostics are collected.");
printError("");
Expand Down Expand Up @@ -2410,22 +2419,16 @@ async function startGatewayWithOptions(
);
}

// When a stale gateway is detected (metadata exists but container is gone,
// e.g. after a Docker/Colima restart), skip the destroy — `gateway start`
// can recover the container without wiping metadata and mTLS certs.
// The retry loop below will destroy only if start genuinely fails.
if (hasStaleGateway(gatewaySnapshot.gwInfo)) {
console.log(" Stale gateway detected — attempting restart without destroy...");
}

// Clear stale SSH host keys from previous gateway (fixes #768)
try {
const { execFileSync } = require("child_process");
execFileSync("ssh-keygen", ["-R", `openshell-${GATEWAY_NAME}`], { stdio: "ignore" });
} catch {
/* ssh-keygen -R may fail if entry doesn't exist — safe to ignore */
}
// Also purge any known_hosts entries matching the gateway hostname pattern
const knownHostsPath = path.join(os.homedir(), ".ssh", "known_hosts");
try {
const kh = fs.readFileSync(knownHostsPath, "utf8");
Expand All @@ -2436,9 +2439,6 @@ async function startGatewayWithOptions(
}

const gwArgs = ["--name", GATEWAY_NAME, "--port", getGatewayPortArg()];
// On NVIDIA hosts, pass --gpu unless the user explicitly opted out. This
// makes direct CUDA tools available in the sandbox by default while still
// supporting host-side inference providers.
if (gpuPassthrough) {
gwArgs.push("--gpu");
}
Expand All @@ -2447,12 +2447,8 @@ async function startGatewayWithOptions(
console.log(` Using pinned OpenShell gateway image: ${gatewayEnv.OPENSHELL_CLUSTER_IMAGE}`);
}

// Retry gateway start with exponential backoff. On some hosts (Horde VMs,
// first-run environments) the embedded k3s needs more time than OpenShell's
// internal health-check window allows. Retrying after a clean destroy lets
// the second attempt benefit from cached images and cleaner cgroup state.
// See: https://github.com/NVIDIA/OpenShell/issues/433
const retries = exitOnFailure ? 2 : 0;
let dockerUnreachable = false;
try {
await pRetry(
async () => {
Expand All @@ -2464,13 +2460,15 @@ async function startGatewayWithOptions(
},
);
if (startResult.status !== 0) {
const lines = String(redact(startResult.output || ""))
.split("\n")
.map((l) => compactText(l.replace(ANSI_RE, "")))
.filter(Boolean)
.map((l) => ` ${l}`);
if (lines.length > 0) {
console.log(` Gateway start returned before healthy:\n${lines.join("\n")}`);
const failure = reportLegacyGatewayStartResultFailure(
startResult.output || "",
console.log,
);
if (failure.kind === "docker_unreachable") {
dockerUnreachable = true;
throw new pRetry.AbortError(
"Docker daemon is not reachable (gateway cannot start).",
);
}
}
console.log(" Waiting for gateway health...");
Expand Down Expand Up @@ -2531,7 +2529,7 @@ async function startGatewayWithOptions(
);
} catch {
if (exitOnFailure) {
handleFinalGatewayStartFailure({ retries });
handleFinalGatewayStartFailure({ retries, dockerUnreachable });
}
throw new Error("Gateway failed to start");
}
Expand Down
48 changes: 39 additions & 9 deletions src/lib/onboard/docker-driver-gateway-failure.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,9 @@ describe("reportDockerDriverGatewayStartFailure (#3111)", () => {
beforeEach(() => {
errSpy = vi.spyOn(console, "error").mockImplementation(() => {});
// Stub process.exit so assertions can still run.
exitSpy = vi
.spyOn(process, "exit")
.mockImplementation(((code?: number) => {
throw new Error(`process.exit(${code})`);
}) as never);
exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
throw new Error(`process.exit(${code})`);
}) as never);
});

afterEach(() => {
Expand All @@ -50,7 +48,9 @@ describe("reportDockerDriverGatewayStartFailure (#3111)", () => {
{ exitOnFailure: false },
),
).not.toThrow();
const joined = errSpy.mock.calls.map((c: string[]) => c.join(" ")).join("\n");
const joined = errSpy.mock.calls
.map((c: string[]) => c.join(" "))
.join("\n");
expect(joined).toContain("Docker-driver gateway failed to start");
expect(joined).toContain("Troubleshooting:");
expect(joined).toContain("tail -100 /tmp/nonexistent-gateway.log");
Expand All @@ -67,7 +67,9 @@ describe("reportDockerDriverGatewayStartFailure (#3111)", () => {
}),
{ exitOnFailure: false },
);
const joined = errSpy.mock.calls.map((c: string[]) => c.join(" ")).join("\n");
const joined = errSpy.mock.calls
.map((c: string[]) => c.join(" "))
.join("\n");
expect(joined).toContain(
"Gateway process exited with code 127 before becoming ready",
);
Expand All @@ -79,7 +81,9 @@ describe("reportDockerDriverGatewayStartFailure (#3111)", () => {
makeExitState(),
{ exitOnFailure: false },
);
const joined = errSpy.mock.calls.map((c: string[]) => c.join(" ")).join("\n");
const joined = errSpy.mock.calls
.map((c: string[]) => c.join(" "))
.join("\n");
expect(joined).not.toContain("before becoming ready");
});

Expand All @@ -94,14 +98,40 @@ describe("reportDockerDriverGatewayStartFailure (#3111)", () => {
reportDockerDriverGatewayStartFailure(log, makeExitState(), {
exitOnFailure: false,
});
const joined = errSpy.mock.calls.map((c: string[]) => c.join(" ")).join("\n");
const joined = errSpy.mock.calls
.map((c: string[]) => c.join(" "))
.join("\n");
expect(joined).toContain("Gateway log tail");
expect(joined).toContain("GLIBC_2.38 not found");
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});

it("prints Docker runtime recovery guidance when gateway log shows Docker is unreachable (#2347)", () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "gw-fail-"));
const log = path.join(dir, "openshell-gateway.log");
fs.writeFileSync(
log,
[
"Error: Failed to create Docker client.",
"Socket not found: /var/run/docker.sock",
].join("\n"),
);
try {
reportDockerDriverGatewayStartFailure(log, makeExitState(), {
exitOnFailure: false,
});
const joined = errSpy.mock.calls
.map((c: string[]) => c.join(" "))
.join("\n");
expect(joined).toContain("Docker daemon is not running");
expect(joined).toContain("Start Docker, then rerun `nemoclaw onboard`");
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});

it("calls process.exit(1) when exitOnFailure is true", () => {
expect(() =>
reportDockerDriverGatewayStartFailure(
Expand Down
5 changes: 5 additions & 0 deletions src/lib/onboard/docker-driver-gateway-failure.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,10 @@
import fs from "node:fs";

import { redact } from "../security/redact";
import { classifyGatewayStartFailure } from "../validation";

import type { ChildExitState } from "./child-exit-tracker";
import { printDockerDaemonRecovery } from "./gateway-start-failure";

export type ReportDockerDriverGatewayStartFailureOpts = {
/**
Expand Down Expand Up @@ -71,6 +73,9 @@ export function reportDockerDriverGatewayStartFailure(
console.error(" Gateway log tail:");
for (const line of tail.split("\n")) console.error(` ${redact(line)}`);
}
if (classifyGatewayStartFailure(tail).kind === "docker_unreachable") {
printDockerDaemonRecovery(console.error);
}
console.error(" Troubleshooting:");
console.error(` tail -100 ${logPath}`);
console.error(" docker info --format '{{json .CDISpecDirs}}'");
Expand Down
24 changes: 24 additions & 0 deletions src/lib/onboard/gateway-start-failure.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it, vi } from "vitest";

import { reportLegacyGatewayStartResultFailure } from "./gateway-start-failure";

describe("reportLegacyGatewayStartResultFailure", () => {
it("classifies Docker-unreachable output after stripping ANSI sequences (#2347)", () => {
const log = vi.fn();
const output = [
"\x1b[31mError: Failed to create Docker client.\x1b[0m",
"\x1b[33mSocket not found: /var/run/docker.sock\x1b[0m",
].join("\n");

expect(reportLegacyGatewayStartResultFailure(output, log)).toEqual({
kind: "docker_unreachable",
});
expect(log).toHaveBeenCalledWith(
expect.stringContaining("Gateway start returned before healthy"),
);
expect(log.mock.calls[0][0]).not.toContain("\x1b");
});
});
40 changes: 40 additions & 0 deletions src/lib/onboard/gateway-start-failure.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { compactText } from "../core/url-utils";
import { redact } from "../security/redact";
import { classifyGatewayStartFailure } from "../validation";

const ANSI_RE = /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/g;

export function reportLegacyGatewayStartResultFailure(
output: string,
log: (message: string) => void,
) {
const cleanedOutput = String(output || "").replace(ANSI_RE, "");
const lines = redact(cleanedOutput)
.split("\n")
.map((l) => compactText(l))
.filter(Boolean)
.map((l) => ` ${l}`);
if (lines.length > 0) {
log(` Gateway start returned before healthy:\n${lines.join("\n")}`);
}
return classifyGatewayStartFailure(cleanedOutput);
}

export function printDockerDaemonRecovery(
printError: (message?: string) => void,
platform: NodeJS.Platform = process.platform,
): void {
printError(" Docker daemon is not running — cannot start the gateway.");
printError("");
printError(" Start Docker, then rerun `nemoclaw onboard`:");
if (platform === "darwin") {
printError(" colima start # or start Docker Desktop");
} else if (platform === "linux") {
printError(" sudo systemctl start docker");
} else {
printError(" Start the Docker daemon.");
}
}
56 changes: 56 additions & 0 deletions src/lib/validation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
classifyValidationFailure,
classifyApplyFailure,
classifySandboxCreateFailure,
classifyGatewayStartFailure,
validateNvidiaApiKeyValue,
isSafeModelId,
isNvcfFunctionNotFoundForAccount,
Expand Down Expand Up @@ -260,6 +261,61 @@ describe("classifySandboxCreateFailure", () => {
});
});

describe("classifyGatewayStartFailure", () => {
// Regression: NemoClaw #2347. When Colima is stopped on macOS, the
// openshell gateway-start stream prints "Failed to create Docker client.
// Socket not found: /var/run/docker.sock" before exiting non-zero. Onboard
// must short-circuit the retry loop with an actionable message instead of
// burning ~15 minutes on health polls against a dead socket.
it("detects colima-stopped signature on macOS (Socket not found)", () => {
const output = [
" Error: Failed to create Docker client.",
" Socket not found: /var/run/docker.sock",
].join("\n");
expect(classifyGatewayStartFailure(output)).toEqual({ kind: "docker_unreachable" });
});

it("detects dockerd-stopped signature on Linux (Cannot connect to the Docker daemon)", () => {
const output =
"Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?";
expect(classifyGatewayStartFailure(output)).toEqual({ kind: "docker_unreachable" });
});

it("detects the standalone 'Failed to create Docker client' error marker", () => {
expect(classifyGatewayStartFailure("Failed to create Docker client")).toEqual({
kind: "docker_unreachable",
});
});

it("does not match historical mentions of 'Failed to create Docker client'", () => {
const output =
"client created successfully; previous Failed to create Docker client issue fixed";
expect(classifyGatewayStartFailure(output)).toEqual({ kind: "unknown" });
});

it("detects free-form 'docker daemon is not running' wording", () => {
expect(classifyGatewayStartFailure("the docker daemon is not running on this host")).toEqual({
kind: "docker_unreachable",
});
});

it("returns unknown for healthy-but-slow output (should not short-circuit)", () => {
// Real output seen during a slow first-time k3s bootstrap — the retry
// loop must stay engaged for these, so we must not misclassify them.
const output = [
"Applying HelmChart openshell",
"openshell-0 still starting",
"Observed pod startup duration 90s",
].join("\n");
expect(classifyGatewayStartFailure(output)).toEqual({ kind: "unknown" });
});

it("returns unknown for empty or missing output", () => {
expect(classifyGatewayStartFailure("")).toEqual({ kind: "unknown" });
expect(classifyGatewayStartFailure()).toEqual({ kind: "unknown" });
});
});

describe("validateNvidiaApiKeyValue", () => {
it("returns null for valid key", () => {
expect(validateNvidiaApiKeyValue("nvapi-abc123")).toBeNull();
Expand Down
Loading
Loading