Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -682,6 +682,7 @@ const {
classifyValidationFailure,
classifyApplyFailure,
classifySandboxCreateFailure,
classifyGatewayStartFailure,
validateNvidiaApiKeyValue,
isSafeModelId,
isNvcfFunctionNotFoundForAccount,
Expand Down Expand Up @@ -3011,6 +3012,7 @@ async function startGatewayWithOptions(_gpu, { exitOnFailure = true } = {}) {
// the second attempt benefit from cached images and cleaner cgroup state.
// See: https://github.com/NVIDIA/OpenShell/issues/433
const retries = exitOnFailure ? 2 : 0;
let dockerUnreachable = false;
try {
await pRetry(
async () => {
Expand All @@ -3030,6 +3032,18 @@ async function startGatewayWithOptions(_gpu, { exitOnFailure = true } = {}) {
if (lines.length > 0) {
console.log(` Gateway start returned before healthy:\n${lines.join("\n")}`);
}
// Fast-fail when the underlying Docker daemon is unreachable
// (e.g. `colima stop` on macOS). Retrying the health poll against
// a dead socket wastes ~5–15 minutes and produces an unactionable
// error; short-circuit with a "start Docker" message instead.
// See NemoClaw #2347.
const failure = classifyGatewayStartFailure(startResult.output || "");
if (failure.kind === "docker_unreachable") {
dockerUnreachable = true;
throw new pRetry.AbortError(
"Docker daemon is not reachable (gateway cannot start).",
);
}
}
console.log(" Waiting for gateway health...");
const healthWait = getGatewayHealthWaitConfig(
Expand Down Expand Up @@ -3084,6 +3098,21 @@ async function startGatewayWithOptions(_gpu, { exitOnFailure = true } = {}) {
);
} catch {
if (exitOnFailure) {
if (dockerUnreachable) {
// Unrecoverable-by-retry: the Docker daemon is stopped. Give the
// user the exact command to run instead of dumping openshell logs.
console.error(" Docker daemon is not running — cannot start the gateway.");
console.error("");
console.error(" Start Docker, then rerun `nemoclaw onboard`:");
if (process.platform === "darwin") {
console.error(" colima start # or start Docker Desktop");
} else if (process.platform === "linux") {
console.error(" sudo systemctl start docker");
} else {
console.error(" Start the Docker daemon.");
}
process.exit(1);
}
console.error(` Gateway failed to start after ${retries + 1} attempts.`);
console.error(" Gateway state preserved for diagnostics.");
console.error("");
Expand Down Expand Up @@ -6816,6 +6845,7 @@ module.exports = {
buildSandboxConfigSyncScript,
compactText,
copyBuildContextDir,
classifyGatewayStartFailure,
classifySandboxCreateFailure,
configureWebSearch,
createSandbox,
Expand Down
50 changes: 50 additions & 0 deletions src/lib/validation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
classifyValidationFailure,
classifyApplyFailure,
classifySandboxCreateFailure,
classifyGatewayStartFailure,
validateNvidiaApiKeyValue,
isSafeModelId,
isNvcfFunctionNotFoundForAccount,
Expand Down Expand Up @@ -184,6 +185,55 @@ describe("classifySandboxCreateFailure", () => {
});
});

describe("classifyGatewayStartFailure", () => {
// Regression: NemoClaw #2347. When Colima is stopped on macOS, the
// openshell gateway-start stream prints "Failed to create Docker client.
// Socket not found: /var/run/docker.sock" before exiting non-zero. Onboard
// must short-circuit the retry loop with an actionable message instead of
// burning ~15 minutes on health polls against a dead socket.
it("detects colima-stopped signature on macOS (Socket not found)", () => {
const output = [
" Error: Failed to create Docker client.",
" Socket not found: /var/run/docker.sock",
].join("\n");
expect(classifyGatewayStartFailure(output)).toEqual({ kind: "docker_unreachable" });
});

it("detects dockerd-stopped signature on Linux (Cannot connect to the Docker daemon)", () => {
const output =
"Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?";
expect(classifyGatewayStartFailure(output)).toEqual({ kind: "docker_unreachable" });
});

it("detects the standalone 'Failed to create Docker client' marker", () => {
expect(classifyGatewayStartFailure("Failed to create Docker client")).toEqual({
kind: "docker_unreachable",
});
});

it("detects free-form 'docker daemon is not running' wording", () => {
expect(classifyGatewayStartFailure("the docker daemon is not running on this host")).toEqual({
kind: "docker_unreachable",
});
});

it("returns unknown for healthy-but-slow output (should not short-circuit)", () => {
// Real output seen during a slow first-time k3s bootstrap — the retry
// loop must stay engaged for these, so we must not misclassify them.
const output = [
"Applying HelmChart openshell",
"openshell-0 still starting",
"Observed pod startup duration 90s",
].join("\n");
expect(classifyGatewayStartFailure(output)).toEqual({ kind: "unknown" });
});

it("returns unknown for empty or missing output", () => {
expect(classifyGatewayStartFailure("")).toEqual({ kind: "unknown" });
expect(classifyGatewayStartFailure()).toEqual({ kind: "unknown" });
});
});

describe("validateNvidiaApiKeyValue", () => {
it("returns null for valid key", () => {
expect(validateNvidiaApiKeyValue("nvapi-abc123")).toBeNull();
Expand Down
39 changes: 39 additions & 0 deletions src/lib/validation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,17 @@ export interface SandboxCreateFailure {
uploadedToGateway: boolean;
}

export interface GatewayStartFailure {
/**
* - `docker_unreachable`: the underlying Docker daemon (Colima on macOS,
* dockerd on Linux) is not responding. Retrying the openshell health
* poll cannot recover from this — the user must start Docker first.
* - `unknown`: any other failure; callers should fall through to the
* normal retry/health-wait behavior.
*/
kind: "docker_unreachable" | "unknown";
}

export function classifyValidationFailure({
httpStatus = 0,
curlStatus = 0,
Expand Down Expand Up @@ -78,6 +89,34 @@ export function classifySandboxCreateFailure(output = ""): SandboxCreateFailure
return { kind: "unknown", uploadedToGateway };
}

/**
* Classify a non-zero `openshell gateway start` result so the onboard retry
* loop can short-circuit on unrecoverable failures.
*
* The only case we special-case today is "Docker daemon not reachable" — on
* macOS this surfaces as `Socket not found: /var/run/docker.sock` (Colima
* stopped) and on Linux as `Cannot connect to the Docker daemon at
* unix:///var/run/docker.sock. Is the docker daemon running?`. Retrying the
* health poll against a stopped daemon wastes ~5–15 minutes and produces an
* unactionable error at the end; bailing out immediately with a clear
* "start Docker" message is strictly better UX. See NemoClaw #2347.
*/
export function classifyGatewayStartFailure(output = ""): GatewayStartFailure {
const text = String(output || "");
// Match both macOS (Colima / Docker Desktop) and Linux docker daemon-down
// signatures. The openshell CLI echoes these verbatim from the underlying
// Docker client error when the gateway controller starts.
if (
/Socket not found:\s*\/var\/run\/docker\.sock/i.test(text) ||
/Cannot connect to the Docker daemon/i.test(text) ||
/Failed to create Docker client/i.test(text) ||
/docker daemon.*(is not running|not responding|unreachable)/i.test(text)
) {
return { kind: "docker_unreachable" };
}
return { kind: "unknown" };
}

export function validateNvidiaApiKeyValue(key: string): string | null {
if (!key) {
return " NVIDIA API Key is required.";
Expand Down
Loading