diff --git a/.github/workflows/regression-e2e.yaml b/.github/workflows/regression-e2e.yaml index e00fb34f750..754030f4e02 100644 --- a/.github/workflows/regression-e2e.yaml +++ b/.github/workflows/regression-e2e.yaml @@ -21,7 +21,7 @@ on: jobs: description: >- Comma-separated regression job names to run (empty = all). - Valid: dashboard-remote-bind-e2e,gateway-health-honest-e2e,gateway-drift-preflight-e2e,openshell-version-pin-e2e,onboard-inference-smoke-e2e,strict-tool-call-probe-e2e,model-router-provider-routed-inference-e2e,openclaw-plugin-runtime-exdev-e2e + Valid: dashboard-remote-bind-e2e,gateway-health-honest-e2e,docker-unreachable-gateway-start-e2e,gateway-drift-preflight-e2e,openshell-version-pin-e2e,onboard-inference-smoke-e2e,strict-tool-call-probe-e2e,model-router-provider-routed-inference-e2e,openclaw-plugin-runtime-exdev-e2e required: false type: string default: "" @@ -46,6 +46,7 @@ jobs: outputs: dashboard: ${{ steps.select.outputs.dashboard }} gateway: ${{ steps.select.outputs.gateway }} + docker_unreachable_gateway_start: ${{ steps.select.outputs.docker_unreachable_gateway_start }} gateway_drift_preflight: ${{ steps.select.outputs.gateway_drift_preflight }} openshell_version_pin: ${{ steps.select.outputs.openshell_version_pin }} onboard_inference_smoke: ${{ steps.select.outputs.onboard_inference_smoke }} @@ -79,6 +80,12 @@ jobs: echo "gateway=false" >> "$GITHUB_OUTPUT" fi + if [ -z "$normalized" ] || includes_job "docker-unreachable-gateway-start-e2e"; then + echo "docker_unreachable_gateway_start=true" >> "$GITHUB_OUTPUT" + else + echo "docker_unreachable_gateway_start=false" >> "$GITHUB_OUTPUT" + fi + if [ -z "$normalized" ] || includes_job "gateway-drift-preflight-e2e"; then echo "gateway_drift_preflight=true" >> "$GITHUB_OUTPUT" else @@ -180,6 +187,43 @@ jobs: /tmp/nemoclaw-e2e-gateway-health-honest-process.log if-no-files-found: ignore + # ── Docker-unreachable gateway-start E2E ─────────────────── + # Coverage guard for #2347's fallback path. Preflight catches Docker + # daemon outages before step [2/8] on the normal path; this focused + # regression simulates Docker dying during `openshell gateway start` and + # verifies onboard exits before health polling with Docker recovery advice. + docker-unreachable-gateway-start-e2e: + needs: select_regression_jobs + if: >- + github.repository == 'NVIDIA/NemoClaw' && + needs.select_regression_jobs.outputs.docker_unreachable_gateway_start == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: "22" + + - name: Run Docker-unreachable gateway-start E2E test + env: + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + run: bash test/e2e/test-docker-unreachable-gateway-start.sh + + - name: Upload Docker-unreachable gateway-start logs on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: docker-unreachable-gateway-start-logs + path: | + /tmp/nemoclaw-e2e-docker-unreachable-gateway-start.log + /tmp/nemoclaw-e2e-docker-unreachable-gateway-start-node.log + /tmp/nemoclaw-e2e-docker-unreachable-gateway-start-openshell.log + if-no-files-found: ignore # ── OpenShell version-pin E2E ────────────────────────────── # Coverage guard for #3474. If a host has sticky OpenShell 0.0.45 on PATH diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index e61222b2f32..d24a5a8a0f7 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -467,6 +467,8 @@ const { trackChildExit } = require("./onboard/child-exit-tracker") as typeof import("./onboard/child-exit-tracker"); const { reportDockerDriverGatewayStartFailure } = require("./onboard/docker-driver-gateway-failure") as typeof import("./onboard/docker-driver-gateway-failure"); +const { printDockerDaemonRecovery, reportLegacyGatewayStartResultFailure } = + require("./onboard/gateway-start-failure") as typeof import("./onboard/gateway-start-failure"); const dockerDriverGatewayEnv: typeof import("./onboard/docker-driver-gateway-env") = require("./onboard/docker-driver-gateway-env"); const { getDockerDriverGatewayEndpoint } = dockerDriverGatewayEnv; @@ -1257,6 +1259,7 @@ function destroyGateway( type FinalGatewayStartFailureOptions = { retries: number; + dockerUnreachable?: boolean; collectDiagnostics?: () => string | null | undefined; cleanupGateway?: () => void; exitProcess?: (code: number) => never; @@ -1265,6 +1268,7 @@ type FinalGatewayStartFailureOptions = { function handleFinalGatewayStartFailure({ retries, + dockerUnreachable = false, collectDiagnostics = () => runCaptureOpenshell(["doctor", "logs", "--name", GATEWAY_NAME], { ignoreError: true, @@ -1274,6 +1278,11 @@ function handleFinalGatewayStartFailure({ exitProcess = (code) => process.exit(code), printError = (message = "") => console.error(message), }: FinalGatewayStartFailureOptions): never { + if (dockerUnreachable) { + printDockerDaemonRecovery(printError); + return exitProcess(1); + } + printError(` Gateway failed to start after ${retries + 1} attempts.`); printError(" Gateway state preserved until diagnostics are collected."); printError(""); @@ -2244,22 +2253,16 @@ async function startGatewayWithOptions( ); } - // When a stale gateway is detected (metadata exists but container is gone, - // e.g. after a Docker/Colima restart), skip the destroy — `gateway start` - // can recover the container without wiping metadata and mTLS certs. - // The retry loop below will destroy only if start genuinely fails. if (hasStaleGateway(gatewaySnapshot.gwInfo)) { console.log(" Stale gateway detected — attempting restart without destroy..."); } - // Clear stale SSH host keys from previous gateway (fixes #768) try { const { execFileSync } = require("child_process"); execFileSync("ssh-keygen", ["-R", `openshell-${GATEWAY_NAME}`], { stdio: "ignore" }); } catch { /* ssh-keygen -R may fail if entry doesn't exist — safe to ignore */ } - // Also purge any known_hosts entries matching the gateway hostname pattern const knownHostsPath = path.join(os.homedir(), ".ssh", "known_hosts"); try { const kh = fs.readFileSync(knownHostsPath, "utf8"); @@ -2270,9 +2273,6 @@ async function startGatewayWithOptions( } const gwArgs = ["--name", GATEWAY_NAME, "--port", getGatewayPortArg()]; - // On NVIDIA hosts, pass --gpu unless the user explicitly opted out. This - // makes direct CUDA tools available in the sandbox by default while still - // supporting host-side inference providers. if (gpuPassthrough) { gwArgs.push("--gpu"); } @@ -2281,12 +2281,8 @@ async function startGatewayWithOptions( console.log(` Using pinned OpenShell gateway image: ${gatewayEnv.OPENSHELL_CLUSTER_IMAGE}`); } - // Retry gateway start with exponential backoff. On some hosts (Horde VMs, - // first-run environments) the embedded k3s needs more time than OpenShell's - // internal health-check window allows. Retrying after a clean destroy lets - // the second attempt benefit from cached images and cleaner cgroup state. - // See: https://github.com/NVIDIA/OpenShell/issues/433 const retries = exitOnFailure ? 2 : 0; + let dockerUnreachable = false; try { await pRetry( async () => { @@ -2298,13 +2294,15 @@ async function startGatewayWithOptions( }, ); if (startResult.status !== 0) { - const lines = String(redact(startResult.output || "")) - .split("\n") - .map((l) => compactText(l.replace(ANSI_RE, ""))) - .filter(Boolean) - .map((l) => ` ${l}`); - if (lines.length > 0) { - console.log(` Gateway start returned before healthy:\n${lines.join("\n")}`); + const failure = reportLegacyGatewayStartResultFailure( + startResult.output || "", + console.log, + ); + if (failure.kind === "docker_unreachable") { + dockerUnreachable = true; + throw new pRetry.AbortError( + "Docker daemon is not reachable (gateway cannot start).", + ); } } console.log(" Waiting for gateway health..."); @@ -2365,7 +2363,7 @@ async function startGatewayWithOptions( ); } catch { if (exitOnFailure) { - handleFinalGatewayStartFailure({ retries }); + handleFinalGatewayStartFailure({ retries, dockerUnreachable }); } throw new Error("Gateway failed to start"); } diff --git a/src/lib/onboard/docker-driver-gateway-failure.test.ts b/src/lib/onboard/docker-driver-gateway-failure.test.ts index 194e5764edf..60cf6497519 100644 --- a/src/lib/onboard/docker-driver-gateway-failure.test.ts +++ b/src/lib/onboard/docker-driver-gateway-failure.test.ts @@ -30,11 +30,9 @@ describe("reportDockerDriverGatewayStartFailure (#3111)", () => { beforeEach(() => { errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); // Stub process.exit so assertions can still run. - exitSpy = vi - .spyOn(process, "exit") - .mockImplementation(((code?: number) => { - throw new Error(`process.exit(${code})`); - }) as never); + exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code})`); + }) as never); }); afterEach(() => { @@ -50,7 +48,9 @@ describe("reportDockerDriverGatewayStartFailure (#3111)", () => { { exitOnFailure: false }, ), ).not.toThrow(); - const joined = errSpy.mock.calls.map((c: string[]) => c.join(" ")).join("\n"); + const joined = errSpy.mock.calls + .map((c: string[]) => c.join(" ")) + .join("\n"); expect(joined).toContain("Docker-driver gateway failed to start"); expect(joined).toContain("Troubleshooting:"); expect(joined).toContain("tail -100 /tmp/nonexistent-gateway.log"); @@ -67,7 +67,9 @@ describe("reportDockerDriverGatewayStartFailure (#3111)", () => { }), { exitOnFailure: false }, ); - const joined = errSpy.mock.calls.map((c: string[]) => c.join(" ")).join("\n"); + const joined = errSpy.mock.calls + .map((c: string[]) => c.join(" ")) + .join("\n"); expect(joined).toContain( "Gateway process exited with code 127 before becoming ready", ); @@ -79,7 +81,9 @@ describe("reportDockerDriverGatewayStartFailure (#3111)", () => { makeExitState(), { exitOnFailure: false }, ); - const joined = errSpy.mock.calls.map((c: string[]) => c.join(" ")).join("\n"); + const joined = errSpy.mock.calls + .map((c: string[]) => c.join(" ")) + .join("\n"); expect(joined).not.toContain("before becoming ready"); }); @@ -94,7 +98,9 @@ describe("reportDockerDriverGatewayStartFailure (#3111)", () => { reportDockerDriverGatewayStartFailure(log, makeExitState(), { exitOnFailure: false, }); - const joined = errSpy.mock.calls.map((c: string[]) => c.join(" ")).join("\n"); + const joined = errSpy.mock.calls + .map((c: string[]) => c.join(" ")) + .join("\n"); expect(joined).toContain("Gateway log tail"); expect(joined).toContain("GLIBC_2.38 not found"); } finally { @@ -102,6 +108,30 @@ describe("reportDockerDriverGatewayStartFailure (#3111)", () => { } }); + it("prints Docker runtime recovery guidance when gateway log shows Docker is unreachable (#2347)", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "gw-fail-")); + const log = path.join(dir, "openshell-gateway.log"); + fs.writeFileSync( + log, + [ + "Error: Failed to create Docker client.", + "Socket not found: /var/run/docker.sock", + ].join("\n"), + ); + try { + reportDockerDriverGatewayStartFailure(log, makeExitState(), { + exitOnFailure: false, + }); + const joined = errSpy.mock.calls + .map((c: string[]) => c.join(" ")) + .join("\n"); + expect(joined).toContain("Docker daemon is not running"); + expect(joined).toContain("Start Docker, then rerun `nemoclaw onboard`"); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + it("calls process.exit(1) when exitOnFailure is true", () => { expect(() => reportDockerDriverGatewayStartFailure( diff --git a/src/lib/onboard/docker-driver-gateway-failure.ts b/src/lib/onboard/docker-driver-gateway-failure.ts index cfddf9b2b3b..ad4a95eb7d4 100644 --- a/src/lib/onboard/docker-driver-gateway-failure.ts +++ b/src/lib/onboard/docker-driver-gateway-failure.ts @@ -25,8 +25,10 @@ import fs from "node:fs"; import { redact } from "../security/redact"; +import { classifyGatewayStartFailure } from "../validation"; import type { ChildExitState } from "./child-exit-tracker"; +import { printDockerDaemonRecovery } from "./gateway-start-failure"; export type ReportDockerDriverGatewayStartFailureOpts = { /** @@ -71,6 +73,9 @@ export function reportDockerDriverGatewayStartFailure( console.error(" Gateway log tail:"); for (const line of tail.split("\n")) console.error(` ${redact(line)}`); } + if (classifyGatewayStartFailure(tail).kind === "docker_unreachable") { + printDockerDaemonRecovery(console.error); + } console.error(" Troubleshooting:"); console.error(` tail -100 ${logPath}`); console.error(" docker info --format '{{json .CDISpecDirs}}'"); diff --git a/src/lib/onboard/gateway-start-failure.test.ts b/src/lib/onboard/gateway-start-failure.test.ts new file mode 100644 index 00000000000..b4bdb789cdf --- /dev/null +++ b/src/lib/onboard/gateway-start-failure.test.ts @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { classifyGatewayStartFailure } from "../validation"; +import { reportLegacyGatewayStartResultFailure } from "./gateway-start-failure"; + +describe("classifyGatewayStartFailure", () => { + // Regression: NemoClaw #2347. When Colima is stopped on macOS, the + // openshell gateway-start stream prints "Failed to create Docker client. + // Socket not found: /var/run/docker.sock" before exiting non-zero. Onboard + // must short-circuit the retry loop with an actionable message instead of + // burning ~15 minutes on health polls against a dead socket. + it("detects colima-stopped signature on macOS (Socket not found)", () => { + const output = [ + " Error: Failed to create Docker client.", + " Socket not found: /var/run/docker.sock", + ].join("\n"); + expect(classifyGatewayStartFailure(output)).toEqual({ kind: "docker_unreachable" }); + }); + + it("detects dockerd-stopped signature on Linux (Cannot connect to the Docker daemon)", () => { + const output = + "Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?"; + expect(classifyGatewayStartFailure(output)).toEqual({ kind: "docker_unreachable" }); + }); + + it("detects the standalone 'Failed to create Docker client' error marker", () => { + expect(classifyGatewayStartFailure("Failed to create Docker client")).toEqual({ + kind: "docker_unreachable", + }); + }); + + it("does not match historical mentions of 'Failed to create Docker client'", () => { + const output = + "client created successfully; previous Failed to create Docker client issue fixed"; + expect(classifyGatewayStartFailure(output)).toEqual({ kind: "unknown" }); + }); + + it("detects free-form 'docker daemon is not running' wording", () => { + expect(classifyGatewayStartFailure("the docker daemon is not running on this host")).toEqual({ + kind: "docker_unreachable", + }); + }); + + it("returns unknown for healthy-but-slow output (should not short-circuit)", () => { + // Real output seen during a slow first-time k3s bootstrap — the retry + // loop must stay engaged for these, so we must not misclassify them. + const output = [ + "Applying HelmChart openshell", + "openshell-0 still starting", + "Observed pod startup duration 90s", + ].join("\n"); + expect(classifyGatewayStartFailure(output)).toEqual({ kind: "unknown" }); + }); + + it("returns unknown for empty or missing output", () => { + expect(classifyGatewayStartFailure("")).toEqual({ kind: "unknown" }); + expect(classifyGatewayStartFailure()).toEqual({ kind: "unknown" }); + }); +}); + +describe("reportLegacyGatewayStartResultFailure", () => { + it("classifies Docker-unreachable output after stripping ANSI sequences (#2347)", () => { + const log = vi.fn(); + const output = [ + "\x1b[31mError: Failed to create Docker client.\x1b[0m", + "\x1b[33mSocket not found: /var/run/docker.sock\x1b[0m", + ].join("\n"); + + expect(reportLegacyGatewayStartResultFailure(output, log)).toEqual({ + kind: "docker_unreachable", + }); + expect(log).toHaveBeenCalledWith( + expect.stringContaining("Gateway start returned before healthy"), + ); + expect(log.mock.calls[0][0]).not.toContain("\x1b"); + }); +}); diff --git a/src/lib/onboard/gateway-start-failure.ts b/src/lib/onboard/gateway-start-failure.ts new file mode 100644 index 00000000000..7154db54c47 --- /dev/null +++ b/src/lib/onboard/gateway-start-failure.ts @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { compactText } from "../core/url-utils"; +import { redact } from "../security/redact"; +import { classifyGatewayStartFailure } from "../validation"; + +const ANSI_RE = /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/g; + +export function reportLegacyGatewayStartResultFailure( + output: string, + log: (message: string) => void, +) { + const cleanedOutput = String(output || "").replace(ANSI_RE, ""); + const lines = redact(cleanedOutput) + .split("\n") + .map((l) => compactText(l)) + .filter(Boolean) + .map((l) => ` ${l}`); + if (lines.length > 0) { + log(` Gateway start returned before healthy:\n${lines.join("\n")}`); + } + return classifyGatewayStartFailure(cleanedOutput); +} + +export function printDockerDaemonRecovery( + printError: (message?: string) => void, + platform: NodeJS.Platform = process.platform, +): void { + printError(" Docker daemon is not running — cannot start the gateway."); + printError(""); + printError(" Start Docker, then rerun `nemoclaw onboard`:"); + if (platform === "darwin") { + printError(" colima start # or start Docker Desktop"); + } else if (platform === "linux") { + printError(" sudo systemctl start docker"); + } else { + printError(" Start the Docker daemon."); + } +} diff --git a/src/lib/validation.ts b/src/lib/validation.ts index 8fb2fb2f493..5ad0a071ef6 100644 --- a/src/lib/validation.ts +++ b/src/lib/validation.ts @@ -17,6 +17,17 @@ export interface SandboxCreateFailure { uploadedToGateway: boolean; } +export interface GatewayStartFailure { + /** + * - `docker_unreachable`: the underlying Docker daemon (Colima on macOS, + * dockerd on Linux) is not responding. Retrying the openshell health + * poll cannot recover from this — the user must start Docker first. + * - `unknown`: any other failure; callers should fall through to the + * normal retry/health-wait behavior. + */ + kind: "docker_unreachable" | "unknown"; +} + export function classifyValidationFailure({ httpStatus = 0, curlStatus = 0, @@ -87,6 +98,34 @@ export function classifySandboxCreateFailure(output = ""): SandboxCreateFailure return { kind: "unknown", uploadedToGateway }; } +/** + * Classify a non-zero `openshell gateway start` result so the onboard retry + * loop can short-circuit on unrecoverable failures. + * + * The only case we special-case today is "Docker daemon not reachable" — on + * macOS this surfaces as `Socket not found: /var/run/docker.sock` (Colima + * stopped) and on Linux as `Cannot connect to the Docker daemon at + * unix:///var/run/docker.sock. Is the docker daemon running?`. Retrying the + * health poll against a stopped daemon wastes ~5–15 minutes and produces an + * unactionable error at the end; bailing out immediately with a clear + * "start Docker" message is strictly better UX. See NemoClaw #2347. + */ +export function classifyGatewayStartFailure(output = ""): GatewayStartFailure { + const text = String(output || ""); + // Match both macOS (Colima / Docker Desktop) and Linux docker daemon-down + // signatures. The openshell CLI echoes these verbatim from the underlying + // Docker client error when the gateway controller starts. + if ( + /Socket not found:\s*\/var\/run\/docker\.sock/i.test(text) || + /Cannot connect to the Docker daemon/i.test(text) || + /^\s*(?:Error:\s*)?Failed to create Docker client(?:[.:]|\b)/im.test(text) || + /docker daemon.*(is not running|not responding|unreachable)/i.test(text) + ) { + return { kind: "docker_unreachable" }; + } + return { kind: "unknown" }; +} + export function validateNvidiaApiKeyValue( key: string, credentialEnv: string = "NVIDIA_API_KEY", diff --git a/test/e2e/test-docker-unreachable-gateway-start.sh b/test/e2e/test-docker-unreachable-gateway-start.sh new file mode 100755 index 00000000000..bcfee8d3256 --- /dev/null +++ b/test/e2e/test-docker-unreachable-gateway-start.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Regression E2E for #2347's gateway-start fallback: if `openshell gateway +# start` reports that Docker/Colima is unreachable, onboard must abort before +# gateway health polling and print Docker recovery guidance. + +set -euo pipefail + +LOG_FILE="/tmp/nemoclaw-e2e-docker-unreachable-gateway-start.log" +START_LOG="/tmp/nemoclaw-e2e-docker-unreachable-gateway-start-node.log" +TRACE_LOG="/tmp/nemoclaw-e2e-docker-unreachable-gateway-start-openshell.log" +exec > >(tee "$LOG_FILE") 2>&1 + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +pass() { echo -e "${GREEN}[PASS]${NC} $1"; } +info() { echo -e "${YELLOW}[INFO]${NC} $1"; } +diag() { echo -e "${YELLOW}[DIAG]${NC} $1"; } +fail() { + echo -e "${RED}[FAIL]${NC} $1" >&2 + diag "node log tail:" + tail -120 "$START_LOG" 2>/dev/null || true + diag "fake openshell trace:" + cat "$TRACE_LOG" 2>/dev/null || true + exit 1 +} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +TMP_DIR="$(mktemp -d)" +FAKE_BIN="${TMP_DIR}/bin" +mkdir -p "$FAKE_BIN" +: >"$TRACE_LOG" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +cd "$REPO_ROOT" + +info "Preparing CLI build" +if [ ! -d node_modules ]; then + npm ci --ignore-scripts +fi +npm run build:cli + +info "Installing fake openshell that reports Docker socket-not-found during gateway start" +cat >"${FAKE_BIN}/openshell" <<'SHIM' +#!/usr/bin/env bash +set -euo pipefail +TRACE_LOG="${NEMOCLAW_FAKE_OPENSHELL_TRACE:?}" +printf '%s\n' "$*" >>"$TRACE_LOG" + +if [[ "$*" == "--version" ]]; then + printf 'openshell 0.0.44\n' + exit 0 +fi + +if [[ "$*" == "gateway --help" ]]; then + printf 'Commands: start select info destroy remove\n' + exit 0 +fi + +if [[ "$*" == *"gateway"*"start"* ]]; then + printf '%s\n' "__GATEWAY_START__" >>"$TRACE_LOG" + printf 'Error: Failed to create Docker client.\n' + printf 'Socket not found: /var/run/docker.sock\n' + exit 1 +fi + +# These probes are allowed before gateway start when startGateway() checks +# reusable gateway state. After __GATEWAY_START__, they prove the regression: +# onboard fell through into health/status polling instead of aborting. +if [[ "$*" == "status" || "$*" == *"gateway"*"info"* ]]; then + printf 'HEALTH POLL REACHED\n' + exit 0 +fi + +if [[ "$*" == *"gateway"*"select"* || "$*" == *"gateway"*"destroy"* || "$*" == *"gateway"*"remove"* ]]; then + exit 0 +fi + +exit 0 +SHIM +chmod 755 "${FAKE_BIN}/openshell" + +info "Invoking legacy startGateway() path with macOS/x64 platform semantics" +set +e +PATH="${FAKE_BIN}:$PATH" \ + HOME="${TMP_DIR}/home" \ + NEMOCLAW_FAKE_OPENSHELL_TRACE="$TRACE_LOG" \ + NEMOCLAW_NON_INTERACTIVE="1" \ + NEMOCLAW_HEALTH_POLL_COUNT="5" \ + node <<'NODE' >"$START_LOG" 2>&1 +Object.defineProperty(process, "platform", { value: "darwin" }); +Object.defineProperty(process, "arch", { value: "x64" }); + +const { startGateway } = require("./dist/lib/onboard"); + +startGateway(null) + .then(() => { + console.log("__startGateway_returned_successfully__"); + process.exit(0); + }) + .catch((error) => { + console.error("__startGateway_threw__"); + console.error(error && error.stack ? error.stack : error); + process.exit(3); + }); +NODE +NODE_EXIT=$? +set -e + +info "node exit code: ${NODE_EXIT}" + +if [ "$NODE_EXIT" -ne 1 ]; then + fail "startGateway should exit 1 when Docker is unreachable, got ${NODE_EXIT}" +fi +pass "startGateway exited 1 on Docker-unreachable gateway-start output" + +if ! grep -q "Docker daemon is not running" "$START_LOG"; then + fail "Docker recovery guidance was not printed" +fi +pass "Docker recovery guidance was printed" + +if ! grep -q "colima start" "$START_LOG"; then + fail "macOS/Colima recovery hint was not printed" +fi +pass "macOS/Colima recovery hint was printed" + +if grep -q "Waiting for gateway health" "$START_LOG"; then + fail "onboard entered gateway health polling after Docker-unreachable gateway-start output" +fi +pass "gateway health polling message was not printed after Docker-unreachable output" + +if grep -q "HEALTH POLL REACHED" "$START_LOG"; then + fail "gateway status/info probe output appeared after Docker-unreachable gateway-start output" +fi +pass "gateway status/info probe output did not appear in user output" + +POST_START_PROBES="$(awk ' + /__GATEWAY_START__/ { seen = 1; next } + seen && ($0 == "status" || $0 ~ /gateway.*info/) { print } +' "$TRACE_LOG")" +if [ -n "$POST_START_PROBES" ]; then + fail "status/gateway-info probes ran after Docker-unreachable gateway start: ${POST_START_PROBES}" +fi +pass "no status/gateway-info probes ran after Docker-unreachable gateway start" + +if grep -q "openshell doctor logs" "$START_LOG"; then + fail "generic OpenShell diagnostics were printed instead of Docker-specific recovery guidance" +fi +pass "generic OpenShell diagnostics were skipped for Docker-unreachable gateway start" + +echo "" +pass "#2347 gateway-start fallback E2E green" diff --git a/test/onboard.test.ts b/test/onboard.test.ts index 3e2be277547..8c4f4330f99 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -218,6 +218,93 @@ startGateway(null).catch(() => {}); ); }); + it("fast-fails gateway start before health polling when Docker is unreachable (#2347)", testTimeoutOptions(20_000), () => { + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-docker-down-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "gateway-docker-down.cjs"); + const onboardPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "onboard.js")); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync( + path.join(fakeBin, "openshell"), + `#!/usr/bin/env bash +if [[ "$*" == "gateway --help" ]]; then + printf "Commands: start destroy\\n" + exit 0 +fi +if [[ "$*" == *"gateway"*"start"* ]]; then + printf "Error: Failed to create Docker client.\\n" + printf "Socket not found: /var/run/docker.sock\\n" + exit 1 +fi +if [[ "$*" == *"status"* || "$*" == *"gateway"*"info"* ]]; then + printf "HEALTH POLL REACHED\\n" + exit 0 +fi +exit 1 +`, + { mode: 0o755 }, + ); + + const script = ` +const mod = require("module"); +const origLoad = mod._load; +mod._load = function(req, parent, isMain) { + if (req === "p-retry") { + const pRetry = async (fn, opts) => { + try { + return await fn({ attemptNumber: 1, retriesLeft: 0 }); + } catch (e) { + if (!(e instanceof pRetry.AbortError) && opts && opts.onFailedAttempt) { + opts.onFailedAttempt(Object.assign(e, { attemptNumber: 1, retriesLeft: 0 })); + } + throw e; + } + }; + pRetry.AbortError = class AbortError extends Error {}; + return pRetry; + } + return origLoad.call(this, req, parent, isMain); +}; +Object.defineProperty(process, "platform", { value: "darwin" }); +Object.defineProperty(process, "arch", { value: "x64" }); +const { startGateway } = require(${onboardPath}); +startGateway(null).catch(() => {}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_HEALTH_POLL_COUNT: "5", + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 1, `unexpected exit code; stderr:\n${result.stderr}`); + assert.ok( + result.stderr.includes("Docker daemon is not running"), + `expected Docker recovery guidance in stderr:\n${result.stderr}`, + ); + assert.ok( + result.stderr.includes("colima start"), + `expected macOS Docker start hint in stderr:\n${result.stderr}`, + ); + assert.ok( + !result.stdout.includes("Waiting for gateway health"), + `health polling should not start after Docker-unreachable output:\n${result.stdout}`, + ); + assert.ok( + !result.stdout.includes("HEALTH POLL REACHED"), + `gateway status/info probes should not run after Docker-unreachable output:\n${result.stdout}`, + ); + }); + it("normalizes sandbox name hints from the environment", () => { const previous = process.env.NEMOCLAW_SANDBOX_NAME; process.env.NEMOCLAW_SANDBOX_NAME = " My-Assistant ";