From 2490607f2e8c19f46f8841a5ed488a3f1c79816e Mon Sep 17 00:00:00 2001 From: Yanyun Liao Date: Fri, 17 Apr 2026 15:33:06 +0800 Subject: [PATCH 1/3] fix(onboard): add gateway start timeout and increase health poll window (#1830) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defensive improvements to gateway startup: 1. streamGatewayStart fallback timeout: Add a 600s hard timeout (configurable via NEMOCLAW_GATEWAY_START_TIMEOUT) that kills the openshell gateway start child process if it never exits. Prevents NemoClaw from hanging indefinitely when Docker daemon is unresponsive or k3s enters a restart loop. 2. Increase post-start health poll defaults from 5×2s=10s to 12×5s=60s on x86. After openshell gateway start returns (container healthy at the Docker layer), the application-layer connectivity check (gRPC, TLS handshake, port mapping) may need additional time. 60s provides a reasonable buffer. ARM64 defaults (30×10s=300s) unchanged. All values remain overridable via NEMOCLAW_HEALTH_POLL_COUNT and NEMOCLAW_HEALTH_POLL_INTERVAL. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Yanyun Liao --- src/lib/onboard.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index d806d000a68..78e42c5c9f6 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -381,14 +381,26 @@ function streamGatewayStart(command, env = process.env) { }, 5000); heartbeatTimer.unref?.(); + // Hard timeout to prevent indefinite hangs if the openshell process + // never exits (e.g. Docker daemon unresponsive, k3s restart loop). (#1830) + const GATEWAY_START_TIMEOUT = envInt("NEMOCLAW_GATEWAY_START_TIMEOUT", 600) * 1000; + const killTimer = setTimeout(() => { + lines.push("[NemoClaw] Gateway start timed out — killing process."); + child.kill("SIGTERM"); + finish({ status: 1, output: lines.join("\n") }); + }, GATEWAY_START_TIMEOUT); + killTimer.unref?.(); + return new Promise((resolve) => { resolvePromise = resolve; child.on("error", (error) => { + clearTimeout(killTimer); const detail = error?.message || String(error); lines.push(detail); finish({ status: 1, output: lines.join("\n") }); }); child.on("close", (code) => { + clearTimeout(killTimer); finish({ status: code ?? 1, output: lines.join("\n") }); }); }); @@ -2595,8 +2607,11 @@ async function startGatewayWithOptions(_gpu, { exitOnFailure = true } = {}) { // ARM64 (e.g. Raspberry Pi) needs more time: k3s takes 90-180s to init const isArm64 = process.arch === "arm64"; - const healthPollCount = envInt("NEMOCLAW_HEALTH_POLL_COUNT", isArm64 ? 30 : 5); - const healthPollInterval = envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", isArm64 ? 10 : 2); + // After openshell gateway start returns (container HEALTHY at Layer 1), + // poll application-layer connectivity (Layer 2: gRPC, TLS, port mapping). + // 60s default gives enough buffer for gRPC init and TLS handshake. (#1830) + const healthPollCount = envInt("NEMOCLAW_HEALTH_POLL_COUNT", isArm64 ? 30 : 12); + const healthPollInterval = envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", isArm64 ? 10 : 5); for (let i = 0; i < healthPollCount; i++) { // Ensure the gateway is selected before each probe (non-TTY environments // like ARM64 may not have it selected automatically) From 4d90b47638d0c272950ba0ad2d6e4896fd2bf375 Mon Sep 17 00:00:00 2001 From: Yanyun Liao Date: Fri, 17 Apr 2026 15:46:33 +0800 Subject: [PATCH 2/3] docs(onboard): add JSDoc to streamGatewayStart and startGatewayWithOptions Co-Authored-By: Claude Opus 4.6 Signed-off-by: Yanyun Liao --- src/lib/onboard.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 78e42c5c9f6..fa6bd7ab8bd 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -269,6 +269,7 @@ function repairRecordedSandbox(sandboxName) { const { streamSandboxCreate } = sandboxCreateStream; +/** Spawn `openshell gateway start` and stream its output with progress heartbeats. */ function streamGatewayStart(command, env = process.env) { const child = spawn("bash", ["-lc", command], { cwd: ROOT, @@ -2524,6 +2525,7 @@ async function preflight() { // ── Step 2: Gateway ────────────────────────────────────────────── +/** Start the OpenShell gateway with retry logic and post-start health polling. */ async function startGatewayWithOptions(_gpu, { exitOnFailure = true } = {}) { step(2, 8, "Starting OpenShell gateway"); From 9a7b255cfd31f9345cbf1a90113580fbd905fd2d Mon Sep 17 00:00:00 2001 From: Yanyun Liao Date: Fri, 17 Apr 2026 15:47:47 +0800 Subject: [PATCH 3/3] fix(onboard): wait for child exit after timeout kill before resolving (#1830) On timeout, send SIGTERM and let the `close` event resolve the promise so the child has actually exited before the caller proceeds to retry. If SIGTERM is ignored, escalate to SIGKILL after 10s. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Yanyun Liao --- src/lib/onboard.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index fa6bd7ab8bd..0a3c0f71d91 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -384,11 +384,18 @@ function streamGatewayStart(command, env = process.env) { // Hard timeout to prevent indefinite hangs if the openshell process // never exits (e.g. Docker daemon unresponsive, k3s restart loop). (#1830) + // On timeout, send SIGTERM and let the `close` event resolve the promise + // so the child has actually exited before the caller proceeds to retry. const GATEWAY_START_TIMEOUT = envInt("NEMOCLAW_GATEWAY_START_TIMEOUT", 600) * 1000; + let killedByTimeout = false; const killTimer = setTimeout(() => { + killedByTimeout = true; lines.push("[NemoClaw] Gateway start timed out — killing process."); child.kill("SIGTERM"); - finish({ status: 1, output: lines.join("\n") }); + // If SIGTERM is ignored, force-kill after 10s. + setTimeout(() => { + if (!settled) child.kill("SIGKILL"); + }, 10_000).unref?.(); }, GATEWAY_START_TIMEOUT); killTimer.unref?.(); @@ -402,7 +409,8 @@ function streamGatewayStart(command, env = process.env) { }); child.on("close", (code) => { clearTimeout(killTimer); - finish({ status: code ?? 1, output: lines.join("\n") }); + const exitCode = killedByTimeout ? 1 : (code ?? 1); + finish({ status: exitCode, output: lines.join("\n") }); }); }); }