diff --git a/bin/lib/onboard.js b/bin/lib/onboard.js index d5606ae51b7..fa5150a980a 100644 --- a/bin/lib/onboard.js +++ b/bin/lib/onboard.js @@ -9,6 +9,7 @@ const fs = require("fs"); const os = require("os"); const path = require("path"); const { spawn, spawnSync } = require("child_process"); +const pRetry = require("p-retry"); const { ROOT, SCRIPTS, run, runCapture, shellQuote } = require("./runner"); const { getDefaultOllamaModel, @@ -1614,36 +1615,52 @@ async function startGatewayWithOptions(_gpu, { exitOnFailure = true } = {}) { console.log(` Using pinned OpenShell gateway image: ${gatewayEnv.OPENSHELL_CLUSTER_IMAGE}`); } - const startResult = runOpenshell(["gateway", "start", ...gwArgs], { ignoreError: true, env: gatewayEnv }); - if (startResult.status !== 0) { - console.error(" Gateway failed to start. Cleaning up stale state..."); - destroyGateway(); + // Retry gateway start with exponential backoff. On some hosts (Horde VMs, + // first-run environments) the embedded k3s needs more time than OpenShell's + // internal health-check window allows. Retrying after a clean destroy lets + // the second attempt benefit from cached images and cleaner cgroup state. + // See: https://github.com/NVIDIA/OpenShell/issues/433 + const retries = exitOnFailure ? 2 : 0; + try { + await pRetry(() => { + runOpenshell(["gateway", "start", ...gwArgs], { ignoreError: true, env: gatewayEnv }); + + for (let i = 0; i < 5; i++) { + const status = runCaptureOpenshell(["status"], { ignoreError: true }); + const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { ignoreError: true }); + const currentInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true }); + if (isGatewayHealthy(status, namedInfo, currentInfo)) { + return; // success + } + if (i < 4) sleep(2); + } + + throw new Error("Gateway failed to start"); + }, { + retries, + minTimeout: 10_000, + factor: 3, + onFailedAttempt: (err) => { + console.log(` Gateway start attempt ${err.attemptNumber} failed. ${err.retriesLeft} retries left...`); + if (err.retriesLeft > 0 && exitOnFailure) { + destroyGateway(); + } + }, + }); + } catch { if (exitOnFailure) { - console.error(" Stale state removed. Please rerun: nemoclaw onboard"); + console.error(` Gateway failed to start after ${retries + 1} attempts.`); + console.error(" Gateway state preserved for diagnostics."); + console.error(""); + console.error(" Troubleshooting:"); + console.error(" openshell doctor logs --name nemoclaw"); + console.error(" openshell doctor check"); process.exit(1); } throw new Error("Gateway failed to start"); } - for (let i = 0; i < 5; i++) { - const status = runCaptureOpenshell(["status"], { ignoreError: true }); - const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { ignoreError: true }); - const currentInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true }); - if (isGatewayHealthy(status, namedInfo, currentInfo)) { - console.log(" ✓ Gateway is healthy"); - break; - } - if (i === 4) { - console.error(" Gateway health check failed. Cleaning up stale state..."); - destroyGateway(); - if (exitOnFailure) { - console.error(" Stale state removed. Please rerun: nemoclaw onboard"); - process.exit(1); - } - throw new Error("Gateway failed to start"); - } - sleep(2); - } + console.log(" ✓ Gateway is healthy"); // CoreDNS fix — k3s-inside-Docker has broken DNS forwarding on all platforms. const runtime = getContainerRuntime(); diff --git a/package-lock.json b/package-lock.json index 8b9e57e0b91..f2197f7913f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,8 @@ "version": "0.1.0", "license": "Apache-2.0", "dependencies": { - "openclaw": "2026.3.11" + "openclaw": "2026.3.11", + "p-retry": "^4.6.2" }, "bin": { "nemoclaw": "bin/nemoclaw.js" @@ -28,7 +29,7 @@ "vitest": "^4.1.0" }, "engines": { - "node": ">=22.0.0" + "node": ">=22.16.0" } }, "node_modules/@agentclientprotocol/sdk": { @@ -947,6 +948,14 @@ "scripts/actions/documentation" ] }, + "node_modules/@buape/carbon/node_modules/opusscript": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/opusscript/-/opusscript-0.0.8.tgz", + "integrity": "sha512-VSTi1aWFuCkRCVq+tx/BQ5q9fMnQ9pVZ3JU4UHKqTkf0ED3fKEPdr+gKAAl3IA2hj9rrP6iyq3hlcJq3HELtNQ==", + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/@buape/carbon/node_modules/prism-media": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/prism-media/-/prism-media-1.3.5.tgz", @@ -1339,6 +1348,14 @@ "url": "https://github.com/discordjs/discord.js?sponsor" } }, + "node_modules/@discordjs/voice/node_modules/opusscript": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/opusscript/-/opusscript-0.0.8.tgz", + "integrity": "sha512-VSTi1aWFuCkRCVq+tx/BQ5q9fMnQ9pVZ3JU4UHKqTkf0ED3fKEPdr+gKAAl3IA2hj9rrP6iyq3hlcJq3HELtNQ==", + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/@discordjs/voice/node_modules/prism-media": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/prism-media/-/prism-media-1.3.5.tgz", diff --git a/package.json b/package.json index 3eec63a48f8..85c49c5c313 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "prepublishOnly": "cd nemoclaw && env -u npm_config_global -u npm_config_prefix -u npm_config_omit npm install --ignore-scripts && ./node_modules/.bin/tsc" }, "dependencies": { - "openclaw": "2026.3.11" + "openclaw": "2026.3.11", + "p-retry": "^4.6.2" }, "files": [ "bin/", @@ -42,8 +43,8 @@ "@j178/prek": "^0.3.6", "@types/node": "^25.5.0", "@vitest/coverage-v8": "^4.1.0", - "execa": "^9.6.1", "eslint": "^10.1.0", + "execa": "^9.6.1", "tsx": "^4.21.0", "typescript": "^6.0.2", "vitest": "^4.1.0" diff --git a/test/gateway-cleanup.test.js b/test/gateway-cleanup.test.js index 799680048a7..7a71d9abea5 100644 --- a/test/gateway-cleanup.test.js +++ b/test/gateway-cleanup.test.js @@ -28,12 +28,10 @@ describe("gateway cleanup: Docker volumes removed on failure (#17)", () => { // Current behavior: // 1. stale gateway metadata is destroyed directly before start, if present - // 2. destroyGateway() runs after start failure - // 3. destroyGateway() runs after health check failure + // 2. destroyGateway() runs inside the retry loop on each failed attempt expect(startGwBlock[0].includes('if (hasStaleGateway(gwInfo))')).toBe(true); expect(startGwBlock[0].includes('runOpenshell(["gateway", "destroy", "-g", GATEWAY_NAME]')).toBe(true); - const destroyCalls = (startGwBlock[0].match(/destroyGateway\(\)/g) || []).length; - expect(destroyCalls).toBeGreaterThanOrEqual(2); + expect(startGwBlock[0]).toContain("destroyGateway()"); }); it("uninstall.sh: includes Docker volume cleanup", () => {