From 196ada1f8f615ee442f2c7aaac9267dc38142351 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Fri, 24 Apr 2026 13:56:56 -0700 Subject: [PATCH 01/11] fix(onboard): auto-allocate dashboard port on multi-sandbox conflict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second nemoclaw onboard crashed mid-flight when port 18789 was held by the first sandbox, blocking 12 P0/P1 QA tests that keep a baseline sandbox alive while onboarding a short-lived second one. The v0.0.21 guard stopped silent forward-stealing but left no recovery path — users had to set CHAT_UI_URL manually and the assigned port was not discoverable post-hoc. Changes: - src/lib/ports.ts: findFreeDashboardPort picker with 10-port window, injectable probe seam, and 1024-65535 range clamp. 6 unit tests. - src/lib/onboard.ts: ensureDashboardForward auto-allocates on loopback conflict (with a real lsof bind probe to avoid ports bound by other processes) and returns the chosen port. On user-pinned CHAT_UI_URL or window exhaustion it rolls back the openshell sandbox before exiting, closing the "leaks ghost sandbox" path from the issue title. Reuse paths seed chatUiUrl from the sandbox's stored dashboardPort via new reuseChatUiUrlFor helper so re-onboarding an auto-allocated sandbox doesn't ratchet its port upward. - src/lib/registry.ts: dashboardPort?: number on SandboxEntry, persisted by registerSandbox. - src/lib/inventory-commands.ts: dashboard URL in nemoclaw list and nemoclaw status. 2 unit tests. - src/nemoclaw.ts: dashboard URL in nemoclaw status. - src/lib/onboard-command.ts: --control-ui-port flag (precedence: flag > CHAT_UI_URL env > stored port > default). 2 unit tests. - src/lib/onboard.ts (banner): getDashboardAccessInfo and getDashboardGuidanceLines read stored dashboardPort as a precedence tier so the completion banner shows the port actually allocated. Fixes #2174 Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Charan Jagwani --- src/lib/inventory-commands.test.ts | 52 +++++++++++ src/lib/inventory-commands.ts | 7 ++ src/lib/onboard-command.test.ts | 39 ++++++++ src/lib/onboard-command.ts | 32 ++++++- src/lib/onboard.ts | 140 ++++++++++++++++++++++++----- src/lib/ports.test.ts | 52 ++++++++++- src/lib/ports.ts | 38 ++++++++ src/lib/registry.ts | 8 ++ src/nemoclaw.ts | 3 + 9 files changed, 348 insertions(+), 23 deletions(-) diff --git a/src/lib/inventory-commands.test.ts b/src/lib/inventory-commands.test.ts index d25e34a47ca..4859a127579 100644 --- a/src/lib/inventory-commands.test.ts +++ b/src/lib/inventory-commands.test.ts @@ -50,6 +50,39 @@ describe("inventory commands", () => { ); }); + it("prints the dashboard URL for each sandbox when dashboardPort is set", async () => { + const lines: string[] = []; + await listSandboxesCommand({ + recoverRegistryEntries: async () => ({ + sandboxes: [ + { + name: "alpha", + model: "m", + provider: "p", + gpuEnabled: false, + policies: [], + dashboardPort: 18789, + }, + { + name: "beta", + model: "m", + provider: "p", + gpuEnabled: false, + policies: [], + dashboardPort: 18790, + }, + ], + defaultSandbox: "alpha", + }), + getLiveInference: () => null, + loadLastSession: () => null, + log: (message = "") => lines.push(message), + }); + + expect(lines).toContain(" dashboard: http://127.0.0.1:18789"); + expect(lines).toContain(" dashboard: http://127.0.0.1:18790"); + }); + it("uses live gateway inference for the default sandbox in list output (#2369)", async () => { const lines: string[] = []; await listSandboxesCommand({ @@ -201,6 +234,25 @@ describe("inventory commands", () => { expect(lines).toContain(" (onboarded: provider=configured-provider)"); }); + it("prints the dashboard URL per sandbox in status when dashboardPort is set", () => { + const lines: string[] = []; + showStatusCommand({ + listSandboxes: () => ({ + sandboxes: [ + { name: "alpha", model: "m", dashboardPort: 18789 }, + { name: "beta", model: "m", dashboardPort: 18790 }, + ], + defaultSandbox: "alpha", + }), + getLiveInference: () => null, + showServiceStatus: vi.fn(), + log: (message = "") => lines.push(message), + }); + + expect(lines).toContain(" dashboard: http://127.0.0.1:18789"); + expect(lines).toContain(" dashboard: http://127.0.0.1:18790"); + }); + it("flags messaging bridge as degraded when checkMessagingBridgeHealth reports conflicts", () => { const lines: string[] = []; const checkMessagingBridgeHealth = vi.fn().mockReturnValue([ diff --git a/src/lib/inventory-commands.ts b/src/lib/inventory-commands.ts index a36561ebfa0..4eb65293f5c 100644 --- a/src/lib/inventory-commands.ts +++ b/src/lib/inventory-commands.ts @@ -11,6 +11,7 @@ export interface SandboxEntry { policies?: string[] | null; messagingChannels?: string[] | null; agent?: string | null; + dashboardPort?: number; } export interface MessagingBridgeHealth { @@ -117,6 +118,9 @@ export async function listSandboxesCommand(deps: ListSandboxesCommandDeps): Prom const connected = sessionCount !== null && sessionCount > 0 ? " ●" : ""; log(` ${sb.name}${def}${connected}`); log(` model: ${model} provider: ${provider} ${gpu} policies: ${presets}`); + if (typeof sb.dashboardPort === "number") { + log(` dashboard: http://127.0.0.1:${sb.dashboardPort}`); + } if (modelDrifted || providerDrifted) { const parts: string[] = []; if (modelDrifted) parts.push(`model=${sb.model || "unknown"}`); @@ -152,6 +156,9 @@ export function showStatusCommand(deps: ShowStatusCommandDeps): void { const liveModel = isDefault && live ? live.model : null; const model = liveModel || sb.model; log(` ${sb.name}${def}${model ? ` (${model})` : ""}`); + if (typeof sb.dashboardPort === "number") { + log(` dashboard: http://127.0.0.1:${sb.dashboardPort}`); + } if (isDefault && liveModel && liveModel !== sb.model) { log(` (onboarded: ${sb.model || "unknown"})`); } diff --git a/src/lib/onboard-command.test.ts b/src/lib/onboard-command.test.ts index a16b948b362..5c6377c5555 100644 --- a/src/lib/onboard-command.test.ts +++ b/src/lib/onboard-command.test.ts @@ -38,6 +38,7 @@ describe("onboard command", () => { acceptThirdPartySoftware: true, agent: null, dangerouslySkipPermissions: false, + controlUiPort: null, }); }); @@ -61,6 +62,7 @@ describe("onboard command", () => { acceptThirdPartySoftware: true, agent: null, dangerouslySkipPermissions: false, + controlUiPort: null, }); }); @@ -83,6 +85,7 @@ describe("onboard command", () => { acceptThirdPartySoftware: false, agent: null, dangerouslySkipPermissions: false, + controlUiPort: null, }); }); @@ -126,9 +129,43 @@ describe("onboard command", () => { acceptThirdPartySoftware: false, agent: null, dangerouslySkipPermissions: false, + controlUiPort: null, }); }); + it("parses --control-ui-port ", () => { + const result = parseOnboardArgs( + ["--resume", "--control-ui-port", "18795"], + "--yes-i-accept-third-party-software", + "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE", + { + env: {}, + error: () => {}, + exit: ((code: number) => { + throw new Error(String(code)); + }) as never, + }, + ); + expect(result.controlUiPort).toBe(18795); + }); + + it("exits when --control-ui-port is out of range", () => { + expect(() => + parseOnboardArgs( + ["--control-ui-port", "80"], + "--yes-i-accept-third-party-software", + "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE", + { + env: {}, + error: () => {}, + exit: ((code: number) => { + throw new Error(`exit:${code}`); + }) as never, + }, + ), + ).toThrow("exit:1"); + }); + it("exits when --from is missing its Dockerfile path", () => { expect(() => parseOnboardArgs( @@ -183,6 +220,7 @@ describe("onboard command", () => { acceptThirdPartySoftware: false, agent: "openclaw", dangerouslySkipPermissions: true, + controlUiPort: null, }); }); @@ -230,6 +268,7 @@ describe("onboard command", () => { acceptThirdPartySoftware: false, agent: null, dangerouslySkipPermissions: false, + controlUiPort: null, }); }); diff --git a/src/lib/onboard-command.ts b/src/lib/onboard-command.ts index f3f4c5d8f9f..978e78aff9d 100644 --- a/src/lib/onboard-command.ts +++ b/src/lib/onboard-command.ts @@ -9,6 +9,7 @@ export interface OnboardCommandOptions { acceptThirdPartySoftware: boolean; agent: string | null; dangerouslySkipPermissions: boolean; + controlUiPort: number | null; } export interface RunOnboardCommandDeps { @@ -36,7 +37,7 @@ const ONBOARD_BASE_ARGS = [ function onboardUsageLines(noticeAcceptFlag: string): string[] { return [ - ` Usage: nemoclaw onboard [--non-interactive] [--resume] [--recreate-sandbox] [--from ] [--agent ] [--dangerously-skip-permissions] [${noticeAcceptFlag}]`, + ` Usage: nemoclaw onboard [--non-interactive] [--resume] [--recreate-sandbox] [--from ] [--agent ] [--control-ui-port ] [--dangerously-skip-permissions] [${noticeAcceptFlag}]`, "", ]; } @@ -69,6 +70,28 @@ export function parseOnboardArgs( parsedArgs.splice(fromIdx, 2); } + let controlUiPort: number | null = null; + const controlUiPortIdx = parsedArgs.indexOf("--control-ui-port"); + if (controlUiPortIdx !== -1) { + const raw = parsedArgs[controlUiPortIdx + 1]; + if (typeof raw !== "string" || raw.startsWith("--")) { + error(" --control-ui-port requires a port number"); + printOnboardUsage(error, noticeAcceptFlag); + exit(1); + } + if (!/^\d+$/.test(raw)) { + error(` --control-ui-port '${raw}' must be an integer between 1024 and 65535`); + exit(1); + } + const parsed = Number(raw); + if (parsed < 1024 || parsed > 65535) { + error(` --control-ui-port '${raw}' must be an integer between 1024 and 65535`); + exit(1); + } + controlUiPort = parsed; + parsedArgs.splice(controlUiPortIdx, 2); + } + let agent: string | null = null; const agentIdx = parsedArgs.indexOf("--agent"); if (agentIdx !== -1) { @@ -105,6 +128,7 @@ export function parseOnboardArgs( parsedArgs.includes(noticeAcceptFlag) || String(deps.env[noticeAcceptEnv] || "") === "1", agent, dangerouslySkipPermissions: parsedArgs.includes("--dangerously-skip-permissions"), + controlUiPort, }; } @@ -116,6 +140,12 @@ export async function runOnboardCommand(deps: RunOnboardCommandDeps): Promise 0) { registry.updateSandbox(sandboxName, { providerCredentialHashes: abortHashes }); } - ensureDashboardForward(sandboxName, chatUiUrl); + const reusePort = ensureDashboardForward( + sandboxName, + reuseChatUiUrlFor(sandboxName, chatUiUrl), + ); + registry.updateSandbox(sandboxName, { dashboardPort: reusePort }); return sandboxName; } } catch (err) { @@ -4058,7 +4071,11 @@ async function createSandbox( if (Object.keys(abortHashes).length > 0) { registry.updateSandbox(sandboxName, { providerCredentialHashes: abortHashes }); } - ensureDashboardForward(sandboxName, chatUiUrl); + const reusePort = ensureDashboardForward( + sandboxName, + reuseChatUiUrlFor(sandboxName, chatUiUrl), + ); + registry.updateSandbox(sandboxName, { dashboardPort: reusePort }); return sandboxName; } } @@ -4477,7 +4494,8 @@ async function createSandbox( // Release any stale forward on the dashboard port before claiming it for the new sandbox. // A previous onboard run may have left the port forwarded to a different sandbox, // which would silently prevent the new sandbox's dashboard from being reachable. - ensureDashboardForward(sandboxName, chatUiUrl); + // ensureDashboardForward may auto-allocate a different port on conflict (#2174). + const allocatedDashboardPort = ensureDashboardForward(sandboxName, chatUiUrl); // Register only after confirmed ready — prevents phantom entries const effectiveAgent = agent || agentDefs.loadAgent("openclaw"); @@ -4501,6 +4519,7 @@ async function createSandbox( Object.keys(providerCredentialHashes).length > 0 ? providerCredentialHashes : undefined, messagingChannels: activeMessagingChannels, disabledChannels: disabledChannels.length > 0 ? [...disabledChannels] : undefined, + dashboardPort: allocatedDashboardPort, }); // Restore workspace state if we backed it up during credential rotation. @@ -6734,27 +6753,90 @@ function findDashboardForwardOwner( return portLine ? (portLine.split(/\s+/)[0] ?? null) : null; } +/** + * Pick the chatUiUrl to use when re-forwarding an already-registered sandbox. + * Precedence: user-set CHAT_UI_URL env > stored dashboardPort > caller-supplied + * default. Prevents reuse paths from always re-requesting the default port and + * ratcheting an auto-allocated sandbox upward on each onboard (#2174). + */ +function reuseChatUiUrlFor(sandboxName: string, fallbackUrl: string): string { + if (process.env.CHAT_UI_URL) return fallbackUrl; + const stored = registry.getSandbox(sandboxName)?.dashboardPort; + return typeof stored === "number" ? `http://127.0.0.1:${stored}` : fallbackUrl; +} + function ensureDashboardForward( sandboxName: string, chatUiUrl = `http://127.0.0.1:${CONTROL_UI_PORT}`, -) { +): number { const chain = buildChain({ chatUiUrl, isWsl: isWsl() }); - const portToStop = String(chain.port); - const forwardTarget = chain.forwardTarget; - // Detect port already claimed by a different sandbox and fail fast with an - // actionable message rather than silently stealing that sandbox's forward. - // (Same sandbox is always allowed — covers reconnect and resume paths.) + const requestedPort = chain.port; const existingForwards = runCaptureOpenshell(["forward", "list"], { ignoreError: true }); - const portOwner = findDashboardForwardOwner(existingForwards, portToStop); + const portOwner = findDashboardForwardOwner(existingForwards, String(requestedPort)); + + let effectivePort = requestedPort; + let forwardTarget = chain.forwardTarget; if (portOwner !== null && portOwner !== sandboxName) { - // Match the preflight pattern (printed error + exit) instead of throwing, - // so the user sees a clean message rather than a raw Node stack trace - // from the top-level IIFE's unhandled rejection. See #2169. - console.error(` Port ${portToStop} is already forwarded for sandbox '${portOwner}'.`); - console.error(` Set CHAT_UI_URL to a different local port (e.g. http://127.0.0.1:18790)`); - console.error(` before onboarding a second sandbox.`); - process.exit(1); + // Port is held by a different sandbox. Auto-allocate on loopback default; + // fail fast (with cleanup) when the user explicitly pinned via CHAT_UI_URL + // env or a non-loopback URL (#2174). + const userPinnedEnv = !!process.env.CHAT_UI_URL; + let isLoopback = true; + try { + const parsed = new URL( + /^[a-z]+:\/\//i.test(chatUiUrl) ? chatUiUrl : `http://${chatUiUrl}`, + ); + isLoopback = isLoopbackHostname(parsed.hostname); + } catch { + isLoopback = true; + } + if (userPinnedEnv || !isLoopback) { + // Pre-existing sandbox (this openshell-create already succeeded). Roll it + // back so `nemoclaw list` and `openshell sandbox list` don't drift + // ("leaks ghost sandbox" from #2174 title). + runOpenshell(["sandbox", "delete", sandboxName], { ignoreError: true }); + console.error(` Port ${requestedPort} is already forwarded for sandbox '${portOwner}'.`); + console.error(` Unset CHAT_UI_URL or pick a free port to onboard '${sandboxName}'.`); + process.exit(1); + } + // Parse held ports from the forward list we already fetched. + const heldPorts = (existingForwards ?? "") + .split("\n") + .map((l) => l.trim()) + .filter(Boolean) + .map((l) => Number(l.split(/\s+/)[2])) + .filter((p) => Number.isFinite(p) && p > 0); + const isPortBoundLocally = (p: number): boolean => { + const out = runCapture(["lsof", "-i", `:${p}`, "-sTCP:LISTEN", "-P", "-n"], { + ignoreError: true, + }); + return typeof out === "string" && out.trim().length > 0; + }; + const chosen = findFreeDashboardPort(requestedPort, { + probe: { + listForwardPorts: () => heldPorts, + probePortFree: (p: number) => !isPortBoundLocally(p), + }, + }); + if (chosen === null) { + runOpenshell(["sandbox", "delete", sandboxName], { ignoreError: true }); + console.error(` All ports ${requestedPort}-${requestedPort + 9} are forwarded.`); + console.error( + ` Destroy an unused sandbox, or set CHAT_UI_URL=http://127.0.0.1:${requestedPort + 10} (or higher) and retry.`, + ); + process.exit(1); + } + effectivePort = chosen; + forwardTarget = buildChain({ + chatUiUrl: `http://127.0.0.1:${effectivePort}`, + isWsl: isWsl(), + }).forwardTarget; + console.log( + ` Dashboard port ${requestedPort} in use by '${portOwner}'; allocated port ${effectivePort} for '${sandboxName}'.`, + ); } + + const portToStop = String(effectivePort); runOpenshell(["forward", "stop", portToStop], { ignoreError: true }); // Use stdio "ignore" to prevent spawnSync from waiting on inherited pipe fds. // The --background flag forks a child that inherits stdout/stderr; if those are @@ -6776,6 +6858,7 @@ function ensureDashboardForward( ); console.warn(` Free the port, then reconnect: nemoclaw ${sandboxName} connect`); } + return effectivePort; } function findFileRecursive(dir: string, filename: string): string | null { @@ -7007,8 +7090,16 @@ function getDashboardAccessInfo( const token = Object.prototype.hasOwnProperty.call(options, "token") ? options.token : fetchGatewayAuthTokenFromSandbox(sandboxName); + // Precedence: caller-supplied > CHAT_UI_URL env > stored dashboardPort > default. + // The stored-port tier ensures the completion banner shows the port actually + // allocated by ensureDashboardForward, not the original default (#2174). + const storedPort = registry.getSandbox(sandboxName)?.dashboardPort; + const storedUrl = typeof storedPort === "number" ? `http://127.0.0.1:${storedPort}` : null; const chatUiUrl = - options.chatUiUrl || process.env.CHAT_UI_URL || `http://127.0.0.1:${CONTROL_UI_PORT}`; + options.chatUiUrl || + process.env.CHAT_UI_URL || + storedUrl || + `http://127.0.0.1:${CONTROL_UI_PORT}`; const chain = buildDashboardChain(chatUiUrl, options); const dashboardAccess = buildControlUiUrls(token, chain.port, chain.accessUrl).map( (url, index) => ({ @@ -7038,10 +7129,17 @@ function getDashboardGuidanceLines( platform?: NodeJS.Platform; release?: string; isWsl?: boolean; + sandboxName?: string; } = {}, ): string[] { + const storedPort = + options.sandboxName && registry.getSandbox(options.sandboxName)?.dashboardPort; + const storedUrl = typeof storedPort === "number" ? `http://127.0.0.1:${storedPort}` : null; const chatUiUrl = - options.chatUiUrl || process.env.CHAT_UI_URL || `http://127.0.0.1:${CONTROL_UI_PORT}`; + options.chatUiUrl || + process.env.CHAT_UI_URL || + storedUrl || + `http://127.0.0.1:${CONTROL_UI_PORT}`; const chain = buildDashboardChain(chatUiUrl, options); const guidance = [`Port ${String(chain.port)} must be forwarded before opening these URLs.`]; if (isWsl(options)) { diff --git a/src/lib/ports.test.ts b/src/lib/ports.test.ts index 3769c2bf8bd..dd1fc2b7d27 100644 --- a/src/lib/ports.test.ts +++ b/src/lib/ports.test.ts @@ -3,7 +3,7 @@ import { describe, it, expect, beforeEach, afterEach } from "vitest"; // Import from compiled dist/ so coverage is attributed correctly. -import { parsePort } from "../../dist/lib/ports"; +import { parsePort, findFreeDashboardPort } from "../../dist/lib/ports"; describe("parsePort", () => { const ENV_KEY = "TEST_PORT"; @@ -70,3 +70,53 @@ describe("parsePort", () => { expect(() => parsePort(ENV_KEY, 8080)).toThrow("Invalid port"); }); }); + +describe("findFreeDashboardPort", () => { + it("skips ports already held by openshell forwards", () => { + const port = findFreeDashboardPort(18789, { + probe: { + listForwardPorts: () => [18789, 18790], + probePortFree: () => true, + }, + }); + expect(port).toBe(18791); + }); + + it("skips ports bound by other host processes", () => { + const bound = new Set([18789, 18790]); + const port = findFreeDashboardPort(18789, { + probe: { + listForwardPorts: () => [], + probePortFree: (p) => !bound.has(p), + }, + }); + expect(port).toBe(18791); + }); + + it("returns null when the 10-port window is exhausted", () => { + const allHeld = Array.from({ length: 10 }, (_, i) => 18789 + i); + const port = findFreeDashboardPort(18789, { + probe: { + listForwardPorts: () => allHeld, + probePortFree: () => true, + }, + }); + expect(port).toBeNull(); + }); + + it("does not allocate ports above 65535 when preferred is near the upper bound", () => { + const port = findFreeDashboardPort(65535, { + probe: { + listForwardPorts: () => [65535], + probePortFree: () => true, + }, + }); + expect(port).toBeNull(); + }); + + it("returns null when preferred port is out of range", () => { + expect(findFreeDashboardPort(0)).toBeNull(); + expect(findFreeDashboardPort(65536)).toBeNull(); + expect(findFreeDashboardPort(1023)).toBeNull(); + }); +}); diff --git a/src/lib/ports.ts b/src/lib/ports.ts index ad5d8166ad7..1d29c0135cc 100644 --- a/src/lib/ports.ts +++ b/src/lib/ports.ts @@ -46,3 +46,41 @@ export const VLLM_PORT = parsePort("NEMOCLAW_VLLM_PORT", 8000); export const OLLAMA_PORT = parsePort("NEMOCLAW_OLLAMA_PORT", 11434); /** Ollama auth proxy port (default 11435, override via NEMOCLAW_OLLAMA_PROXY_PORT). */ export const OLLAMA_PROXY_PORT = parsePort("NEMOCLAW_OLLAMA_PROXY_PORT", 11435); + +/** + * Injectable probes so tests can drive `findFreeDashboardPort` without touching + * real openshell or real sockets. + */ +export interface PortProbe { + listForwardPorts: () => number[]; + probePortFree: (port: number) => boolean; +} + +/** + * Try up to 10 ports starting at the preferred port before giving up. + * Anything beyond that and the operator has bigger problems; fail loudly. + */ +const PORT_WINDOW = 10; + +/** + * Find a free dashboard port, preferring `preferred` and walking upward. + * Skips ports already claimed by openshell forwards and ports bound by + * other host processes. Returns null if the 10-port window is exhausted. + */ +export function findFreeDashboardPort( + preferred: number, + options: { probe?: PortProbe } = {}, +): number | null { + if (!Number.isInteger(preferred) || preferred < 1024 || preferred > 65535) { + return null; + } + const probe = options.probe; + const held = new Set(probe?.listForwardPorts() ?? []); + const isFree = probe?.probePortFree ?? (() => true); + for (let offset = 0; offset < PORT_WINDOW; offset++) { + const port = preferred + offset; + if (port > 65535) break; + if (!held.has(port) && isFree(port)) return port; + } + return null; +} diff --git a/src/lib/registry.ts b/src/lib/registry.ts index 7975dd30054..527d3c4c520 100644 --- a/src/lib/registry.ts +++ b/src/lib/registry.ts @@ -28,6 +28,8 @@ export interface SandboxEntry { providerCredentialHashes?: Record; messagingChannels?: string[]; disabledChannels?: string[]; + /** Host-side port for the openshell forward to the OpenClaw dashboard (#2174). */ + dashboardPort?: number; } export interface SandboxRegistry { @@ -198,6 +200,12 @@ export function registerSandbox(entry: SandboxEntry): void { Array.isArray(entry.disabledChannels) && entry.disabledChannels.length > 0 ? [...entry.disabledChannels] : undefined, + dashboardPort: + Number.isInteger(entry.dashboardPort) && + (entry.dashboardPort as number) >= 1024 && + (entry.dashboardPort as number) <= 65535 + ? entry.dashboardPort + : undefined, }; if (!data.defaultSandbox) { data.defaultSandbox = entry.name; diff --git a/src/nemoclaw.ts b/src/nemoclaw.ts index 069f123014c..4393d3b34d0 100644 --- a/src/nemoclaw.ts +++ b/src/nemoclaw.ts @@ -1662,6 +1662,9 @@ async function sandboxStatus(sandboxName: string) { } console.log(` GPU: ${sb.gpuEnabled ? "yes" : "no"}`); console.log(` Policies: ${(sb.policies || []).join(", ") || "none"}`); + if (typeof sb.dashboardPort === "number") { + console.log(` Dashboard: http://127.0.0.1:${sb.dashboardPort}`); + } // Active session indicator try { From f13c77602ee703db5682b30f5255d1ef9a9e4a94 Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Fri, 24 Apr 2026 21:28:06 +0000 Subject: [PATCH 02/11] fix: apply CodeRabbit auto-fixes Fixed 3 file(s) based on 3 unresolved review comments. Co-authored-by: CodeRabbit --- nemoclaw-blueprint/scripts/ws-proxy-fix.js | 4 +++- src/lib/onboard-command.ts | 16 +++++++++++++--- src/lib/onboard.ts | 19 +++++++++++++++---- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/nemoclaw-blueprint/scripts/ws-proxy-fix.js b/nemoclaw-blueprint/scripts/ws-proxy-fix.js index 1b9a9626750..9b1e1b5d348 100644 --- a/nemoclaw-blueprint/scripts/ws-proxy-fix.js +++ b/nemoclaw-blueprint/scripts/ws-proxy-fix.js @@ -176,7 +176,9 @@ const _PATCHED = Symbol.for("nemoclaw.wsProxyFix"); if (isDiscordWsUpgrade(host, opts.headers)) { // Guard: if isDiscordWsUpgrade matched but host resolved to // undefined, we cannot construct a CONNECT tunnel (no target). - // Fall through to the original https.request unchanged. + // Fall through to the original https.request unchanged. Before + // PR #2422 this path would have attempted the tunnel with an + // undefined host, which would fail in createTunnelAgent anyway. if (!host) { return callOriginalRequest(input, options, callback); } diff --git a/src/lib/onboard-command.ts b/src/lib/onboard-command.ts index 978e78aff9d..b86a365d2da 100644 --- a/src/lib/onboard-command.ts +++ b/src/lib/onboard-command.ts @@ -143,10 +143,20 @@ export async function runOnboardCommand(deps: RunOnboardCommandDeps): Promise/dev/null", { ignoreError: true }) || "").trim().split(/\s+/)[0] || null) : null; const chain = buildChain({ chatUiUrl, isWsl: isWsl(), wslHostAddress: wslAddr }); From d2a74d23764d3209d2eed3e9bddb19e3065c2379 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 28 Apr 2026 09:45:08 -0700 Subject: [PATCH 03/11] diag: dump fd/process state before exit 0 in survival test Temporary instrumentation to root-cause the post-success-message hang on sandbox-survival-e2e for PR #2454. Captures lsof output, ps tree, and every process holding bash's stdout/stderr pipe FDs into the upload artifact path. exit 99 forces the failure step so the artifact uploads. Revert before merge. --- test/e2e/test-sandbox-survival.sh | 47 ++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/test/e2e/test-sandbox-survival.sh b/test/e2e/test-sandbox-survival.sh index 621ccd4d511..7d7316135fe 100755 --- a/test/e2e/test-sandbox-survival.sh +++ b/test/e2e/test-sandbox-survival.sh @@ -714,7 +714,52 @@ echo "========================================" if [ "$FAIL" -eq 0 ]; then printf '\n\033[1;32m Sandbox survival PASSED — all state persisted, live inference verified before AND after gateway restart.\033[0m\n' printf '\033[1;32m Issues validated: #486, #888, #859, #1086\033[0m\n' - exit 0 + + # ── DIAGNOSTIC: capture process state right before exit 0 ────────────────── + # Investigates why bash hangs ~1-2 minutes after this point on PR #2454 CI. + # Writes to /tmp/nemoclaw-e2e-install.log (path the upload-artifact step grabs). + DIAG_FILE=/tmp/nemoclaw-e2e-install.log + { + printf '\n=== EXIT-DIAG bash_pid=%s ts=%s ===\n' "$$" "$(date +%s.%N)" + printf -- '--- /proc/%s/fd ---\n' "$$" + ls -la "/proc/$$/fd" 2>&1 || true + printf -- '--- lsof for bash %s ---\n' "$$" + lsof -p "$$" 2>&1 | head -100 || true + printf -- '--- full ps -ef ---\n' + ps -ef 2>&1 || true + printf -- '--- pstree of session ---\n' + pstree -p -s "$$" 2>&1 || true + # Find every process holding the same pipe FDs as bash (stdout, stderr). + # If anyone other than bash + timeout has these open, that process is what + # keeps the runner's pipe alive after bash tries to exit. + for which in 1 2; do + tgt=$(readlink "/proc/$$/fd/$which" 2>/dev/null) + printf -- '--- consumers of bash fd %s -> %s ---\n' "$which" "$tgt" + case "$tgt" in + pipe:\[*\]) + for pid_dir in /proc/[0-9]*; do + pid=${pid_dir##*/} + [ -d "$pid_dir/fd" ] || continue + for fd in "$pid_dir/fd"/*; do + [ -L "$fd" ] || continue + ftgt=$(readlink "$fd" 2>/dev/null) + if [ "$ftgt" = "$tgt" ]; then + cmd=$(tr '\0' ' ' < "$pid_dir/cmdline" 2>/dev/null) + comm=$(cat "$pid_dir/comm" 2>/dev/null) + printf ' pid=%s comm=%s fd=%s cmdline=%s\n' "$pid" "$comm" "$(basename "$fd")" "$cmd" + fi + done + done + ;; + esac + done + printf '=== EXIT-DIAG end ts=%s ===\n' "$(date +%s.%N)" + } > "$DIAG_FILE" 2>&1 + printf ' [diag] wrote %s (%s lines)\n' "$DIAG_FILE" "$(wc -l < "$DIAG_FILE" 2>/dev/null || echo 0)" + + # Force the job to fail so actions/upload-artifact runs and captures the diag. + # Once we have the diag, revert this and fix the real cause. + exit 99 else printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" exit 1 From d1193383f78e717fbec23ce8f436a58f1b122210 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 28 Apr 2026 10:08:45 -0700 Subject: [PATCH 04/11] diag v2: save orig stdout/stderr before redirect v1 saw fd 1 = the diag file (its own redirection) instead of the runner pipe. Save the original to fds 9/8 first, then walk /proc to find every process holding the same pipe inode. --- test/e2e/test-sandbox-survival.sh | 34 +++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/test/e2e/test-sandbox-survival.sh b/test/e2e/test-sandbox-survival.sh index 7d7316135fe..af964182179 100755 --- a/test/e2e/test-sandbox-survival.sh +++ b/test/e2e/test-sandbox-survival.sh @@ -717,24 +717,32 @@ if [ "$FAIL" -eq 0 ]; then # ── DIAGNOSTIC: capture process state right before exit 0 ────────────────── # Investigates why bash hangs ~1-2 minutes after this point on PR #2454 CI. - # Writes to /tmp/nemoclaw-e2e-install.log (path the upload-artifact step grabs). + # v2: save the ORIGINAL stdout/stderr to fds 9/8 before the redirect, so the + # diag can inspect what was actually on fd 1/2 before this group hijacked + # them. v1 saw fd 1 = the diag file (its own redirection) — useless. DIAG_FILE=/tmp/nemoclaw-e2e-install.log + exec 9>&1 8>&2 + ORIG_STDOUT_TGT=$(readlink "/proc/$$/fd/9" 2>/dev/null) + ORIG_STDERR_TGT=$(readlink "/proc/$$/fd/8" 2>/dev/null) { - printf '\n=== EXIT-DIAG bash_pid=%s ts=%s ===\n' "$$" "$(date +%s.%N)" - printf -- '--- /proc/%s/fd ---\n' "$$" + printf '\n=== EXIT-DIAG v2 bash_pid=%s ts=%s ===\n' "$$" "$(date +%s.%N)" + printf 'ORIG_STDOUT_TGT=%s\n' "$ORIG_STDOUT_TGT" + printf 'ORIG_STDERR_TGT=%s\n' "$ORIG_STDERR_TGT" + printf -- '--- /proc/%s/fd (note: 1/2 are diag-redirected, 9/8 are saved orig) ---\n' "$$" ls -la "/proc/$$/fd" 2>&1 || true printf -- '--- lsof for bash %s ---\n' "$$" - lsof -p "$$" 2>&1 | head -100 || true + lsof -p "$$" 2>&1 | head -200 || true printf -- '--- full ps -ef ---\n' ps -ef 2>&1 || true printf -- '--- pstree of session ---\n' pstree -p -s "$$" 2>&1 || true - # Find every process holding the same pipe FDs as bash (stdout, stderr). - # If anyone other than bash + timeout has these open, that process is what - # keeps the runner's pipe alive after bash tries to exit. - for which in 1 2; do - tgt=$(readlink "/proc/$$/fd/$which" 2>/dev/null) - printf -- '--- consumers of bash fd %s -> %s ---\n' "$which" "$tgt" + # Find every process holding the same pipe inode as bash's ORIGINAL stdout + # and stderr. Anyone other than bash + timeout is what keeps the runner's + # pipe alive after bash tries to exit. + for label in stdout:$ORIG_STDOUT_TGT stderr:$ORIG_STDERR_TGT; do + tgt=${label#*:} + name=${label%%:*} + printf -- '--- consumers of bash %s (%s) ---\n' "$name" "$tgt" case "$tgt" in pipe:\[*\]) for pid_dir in /proc/[0-9]*; do @@ -746,15 +754,19 @@ if [ "$FAIL" -eq 0 ]; then if [ "$ftgt" = "$tgt" ]; then cmd=$(tr '\0' ' ' < "$pid_dir/cmdline" 2>/dev/null) comm=$(cat "$pid_dir/comm" 2>/dev/null) - printf ' pid=%s comm=%s fd=%s cmdline=%s\n' "$pid" "$comm" "$(basename "$fd")" "$cmd" + ppid=$(awk '{print $4}' "$pid_dir/stat" 2>/dev/null) + printf ' pid=%s ppid=%s comm=%s fd=%s cmdline=%s\n' \ + "$pid" "$ppid" "$comm" "$(basename "$fd")" "$cmd" fi done done ;; + *) printf ' (not a pipe — nothing to scan)\n' ;; esac done printf '=== EXIT-DIAG end ts=%s ===\n' "$(date +%s.%N)" } > "$DIAG_FILE" 2>&1 + exec 9>&- 8>&- printf ' [diag] wrote %s (%s lines)\n' "$DIAG_FILE" "$(wc -l < "$DIAG_FILE" 2>/dev/null || echo 0)" # Force the job to fail so actions/upload-artifact runs and captures the diag. From 2ddd01358e15dccae2ba7a9a2aa8dae41657cdcd Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 28 Apr 2026 10:36:29 -0700 Subject: [PATCH 05/11] diag v3: EXIT trap timestamp to localize the hang Records when bash actually starts to exit (via EXIT trap, fires right before exit syscall). Comparing that to the runner's Process completed timestamp tells us whether the ~60s post-printf gap is in bash or in the runner's exit-detection path. --- test/e2e/test-sandbox-survival.sh | 69 +++++++------------------------ 1 file changed, 16 insertions(+), 53 deletions(-) diff --git a/test/e2e/test-sandbox-survival.sh b/test/e2e/test-sandbox-survival.sh index af964182179..0098c331fa5 100755 --- a/test/e2e/test-sandbox-survival.sh +++ b/test/e2e/test-sandbox-survival.sh @@ -715,62 +715,25 @@ if [ "$FAIL" -eq 0 ]; then printf '\n\033[1;32m Sandbox survival PASSED — all state persisted, live inference verified before AND after gateway restart.\033[0m\n' printf '\033[1;32m Issues validated: #486, #888, #859, #1086\033[0m\n' - # ── DIAGNOSTIC: capture process state right before exit 0 ────────────────── - # Investigates why bash hangs ~1-2 minutes after this point on PR #2454 CI. - # v2: save the ORIGINAL stdout/stderr to fds 9/8 before the redirect, so the - # diag can inspect what was actually on fd 1/2 before this group hijacked - # them. v1 saw fd 1 = the diag file (its own redirection) — useless. - DIAG_FILE=/tmp/nemoclaw-e2e-install.log - exec 9>&1 8>&2 - ORIG_STDOUT_TGT=$(readlink "/proc/$$/fd/9" 2>/dev/null) - ORIG_STDERR_TGT=$(readlink "/proc/$$/fd/8" 2>/dev/null) + # ── DIAGNOSTIC v3: pinpoint where the post-printf delay actually lives ───── + # Hypothesis: bash exits cleanly within ms, but runner takes ~60s to detect + # the exit (possibly due to processing buffered stdout from k3s log spam in + # Phase 6). EXIT trap records the moment bash actually exits to a file the + # next workflow step can read; comparing that to the runner's "Process + # completed" timestamp tells us if the gap is bash-side or runner-side. + EXIT_MARKER=/tmp/nemoclaw-e2e-install.log + trap 'echo "TRAP_EXIT pid=$$ ts=$(date +%s.%N) exit_arg=$?" >> "$EXIT_MARKER"' EXIT + + # Quick fd dump for completeness — keep tiny so it can't affect timing. { - printf '\n=== EXIT-DIAG v2 bash_pid=%s ts=%s ===\n' "$$" "$(date +%s.%N)" - printf 'ORIG_STDOUT_TGT=%s\n' "$ORIG_STDOUT_TGT" - printf 'ORIG_STDERR_TGT=%s\n' "$ORIG_STDERR_TGT" - printf -- '--- /proc/%s/fd (note: 1/2 are diag-redirected, 9/8 are saved orig) ---\n' "$$" + printf '\n=== EXIT-DIAG v3 pid=%s ts=%s ===\n' "$$" "$(date +%s.%N)" ls -la "/proc/$$/fd" 2>&1 || true - printf -- '--- lsof for bash %s ---\n' "$$" - lsof -p "$$" 2>&1 | head -200 || true - printf -- '--- full ps -ef ---\n' - ps -ef 2>&1 || true - printf -- '--- pstree of session ---\n' - pstree -p -s "$$" 2>&1 || true - # Find every process holding the same pipe inode as bash's ORIGINAL stdout - # and stderr. Anyone other than bash + timeout is what keeps the runner's - # pipe alive after bash tries to exit. - for label in stdout:$ORIG_STDOUT_TGT stderr:$ORIG_STDERR_TGT; do - tgt=${label#*:} - name=${label%%:*} - printf -- '--- consumers of bash %s (%s) ---\n' "$name" "$tgt" - case "$tgt" in - pipe:\[*\]) - for pid_dir in /proc/[0-9]*; do - pid=${pid_dir##*/} - [ -d "$pid_dir/fd" ] || continue - for fd in "$pid_dir/fd"/*; do - [ -L "$fd" ] || continue - ftgt=$(readlink "$fd" 2>/dev/null) - if [ "$ftgt" = "$tgt" ]; then - cmd=$(tr '\0' ' ' < "$pid_dir/cmdline" 2>/dev/null) - comm=$(cat "$pid_dir/comm" 2>/dev/null) - ppid=$(awk '{print $4}' "$pid_dir/stat" 2>/dev/null) - printf ' pid=%s ppid=%s comm=%s fd=%s cmdline=%s\n' \ - "$pid" "$ppid" "$comm" "$(basename "$fd")" "$cmd" - fi - done - done - ;; - *) printf ' (not a pipe — nothing to scan)\n' ;; - esac - done - printf '=== EXIT-DIAG end ts=%s ===\n' "$(date +%s.%N)" - } > "$DIAG_FILE" 2>&1 - exec 9>&- 8>&- - printf ' [diag] wrote %s (%s lines)\n' "$DIAG_FILE" "$(wc -l < "$DIAG_FILE" 2>/dev/null || echo 0)" + ps -eo pid,ppid,pgid,sid,stat,comm 2>&1 | grep -E "PID|runner|test|timeout|bash|node|openshell|Runner" || true + } >> "$EXIT_MARKER" 2>&1 + + printf ' [diag v3] about to exit, ts=%s\n' "$(date +%s.%N)" - # Force the job to fail so actions/upload-artifact runs and captures the diag. - # Once we have the diag, revert this and fix the real cause. + # Force failure so the artifact uploads. exit 99 else printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" From d99edca3318798386bf1e39b2b88faed8eeedaa0 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 28 Apr 2026 10:53:46 -0700 Subject: [PATCH 06/11] diag v4: exit 0 + EXIT trap + always-upload artifact If the trap fires immediately after diag (within ms), bash exits cleanly on exit 0 and the 50s+ PR-specific extra tail is runner-side. If trap fires 50s+ later, bash is hanging in fflush() before exit. --- .github/workflows/nightly-e2e.yaml | 2 +- test/e2e/test-sandbox-survival.sh | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index b61a6d3a234..1900cfeab02 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -158,7 +158,7 @@ jobs: run: bash test/e2e/test-sandbox-survival.sh - name: Upload install log on failure - if: failure() + if: always() # diag v4: capture artifact on exit 0 too uses: actions/upload-artifact@v4 with: name: sandbox-survival-install-log diff --git a/test/e2e/test-sandbox-survival.sh b/test/e2e/test-sandbox-survival.sh index 0098c331fa5..40c6f53e63c 100755 --- a/test/e2e/test-sandbox-survival.sh +++ b/test/e2e/test-sandbox-survival.sh @@ -733,8 +733,13 @@ if [ "$FAIL" -eq 0 ]; then printf ' [diag v3] about to exit, ts=%s\n' "$(date +%s.%N)" - # Force failure so the artifact uploads. - exit 99 + # v4: exit 0 (the actual code path on real success). EXIT trap will record + # the moment bash invokes exit. If trap timestamp == diag timestamp, bash + # exits immediately and the 50s+ PR-specific extra tail is runner-side. + # If trap fires 50s+ after diag, bash is hanging in fflush() before exit. + # Keep the artifact path populated by the trap so this still uploads even + # without forced failure: failing post-step covers it on the SIGTERM path. + exit 0 else printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" exit 1 From 9aaadf94ba0deeacfc490f8c695e9471015fc30d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 28 Apr 2026 11:16:14 -0700 Subject: [PATCH 07/11] ci: always-upload sandbox-survival artifact to fix runner-batch flake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sandbox-survival-e2e job has a 900s internal wall clock and was flaking on PRs whose install hits the slow side of GitHub runner variance. Investigation traced the failure to runner-side log batching: on a clean exit 0 with no follow-up step (if: failure() skipped), the runner sat in its ~60s log-flush window before reporting the step's exit code. A guaranteed follow-up step (if: always()) makes the runner finalize immediately — tail dropped from 60s to 1.7s in a controlled test, reclaiming ~58s of the 900s budget. The artifact upload is harmless on success: the production script doesn't write /tmp/nemoclaw-e2e-install.log, and if-no-files-found: ignore no-ops the upload. --- .github/workflows/nightly-e2e.yaml | 8 +++++++- test/e2e/test-sandbox-survival.sh | 25 ------------------------- 2 files changed, 7 insertions(+), 26 deletions(-) diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index 1900cfeab02..767546f1afb 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -158,7 +158,13 @@ jobs: run: bash test/e2e/test-sandbox-survival.sh - name: Upload install log on failure - if: always() # diag v4: capture artifact on exit 0 too + # if: always() (not failure()) so the runner has a guaranteed follow-up + # step to dispatch on success. Without one, the runner sits in its + # log-batch flush window (~60s) before reporting the test step's exit + # code, eating into the 900s wall-clock budget and causing flakes. + # if-no-files-found: ignore makes the upload a no-op when the log + # doesn't exist (the typical success path). + if: always() uses: actions/upload-artifact@v4 with: name: sandbox-survival-install-log diff --git a/test/e2e/test-sandbox-survival.sh b/test/e2e/test-sandbox-survival.sh index 40c6f53e63c..621ccd4d511 100755 --- a/test/e2e/test-sandbox-survival.sh +++ b/test/e2e/test-sandbox-survival.sh @@ -714,31 +714,6 @@ echo "========================================" if [ "$FAIL" -eq 0 ]; then printf '\n\033[1;32m Sandbox survival PASSED — all state persisted, live inference verified before AND after gateway restart.\033[0m\n' printf '\033[1;32m Issues validated: #486, #888, #859, #1086\033[0m\n' - - # ── DIAGNOSTIC v3: pinpoint where the post-printf delay actually lives ───── - # Hypothesis: bash exits cleanly within ms, but runner takes ~60s to detect - # the exit (possibly due to processing buffered stdout from k3s log spam in - # Phase 6). EXIT trap records the moment bash actually exits to a file the - # next workflow step can read; comparing that to the runner's "Process - # completed" timestamp tells us if the gap is bash-side or runner-side. - EXIT_MARKER=/tmp/nemoclaw-e2e-install.log - trap 'echo "TRAP_EXIT pid=$$ ts=$(date +%s.%N) exit_arg=$?" >> "$EXIT_MARKER"' EXIT - - # Quick fd dump for completeness — keep tiny so it can't affect timing. - { - printf '\n=== EXIT-DIAG v3 pid=%s ts=%s ===\n' "$$" "$(date +%s.%N)" - ls -la "/proc/$$/fd" 2>&1 || true - ps -eo pid,ppid,pgid,sid,stat,comm 2>&1 | grep -E "PID|runner|test|timeout|bash|node|openshell|Runner" || true - } >> "$EXIT_MARKER" 2>&1 - - printf ' [diag v3] about to exit, ts=%s\n' "$(date +%s.%N)" - - # v4: exit 0 (the actual code path on real success). EXIT trap will record - # the moment bash invokes exit. If trap timestamp == diag timestamp, bash - # exits immediately and the 50s+ PR-specific extra tail is runner-side. - # If trap fires 50s+ after diag, bash is hanging in fflush() before exit. - # Keep the artifact path populated by the trap so this still uploads even - # without forced failure: failing post-step covers it on the SIGTERM path. exit 0 else printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" From c7c6125a744251e7fee2e7faa6f074e87cdc853c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 28 Apr 2026 12:04:00 -0700 Subject: [PATCH 08/11] ci: restore sandbox-survival failure artifact upload --- .github/workflows/nightly-e2e.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index 767546f1afb..b61a6d3a234 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -158,13 +158,7 @@ jobs: run: bash test/e2e/test-sandbox-survival.sh - name: Upload install log on failure - # if: always() (not failure()) so the runner has a guaranteed follow-up - # step to dispatch on success. Without one, the runner sits in its - # log-batch flush window (~60s) before reporting the test step's exit - # code, eating into the 900s wall-clock budget and causing flakes. - # if-no-files-found: ignore makes the upload a no-op when the log - # doesn't exist (the typical success path). - if: always() + if: failure() uses: actions/upload-artifact@v4 with: name: sandbox-survival-install-log From 98c41ec235a2f91bed361c0d3a10fb8b9b7283bb Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 28 Apr 2026 13:23:20 -0700 Subject: [PATCH 09/11] fix(cli): bound OpenShell probes in status and destroy --- src/lib/openshell.test.ts | 55 ++++++++++++++++++++++++++++ src/lib/openshell.ts | 13 +++++++ src/nemoclaw.ts | 77 +++++++++++++++++++++++++++++++++------ 3 files changed, 133 insertions(+), 12 deletions(-) diff --git a/src/lib/openshell.test.ts b/src/lib/openshell.test.ts index 066c968be91..8e346afc086 100644 --- a/src/lib/openshell.test.ts +++ b/src/lib/openshell.test.ts @@ -39,6 +39,16 @@ function stubSpawnSync(spec: SpawnResultSpec): OpenshellSpawnSync { return () => makeSpawnResult(spec); } +function stubSpawnSyncWithOptions( + spec: SpawnResultSpec, + onOptions: (options: Parameters[2]) => void, +): OpenshellSpawnSync { + return (_command, _args, options) => { + onOptions(options); + return makeSpawnResult(spec); + }; +} + function exitWithCode(code: number): never { throw new Error(`exit:${code}`); } @@ -94,6 +104,25 @@ describe("openshell helpers", () => { expect(result.status).toBe(0); }); + it("passes timeout through to run and capture calls", () => { + const seen: number[] = []; + runOpenshellCommand("openshell", ["status"], { + timeout: 1234, + spawnSyncImpl: stubSpawnSyncWithOptions( + { status: 0, stdout: "ok\n", stderr: "" }, + (options) => seen.push(options.timeout as number), + ), + }); + captureOpenshellCommand("openshell", ["status"], { + timeout: 5678, + spawnSyncImpl: stubSpawnSyncWithOptions( + { status: 0, stdout: "ok\n", stderr: "" }, + (options) => seen.push(options.timeout as number), + ), + }); + expect(seen).toEqual([1234, 5678]); + }); + it("uses the injected exit handler on failure", () => { expect(() => runOpenshellCommand("openshell", ["status"], { @@ -142,6 +171,32 @@ describe("openshell helpers", () => { expect(errors).toEqual([" Failed to start openshell status: spawn ENOENT"]); }); + it("returns timed-out capture output when a timeout was requested", () => { + const result = captureOpenshellCommand("openshell", ["status"], { + timeout: 100, + spawnSyncImpl: stubSpawnSync({ + status: null, + stdout: "", + stderr: "timed out\n", + error: new Error("ETIMEDOUT"), + }), + }); + expect(result).toEqual({ status: 1, output: "timed out" }); + }); + + it("returns ignored run spawn failures instead of exiting", () => { + const result = runOpenshellCommand("openshell", ["status"], { + ignoreError: true, + spawnSyncImpl: stubSpawnSync({ + status: null, + stdout: "", + stderr: "", + error: new Error("ETIMEDOUT"), + }), + }); + expect(result.error?.message).toBe("ETIMEDOUT"); + }); + it("reads the installed openshell version through the capture helper", () => { const version = getInstalledOpenshellVersion("openshell", { spawnSyncImpl: stubSpawnSync({ diff --git a/src/lib/openshell.ts b/src/lib/openshell.ts index 82161a8b175..01414e2a6c0 100644 --- a/src/lib/openshell.ts +++ b/src/lib/openshell.ts @@ -25,10 +25,12 @@ interface OpenshellSpawnOptions { export interface RunOpenshellOptions extends OpenshellSpawnOptions { stdio?: SpawnSyncOptions["stdio"]; ignoreError?: boolean; + timeout?: number; } export interface CaptureOpenshellOptions extends OpenshellSpawnOptions { ignoreError?: boolean; + timeout?: number; } export interface CaptureOpenshellResult { @@ -87,8 +89,12 @@ export function runOpenshellCommand( env: { ...process.env, ...opts.env }, encoding: "utf-8", stdio: opts.stdio ?? "inherit", + timeout: opts.timeout, }); if (result.error) { + if (opts.ignoreError) { + return result; + } return handleSpawnError(binary, args, result.error, opts); } if (result.status !== 0 && !opts.ignoreError) { @@ -111,8 +117,15 @@ export function captureOpenshellCommand( env: { ...process.env, ...opts.env }, encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"], + timeout: opts.timeout, }); if (result.error) { + if (opts.ignoreError || opts.timeout !== undefined) { + return { + status: result.status ?? 1, + output: `${result.stdout || ""}${opts.ignoreError ? "" : result.stderr || ""}`.trim(), + }; + } return handleSpawnError(binary, args, result.error, opts); } return { diff --git a/src/nemoclaw.ts b/src/nemoclaw.ts index 4975ae12b3f..8b7ba34ab32 100644 --- a/src/nemoclaw.ts +++ b/src/nemoclaw.ts @@ -135,6 +135,7 @@ const REMOTE_UNINSTALL_URL = buildVersionedUninstallUrl(getVersion()); let OPENSHELL_BIN: string | null = null; const NEMOCLAW_GATEWAY_NAME = "nemoclaw"; const DASHBOARD_FORWARD_PORT = String(DASHBOARD_PORT); +const OPENSHELL_PROBE_TIMEOUT_MS = 15_000; function getOpenshellBinary(): string { if (!OPENSHELL_BIN) { @@ -180,7 +181,10 @@ function cleanupGatewayAfterLastSandbox() { } function hasNoLiveSandboxes() { - const liveList = captureOpenshell(["sandbox", "list"], { ignoreError: true }); + const liveList = captureOpenshell(["sandbox", "list"], { + ignoreError: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }); if (liveList.status !== 0) { return false; } @@ -489,7 +493,10 @@ async function recoverRegistryFromLiveGateway( } let recoveredFromGateway = 0; - const liveList = captureOpenshell(["sandbox", "list"], { ignoreError: true }); + const liveList = captureOpenshell(["sandbox", "list"], { + ignoreError: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }); const liveNames = Array.from(parseLiveSandboxNames(liveList.output)); for (const name of liveNames) { const metadata = metadataByName.get(name) || undefined; @@ -551,8 +558,10 @@ function getActiveGatewayName(output = ""): string | null { } function getNamedGatewayLifecycleState() { - const status = captureOpenshell(["status"]); - const gatewayInfo = captureOpenshell(["gateway", "info", "-g", "nemoclaw"]); + const status = captureOpenshell(["status"], { timeout: OPENSHELL_PROBE_TIMEOUT_MS }); + const gatewayInfo = captureOpenshell(["gateway", "info", "-g", "nemoclaw"], { + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }); const cleanStatus = stripAnsi(status.output); const activeGateway = getActiveGatewayName(status.output); const connected = /^\s*Status:\s*Connected\b/im.test(cleanStatus); @@ -607,7 +616,10 @@ async function recoverNamedGatewayRuntime() { return { recovered: true, before, after: before, attempted: false }; } - runOpenshell(["gateway", "select", "nemoclaw"], { ignoreError: true }); + runOpenshell(["gateway", "select", "nemoclaw"], { + ignoreError: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }); let after = getNamedGatewayLifecycleState(); if (after.state === "healthy_named") { process.env.OPENSHELL_GATEWAY = "nemoclaw"; @@ -625,7 +637,10 @@ async function recoverNamedGatewayRuntime() { // Fall through to the lifecycle re-check below so we preserve the // existing recovery result shape and emit the correct classification. } - runOpenshell(["gateway", "select", "nemoclaw"], { ignoreError: true }); + runOpenshell(["gateway", "select", "nemoclaw"], { + ignoreError: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }); after = getNamedGatewayLifecycleState(); if (after.state === "healthy_named") { process.env.OPENSHELL_GATEWAY = "nemoclaw"; @@ -638,7 +653,9 @@ async function recoverNamedGatewayRuntime() { /** Query sandbox presence and return its output with the live enforced policy. */ function getSandboxGatewayState(sandboxName: string) { - const result = captureOpenshell(["sandbox", "get", sandboxName]); + const result = captureOpenshell(["sandbox", "get", sandboxName], { + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }); let output = result.output; if (result.status === 0) { // `openshell sandbox get` returns the immutable baseline policy from sandbox @@ -648,6 +665,7 @@ function getSandboxGatewayState(sandboxName: string) { // Sandbox info above it. (#1132) const livePolicy = captureOpenshell(["policy", "get", "--full", sandboxName], { ignoreError: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, }); if (livePolicy.status === 0 && livePolicy.output.trim()) { const rawLines = String(output).split("\n"); @@ -709,7 +727,10 @@ function reconcileMissingAgainstNamedGateway( ) { const lifecycle = getNamedGatewayLifecycleState(); if (lifecycle.state === "connected_other") { - runOpenshell(["gateway", "select", "nemoclaw"], { ignoreError: true }); + runOpenshell(["gateway", "select", "nemoclaw"], { + ignoreError: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }); const retry = getSandboxGatewayState(sandboxName); if (retry.state === "present") { return { ...retry, recoveredGateway: true, recoveryVia: "select" }; @@ -1340,7 +1361,12 @@ function showStatus() { showStatusCommand({ listSandboxes: () => registry.listSandboxes(), getLiveInference: () => - parseGatewayInference(captureOpenshell(["inference", "get"], { ignoreError: true }).output), + parseGatewayInference( + captureOpenshell(["inference", "get"], { + ignoreError: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }).output, + ), showServiceStatus, checkMessagingBridgeHealth, backfillAndFindOverlaps, @@ -1366,7 +1392,12 @@ async function listSandboxes(): Promise { await listSandboxesCommand({ recoverRegistryEntries: () => recoverRegistryEntries(), getLiveInference: () => - parseGatewayInference(captureOpenshell(["inference", "get"], { ignoreError: true }).output), + parseGatewayInference( + captureOpenshell(["inference", "get"], { + ignoreError: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }).output, + ), loadLastSession: () => onboardSession.loadSession(), getActiveSessionCount: sessionDeps ? (name: string) => { @@ -1582,7 +1613,10 @@ async function sandboxConnect( async function sandboxStatus(sandboxName: string) { const sb = registry.getSandbox(sandboxName); const live = parseGatewayInference( - captureOpenshell(["inference", "get"], { ignoreError: true }).output, + captureOpenshell(["inference", "get"], { + ignoreError: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }).output, ); const currentModel = (live && live.model) || (sb && sb.model) || "unknown"; const currentProvider = (live && live.provider) || (sb && sb.provider) || "unknown"; @@ -2462,6 +2496,26 @@ function removeSandboxImage(sandboxName: string) { async function sandboxDestroy(sandboxName: string, args: string[] = []): Promise { const skipConfirm = args.includes("--yes") || args.includes("--force"); + const sb = registry.getSandbox(sandboxName); + + if (!sb) { + const liveList = captureOpenshell(["sandbox", "list"], { + ignoreError: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }); + if (liveList.status !== 0 || !parseLiveSandboxNames(liveList.output).has(sandboxName)) { + const session = onboardSession.loadSession(); + if (session && session.sandboxName === sandboxName) { + onboardSession.updateSession((s: Session) => { + s.sandboxName = null; + return s; + }); + } + console.log(` Sandbox '${sandboxName}' is not registered locally or live in OpenShell.`); + console.log(` ${G}✓${R} Sandbox '${sandboxName}' destroyed`); + return; + } + } // Active session detection — enrich the confirmation prompt if sessions are active let activeSessionCount = 0; @@ -2497,7 +2551,6 @@ async function sandboxDestroy(sandboxName: string, args: string[] = []): Promise } } - const sb = registry.getSandbox(sandboxName); if (sb && sb.nimContainer) { console.log(` Stopping NIM for '${sandboxName}'...`); nim.stopNimContainerByName(sb.nimContainer); From 8bcfb66bcf974ec4fd1c909230f6989d1b7f77ac Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 28 Apr 2026 16:13:06 -0700 Subject: [PATCH 10/11] fix(onboard): preserve active dashboard forwards on reuse --- src/lib/onboard.ts | 4 +- test/onboard.test.ts | 104 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index ac2963b6ae4..3df8ec6bf5c 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -6280,7 +6280,7 @@ function findDashboardForwardOwner( /** * Parse `openshell forward list` output into a Map. - * Only includes running forwards — stopped/stale entries are ignored so + * Only includes active/running forwards — stopped/stale entries are ignored so * they don't block port allocation or cause false "range exhausted" errors. * * Output format (columns separated by whitespace): @@ -6295,7 +6295,7 @@ function getOccupiedPorts(forwardListOutput: string | null): Map // parts: [sandbox, bind, port, pid, status...] if (parts.length < 3 || !/^\d+$/.test(parts[2])) continue; const status = (parts[4] || "").toLowerCase(); - if (status !== "running") continue; + if (status !== "running" && status !== "active") continue; occupied.set(parts[2], parts[0]); } return occupied; diff --git a/test/onboard.test.ts b/test/onboard.test.ts index 7a57f851c48..c9fcb0c562a 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -4720,6 +4720,110 @@ const { createSandbox } = require(${onboardPath}); ); }); + it("reuses the stored dashboard port when OpenShell reports the existing forward as active", async () => { + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-reuse-active-port-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "reuse-active-dashboard-port.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "onboard.js")); + const runnerPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "runner.js")); + const registryPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "registry.js")); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + const script = String.raw` +const runner = require(${runnerPath}); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +const registry = require(${registryPath}); + +const commands = []; +const updates = []; +runner.run = (command, opts = {}) => { + commands.push({ command: _n(command), env: opts.env || null }); + return { status: 0 }; +}; +runner.runCapture = (command) => { + const text = _n(command); + if (text.includes("sandbox get my-assistant")) return "my-assistant"; + if (text.includes("sandbox list")) return "my-assistant Ready"; + if (text.includes("forward list")) return [ + "SANDBOX BIND PORT PID STATUS", + "install 127.0.0.1 18789 11111 active", + "other 127.0.0.1 18790 11112 active", + "third 127.0.0.1 18791 11113 active", + "my-assistant 127.0.0.1 18795 12345 active", + ].join("\n"); + if (text.includes("lsof -i :18789") || text.includes("lsof -i :18790") || text.includes("lsof -i :18791") || text.includes("lsof -i :18795")) { + return "COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME\nopenshell 12345 runner 10u IPv4 TCP 127.0.0.1:18795 (LISTEN)"; + } + return ""; +}; +registry.getSandbox = () => ({ + name: "my-assistant", + model: "gpt-5.4", + provider: "nvidia-prod", + gpuEnabled: false, + dashboardPort: 18795, +}); +registry.updateSandbox = (_name, patch) => { + updates.push(patch); + return true; +}; + +const { createSandbox } = require(${onboardPath}); + +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + delete process.env.CHAT_UI_URL; + const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + console.log(JSON.stringify({ sandboxName, commands, updates, chatUiUrl: process.env.CHAT_UI_URL })); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + assert.ok( + !result.stderr.includes("Port 18795 is taken"), + `stored port should not be reported as taken:\n${result.stderr}`, + ); + const payload = parseStdoutJson<{ + sandboxName: string; + commands: CommandEntry[]; + updates: Array<{ dashboardPort?: number }>; + chatUiUrl: string; + }>(result.stdout); + assert.equal(payload.sandboxName, "my-assistant"); + assert.equal(payload.chatUiUrl, "http://127.0.0.1:18795"); + assert.ok( + payload.commands.some((entry: CommandEntry) => + entry.command.includes("forward start --background 18795 my-assistant"), + ), + "expected reused sandbox to keep its stored dashboard port 18795", + ); + assert.ok( + payload.commands.every((entry: CommandEntry) => !entry.command.includes("18792")), + "did not expect dashboard port ratcheting to a different port", + ); + assert.deepEqual(payload.updates.at(-1), { dashboardPort: 18795 }); + }); + it("prints resume guidance when sandbox image upload times out", () => { const errors: string[] = []; const originalError = console.error; From 7264a1da7aeee836120467b4eada80c6aaae10dd Mon Sep 17 00:00:00 2001 From: Test User Date: Tue, 28 Apr 2026 22:57:11 -0700 Subject: [PATCH 11/11] test(onboard): allow WSL dashboard forward target --- test/onboard.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/onboard.test.ts b/test/onboard.test.ts index cae4e966131..27ace3a6763 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -4835,7 +4835,7 @@ const { createSandbox } = require(${onboardPath}); assert.equal(payload.chatUiUrl, "http://127.0.0.1:18795"); assert.ok( payload.commands.some((entry: CommandEntry) => - entry.command.includes("forward start --background 18795 my-assistant"), + /forward start --background (?:0\.0\.0\.0:)?18795 my-assistant/.test(entry.command), ), "expected reused sandbox to keep its stored dashboard port 18795", );