diff --git a/docs/reference/troubleshooting.md b/docs/reference/troubleshooting.md index cbf0a9526f7..dd275277450 100644 --- a/docs/reference/troubleshooting.md +++ b/docs/reference/troubleshooting.md @@ -174,6 +174,9 @@ The NemoClaw dashboard uses port `18789` by default and the gateway uses port `8 If another sandbox already owns the dashboard port, onboarding scans ports `18789` through `18799` and uses the next free port. If all ports in that range are occupied, the error lists the owner for each port and suggests using `--control-ui-port` with a port outside the range. +When a previous onboard, upgrade, or sandbox crash leaves a stale `openclaw-gateway` host process holding the dashboard port, `nemoclaw onboard --fresh`, `nemoclaw destroy` (when destroying the last sandbox), and `nemoclaw uninstall` automatically sweep the dashboard port range and signal `SIGTERM` then `SIGKILL` to recover. +The sweep only targets processes owned by the current user whose command line matches `openclaw-gateway` or `openshell forward` markers, and skips dashboard ports owned by other live sandboxes. + If a non-NemoClaw process is already bound to the dashboard port or the gateway port, identify the conflicting process, verify it is safe to stop, and terminate it: ```console diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index d359b87df2d..557999701dc 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -16,6 +16,7 @@ import * as onboardSession from "../../state/onboard-session"; import type { Session } from "../../state/onboard-session"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; import { DASHBOARD_PORT } from "../../core/ports"; +import { stopStaleDashboardListeners } from "../../onboard/stale-gateway-cleanup"; import * as registry from "../../state/registry"; import { resolveOpenshell } from "../../adapters/openshell/resolve"; import { parseLiveSandboxNames } from "../../runtime-recovery"; @@ -69,7 +70,7 @@ function dockerDriverGatewayPidFile(): string { function isDockerDriverGatewayPid(pid: number): boolean { try { const cmdline = fs.readFileSync(`/proc/${pid}/cmdline`, "utf-8").replace(/\0/g, " "); - return cmdline.includes("openshell-gateway"); + return cmdline.includes("openshell-gateway") || cmdline.includes("openclaw-gateway"); } catch { return false; } @@ -112,6 +113,11 @@ function cleanupGatewayAfterLastSandbox(): void { ignoreError: true, stdio: ["ignore", "ignore", "ignore"], }); + // After the cooperative forward-stop, sweep the dashboard port range for + // stale host-side gateway-forward processes (#3397, #3398). The forward-stop + // above releases ports the live openshell tracks; this catches orphans whose + // openshell record was lost across upgrades or failed onboards. + stopStaleDashboardListeners(); if (process.platform === "linux") { stopDockerDriverGatewayProcess(); const removeResult = runOpenshell(["gateway", "remove", NEMOCLAW_GATEWAY_NAME], { diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index 9642b7d225e..f8fd7fcccb0 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -11,6 +11,7 @@ import { getAgentBranding, type AgentBranding } from "../../cli/branding"; import { sleepMs } from "../../core/wait"; import { defaultUninstallPaths, NEMOCLAW_OLLAMA_MODELS, NEMOCLAW_PROVIDERS, type UninstallPaths } from "../../domain/uninstall/paths"; import { buildUninstallPlan, type UninstallPlan } from "../../domain/uninstall/plan"; +import { stopStaleDashboardListeners } from "../../onboard/stale-gateway-cleanup"; import { classifyShimPath, type FileSystemDeps } from "./plan"; export interface RunResult { @@ -556,6 +557,14 @@ function executePlan(plan: UninstallPlan, paths: UninstallPaths, options: Uninst stopHelperServices(paths, runtime); removeGlob(paths.helperServiceGlob, runtime); stopMatchingPids(`openshell.*forward.*${runtime.env.NEMOCLAW_DASHBOARD_PORT || "18789"}`, runtime, "local OpenShell forward processes"); + stopStaleDashboardListeners({ + run: runtime.run, + kill: runtime.kill, + env: runtime.env, + log: runtime.log, + warn: runtime.warn, + commandExists: runtime.commandExists, + }); stopOrphanedOpenShell(runtime); stopOllamaAuthProxy(paths, runtime); } else if (step.name === "OpenShell resources") { diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index fafa2b9a9dc..4083552b832 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -20,6 +20,7 @@ const { cleanupTempDir, secureTempFile, }: typeof import("./onboard/temp-files") = require("./onboard/temp-files"); +const { stopStaleDashboardListenersForSandbox } = require("./onboard/stale-gateway-cleanup"); const { buildDirectGpuPolicyYaml, buildDirectSandboxGpuProofCommands, @@ -11537,11 +11538,6 @@ async function onboard(opts: OnboardOptions = {}): Promise { break; } - // Prompt for the sandbox name and show the review gate BEFORE - // setupInference runs upsertProvider / `inference set` on the gateway. - // On retry (inferenceResult.retry === "selection") the user is re-prompted - // for provider/model above and sees this gate again with the new config. - // See #2221 (CodeRabbit). if (!sandboxName) { sandboxName = await promptValidatedSandboxName(agent); } @@ -11725,10 +11721,16 @@ async function onboard(opts: OnboardOptions = {}): Promise { current.messagingChannelConfig = messagingChannelConfig; return current; }); + if (!sandboxName) { + sandboxName = await promptValidatedSandboxName(agent); + } if (typeof model !== "string" || typeof provider !== "string") { console.error(" Inference selection is incomplete; cannot create sandbox."); process.exit(1); } + if (fresh) { + stopStaleDashboardListenersForSandbox(registry.listSandboxes().sandboxes, sandboxName); + } sandboxName = await createSandbox( gpu, model, @@ -11743,9 +11745,6 @@ async function onboard(opts: OnboardOptions = {}): Promise { sandboxGpuConfig, ); webSearchConfig = nextWebSearchConfig; - // Persist model and provider after the sandbox entry exists in the registry. - // updateSandbox() silently no-ops when the entry is missing, so this must - // run after createSandbox() / registerSandbox() — not before. Fixes #1881. registry.updateSandbox(sandboxName, { model, provider, diff --git a/src/lib/onboard/stale-gateway-cleanup.test.ts b/src/lib/onboard/stale-gateway-cleanup.test.ts new file mode 100644 index 00000000000..d85426da0ec --- /dev/null +++ b/src/lib/onboard/stale-gateway-cleanup.test.ts @@ -0,0 +1,237 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + getProtectedDashboardPortsForSandbox, + stopStaleDashboardListeners, + type RunResult, + type StaleGatewayDeps, +} from "./stale-gateway-cleanup"; + +interface RunArgs { + command: string; + args: string[]; +} + +function emptyResult(): RunResult { + return { status: 0, stdout: "", stderr: "" }; +} + +function makeRun( + responses: Map RunResult)>, +): { + run: StaleGatewayDeps["run"]; + calls: RunArgs[]; +} { + const calls: RunArgs[] = []; + const run: StaleGatewayDeps["run"] = (command, args) => { + calls.push({ command, args }); + const key = `${command} ${args.join(" ")}`; + const exact = responses.get(key); + if (exact !== undefined) { + return typeof exact === "function" ? exact(args) : exact; + } + // Default lsof to empty (no listener) and ps to non-existent pid. + if (command === "lsof") return { status: 1, stdout: "", stderr: "" }; + if (command === "ps") return { status: 1, stdout: "", stderr: "" }; + return emptyResult(); + }; + return { run, calls }; +} + +function baseDeps(overrides: Partial = {}): StaleGatewayDeps { + return { + run: overrides.run ?? (() => emptyResult()), + kill: overrides.kill ?? vi.fn(() => true), + env: overrides.env ?? { USER: "tester" }, + log: overrides.log ?? vi.fn(), + warn: overrides.warn ?? vi.fn(), + commandExists: overrides.commandExists ?? (() => true), + }; +} + +describe("stopStaleDashboardListeners", () => { + it("protects registered sandbox dashboard ports except the fresh target", () => { + expect( + getProtectedDashboardPortsForSandbox( + [ + { name: "my-assistant", dashboardPort: 18789 }, + { name: "other", dashboardPort: 18790 }, + { name: "missing" }, + ], + "my-assistant", + ), + ).toEqual([18790]); + }); + + it("returns without scanning when lsof is missing", () => { + const run = vi.fn(() => emptyResult()); + const result = stopStaleDashboardListeners({ + ...baseDeps({ commandExists: () => false }), + run, + }); + expect(result).toEqual({ stopped: [], skippedForeignPids: [], skippedNonMatchingPids: [], skippedProtectedPorts: [] }); + expect(run).not.toHaveBeenCalled(); + }); + + it("returns no work when lsof reports no listeners across the range", () => { + const { run } = makeRun(new Map()); + const result = stopStaleDashboardListeners(baseDeps({ run })); + expect(result).toEqual({ stopped: [], skippedForeignPids: [], skippedNonMatchingPids: [], skippedProtectedPorts: [] }); + }); + + it("kills a user-owned openclaw-gateway process holding the dashboard port", () => { + const kill = vi.fn<(pid: number, signal?: NodeJS.Signals | number) => boolean>(() => true); + let pidGone = false; + const responses = new Map RunResult)>([ + ["lsof -ti :18789 -sTCP:LISTEN", { status: 0, stdout: "2522044\n", stderr: "" }], + [ + "ps -p 2522044 -o user=", + { status: 0, stdout: "tester\n", stderr: "" }, + ], + [ + "ps -p 2522044 -o args=", + { status: 0, stdout: "openclaw-gateway --port 18789\n", stderr: "" }, + ], + [ + "ps -p 2522044 -o pid=", + () => (pidGone ? { status: 1, stdout: "", stderr: "" } : { status: 0, stdout: "2522044\n", stderr: "" }), + ], + ]); + const { run } = makeRun(responses); + const customKill: StaleGatewayDeps["kill"] = (pid, signal) => { + kill(pid, signal); + if (signal === "SIGTERM") pidGone = true; + return true; + }; + const log = vi.fn(); + const result = stopStaleDashboardListeners({ + ...baseDeps({ run, kill: customKill, log }), + }); + expect(result.stopped).toEqual([2522044]); + expect(kill).toHaveBeenCalledWith(2522044, "SIGTERM"); + expect(log).toHaveBeenCalledWith(expect.stringContaining("Stopped stale dashboard gateway listener 2522044")); + }); + + it("escalates to SIGKILL when SIGTERM does not free the process", () => { + const sentSignals: NodeJS.Signals[] = []; + let pidGone = false; + const responses = new Map RunResult)>([ + ["lsof -ti :18789 -sTCP:LISTEN", { status: 0, stdout: "999\n", stderr: "" }], + ["ps -p 999 -o user=", { status: 0, stdout: "tester\n", stderr: "" }], + [ + "ps -p 999 -o args=", + { status: 0, stdout: "openclaw-gateway\n", stderr: "" }, + ], + [ + "ps -p 999 -o pid=", + () => (pidGone ? { status: 1, stdout: "", stderr: "" } : { status: 0, stdout: "999\n", stderr: "" }), + ], + ]); + const { run } = makeRun(responses); + const kill: StaleGatewayDeps["kill"] = (_pid, signal) => { + sentSignals.push(signal as NodeJS.Signals); + if (signal === "SIGKILL") pidGone = true; + return true; + }; + const result = stopStaleDashboardListeners({ + ...baseDeps({ run, kill }), + }); + expect(result.stopped).toEqual([999]); + expect(sentSignals).toEqual(["SIGTERM", "SIGKILL"]); + }); + + it("skips PIDs owned by another user", () => { + const kill = vi.fn(() => true); + const responses = new Map RunResult)>([ + ["lsof -ti :18789 -sTCP:LISTEN", { status: 0, stdout: "42\n", stderr: "" }], + ["ps -p 42 -o user=", { status: 0, stdout: "root\n", stderr: "" }], + ]); + const { run } = makeRun(responses); + const result = stopStaleDashboardListeners({ + ...baseDeps({ run, kill, env: { USER: "tester" } }), + }); + expect(result).toEqual({ stopped: [], skippedForeignPids: [42], skippedNonMatchingPids: [], skippedProtectedPorts: [] }); + expect(kill).not.toHaveBeenCalled(); + }); + + it("does not kill listeners on ports protected by registered sandboxes (#3260)", () => { + const kill = vi.fn(() => true); + const responses = new Map RunResult)>([ + ["lsof -ti :18789 -sTCP:LISTEN", { status: 0, stdout: "4242\n", stderr: "" }], + ]); + const { run, calls } = makeRun(responses); + const result = stopStaleDashboardListeners( + { ...baseDeps({ run, kill }) }, + { protectedPorts: [18789] }, + ); + expect(result.stopped).toEqual([]); + expect(result.skippedProtectedPorts).toEqual([18789]); + expect(kill).not.toHaveBeenCalled(); + expect(calls.some((c) => c.command === "ps" && c.args.includes("user="))).toBe(false); + expect(calls.some((c) => c.command === "ps" && c.args.includes("args="))).toBe(false); + }); + + it("does not revisit a PID seen on a protected port when it also appears on an unprotected port", () => { + const kill = vi.fn(() => true); + const responses = new Map RunResult)>([ + ["lsof -ti :18789 -sTCP:LISTEN", { status: 0, stdout: "777\n", stderr: "" }], + ["lsof -ti :18790 -sTCP:LISTEN", { status: 0, stdout: "777\n", stderr: "" }], + ]); + const { run } = makeRun(responses); + const result = stopStaleDashboardListeners( + { ...baseDeps({ run, kill }) }, + { protectedPorts: [18789] }, + ); + expect(result.stopped).toEqual([]); + expect(result.skippedProtectedPorts).toEqual([18789]); + expect(kill).not.toHaveBeenCalled(); + }); + + it("skips PIDs whose cmdline does not match a gateway marker", () => { + const kill = vi.fn(() => true); + const responses = new Map RunResult)>([ + ["lsof -ti :18789 -sTCP:LISTEN", { status: 0, stdout: "777\n", stderr: "" }], + ["ps -p 777 -o user=", { status: 0, stdout: "tester\n", stderr: "" }], + ["ps -p 777 -o args=", { status: 0, stdout: "python -m http.server 18789\n", stderr: "" }], + ]); + const { run } = makeRun(responses); + const result = stopStaleDashboardListeners({ + ...baseDeps({ run, kill }), + }); + expect(result).toEqual({ stopped: [], skippedForeignPids: [], skippedNonMatchingPids: [777], skippedProtectedPorts: [] }); + expect(kill).not.toHaveBeenCalled(); + }); + + it("does not double-process a PID that appears on multiple ports in the range", () => { + let pidGone = false; + const responses = new Map RunResult)>([ + ["lsof -ti :18789 -sTCP:LISTEN", { status: 0, stdout: "501\n", stderr: "" }], + ["lsof -ti :18790 -sTCP:LISTEN", { status: 0, stdout: "501\n", stderr: "" }], + ["ps -p 501 -o user=", { status: 0, stdout: "tester\n", stderr: "" }], + ["ps -p 501 -o args=", { status: 0, stdout: "openclaw-gateway\n", stderr: "" }], + [ + "ps -p 501 -o pid=", + () => (pidGone ? { status: 1, stdout: "", stderr: "" } : { status: 0, stdout: "501\n", stderr: "" }), + ], + ]); + const { run, calls } = makeRun(responses); + const kill: StaleGatewayDeps["kill"] = (_pid, signal) => { + if (signal === "SIGTERM") pidGone = true; + return true; + }; + const result = stopStaleDashboardListeners({ + ...baseDeps({ run, kill }), + }); + expect(result.stopped).toEqual([501]); + // user=/args= lookup must run exactly once per unique PID even when seen twice. + expect( + calls.filter( + (c) => + c.command === "ps" && c.args[0] === "-p" && c.args[1] === "501" && c.args[3] === "user=", + ), + ).toHaveLength(1); + }); +}); diff --git a/src/lib/onboard/stale-gateway-cleanup.ts b/src/lib/onboard/stale-gateway-cleanup.ts new file mode 100644 index 00000000000..a0eb2cbd3c0 --- /dev/null +++ b/src/lib/onboard/stale-gateway-cleanup.ts @@ -0,0 +1,273 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Stale host-side gateway-forward cleanup. + * + * Background: when `openshell forward start` is killed unexpectedly (failed + * onboard, container crash mid-build, upgrade across versions), the host-side + * process that holds the NemoClaw dashboard port can survive. It commonly + * shows up in `ss -tlnp` as `openclaw-gatewa(y)` because the forward shim + * re-execs into the binary it proxies for. The next `nemoclaw onboard` + * detects the port conflict and falls back to a different port, but the new + * sandbox is baked with the original port and never becomes reachable. See + * #3397 and #3398. + * + * This module finds those orphans by scanning the dashboard port range, + * verifying ownership and cmdline, then sending SIGTERM followed by SIGKILL + * with bounded waits — mirroring the proven `tryStopOllamaProxyPid` pattern + * in `src/lib/actions/uninstall/run-plan.ts`. + */ + +import { spawnSync, type SpawnSyncOptions } from "node:child_process"; +import os from "node:os"; + +import { DASHBOARD_PORT_RANGE_END, DASHBOARD_PORT_RANGE_START } from "../core/ports"; +import { sleepMs } from "../core/wait"; + +export interface RunResult { + status: number | null; + stdout: string; + stderr: string; +} + +export interface StaleGatewayDeps { + /** Spawn a command synchronously. Mirrors `child_process.spawnSync` shape. */ + run: (command: string, args: string[], options?: SpawnSyncOptions) => RunResult; + /** Send a signal to a PID. Returns true if the signal was accepted. */ + kill: (pid: number, signal?: NodeJS.Signals | number) => boolean; + /** Environment used for resolving the expected process owner. */ + env: NodeJS.ProcessEnv; + /** Informational log sink (used for successful stops). Defaults to console.log. */ + log?: (message: string) => void; + /** Warning sink for partial failures. Defaults to console.warn. */ + warn?: (message: string) => void; + /** Returns true if the named CLI exists on PATH. Defaults to a `command -v` probe. */ + commandExists?: (command: string) => boolean; +} + +export interface CleanupResult { + stopped: number[]; + skippedForeignPids: number[]; + skippedNonMatchingPids: number[]; + skippedProtectedPorts: number[]; +} + +export interface SandboxDashboardPortEntry { + name: string; + dashboardPort?: number | null; +} + +export interface StaleGatewayOptions { + /** + * Ports that must not be swept even if a matching gateway-forward process is + * holding them. The onboard `--fresh` path passes the dashboard ports of + * currently-registered sandboxes so a fresh onboard for a new name does not + * disrupt the forward of an existing sandbox (#3260). + */ + protectedPorts?: Iterable; +} + +const CMDLINE_MARKERS = ["openclaw-gateway", "openshell-forward", "openshell forward"]; + +const TERM_WAIT_MS = 1000; +const KILL_WAIT_MS = 1000; + +function toRunResult( + result: ReturnType, +): RunResult { + return { + status: result.status, + stdout: typeof result.stdout === "string" ? result.stdout : String(result.stdout ?? ""), + stderr: typeof result.stderr === "string" ? result.stderr : String(result.stderr ?? ""), + }; +} + +function defaultRun( + command: string, + args: string[], + options: SpawnSyncOptions = {}, +): RunResult { + return toRunResult(spawnSync(command, args, { encoding: "utf-8", ...options })); +} + +function defaultKill(pid: number, signal?: NodeJS.Signals | number): boolean { + try { + process.kill(pid, signal); + return true; + } catch { + return false; + } +} + +function defaultCommandExists(command: string, env: NodeJS.ProcessEnv): boolean { + const probe = spawnSync( + "sh", + ["-c", `command -v ${JSON.stringify(command)} >/dev/null 2>&1`], + { env, encoding: "utf-8" }, + ); + return probe.status === 0; +} + +export function defaultStaleGatewayDeps( + overrides: Partial = {}, +): StaleGatewayDeps { + const env = overrides.env ?? process.env; + return { + run: overrides.run ?? defaultRun, + kill: overrides.kill ?? defaultKill, + env, + log: overrides.log, + warn: overrides.warn, + commandExists: overrides.commandExists ?? ((cmd) => defaultCommandExists(cmd, env)), + }; +} + +function parsePidLines(output: string): number[] { + return output + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => /^\d+$/.test(line)) + .map(Number); +} + +function pidOwnedByCurrentUser(pid: number, deps: StaleGatewayDeps): boolean { + const expected = + deps.env.SUDO_USER || deps.env.LOGNAME || deps.env.USER || os.userInfo().username; + if (!expected) return false; + const result = deps.run("ps", ["-p", String(pid), "-o", "user="], { env: deps.env }); + return result.status === 0 && result.stdout.trim() === expected; +} + +function pidExists(pid: number, deps: StaleGatewayDeps): boolean { + return ( + deps.run("ps", ["-p", String(pid), "-o", "pid="], { env: deps.env }).status === 0 + ); +} + +function waitForExit(pid: number, deps: StaleGatewayDeps, timeoutMs: number): boolean { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (!pidExists(pid, deps)) return true; + sleepMs(50); + } + return !pidExists(pid, deps); +} + +function pidCmdlineMatches(pid: number, deps: StaleGatewayDeps): boolean { + const result = deps.run("ps", ["-p", String(pid), "-o", "args="], { env: deps.env }); + if (result.status !== 0) return false; + const cmdline = result.stdout.trim(); + return CMDLINE_MARKERS.some((marker) => cmdline.includes(marker)); +} + +function lsofPidsForPort(port: number, deps: StaleGatewayDeps): number[] { + // Restrict to listening sockets so we never kill a process that is only + // an in-flight client of the port (matches the `-sTCP:LISTEN` pattern in + // preflight). Anything else under SIGTERM/SIGKILL would be unsafe. + const result = deps.run("lsof", ["-ti", `:${port}`, "-sTCP:LISTEN"], { env: deps.env }); + if (result.status !== 0 && result.status !== 1) { + // Status 1 from lsof is "no listeners" — normal. Anything else is a real error. + const warn = deps.warn ?? ((m: string) => console.warn(m)); + const detail = result.stderr.trim() || `status ${String(result.status)}`; + warn(`lsof failed while scanning dashboard port ${port}: ${detail}`); + return []; + } + return parsePidLines(result.stdout); +} + +export function getProtectedDashboardPortsForSandbox( + sandboxes: Iterable, + sandboxName: string, +): number[] { + return Array.from(sandboxes) + .filter((sb) => sb.name !== sandboxName) + .map((sb) => sb.dashboardPort) + .filter((p): p is number => typeof p === "number" && Number.isFinite(p)); +} + +export function stopStaleDashboardListenersForSandbox( + sandboxes: Iterable, + sandboxName: string, + depsOverrides: Partial = {}, +): CleanupResult { + return stopStaleDashboardListeners(depsOverrides, { + protectedPorts: getProtectedDashboardPortsForSandbox(sandboxes, sandboxName), + }); +} + +function tryStopPid(pid: number, deps: StaleGatewayDeps): boolean { + const log = deps.log ?? ((m) => console.log(m)); + const warn = deps.warn ?? ((m) => console.warn(m)); + + deps.kill(pid, "SIGTERM"); + if (waitForExit(pid, deps, TERM_WAIT_MS)) { + log(`Stopped stale dashboard gateway listener ${pid}`); + return true; + } + deps.kill(pid, "SIGKILL"); + if (waitForExit(pid, deps, KILL_WAIT_MS)) { + log(`Stopped stale dashboard gateway listener ${pid} (after SIGKILL)`); + return true; + } + warn(`Failed to stop stale dashboard gateway listener ${pid}`); + return false; +} + +/** + * Scan the dashboard port range for stale host-side gateway-forward processes + * left over from a previous `nemoclaw onboard` / `openshell forward start` and + * stop them. Safe to call repeatedly — if no orphans are found the function + * exits cleanly. + * + * Conservative by design: + * - Only PIDs the current user can signal are considered. + * - Only PIDs whose cmdline matches one of [[CMDLINE_MARKERS]] are killed. + * - Two-phase TERM-then-KILL with bounded waits prevents zombie kills. + * + * When `lsof` is unavailable the scan returns without warning — the caller + * shouldn't block uninstall/destroy on missing tooling. Other unexpected + * states are surfaced through `deps.warn`. + */ +export function stopStaleDashboardListeners( + depsOverrides: Partial = {}, + options: StaleGatewayOptions = {}, +): CleanupResult { + const deps = defaultStaleGatewayDeps(depsOverrides); + const protectedPorts = new Set( + options.protectedPorts ? Array.from(options.protectedPorts).filter(Number.isFinite) : [], + ); + const result: CleanupResult = { + stopped: [], + skippedForeignPids: [], + skippedNonMatchingPids: [], + skippedProtectedPorts: [], + }; + if (deps.commandExists && !deps.commandExists("lsof")) return result; + + const seen = new Set(); + for (let port = DASHBOARD_PORT_RANGE_START; port <= DASHBOARD_PORT_RANGE_END; port += 1) { + if (protectedPorts.has(port)) { + const pids = lsofPidsForPort(port, deps); + if (pids.length > 0) { + result.skippedProtectedPorts.push(port); + for (const pid of pids) seen.add(pid); + } + continue; + } + for (const pid of lsofPidsForPort(port, deps)) { + if (seen.has(pid)) continue; + seen.add(pid); + if (!pidOwnedByCurrentUser(pid, deps)) { + result.skippedForeignPids.push(pid); + continue; + } + if (!pidCmdlineMatches(pid, deps)) { + result.skippedNonMatchingPids.push(pid); + continue; + } + if (tryStopPid(pid, deps)) result.stopped.push(pid); + } + } + return result; +} diff --git a/test/onboard.test.ts b/test/onboard.test.ts index 70f98ae1e65..f70abbe529e 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -4878,6 +4878,25 @@ const { setupInference } = require(${onboardPath}); ); }); + it("runs fresh stale-gateway cleanup after the sandbox name is known but before createSandbox", () => { + const source = fs.readFileSync( + path.join(import.meta.dirname, "..", "src", "lib", "onboard.ts"), + "utf-8", + ); + const promptPos = source.indexOf( + "if (!sandboxName) {\n sandboxName = await promptValidatedSandboxName(agent);", + ); + const cleanupPos = source.indexOf( + "stopStaleDashboardListenersForSandbox(registry.listSandboxes().sandboxes, sandboxName);", + promptPos, + ); + const createPos = source.indexOf("sandboxName = await createSandbox(", promptPos); + + assert.ok(promptPos !== -1, "sandbox-name resolution block not found"); + assert.ok(cleanupPos > promptPos, "fresh cleanup should run after sandboxName is known"); + assert.ok(cleanupPos < createPos, "fresh cleanup should run before createSandbox allocates a port"); + }); + it("defaults GPU passthrough on for detected NVIDIA GPUs unless opted out", () => { const source = fs.readFileSync( path.join(import.meta.dirname, "..", "src", "lib", "onboard.ts"),