fix(onboard): clean up stale openclaw-gateway dashboard listeners

docs/reference/troubleshooting.md

@@ -174,6 +174,9 @@ The NemoClaw dashboard uses port `18789` by default and the gateway uses port `8
 If another sandbox already owns the dashboard port, onboarding scans ports `18789` through `18799` and uses the next free port.
 If all ports in that range are occupied, the error lists the owner for each port and suggests using `--control-ui-port` with a port outside the range.
 
+When a previous onboard, upgrade, or sandbox crash leaves a stale `openclaw-gateway` host process holding the dashboard port, `nemoclaw onboard --fresh`, `nemoclaw <name> destroy` (when destroying the last sandbox), and `nemoclaw uninstall` automatically sweep the dashboard port range and signal `SIGTERM` then `SIGKILL` to recover.
+The sweep only targets processes owned by the current user whose command line matches `openclaw-gateway` or `openshell forward` markers, and skips dashboard ports owned by other live sandboxes.
+
 If a non-NemoClaw process is already bound to the dashboard port or the gateway port, identify the conflicting process, verify it is safe to stop, and terminate it:
 
 ```console

src/lib/actions/sandbox/destroy.ts

@@ -16,6 +16,7 @@ import * as onboardSession from "../../state/onboard-session";
 import type { Session } from "../../state/onboard-session";
 import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts";
 import { DASHBOARD_PORT } from "../../core/ports";
+import { stopStaleDashboardListeners } from "../../onboard/stale-gateway-cleanup";
 import * as registry from "../../state/registry";
 import { resolveOpenshell } from "../../adapters/openshell/resolve";
 import { parseLiveSandboxNames } from "../../runtime-recovery";
@@ -69,7 +70,7 @@ function dockerDriverGatewayPidFile(): string {
 function isDockerDriverGatewayPid(pid: number): boolean {
   try {
     const cmdline = fs.readFileSync(`/proc/${pid}/cmdline`, "utf-8").replace(/\0/g, " ");
-    return cmdline.includes("openshell-gateway");
+    return cmdline.includes("openshell-gateway") || cmdline.includes("openclaw-gateway");
   } catch {
     return false;
   }
@@ -112,6 +113,11 @@ function cleanupGatewayAfterLastSandbox(): void {
     ignoreError: true,
     stdio: ["ignore", "ignore", "ignore"],
   });
+  // After the cooperative forward-stop, sweep the dashboard port range for
+  // stale host-side gateway-forward processes (#3397, #3398). The forward-stop
+  // above releases ports the live openshell tracks; this catches orphans whose
+  // openshell record was lost across upgrades or failed onboards.
+  stopStaleDashboardListeners();
   if (process.platform === "linux") {
     stopDockerDriverGatewayProcess();
     const removeResult = runOpenshell(["gateway", "remove", NEMOCLAW_GATEWAY_NAME], {

src/lib/actions/uninstall/run-plan.ts

@@ -11,6 +11,7 @@ import { getAgentBranding, type AgentBranding } from "../../cli/branding";
 import { sleepMs } from "../../core/wait";
 import { defaultUninstallPaths, NEMOCLAW_OLLAMA_MODELS, NEMOCLAW_PROVIDERS, type UninstallPaths } from "../../domain/uninstall/paths";
 import { buildUninstallPlan, type UninstallPlan } from "../../domain/uninstall/plan";
+import { stopStaleDashboardListeners } from "../../onboard/stale-gateway-cleanup";
 import { classifyShimPath, type FileSystemDeps } from "./plan";
 
 export interface RunResult {
@@ -556,6 +557,14 @@ function executePlan(plan: UninstallPlan, paths: UninstallPaths, options: Uninst
       stopHelperServices(paths, runtime);
       removeGlob(paths.helperServiceGlob, runtime);
       stopMatchingPids(`openshell.*forward.*${runtime.env.NEMOCLAW_DASHBOARD_PORT || "18789"}`, runtime, "local OpenShell forward processes");
+      stopStaleDashboardListeners({
+        run: runtime.run,
+        kill: runtime.kill,
+        env: runtime.env,
+        log: runtime.log,
+        warn: runtime.warn,
+        commandExists: runtime.commandExists,
+      });
       stopOrphanedOpenShell(runtime);
       stopOllamaAuthProxy(paths, runtime);
     } else if (step.name === "OpenShell resources") {

src/lib/onboard.ts

@@ -20,6 +20,7 @@ const {
   cleanupTempDir,
   secureTempFile,
 }: typeof import("./onboard/temp-files") = require("./onboard/temp-files");
+const { stopStaleDashboardListenersForSandbox } = require("./onboard/stale-gateway-cleanup");
 const {
   buildDirectGpuPolicyYaml,
   buildDirectSandboxGpuProofCommands,
@@ -11537,11 +11538,6 @@ async function onboard(opts: OnboardOptions = {}): Promise<void> {
         break;
       }
 
-      // Prompt for the sandbox name and show the review gate BEFORE
-      // setupInference runs upsertProvider / `inference set` on the gateway.
-      // On retry (inferenceResult.retry === "selection") the user is re-prompted
-      // for provider/model above and sees this gate again with the new config.
-      // See #2221 (CodeRabbit).
       if (!sandboxName) {
         sandboxName = await promptValidatedSandboxName(agent);
       }
@@ -11725,10 +11721,16 @@ async function onboard(opts: OnboardOptions = {}): Promise<void> {
         current.messagingChannelConfig = messagingChannelConfig;
         return current;
       });
+      if (!sandboxName) {
+        sandboxName = await promptValidatedSandboxName(agent);
+      }
       if (typeof model !== "string" || typeof provider !== "string") {
         console.error("  Inference selection is incomplete; cannot create sandbox.");
         process.exit(1);
       }
+      if (fresh) {
+        stopStaleDashboardListenersForSandbox(registry.listSandboxes().sandboxes, sandboxName);
+      }
       sandboxName = await createSandbox(
         gpu,
         model,
@@ -11743,9 +11745,6 @@ async function onboard(opts: OnboardOptions = {}): Promise<void> {
         sandboxGpuConfig,
       );
       webSearchConfig = nextWebSearchConfig;
-      // Persist model and provider after the sandbox entry exists in the registry.
-      // updateSandbox() silently no-ops when the entry is missing, so this must
-      // run after createSandbox() / registerSandbox() — not before. Fixes #1881.
       registry.updateSandbox(sandboxName, {
         model,
         provider,

src/lib/onboard/stale-gateway-cleanup.test.ts

@@ -0,0 +1,237 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it, vi } from "vitest";
+
+import {
+  getProtectedDashboardPortsForSandbox,
+  stopStaleDashboardListeners,
+  type RunResult,
+  type StaleGatewayDeps,
+} from "./stale-gateway-cleanup";
+
+interface RunArgs {
+  command: string;
+  args: string[];
+}
+
+function emptyResult(): RunResult {
+  return { status: 0, stdout: "", stderr: "" };
+}
+
+function makeRun(
+  responses: Map<string, RunResult | ((args: string[]) => RunResult)>,
+): {
+  run: StaleGatewayDeps["run"];
+  calls: RunArgs[];
+} {
+  const calls: RunArgs[] = [];
+  const run: StaleGatewayDeps["run"] = (command, args) => {
+    calls.push({ command, args });
+    const key = `${command} ${args.join(" ")}`;
+    const exact = responses.get(key);
+    if (exact !== undefined) {
+      return typeof exact === "function" ? exact(args) : exact;
+    }
+    // Default lsof to empty (no listener) and ps to non-existent pid.
+    if (command === "lsof") return { status: 1, stdout: "", stderr: "" };
+    if (command === "ps") return { status: 1, stdout: "", stderr: "" };
+    return emptyResult();
+  };
+  return { run, calls };
+}
+
+function baseDeps(overrides: Partial<StaleGatewayDeps> = {}): StaleGatewayDeps {
+  return {
+    run: overrides.run ?? (() => emptyResult()),
+    kill: overrides.kill ?? vi.fn(() => true),
+    env: overrides.env ?? { USER: "tester" },
+    log: overrides.log ?? vi.fn(),
+    warn: overrides.warn ?? vi.fn(),
+    commandExists: overrides.commandExists ?? (() => true),
+  };
+}
+
+describe("stopStaleDashboardListeners", () => {
+  it("protects registered sandbox dashboard ports except the fresh target", () => {
+    expect(
+      getProtectedDashboardPortsForSandbox(
+        [
+          { name: "my-assistant", dashboardPort: 18789 },
+          { name: "other", dashboardPort: 18790 },
+          { name: "missing" },
+        ],
+        "my-assistant",
+      ),
+    ).toEqual([18790]);
+  });
+
+  it("returns without scanning when lsof is missing", () => {
+    const run = vi.fn(() => emptyResult());
+    const result = stopStaleDashboardListeners({
+      ...baseDeps({ commandExists: () => false }),
+      run,
+    });
+    expect(result).toEqual({ stopped: [], skippedForeignPids: [], skippedNonMatchingPids: [], skippedProtectedPorts: [] });
+    expect(run).not.toHaveBeenCalled();
+  });
+
+  it("returns no work when lsof reports no listeners across the range", () => {
+    const { run } = makeRun(new Map());
+    const result = stopStaleDashboardListeners(baseDeps({ run }));
+    expect(result).toEqual({ stopped: [], skippedForeignPids: [], skippedNonMatchingPids: [], skippedProtectedPorts: [] });
+  });
+
+  it("kills a user-owned openclaw-gateway process holding the dashboard port", () => {
+    const kill = vi.fn<(pid: number, signal?: NodeJS.Signals | number) => boolean>(() => true);
+    let pidGone = false;
+    const responses = new Map<string, RunResult | ((args: string[]) => RunResult)>([
+      ["lsof -ti :18789 -sTCP:LISTEN", { status: 0, stdout: "2522044\n", stderr: "" }],
+      [
+        "ps -p 2522044 -o user=",
+        { status: 0, stdout: "tester\n", stderr: "" },
+      ],
+      [
+        "ps -p 2522044 -o args=",
+        { status: 0, stdout: "openclaw-gateway --port 18789\n", stderr: "" },
+      ],
+      [
+        "ps -p 2522044 -o pid=",
+        () => (pidGone ? { status: 1, stdout: "", stderr: "" } : { status: 0, stdout: "2522044\n", stderr: "" }),
+      ],
+    ]);
+    const { run } = makeRun(responses);
+    const customKill: StaleGatewayDeps["kill"] = (pid, signal) => {
+      kill(pid, signal);
+      if (signal === "SIGTERM") pidGone = true;
+      return true;
+    };
+    const log = vi.fn();
+    const result = stopStaleDashboardListeners({
+      ...baseDeps({ run, kill: customKill, log }),
+    });
+    expect(result.stopped).toEqual([2522044]);
+    expect(kill).toHaveBeenCalledWith(2522044, "SIGTERM");
+    expect(log).toHaveBeenCalledWith(expect.stringContaining("Stopped stale dashboard gateway listener 2522044"));
+  });
+
+  it("escalates to SIGKILL when SIGTERM does not free the process", () => {
+    const sentSignals: NodeJS.Signals[] = [];
+    let pidGone = false;
+    const responses = new Map<string, RunResult | ((args: string[]) => RunResult)>([
+      ["lsof -ti :18789 -sTCP:LISTEN", { status: 0, stdout: "999\n", stderr: "" }],
+      ["ps -p 999 -o user=", { status: 0, stdout: "tester\n", stderr: "" }],
+      [
+        "ps -p 999 -o args=",
+        { status: 0, stdout: "openclaw-gateway\n", stderr: "" },
+      ],
+      [
+        "ps -p 999 -o pid=",
+        () => (pidGone ? { status: 1, stdout: "", stderr: "" } : { status: 0, stdout: "999\n", stderr: "" }),
+      ],
+    ]);
+    const { run } = makeRun(responses);
+    const kill: StaleGatewayDeps["kill"] = (_pid, signal) => {
+      sentSignals.push(signal as NodeJS.Signals);
+      if (signal === "SIGKILL") pidGone = true;
+      return true;
+    };
+    const result = stopStaleDashboardListeners({
+      ...baseDeps({ run, kill }),
+    });
+    expect(result.stopped).toEqual([999]);
+    expect(sentSignals).toEqual(["SIGTERM", "SIGKILL"]);
+  });
+
+  it("skips PIDs owned by another user", () => {
+    const kill = vi.fn(() => true);
+    const responses = new Map<string, RunResult | ((args: string[]) => RunResult)>([
+      ["lsof -ti :18789 -sTCP:LISTEN", { status: 0, stdout: "42\n", stderr: "" }],
+      ["ps -p 42 -o user=", { status: 0, stdout: "root\n", stderr: "" }],
+    ]);
+    const { run } = makeRun(responses);
+    const result = stopStaleDashboardListeners({
+      ...baseDeps({ run, kill, env: { USER: "tester" } }),
+    });
+    expect(result).toEqual({ stopped: [], skippedForeignPids: [42], skippedNonMatchingPids: [], skippedProtectedPorts: [] });
+    expect(kill).not.toHaveBeenCalled();
+  });
+
+  it("does not kill listeners on ports protected by registered sandboxes (#3260)", () => {
+    const kill = vi.fn(() => true);
+    const responses = new Map<string, RunResult | ((args: string[]) => RunResult)>([
+      ["lsof -ti :18789 -sTCP:LISTEN", { status: 0, stdout: "4242\n", stderr: "" }],
+    ]);
+    const { run, calls } = makeRun(responses);
+    const result = stopStaleDashboardListeners(
+      { ...baseDeps({ run, kill }) },
+      { protectedPorts: [18789] },
+    );
+    expect(result.stopped).toEqual([]);
+    expect(result.skippedProtectedPorts).toEqual([18789]);
+    expect(kill).not.toHaveBeenCalled();
+    expect(calls.some((c) => c.command === "ps" && c.args.includes("user="))).toBe(false);
+    expect(calls.some((c) => c.command === "ps" && c.args.includes("args="))).toBe(false);
+  });
+
+  it("does not revisit a PID seen on a protected port when it also appears on an unprotected port", () => {
+    const kill = vi.fn(() => true);
+    const responses = new Map<string, RunResult | ((args: string[]) => RunResult)>([
+      ["lsof -ti :18789 -sTCP:LISTEN", { status: 0, stdout: "777\n", stderr: "" }],
+      ["lsof -ti :18790 -sTCP:LISTEN", { status: 0, stdout: "777\n", stderr: "" }],
+    ]);
+    const { run } = makeRun(responses);
+    const result = stopStaleDashboardListeners(
+      { ...baseDeps({ run, kill }) },
+      { protectedPorts: [18789] },
+    );
+    expect(result.stopped).toEqual([]);
+    expect(result.skippedProtectedPorts).toEqual([18789]);
+    expect(kill).not.toHaveBeenCalled();
+  });
+
+  it("skips PIDs whose cmdline does not match a gateway marker", () => {
+    const kill = vi.fn(() => true);
+    const responses = new Map<string, RunResult | ((args: string[]) => RunResult)>([
+      ["lsof -ti :18789 -sTCP:LISTEN", { status: 0, stdout: "777\n", stderr: "" }],
+      ["ps -p 777 -o user=", { status: 0, stdout: "tester\n", stderr: "" }],
+      ["ps -p 777 -o args=", { status: 0, stdout: "python -m http.server 18789\n", stderr: "" }],
+    ]);
+    const { run } = makeRun(responses);
+    const result = stopStaleDashboardListeners({
+      ...baseDeps({ run, kill }),
+    });
+    expect(result).toEqual({ stopped: [], skippedForeignPids: [], skippedNonMatchingPids: [777], skippedProtectedPorts: [] });
+    expect(kill).not.toHaveBeenCalled();
+  });
+
+  it("does not double-process a PID that appears on multiple ports in the range", () => {
+    let pidGone = false;
+    const responses = new Map<string, RunResult | ((args: string[]) => RunResult)>([
+      ["lsof -ti :18789 -sTCP:LISTEN", { status: 0, stdout: "501\n", stderr: "" }],
+      ["lsof -ti :18790 -sTCP:LISTEN", { status: 0, stdout: "501\n", stderr: "" }],
+      ["ps -p 501 -o user=", { status: 0, stdout: "tester\n", stderr: "" }],
+      ["ps -p 501 -o args=", { status: 0, stdout: "openclaw-gateway\n", stderr: "" }],
+      [
+        "ps -p 501 -o pid=",
+        () => (pidGone ? { status: 1, stdout: "", stderr: "" } : { status: 0, stdout: "501\n", stderr: "" }),
+      ],
+    ]);
+    const { run, calls } = makeRun(responses);
+    const kill: StaleGatewayDeps["kill"] = (_pid, signal) => {
+      if (signal === "SIGTERM") pidGone = true;
+      return true;
+    };
+    const result = stopStaleDashboardListeners({
+      ...baseDeps({ run, kill }),
+    });
+    expect(result.stopped).toEqual([501]);
+    // user=/args= lookup must run exactly once per unique PID even when seen twice.
+    expect(
+      calls.filter(
+        (c) =>
+          c.command === "ps" && c.args[0] === "-p" && c.args[1] === "501" && c.args[3] === "user=",
+      ),
+    ).toHaveLength(1);
+  });
+});