fix(onboard): preserve concurrent instance gateway and dashboard during onboard

.github/workflows/nightly-e2e.yaml

@@ -68,6 +68,14 @@
 #   openclaw-anthropic-inference-switch-e2e
 #                            Switches a running OpenClaw sandbox to a compatible
 #                            Anthropic Messages provider and verifies agent traffic.
+#   openclaw-skill-cli-e2e   Validates workspace-installed OpenClaw skills survive sandbox
+#                            lifecycle through OPENCLAW_HOME/STATE_DIR/WORKSPACE_DIR pinning
+#                            (#4766 / #4709). Seven-phase deterministic skill-CLI exercise
+#                            inside a real onboarded sandbox (install, list, info, check).
+#   channels-add-remove-e2e  Telegram/Discord/Slack channel add/remove lifecycle plus
+#                            gateway-credential reuse on rebuild (#4745 / #3895). Exercises
+#                            the path where the host env credential is empty but the
+#                            gateway already holds the provider credential.
 #   issue-4434-tui-unreachable-inference-e2e
 #                            Recreates #4434's NVIDIA endpoint firewall block and verifies
 #                            OpenClaw TUI shows a visible error and stops the active spinner.
@@ -77,6 +85,10 @@
 #   launchable-smoke-e2e     Community install path (brev-launchable-ci-cpu.sh) on ubuntu-latest.
 #   gpu-e2e                  Local Ollama inference on an NVKS ephemeral GPU runner.
 #   gpu-double-onboard-e2e   Ollama proxy token consistency after re-onboard (#2553).
+#   concurrent-gateway-ports-e2e
+#                            Two sandboxes coexisting on the same host with distinct
+#                            NEMOCLAW_GATEWAY_PORT values; verifies per-instance
+#                            gateway and dashboard segregation.
 #   notify-on-failure        Auto-creates a GitHub issue when any E2E job fails.
 #
 # Runs directly on the runner (not inside Docker) because OpenShell bootstraps
@@ -126,7 +138,8 @@ on:
           rebuild-hermes-stale-base-e2e, double-onboard-e2e, onboard-repair-e2e,
           onboard-resume-e2e, onboard-negative-paths-e2e, runtime-overrides-e2e,
           credential-sanitization-e2e, telegram-injection-e2e, overlayfs-autofix-e2e,
-          device-auth-health-e2e, launchable-smoke-e2e, gpu-e2e, gpu-double-onboard-e2e
+          device-auth-health-e2e, launchable-smoke-e2e, gpu-e2e, gpu-double-onboard-e2e,
+          concurrent-gateway-ports-e2e
         required: false
         type: string
         default: ""
@@ -2063,6 +2076,58 @@ jobs:
           path: /tmp/nemoclaw-gpu-double-onboard-test.log
           if-no-files-found: ignore
 
+  concurrent-gateway-ports-e2e:
+    if: >-
+      github.repository == 'NVIDIA/NemoClaw' &&
+      (github.event_name != 'workflow_dispatch' ||
+       inputs.jobs == '' ||
+       contains(format(',{0},', inputs.jobs), ',concurrent-gateway-ports-e2e,'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      NEMOCLAW_NON_INTERACTIVE: "1"
+      NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
+      NEMOCLAW_E2E_PHASE_TIMEOUT: "1200"
+    steps:
+      - *target-ref-checkout
+      - *dockerhub-auth-step
+      - name: Install NemoClaw
+        env:
+          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+          NEMOCLAW_NON_INTERACTIVE: "1"
+          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
+        run: bash install.sh --non-interactive --yes-i-accept-third-party-software
+      - name: Run concurrent gateway ports E2E test
+        env:
+          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+        run: |
+          [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true
+          export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"
+          [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"
+          [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH"
+          bash test/e2e/test-concurrent-gateway-ports.sh
+      - name: Upload sandbox A onboard log on failure
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: concurrent-gateway-ports-sandbox-a-onboard-log
+          path: /tmp/e2e-cgp-a-onboard.log
+          if-no-files-found: ignore
+      - name: Upload sandbox B onboard log on failure
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: concurrent-gateway-ports-sandbox-b-onboard-log
+          path: /tmp/e2e-cgp-b-onboard.log
+          if-no-files-found: ignore
+      - name: Upload sandbox B destroy log on failure
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: concurrent-gateway-ports-sandbox-b-destroy-log
+          path: /tmp/e2e-cgp-b-destroy.log
+          if-no-files-found: ignore
+
   notify-on-failure:
     runs-on: ubuntu-latest
     needs:
@@ -2131,6 +2196,7 @@ jobs:
         launchable-smoke-e2e,
         gpu-e2e,
         gpu-double-onboard-e2e,
+        concurrent-gateway-ports-e2e,
       ]
     if: ${{ always() && github.event_name == 'schedule' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}
     permissions:
@@ -2245,6 +2311,7 @@ jobs:
         launchable-smoke-e2e,
         gpu-e2e,
         gpu-double-onboard-e2e,
+        concurrent-gateway-ports-e2e,
       ]
     if: ${{ always() && github.event_name == 'workflow_dispatch' }}
     permissions:
@@ -2416,6 +2483,7 @@ jobs:
         launchable-smoke-e2e,
         gpu-e2e,
         gpu-double-onboard-e2e,
+        concurrent-gateway-ports-e2e,
       ]
     if: ${{ always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}
     permissions:

src/lib/onboard.ts

@@ -439,6 +439,8 @@ const {
   findAvailableDashboardPort,
   preflightDashboardPortRangeAvailability,
 } = require("./onboard/dashboard-port") as typeof import("./onboard/dashboard-port");
+const { tryCleanupOrphanedDashboardForward } =
+  require("./onboard/orphaned-dashboard-forward") as typeof import("./onboard/orphaned-dashboard-forward");
 const { destroyGatewayForReuse } = require("./onboard/gateway-cleanup") as typeof import("./onboard/gateway-cleanup");
 const { applyPreflightGatewayCleanup } =
   require("./onboard/preflight-gateway-cleanup-decision") as typeof import("./onboard/preflight-gateway-cleanup-decision");
@@ -2079,20 +2081,12 @@ async function preflight(
       // if its command line contains "openshell" to avoid killing unrelated SSH
       // tunnels the user may have set up on the same port. (#1950)
       if (port === DASHBOARD_PORT && portCheck.process === "ssh" && portCheck.pid) {
-        // Use `ps` to get the command line — works on Linux, macOS, and WSL.
-        const cmdline = captureProcessArgs(portCheck.pid);
-        if (cmdline.includes("openshell")) {
-          console.log(
-            `  Cleaning up orphaned SSH port-forward on port ${port} (PID ${portCheck.pid})...`,
-          );
-          run(["kill", String(portCheck.pid)], { ignoreError: true });
-          sleepSeconds(1);
-          portCheck = await checkPortAvailable(port, portCheckOptions);
-          if (portCheck.ok) {
-            console.log(`  ✓ Port ${port} available after orphaned forward cleanup (${label})`);
-            continue;
-          }
-        }
+        const outcome = await tryCleanupOrphanedDashboardForward({
+          port, pid: portCheck.pid, label, portCheckOptions,
+          captureProcessArgs, runCaptureOpenshell, run, sleepSeconds, checkPortAvailable,
+        });
+        if (outcome.kind === "killed-still-blocked") portCheck = outcome.portCheck;
+        else if (outcome.kind !== "not-openshell") continue;
       }
       console.error("");
       console.error(`  !! Port ${port} is not available.`);

src/lib/onboard/machine/handlers/gateway.test.ts

@@ -314,4 +314,22 @@ describe("handleGatewayState", () => {
       "  Replacing legacy OpenShell gateway metadata with Docker-driver gateway.",
     );
   });
+
+  it("does not retire a foreign-active Docker-driver gateway (concurrent instances)", async () => {
+    const { deps, calls } = createDeps({
+      isLinuxDockerDriverGatewayEnabled: vi.fn(() => true),
+      reconcileGatewayGpuReuseForGpuIntent: vi.fn(
+        () => "foreign-active" as GatewayReuseState,
+      ),
+    });
+
+    const result = await handleGatewayState(baseOptions(deps, "foreign-active"));
+
+    expect(calls.retireLegacy).not.toHaveBeenCalled();
+    expect(calls.note).not.toHaveBeenCalledWith(
+      "  Replacing legacy OpenShell gateway metadata with Docker-driver gateway.",
+    );
+    expect(calls.startGateway).toHaveBeenCalledOnce();
+    expect(result.gatewayReuseState).toBe("missing");
+  });
 });

src/lib/onboard/machine/handlers/gateway.ts

@@ -204,10 +204,16 @@ export async function handleGatewayState<Gpu>({
       }
     }
     await deps.startRecordedStep("gateway");
-    if (deps.isLinuxDockerDriverGatewayEnabled() && gatewayReuseState !== "missing") {
+    if (
+      deps.isLinuxDockerDriverGatewayEnabled() &&
+      gatewayReuseState !== "missing" &&
+      gatewayReuseState !== "foreign-active"
+    ) {
       deps.note("  Replacing legacy OpenShell gateway metadata with Docker-driver gateway.");
       deps.retireLegacyGatewayForDockerDriverUpgrade();
       gatewayReuseState = "missing";
+    } else if (gatewayReuseState === "foreign-active") {
+      gatewayReuseState = "missing";
     }
     await withGatewayTrace(gatewayReuseState, gpuPassthrough, () =>
       deps.startGateway(gpu, { gpuPassthrough }),

src/lib/onboard/orphaned-dashboard-forward.test.ts

@@ -0,0 +1,124 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it, vi } from "vitest";
+
+import {
+  tryCleanupOrphanedDashboardForward,
+  type OrphanedDashboardForwardDeps,
+} from "../../../dist/lib/onboard/orphaned-dashboard-forward";
+
+function forwardListWith(
+  entries: Array<{ sandbox: string; port: number; status?: string }>,
+): string {
+  const header = "SANDBOX   BIND        PORT   PID    STATUS";
+  const rows = entries.map(
+    (e) => `${e.sandbox}  127.0.0.1   ${e.port}   1234   ${e.status ?? "running"}`,
+  );
+  return [header, ...rows].join("\n");
+}
+
+interface MakeDepsOverrides {
+  cmdline?: string;
+  listFn?: () => string;
+  portCheckResult?: { ok: boolean; process?: string; pid?: number | null; reason?: string };
+}
+
+function makeDeps(overrides: MakeDepsOverrides = {}) {
+  const calls = {
+    captureProcessArgs: vi.fn((_pid: number) => overrides.cmdline ?? "ssh -L openshell-forward 18789:..."),
+    runCaptureOpenshell: vi.fn(
+      overrides.listFn ?? (() => forwardListWith([])),
+    ) as OrphanedDashboardForwardDeps["runCaptureOpenshell"],
+    run: vi.fn() as unknown as OrphanedDashboardForwardDeps["run"],
+    sleepSeconds: vi.fn() as OrphanedDashboardForwardDeps["sleepSeconds"],
+    checkPortAvailable: vi.fn(async () => overrides.portCheckResult ?? { ok: true }) as unknown as OrphanedDashboardForwardDeps["checkPortAvailable"],
+    log: vi.fn(),
+  };
+  const deps: OrphanedDashboardForwardDeps = {
+    port: 18789,
+    pid: 4321,
+    label: "Test dashboard",
+    captureProcessArgs: calls.captureProcessArgs,
+    runCaptureOpenshell: calls.runCaptureOpenshell,
+    run: calls.run,
+    sleepSeconds: calls.sleepSeconds,
+    checkPortAvailable: calls.checkPortAvailable,
+    log: calls.log,
+  };
+  return { deps, calls };
+}
+
+describe("tryCleanupOrphanedDashboardForward", () => {
+  it("returns not-openshell when the listener is unrelated SSH", async () => {
+    const { deps, calls } = makeDeps({ cmdline: "ssh -L 18789:remote-host:80 user@bastion" });
+    const outcome = await tryCleanupOrphanedDashboardForward(deps);
+    expect(outcome).toEqual({ kind: "not-openshell" });
+    expect(calls.runCaptureOpenshell).not.toHaveBeenCalled();
+    expect(calls.run).not.toHaveBeenCalled();
+  });
+
+  it("returns list-failed and skips the kill when forward list throws", async () => {
+    const { deps, calls } = makeDeps({
+      listFn: () => {
+        throw new Error("gateway probe timed out");
+      },
+    });
+    const outcome = await tryCleanupOrphanedDashboardForward(deps);
+    expect(outcome).toEqual({ kind: "list-failed" });
+    expect(calls.run).not.toHaveBeenCalled();
+    expect(calls.checkPortAvailable).not.toHaveBeenCalled();
+    expect(calls.log).toHaveBeenCalledWith(
+      expect.stringContaining("Could not enumerate OpenShell forwards"),
+    );
+  });
+
+  it("does not pass ignoreError to runCaptureOpenshell (failures must throw to be classified list-failed)", async () => {
+    const { deps, calls } = makeDeps();
+    await tryCleanupOrphanedDashboardForward(deps);
+    expect(calls.runCaptureOpenshell).toHaveBeenCalledWith(
+      ["forward", "list"],
+      expect.objectContaining({ timeout: 10_000, suppressOutput: true }),
+    );
+    expect(calls.runCaptureOpenshell).not.toHaveBeenCalledWith(
+      ["forward", "list"],
+      expect.objectContaining({ ignoreError: true }),
+    );
+  });
+
+  it("returns owned-by-live when another live sandbox owns the port", async () => {
+    const { deps, calls } = makeDeps({
+      listFn: () => forwardListWith([{ sandbox: "other-sandbox", port: 18789 }]),
+    });
+    const outcome = await tryCleanupOrphanedDashboardForward(deps);
+    expect(outcome).toEqual({ kind: "owned-by-live", owner: "other-sandbox" });
+    expect(calls.run).not.toHaveBeenCalled();
+    expect(calls.checkPortAvailable).not.toHaveBeenCalled();
+  });
+
+  it("returns killed-cleared when the kill frees the port", async () => {
+    const { deps, calls } = makeDeps({ portCheckResult: { ok: true } });
+    const outcome = await tryCleanupOrphanedDashboardForward(deps);
+    expect(outcome).toEqual({ kind: "killed-cleared" });
+    expect(calls.run).toHaveBeenCalledWith(["kill", "4321"], { ignoreError: true });
+    expect(calls.sleepSeconds).toHaveBeenCalledWith(1);
+    expect(calls.checkPortAvailable).toHaveBeenCalledWith(18789, undefined);
+  });
+
+  it("returns killed-still-blocked when the kill ran but the port stayed blocked", async () => {
+    const refreshedCheck = { ok: false, process: "ssh", pid: 4321, reason: "still busy" };
+    const { deps, calls } = makeDeps({ portCheckResult: refreshedCheck });
+    const outcome = await tryCleanupOrphanedDashboardForward(deps);
+    expect(outcome).toEqual({ kind: "killed-still-blocked", portCheck: refreshedCheck });
+    expect(calls.run).toHaveBeenCalledTimes(1);
+  });
+
+  it("ignores non-live forward statuses when deciding ownership", async () => {
+    const { deps, calls } = makeDeps({
+      listFn: () => forwardListWith([{ sandbox: "other-sandbox", port: 18789, status: "stopped" }]),
+    });
+    const outcome = await tryCleanupOrphanedDashboardForward(deps);
+    expect(outcome.kind).toBe("killed-cleared");
+    expect(calls.run).toHaveBeenCalledTimes(1);
+  });
+});