fix(tunnel): stop agent gateway forwards on shutdown

src/lib/tunnel/agent-forward-stop.test.ts

@@ -0,0 +1,192 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it, vi } from "vitest";
+
+import { stopAgentForwardPortsForStop } from "./agent-forward-stop";
+
+function forwardList(entries: Array<{ sandbox: string; port: number; status?: string }>): string {
+  return [
+    "SANDBOX BIND PORT PID STATUS",
+    ...entries.map(
+      (entry, index) =>
+        `${entry.sandbox} 127.0.0.1 ${String(entry.port)} ${String(1000 + index)} ${
+          entry.status ?? "running"
+        }`,
+    ),
+  ].join("\n");
+}
+
+describe("stopAgentForwardPortsForStop", () => {
+  it("stops declared and runtime dashboard forwards with sandbox-scoped commands", () => {
+    const runOpenshell = vi.fn();
+    const runCaptureOpenshell = vi.fn(() =>
+      forwardList([
+        { sandbox: "nemohermes", port: 18789 },
+        { sandbox: "nemohermes", port: 8642 },
+        { sandbox: "nemohermes", port: 18792 },
+      ]),
+    );
+    const info = vi.fn<(message: string) => void>();
+
+    stopAgentForwardPortsForStop("nemohermes", {
+      getSessionAgent: () => ({
+        displayName: "Hermes Agent",
+        forward_ports: [18789, 8642, "8642", 0, 80, 70000, "not-a-port"],
+      }),
+      getAgentDisplayName: (agent) => agent?.displayName ?? "OpenClaw",
+      getSandbox: () => ({
+        dashboardPort: 18792,
+        gatewayName: "nemoclaw-18080",
+        gatewayPort: 18080,
+      }),
+      resolveOpenshell: () => "/usr/local/bin/openshell",
+      runOpenshell,
+      runCaptureOpenshell,
+      confirmPortReleased: () => true,
+      info,
+    });
+
+    expect(runOpenshell).toHaveBeenCalledTimes(3);
+    expect(runOpenshell).toHaveBeenNthCalledWith(
+      1,
+      ["forward", "stop", "18789", "nemohermes", "--gateway", "nemoclaw-18080"],
+      {
+        ignoreError: true,
+        suppressOutput: true,
+      },
+    );
+    expect(runOpenshell).toHaveBeenNthCalledWith(
+      2,
+      ["forward", "stop", "8642", "nemohermes", "--gateway", "nemoclaw-18080"],
+      {
+        ignoreError: true,
+        suppressOutput: true,
+      },
+    );
+    expect(runOpenshell).toHaveBeenNthCalledWith(
+      3,
+      ["forward", "stop", "18792", "nemohermes", "--gateway", "nemoclaw-18080"],
+      {
+        ignoreError: true,
+        suppressOutput: true,
+      },
+    );
+    expect(runCaptureOpenshell).toHaveBeenCalledTimes(3);
+    expect(runCaptureOpenshell).toHaveBeenCalledWith(
+      ["forward", "list", "--gateway", "nemoclaw-18080"],
+      expect.any(Object),
+    );
+    expect(info.mock.calls.map((call) => call[0]).join("\n")).toContain(
+      "Stopped Hermes Agent host port forward 8642",
+    );
+  });
+
+  it("skips OpenClaw or agents without declared forwards", () => {
+    const resolveOpenshell = vi.fn(() => "/usr/local/bin/openshell");
+    const runOpenshell = vi.fn();
+
+    stopAgentForwardPortsForStop("openclaw-sandbox", {
+      getSessionAgent: () => null,
+      resolveOpenshell,
+      runOpenshell,
+    });
+
+    stopAgentForwardPortsForStop("empty-agent", {
+      getSessionAgent: () => ({ displayName: "Empty Agent", forward_ports: [] }),
+      getSandbox: () => ({ gatewayPort: 8080 }),
+      resolveOpenshell,
+      runOpenshell,
+    });
+
+    expect(resolveOpenshell).not.toHaveBeenCalled();
+    expect(runOpenshell).not.toHaveBeenCalled();
+  });
+
+  it("leaves forwards alone when OpenShell reports a different owner", () => {
+    const runOpenshell = vi.fn();
+    const warn = vi.fn<(message: string) => void>();
+
+    stopAgentForwardPortsForStop("nemohermes", {
+      getSessionAgent: () => ({ displayName: "Hermes Agent", forward_ports: [8642] }),
+      getAgentDisplayName: (agent) => agent?.displayName ?? "OpenClaw",
+      getSandbox: () => ({ gatewayPort: 8080 }),
+      resolveOpenshell: () => "/usr/local/bin/openshell",
+      runOpenshell,
+      runCaptureOpenshell: () => forwardList([{ sandbox: "other-sandbox", port: 8642 }]),
+      warn,
+    });
+
+    expect(runOpenshell).not.toHaveBeenCalled();
+    expect(warn.mock.calls.map((call) => call[0]).join("\n")).toContain(
+      "belongs to another sandbox",
+    );
+  });
+
+  it("does not stop a forward when ownership cannot be enumerated", () => {
+    const runOpenshell = vi.fn();
+    const warn = vi.fn<(message: string) => void>();
+
+    stopAgentForwardPortsForStop("nemohermes", {
+      getSessionAgent: () => ({ displayName: "Hermes Agent", forward_ports: [8642] }),
+      getAgentDisplayName: (agent) => agent?.displayName ?? "OpenClaw",
+      getSandbox: () => ({ gatewayPort: 8080 }),
+      resolveOpenshell: () => "/usr/local/bin/openshell",
+      runOpenshell,
+      runCaptureOpenshell: () => {
+        throw new Error("forward list failed");
+      },
+      warn,
+    });
+
+    expect(runOpenshell).not.toHaveBeenCalled();
+    expect(warn.mock.calls.map((call) => call[0]).join("\n")).toContain(
+      "Could not enumerate OpenShell forwards",
+    );
+  });
+
+  it("warns instead of claiming success when the listener remains bound (#6392)", () => {
+    const runOpenshell = vi.fn();
+    const info = vi.fn<(message: string) => void>();
+    const warn = vi.fn<(message: string) => void>();
+
+    stopAgentForwardPortsForStop("nemohermes", {
+      getSessionAgent: () => ({ displayName: "Hermes Agent", forward_ports: [8642] }),
+      getAgentDisplayName: (agent) => agent?.displayName ?? "OpenClaw",
+      getSandbox: () => ({ gatewayName: "nemoclaw-18080", gatewayPort: 18080 }),
+      resolveOpenshell: () => "/usr/local/bin/openshell",
+      runOpenshell,
+      runCaptureOpenshell: () => forwardList([{ sandbox: "nemohermes", port: 8642 }]),
+      confirmPortReleased: () => false,
+      info,
+      warn,
+    });
+
+    expect(runOpenshell).toHaveBeenCalledWith(
+      ["forward", "stop", "8642", "nemohermes", "--gateway", "nemoclaw-18080"],
+      expect.any(Object),
+    );
+    expect(info).not.toHaveBeenCalled();
+    expect(warn.mock.calls.map((call) => call[0]).join("\n")).toContain(
+      "Could not confirm Hermes Agent host port forward 8642 was released within 5 seconds",
+    );
+  });
+
+  it("fails closed when the sandbox gateway binding is unavailable", () => {
+    const resolveOpenshell = vi.fn(() => "/usr/local/bin/openshell");
+    const warn = vi.fn<(message: string) => void>();
+
+    stopAgentForwardPortsForStop("nemohermes", {
+      getSessionAgent: () => ({ displayName: "Hermes Agent", forward_ports: [8642] }),
+      getAgentDisplayName: (agent) => agent?.displayName ?? "OpenClaw",
+      getSandbox: () => null,
+      resolveOpenshell,
+      warn,
+    });
+
+    expect(resolveOpenshell).not.toHaveBeenCalled();
+    expect(warn.mock.calls.map((call) => call[0]).join("\n")).toContain(
+      "cannot safely stop Hermes Agent host port forwards",
+    );
+  });
+});

src/lib/tunnel/agent-forward-stop.ts

@@ -0,0 +1,195 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { spawnSync } from "node:child_process";
+
+import { resolveOpenshell } from "../adapters/openshell/resolve";
+import { OPENSHELL_OPERATION_TIMEOUT_MS } from "../adapters/openshell/timeouts";
+import * as agentRuntime from "../agent/runtime";
+import { waitUntil } from "../core/wait";
+import {
+  bestEffortForwardStopForSandbox,
+  type ForwardListRunner,
+  type ForwardStopRunner,
+} from "../onboard/forward-cleanup";
+import { resolveSandboxGatewayName } from "../onboard/gateway-binding";
+import * as registry from "../state/registry";
+import { defaultProbePortFree } from "./gateway-port-confirmation";
+
+type Reporter = (message: string) => void;
+
+type AgentWithForwards = {
+  displayName?: string;
+  forward_ports?: unknown;
+};
+
+type SandboxWithDashboardPort = {
+  dashboardPort?: unknown;
+  gatewayName?: string | null;
+  gatewayPort?: number | null;
+};
+
+type StopAgentForwardPortsDeps = {
+  getSessionAgent?: (sandboxName?: string) => AgentWithForwards | null;
+  getAgentDisplayName?: (agent: AgentWithForwards | null) => string;
+  getSandbox?: (sandboxName: string) => SandboxWithDashboardPort | null;
+  resolveOpenshell?: () => string | null;
+  runOpenshell?: ForwardStopRunner;
+  runCaptureOpenshell?: ForwardListRunner;
+  confirmPortReleased?: (port: number) => boolean;
+  info?: Reporter;
+  warn?: Reporter;
+};
+
+const FORWARD_RELEASE_TIMEOUT_MS = 5000;
+const FORWARD_RELEASE_POLL_MS = 250;
+
+function confirmForwardPortReleased(port: number): boolean {
+  const now = Date.now;
+  return waitUntil(() => defaultProbePortFree(port), {
+    deadlineMs: now() + FORWARD_RELEASE_TIMEOUT_MS,
+    maxAttempts: 20,
+    initialIntervalMs: FORWARD_RELEASE_POLL_MS,
+    maxIntervalMs: FORWARD_RELEASE_POLL_MS,
+    backoffFactor: 1,
+    now,
+  });
+}
+
+function getAgentForwardPorts(agent: AgentWithForwards, dashboardPort: unknown): number[] {
+  const ports = new Set<number>();
+  const candidates = [
+    ...(Array.isArray(agent.forward_ports) ? agent.forward_ports : []),
+    dashboardPort,
+  ];
+  for (const rawPort of candidates) {
+    const port =
+      typeof rawPort === "number"
+        ? rawPort
+        : typeof rawPort === "string" && /^\d+$/.test(rawPort.trim())
+          ? Number(rawPort.trim())
+          : NaN;
+    if (Number.isInteger(port) && port >= 1024 && port <= 65535) {
+      ports.add(port);
+    }
+  }
+  return [...ports];
+}
+
+function makeRunOpenshell(openshell: string): ForwardStopRunner {
+  return (args, opts) => {
+    const result = spawnSync(openshell, args, {
+      encoding: "utf-8",
+      stdio: opts.suppressOutput ? "ignore" : "inherit",
+      timeout: OPENSHELL_OPERATION_TIMEOUT_MS,
+    });
+    if (!opts.ignoreError && result.status !== 0) {
+      throw new Error(`openshell ${args.join(" ")} failed`);
+    }
+    return result;
+  };
+}
+
+function makeRunCaptureOpenshell(openshell: string): ForwardListRunner {
+  return (args, opts) => {
+    const result = spawnSync(openshell, args, {
+      encoding: "utf-8",
+      stdio: ["ignore", "pipe", "pipe"],
+      timeout: opts.timeout,
+    });
+    if (result.status !== 0) {
+      throw new Error(`openshell ${args.join(" ")} failed`);
+    }
+    return result.stdout || "";
+  };
+}
+
+export function stopAgentForwardPortsForStop(
+  sandboxName: string | undefined,
+  deps: StopAgentForwardPortsDeps = {},
+): void {
+  if (!sandboxName) return;
+
+  const getSessionAgent = deps.getSessionAgent ?? agentRuntime.getSessionAgent;
+  const agent = getSessionAgent(sandboxName);
+  if (!agent) return;
+
+  const warn = deps.warn ?? (() => {});
+  const info = deps.info ?? (() => {});
+  const displayName = deps.getAgentDisplayName
+    ? deps.getAgentDisplayName(agent)
+    : agentRuntime.getAgentDisplayName(
+        agent as Parameters<typeof agentRuntime.getAgentDisplayName>[0],
+      );
+  const getSandbox = deps.getSandbox ?? registry.getSandbox;
+  const sandbox = getSandbox(sandboxName);
+  if (!sandbox) {
+    warn(
+      `Could not resolve sandbox '${sandboxName}' - cannot safely stop ${displayName} host port forwards.`,
+    );
+    return;
+  }
+  let gatewayName: string;
+  try {
+    gatewayName = resolveSandboxGatewayName(sandbox);
+  } catch (error) {
+    warn(
+      `Could not resolve the OpenShell gateway for sandbox '${sandboxName}': ` +
+        `${(error as Error).message ?? String(error)}. ` +
+        `Skipping ${displayName} host port forward cleanup.`,
+    );
+    return;
+  }
+
+  const ports = getAgentForwardPorts(agent, sandbox.dashboardPort);
+  if (ports.length === 0) return;
+
+  const openshell = (deps.resolveOpenshell ?? resolveOpenshell)();
+  if (!openshell) {
+    warn(`openshell not found - cannot stop ${displayName} host port forwards.`);
+    return;
+  }
+
+  const runOpenshell = deps.runOpenshell ?? makeRunOpenshell(openshell);
+  const runCaptureOpenshell = deps.runCaptureOpenshell ?? makeRunCaptureOpenshell(openshell);
+  const scopedRunOpenshell: ForwardStopRunner = (args, opts) =>
+    runOpenshell([...args, "--gateway", gatewayName], opts);
+  const scopedRunCaptureOpenshell: ForwardListRunner = (args, opts) =>
+    runCaptureOpenshell([...args, "--gateway", gatewayName], opts);
+  const confirmPortReleased = deps.confirmPortReleased ?? confirmForwardPortReleased;
+
+  for (const port of ports) {
+    const result = bestEffortForwardStopForSandbox(
+      scopedRunOpenshell,
+      scopedRunCaptureOpenshell,
+      port,
+      sandboxName,
+    );
+    if (result === "owned-other") {
+      warn(
+        `Keeping ${displayName} host port forward ${String(port)}; it belongs to another sandbox.`,
+      );
+      continue;
+    }
+    if (result === "list-failed") {
+      warn(
+        `Could not enumerate OpenShell forwards; skipping ${displayName} host port forward ${String(
+          port,
+        )} cleanup.`,
+      );
+      continue;
+    }
+
+    if (!confirmPortReleased(port)) {
+      warn(
+        `Could not confirm ${displayName} host port forward ${String(port)} was released ` +
+          `within ${String(FORWARD_RELEASE_TIMEOUT_MS / 1000)} seconds; ` +
+          "the listener may still be running.",
+      );
+    } else if (result === "stopped") {
+      info(
+        `Stopped ${displayName} host port forward ${String(port)} for sandbox '${sandboxName}'.`,
+      );
+    }
+  }
+}