From 52577bdda51c4db13358f1b51929e81f0ff9333e Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Fri, 14 Aug 2026 12:46:12 +0000 Subject: [PATCH 1/3] fix(readiness): name the process holding the gateway port The public readiness collector probes the port with lsof skipped, so the conflict diagnostic read an owner of "unknown" from the bind probe while the collector's own unprivileged listener scan already held the owning PID. The message now names each listener and gives the command that stops it, once the operator has confirmed it is not a second NemoClaw gateway. Signed-off-by: Tinson Lai --- docs/reference/system-readiness.mdx | 1 + docs/reference/troubleshooting.mdx | 5 ++ src/lib/readiness/gateway-production.test.ts | 45 ++++++++++++ src/lib/readiness/gateway-production.ts | 73 ++++++++++++++++--- ...rd-gateway-port-conflict-fast-fail.test.ts | 3 + 5 files changed, 118 insertions(+), 9 deletions(-) diff --git a/docs/reference/system-readiness.mdx b/docs/reference/system-readiness.mdx index a0c39eb891d..3582976df71 100644 --- a/docs/reference/system-readiness.mdx +++ b/docs/reference/system-readiness.mdx @@ -253,6 +253,7 @@ Use finding IDs to handle gateway failures without parsing summaries: | `gateway.port.inconclusive` | Warning | NemoClaw could not establish gateway port ownership. | The report includes bounded evidence for the resolved owner and applicable attachment, port, collection, or stale-observation failures. +Port-conflict evidence names each listener that holds the gateway port by process name and PID, and gives the command that stops those PIDs. It omits the gateway state directory, removes process environments, redacts credential-shaped content, bounds diagnostic length, and renders control characters visibly. Managed gateway metadata is reusable only when its endpoint is bound to loopback on the configured gateway port. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 0c53b560624..e70fefc7095 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -318,6 +318,11 @@ If forwarding then fails, onboarding removes the new sandbox and tells you to re When a previous onboard, upgrade, or sandbox crash leaves a stale `openclaw-gateway` host process holding the dashboard port, `$$nemoclaw onboard --fresh`, `$$nemoclaw destroy` (when destroying the last sandbox), and `$$nemoclaw uninstall` automatically sweep the dashboard port range and signal `SIGTERM` then `SIGKILL` to recover. The sweep only targets processes owned by the current user whose command line matches `openclaw-gateway` or `openshell forward` markers, and skips dashboard ports owned by other live sandboxes. +When onboarding preflight blocks a gateway port conflict, it names each listener that holds the port by process name and PID. +It prints a PID alone when it cannot read the process name, and an inspection command when it resolves no listener. +Confirm that a named listener is not a second NemoClaw gateway environment before you stop it. +Release that environment with `NEMOCLAW_GATEWAY_PORT= $$nemoclaw uninstall` instead of stopping its process. + If a non-NemoClaw process is already bound to the dashboard port or the gateway port, identify the conflicting process, verify it is safe to stop, and terminate it: ```bash diff --git a/src/lib/readiness/gateway-production.test.ts b/src/lib/readiness/gateway-production.test.ts index d353c846207..b1c1a090175 100644 --- a/src/lib/readiness/gateway-production.test.ts +++ b/src/lib/readiness/gateway-production.test.ts @@ -2,6 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 import fs from "node:fs"; +import type { AddressInfo } from "node:net"; +import net from "node:net"; import os from "node:os"; import path from "node:path"; @@ -33,6 +35,7 @@ import { classifyManagedGatewayVersionDrift, classifyManagedGatewayVersionSource, createProductionGatewayReadinessDependencies, + gatewayPortConflictDetail, gatewayProcessIdentityMatchesTrustedBinary, gatewayProcessSamplesMatchTrustedBinary, parseDarwinLsofExecutable, @@ -493,4 +496,46 @@ describe("managed gateway port readiness (#7411)", () => { fs.rmSync(traceDir, { force: true, recursive: true }); } }); + + it("names the foreign listener the unprivileged scan resolved and how to stop it (#9118)", async () => { + const foreignListener = net.createServer(); + await new Promise((resolve, reject) => { + foreignListener.once("error", reject); + foreignListener.listen(0, "127.0.0.1", resolve); + }); + const gatewayPort = (foreignListener.address() as AddressInfo).port; + subprocess.spawnSync.mockImplementation((command: string, args: readonly string[] = []) => { + if (command === "lsof" && args.includes("-ti")) return commandResult(`${process.pid}\n`, 0); + if (command === "ps" && args.includes("comm=")) return commandResult("python3\n", 0); + return commandResult(); + }); + + try { + const deps = createProductionGatewayReadinessDependencies({ + gatewayName: () => "nemoclaw-readiness-test", + gatewayPort: () => gatewayPort, + }); + + const observed = await deps.observeManagedGateway(managedOwner(gatewayPort)); + + expect(observed.portConflictState).not.toBe("none"); + expect(observed.portConflictDetail).toContain(`python3 (PID ${process.pid})`); + expect(observed.portConflictDetail).toContain(`sudo kill ${process.pid}`); + expect(observed.portConflictDetail).not.toContain("occupied by unknown"); + } finally { + await new Promise((resolve) => foreignListener.close(() => resolve())); + } + }); + + it("offers an inspection command when no listener could be resolved (#9118)", () => { + const detail = gatewayPortConflictDetail( + 8080, + { ok: false, process: "unknown", pid: null, reason: "port 8080 is in use (EADDRINUSE)" }, + "occupied", + { pids: [], text: null }, + ); + + expect(detail).toContain("is occupied by an unknown listener"); + expect(detail).toContain("sudo lsof -i :8080 -sTCP:LISTEN -P -n"); + }); }); diff --git a/src/lib/readiness/gateway-production.ts b/src/lib/readiness/gateway-production.ts index d1eee202868..3c36b6fea7c 100644 --- a/src/lib/readiness/gateway-production.ts +++ b/src/lib/readiness/gateway-production.ts @@ -261,6 +261,13 @@ function readLinuxProcessExecutable(pid: number): string | null { } } +function readProcessName(pid: number, env: NodeJS.ProcessEnv): string | null { + const result = captureReadonly(["ps", "-p", String(pid), "-o", "comm="], env); + if (result.exitCode !== 0) return null; + const name = result.stdout.split(/\r?\n/)[0]?.trim(); + return name ? path.basename(name) : null; +} + export function parseDarwinLsofExecutable(output: string): string | null { // macOS reports the process executable first, followed by other text vnodes // such as /usr/lib/dyld. Selecting the first record prevents a process from @@ -520,25 +527,64 @@ export function classifyManagedGatewayVersionSource( return null; } -function gatewayPortConflictDetail( +export interface GatewayPortOwners { + pids: number[]; + text: string | null; +} + +export function describeGatewayPortOwners( + listenerScan: { pids: readonly number[]; unverifiedPids: readonly number[] }, + describeProcess: (pid: number) => string | null, +): GatewayPortOwners { + const pids = + listenerScan.unverifiedPids.length > 0 + ? [...listenerScan.unverifiedPids] + : [...listenerScan.pids]; + if (pids.length === 0) return { pids, text: null }; + const text = pids + .map((pid) => { + const name = describeProcess(pid); + return name ? `${name} (PID ${pid})` : `PID ${pid}`; + }) + .join(", "); + return { pids, text }; +} + +function gatewayPortConflictRemediation(gatewayPort: number, ownerPids: readonly number[]): string { + if (ownerPids.length === 0) { + return `Identify its listener before retrying: sudo lsof -i :${gatewayPort} -sTCP:LISTEN -P -n`; + } + const subject = ownerPids.length === 1 ? "that process is" : "those processes are"; + const object = ownerPids.length === 1 ? "it" : "them"; + return ( + `Confirm ${subject} not another NemoClaw gateway, then stop only ${object} before retrying: ` + + `sudo kill ${ownerPids.join(" ")}` + ); +} + +export function gatewayPortConflictDetail( gatewayPort: number, portCheck: Awaited>, state: GatewayPortConflictState, + owners: GatewayPortOwners = { pids: [], text: null }, ): string | undefined { if (state === "none") return undefined; - const owner = portCheck.process - ? `${portCheck.process}${portCheck.pid ? ` (PID ${portCheck.pid})` : ""}` - : "an unknown listener"; + const probedOwner = + portCheck.process && portCheck.process !== "unknown" + ? `${portCheck.process}${portCheck.pid ? ` (PID ${portCheck.pid})` : ""}` + : null; + const owner = owners.text ?? probedOwner ?? "an unknown listener"; const condition = state === "multiple-owners" - ? "has multiple listeners" + ? `has multiple listeners: ${owner}` : state === "unknown" ? "could not be observed completely" : `is occupied by ${owner}`; - return ( - `Gateway port ${gatewayPort} ${condition}. ` + - `Inspect port ${gatewayPort} and stop only its owning process before retrying.` + const remediation = gatewayPortConflictRemediation( + gatewayPort, + state === "unknown" ? [] : owners.pids, ); + return `Gateway port ${gatewayPort} ${condition}. ${remediation}`; } function rejectUnexpectedGatewayEffect(): never { @@ -782,11 +828,20 @@ export function createProductionGatewayReadinessDependencies( endpointBinding, listenerScan.pids.length === 1 && trustedTargetBoundPids.has(listenerScan.pids[0] ?? -1), ); + const portConflictOwners = + portConflictState === "none" + ? { pids: [], text: null } + : describeGatewayPortOwners(listenerScan, (pid) => readProcessName(pid, probeEnv)); return { reuseState, driftState, portConflictState, - portConflictDetail: gatewayPortConflictDetail(gatewayPort, portCheck, portConflictState), + portConflictDetail: gatewayPortConflictDetail( + gatewayPort, + portCheck, + portConflictState, + portConflictOwners, + ), }; } diff --git a/test/onboard-gateway-port-conflict-fast-fail.test.ts b/test/onboard-gateway-port-conflict-fast-fail.test.ts index 89f941f947c..1719b87befe 100644 --- a/test/onboard-gateway-port-conflict-fast-fail.test.ts +++ b/test/onboard-gateway-port-conflict-fast-fail.test.ts @@ -116,6 +116,9 @@ describe("onboard gateway port conflict readiness (#6752)", () => { expect(combined).toMatch( /The gateway port is held by an incompatible or ambiguous owner|OpenShell gateway needs this port/, ); + expect(combined).not.toMatch(/occupied by unknown/); + expect(combined).toMatch(/\(PID \d+\)/); + expect(combined).toMatch(/sudo kill \d+/); }, ); From 3f0467409e43e0bebc15f41f65999c1728b392ae Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 14 Aug 2026 10:42:18 -0700 Subject: [PATCH 2/3] fix(readiness): preserve verified gateway listeners Signed-off-by: Prekshi Vyas --- docs/reference/system-readiness.mdx | 5 +- docs/reference/troubleshooting.mdx | 9 ++-- src/lib/readiness/gateway-production.test.ts | 52 ++++++++++++++++++-- src/lib/readiness/gateway-production.ts | 44 +++++++++++------ 4 files changed, 86 insertions(+), 24 deletions(-) diff --git a/docs/reference/system-readiness.mdx b/docs/reference/system-readiness.mdx index 3582976df71..acdadeda4f5 100644 --- a/docs/reference/system-readiness.mdx +++ b/docs/reference/system-readiness.mdx @@ -253,7 +253,10 @@ Use finding IDs to handle gateway failures without parsing summaries: | `gateway.port.inconclusive` | Warning | NemoClaw could not establish gateway port ownership. | The report includes bounded evidence for the resolved owner and applicable attachment, port, collection, or stale-observation failures. -Port-conflict evidence names each listener that holds the gateway port by process name and PID, and gives the command that stops those PIDs. +When NemoClaw resolves the complete listener set, port-conflict evidence lists every listener by process name and PID, or by PID when no name is available. +A stop command targets only listeners that fail ownership verification. +NemoClaw does not provide a stop command for a verified managed listener. +If NemoClaw resolves no listener, the diagnostic provides an `lsof` inspection command. It omits the gateway state directory, removes process environments, redacts credential-shaped content, bounds diagnostic length, and renders control characters visibly. Managed gateway metadata is reusable only when its endpoint is bound to loopback on the configured gateway port. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index e70fefc7095..95995333da9 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -318,9 +318,12 @@ If forwarding then fails, onboarding removes the new sandbox and tells you to re When a previous onboard, upgrade, or sandbox crash leaves a stale `openclaw-gateway` host process holding the dashboard port, `$$nemoclaw onboard --fresh`, `$$nemoclaw destroy` (when destroying the last sandbox), and `$$nemoclaw uninstall` automatically sweep the dashboard port range and signal `SIGTERM` then `SIGKILL` to recover. The sweep only targets processes owned by the current user whose command line matches `openclaw-gateway` or `openshell forward` markers, and skips dashboard ports owned by other live sandboxes. -When onboarding preflight blocks a gateway port conflict, it names each listener that holds the port by process name and PID. -It prints a PID alone when it cannot read the process name, and an inspection command when it resolves no listener. -Confirm that a named listener is not a second NemoClaw gateway environment before you stop it. +If onboarding preflight resolves the complete listener set for a gateway port conflict, the diagnostic lists every listener. +Each entry contains the process name and PID, or only the PID when NemoClaw cannot read the process name. +The stop command targets only listeners that fail ownership verification. +It does not target a verified managed listener. +If NemoClaw resolves no listener, the diagnostic provides an `lsof` inspection command. +Before you run a stop command, confirm that each targeted listener is not part of a second NemoClaw gateway environment. Release that environment with `NEMOCLAW_GATEWAY_PORT= $$nemoclaw uninstall` instead of stopping its process. If a non-NemoClaw process is already bound to the dashboard port or the gateway port, identify the conflicting process, verify it is safe to stop, and terminate it: diff --git a/src/lib/readiness/gateway-production.test.ts b/src/lib/readiness/gateway-production.test.ts index b1c1a090175..31d4cfa02f6 100644 --- a/src/lib/readiness/gateway-production.test.ts +++ b/src/lib/readiness/gateway-production.test.ts @@ -35,6 +35,7 @@ import { classifyManagedGatewayVersionDrift, classifyManagedGatewayVersionSource, createProductionGatewayReadinessDependencies, + describeGatewayPortOwners, gatewayPortConflictDetail, gatewayProcessIdentityMatchesTrustedBinary, gatewayProcessSamplesMatchTrustedBinary, @@ -505,9 +506,13 @@ describe("managed gateway port readiness (#7411)", () => { }); const gatewayPort = (foreignListener.address() as AddressInfo).port; subprocess.spawnSync.mockImplementation((command: string, args: readonly string[] = []) => { - if (command === "lsof" && args.includes("-ti")) return commandResult(`${process.pid}\n`, 0); - if (command === "ps" && args.includes("comm=")) return commandResult("python3\n", 0); - return commandResult(); + const resolvesListener = command === "lsof" && args.includes("-ti"); + const resolvesName = command === "ps" && args.includes("comm="); + return resolvesListener + ? commandResult(`${process.pid}\n`, 0) + : resolvesName + ? commandResult("python3\n", 0) + : commandResult(); }); try { @@ -532,10 +537,49 @@ describe("managed gateway port readiness (#7411)", () => { 8080, { ok: false, process: "unknown", pid: null, reason: "port 8080 is in use (EADDRINUSE)" }, "occupied", - { pids: [], text: null }, + { stopPids: [], text: null }, ); expect(detail).toContain("is occupied by an unknown listener"); expect(detail).toContain("sudo lsof -i :8080 -sTCP:LISTEN -P -n"); }); + + it("lists every listener and limits the stop command to the unverified listener (#9118)", () => { + const processNames = new Map([ + [100, "openshell-gateway"], + [200, "python3"], + ]); + const owners = describeGatewayPortOwners( + { pids: [100], unverifiedPids: [200] }, + (pid) => processNames.get(pid) ?? null, + ); + const detail = gatewayPortConflictDetail( + 8080, + { ok: false, process: "unknown", pid: null, reason: "port 8080 is in use (EADDRINUSE)" }, + "multiple-owners", + owners, + ); + + expect(detail).toContain("openshell-gateway (PID 100), python3 (PID 200)"); + expect(detail).toContain("Confirm PID 200 is not another NemoClaw gateway"); + expect(detail).toContain("sudo kill 200"); + expect(detail).not.toContain("sudo kill 100"); + }); + + it("recommends releasing a verified gateway environment without a process stop command (#9118)", () => { + const owners = describeGatewayPortOwners( + { pids: [100], unverifiedPids: [] }, + () => "openshell-gateway", + ); + const detail = gatewayPortConflictDetail( + 8080, + { ok: false, process: "unknown", pid: null, reason: "port 8080 is in use (EADDRINUSE)" }, + "owner-mismatch", + owners, + ); + + expect(detail).toContain("openshell-gateway (PID 100)"); + expect(detail).toContain("NEMOCLAW_GATEWAY_PORT=8080 nemoclaw uninstall"); + expect(detail).not.toContain("sudo kill"); + }); }); diff --git a/src/lib/readiness/gateway-production.ts b/src/lib/readiness/gateway-production.ts index 3c36b6fea7c..cbb72ed14c0 100644 --- a/src/lib/readiness/gateway-production.ts +++ b/src/lib/readiness/gateway-production.ts @@ -528,7 +528,7 @@ export function classifyManagedGatewayVersionSource( } export interface GatewayPortOwners { - pids: number[]; + stopPids: number[]; text: string | null; } @@ -536,29 +536,39 @@ export function describeGatewayPortOwners( listenerScan: { pids: readonly number[]; unverifiedPids: readonly number[] }, describeProcess: (pid: number) => string | null, ): GatewayPortOwners { - const pids = - listenerScan.unverifiedPids.length > 0 - ? [...listenerScan.unverifiedPids] - : [...listenerScan.pids]; - if (pids.length === 0) return { pids, text: null }; - const text = pids + const allPids = [...new Set([...listenerScan.pids, ...listenerScan.unverifiedPids])]; + const stopPids = [...listenerScan.unverifiedPids]; + if (allPids.length === 0) return { stopPids, text: null }; + const text = allPids .map((pid) => { const name = describeProcess(pid); return name ? `${name} (PID ${pid})` : `PID ${pid}`; }) .join(", "); - return { pids, text }; + return { stopPids, text }; } -function gatewayPortConflictRemediation(gatewayPort: number, ownerPids: readonly number[]): string { - if (ownerPids.length === 0) { +function gatewayPortConflictRemediation( + gatewayPort: number, + stopPids: readonly number[], + hasResolvedOwner: boolean, + state: GatewayPortConflictState, +): string { + if (state === "unknown" || !hasResolvedOwner) { return `Identify its listener before retrying: sudo lsof -i :${gatewayPort} -sTCP:LISTEN -P -n`; } - const subject = ownerPids.length === 1 ? "that process is" : "those processes are"; - const object = ownerPids.length === 1 ? "it" : "them"; + if (stopPids.length === 0) { + return ( + "Confirm which managed gateway environment owns the verified listener, then release that " + + `environment with \`NEMOCLAW_GATEWAY_PORT=${gatewayPort} nemoclaw uninstall\` before retrying.` + ); + } + const subject = + stopPids.length === 1 ? `PID ${stopPids[0]} is` : `PIDs ${stopPids.join(", ")} are`; + const object = stopPids.length === 1 ? "it" : "them"; return ( `Confirm ${subject} not another NemoClaw gateway, then stop only ${object} before retrying: ` + - `sudo kill ${ownerPids.join(" ")}` + `sudo kill ${stopPids.join(" ")}` ); } @@ -566,7 +576,7 @@ export function gatewayPortConflictDetail( gatewayPort: number, portCheck: Awaited>, state: GatewayPortConflictState, - owners: GatewayPortOwners = { pids: [], text: null }, + owners: GatewayPortOwners = { stopPids: [], text: null }, ): string | undefined { if (state === "none") return undefined; const probedOwner = @@ -582,7 +592,9 @@ export function gatewayPortConflictDetail( : `is occupied by ${owner}`; const remediation = gatewayPortConflictRemediation( gatewayPort, - state === "unknown" ? [] : owners.pids, + owners.stopPids, + owners.text !== null, + state, ); return `Gateway port ${gatewayPort} ${condition}. ${remediation}`; } @@ -830,7 +842,7 @@ export function createProductionGatewayReadinessDependencies( ); const portConflictOwners = portConflictState === "none" - ? { pids: [], text: null } + ? { stopPids: [], text: null } : describeGatewayPortOwners(listenerScan, (pid) => readProcessName(pid, probeEnv)); return { reuseState, From 7593744bdb8d4c13e6b8b000e4b957e26e57c36d Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 14 Aug 2026 11:49:27 -0700 Subject: [PATCH 3/3] fix(readiness): use invoked CLI in release guidance Signed-off-by: Carlos Villela --- src/lib/readiness/gateway-production.test.ts | 18 ++++++++++++++++++ src/lib/readiness/gateway-production.ts | 4 +++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/lib/readiness/gateway-production.test.ts b/src/lib/readiness/gateway-production.test.ts index 31d4cfa02f6..58609246617 100644 --- a/src/lib/readiness/gateway-production.test.ts +++ b/src/lib/readiness/gateway-production.test.ts @@ -582,4 +582,22 @@ describe("managed gateway port readiness (#7411)", () => { expect(detail).toContain("NEMOCLAW_GATEWAY_PORT=8080 nemoclaw uninstall"); expect(detail).not.toContain("sudo kill"); }); + + it("uses the invoked CLI name in verified gateway release guidance (#9118)", () => { + vi.stubEnv("NEMOCLAW_INVOKED_AS", "nemohermes"); + const owners = describeGatewayPortOwners( + { pids: [100], unverifiedPids: [] }, + () => "openshell-gateway", + ); + + const detail = gatewayPortConflictDetail( + 8080, + { ok: false, process: "unknown", pid: null, reason: "port 8080 is in use (EADDRINUSE)" }, + "owner-mismatch", + owners, + ); + + expect(detail).toContain("NEMOCLAW_GATEWAY_PORT=8080 nemohermes uninstall"); + expect(detail).not.toContain("nemoclaw uninstall"); + }); }); diff --git a/src/lib/readiness/gateway-production.ts b/src/lib/readiness/gateway-production.ts index cbb72ed14c0..5f1e8724fa0 100644 --- a/src/lib/readiness/gateway-production.ts +++ b/src/lib/readiness/gateway-production.ts @@ -17,6 +17,7 @@ import { parseVersionFromText, stripAnsi, } from "../adapters/openshell/gateway-drift"; +import { cliName as resolveCliName } from "../onboard/branding"; import { getConfiguredGatewayPort, getDockerDriverGatewayEndpoint, @@ -558,9 +559,10 @@ function gatewayPortConflictRemediation( return `Identify its listener before retrying: sudo lsof -i :${gatewayPort} -sTCP:LISTEN -P -n`; } if (stopPids.length === 0) { + const cliName = resolveCliName(); return ( "Confirm which managed gateway environment owns the verified listener, then release that " + - `environment with \`NEMOCLAW_GATEWAY_PORT=${gatewayPort} nemoclaw uninstall\` before retrying.` + `environment with \`NEMOCLAW_GATEWAY_PORT=${gatewayPort} ${cliName} uninstall\` before retrying.` ); } const subject =