diff --git a/docs/reference/system-readiness.mdx b/docs/reference/system-readiness.mdx index a0c39eb891d..acdadeda4f5 100644 --- a/docs/reference/system-readiness.mdx +++ b/docs/reference/system-readiness.mdx @@ -253,6 +253,10 @@ Use finding IDs to handle gateway failures without parsing summaries: | `gateway.port.inconclusive` | Warning | NemoClaw could not establish gateway port ownership. | The report includes bounded evidence for the resolved owner and applicable attachment, port, collection, or stale-observation failures. +When NemoClaw resolves the complete listener set, port-conflict evidence lists every listener by process name and PID, or by PID when no name is available. +A stop command targets only listeners that fail ownership verification. +NemoClaw does not provide a stop command for a verified managed listener. +If NemoClaw resolves no listener, the diagnostic provides an `lsof` inspection command. It omits the gateway state directory, removes process environments, redacts credential-shaped content, bounds diagnostic length, and renders control characters visibly. Managed gateway metadata is reusable only when its endpoint is bound to loopback on the configured gateway port. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 0c53b560624..95995333da9 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -318,6 +318,14 @@ If forwarding then fails, onboarding removes the new sandbox and tells you to re When a previous onboard, upgrade, or sandbox crash leaves a stale `openclaw-gateway` host process holding the dashboard port, `$$nemoclaw onboard --fresh`, `$$nemoclaw destroy` (when destroying the last sandbox), and `$$nemoclaw uninstall` automatically sweep the dashboard port range and signal `SIGTERM` then `SIGKILL` to recover. The sweep only targets processes owned by the current user whose command line matches `openclaw-gateway` or `openshell forward` markers, and skips dashboard ports owned by other live sandboxes. +If onboarding preflight resolves the complete listener set for a gateway port conflict, the diagnostic lists every listener. +Each entry contains the process name and PID, or only the PID when NemoClaw cannot read the process name. +The stop command targets only listeners that fail ownership verification. +It does not target a verified managed listener. +If NemoClaw resolves no listener, the diagnostic provides an `lsof` inspection command. +Before you run a stop command, confirm that each targeted listener is not part of a second NemoClaw gateway environment. +Release that environment with `NEMOCLAW_GATEWAY_PORT= $$nemoclaw uninstall` instead of stopping its process. + If a non-NemoClaw process is already bound to the dashboard port or the gateway port, identify the conflicting process, verify it is safe to stop, and terminate it: ```bash diff --git a/src/lib/readiness/gateway-production.test.ts b/src/lib/readiness/gateway-production.test.ts index d353c846207..58609246617 100644 --- a/src/lib/readiness/gateway-production.test.ts +++ b/src/lib/readiness/gateway-production.test.ts @@ -2,6 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 import fs from "node:fs"; +import type { AddressInfo } from "node:net"; +import net from "node:net"; import os from "node:os"; import path from "node:path"; @@ -33,6 +35,8 @@ import { classifyManagedGatewayVersionDrift, classifyManagedGatewayVersionSource, createProductionGatewayReadinessDependencies, + describeGatewayPortOwners, + gatewayPortConflictDetail, gatewayProcessIdentityMatchesTrustedBinary, gatewayProcessSamplesMatchTrustedBinary, parseDarwinLsofExecutable, @@ -493,4 +497,107 @@ describe("managed gateway port readiness (#7411)", () => { fs.rmSync(traceDir, { force: true, recursive: true }); } }); + + it("names the foreign listener the unprivileged scan resolved and how to stop it (#9118)", async () => { + const foreignListener = net.createServer(); + await new Promise((resolve, reject) => { + foreignListener.once("error", reject); + foreignListener.listen(0, "127.0.0.1", resolve); + }); + const gatewayPort = (foreignListener.address() as AddressInfo).port; + subprocess.spawnSync.mockImplementation((command: string, args: readonly string[] = []) => { + const resolvesListener = command === "lsof" && args.includes("-ti"); + const resolvesName = command === "ps" && args.includes("comm="); + return resolvesListener + ? commandResult(`${process.pid}\n`, 0) + : resolvesName + ? commandResult("python3\n", 0) + : commandResult(); + }); + + try { + const deps = createProductionGatewayReadinessDependencies({ + gatewayName: () => "nemoclaw-readiness-test", + gatewayPort: () => gatewayPort, + }); + + const observed = await deps.observeManagedGateway(managedOwner(gatewayPort)); + + expect(observed.portConflictState).not.toBe("none"); + expect(observed.portConflictDetail).toContain(`python3 (PID ${process.pid})`); + expect(observed.portConflictDetail).toContain(`sudo kill ${process.pid}`); + expect(observed.portConflictDetail).not.toContain("occupied by unknown"); + } finally { + await new Promise((resolve) => foreignListener.close(() => resolve())); + } + }); + + it("offers an inspection command when no listener could be resolved (#9118)", () => { + const detail = gatewayPortConflictDetail( + 8080, + { ok: false, process: "unknown", pid: null, reason: "port 8080 is in use (EADDRINUSE)" }, + "occupied", + { stopPids: [], text: null }, + ); + + expect(detail).toContain("is occupied by an unknown listener"); + expect(detail).toContain("sudo lsof -i :8080 -sTCP:LISTEN -P -n"); + }); + + it("lists every listener and limits the stop command to the unverified listener (#9118)", () => { + const processNames = new Map([ + [100, "openshell-gateway"], + [200, "python3"], + ]); + const owners = describeGatewayPortOwners( + { pids: [100], unverifiedPids: [200] }, + (pid) => processNames.get(pid) ?? null, + ); + const detail = gatewayPortConflictDetail( + 8080, + { ok: false, process: "unknown", pid: null, reason: "port 8080 is in use (EADDRINUSE)" }, + "multiple-owners", + owners, + ); + + expect(detail).toContain("openshell-gateway (PID 100), python3 (PID 200)"); + expect(detail).toContain("Confirm PID 200 is not another NemoClaw gateway"); + expect(detail).toContain("sudo kill 200"); + expect(detail).not.toContain("sudo kill 100"); + }); + + it("recommends releasing a verified gateway environment without a process stop command (#9118)", () => { + const owners = describeGatewayPortOwners( + { pids: [100], unverifiedPids: [] }, + () => "openshell-gateway", + ); + const detail = gatewayPortConflictDetail( + 8080, + { ok: false, process: "unknown", pid: null, reason: "port 8080 is in use (EADDRINUSE)" }, + "owner-mismatch", + owners, + ); + + expect(detail).toContain("openshell-gateway (PID 100)"); + expect(detail).toContain("NEMOCLAW_GATEWAY_PORT=8080 nemoclaw uninstall"); + expect(detail).not.toContain("sudo kill"); + }); + + it("uses the invoked CLI name in verified gateway release guidance (#9118)", () => { + vi.stubEnv("NEMOCLAW_INVOKED_AS", "nemohermes"); + const owners = describeGatewayPortOwners( + { pids: [100], unverifiedPids: [] }, + () => "openshell-gateway", + ); + + const detail = gatewayPortConflictDetail( + 8080, + { ok: false, process: "unknown", pid: null, reason: "port 8080 is in use (EADDRINUSE)" }, + "owner-mismatch", + owners, + ); + + expect(detail).toContain("NEMOCLAW_GATEWAY_PORT=8080 nemohermes uninstall"); + expect(detail).not.toContain("nemoclaw uninstall"); + }); }); diff --git a/src/lib/readiness/gateway-production.ts b/src/lib/readiness/gateway-production.ts index d1eee202868..5f1e8724fa0 100644 --- a/src/lib/readiness/gateway-production.ts +++ b/src/lib/readiness/gateway-production.ts @@ -17,6 +17,7 @@ import { parseVersionFromText, stripAnsi, } from "../adapters/openshell/gateway-drift"; +import { cliName as resolveCliName } from "../onboard/branding"; import { getConfiguredGatewayPort, getDockerDriverGatewayEndpoint, @@ -261,6 +262,13 @@ function readLinuxProcessExecutable(pid: number): string | null { } } +function readProcessName(pid: number, env: NodeJS.ProcessEnv): string | null { + const result = captureReadonly(["ps", "-p", String(pid), "-o", "comm="], env); + if (result.exitCode !== 0) return null; + const name = result.stdout.split(/\r?\n/)[0]?.trim(); + return name ? path.basename(name) : null; +} + export function parseDarwinLsofExecutable(output: string): string | null { // macOS reports the process executable first, followed by other text vnodes // such as /usr/lib/dyld. Selecting the first record prevents a process from @@ -520,25 +528,77 @@ export function classifyManagedGatewayVersionSource( return null; } -function gatewayPortConflictDetail( +export interface GatewayPortOwners { + stopPids: number[]; + text: string | null; +} + +export function describeGatewayPortOwners( + listenerScan: { pids: readonly number[]; unverifiedPids: readonly number[] }, + describeProcess: (pid: number) => string | null, +): GatewayPortOwners { + const allPids = [...new Set([...listenerScan.pids, ...listenerScan.unverifiedPids])]; + const stopPids = [...listenerScan.unverifiedPids]; + if (allPids.length === 0) return { stopPids, text: null }; + const text = allPids + .map((pid) => { + const name = describeProcess(pid); + return name ? `${name} (PID ${pid})` : `PID ${pid}`; + }) + .join(", "); + return { stopPids, text }; +} + +function gatewayPortConflictRemediation( + gatewayPort: number, + stopPids: readonly number[], + hasResolvedOwner: boolean, + state: GatewayPortConflictState, +): string { + if (state === "unknown" || !hasResolvedOwner) { + return `Identify its listener before retrying: sudo lsof -i :${gatewayPort} -sTCP:LISTEN -P -n`; + } + if (stopPids.length === 0) { + const cliName = resolveCliName(); + return ( + "Confirm which managed gateway environment owns the verified listener, then release that " + + `environment with \`NEMOCLAW_GATEWAY_PORT=${gatewayPort} ${cliName} uninstall\` before retrying.` + ); + } + const subject = + stopPids.length === 1 ? `PID ${stopPids[0]} is` : `PIDs ${stopPids.join(", ")} are`; + const object = stopPids.length === 1 ? "it" : "them"; + return ( + `Confirm ${subject} not another NemoClaw gateway, then stop only ${object} before retrying: ` + + `sudo kill ${stopPids.join(" ")}` + ); +} + +export function gatewayPortConflictDetail( gatewayPort: number, portCheck: Awaited>, state: GatewayPortConflictState, + owners: GatewayPortOwners = { stopPids: [], text: null }, ): string | undefined { if (state === "none") return undefined; - const owner = portCheck.process - ? `${portCheck.process}${portCheck.pid ? ` (PID ${portCheck.pid})` : ""}` - : "an unknown listener"; + const probedOwner = + portCheck.process && portCheck.process !== "unknown" + ? `${portCheck.process}${portCheck.pid ? ` (PID ${portCheck.pid})` : ""}` + : null; + const owner = owners.text ?? probedOwner ?? "an unknown listener"; const condition = state === "multiple-owners" - ? "has multiple listeners" + ? `has multiple listeners: ${owner}` : state === "unknown" ? "could not be observed completely" : `is occupied by ${owner}`; - return ( - `Gateway port ${gatewayPort} ${condition}. ` + - `Inspect port ${gatewayPort} and stop only its owning process before retrying.` + const remediation = gatewayPortConflictRemediation( + gatewayPort, + owners.stopPids, + owners.text !== null, + state, ); + return `Gateway port ${gatewayPort} ${condition}. ${remediation}`; } function rejectUnexpectedGatewayEffect(): never { @@ -782,11 +842,20 @@ export function createProductionGatewayReadinessDependencies( endpointBinding, listenerScan.pids.length === 1 && trustedTargetBoundPids.has(listenerScan.pids[0] ?? -1), ); + const portConflictOwners = + portConflictState === "none" + ? { stopPids: [], text: null } + : describeGatewayPortOwners(listenerScan, (pid) => readProcessName(pid, probeEnv)); return { reuseState, driftState, portConflictState, - portConflictDetail: gatewayPortConflictDetail(gatewayPort, portCheck, portConflictState), + portConflictDetail: gatewayPortConflictDetail( + gatewayPort, + portCheck, + portConflictState, + portConflictOwners, + ), }; } diff --git a/test/onboard-gateway-port-conflict-fast-fail.test.ts b/test/onboard-gateway-port-conflict-fast-fail.test.ts index 89f941f947c..1719b87befe 100644 --- a/test/onboard-gateway-port-conflict-fast-fail.test.ts +++ b/test/onboard-gateway-port-conflict-fast-fail.test.ts @@ -116,6 +116,9 @@ describe("onboard gateway port conflict readiness (#6752)", () => { expect(combined).toMatch( /The gateway port is held by an incompatible or ambiguous owner|OpenShell gateway needs this port/, ); + expect(combined).not.toMatch(/occupied by unknown/); + expect(combined).toMatch(/\(PID \d+\)/); + expect(combined).toMatch(/sudo kill \d+/); }, );