diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 0ab334c8aa1..dff723935af 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3074,6 +3074,9 @@ Earlier releases only stopped `openshell forward` processes, so those orphans ac For Local Ollama setups, uninstall also stops matching Ollama auth proxy processes before deleting `~/.nemoclaw` state so stale proxy listeners do not block a later reinstall. +For Hermes setups, uninstall inspects the selected gateway's managed port-forward watcher state, stops each verified watcher process and its sandbox-scoped forward, and leaves sibling gateway state untouched. +If any watcher or forward cleanup cannot be confirmed, uninstall exits nonzero and preserves the selected gateway's watcher state so you can retry cleanup. + On Linux, uninstall removes `~/.local/state/nemoclaw`, which contains Docker-driver gateway SQLite data, audit logs, VM-driver state, and standalone-fallback gateway PID files. | Flag | Effect | diff --git a/scripts/install.sh b/scripts/install.sh index 9fb594e114f..89ae7297990 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -423,7 +423,7 @@ resolve_onboarded_agent() { } restore_onboard_forward_after_post_checks() { - local sandbox_name agent_name agent_display port openshell_bin attempt selected_state_dir state_dir pid_file watcher_script watcher_pid + local sandbox_name agent_name agent_display port openshell_bin openshell_dir attempt selected_state_dir state_dir pid_file watcher_script watcher_pid sandbox_name="$(resolve_default_sandbox_name)" agent_name="$(resolve_onboarded_agent)" agent_display="$(agent_display_name "$agent_name")" @@ -440,6 +440,12 @@ restore_onboard_forward_after_post_checks() { else return 0 fi + if [[ "$openshell_bin" != /* ]]; then + openshell_dir="${openshell_bin%/*}" + [[ "$openshell_dir" == "$openshell_bin" ]] && openshell_dir="." + openshell_dir="$(cd -- "$openshell_dir" && pwd -P)" || return 1 + openshell_bin="${openshell_dir}/${openshell_bin##*/}" + fi selected_state_dir="$(ensure_nemoclaw_state_dir)" || return 1 state_dir="${selected_state_dir}/state" diff --git a/src/lib/actions/uninstall/hermes-forward-watcher-cleanup.ts b/src/lib/actions/uninstall/hermes-forward-watcher-cleanup.ts new file mode 100644 index 00000000000..45f663c55af --- /dev/null +++ b/src/lib/actions/uninstall/hermes-forward-watcher-cleanup.ts @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + type HermesForwardWatcherHost, + stopHermesForwardWatcherProcess, + stopHermesSandboxForward, +} from "../../adapters/openshell/hermes-forward-watcher"; +import { readHermesForwardWatcherState } from "../../state/hermes-forward-watcher"; + +export function stopHermesForwardWatchers( + nemoclawStateDir: string, + host: HermesForwardWatcherHost, +): boolean { + const state = readHermesForwardWatcherState(nemoclawStateDir); + if (!state.readable) { + host.warn(`Failed to inspect Hermes forward watcher state under ${nemoclawStateDir}.`); + return false; + } + if (state.watchers.length === 0) { + host.log("No Hermes forward watchers found"); + return true; + } + + let allStopped = true; + for (const watcher of state.watchers) { + if (!stopHermesForwardWatcherProcess(watcher, host)) allStopped = false; + if (!stopHermesSandboxForward(watcher, host)) allStopped = false; + } + return allStopped; +} diff --git a/src/lib/actions/uninstall/hermes-forward-watcher-installer.test.ts b/src/lib/actions/uninstall/hermes-forward-watcher-installer.test.ts new file mode 100644 index 00000000000..9038bfb1540 --- /dev/null +++ b/src/lib/actions/uninstall/hermes-forward-watcher-installer.test.ts @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const REPOSITORY_ROOT = path.resolve(import.meta.dirname, "../../../.."); +const INSTALLER = path.join(REPOSITORY_ROOT, "scripts", "install.sh"); + +function writeExecutable(target: string, contents: string): void { + fs.writeFileSync(target, contents, { mode: 0o755 }); +} + +describe("Hermes forward watcher installer contract", () => { + it("gives the watcher an absolute OpenShell path for a relative override (#7163)", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemohermes-forward-relative-")); + try { + const fakeBin = path.join(tmp, "bin"); + const stateDir = path.join(tmp, ".nemoclaw"); + const watcherLog = path.join(tmp, "watcher.log"); + const openshell = path.join(fakeBin, "openshell"); + fs.mkdirSync(fakeBin, { recursive: true }); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync( + path.join(stateDir, "onboard-session.json"), + JSON.stringify({ sandboxName: "created-by-onboard", agent: "hermes" }), + ); + writeExecutable(openshell, "#!/usr/bin/env bash\nexit 0\n"); + writeExecutable( + path.join(fakeBin, "node"), + `#!/usr/bin/env bash +if [ "\${1:-}" = "-e" ] && [[ "\${2:-}" == *"const { spawn }"* ]]; then + printf '%s\n' "$4" > "$WATCHER_LOG" + exit 0 +fi +exec ${JSON.stringify(process.execPath)} "$@" +`, + ); + for (const command of ["curl", "sleep"]) { + writeExecutable(path.join(fakeBin, command), "#!/usr/bin/env bash\nexit 0\n"); + } + const relativeOpenshell = path.relative(REPOSITORY_ROOT, openshell); + const result = spawnSync( + "bash", + ["-c", 'source "$INSTALLER" 2>/dev/null; restore_onboard_forward_after_post_checks'], + { + cwd: REPOSITORY_ROOT, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmp, + INSTALLER, + NEMOCLAW_OPENSHELL_BIN: relativeOpenshell, + PATH: `${fakeBin}:/usr/bin:/bin`, + WATCHER_LOG: watcherLog, + }, + }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(fs.readFileSync(watcherLog, "utf-8").trim()).toBe(openshell); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/actions/uninstall/run-plan-hermes-forward-watcher.test.ts b/src/lib/actions/uninstall/run-plan-hermes-forward-watcher.test.ts new file mode 100644 index 00000000000..5d2ac34accc --- /dev/null +++ b/src/lib/actions/uninstall/run-plan-hermes-forward-watcher.test.ts @@ -0,0 +1,465 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { type RunResult, runUninstallPlan, type UninstallRunDeps } from "./run-plan"; + +const SANDBOX = "default-sandbox"; +const PORT = "8642"; + +type WatcherFixture = { + pidFile: string; + port: string; + sandbox: string; + watcherScript: string; +}; + +type ProcessFixture = { + argv?: readonly string[] | null; + commandLine?: string; + commandLineAfterSignal?: string; + commandLineReadable?: boolean; + exitsOnSignal?: boolean; + owner?: string; + pid: number; + running?: boolean; + watcher: WatcherFixture; +}; + +type RunPlan = typeof runUninstallPlan; + +function result(status: number | null, stdout = "", stderr = ""): RunResult { + return { status, stdout, stderr }; +} + +function ok(stdout = ""): RunResult { + return result(0, stdout); +} + +function notFound(): RunResult { + return result(1); +} + +function commandKey(command: string, args: readonly string[]): string { + return [command, ...args].join("\0"); +} + +function seedWatcher( + stateRoot: string, + pidContent: string, + sandbox = SANDBOX, + port = PORT, +): WatcherFixture { + const stateDir = path.join(stateRoot, "state"); + fs.mkdirSync(stateDir, { recursive: true }); + const pidFile = path.join(stateDir, `hermes-${sandbox}-${port}.forward.pid`); + fs.writeFileSync(pidFile, pidContent); + return { pidFile, port, sandbox, watcherScript: `${pidFile}.js` }; +} + +function managedArgv(watcher: WatcherFixture): readonly string[] { + return [ + "/usr/bin/node", + watcher.watcherScript, + "/usr/local/bin/openshell", + watcher.port, + watcher.sandbox, + ]; +} + +function managedCommandLine(watcher: WatcherFixture): string { + return `${managedArgv(watcher).join(" ")}\n`; +} + +function defaultRun(command: string, args: readonly string[]): RunResult { + const defaults = new Map([["lsof", ok("")]]); + const shellProbe = args[0] === "-c" ? ok("/fake/bin/tool\n") : ok(""); + return defaults.get(command) ?? shellProbe; +} + +function createHarness( + tmpHome: string, + processes: readonly ProcessFixture[], + forwardStatuses: ReadonlyMap = new Map(), +) { + const calls: Array<{ args: string[]; command: string }> = []; + const killed: number[] = []; + const logs: string[] = []; + const warnings: string[] = []; + const processByPid = new Map(processes.map((process) => [process.pid, process])); + const processCommandLines = new Map( + processes.map((process) => [ + process.pid, + process.commandLine ?? managedCommandLine(process.watcher), + ]), + ); + const alive = new Set( + processes.filter((process) => process.running !== false).map((process) => process.pid), + ); + const routes = new Map RunResult>(); + + for (const process of processes) { + const pid = String(process.pid); + routes.set(commandKey("ps", ["-p", pid, "-o", "pid="]), () => + alive.has(process.pid) ? ok(`${pid}\n`) : notFound(), + ); + routes.set(commandKey("ps", ["-p", pid, "-o", "user="]), () => + ok(`${process.owner ?? "testuser"}\n`), + ); + routes.set(commandKey("ps", ["-ww", "-p", pid, "-o", "args="]), () => + process.commandLineReadable === false + ? result(2, "", "process inspection failed") + : ok(processCommandLines.get(process.pid) ?? ""), + ); + } + for (const [forward, status] of forwardStatuses) { + const [port, sandbox] = forward.split("\0"); + routes.set(commandKey("openshell", ["forward", "stop", port ?? "", sandbox ?? ""]), () => + result(status), + ); + } + + const run = vi.fn((command: string, args: string[]): RunResult => { + calls.push({ args: [...args], command }); + return routes.get(commandKey(command, args))?.() ?? defaultRun(command, args); + }); + const deps: UninstallRunDeps = { + commandExists: (command) => !["docker", "pgrep"].includes(command), + env: { HOME: tmpHome, LOGNAME: "testuser" }, + error: (line) => warnings.push(line), + existsSync: (target) => fs.existsSync(target), + isTty: false, + kill: (pid) => { + killed.push(pid); + const process = processByPid.get(pid); + const replacement = process?.commandLineAfterSignal; + const exits = process?.exitsOnSignal !== false; + const transition = + replacement !== undefined + ? () => processCommandLines.set(pid, replacement) + : exits + ? () => alive.delete(pid) + : () => alive.has(pid); + transition(); + return true; + }, + log: (line) => logs.push(line), + readProcessArgv: (pid) => processByPid.get(pid)?.argv ?? null, + rmSync: vi.fn(), + run, + runDocker: () => ok(), + }; + return { calls, deps, killed, logs, warnings }; +} + +function uninstall( + tmpHome: string, + harness: ReturnType, + runPlan: RunPlan = runUninstallPlan, + env: NodeJS.ProcessEnv = {}, +) { + return runPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { ...harness.deps, env: { HOME: tmpHome, LOGNAME: "testuser", ...env } }, + ); +} + +function forwardStops(harness: ReturnType): string[][] { + return harness.calls + .filter(({ command, args }) => command === "openshell" && args[0] === "forward") + .map(({ args }) => args); +} + +afterEach(() => { + vi.unstubAllEnvs(); + vi.resetModules(); +}); + +describe("uninstall Hermes forward watcher cleanup (#7163)", () => { + it("stops an owned exact-argv watcher and its sandbox-scoped forward", () => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-7163-stop-")); + try { + const watcher = seedWatcher(path.join(tmpHome, ".nemoclaw"), "60642\n"); + const harness = createHarness(tmpHome, [{ argv: managedArgv(watcher), pid: 60642, watcher }]); + const outcome = uninstall(tmpHome, harness); + + expect(outcome.exitCode).toBe(0); + expect(harness.killed).toContain(60642); + expect(harness.logs).toContain("Stopped Hermes forward watcher 60642"); + expect(forwardStops(harness)).toContainEqual(["forward", "stop", PORT, SANDBOX]); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("stops an owned watcher through the exact macOS ps fallback", () => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-7163-ps-")); + try { + const watcher = seedWatcher(path.join(tmpHome, ".nemoclaw"), "69642\n"); + const harness = createHarness(tmpHome, [{ pid: 69642, watcher }]); + const outcome = uninstall(tmpHome, harness); + + expect(outcome.exitCode).toBe(0); + expect(harness.killed).toContain(69642); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("never signals a foreign-owned watcher even when its argv matches", () => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-7163-foreign-")); + try { + const watcher = seedWatcher(path.join(tmpHome, ".nemoclaw"), "70642\n"); + const harness = createHarness(tmpHome, [ + { argv: managedArgv(watcher), owner: "someone-else", pid: 70642, watcher }, + ]); + const outcome = uninstall(tmpHome, harness); + + expect(outcome.exitCode).toBe(0); + expect(harness.killed).not.toContain(70642); + expect(forwardStops(harness)).toContainEqual(["forward", "stop", PORT, SANDBOX]); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("rejects an adversarial command line that only embeds the watcher argv", () => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-7163-reused-")); + try { + const watcher = seedWatcher(path.join(tmpHome, ".nemoclaw"), "80642\n"); + const harness = createHarness(tmpHome, [ + { + commandLine: `/bin/sh -c ${managedCommandLine(watcher).trim()}\n`, + pid: 80642, + watcher, + }, + ]); + const outcome = uninstall(tmpHome, harness); + + expect(outcome.exitCode).toBe(0); + expect(harness.killed).not.toContain(80642); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("handles a stale numeric PID file without signaling it", () => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-7163-stale-")); + try { + const root = path.join(tmpHome, ".nemoclaw"); + const stale = seedWatcher(root, "90642\n", "stale-sandbox"); + const harness = createHarness(tmpHome, [{ pid: 90642, running: false, watcher: stale }]); + const outcome = uninstall(tmpHome, harness); + + expect(outcome.exitCode).toBe(0); + expect(harness.killed).toHaveLength(0); + expect(forwardStops(harness)).toContainEqual(["forward", "stop", PORT, "stale-sandbox"]); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("preserves retry state when the watcher PID file is invalid", () => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-7163-invalid-")); + try { + const watcher = seedWatcher(path.join(tmpHome, ".nemoclaw"), "91642junk\n"); + const harness = createHarness(tmpHome, []); + harness.deps.rmSync = fs.rmSync; + const outcome = uninstall(tmpHome, harness); + + expect(outcome.exitCode).toBe(1); + expect(harness.killed).toHaveLength(0); + expect(fs.existsSync(watcher.pidFile)).toBe(true); + expect(harness.warnings).toContain( + `Failed to read a valid Hermes forward watcher PID from ${watcher.pidFile}.`, + ); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("does not follow a watcher PID-file symlink", () => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-7163-symlink-")); + try { + const root = path.join(tmpHome, ".nemoclaw"); + const stateDir = path.join(root, "state"); + const target = path.join(tmpHome, "foreign-pid"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(target, "92642\n"); + const pidFile = path.join(stateDir, `hermes-${SANDBOX}-${PORT}.forward.pid`); + fs.symlinkSync(target, pidFile); + const watcher = { pidFile, port: PORT, sandbox: SANDBOX, watcherScript: `${pidFile}.js` }; + const harness = createHarness(tmpHome, [{ pid: 92642, watcher }]); + harness.deps.rmSync = fs.rmSync; + const outcome = uninstall(tmpHome, harness); + + expect(outcome.exitCode).toBe(1); + expect(harness.killed).not.toContain(92642); + expect(forwardStops(harness)).toContainEqual(["forward", "stop", PORT, SANDBOX]); + expect(fs.existsSync(pidFile)).toBe(true); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("returns nonzero when an owned watcher cannot be stopped", () => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-7163-stuck-")); + try { + const watcher = seedWatcher(path.join(tmpHome, ".nemoclaw"), "61642\n"); + const harness = createHarness(tmpHome, [ + { argv: managedArgv(watcher), exitsOnSignal: false, pid: 61642, watcher }, + ]); + harness.deps.rmSync = fs.rmSync; + const outcome = uninstall(tmpHome, harness); + + expect(outcome.exitCode).toBe(1); + expect(harness.warnings).toContain("Failed to stop Hermes forward watcher 61642"); + expect(fs.existsSync(watcher.pidFile)).toBe(true); + expect(harness.logs).not.toContain("Claws retracted. Until next time."); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("preserves retry state when a live watcher cannot be inspected", () => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-7163-inspect-")); + try { + const watcher = seedWatcher(path.join(tmpHome, ".nemoclaw"), "65642\n"); + const harness = createHarness(tmpHome, [{ commandLineReadable: false, pid: 65642, watcher }]); + harness.deps.rmSync = fs.rmSync; + const outcome = uninstall(tmpHome, harness); + + expect(outcome.exitCode).toBe(1); + expect(harness.killed).toHaveLength(0); + expect(fs.existsSync(watcher.pidFile)).toBe(true); + expect(harness.warnings).toContain( + "Failed to inspect Hermes forward watcher 65642; preserving state for retry.", + ); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("retries from preserved state after a transient watcher stop failure", () => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-7163-retry-")); + try { + const watcher = seedWatcher(path.join(tmpHome, ".nemoclaw"), "66642\n"); + const process: ProcessFixture = { + argv: managedArgv(watcher), + exitsOnSignal: false, + pid: 66642, + watcher, + }; + const harness = createHarness(tmpHome, [process]); + harness.deps.rmSync = fs.rmSync; + + const firstAttempt = uninstall(tmpHome, harness); + expect(firstAttempt.exitCode).toBe(1); + expect(fs.existsSync(watcher.pidFile)).toBe(true); + + process.exitsOnSignal = true; + const retry = uninstall(tmpHome, harness); + + expect(retry.exitCode).toBe(0); + expect(harness.killed).toEqual([66642, 66642, 66642]); + expect(forwardStops(harness)).toEqual([ + ["forward", "stop", PORT, SANDBOX], + ["forward", "stop", PORT, SANDBOX], + ]); + expect(fs.existsSync(watcher.pidFile)).toBe(false); + expect(harness.logs).toContain("Claws retracted. Until next time."); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("does not send SIGKILL after the watcher PID is recycled", () => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-7163-recycle-")); + try { + const watcher = seedWatcher(path.join(tmpHome, ".nemoclaw"), "67642\n"); + const harness = createHarness(tmpHome, [ + { + commandLineAfterSignal: "/usr/bin/sleep 99\n", + exitsOnSignal: false, + pid: 67642, + watcher, + }, + ]); + const outcome = uninstall(tmpHome, harness); + + expect(outcome.exitCode).toBe(0); + expect(harness.killed).toEqual([67642]); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("returns nonzero when the sandbox-scoped forward stop fails", () => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-7163-forward-")); + try { + const watcher = seedWatcher(path.join(tmpHome, ".nemoclaw"), "62642\n"); + const harness = createHarness( + tmpHome, + [{ argv: managedArgv(watcher), pid: 62642, watcher }], + new Map([[`${PORT}\0${SANDBOX}`, 7]]), + ); + harness.deps.rmSync = fs.rmSync; + const outcome = uninstall(tmpHome, harness); + + expect(outcome.exitCode).toBe(1); + expect(harness.killed).toContain(62642); + expect(fs.existsSync(watcher.pidFile)).toBe(true); + expect(harness.warnings).toContain( + "Failed to stop Hermes forward for sandbox 'default-sandbox' on port 8642 (exit 7).", + ); + expect(harness.logs).not.toContain("Claws retracted. Until next time."); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("cleans only the selected custom gateway when a sibling remains", async () => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-7163-sibling-")); + try { + vi.stubEnv("NEMOCLAW_GATEWAY_PORT", "9123"); + vi.resetModules(); + const { runUninstallPlan: runPortUninstall } = await import("./run-plan"); + const gatewayRoot = path.join(tmpHome, ".nemoclaw", "gateways"); + const selected = seedWatcher(path.join(gatewayRoot, "9123"), "63642\n", "selected-box"); + const sibling = seedWatcher(path.join(gatewayRoot, "9124"), "64642\n", "sibling-box"); + const harness = createHarness(tmpHome, [ + { argv: managedArgv(selected), pid: 63642, watcher: selected }, + { argv: managedArgv(sibling), pid: 64642, watcher: sibling }, + ]); + const outcome = uninstall(tmpHome, harness, runPortUninstall, { + NEMOCLAW_GATEWAY_PORT: "9123", + }); + + expect(outcome.exitCode).toBe(0); + expect(harness.killed).toContain(63642); + expect(harness.killed).not.toContain(64642); + expect(forwardStops(harness)).toContainEqual(["forward", "stop", PORT, "selected-box"]); + expect(forwardStops(harness)).not.toContainEqual(["forward", "stop", PORT, "sibling-box"]); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("logs and continues when no watcher PID file exists", () => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-7163-none-")); + try { + const harness = createHarness(tmpHome, []); + const outcome = uninstall(tmpHome, harness); + + expect(outcome.exitCode).toBe(0); + expect(harness.logs).toContain("No Hermes forward watchers found"); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index bff09d9b698..88098a6e453 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -37,6 +37,7 @@ import { registryEntryGatewayPort, } from "../../state/gateway-registry"; import { GATEWAYS_SUBDIR } from "../../state/state-root"; +import { stopHermesForwardWatchers } from "./hermes-forward-watcher-cleanup"; import { stopOpenRouterRuntimeAdapter } from "./openrouter-runtime-adapter-cleanup"; import { classifyShimPath, type FileSystemDeps } from "./plan"; @@ -63,6 +64,7 @@ export interface UninstallRunDeps { isTty?: boolean; kill?: (pid: number, signal?: NodeJS.Signals | number) => boolean; log?: (message: string) => void; + readProcessArgv?: (pid: number) => readonly string[] | null; readLine?: () => string | null; rmSync?: typeof fs.rmSync; run?: (command: string, args: string[], options?: SpawnSyncOptions) => RunResult; @@ -287,6 +289,7 @@ interface UninstallRuntime { isTty: boolean; kill: (pid: number, signal?: NodeJS.Signals | number) => boolean; log: (message: string) => void; + readProcessArgv: ((pid: number) => readonly string[] | null) | undefined; readLine: () => string | null; rmSync: typeof fs.rmSync; run: (command: string, args: string[], options?: SpawnSyncOptions) => RunResult; @@ -315,6 +318,7 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime { } }), log: deps.log ?? ((message) => console.log(message)), + readProcessArgv: deps.readProcessArgv, readLine: deps.readLine ?? readLineFromStdin, rmSync: deps.rmSync ?? fs.rmSync, run: deps.run ?? defaultRun, @@ -1296,8 +1300,9 @@ function executePlan( }); stopOrphanedOpenShell(runtime); } else { - runtime.log("Sibling gateways remain; kept shared helper and forward services."); + runtime.log("Sibling gateways remain; kept shared helper services and sibling forwards."); } + if (!stopHermesForwardWatchers(paths.nemoclawStateDir, runtime)) return { ok: false }; if (!scopedToSelectedGateway) { stopHostGatewayProcesses( { diff --git a/src/lib/adapters/openshell/hermes-forward-watcher.ts b/src/lib/adapters/openshell/hermes-forward-watcher.ts new file mode 100644 index 00000000000..28f5925b0c0 --- /dev/null +++ b/src/lib/adapters/openshell/hermes-forward-watcher.ts @@ -0,0 +1,173 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SpawnSyncOptions } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; + +import { sleepMs } from "../../core/wait"; +import { + type HermesForwardWatcherCommandLine, + type HermesForwardWatcherState, + isManagedHermesForwardWatcherProcess, +} from "../../domain/uninstall/hermes-forward-watcher"; + +interface RunResult { + status: number | null; + stderr: string; + stdout: string; +} + +export interface HermesForwardWatcherHost { + commandExists: (command: string) => boolean; + env: NodeJS.ProcessEnv; + kill: (pid: number, signal?: NodeJS.Signals | number) => boolean; + log: (message: string) => void; + readProcessArgv: ((pid: number) => readonly string[] | null) | undefined; + run: (command: string, args: string[], options?: SpawnSyncOptions) => RunResult; + warn: (message: string) => void; +} + +type ManagedWatcherProcessStatus = "absent" | "managed" | "other" | "unknown"; + +function pidExists(pid: number, host: HermesForwardWatcherHost): boolean | null { + const result = host.run("ps", ["-p", String(pid), "-o", "pid="], { env: host.env }); + if (result.status === 0) return result.stdout.trim() ? true : null; + return result.status === 1 ? false : null; +} + +function readProcCommandLine(pid: number): HermesForwardWatcherCommandLine | null { + try { + const argv = fs.readFileSync(`/proc/${pid}/cmdline`, "utf-8").split("\0").filter(Boolean); + return argv.length > 0 ? { kind: "argv", value: argv } : null; + } catch { + return null; + } +} + +function readProcessCommandLine( + pid: number, + host: HermesForwardWatcherHost, +): HermesForwardWatcherCommandLine | null { + const injectedArgv = host.readProcessArgv?.(pid); + const procCommandLine = host.readProcessArgv + ? injectedArgv && injectedArgv.length > 0 + ? { kind: "argv" as const, value: injectedArgv } + : null + : readProcCommandLine(pid); + if (procCommandLine) return procCommandLine; + const result = host.run("ps", ["-ww", "-p", String(pid), "-o", "args="], { env: host.env }); + return result.status === 0 && result.stdout.trim() ? { kind: "ps", value: result.stdout } : null; +} + +function currentUser(host: HermesForwardWatcherHost): string { + return host.env.SUDO_USER || host.env.LOGNAME || os.userInfo().username; +} + +function processUser(pid: number, host: HermesForwardWatcherHost): string | null { + const result = host.run("ps", ["-p", String(pid), "-o", "user="], { env: host.env }); + const user = result.status === 0 ? result.stdout.trim() : ""; + return user || null; +} + +function managedWatcherProcessStatus( + watcher: HermesForwardWatcherState, + host: HermesForwardWatcherHost, +): ManagedWatcherProcessStatus { + const pid = watcher.pid; + if (pid === null) return "unknown"; + const exists = pidExists(pid, host); + if (exists === false) return "absent"; + if (exists === null) return "unknown"; + + const commandLine = readProcessCommandLine(pid, host); + const observedUser = processUser(pid, host); + if (!commandLine || observedUser === null) { + const stillExists = pidExists(pid, host); + return stillExists === false ? "absent" : "unknown"; + } + return isManagedHermesForwardWatcherProcess({ + commandLine, + expectedUser: currentUser(host), + observedUser, + watcher, + }) + ? "managed" + : "other"; +} + +function waitForWatcherExit( + watcher: HermesForwardWatcherState, + host: HermesForwardWatcherHost, + timeoutMs: number, +): boolean { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const status = managedWatcherProcessStatus(watcher, host); + if (status === "absent" || status === "other") return true; + sleepMs(50); + } + const status = managedWatcherProcessStatus(watcher, host); + return status === "absent" || status === "other"; +} + +export function stopHermesForwardWatcherProcess( + watcher: HermesForwardWatcherState, + host: HermesForwardWatcherHost, +): boolean { + const pid = watcher.pid; + if (pid === null) { + host.warn(`Failed to read a valid Hermes forward watcher PID from ${watcher.pidFile}.`); + return false; + } + const initialStatus = managedWatcherProcessStatus(watcher, host); + if (initialStatus === "absent" || initialStatus === "other") return true; + if (initialStatus === "unknown") { + host.warn(`Failed to inspect Hermes forward watcher ${pid}; preserving state for retry.`); + return false; + } + + host.kill(pid); + if (waitForWatcherExit(watcher, host, 1000)) { + host.log(`Stopped Hermes forward watcher ${pid}`); + return true; + } + const beforeForceKill = managedWatcherProcessStatus(watcher, host); + if (beforeForceKill === "absent" || beforeForceKill === "other") { + host.log(`Stopped Hermes forward watcher ${pid}`); + return true; + } + if (beforeForceKill === "unknown") { + host.warn( + `Failed to confirm Hermes forward watcher ${pid} identity; preserving state for retry.`, + ); + return false; + } + host.kill(pid, "SIGKILL"); + if (waitForWatcherExit(watcher, host, 1000)) { + host.log(`Stopped Hermes forward watcher ${pid}`); + return true; + } + host.warn(`Failed to stop Hermes forward watcher ${pid}`); + return false; +} + +export function stopHermesSandboxForward( + watcher: HermesForwardWatcherState, + host: HermesForwardWatcherHost, +): boolean { + if (!host.commandExists("openshell")) { + host.warn( + `Failed to stop Hermes forward for sandbox '${watcher.sandbox}' on port ${watcher.port}: openshell is unavailable.`, + ); + return false; + } + const result = host.run("openshell", ["forward", "stop", watcher.port, watcher.sandbox], { + env: host.env, + }); + if (result.status === 0) return true; + host.warn( + `Failed to stop Hermes forward for sandbox '${watcher.sandbox}' on port ${watcher.port} (exit ${String(result.status ?? "unknown")}).`, + ); + return false; +} diff --git a/src/lib/domain/uninstall/hermes-forward-watcher.ts b/src/lib/domain/uninstall/hermes-forward-watcher.ts new file mode 100644 index 00000000000..7da8dfbc9f7 --- /dev/null +++ b/src/lib/domain/uninstall/hermes-forward-watcher.ts @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +export interface HermesForwardWatcherState { + pid: number | null; + pidFile: string; + port: string; + sandbox: string; + watcherScript: string; +} + +export type HermesForwardWatcherCommandLine = + | { kind: "argv"; value: readonly string[] } + | { kind: "ps"; value: string }; + +function hasExpectedExecutableName(executable: string, expected: "node" | "openshell"): boolean { + const basename = path.basename(executable); + return expected === "node" ? basename === "node" || basename === "nodejs" : basename === expected; +} + +function matchesExactArgv(argv: readonly string[], watcher: HermesForwardWatcherState): boolean { + return ( + argv.length === 5 && + path.isAbsolute(argv[0] ?? "") && + hasExpectedExecutableName(argv[0] ?? "", "node") && + argv[1] === watcher.watcherScript && + path.isAbsolute(argv[2] ?? "") && + hasExpectedExecutableName(argv[2] ?? "", "openshell") && + argv[3] === watcher.port && + argv[4] === watcher.sandbox + ); +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function matchesExactPsCommandLine( + commandLine: string, + watcher: HermesForwardWatcherState, +): boolean { + const nodeExecutable = String.raw`\/(?:\S*\/)?node(?:js)?`; + const openshellExecutable = String.raw`\/(?:\S*\/)?openshell`; + const expected = new RegExp( + `^${nodeExecutable}[ \\t]+${escapeRegExp(watcher.watcherScript)}[ \\t]+${openshellExecutable}[ \\t]+${escapeRegExp(watcher.port)}[ \\t]+${escapeRegExp(watcher.sandbox)}$`, + ); + return expected.test(commandLine.trim()); +} + +export function isManagedHermesForwardWatcherProcess(input: { + commandLine: HermesForwardWatcherCommandLine | null; + expectedUser: string; + observedUser: string; + watcher: HermesForwardWatcherState; +}): boolean { + const { commandLine, expectedUser, observedUser, watcher } = input; + if (!commandLine || observedUser !== expectedUser) return false; + return commandLine.kind === "argv" + ? matchesExactArgv(commandLine.value, watcher) + : matchesExactPsCommandLine(commandLine.value, watcher); +} diff --git a/src/lib/state/hermes-forward-watcher.ts b/src/lib/state/hermes-forward-watcher.ts new file mode 100644 index 00000000000..803e6aa4adb --- /dev/null +++ b/src/lib/state/hermes-forward-watcher.ts @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import type { HermesForwardWatcherState } from "../domain/uninstall/hermes-forward-watcher"; + +const HERMES_FORWARD_WATCHER_STATE_SUBDIR = "state"; +const HERMES_FORWARD_WATCHER_FILE_PATTERN = /^hermes-(.+)-(\d+)\.forward\.pid$/; + +export interface HermesForwardWatcherStateResult { + readable: boolean; + watchers: HermesForwardWatcherState[]; +} + +function parsePid(raw: string): number | null { + const value = raw.trim(); + if (!/^[1-9]\d*$/.test(value)) return null; + const pid = Number(value); + return Number.isSafeInteger(pid) ? pid : null; +} + +function readPid(pidFile: string): number | null { + let descriptor: number | null = null; + try { + descriptor = fs.openSync(pidFile, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + if (!fs.fstatSync(descriptor).isFile()) return null; + return parsePid(fs.readFileSync(descriptor, "utf-8")); + } catch { + return null; + } finally { + if (descriptor !== null) fs.closeSync(descriptor); + } +} + +export function readHermesForwardWatcherState( + nemoclawStateDir: string, +): HermesForwardWatcherStateResult { + const stateDir = path.join(nemoclawStateDir, HERMES_FORWARD_WATCHER_STATE_SUBDIR); + if (!fs.existsSync(stateDir)) return { readable: true, watchers: [] }; + + let entries: string[]; + try { + const stat = fs.lstatSync(stateDir); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + return { readable: false, watchers: [] }; + } + entries = fs.readdirSync(stateDir); + } catch { + return { readable: false, watchers: [] }; + } + + const watchers = entries.flatMap((name): HermesForwardWatcherState[] => { + const match = HERMES_FORWARD_WATCHER_FILE_PATTERN.exec(name); + if (!match || path.basename(name) !== name) return []; + const [, sandbox, port] = match; + const portNumber = Number(port); + if (!Number.isInteger(portNumber) || portNumber < 1 || portNumber > 65_535) return []; + const pidFile = path.join(stateDir, name); + return [ + { + pid: readPid(pidFile), + pidFile, + port, + sandbox, + watcherScript: `${pidFile}.js`, + }, + ]; + }); + return { readable: true, watchers }; +}