diff --git a/src/lib/actions/uninstall/run-plan.test.ts b/src/lib/actions/uninstall/run-plan.test.ts index f7813c50574..32ae0ae9611 100644 --- a/src/lib/actions/uninstall/run-plan.test.ts +++ b/src/lib/actions/uninstall/run-plan.test.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; + import { describe, expect, it, vi } from "vitest"; import { buildRunPlan, runUninstallPlan, type RunResult } from "./run-plan"; @@ -9,6 +11,28 @@ function ok(stdout = ""): RunResult { return { status: 0, stdout, stderr: "" }; } +function notFound(): RunResult { + return { status: 1, stdout: "", stderr: "" }; +} + +const PROXY_CMDLINE = "/usr/bin/node /opt/nemoclaw/scripts/ollama-auth-proxy.js\n"; + +function psStub( + pidStr: string, + opts: { exited: Set; cmdline?: string; owner?: string }, +) { + return (args: readonly string[]): RunResult | null => { + if (args[0] !== "-p" || args[1] !== pidStr || args[2] !== "-o") return null; + const pid = Number(pidStr); + if (args[3] === "pid=") { + return opts.exited.has(pid) ? notFound() : ok(`${pidStr}\n`); + } + if (args[3] === "user=") return ok(`${opts.owner ?? "testuser"}\n`); + if (args[3] === "args=") return ok(opts.cmdline ?? PROXY_CMDLINE); + return null; + }; +} + describe("uninstall run plan", () => { it("builds a plan using host paths and shim classification", () => { const { paths, plan } = buildRunPlan( @@ -152,6 +176,338 @@ describe("uninstall run plan", () => { expect(run).not.toHaveBeenCalled(); }); + it("kills the Ollama auth proxy via the persisted PID file (#2759)", () => { + const logs: string[] = []; + const killed: number[] = []; + const exited = new Set(); + // Simulate the persisted PID file under ~/.nemoclaw/. + const tmpHome = "/tmp/nemoclaw-uninstall-test-2759-pidfile"; + const pidFile = `${tmpHome}/.nemoclaw/ollama-auth-proxy.pid`; + fs.mkdirSync(`${tmpHome}/.nemoclaw`, { recursive: true }); + fs.writeFileSync(pidFile, "44321\n"); + + try { + const stub = psStub("44321", { exited }); + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: () => true, + env: { HOME: tmpHome, LOGNAME: "testuser" } as NodeJS.ProcessEnv, + existsSync: (target) => target === pidFile, + isTty: false, + kill: (pid, _signal) => { + killed.push(pid); + exited.add(pid); + return true; + }, + log: (line) => logs.push(line), + rmSync: vi.fn(), + run: (command, args) => { + if (command === "ps") { + const result = stub(args); + if (result) return result; + } + // lsof fallback returns nothing — PID-file branch should win. + if (command === "lsof") return ok(""); + if (args[0] === "-c") return ok("/fake/bin/tool\n"); + if (args[0] === "-f") return ok(""); + return ok(); + }, + runDocker: () => ok(""), + }, + ); + + expect(result.exitCode).toBe(0); + expect(killed).toContain(44321); + expect(logs).toContain("Stopped Ollama auth proxy 44321"); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("kills an orphan auth proxy via lsof :11435 when the PID file is gone", () => { + const logs: string[] = []; + const killed: number[] = []; + const exited = new Set(); + const stub = psStub("55678", { exited }); + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: () => true, + env: { + HOME: "/tmp/nemoclaw-uninstall-test-2759-lsof", + LOGNAME: "testuser", + } as NodeJS.ProcessEnv, + existsSync: () => false, + isTty: false, + kill: (pid, _signal) => { + killed.push(pid); + exited.add(pid); + return true; + }, + log: (line) => logs.push(line), + rmSync: vi.fn(), + run: (command, args) => { + if (command === "lsof" && args[0] === "-ti" && args[1] === ":11435") { + return ok("55678\n"); + } + if (command === "ps") { + const result = stub(args); + if (result) return result; + } + if (args[0] === "-c") return ok("/fake/bin/tool\n"); + if (args[0] === "-f") return ok(""); + return ok(); + }, + runDocker: () => ok(""), + }, + ); + + expect(result.exitCode).toBe(0); + expect(killed).toContain(55678); + expect(logs).toContain("Stopped Ollama auth proxy 55678"); + }); + + it("never stops a foreign-owned auth proxy on :11435 even if cmdline matches", () => { + const logs: string[] = []; + const killed: number[] = []; + const stub = psStub("77777", { exited: new Set(), owner: "someone-else" }); + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: () => true, + env: { + HOME: "/tmp/nemoclaw-uninstall-test-2759-foreign-owner", + LOGNAME: "testuser", + } as NodeJS.ProcessEnv, + existsSync: () => false, + isTty: false, + kill: (pid) => { + killed.push(pid); + return true; + }, + log: (line) => logs.push(line), + rmSync: vi.fn(), + run: (command, args) => { + if (command === "lsof" && args[0] === "-ti" && args[1] === ":11435") { + return ok("77777\n"); + } + if (command === "ps") { + const result = stub(args); + if (result) return result; + } + if (args[0] === "-c") return ok("/fake/bin/tool\n"); + if (args[0] === "-f") return ok(""); + return ok(); + }, + runDocker: () => ok(""), + }, + ); + + expect(result.exitCode).toBe(0); + expect(killed).not.toContain(77777); + expect(logs).toContain("No Ollama auth proxy processes found"); + }); + + it("scans the custom NEMOCLAW_OLLAMA_PROXY_PORT for orphan auth proxies", () => { + const logs: string[] = []; + const killed: number[] = []; + const exited = new Set(); + const stub = psStub("33333", { exited }); + const lsofPorts: string[] = []; + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: () => true, + env: { + HOME: "/tmp/nemoclaw-uninstall-test-2759-custom-port", + LOGNAME: "testuser", + NEMOCLAW_OLLAMA_PROXY_PORT: "12000", + } as NodeJS.ProcessEnv, + existsSync: () => false, + isTty: false, + kill: (pid, _signal) => { + killed.push(pid); + exited.add(pid); + return true; + }, + log: (line) => logs.push(line), + rmSync: vi.fn(), + run: (command, args) => { + if (command === "lsof" && args[0] === "-ti") { + lsofPorts.push(args[1] ?? ""); + // Only return a hit when the scan is asking about the custom port. + if (args[1] === ":12000") return ok("33333\n"); + return ok(""); + } + if (command === "ps") { + const result = stub(args); + if (result) return result; + } + if (args[0] === "-c") return ok("/fake/bin/tool\n"); + if (args[0] === "-f") return ok(""); + return ok(); + }, + runDocker: () => ok(""), + }, + ); + + expect(result.exitCode).toBe(0); + expect(lsofPorts).toContain(":12000"); + expect(lsofPorts).not.toContain(":11435"); + expect(killed).toContain(33333); + expect(logs).toContain("Stopped Ollama auth proxy 33333"); + }); + + it("never kills a process on :11435 whose cmdline is not the auth proxy", () => { + const logs: string[] = []; + const killed: number[] = []; + // Same owner, different cmdline — exercises the cmdline gate specifically. + const stub = psStub("99999", { + exited: new Set(), + cmdline: "/usr/sbin/nginx -g daemon off;\n", + }); + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: () => true, + env: { + HOME: "/tmp/nemoclaw-uninstall-test-2759-foreign", + LOGNAME: "testuser", + } as NodeJS.ProcessEnv, + existsSync: () => false, + isTty: false, + kill: (pid) => { + killed.push(pid); + return true; + }, + log: (line) => logs.push(line), + rmSync: vi.fn(), + run: (command, args) => { + if (command === "lsof" && args[0] === "-ti" && args[1] === ":11435") { + return ok("99999\n"); + } + if (command === "ps") { + const result = stub(args); + if (result) return result; + } + if (args[0] === "-c") return ok("/fake/bin/tool\n"); + if (args[0] === "-f") return ok(""); + return ok(); + }, + runDocker: () => ok(""), + }, + ); + + expect(result.exitCode).toBe(0); + expect(killed).not.toContain(99999); + expect(logs).toContain("No Ollama auth proxy processes found"); + }); + + it("escalates to SIGKILL and reports failure when SIGTERM is ignored", () => { + const logs: string[] = []; + const warnings: string[] = []; + const signals: NodeJS.Signals[] = []; + const tmpHome = "/tmp/nemoclaw-uninstall-test-2759-stuck"; + const pidFile = `${tmpHome}/.nemoclaw/ollama-auth-proxy.pid`; + fs.mkdirSync(`${tmpHome}/.nemoclaw`, { recursive: true }); + fs.writeFileSync(pidFile, "44322\n"); + + try { + // exited stays empty — pidExists() always reports alive, simulating a + // process that ignores SIGTERM and survives SIGKILL. + const stub = psStub("44322", { exited: new Set() }); + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: () => true, + env: { HOME: tmpHome, LOGNAME: "testuser" } as NodeJS.ProcessEnv, + existsSync: (target) => target === pidFile, + isTty: false, + kill: (_pid, signal) => { + if (typeof signal === "string") signals.push(signal); + return true; + }, + log: (line: string) => logs.push(line), + error: (line: string) => warnings.push(line), + rmSync: vi.fn(), + run: (command, args) => { + if (command === "ps") { + const result = stub(args); + if (result) return result; + } + if (command === "lsof") return ok(""); + if (args[0] === "-c") return ok("/fake/bin/tool\n"); + if (args[0] === "-f") return ok(""); + return ok(); + }, + runDocker: () => ok(""), + }, + ); + + expect(result.exitCode).toBe(0); + expect(signals).toContain("SIGKILL"); + expect(warnings).toContain("Failed to stop Ollama auth proxy 44322"); + expect(logs).not.toContain("Stopped Ollama auth proxy 44322"); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("warns instead of claiming success when lsof is unavailable for orphan scan", () => { + const logs: string[] = []; + const warnings: string[] = []; + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: (command) => command !== "lsof", + env: { HOME: "/tmp/nemoclaw-uninstall-test-2759-no-lsof" } as NodeJS.ProcessEnv, + existsSync: () => false, + isTty: false, + kill: () => true, + log: (line: string) => logs.push(line), + error: (line: string) => warnings.push(line), + rmSync: vi.fn(), + run: (_command, args) => { + if (args[0] === "-c") return ok("/fake/bin/tool\n"); + if (args[0] === "-f") return ok(""); + return ok(); + }, + runDocker: () => ok(""), + }, + ); + + expect(result.exitCode).toBe(0); + expect(warnings).toContain("lsof not found; skipping orphan Ollama auth proxy scan."); + expect(logs).not.toContain("No Ollama auth proxy processes found"); + }); + + it("logs and continues when no Ollama auth proxy is running", () => { + const logs: string[] = []; + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: () => true, + env: { HOME: "/tmp/nemoclaw-uninstall-test-2759-empty" } as NodeJS.ProcessEnv, + existsSync: () => false, + isTty: false, + kill: () => true, + log: (line) => logs.push(line), + rmSync: vi.fn(), + run: (command, args) => { + if (command === "lsof") return ok(""); + if (args[0] === "-c") return ok("/fake/bin/tool\n"); + if (args[0] === "-f") return ok(""); + return ok(); + }, + runDocker: () => ok(""), + }, + ); + + expect(result.exitCode).toBe(0); + expect(logs).toContain("No Ollama auth proxy processes found"); + }); + it("does not report swap cleanup success when swapoff fails", () => { const warnings: string[] = []; const logs: string[] = []; diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index 17c6471c55e..d89670404f5 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -9,6 +9,7 @@ import path from "node:path"; import { dockerSpawnSync } from "../../adapters/docker/exec"; import { getAgentBranding, type AgentBranding } from "../../cli/branding"; +import { sleepMs } from "../../core/wait"; import { defaultUninstallPaths, NEMOCLAW_OLLAMA_MODELS, NEMOCLAW_PROVIDERS, type UninstallPaths } from "../../domain/uninstall/paths"; import { buildUninstallPlan, type UninstallPlan } from "../../domain/uninstall/plan"; import { classifyShimPath, type FileSystemDeps } from "./plan"; @@ -234,6 +235,133 @@ function stopMatchingPids(pattern: string, runtime: UninstallRuntime, label: str } } +// Identifier we look for in `/proc//cmdline` (via `ps -p -o args=`) +// to confirm a candidate PID is the Ollama auth proxy and not another node +// process that happens to be on the same port. Mirrors the +// `isOllamaProxyProcess` check in `src/lib/onboard-ollama-proxy.ts`. +const OLLAMA_AUTH_PROXY_CMDLINE_MARK = "ollama-auth-proxy.js"; + +// Resolve the proxy port from runtime.env (rather than `process.env` at +// module-load time) so a user who onboarded with NEMOCLAW_OLLAMA_PROXY_PORT +// set to a custom value sees uninstall scan that same port. Mirrors the +// validation in `src/lib/core/ports.ts::parsePort`; falls back silently to +// the default (11435) on malformed input — uninstall is best-effort. +const DEFAULT_OLLAMA_PROXY_PORT = 11435; + +function resolveOllamaProxyPort(runtime: UninstallRuntime): number { + const raw = runtime.env.NEMOCLAW_OLLAMA_PROXY_PORT; + if (raw === undefined || raw === "") return DEFAULT_OLLAMA_PROXY_PORT; + const trimmed = String(raw).trim(); + if (!/^\d+$/.test(trimmed)) return DEFAULT_OLLAMA_PROXY_PORT; + const parsed = Number(trimmed); + if (parsed < 1024 || parsed > 65535) return DEFAULT_OLLAMA_PROXY_PORT; + return parsed; +} + +function isOllamaAuthProxyPid(pid: number, runtime: UninstallRuntime): boolean { + if (!Number.isInteger(pid) || pid <= 0) return false; + const result = runtime.run("ps", ["-p", String(pid), "-o", "args="], { env: runtime.env }); + return result.status === 0 && result.stdout.includes(OLLAMA_AUTH_PROXY_CMDLINE_MARK); +} + +// `ps -p ` is preferred over `kill(pid, 0)` for existence probing here: +// `runtime.kill()` collapses every `process.kill` error to `false`, so a foreign +// PID throwing EPERM (process exists but caller can't signal it) would look +// identical to ESRCH (gone) and we'd falsely log it as Stopped. `ps` reports +// existence regardless of signalling permission. +function pidExists(pid: number, runtime: UninstallRuntime): boolean { + return runtime.run("ps", ["-p", String(pid), "-o", "pid="], { env: runtime.env }).status === 0; +} + +function waitForPidExit(pid: number, runtime: UninstallRuntime, timeoutMs: number): boolean { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (!pidExists(pid, runtime)) return true; + sleepMs(50); + } + return !pidExists(pid, runtime); +} + +function pidOwnedByCurrentUser(pid: number, runtime: UninstallRuntime): boolean { + const expected = runtime.env.SUDO_USER || runtime.env.LOGNAME || os.userInfo().username; + if (!expected) return true; + const result = runtime.run("ps", ["-p", String(pid), "-o", "user="], { env: runtime.env }); + return result.status === 0 && result.stdout.trim() === expected; +} + +function tryStopOllamaProxyPid(pid: number, runtime: UninstallRuntime): boolean { + // `runtime.kill()` only confirms the signal was sent; the proxy may ignore + // SIGTERM, take time to clean up, or linger as a zombie. Verify the PID is + // actually gone before claiming success — otherwise the next install fails + // with `Ollama auth proxy failed to start on :11435`. + runtime.kill(pid); + if (waitForPidExit(pid, runtime, 1000)) { + runtime.log(`Stopped Ollama auth proxy ${pid}`); + return true; + } + runtime.kill(pid, "SIGKILL"); + if (waitForPidExit(pid, runtime, 1000)) { + runtime.log(`Stopped Ollama auth proxy ${pid}`); + return true; + } + runtime.warn(`Failed to stop Ollama auth proxy ${pid}`); + return false; +} + +function stopOllamaAuthProxy(paths: UninstallPaths, runtime: UninstallRuntime): void { + // The auth proxy is a detached node child started by the Local Ollama + // onboard path that listens on `NEMOCLAW_OLLAMA_PROXY_PORT` (default + // 11435). Without this cleanup, + // uninstall + reinstall fails with `Ollama auth proxy failed to start on + // :11435` — the on-disk PID file is removed by the "State and binaries" + // step but the process keeps running. The two-prong check (persisted PID + // first, then port-bound listeners) mirrors `killStaleProxy()` in + // `src/lib/onboard-ollama-proxy.ts` and verifies cmdline on every PID, so + // an unrelated process on the same port (custom proxy, test setup) is + // never killed. See issue #2759. + const stopped = new Set(); + + // 1. Try the persisted PID file. The proxy stays bound across NemoClaw + // sessions; the PID file is the most reliable signal. The path mirrors + // `PROXY_PID_PATH` in `src/lib/onboard-ollama-proxy.ts` (`~/.nemoclaw`). + const pidFile = path.join(paths.nemoclawStateDir, "ollama-auth-proxy.pid"); + if (runtime.existsSync(pidFile)) { + try { + const raw = fs.readFileSync(pidFile, "utf-8").trim(); + const pid = Number.parseInt(raw, 10); + if (Number.isFinite(pid) && pid > 0 && isOllamaAuthProxyPid(pid, runtime)) { + if (tryStopOllamaProxyPid(pid, runtime)) stopped.add(pid); + } + } catch { + /* ignore — the State step deletes the file shortly anyway */ + } + } + + // 2. Fall back to the configured proxy port for orphans whose PID file is + // gone (e.g. a previous uninstall already wiped state but the process + // survived). Filter via cmdline so we never kill unrelated listeners. + if (!runtime.commandExists("lsof")) { + if (stopped.size === 0) { + runtime.warn("lsof not found; skipping orphan Ollama auth proxy scan."); + } + return; + } + const proxyPort = resolveOllamaProxyPort(runtime); + const lsof = runtime.run("lsof", ["-ti", `:${proxyPort}`], { env: runtime.env }); + const pids = splitNonEmptyLines(lsof.stdout).map(Number).filter(Number.isFinite); + for (const pid of pids) { + if (stopped.has(pid)) continue; + // Skip foreign-owned PIDs even if the cmdline matches: signalling them + // would either no-op under EPERM or escalate via sudo, neither of which + // is appropriate during a per-user uninstall. + if (!pidOwnedByCurrentUser(pid, runtime)) continue; + if (!isOllamaAuthProxyPid(pid, runtime)) continue; + if (tryStopOllamaProxyPid(pid, runtime)) stopped.add(pid); + } + + if (stopped.size === 0) runtime.log("No Ollama auth proxy processes found"); +} + function stopOrphanedOpenShell(runtime: UninstallRuntime): void { if (!runtime.commandExists("pgrep")) { runtime.warn("pgrep not found; skipping orphaned openshell process cleanup."); @@ -430,6 +558,7 @@ function executePlan(plan: UninstallPlan, paths: UninstallPaths, options: Uninst removeGlob(paths.helperServiceGlob, runtime); stopMatchingPids(`openshell.*forward.*${runtime.env.NEMOCLAW_DASHBOARD_PORT || "18789"}`, runtime, "local OpenShell forward processes"); stopOrphanedOpenShell(runtime); + stopOllamaAuthProxy(paths, runtime); } else if (step.name === "OpenShell resources") { removeOpenShellResources(options, runtime); } else if (step.name === "NemoClaw CLI") { diff --git a/src/lib/domain/uninstall/plan.test.ts b/src/lib/domain/uninstall/plan.test.ts index 011655806f7..a3e9c61cde0 100644 --- a/src/lib/domain/uninstall/plan.test.ts +++ b/src/lib/domain/uninstall/plan.test.ts @@ -34,8 +34,18 @@ describe("uninstall plan", () => { { kind: "preserve-ollama-models", names: ["nemotron-3-super:120b", "nemotron-3-nano:30b"] }, { kind: "delete-managed-swap" }, { kind: "delete-openshell-binary", path: "/usr/local/bin/openshell" }, + { kind: "stop-ollama-auth-proxy" }, ]), ); + + // The Ollama auth proxy must be stopped during the "Stopping services" + // step, before any "State and binaries" cleanup deletes the PID file. + // Otherwise a stale proxy on :11435 blocks reinstall (issue #2759). + const stoppingServicesStep = plan.steps.find((step) => step.name === "Stopping services"); + expect(stoppingServicesStep).toBeTruthy(); + expect(stoppingServicesStep?.actions).toEqual( + expect.arrayContaining([{ kind: "stop-ollama-auth-proxy" }]), + ); }); it("respects delete-models, keep-openshell, custom gateway, and foreign shim decisions", () => { diff --git a/src/lib/domain/uninstall/plan.ts b/src/lib/domain/uninstall/plan.ts index 4fd504c9036..577a631a4bf 100644 --- a/src/lib/domain/uninstall/plan.ts +++ b/src/lib/domain/uninstall/plan.ts @@ -35,6 +35,7 @@ export type UninstallPlanAction = | { kind: "preserve-openshell-binary"; paths: string[] } | { kind: "preserve-shim"; reason: string } | { kind: "stop-helper-services" } + | { kind: "stop-ollama-auth-proxy" } | { kind: "stop-openshell-forward-processes" } | { kind: "stop-orphaned-openshell-processes" } | { kind: "uninstall-npm-package"; name: "nemoclaw" }; @@ -68,6 +69,7 @@ export function buildUninstallPlan(paths: UninstallPaths, options: UninstallPlan { kind: "delete-runtime-glob", pattern: paths.helperServiceGlob }, { kind: "stop-openshell-forward-processes" }, { kind: "stop-orphaned-openshell-processes" }, + { kind: "stop-ollama-auth-proxy" }, ], }, {