Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
193 changes: 193 additions & 0 deletions src/lib/actions/uninstall/run-plan.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ function notFound(): RunResult {
}

const PROXY_CMDLINE = "/usr/bin/node /opt/nemoclaw/scripts/ollama-auth-proxy.js\n";
// Real-world: model-router is a Python venv script so the OS interposes the
// interpreter — args[0]=python, args[1]=model-router (issue #5169).
const MODEL_ROUTER_CMDLINE =
"/home/test/.nemoclaw/model-router-venv/bin/python /home/test/.nemoclaw/model-router-venv/bin/model-router proxy --port 4000\n";

function psStub(pidStr: string, opts: { exited: Set<number>; cmdline?: string; owner?: string }) {
return (args: readonly string[]): RunResult | null => {
Expand Down Expand Up @@ -517,6 +521,195 @@ describe("uninstall run plan", () => {
expect(logs).toContain("No Ollama auth proxy processes found");
});

it("kills the model router via onboard-session routerPid (#5169)", () => {
const logs: string[] = [];
const killed: number[] = [];
const exited = new Set<number>();
const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-test-5169-session-"));
const stateDir = path.join(tmpHome, ".nemoclaw");
const sessionFile = path.join(stateDir, "onboard-session.json");
fs.mkdirSync(stateDir, { recursive: true });
fs.writeFileSync(sessionFile, JSON.stringify({ routerPid: 55432 }));

try {
const stub = psStub("55432", { exited, cmdline: MODEL_ROUTER_CMDLINE });
const result = runUninstallPlan(
{ assumeYes: true, deleteModels: false, keepOpenShell: true },
{
commandExists: () => true,
env: { HOME: tmpHome, LOGNAME: "testuser" } as NodeJS.ProcessEnv,
existsSync: () => false,
isTty: false,
kill: (pid, _signal) => {
killed.push(pid);
exited.add(pid);
return true;
},
log: (line) => logs.push(line),
rmSync: vi.fn(),
run: (command, args) => {
if (command === "ps") {
const result = stub(args);
if (result) return result;
}
if (command === "lsof") return ok("");
if (args[0] === "-c") return ok("/fake/bin/tool\n");
if (args[0] === "-f") return ok("");
return ok();
},
runDocker: () => ok(""),
},
);

expect(result.exitCode).toBe(0);
expect(killed).toContain(55432);
expect(logs).toContain("Stopped model router 55432");
} finally {
fs.rmSync(tmpHome, { recursive: true, force: true });
}
});

it("kills an orphan model router via lsof :4000 when onboard-session is gone", () => {
const logs: string[] = [];
const killed: number[] = [];
const exited = new Set<number>();
const stub = psStub("55679", { exited, cmdline: MODEL_ROUTER_CMDLINE });
const result = runUninstallPlan(
{ assumeYes: true, deleteModels: false, keepOpenShell: true },
{
commandExists: () => true,
env: {
HOME: "/tmp/nemoclaw-uninstall-test-5169-lsof",
LOGNAME: "testuser",
} as NodeJS.ProcessEnv,
existsSync: () => false,
isTty: false,
kill: (pid, _signal) => {
killed.push(pid);
exited.add(pid);
return true;
},
log: (line) => logs.push(line),
rmSync: vi.fn(),
run: (command, args) => {
if (command === "lsof" && args[0] === "-ti" && args[1] === ":4000") {
return ok("55679\n");
}
if (command === "lsof" && args[0] === "-ti" && args[1] === ":11435") {
return ok("");
}
if (command === "ps") {
const result = stub(args);
if (result) return result;
}
if (args[0] === "-c") return ok("/fake/bin/tool\n");
if (args[0] === "-f") return ok("");
return ok();
},
runDocker: () => ok(""),
},
);

expect(result.exitCode).toBe(0);
expect(killed).toContain(55679);
expect(logs).toContain("Stopped model router 55679");
});

it("never stops a foreign-owned model router on :4000 even if cmdline matches", () => {
const logs: string[] = [];
const killed: number[] = [];
const stub = psStub("77888", {
exited: new Set(),
owner: "someone-else",
cmdline: MODEL_ROUTER_CMDLINE,
});
const result = runUninstallPlan(
{ assumeYes: true, deleteModels: false, keepOpenShell: true },
{
commandExists: () => true,
env: {
HOME: "/tmp/nemoclaw-uninstall-test-5169-foreign-owner",
LOGNAME: "testuser",
} as NodeJS.ProcessEnv,
existsSync: () => false,
isTty: false,
kill: (pid) => {
killed.push(pid);
return true;
},
log: (line) => logs.push(line),
rmSync: vi.fn(),
run: (command, args) => {
if (command === "lsof" && args[0] === "-ti" && args[1] === ":4000") {
return ok("77888\n");
}
if (command === "lsof" && args[0] === "-ti" && args[1] === ":11435") {
return ok("");
}
if (command === "ps") {
const result = stub(args);
if (result) return result;
}
if (args[0] === "-c") return ok("/fake/bin/tool\n");
if (args[0] === "-f") return ok("");
return ok();
},
runDocker: () => ok(""),
},
);

expect(result.exitCode).toBe(0);
expect(killed).not.toContain(77888);
expect(logs).toContain("No model router processes found");
});

it("never kills a process on :4000 whose cmdline is not the model router", () => {
const logs: string[] = [];
const killed: number[] = [];
const stub = psStub("88888", {
exited: new Set(),
cmdline: "/usr/sbin/nginx -g daemon off;\n",
});
const result = runUninstallPlan(
{ assumeYes: true, deleteModels: false, keepOpenShell: true },
{
commandExists: () => true,
env: {
HOME: "/tmp/nemoclaw-uninstall-test-5169-foreign-cmdline",
LOGNAME: "testuser",
} as NodeJS.ProcessEnv,
existsSync: () => false,
isTty: false,
kill: (pid) => {
killed.push(pid);
return true;
},
log: (line) => logs.push(line),
rmSync: vi.fn(),
run: (command, args) => {
if (command === "lsof" && args[0] === "-ti" && args[1] === ":4000") {
return ok("88888\n");
}
if (command === "lsof" && args[0] === "-ti" && args[1] === ":11435") {
return ok("");
}
if (command === "ps") {
const result = stub(args);
if (result) return result;
}
if (args[0] === "-c") return ok("/fake/bin/tool\n");
if (args[0] === "-f") return ok("");
return ok();
},
runDocker: () => ok(""),
},
);

expect(result.exitCode).toBe(0);
expect(killed).not.toContain(88888);
expect(logs).toContain("No model router processes found");
});

it("escalates to SIGKILL and reports failure when SIGTERM is ignored", () => {
const logs: string[] = [];
const warnings: string[] = [];
Expand Down
83 changes: 83 additions & 0 deletions src/lib/actions/uninstall/run-plan.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import {
type UninstallPaths,
} from "../../domain/uninstall/paths";
import { buildUninstallPlan, type UninstallPlan } from "../../domain/uninstall/plan";
import { isModelRouterCommandLineForPort } from "../../onboard/model-router-process";
import { stopHostGatewayProcesses } from "../../onboard/host-gateway-process";
import { stopStaleDashboardListeners } from "../../onboard/stale-gateway-cleanup";
import { classifyShimPath, type FileSystemDeps } from "./plan";
Expand Down Expand Up @@ -481,6 +482,87 @@ function stopOllamaAuthProxy(paths: UninstallPaths, runtime: UninstallRuntime):
if (stopped.size === 0) runtime.log("No Ollama auth proxy processes found");
}

const DEFAULT_MODEL_ROUTER_PORT = 4000;

function resolveModelRouterPort(_runtime: UninstallRuntime): number {
// Routed onboard profiles use blueprint port 4000 by default; a custom port
// would require reading the blueprint, which uninstall does not do today.
return DEFAULT_MODEL_ROUTER_PORT;
}

function readOnboardSessionRouterPid(paths: UninstallPaths): number | null {
const sessionFile = path.join(paths.nemoclawStateDir, "onboard-session.json");
try {
const raw = fs.readFileSync(sessionFile, "utf-8");
const data = JSON.parse(raw) as { routerPid?: unknown };
const pid = data.routerPid;
if (typeof pid === "number" && Number.isInteger(pid) && pid > 0) return pid;
} catch {
/* ignore — State step deletes the file shortly anyway */
}
return null;
}

function isModelRouterPid(pid: number, port: number, runtime: UninstallRuntime): boolean {
if (!Number.isInteger(pid) || pid <= 0) return false;
if (!pidExists(pid, runtime)) return false;
const result = runtime.run("ps", ["-p", String(pid), "-o", "args="], { env: runtime.env });
if (result.status !== 0) return false;
const args = result.stdout.trim().split(/\s+/).filter(Boolean);
return isModelRouterCommandLineForPort(args, port);
}

function tryStopModelRouterPid(pid: number, runtime: UninstallRuntime): boolean {
runtime.kill(pid);
if (waitForPidExit(pid, runtime, 1000)) {
runtime.log(`Stopped model router ${pid}`);
return true;
}
runtime.kill(pid, "SIGKILL");
if (waitForPidExit(pid, runtime, 1000)) {
runtime.log(`Stopped model router ${pid}`);
return true;
}
runtime.warn(`Failed to stop model router ${pid}`);
return false;
}

function stopModelRouter(paths: UninstallPaths, runtime: UninstallRuntime): void {
// The model router is a detached child started during routed onboard that
// listens on port 4000 by default. Without this cleanup, uninstall +
// reinstall fails with "Port 4000 already has a healthy router endpoint".
// The tracked PID lives in ~/.nemoclaw/onboard-session.json (routerPid), not
// a dedicated .pid file. Mirrors stopOllamaAuthProxy() and issue #5169.
const stopped = new Set<number>();
const routerPort = resolveModelRouterPort(runtime);

const recordedPid = readOnboardSessionRouterPid(paths);
if (
recordedPid !== null &&
pidOwnedByCurrentUser(recordedPid, runtime) &&
isModelRouterPid(recordedPid, routerPort, runtime)
) {
if (tryStopModelRouterPid(recordedPid, runtime)) stopped.add(recordedPid);
}

if (!runtime.commandExists("lsof")) {
if (stopped.size === 0) {
runtime.warn("lsof not found; skipping orphan model router scan.");
}
return;
}
const lsof = runtime.run("lsof", ["-ti", `:${routerPort}`], { env: runtime.env });
const pids = splitNonEmptyLines(lsof.stdout).map(Number).filter(Number.isFinite);
for (const pid of pids) {
if (stopped.has(pid)) continue;
if (!pidOwnedByCurrentUser(pid, runtime)) continue;
if (!isModelRouterPid(pid, routerPort, runtime)) continue;
if (tryStopModelRouterPid(pid, runtime)) stopped.add(pid);
}

if (stopped.size === 0) runtime.log("No model router processes found");
}

function stopOrphanedOpenShell(runtime: UninstallRuntime): void {
if (!runtime.commandExists("pgrep")) {
runtime.warn("pgrep not found; skipping orphaned openshell process cleanup.");
Expand Down Expand Up @@ -788,6 +870,7 @@ function executePlan(
{ logNoProcesses: true },
);
stopOllamaAuthProxy(paths, runtime);
stopModelRouter(paths, runtime);
} else if (step.name === "OpenShell resources") {
removeOpenShellResources(options, runtime);
} else if (step.name === "NemoClaw CLI") {
Expand Down
10 changes: 6 additions & 4 deletions src/lib/domain/uninstall/plan.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,16 +40,18 @@ describe("uninstall plan", () => {
path: path.join("/usr/local/bin", binary),
})),
{ kind: "stop-ollama-auth-proxy" },
{ kind: "stop-model-router" },
]),
);

// The Ollama auth proxy must be stopped during the "Stopping services"
// step, before any "State and binaries" cleanup deletes the PID file.
// Otherwise a stale proxy on :11435 blocks reinstall (issue #2759).
// Both the Ollama auth proxy and the model router must be stopped during
// the "Stopping services" step, before "State and binaries" deletes PID
// files. A stale proxy on :11435 blocks reinstall (#2759); a stale router
// on :4000 blocks reinstall (#5169).
const stoppingServicesStep = plan.steps.find((step) => step.name === "Stopping services");
expect(stoppingServicesStep).toBeTruthy();
expect(stoppingServicesStep?.actions).toEqual(
expect.arrayContaining([{ kind: "stop-ollama-auth-proxy" }]),
expect.arrayContaining([{ kind: "stop-ollama-auth-proxy" }, { kind: "stop-model-router" }]),
);
});

Expand Down
2 changes: 2 additions & 0 deletions src/lib/domain/uninstall/plan.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ export type UninstallPlanAction =
| { kind: "preserve-openshell-install-paths"; paths: string[] }
| { kind: "preserve-shim"; reason: string }
| { kind: "stop-helper-services" }
| { kind: "stop-model-router" }
| { kind: "stop-ollama-auth-proxy" }
| { kind: "stop-openshell-forward-processes" }
| { kind: "stop-orphaned-openshell-processes" }
Expand Down Expand Up @@ -76,6 +77,7 @@ export function buildUninstallPlan(
{ kind: "stop-openshell-forward-processes" },
{ kind: "stop-orphaned-openshell-processes" },
{ kind: "stop-ollama-auth-proxy" },
{ kind: "stop-model-router" },
],
},
{
Expand Down
Loading