From b157d5fbd88f736eab41c1407c0e13b1602b8a86 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sun, 9 Aug 2026 20:50:46 -0700 Subject: [PATCH 1/5] fix(uninstall): isolate scoped gateway shutdown Signed-off-by: Apurv Kumaria --- ...run-plan-gateway-process-isolation.test.ts | 315 +++++++++ .../run-plan-gateway-segregation.test.ts | 7 +- .../run-plan-gateway-service.test.ts | 394 +++++++++++- src/lib/actions/uninstall/run-plan.ts | 594 ++++++++++++++++- .../docker-driver-gateway-prelaunch.test.ts | 1 + .../docker-driver-gateway-prelaunch.ts | 1 + src/lib/onboard/gateway-process-identity.ts | 13 +- .../gateway-process-target-identity.ts | 24 +- .../host-gateway-process-target.test.ts | 38 ++ src/lib/onboard/host-gateway-process.test.ts | 342 ++++++++++ src/lib/onboard/host-gateway-process.ts | 605 +++++++++++++++++- .../gateway-port-release-test-helpers.ts | 1 + .../e2e/live/concurrent-gateway-ports.test.ts | 391 ++++++++++- 13 files changed, 2694 insertions(+), 32 deletions(-) create mode 100644 src/lib/actions/uninstall/run-plan-gateway-process-isolation.test.ts diff --git a/src/lib/actions/uninstall/run-plan-gateway-process-isolation.test.ts b/src/lib/actions/uninstall/run-plan-gateway-process-isolation.test.ts new file mode 100644 index 00000000000..66445c9c45d --- /dev/null +++ b/src/lib/actions/uninstall/run-plan-gateway-process-isolation.test.ts @@ -0,0 +1,315 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { writeDockerDriverGatewayRuntimeMarkerForStateDir } from "../../onboard/docker-driver-gateway-runtime-marker"; +import { + type RunResult, + runUninstallPlan as runUninstallPlanBase, + type UninstallRunDeps, + type UninstallRunOptions, +} from "./run-plan"; + +function ok(stdout = ""): RunResult { + return { status: 0, stdout, stderr: "" }; +} + +function withManagedGatewayAuthority(deps: UninstallRunDeps): UninstallRunDeps { + const commandExists = deps.commandExists; + return { + resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + ...deps, + isPortFree: deps.isPortFree ?? (() => true), + commandExists: (command) => command === "lsof" || (commandExists?.(command) ?? false), + }; +} + +function bindManagedGatewayAuthority(run: typeof runUninstallPlanBase) { + return (options: UninstallRunOptions, deps: UninstallRunDeps) => + run(options, withManagedGatewayAuthority(deps)); +} + +function writeScopedGatewayPairState(options: { + markerPid: number; + pidFilePid: number; + selectedPort: number; + siblingPid: number; + tmpHome: string; +}) { + const { markerPid, pidFilePid, selectedPort, siblingPid, tmpHome } = options; + const sharedStateDir = path.join(tmpHome, ".nemoclaw"); + const selectedStateDir = path.join(sharedStateDir, "gateways", String(selectedPort)); + const gatewayRuntimeRoot = path.join(tmpHome, ".local", "state", "nemoclaw"); + const selectedGatewayRuntimeDir = path.join( + gatewayRuntimeRoot, + `openshell-docker-gateway-${String(selectedPort)}`, + ); + const siblingGatewayRuntimeDir = path.join(gatewayRuntimeRoot, "openshell-docker-gateway"); + fs.mkdirSync(selectedStateDir, { recursive: true }); + fs.mkdirSync(selectedGatewayRuntimeDir, { recursive: true }); + fs.mkdirSync(siblingGatewayRuntimeDir, { recursive: true }); + fs.writeFileSync( + path.join(sharedStateDir, "sandboxes.json"), + JSON.stringify({ + defaultSandbox: "sibling-box", + sandboxes: { + "sibling-box": { + name: "sibling-box", + gatewayName: "nemoclaw", + gatewayPort: 8080, + }, + }, + }), + ); + fs.writeFileSync( + path.join(selectedStateDir, "sandboxes.json"), + JSON.stringify({ + defaultSandbox: "selected-box", + sandboxes: { + "selected-box": { + name: "selected-box", + gatewayName: `nemoclaw-${String(selectedPort)}`, + gatewayPort: selectedPort, + }, + }, + }), + ); + const pidFile = path.join(selectedGatewayRuntimeDir, "openshell-gateway.pid"); + fs.writeFileSync(pidFile, `${String(pidFilePid)}\n`); + writeDockerDriverGatewayRuntimeMarkerForStateDir(selectedGatewayRuntimeDir, { + desiredEnv: {}, + endpoint: `https://127.0.0.1:${String(selectedPort)}`, + gatewayBin: "/opt/openshell-gateway", + pid: markerPid, + }); + fs.writeFileSync(path.join(selectedGatewayRuntimeDir, "selected-state"), "keep\n"); + fs.writeFileSync( + path.join(siblingGatewayRuntimeDir, "openshell-gateway.pid"), + `${String(siblingPid)}\n`, + ); + fs.writeFileSync(path.join(siblingGatewayRuntimeDir, "sibling-state"), "keep\n"); + return { + pidFile, + selectedGatewayRuntimeDir, + selectedStateDir, + sharedStateDir, + siblingGatewayRuntimeDir, + }; +} + +afterEach(() => { + vi.unstubAllEnvs(); + vi.resetModules(); +}); + +describe("scoped uninstall gateway process isolation", () => { + it("proves and stops only the selected gateway process during scoped uninstall (#8663)", async () => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-exact-process-")); + const selectedPort = 18_080; + const selectedPid = 987_650; + const siblingPid = 987_651; + try { + vi.stubEnv("NEMOCLAW_GATEWAY_PORT", String(selectedPort)); + vi.resetModules(); + const runPortUninstall = bindManagedGatewayAuthority( + (await import("./run-plan")).runUninstallPlan, + ); + const { + selectedGatewayRuntimeDir, + selectedStateDir, + sharedStateDir, + siblingGatewayRuntimeDir, + } = writeScopedGatewayPairState({ + markerPid: selectedPid, + pidFilePid: selectedPid, + selectedPort, + siblingPid, + tmpHome, + }); + + const events: string[] = []; + const signals: Array<{ pid: number; signal?: NodeJS.Signals | number }> = []; + const selectedUid = fs.statSync( + path.join(selectedGatewayRuntimeDir, "openshell-gateway.pid"), + ).uid; + let selectedAlive = true; + const result = runPortUninstall( + { + assumeYes: true, + deleteModels: false, + destroyUserData: true, + gatewayName: `nemoclaw-${String(selectedPort)}`, + keepOpenShell: false, + }, + { + commandExists: (command) => ["lsof", "openshell", "pgrep"].includes(command), + env: { + HOME: tmpHome, + LOGNAME: "tester", + NEMOCLAW_GATEWAY_PORT: String(selectedPort), + } as NodeJS.ProcessEnv, + existsSync: (target) => target.startsWith(tmpHome) && fs.existsSync(target), + isPortFree: () => !selectedAlive, + isTty: false, + kill: (pid, signal) => { + events.push(`kill ${String(pid)} ${String(signal)}`); + signals.push({ pid, signal }); + if (pid !== selectedPid || signal !== "SIGKILL") return false; + selectedAlive = false; + return true; + }, + log: vi.fn(), + run: (command, args) => { + events.push([command, ...args].join(" ")); + if (command === "openshell" && args[0] === "gateway" && args[1] === "list") { + return ok( + JSON.stringify([ + { name: "nemoclaw" }, + { name: `nemoclaw-${String(selectedPort)}` }, + ]), + ); + } + if (command === "lsof" && args.includes(`:${String(selectedPort)}`)) { + return selectedAlive ? ok(`${String(selectedPid)}\n`) : { ...ok(), status: 1 }; + } + if (command === "ps" && args[1] === String(selectedPid)) { + if (args.includes("pid=")) { + return selectedAlive ? ok(`${String(selectedPid)}\n`) : { ...ok(), status: 1 }; + } + if (args.includes("uid=")) return ok(`${String(selectedUid)}\n`); + if (args.includes("comm=")) return ok("/opt/openshell-gateway\n"); + if (args.includes("lstart=")) return ok("fixture-start-identity\n"); + if (args.includes("args=")) { + return ok( + `openshell-gateway[nemoclaw=nemoclaw-${String(selectedPort)};port=${String(selectedPort)}]\n`, + ); + } + } + if (command === "pgrep") return ok(`${String(siblingPid)}\n${String(selectedPid)}\n`); + return ok(); + }, + runDocker: () => ok(), + }, + ); + + expect(result.exitCode).toBe(0); + expect(signals).toEqual([{ pid: selectedPid, signal: "SIGKILL" }]); + expect(events.some((event) => event.startsWith("pgrep "))).toBe(false); + expect(events.indexOf("openshell sandbox delete selected-box")).toBeLessThan( + events.indexOf(`kill ${String(selectedPid)} SIGKILL`), + ); + expect(fs.existsSync(selectedStateDir)).toBe(false); + expect( + fs.readFileSync(path.join(siblingGatewayRuntimeDir, "openshell-gateway.pid"), "utf8"), + ).toBe(`${String(siblingPid)}\n`); + expect(fs.readFileSync(path.join(siblingGatewayRuntimeDir, "sibling-state"), "utf8")).toBe( + "keep\n", + ); + expect(fs.existsSync(path.join(sharedStateDir, "sandboxes.json"))).toBe(true); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("fails closed and preserves runtime evidence when scoped PID identities cross-match (#8663)", async () => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-crossed-process-")); + const selectedPort = 18_080; + const siblingPid = 987_652; + const selectedMarkerPid = 987_653; + try { + vi.stubEnv("NEMOCLAW_GATEWAY_PORT", String(selectedPort)); + vi.resetModules(); + const runPortUninstall = bindManagedGatewayAuthority( + (await import("./run-plan")).runUninstallPlan, + ); + const { + pidFile, + selectedGatewayRuntimeDir, + selectedStateDir, + sharedStateDir, + siblingGatewayRuntimeDir, + } = writeScopedGatewayPairState({ + markerPid: selectedMarkerPid, + pidFilePid: siblingPid, + selectedPort, + siblingPid, + tmpHome, + }); + + const errors: string[] = []; + const kill = vi.fn(() => true); + const calls: string[] = []; + const result = runPortUninstall( + { + assumeYes: true, + deleteModels: false, + destroyUserData: true, + gatewayName: `nemoclaw-${String(selectedPort)}`, + keepOpenShell: false, + }, + { + commandExists: (command) => ["lsof", "openshell", "pgrep"].includes(command), + env: { HOME: tmpHome, NEMOCLAW_GATEWAY_PORT: String(selectedPort) } as NodeJS.ProcessEnv, + error: (message) => errors.push(message), + existsSync: (target) => target.startsWith(tmpHome) && fs.existsSync(target), + isTty: false, + kill, + log: vi.fn(), + run: (command, args) => { + calls.push([command, ...args].join(" ")); + if (command === "openshell" && args[0] === "gateway" && args[1] === "list") { + return ok( + JSON.stringify([ + { name: "nemoclaw" }, + { name: `nemoclaw-${String(selectedPort)}` }, + ]), + ); + } + if (command === "ps" && args[1] === String(siblingPid) && args.includes("pid=")) { + return ok(`${String(siblingPid)}\n`); + } + if (command === "pgrep") { + return ok(`${String(siblingPid)}\n${String(selectedMarkerPid)}\n`); + } + return ok(); + }, + runDocker: () => ok(), + }, + ); + + expect(result.exitCode).toBe(1); + expect(kill).not.toHaveBeenCalled(); + expect(calls.some((call) => call.startsWith("pgrep "))).toBe(false); + expect(errors.join("\n")).toContain( + `runtime marker PID ${String(selectedMarkerPid)} does not match PID file ${String(siblingPid)}`, + ); + expect(fs.readFileSync(pidFile, "utf8")).toBe(`${String(siblingPid)}\n`); + expect(fs.readFileSync(path.join(selectedGatewayRuntimeDir, "selected-state"), "utf8")).toBe( + "keep\n", + ); + expect(fs.readFileSync(path.join(siblingGatewayRuntimeDir, "sibling-state"), "utf8")).toBe( + "keep\n", + ); + expect( + fs.readFileSync(path.join(siblingGatewayRuntimeDir, "openshell-gateway.pid"), "utf8"), + ).toBe(`${String(siblingPid)}\n`); + expect(fs.existsSync(selectedStateDir)).toBe(true); + expect(fs.existsSync(path.join(sharedStateDir, "sandboxes.json"))).toBe(true); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts index 48cddd87208..2d4d49cef1f 100644 --- a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts +++ b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts @@ -6,7 +6,6 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; - import { NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER_LINE } from "../../onboard/docker-driver-gateway-service"; import { readGatewayRegistryFile } from "../../state/gateway-registry"; import { migrateLegacyPortState } from "../../state/legacy-port-migration"; @@ -22,6 +21,7 @@ function ok(stdout = ""): RunResult { } function withManagedGatewayAuthority(deps: UninstallRunDeps): UninstallRunDeps { + const commandExists = deps.commandExists; return { resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ gatewayName, @@ -34,6 +34,11 @@ function withManagedGatewayAuthority(deps: UninstallRunDeps): UninstallRunDeps { requiredCapabilities: [], }), ...deps, + isPortFree: deps.isPortFree ?? (() => true), + // Scoped teardown must distinguish an absent selected gateway from an + // unobservable listener. These unit fixtures model a successful empty + // lsof query unless a test overrides the command response with a PID. + commandExists: (command) => command === "lsof" || (commandExists?.(command) ?? false), }; } diff --git a/src/lib/actions/uninstall/run-plan-gateway-service.test.ts b/src/lib/actions/uninstall/run-plan-gateway-service.test.ts index 84e88b18730..59b22733a01 100644 --- a/src/lib/actions/uninstall/run-plan-gateway-service.test.ts +++ b/src/lib/actions/uninstall/run-plan-gateway-service.test.ts @@ -27,6 +27,7 @@ interface Fixture { } const tempRoots: string[] = []; +const CURRENT_UID = typeof process.getuid === "function" ? process.getuid() : 0; afterEach(() => { for (const root of tempRoots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); @@ -97,19 +98,51 @@ function writeGatewayState(test: Fixture): string { return configPath; } +function managedSystemdShow( + test: Fixture, + options: { + active: boolean; + effectiveScopedStop?: boolean; + execStartPath?: string; + fragmentPath?: string; + mainPid: number; + }, +): string { + const gatewayBin = options.execStartPath ?? `${test.home}/.local/bin/openshell-gateway`; + const servicePath = + options.fragmentPath ?? getNemoclawOpenShellGatewayUserServicePath(test.home, test.env); + return [ + `FragmentPath=${servicePath}`, + `ExecStart={ path=${gatewayBin} ; argv[]=${gatewayBin} ; }`, + "ExecStop=", + "ExecStopPost=", + `ActiveState=${options.active ? "active" : "inactive"}`, + `MainPID=${String(options.active ? options.mainPid : 0)}`, + `Restart=${options.effectiveScopedStop ? "no" : "on-failure"}`, + `KillSignal=${options.effectiveScopedStop ? "SIGKILL" : "SIGTERM"}`, + "KillMode=control-group", + ].join("\n"); +} + function uninstall( test: Fixture, keepOpenShell: boolean, deps: Partial = {}, gateways: { name: string }[] = [{ name: "nemoclaw" }], ) { - const { commandExists = () => false, run = () => ok(), ...overrides } = deps; + const { + commandExists = () => false, + isPortFree = () => true, + run = () => ok(), + ...overrides + } = deps; return runUninstallPlan( { assumeYes: true, deleteModels: false, keepOpenShell }, { env: test.env, existsSync: (target) => String(target).startsWith(test.root) && fs.existsSync(target), isTty: false, + isPortFree, platform: "linux", resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ gatewayName, @@ -124,11 +157,39 @@ function uninstall( rmSync: fs.rmSync, runDocker: () => ok(), ...overrides, - commandExists: (command) => command === "openshell" || commandExists(command), - run: (command, args, options) => - command === "openshell" && args[0] === "gateway" && args[1] === "list" - ? ok(JSON.stringify(gateways)) - : run(command, args, options), + // A scoped uninstall with no PID evidence may only conclude that the + // gateway is absent after a complete empty listener observation. + commandExists: (command) => + command === "openshell" || command === "lsof" || commandExists(command), + run: (command, args, options) => { + if (command === "openshell" && args[0] === "gateway" && args[1] === "list") { + return ok(JSON.stringify(gateways)); + } + const response = run(command, args, options); + if ( + command === "systemctl" && + args.includes("show") && + response.status === 0 && + response.stdout === "" + ) { + const servicePath = getNemoclawOpenShellGatewayUserServicePath(test.home, test.env); + const gatewayBin = `${test.home}/.local/bin/openshell-gateway`; + return ok( + [ + `FragmentPath=${servicePath}`, + `ExecStart={ path=${gatewayBin} ; argv[]=${gatewayBin} ; }`, + "ExecStop=", + "ExecStopPost=", + "ActiveState=inactive", + "MainPID=0", + "Restart=on-failure", + "KillSignal=15", + "KillMode=control-group", + ].join("\n"), + ); + } + return response; + }, }, ); } @@ -215,8 +276,8 @@ describe("uninstall OpenShell gateway user service", () => { run: (command, args) => { calls.push([command, ...args]); gatewayStopped ||= command === "systemctl" && args.includes("disable"); - // `systemctl disable --now` also stops the OpenShell gateway service, - // so every scoped `openshell` call fails once the unit is disabled. + // Scoped cleanup must finish its OpenShell calls before disabling + // the unit. The disable intentionally omits --now. return command === "openshell" && gatewayStopped ? { status: 1, stdout: "", stderr: "gateway unreachable" } : ok(); @@ -241,10 +302,327 @@ describe("uninstall OpenShell gateway user service", () => { expect(result.exitCode).toBe(0); expect(deletedAt).toBeGreaterThanOrEqual(0); expect(disabledAt).toBeGreaterThan(deletedAt); + expect(calls[disabledAt]).toEqual([ + "systemctl", + "--user", + "disable", + NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE, + ]); + expect(calls[disabledAt]).not.toContain("--now"); expect(dockerCalls).toContainEqual(["rm", "-f", "sandbox-id"]); expect(fs.existsSync(servicePath)).toBe(false); }); + it("proves and SIGKILL-stops only the active managed service after scoped sandbox cleanup (#8663)", () => { + const test = fixture(true); + test.env.LOGNAME = "gateway-owner"; + const servicePath = writeManagedService(test); + writeSelectedSandboxRegistry(test, "my-assistant"); + const gatewayBin = `${test.home}/.local/bin/openshell-gateway`; + const mainPid = 41_101; + const calls: string[][] = []; + const events: string[] = []; + let scopedOverrideLoaded = false; + let serviceStopped = false; + const isPortFree = vi.fn(() => serviceStopped); + const kill = vi.fn(() => true); + const readProcessExecutable = vi.fn(() => gatewayBin); + const readProcessStartIdentity = vi.fn(() => "boot-identity:12345"); + + const result = uninstall( + test, + false, + { + commandExists: (command) => command === "systemctl", + isPortFree, + kill, + readProcessExecutable, + readProcessStartIdentity, + run: (command, args) => { + calls.push([command, ...args]); + if (command === "openshell" && args[0] === "sandbox" && args[1] === "delete") { + events.push("sandbox-delete"); + return ok(); + } + if (command === "systemctl" && args.includes("show")) { + return ok( + managedSystemdShow(test, { + active: !serviceStopped, + effectiveScopedStop: scopedOverrideLoaded, + mainPid, + }), + ); + } + if (command === "systemctl" && args.includes("daemon-reload")) { + events.push("daemon-reload"); + if (!serviceStopped) { + const dropInPath = path.join(`${servicePath}.d`, "99-nemoclaw-scoped-uninstall.conf"); + expect(fs.readFileSync(dropInPath, "utf-8")).toBe( + "[Service]\nRestart=no\nKillSignal=SIGKILL\nKillMode=control-group\n", + ); + scopedOverrideLoaded = true; + } + return ok(); + } + if (command === "systemctl" && args.includes("disable")) { + events.push("disable-now"); + expect(scopedOverrideLoaded).toBe(true); + serviceStopped = true; + return ok(); + } + if (command === "ps" && args.at(-1) === "uid=") return ok(`${String(CURRENT_UID)}\n`); + if (command === "ps" && args.at(-1) === "pid=") { + return serviceStopped ? { status: 1, stdout: "", stderr: "" } : ok(`${mainPid}\n`); + } + if (command === "lsof" && args.includes(":8080")) { + return serviceStopped ? ok() : ok(`${mainPid}\n`); + } + return ok(); + }, + }, + [{ name: "nemoclaw" }, { name: "nemoclaw-8081" }], + ); + + expect(result.exitCode).toBe(0); + expect(events.slice(0, 3)).toEqual(["sandbox-delete", "daemon-reload", "disable-now"]); + expect(calls).toContainEqual([ + "systemctl", + "--user", + "disable", + "--now", + NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE, + ]); + expect(readProcessExecutable).toHaveBeenCalledTimes(2); + expect(readProcessExecutable).toHaveBeenNthCalledWith(1, mainPid); + expect(readProcessExecutable).toHaveBeenNthCalledWith(2, mainPid); + expect(readProcessStartIdentity).toHaveBeenCalledTimes(2); + expect(readProcessStartIdentity).toHaveBeenNthCalledWith(1, mainPid); + expect(readProcessStartIdentity).toHaveBeenNthCalledWith(2, mainPid); + expect(serviceStopped).toBe(true); + expect(isPortFree).toHaveBeenCalledWith(8080); + expect( + calls.some( + (call) => + call[0] === "ps" && + call[1] === "-p" && + call[2] === String(mainPid) && + call.at(-1) === "pid=", + ), + ).toBe(true); + expect(fs.existsSync(servicePath)).toBe(false); + expect(fs.existsSync(`${servicePath}.d`)).toBe(false); + expect(kill).not.toHaveBeenCalled(); + expect(calls.some((call) => call[0] === "pgrep")).toBe(false); + }); + + it("SIGKILL-stops but preserves a trusted package-owned gateway service (#8663)", () => { + const test = fixture(true); + test.env.LOGNAME = "gateway-owner"; + writeSelectedSandboxRegistry(test, "my-assistant"); + const servicePath = "/usr/lib/systemd/user/openshell-gateway.service"; + const gatewayBin = "/usr/bin/openshell-gateway"; + const mainPid = 41_301; + const dropInPath = path.join( + getOpenShellUserConfigHome(test.home, test.env), + "systemd", + "user", + "openshell-gateway.service.d", + "99-nemoclaw-scoped-uninstall.conf", + ); + const calls: string[][] = []; + let scopedOverrideLoaded = false; + let serviceStopped = false; + + const result = uninstall( + test, + false, + { + commandExists: (command) => command === "systemctl", + existsSync: (target) => + target === servicePath || + (String(target).startsWith(test.root) && fs.existsSync(String(target))), + isPortFree: () => serviceStopped, + readProcessExecutable: () => gatewayBin, + readProcessStartIdentity: () => "boot-identity:package-service", + run: (command, args) => { + calls.push([command, ...args]); + if (command === "systemctl" && args.includes("show")) { + return ok( + managedSystemdShow(test, { + active: !serviceStopped, + effectiveScopedStop: scopedOverrideLoaded, + execStartPath: gatewayBin, + fragmentPath: servicePath, + mainPid, + }), + ); + } + if (command === "systemctl" && args.includes("daemon-reload")) { + scopedOverrideLoaded = fs.existsSync(dropInPath); + return ok(); + } + if (command === "systemctl" && args.includes("stop")) { + expect(scopedOverrideLoaded).toBe(true); + serviceStopped = true; + return ok(); + } + if (command === "ps" && args.at(-1) === "uid=") return ok(`${String(CURRENT_UID)}\n`); + if (command === "ps" && args.at(-1) === "pid=") { + return serviceStopped ? { status: 1, stdout: "", stderr: "" } : ok(`${mainPid}\n`); + } + if (command === "lsof" && args.includes(":8080")) { + return serviceStopped ? ok() : ok(`${mainPid}\n`); + } + return ok(); + }, + }, + [{ name: "nemoclaw" }, { name: "nemoclaw-8081" }], + ); + + expect(result.exitCode).toBe(0); + expect(calls).toContainEqual(["systemctl", "--user", "stop", "openshell-gateway"]); + expect(calls.some((call) => call[0] === "systemctl" && call.includes("disable"))).toBe(false); + expect(calls.some((call) => call[0] === "pgrep")).toBe(false); + expect(serviceStopped).toBe(true); + expect(fs.existsSync(dropInPath)).toBe(false); + }); + + it("removes an owned scoped-stop override when retrying inactive service cleanup (#8663)", () => { + const test = fixture(true); + test.env.LOGNAME = "gateway-owner"; + const servicePath = writeManagedService(test); + writeSelectedSandboxRegistry(test, "my-assistant"); + const gatewayBin = `${test.home}/.local/bin/openshell-gateway`; + const mainPid = 41_302; + const dropInPath = path.join(`${servicePath}.d`, "99-nemoclaw-scoped-uninstall.conf"); + let failFirstDropInRemoval = true; + let scopedOverrideLoaded = false; + let serviceStopped = false; + const run = (command: string, args: string[]) => { + if (command === "systemctl" && args.includes("show")) { + return ok( + managedSystemdShow(test, { + active: !serviceStopped, + effectiveScopedStop: scopedOverrideLoaded, + mainPid, + }), + ); + } + if (command === "systemctl" && args.includes("daemon-reload")) { + scopedOverrideLoaded = fs.existsSync(dropInPath); + return ok(); + } + if (command === "systemctl" && args.includes("disable")) { + serviceStopped = true; + return ok(); + } + if (command === "ps" && args.at(-1) === "uid=") return ok(`${String(CURRENT_UID)}\n`); + if (command === "ps" && args.at(-1) === "pid=") { + return serviceStopped ? { status: 1, stdout: "", stderr: "" } : ok(`${mainPid}\n`); + } + if (command === "lsof" && args.includes(":8080")) { + return serviceStopped ? ok() : ok(`${mainPid}\n`); + } + return ok(); + }; + const deps: Partial = { + commandExists: (command) => command === "systemctl", + isPortFree: () => serviceStopped, + readProcessExecutable: () => gatewayBin, + readProcessStartIdentity: () => "boot-identity:retry", + rmSync: (target, options) => { + if (String(target) === dropInPath && failFirstDropInRemoval) { + failFirstDropInRemoval = false; + throw new Error("injected drop-in removal failure"); + } + fs.rmSync(target, options); + }, + run, + }; + + const first = uninstall(test, false, deps, [{ name: "nemoclaw" }, { name: "nemoclaw-8081" }]); + + expect(first.exitCode).toBe(1); + expect(serviceStopped).toBe(true); + expect(fs.existsSync(servicePath)).toBe(true); + expect(fs.existsSync(dropInPath)).toBe(true); + + const retry = uninstall(test, false, deps, [{ name: "nemoclaw" }, { name: "nemoclaw-8081" }]); + + expect(retry.exitCode).toBe(0); + expect(fs.existsSync(servicePath)).toBe(false); + expect(fs.existsSync(dropInPath)).toBe(false); + }); + + it.each([ + { + label: "the port-8080 listener belongs to a sibling MainPID", + listenerPid: 41_202, + }, + { + fragmentPath: "/tmp/foreign-openshell-gateway.service", + label: "the loaded unit fragment is ambiguous", + listenerPid: 41_201, + }, + { + label: "the managed process owner differs from the current user", + listenerPid: 41_201, + processUid: CURRENT_UID + 1, + }, + ])("fails closed and preserves the active managed unit when $label (#8663)", (identity) => { + const test = fixture(true); + test.env.LOGNAME = "gateway-owner"; + const servicePath = writeManagedService(test); + writeSelectedSandboxRegistry(test, "my-assistant"); + const gatewayBin = `${test.home}/.local/bin/openshell-gateway`; + const mainPid = 41_201; + const calls: string[][] = []; + const kill = vi.fn(() => true); + + const result = uninstall( + test, + false, + { + commandExists: (command) => command === "systemctl", + isPortFree: () => false, + kill, + readProcessExecutable: () => gatewayBin, + readProcessStartIdentity: () => "boot-identity:12345", + run: (command, args) => { + calls.push([command, ...args]); + if (command === "systemctl" && args.includes("show")) { + return ok( + managedSystemdShow(test, { + active: true, + fragmentPath: identity.fragmentPath, + mainPid, + }), + ); + } + if (command === "ps" && args.at(-1) === "uid=") { + return ok(`${String(identity.processUid ?? CURRENT_UID)}\n`); + } + if (command === "lsof" && args.includes(":8080")) { + return ok(`${identity.listenerPid}\n`); + } + return ok(); + }, + }, + [{ name: "nemoclaw" }, { name: "nemoclaw-8081" }], + ); + + expect(result.exitCode).toBe(1); + expect(calls).toContainEqual(["openshell", "sandbox", "delete", "my-assistant"]); + expect(calls.some((call) => call[0] === "systemctl" && call.includes("disable"))).toBe(false); + expect(calls.some((call) => call[0] === "systemctl" && call.includes("daemon-reload"))).toBe( + false, + ); + expect(fs.existsSync(servicePath)).toBe(true); + expect(fs.existsSync(`${servicePath}.d`)).toBe(false); + expect(kill).not.toHaveBeenCalled(); + expect(calls.some((call) => call[0] === "pgrep")).toBe(false); + }); + it("preserves the marked Linux unit when scoped sandbox deletion fails (#8220)", () => { const test = fixture(true); const servicePath = writeManagedService(test); diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index b2e6f5be30d..eb10dcb6ccb 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -47,9 +47,12 @@ import { import { buildDockerGatewayDebEnvFile } from "../../onboard/docker-driver-gateway-env"; import { getNemoclawOpenShellGatewayUserServicePath, + getOpenShellGatewayUserServiceBinaryPaths, + getOpenShellGatewayUserServicePaths, getOpenShellUserConfigHome, NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE, NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER_LINE, + OPENSHELL_GATEWAY_USER_SERVICE, } from "../../onboard/docker-driver-gateway-service"; import { resolveGatewayName, resolveGatewayPortFromName } from "../../onboard/gateway-binding"; import { isExternallySupervised } from "../../onboard/gateway-ownership"; @@ -58,6 +61,7 @@ import { resolveGatewayTeardownAuthority, } from "../../onboard/gateway-teardown-authority"; import { + isHostPortFree, type StopHostGatewayOptions, stopHostGatewayProcesses, } from "../../onboard/host-gateway-process"; @@ -98,12 +102,17 @@ export interface UninstallRunDeps { error?: (message: string) => void; existsSync?: (target: string) => boolean; fs?: FileSystemDeps; + /** Test seam for confirming a scoped gateway released only its selected listener. */ + isPortFree?: (port: number) => boolean; isTty?: boolean; kill?: (pid: number, signal?: NodeJS.Signals | number) => boolean; log?: (message: string) => void; openRegularFile?: typeof openRegularFileNoFollow; platform?: NodeJS.Platform; readProcessArgv?: (pid: number) => readonly string[] | null; + readProcessExecutable?: (pid: number) => string | null; + readProcessEnvironment?: (pid: number) => Record | null; + readProcessStartIdentity?: (pid: number) => string | null; readLine?: () => string | null; requireCompleteGatewayProcessCleanup?: boolean; resolveGatewayTeardownAuthority?: GatewayTeardownAuthorityResolver; @@ -406,12 +415,16 @@ interface UninstallRuntime { env: NodeJS.ProcessEnv; error: (message: string) => void; existsSync: (target: string) => boolean; + isPortFree: ((port: number) => boolean) | undefined; isTty: boolean; kill: (pid: number, signal?: NodeJS.Signals | number) => boolean; log: (message: string) => void; openRegularFile: typeof openRegularFileNoFollow; platform: NodeJS.Platform; readProcessArgv: ((pid: number) => readonly string[] | null) | undefined; + readProcessExecutable: ((pid: number) => string | null) | undefined; + readProcessEnvironment: ((pid: number) => Record | null) | undefined; + readProcessStartIdentity: ((pid: number) => string | null) | undefined; readLine: () => string | null; requireCompleteGatewayProcessCleanup: boolean; resolveGatewayTeardownAuthority: GatewayTeardownAuthorityResolver; @@ -432,6 +445,7 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime { env, error: deps.error ?? ((message) => console.error(message)), existsSync: deps.existsSync ?? ((target) => fs.existsSync(target)), + isPortFree: deps.isPortFree, // Side-effect-free TTY check + EAGAIN-tolerant reader; the // process.stdin/non-blocking-fd hazard is documented in core/stdin.ts. isTty: deps.isTty ?? isStdinTty(), @@ -449,6 +463,9 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime { openRegularFile: deps.openRegularFile ?? openRegularFileNoFollow, platform: deps.platform ?? process.platform, readProcessArgv: deps.readProcessArgv, + readProcessExecutable: deps.readProcessExecutable, + readProcessEnvironment: deps.readProcessEnvironment, + readProcessStartIdentity: deps.readProcessStartIdentity, readLine: deps.readLine ?? readLineFromStdin, requireCompleteGatewayProcessCleanup: deps.requireCompleteGatewayProcessCleanup ?? false, resolveGatewayTeardownAuthority: @@ -768,6 +785,14 @@ function pidOwnedByCurrentUser(pid: number, runtime: UninstallRuntime): boolean return result.status === 0 && result.stdout.trim() === expected; } +function scopedServicePidOwnedByCurrentUser(pid: number, runtime: UninstallRuntime): boolean { + if (typeof process.getuid !== "function") return false; + const result = runtime.run("ps", ["-p", String(pid), "-o", "uid="], { env: runtime.env }); + if (result.status !== 0) return false; + const uid = Number.parseInt(result.stdout.trim(), 10); + return Number.isSafeInteger(uid) && uid === process.getuid(); +} + function tryStopOllamaProxyPid(pid: number, runtime: UninstallRuntime): boolean { // `runtime.kill()` only confirms the signal was sent; the proxy may ignore // SIGTERM, take time to clean up, or linger as a zombie. Verify the PID is @@ -1035,6 +1060,549 @@ function removeNemoclawOpenShellGatewayUserService(runtime: UninstallRuntime): b return true; } +const SCOPED_GATEWAY_STOP_DROP_IN = "99-nemoclaw-scoped-uninstall.conf"; +const SCOPED_GATEWAY_STOP_DROP_IN_CONTENT = `[Service] +Restart=no +KillSignal=SIGKILL +KillMode=control-group +`; + +type ScopedGatewayServiceIdentity = { + active: boolean; + execStartPath: string; + fragmentPath: string; + killMode: string; + killSignal: string; + mainPid: number; + restart: string; +}; + +type ScopedGatewayServiceTarget = { + removeUnit: boolean; + serviceName: string; + trustedBinaryPaths: readonly string[]; + trustedUnitPaths: readonly string[]; +}; + +function parseSystemctlProperties(output: string): Record { + return Object.fromEntries( + output + .split(/\r?\n/) + .map((line) => { + const separator = line.indexOf("="); + return separator > 0 ? [line.slice(0, separator), line.slice(separator + 1).trim()] : null; + }) + .filter((entry): entry is [string, string] => entry !== null), + ); +} + +function systemdExecStartPath(value: string): string | null { + const candidate = /(?:^|[\s;])path=([^\s;]+)/.exec(value)?.[1]?.trim(); + return candidate && path.isAbsolute(candidate) ? path.normalize(candidate) : null; +} + +function declaredGatewayServiceBinary(contents: string): string | null { + const values = contents + .split(/\r?\n/) + .map((line) => /^ExecStart=(\S+)$/.exec(line.trim())?.[1] ?? null) + .filter((value): value is string => value !== null); + return values.length === 1 && path.isAbsolute(values[0]) ? path.normalize(values[0]) : null; +} + +function isTrustedManagedGatewayServiceBinary( + binaryPath: string, + runtime: UninstallRuntime, +): boolean { + const home = runtime.env.HOME || os.homedir(); + const configuredBinHome = runtime.env.XDG_BIN_HOME?.trim(); + const userBinHome = + configuredBinHome && path.isAbsolute(configuredBinHome) + ? path.normalize(configuredBinHome) + : path.join(home, ".local", "bin"); + return [ + path.join(userBinHome, "openshell-gateway"), + "/usr/local/bin/openshell-gateway", + "/usr/bin/openshell-gateway", + ].some((candidate) => path.normalize(candidate) === binaryPath); +} + +function inspectScopedGatewayService( + runtime: UninstallRuntime, + target: ScopedGatewayServiceTarget, +): ScopedGatewayServiceIdentity | null { + const result = runtime.run( + "systemctl", + [ + "--user", + "show", + target.serviceName, + "--property=FragmentPath", + "--property=ExecStart", + "--property=ExecStop", + "--property=ExecStopPost", + "--property=ActiveState", + "--property=MainPID", + "--property=Restart", + "--property=KillSignal", + "--property=KillMode", + ], + { env: runtime.env }, + ); + if (result.status !== 0) return null; + const properties = parseSystemctlProperties(result.stdout); + const fragmentPath = path.normalize(properties.FragmentPath ?? ""); + const execStartPath = systemdExecStartPath(properties.ExecStart ?? ""); + const mainPid = Number(properties.MainPID); + const activeState = properties.ActiveState; + if ( + !target.trustedUnitPaths.some((candidate) => path.normalize(candidate) === fragmentPath) || + !execStartPath || + !target.trustedBinaryPaths.some((candidate) => path.normalize(candidate) === execStartPath) || + (properties.ExecStop ?? "") !== "" || + (properties.ExecStopPost ?? "") !== "" || + (activeState !== "active" && activeState !== "inactive" && activeState !== "failed") || + !Number.isSafeInteger(mainPid) || + mainPid < 0 || + (activeState === "active" ? mainPid <= 0 : mainPid !== 0) + ) { + return null; + } + return { + active: activeState === "active", + execStartPath, + fragmentPath, + killMode: properties.KillMode ?? "", + killSignal: properties.KillSignal ?? "", + mainPid, + restart: properties.Restart ?? "", + }; +} + +function runtimeProcessExecutable(pid: number, runtime: UninstallRuntime): string | null { + if (runtime.readProcessExecutable) return runtime.readProcessExecutable(pid); + try { + return fs.realpathSync.native(`/proc/${String(pid)}/exe`); + } catch { + const result = runtime.run("lsof", ["-a", "-p", String(pid), "-d", "txt", "-Fn"], { + env: runtime.env, + }); + const executable = + result.status === 0 + ? result.stdout + .split(/\r?\n/) + .find((line) => line.startsWith("n/") && line.length > 2) + ?.slice(1) + : undefined; + return executable ?? null; + } +} + +function runtimeProcessStartIdentity(pid: number, runtime: UninstallRuntime): string | null { + if (runtime.readProcessStartIdentity) return runtime.readProcessStartIdentity(pid); + try { + const stat = fs.readFileSync(`/proc/${String(pid)}/stat`, "utf-8"); + const commandEnd = stat.lastIndexOf(")"); + if (commandEnd < 0) return null; + return ( + stat + .slice(commandEnd + 1) + .trim() + .split(/\s+/)[19] ?? null + ); + } catch { + const result = runtime.run("ps", ["-p", String(pid), "-o", "lstart="], { + env: runtime.env, + }); + return result.status === 0 && result.stdout.trim() ? result.stdout.trim() : null; + } +} + +function scopedGatewayListenerPids(runtime: UninstallRuntime, port: number): number[] | null { + if (!runtime.commandExists("lsof")) return null; + const result = runtime.run("lsof", ["-ti", `:${String(port)}`, "-sTCP:LISTEN"], { + env: runtime.env, + }); + if (result.status !== 0 && result.status !== 1) return null; + return [ + ...new Set( + splitNonEmptyLines(result.stdout) + .map((line) => Number.parseInt(line, 10)) + .filter((pid) => Number.isSafeInteger(pid) && pid > 0), + ), + ]; +} + +function scopedGatewayPortFree(runtime: UninstallRuntime, port: number): boolean { + return (runtime.isPortFree ?? isHostPortFree)(port); +} + +function normalizedExecutablePath(value: string): string { + try { + return fs.realpathSync.native(value); + } catch { + return path.normalize(value); + } +} + +type ScopedGatewayStopDropIn = { dir: string; path: string }; + +function scopedGatewayStopDropIn( + runtime: UninstallRuntime, + serviceName: string, +): ScopedGatewayStopDropIn { + const home = runtime.env.HOME || os.homedir(); + const userUnitDir = path.join(getOpenShellUserConfigHome(home, runtime.env), "systemd", "user"); + const dir = path.join(userUnitDir, `${serviceName}.service.d`); + return { dir, path: path.join(dir, SCOPED_GATEWAY_STOP_DROP_IN) }; +} + +function existingScopedGatewayStopDropIn( + runtime: UninstallRuntime, + serviceName: string, +): ScopedGatewayStopDropIn | null | false { + const dropIn = scopedGatewayStopDropIn(runtime, serviceName); + if (!fs.existsSync(dropIn.path)) return null; + try { + const currentUid = typeof process.getuid === "function" ? process.getuid() : null; + const dirStat = fs.lstatSync(dropIn.dir); + const fileStat = fs.lstatSync(dropIn.path); + if ( + dirStat.isSymbolicLink() || + !dirStat.isDirectory() || + fileStat.isSymbolicLink() || + !fileStat.isFile() || + (currentUid !== null && (dirStat.uid !== currentUid || fileStat.uid !== currentUid)) + ) { + return false; + } + const existing = runtime.openRegularFile(dropIn.path); + try { + return existing.readUtf8() === SCOPED_GATEWAY_STOP_DROP_IN_CONTENT ? dropIn : false; + } finally { + existing.close(); + } + } catch { + return false; + } +} + +function removeScopedGatewayStopDropIn( + runtime: UninstallRuntime, + dropInPath: string, + dropInDir: string, +): boolean { + try { + runtime.rmSync(dropInPath, { force: true }); + if (fs.existsSync(dropInDir) && fs.readdirSync(dropInDir).length === 0) { + runtime.rmSync(dropInDir, { force: true, recursive: true }); + } + return true; + } catch { + runtime.warn(`Failed to remove temporary scoped gateway service override ${dropInPath}`); + return false; + } +} + +function rollbackScopedGatewayStopDropIn( + runtime: UninstallRuntime, + dropIn: ScopedGatewayStopDropIn, +): boolean { + const removed = removeScopedGatewayStopDropIn(runtime, dropIn.path, dropIn.dir); + const reloaded = runtime.run("systemctl", ["--user", "daemon-reload"], { + env: runtime.env, + stdio: "ignore", + }); + if (!removed || reloaded.status !== 0) { + runtime.warn("Failed to roll back the temporary scoped gateway service override."); + return false; + } + return true; +} + +function installScopedGatewayStopDropIn( + runtime: UninstallRuntime, + serviceName: string, +): ScopedGatewayStopDropIn | null { + const dropIn = scopedGatewayStopDropIn(runtime, serviceName); + try { + if (fs.existsSync(dropIn.dir)) { + const stat = fs.lstatSync(dropIn.dir); + const currentUid = typeof process.getuid === "function" ? process.getuid() : null; + if ( + stat.isSymbolicLink() || + !stat.isDirectory() || + (currentUid !== null && stat.uid !== currentUid) + ) { + return null; + } + } else { + fs.mkdirSync(dropIn.dir, { mode: 0o700, recursive: true }); + } + const existing = existingScopedGatewayStopDropIn(runtime, serviceName); + if (existing === false) return null; + if (existing) return existing; + const created = runtime.openRegularFile(dropIn.path, { + create: true, + mode: 0o600, + writable: true, + }); + try { + created.replaceUtf8(SCOPED_GATEWAY_STOP_DROP_IN_CONTENT, 0o600); + } finally { + created.close(); + } + return existingScopedGatewayStopDropIn(runtime, serviceName) || null; + } catch { + return null; + } +} + +function removeManagedDefaultGatewayUserServiceScoped( + runtime: UninstallRuntime, + options: UninstallRunOptions, + externallySupervised: boolean, +): boolean { + if ( + options.keepOpenShell || + externallySupervised || + GATEWAY_PORT !== DEFAULT_GATEWAY_PORT || + runtime.platform !== "linux" + ) { + return true; + } + const nemoclawServicePath = getNemoclawOpenShellGatewayUserServicePath( + runtime.env.HOME || os.homedir(), + runtime.env, + ); + let target: ScopedGatewayServiceTarget | null = null; + if (runtime.existsSync(nemoclawServicePath)) { + let serviceContents: string; + try { + const service = runtime.openRegularFile(nemoclawServicePath); + try { + serviceContents = service.readUtf8(); + } finally { + service.close(); + } + } catch { + runtime.warn( + `Failed to validate ${nemoclawServicePath}; leaving gateway user service in place.`, + ); + return false; + } + if ( + !serviceContents + .split(/\r?\n/) + .some((line) => line.trimEnd() === NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER_LINE) + ) { + runtime.warn(`Leaving ${nemoclawServicePath} in place because it is not NemoClaw-managed.`); + return false; + } + const declaredBinary = declaredGatewayServiceBinary(serviceContents); + if (!declaredBinary || !isTrustedManagedGatewayServiceBinary(declaredBinary, runtime)) { + runtime.warn("The managed gateway service executable is not trusted; leaving it running."); + return false; + } + target = { + removeUnit: true, + serviceName: NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE, + trustedBinaryPaths: [declaredBinary], + trustedUnitPaths: [nemoclawServicePath], + }; + } else if ( + getOpenShellGatewayUserServicePaths().some((candidate) => runtime.existsSync(candidate)) + ) { + target = { + removeUnit: false, + serviceName: OPENSHELL_GATEWAY_USER_SERVICE, + trustedBinaryPaths: getOpenShellGatewayUserServiceBinaryPaths(), + trustedUnitPaths: getOpenShellGatewayUserServicePaths(), + }; + } + if (!target) return true; + if (!runtime.commandExists("systemctl")) { + runtime.warn("systemctl not found; cannot safely stop the scoped managed gateway service."); + return false; + } + const before = inspectScopedGatewayService(runtime, target); + if (!before) { + runtime.warn( + "Could not prove the scoped managed gateway service identity; leaving it running.", + ); + return false; + } + if (!before.active) { + if (target.removeUnit) { + // A staged NemoClaw unit can be inactive while onboarding has fallen + // back to the standalone per-gateway process. Disable only that unit; + // the PID/marker proof in the caller owns any live listener teardown. + const disabled = runtime.run("systemctl", ["--user", "disable", target.serviceName], { + env: runtime.env, + stdio: "ignore", + }); + const stillInactive = inspectScopedGatewayService(runtime, target); + if ( + disabled.status !== 0 || + !stillInactive || + stillInactive.active || + stillInactive.fragmentPath !== before.fragmentPath || + stillInactive.execStartPath !== before.execStartPath + ) { + runtime.warn("The inactive managed gateway service changed while disabling it."); + return false; + } + } + const leftoverDropIn = existingScopedGatewayStopDropIn(runtime, target.serviceName); + if (leftoverDropIn === false) { + runtime.warn("The temporary scoped gateway service override is not safely owned."); + return false; + } + if ( + leftoverDropIn && + !removeScopedGatewayStopDropIn(runtime, leftoverDropIn.path, leftoverDropIn.dir) + ) { + return false; + } + if (target.removeUnit) { + try { + runtime.rmSync(before.fragmentPath, { force: true }); + } catch { + runtime.run("systemctl", ["--user", "daemon-reload"], { + env: runtime.env, + stdio: "ignore", + }); + runtime.warn(`Failed to remove ${before.fragmentPath}; leaving it in place.`); + return false; + } + } + const reload = runtime.run("systemctl", ["--user", "daemon-reload"], { + env: runtime.env, + stdio: "ignore", + }); + if (reload.status !== 0) { + runtime.warn("Failed to reload the user systemd manager."); + return false; + } + runtime.log( + target.removeUnit + ? `Disabled and removed ${target.serviceName}.service` + : `Preserved inactive package-owned ${target.serviceName}.service`, + ); + return true; + } + + const listenerPids = scopedGatewayListenerPids(runtime, DEFAULT_GATEWAY_PORT); + const processExecutable = runtimeProcessExecutable(before.mainPid, runtime); + const startIdentity = runtimeProcessStartIdentity(before.mainPid, runtime); + if ( + !scopedServicePidOwnedByCurrentUser(before.mainPid, runtime) || + !processExecutable || + processExecutable.endsWith(" (deleted)") || + normalizedExecutablePath(processExecutable) !== + normalizedExecutablePath(before.execStartPath) || + !startIdentity || + listenerPids?.length !== 1 || + listenerPids[0] !== before.mainPid + ) { + runtime.warn( + "Could not prove the managed gateway process and listener ownership; leaving it running.", + ); + return false; + } + + const dropIn = installScopedGatewayStopDropIn(runtime, target.serviceName); + if (!dropIn) { + runtime.warn("Could not install the temporary scoped gateway SIGKILL override."); + return false; + } + const reload = runtime.run("systemctl", ["--user", "daemon-reload"], { + env: runtime.env, + stdio: "ignore", + }); + if (reload.status !== 0) { + rollbackScopedGatewayStopDropIn(runtime, dropIn); + runtime.warn("Failed to reload the user systemd manager for scoped gateway cleanup."); + return false; + } + const proven = inspectScopedGatewayService(runtime, target); + const listenerPidsAfterReload = scopedGatewayListenerPids(runtime, DEFAULT_GATEWAY_PORT); + const processExecutableAfterReload = runtimeProcessExecutable(before.mainPid, runtime); + if ( + !proven?.active || + proven.mainPid !== before.mainPid || + proven.fragmentPath !== before.fragmentPath || + proven.execStartPath !== before.execStartPath || + proven.restart !== "no" || + !["9", "KILL", "SIGKILL"].includes(proven.killSignal) || + proven.killMode !== "control-group" || + !scopedServicePidOwnedByCurrentUser(before.mainPid, runtime) || + !processExecutableAfterReload || + processExecutableAfterReload.endsWith(" (deleted)") || + normalizedExecutablePath(processExecutableAfterReload) !== + normalizedExecutablePath(before.execStartPath) || + runtimeProcessStartIdentity(before.mainPid, runtime) !== startIdentity || + listenerPidsAfterReload?.length !== 1 || + listenerPidsAfterReload[0] !== before.mainPid + ) { + rollbackScopedGatewayStopDropIn(runtime, dropIn); + runtime.warn( + "Managed gateway identity changed before the scoped service stop; leaving it running.", + ); + return false; + } + + const stopArgs = target.removeUnit + ? ["--user", "disable", "--now", target.serviceName] + : ["--user", "stop", target.serviceName]; + const stopped = runtime.run("systemctl", stopArgs, { env: runtime.env, stdio: "ignore" }); + if (stopped.status !== 0) { + rollbackScopedGatewayStopDropIn(runtime, dropIn); + runtime.warn(`Failed to stop ${target.serviceName}.service`); + return false; + } + const after = inspectScopedGatewayService(runtime, target); + if ( + !after || + after.active || + after.fragmentPath !== before.fragmentPath || + after.execStartPath !== before.execStartPath || + !waitForPidExit(before.mainPid, runtime, 1000) || + !scopedGatewayPortFree(runtime, DEFAULT_GATEWAY_PORT) + ) { + rollbackScopedGatewayStopDropIn(runtime, dropIn); + runtime.warn("Scoped managed gateway service stop did not release only its selected process."); + return false; + } + + try { + if (!removeScopedGatewayStopDropIn(runtime, dropIn.path, dropIn.dir)) return false; + if (target.removeUnit) runtime.rmSync(before.fragmentPath, { force: true }); + } catch { + runtime.run("systemctl", ["--user", "daemon-reload"], { + env: runtime.env, + stdio: "ignore", + }); + runtime.warn( + `Failed to finalize ${target.serviceName}.service; leaving remaining service state in place.`, + ); + return false; + } + const finalReload = runtime.run("systemctl", ["--user", "daemon-reload"], { + env: runtime.env, + stdio: "ignore", + }); + if (finalReload.status !== 0) { + runtime.warn("Failed to reload the user systemd manager."); + return false; + } + runtime.log( + target.removeUnit + ? `Stopped and removed ${target.serviceName}.service` + : `Stopped and preserved package-owned ${target.serviceName}.service`, + ); + return true; +} + // scripts/install.sh stages the NemoClaw-managed gateway user service only for // the default gateway port, so an uninstall run for any other port leaves it in // place. `--keep-openshell` and an externally supervised authority leave it in @@ -2099,7 +2667,11 @@ function executePlan( return { ok: false }; } if (scopedToSelectedGateway && !options.keepOpenShell && !externallySupervised) { - if (!removeManagedDefaultGatewayUserService(runtime, options, externallySupervised)) { + // A marked default-port user service gets an exact, temporary + // systemd SIGKILL override. Standalone gateways use the PID/runtime + // ownership proof below. Neither path invokes OpenShell's shared + // graceful Docker cleanup. + if (!removeManagedDefaultGatewayUserServiceScoped(runtime, options, externallySupervised)) { return { ok: false }; } stopHostGatewayProcessesForUninstall(runtime, { @@ -2108,7 +2680,9 @@ function executePlan( openShellGatewayName: options.gatewayName || resolveGatewayName(GATEWAY_PORT), openShellGatewayPort: GATEWAY_PORT, preserveRuntimeFilesOnNonMatching: true, + scopedGatewayStop: true, stateDir: paths.selectedGatewayLocalStateDir, + usePgrepFallback: false, }); } else if (scopedToSelectedGateway && externallySupervised) { runtime.log("Kept the externally supervised OpenShell gateway process running."); @@ -2258,9 +2832,27 @@ function stopHostGatewayProcessesForUninstall( log: runtime.log, warn: runtime.warn, commandExists: runtime.commandExists, + isPortFree: runtime.isPortFree, + readProcessExecutable: runtime.readProcessExecutable, + readProcessEnvironment: runtime.readProcessEnvironment, + readProcessStartIdentity: runtime.readProcessStartIdentity, }, options, ); + const scopedIncomplete = + options.scopedGatewayStop === true && + (result.failed.length > 0 || + result.ownershipFailures.length > 0 || + result.skippedNonMatchingPids.length > 0); + if (scopedIncomplete) { + for (const failure of result.ownershipFailures) { + runtime.error(`Scoped gateway ownership check failed: ${failure}`); + } + runtime.error( + "Cannot continue scoped uninstall because the selected gateway process was not proven and stopped.", + ); + throw new IncompleteHostGatewayCleanupError(); + } if (!runtime.requireCompleteGatewayProcessCleanup) return; if (result.failed.length === 0 && result.orphanScanComplete !== false) return; runtime.error("Cannot continue uninstall because host gateway process cleanup did not complete."); diff --git a/src/lib/onboard/docker-driver-gateway-prelaunch.test.ts b/src/lib/onboard/docker-driver-gateway-prelaunch.test.ts index 0f075d273d2..282c87bcfab 100644 --- a/src/lib/onboard/docker-driver-gateway-prelaunch.test.ts +++ b/src/lib/onboard/docker-driver-gateway-prelaunch.test.ts @@ -14,6 +14,7 @@ import type { StopHostGatewayOptions, StopHostGatewayResult } from "./host-gatew function emptyResult(overrides: Partial = {}): StopHostGatewayResult { return { failed: [], + ownershipFailures: [], skippedDeadPids: [], skippedNonMatchingPids: [], stopped: [], diff --git a/src/lib/onboard/docker-driver-gateway-prelaunch.ts b/src/lib/onboard/docker-driver-gateway-prelaunch.ts index 1effffe9e3f..3f8abe920a4 100644 --- a/src/lib/onboard/docker-driver-gateway-prelaunch.ts +++ b/src/lib/onboard/docker-driver-gateway-prelaunch.ts @@ -54,6 +54,7 @@ export interface ReapHostGatewayBeforeLaunchOptions { function emptyStopResult(): StopHostGatewayResult { return { failed: [], + ownershipFailures: [], skippedDeadPids: [], skippedNonMatchingPids: [], stopped: [], diff --git a/src/lib/onboard/gateway-process-identity.ts b/src/lib/onboard/gateway-process-identity.ts index 4f221fe186c..1aaf1fa95cb 100644 --- a/src/lib/onboard/gateway-process-identity.ts +++ b/src/lib/onboard/gateway-process-identity.ts @@ -13,6 +13,8 @@ import { export { buildOwnedHostGatewayArgv0, + canonicalGatewayTargetMatches, + gatewayCompatContainerNameForPort, type OpenShellGatewayProcessTarget, } from "./gateway-process-target-identity"; @@ -43,6 +45,7 @@ export function gatewayProcessCmdlineMatches( opts: { expectedOpenShellGateway?: OpenShellGatewayProcessTarget; processNames?: ReadonlySet; + requireExpectedFlags?: boolean; resolveExecutablePath?: ResolveExecutablePath; } = {}, ): boolean { @@ -59,12 +62,12 @@ export function gatewayProcessCmdlineMatches( if (processNames.has(base)) { if (processNames.has("openshell-gateway") && base === "openshell-gateway") { return openShellGatewayMatchesTarget(tokens, opts.expectedOpenShellGateway, { - requireExpectedFlags: false, + requireExpectedFlags: opts.requireExpectedFlags ?? false, }); } if (base === "openclaw-gateway") { return openShellGatewayMatchesTarget(tokens, opts.expectedOpenShellGateway, { - requireExpectedFlags: true, + requireExpectedFlags: opts.requireExpectedFlags ?? true, }); } return true; @@ -76,7 +79,7 @@ export function gatewayProcessCmdlineMatches( tokens[2] === "start" ) { return openShellGatewayMatchesTarget(tokens, opts.expectedOpenShellGateway, { - requireExpectedFlags: true, + requireExpectedFlags: opts.requireExpectedFlags ?? true, }); } @@ -86,7 +89,7 @@ export function gatewayProcessCmdlineMatches( const expected = normalize(gatewayBin); if (actual && expected && actual === expected) { return openShellGatewayMatchesTarget(tokens, opts.expectedOpenShellGateway, { - requireExpectedFlags: false, + requireExpectedFlags: opts.requireExpectedFlags ?? false, }); } } @@ -108,8 +111,10 @@ export function hostGatewayCmdlineMatches( cmdline: string, gatewayBin: string | null | undefined, expectedOpenShellGateway?: OpenShellGatewayProcessTarget, + opts: { requireExpectedFlags?: boolean } = {}, ): boolean { return gatewayProcessCmdlineMatches(cmdline, gatewayBin, { expectedOpenShellGateway, + requireExpectedFlags: opts.requireExpectedFlags, }); } diff --git a/src/lib/onboard/gateway-process-target-identity.ts b/src/lib/onboard/gateway-process-target-identity.ts index e446c6be2b8..e78ec17caa1 100644 --- a/src/lib/onboard/gateway-process-target-identity.ts +++ b/src/lib/onboard/gateway-process-target-identity.ts @@ -45,15 +45,31 @@ export function gatewayTargetMatches( return true; } +export function canonicalGatewayTargetMatches(name: string, port: number): boolean { + return resolveGatewayName(port) === name; +} + +export function gatewayCompatContainerNameForPort(port: number): string { + return resolveGatewayCompatContainerName(port); +} + function cliFlagValue(tokens: string[], names: string[]): string | null { + const values: string[] = []; for (let index = 0; index < tokens.length; index += 1) { const token = tokens[index]; for (const name of names) { - if (token === name) return tokens[index + 1] ?? null; - if (token.startsWith(`${name}=`)) return token.slice(name.length + 1); + if (token === name) { + const value = tokens[index + 1]; + if (!value) return null; + values.push(value); + } else if (token.startsWith(`${name}=`)) { + const value = token.slice(name.length + 1); + if (!value) return null; + values.push(value); + } } } - return null; + return values.length === 1 ? values[0] : null; } export function openShellGatewayMatchesTarget( @@ -99,5 +115,5 @@ export function dockerCompatGatewayMatchesTarget( const port = Number(target.port); if (!Number.isInteger(port) || port < 1 || port > 65535) return false; if (target.name && target.name !== resolveGatewayName(port)) return false; - return cliFlagValue(tokens, ["--name"]) === resolveGatewayCompatContainerName(port); + return cliFlagValue(tokens, ["--name"]) === gatewayCompatContainerNameForPort(port); } diff --git a/src/lib/onboard/host-gateway-process-target.test.ts b/src/lib/onboard/host-gateway-process-target.test.ts index 80d23009608..d9a951413ba 100644 --- a/src/lib/onboard/host-gateway-process-target.test.ts +++ b/src/lib/onboard/host-gateway-process-target.test.ts @@ -10,6 +10,7 @@ import { describe, expect, it, vi } from "vitest"; import { HOST_GATEWAY_PGREP_PATTERN, type HostGatewayProcessDeps, + hostGatewayCmdlineMatches, type RunResult, stopHostGatewayProcesses, } from "./host-gateway-process"; @@ -182,3 +183,40 @@ describe("stopHostGatewayProcesses target filtering", () => { expect(fs.existsSync(pidFile)).toBe(false); }); }); + +describe("exact OpenShell gateway command-line identity (#8663)", () => { + const target = { name: "nemoclaw-18080", port: 18_080 }; + + it("accepts one exact gateway name and port", () => { + expect( + hostGatewayCmdlineMatches( + "/opt/openshell/openshell gateway start --name nemoclaw-18080 --port 18080", + null, + target, + { requireExpectedFlags: true }, + ), + ).toBe(true); + }); + + it.each([ + ["missing gateway name", "/opt/openshell/openshell gateway start --port 18080"], + ["missing gateway port", "/opt/openshell/openshell gateway start --name nemoclaw-18080"], + ["sibling gateway name", "/opt/openshell/openshell gateway start --name nemoclaw --port 18080"], + [ + "sibling gateway port", + "/opt/openshell/openshell gateway start --name nemoclaw-18080 --port 8080", + ], + [ + "duplicate gateway names", + "/opt/openshell/openshell gateway start --name nemoclaw-18080 --name nemoclaw --port 18080", + ], + [ + "duplicate gateway ports", + "/opt/openshell/openshell gateway start --name nemoclaw-18080 --port 18080 --port 8080", + ], + ])("rejects %s", (_case, cmdline) => { + expect(hostGatewayCmdlineMatches(cmdline, null, target, { requireExpectedFlags: true })).toBe( + false, + ); + }); +}); diff --git a/src/lib/onboard/host-gateway-process.test.ts b/src/lib/onboard/host-gateway-process.test.ts index 99b4a97ba49..f72267669a0 100644 --- a/src/lib/onboard/host-gateway-process.test.ts +++ b/src/lib/onboard/host-gateway-process.test.ts @@ -7,6 +7,10 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { + getDockerDriverGatewayRuntimeMarkerPath, + writeDockerDriverGatewayRuntimeMarkerForStateDir, +} from "./docker-driver-gateway-runtime-marker"; import { clearHostGatewayRuntimeFiles, HOST_GATEWAY_PGREP_PATTERN, @@ -68,6 +72,155 @@ function psResponses( ]; } +const CURRENT_UID = typeof process.getuid === "function" ? process.getuid() : 1_000; + +type ScopedGatewayFixtureOptions = { + cmdline?: string; + compatContainerPid?: number; + listenerPids?: readonly number[]; + markerPid?: number; + markerPort?: number; + omitMarker?: boolean; + pidFilePid?: number; + processUid?: number; + startIdentities?: readonly string[]; + usePgrepFallback?: boolean; +}; + +function scopedGatewayFixture(options: ScopedGatewayFixtureOptions = {}) { + const selectedPid = 9_991_880; + const siblingPid = 9_990_808; + const selectedPort = 18_080; + const selectedName = "nemoclaw-18080"; + const directGatewayBin = "/opt/openshell/openshell"; + const selectedCompatContainerName = "nemoclaw-openshell-gateway-18080"; + const selectedCompatContainerId = "a".repeat(64); + const siblingCompatContainerName = "nemoclaw-openshell-gateway"; + const selectedStateDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-host-gateway-scoped-selected-"), + ); + const siblingStateDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-host-gateway-scoped-sibling-"), + ); + const selectedPidFile = path.join(selectedStateDir, "openshell-gateway.pid"); + const siblingPidFile = path.join(siblingStateDir, "openshell-gateway.pid"); + const selectedCmdline = + options.cmdline ?? + `${directGatewayBin} gateway start --name ${selectedName} --port ${String(selectedPort)}\n`; + const compatibilityMode = selectedCmdline.includes("/opt/nemoclaw/openshell-gateway"); + fs.writeFileSync(selectedPidFile, `${String(options.pidFilePid ?? selectedPid)}\n`); + fs.writeFileSync(siblingPidFile, `${String(siblingPid)}\n`); + if (!options.omitMarker) { + writeDockerDriverGatewayRuntimeMarkerForStateDir(selectedStateDir, { + desiredEnv: {}, + dockerHost: null, + endpoint: `https://127.0.0.1:${String(options.markerPort ?? selectedPort)}`, + gatewayBin: compatibilityMode ? null : directGatewayBin, + pid: options.markerPid ?? selectedPid, + }); + } + writeDockerDriverGatewayRuntimeMarkerForStateDir(siblingStateDir, { + desiredEnv: {}, + endpoint: "https://127.0.0.1:8080", + pid: siblingPid, + }); + + const recordedPid = options.pidFilePid ?? selectedPid; + const exited = new Set(); + const responses = new Map RunResult)>([ + // A host-wide fallback would discover both gateways. Scoped teardown must + // never execute this response. + [PGREP_KEY, ok(`${String(siblingPid)}\n${String(selectedPid)}\n`)], + [ + `ps -p ${String(recordedPid)} -o pid=`, + () => (exited.has(recordedPid) ? notFound() : ok(`${String(recordedPid)}\n`)), + ], + [`ps -p ${String(recordedPid)} -o uid=`, ok(`${String(options.processUid ?? CURRENT_UID)}\n`)], + [`ps -p ${String(recordedPid)} -o args=`, ok(selectedCmdline)], + [ + `lsof -ti :${String(selectedPort)} -sTCP:LISTEN`, + ok((options.listenerPids ?? [recordedPid]).map(String).join("\n") + "\n"), + ], + ]); + if (options.compatContainerPid !== undefined) { + responses.set( + `docker inspect --type container ${selectedCompatContainerName}`, + ok( + `${JSON.stringify([ + { + Args: [], + HostConfig: { NetworkMode: "host" }, + Id: selectedCompatContainerId, + Name: `/${selectedCompatContainerName}`, + Path: "/opt/nemoclaw/openshell-gateway", + State: { Pid: options.compatContainerPid, Running: true }, + }, + ])}\n`, + ), + ); + responses.set(`docker rm -f ${selectedCompatContainerId}`, () => { + exited.add(recordedPid); + return ok(`${selectedCompatContainerName}\n`); + }); + } + const { calls, run } = makeRun(responses); + let startIdentityRead = 0; + const kill = vi.fn((pid, signal) => { + if (signal === "SIGKILL") exited.add(pid); + return true; + }); + + const result = stopHostGatewayProcesses( + { + run, + kill, + env: { USER: "tester" }, + commandExists: () => true, + isPortFree: () => true, + log: vi.fn(), + readProcessExecutable: () => directGatewayBin, + readProcessEnvironment: () => ({}), + readProcessStartIdentity: (pid) => { + if (exited.has(pid)) return null; + const identities = options.startIdentities ?? ["fixture-start-identity"]; + const identity = identities[Math.min(startIdentityRead, identities.length - 1)] ?? null; + startIdentityRead += 1; + return identity; + }, + }, + { + killWaitMs: 0, + gatewayBin: directGatewayBin, + openShellGatewayName: selectedName, + openShellGatewayPort: selectedPort, + pollIntervalMs: 0, + scopedGatewayStop: true, + stateDir: selectedStateDir, + usePgrepFallback: options.usePgrepFallback, + }, + ); + + return { + calls, + cleanup: () => { + fs.rmSync(selectedStateDir, { recursive: true, force: true }); + fs.rmSync(siblingStateDir, { recursive: true, force: true }); + }, + kill, + recordedPid, + result, + selectedPid, + selectedPidFile, + selectedCompatContainerName, + selectedCompatContainerId, + selectedRuntimeMarker: getDockerDriverGatewayRuntimeMarkerPath(selectedStateDir), + siblingCompatContainerName, + siblingPid, + siblingPidFile, + siblingRuntimeMarker: getDockerDriverGatewayRuntimeMarkerPath(siblingStateDir), + }; +} + describe("host gateway cleanup boundaries", () => { it.each([ ["free", 0, true], @@ -424,3 +577,192 @@ describe("stopHostGatewayProcesses", () => { expect(fs.existsSync(pidFile)).toBe(false); }); }); + +describe("scoped host gateway stop isolation (#8663)", () => { + it("stops only the selected gateway after proving its PID, marker, owner, command line, and listener", () => { + const fixture = scopedGatewayFixture(); + try { + expect(fixture.result).toMatchObject({ + failed: [], + ownershipFailures: [], + skippedNonMatchingPids: [], + stopped: [fixture.selectedPid], + }); + expect(fixture.kill.mock.calls).toEqual([[fixture.selectedPid, "SIGKILL"]]); + expect(fixture.kill).not.toHaveBeenCalledWith(fixture.siblingPid, expect.anything()); + expect(fixture.calls.filter(({ command }) => command === "pgrep")).toEqual([]); + expect(fs.existsSync(fixture.selectedPidFile)).toBe(false); + expect(fs.existsSync(fixture.selectedRuntimeMarker)).toBe(false); + expect(fs.readFileSync(fixture.siblingPidFile, "utf-8")).toBe( + `${String(fixture.siblingPid)}\n`, + ); + expect(fs.existsSync(fixture.siblingRuntimeMarker)).toBe(true); + } finally { + fixture.cleanup(); + } + }); + + it("removes only the selected per-port Docker compatibility container after correlating its listener", () => { + const compatContainerPid = 7_771_880; + const fixture = scopedGatewayFixture({ + cmdline: + "/usr/local/bin/docker run --rm --name nemoclaw-openshell-gateway-18080 --network host ubuntu:24.04 /opt/nemoclaw/openshell-gateway\n", + compatContainerPid, + listenerPids: [compatContainerPid], + }); + try { + expect(fixture.result).toMatchObject({ + failed: [], + ownershipFailures: [], + skippedNonMatchingPids: [], + stopped: [fixture.selectedPid], + }); + expect(fixture.kill).not.toHaveBeenCalled(); + const dockerCalls = fixture.calls.filter(({ command }) => command === "docker"); + expect(dockerCalls).toContainEqual({ + args: ["inspect", "--type", "container", fixture.selectedCompatContainerName], + command: "docker", + }); + expect(dockerCalls).toContainEqual({ + args: ["rm", "-f", fixture.selectedCompatContainerId], + command: "docker", + }); + expect( + dockerCalls.some(({ args }) => args.includes(fixture.siblingCompatContainerName)), + ).toBe(false); + expect(fs.existsSync(fixture.selectedPidFile)).toBe(false); + expect(fs.existsSync(fixture.selectedRuntimeMarker)).toBe(false); + expect(fs.existsSync(fixture.siblingPidFile)).toBe(true); + expect(fs.existsSync(fixture.siblingRuntimeMarker)).toBe(true); + } finally { + fixture.cleanup(); + } + }); + + it("preserves selected state when the Docker compatibility container does not own the selected listener", () => { + const compatContainerPid = 7_771_880; + const siblingContainerPid = 7_770_808; + const fixture = scopedGatewayFixture({ + cmdline: + "/usr/local/bin/docker run --rm --name nemoclaw-openshell-gateway-18080 --network host ubuntu:24.04 /opt/nemoclaw/openshell-gateway\n", + compatContainerPid, + listenerPids: [siblingContainerPid], + }); + try { + expect(fixture.result.stopped).toEqual([]); + expect(fixture.result.skippedNonMatchingPids).toEqual([fixture.selectedPid]); + expect(fixture.result.ownershipFailures).toEqual([ + `PID ${String(fixture.selectedPid)}: compatibility container '${fixture.selectedCompatContainerName}' does not solely own the listener on port 18080`, + ]); + expect(fixture.kill).not.toHaveBeenCalled(); + const dockerCalls = fixture.calls.filter(({ command }) => command === "docker"); + expect(dockerCalls).toContainEqual({ + args: ["inspect", "--type", "container", fixture.selectedCompatContainerName], + command: "docker", + }); + expect(dockerCalls.some(({ args }) => args[0] === "rm")).toBe(false); + expect( + dockerCalls.some(({ args }) => args.includes(fixture.siblingCompatContainerName)), + ).toBe(false); + expect(fs.existsSync(fixture.selectedPidFile)).toBe(true); + expect(fs.existsSync(fixture.selectedRuntimeMarker)).toBe(true); + expect(fs.existsSync(fixture.siblingPidFile)).toBe(true); + expect(fs.existsSync(fixture.siblingRuntimeMarker)).toBe(true); + } finally { + fixture.cleanup(); + } + }); + + it.each([ + { + label: "the selected PID file points at the sibling PID", + options: { pidFilePid: 9_990_808 }, + reason: "runtime marker PID 9991880 does not match PID file 9990808", + }, + { + label: "the process command line names the sibling gateway", + options: { + cmdline: "/opt/openshell/openshell gateway start --name nemoclaw --port 18080\n", + }, + reason: "process command line does not prove gateway 'nemoclaw-18080' on port 18080", + }, + { + label: "the process command line names the selected gateway on the sibling port", + options: { + cmdline: "/opt/openshell/openshell gateway start --name nemoclaw-18080 --port 8080\n", + }, + reason: "process command line does not prove gateway 'nemoclaw-18080' on port 18080", + }, + { + label: "the runtime marker identifies the sibling port", + options: { markerPort: 8_080 }, + reason: "runtime marker endpoint does not identify port 18080", + }, + { + label: "the runtime marker is missing", + options: { omitMarker: true }, + reason: "runtime marker is missing or not a regular file", + }, + { + label: "the process owner differs from the runtime evidence owner", + options: { processUid: CURRENT_UID + 1 }, + reason: "gateway process owner does not match the scoped runtime evidence owner", + }, + { + label: "the selected port listener belongs to the sibling PID", + options: { listenerPids: [9_990_808] }, + reason: "PID 9991880 is not the sole listener owner for port 18080", + }, + ] as const)("fails closed and preserves evidence when $label", ({ options, reason }) => { + const fixture = scopedGatewayFixture(options); + try { + expect(fixture.result.stopped).toEqual([]); + expect(fixture.result.failed).toEqual([]); + expect(fixture.result.skippedNonMatchingPids).toEqual([fixture.recordedPid]); + expect(fixture.result.ownershipFailures).toEqual([ + `PID ${String(fixture.recordedPid)}: ${reason}`, + ]); + expect(fixture.kill).not.toHaveBeenCalled(); + expect(fixture.calls.filter(({ command }) => command === "pgrep")).toEqual([]); + expect(fs.existsSync(fixture.selectedPidFile)).toBe(true); + expect(fs.existsSync(fixture.selectedRuntimeMarker)).toBe(!options.omitMarker); + expect(fs.existsSync(fixture.siblingPidFile)).toBe(true); + expect(fs.existsSync(fixture.siblingRuntimeMarker)).toBe(true); + } finally { + fixture.cleanup(); + } + }); + + it("rejects a requested host-wide fallback without scanning or signaling", () => { + const fixture = scopedGatewayFixture({ usePgrepFallback: true }); + try { + expect(fixture.result.ownershipFailures).toEqual([ + "scoped gateway stop forbids host-wide process discovery", + ]); + expect(fixture.calls.filter(({ command }) => command === "pgrep")).toEqual([]); + expect(fixture.kill).not.toHaveBeenCalled(); + expect(fs.existsSync(fixture.selectedPidFile)).toBe(true); + expect(fs.existsSync(fixture.selectedRuntimeMarker)).toBe(true); + } finally { + fixture.cleanup(); + } + }); + + it("fails closed when the selected process identity changes immediately before signaling", () => { + const fixture = scopedGatewayFixture({ startIdentities: ["original", "replacement"] }); + try { + expect(fixture.result.stopped).toEqual([]); + expect(fixture.result.skippedNonMatchingPids).toEqual([fixture.selectedPid]); + expect(fixture.result.ownershipFailures).toEqual([ + `PID ${String(fixture.selectedPid)}: gateway process identity changed immediately before signaling`, + ]); + expect(fixture.kill).not.toHaveBeenCalled(); + expect(fs.existsSync(fixture.selectedPidFile)).toBe(true); + expect(fs.existsSync(fixture.selectedRuntimeMarker)).toBe(true); + expect(fs.existsSync(fixture.siblingPidFile)).toBe(true); + expect(fs.existsSync(fixture.siblingRuntimeMarker)).toBe(true); + } finally { + fixture.cleanup(); + } + }); +}); diff --git a/src/lib/onboard/host-gateway-process.ts b/src/lib/onboard/host-gateway-process.ts index 11c3e217702..40d25876a2b 100644 --- a/src/lib/onboard/host-gateway-process.ts +++ b/src/lib/onboard/host-gateway-process.ts @@ -7,8 +7,17 @@ import os from "node:os"; import path from "node:path"; import { waitUntil } from "../core/wait"; -import { clearDockerDriverGatewayRuntimeMarker } from "./docker-driver-gateway-runtime-marker"; import { + clearDockerDriverGatewayRuntimeMarker, + getDockerDriverGatewayRuntimeMarkerPath, + readDockerDriverGatewayRuntimeMarker, +} from "./docker-driver-gateway-runtime-marker"; +import { + canonicalGatewayTargetMatches, + cleanGatewayProcessToken, + DOCKER_DRIVER_GATEWAY_COMPAT_MOUNT_PATH, + DOCKER_DRIVER_GATEWAY_CONTAINER_RUNTIME_NAMES, + gatewayCompatContainerNameForPort, type OpenShellGatewayProcessTarget, hostGatewayCmdlineMatches as sharedHostGatewayCmdlineMatches, } from "./gateway-process-identity"; @@ -24,7 +33,11 @@ export interface HostGatewayProcessDeps { kill: (pid: number, signal?: NodeJS.Signals | number) => boolean; env: NodeJS.ProcessEnv; commandExists?: (command: string) => boolean; + isPortFree?: (port: number) => boolean; log?: (message: string) => void; + readProcessExecutable?: (pid: number) => string | null; + readProcessEnvironment?: (pid: number) => Record | null; + readProcessStartIdentity?: (pid: number) => string | null; warn?: (message: string) => void; } @@ -41,6 +54,11 @@ export interface StopHostGatewayOptions { pollIntervalMs?: number; /** Keep PID/runtime evidence when a PID-file process does not match the cleanup target. */ preserveRuntimeFilesOnNonMatching?: boolean; + /** + * Stop one gateway without invoking OpenShell's shared Docker shutdown cleanup. + * Requires exact per-gateway PID, runtime-marker, owner, cmdline, and listener proof. + */ + scopedGatewayStop?: boolean; stateDir?: string; termWaitMs?: number; /** Whether to read and act on the resolved pid file. */ @@ -52,6 +70,7 @@ export interface StopHostGatewayResult { failed: number[]; /** Whether a requested pgrep fallback completed with a usable result. */ orphanScanComplete?: boolean; + ownershipFailures: string[]; skippedDeadPids: number[]; skippedNonMatchingPids: number[]; stopped: number[]; @@ -125,7 +144,11 @@ function defaultDeps(overrides: Partial = {}): HostGatew kill: overrides.kill ?? defaultKill, env, commandExists: overrides.commandExists ?? ((cmd) => defaultCommandExists(cmd, env)), + isPortFree: overrides.isPortFree ?? ((port) => isHostPortFree(port)), log: overrides.log, + readProcessExecutable: overrides.readProcessExecutable, + readProcessEnvironment: overrides.readProcessEnvironment, + readProcessStartIdentity: overrides.readProcessStartIdentity, warn: overrides.warn, }; } @@ -161,6 +184,47 @@ function processArgs(pid: number, deps: HostGatewayProcessDeps): string { return result.status === 0 ? result.stdout.trim() : ""; } +function processExecutable(pid: number, deps: HostGatewayProcessDeps): string | null { + if (deps.readProcessExecutable) return deps.readProcessExecutable(pid); + try { + return fs.readlinkSync(`/proc/${String(pid)}/exe`); + } catch { + const lsof = deps.run("lsof", ["-a", "-p", String(pid), "-d", "txt", "-Fn"], { + env: deps.env, + }); + const lsofPath = + lsof.status === 0 + ? lsof.stdout + .split(/\r?\n/) + .find((line) => line.startsWith("n/") && line.length > 2) + ?.slice(1) + : undefined; + if (lsofPath) return lsofPath; + const result = deps.run("ps", ["-p", String(pid), "-o", "comm="], { env: deps.env }); + const executable = result.status === 0 ? result.stdout.trim() : ""; + return executable && path.isAbsolute(executable) ? executable : null; + } +} + +function processStartIdentity(pid: number, deps: HostGatewayProcessDeps): string | null { + if (deps.readProcessStartIdentity) return deps.readProcessStartIdentity(pid); + try { + const stat = fs.readFileSync(`/proc/${String(pid)}/stat`, "utf-8"); + const commandEnd = stat.lastIndexOf(")"); + if (commandEnd < 0) return null; + return ( + stat + .slice(commandEnd + 1) + .trim() + .split(/\s+/)[19] ?? null + ); + } catch { + const result = deps.run("ps", ["-p", String(pid), "-o", "lstart="], { env: deps.env }); + const started = result.status === 0 ? result.stdout.trim() : ""; + return started ? started : null; + } +} + function pidExists(pid: number, deps: HostGatewayProcessDeps): boolean { return deps.run("ps", ["-p", String(pid), "-o", "pid="], { env: deps.env }).status === 0; } @@ -171,12 +235,375 @@ function pidOwner(pid: number, deps: HostGatewayProcessDeps): string | null { return result.stdout.trim() || null; } +function pidUid(pid: number, deps: HostGatewayProcessDeps): number | null { + const result = deps.run("ps", ["-p", String(pid), "-o", "uid="], { env: deps.env }); + if (result.status !== 0) return null; + const uid = Number.parseInt(result.stdout.trim(), 10); + return Number.isInteger(uid) && uid >= 0 ? uid : null; +} + +function regularFileUid(filePath: string): number | null { + try { + const stat = fs.lstatSync(filePath); + return stat.isFile() && !stat.isSymbolicLink() ? stat.uid : null; + } catch { + return null; + } +} + +function ownedStateDirUid(stateDir: string): number | null { + try { + const stat = fs.lstatSync(stateDir); + return stat.isDirectory() && !stat.isSymbolicLink() ? stat.uid : null; + } catch { + return null; + } +} + +function gatewayEndpointPort(endpoint: string): number | null { + try { + const parsed = new URL(endpoint); + const port = Number.parseInt(parsed.port, 10); + return Number.isInteger(port) && port > 0 && port <= 65_535 ? port : null; + } catch { + return null; + } +} + +function listeningPids( + port: number, + deps: HostGatewayProcessDeps, +): { complete: boolean; pids: number[] } { + if (deps.commandExists && !deps.commandExists("lsof")) { + return { complete: false, pids: [] }; + } + const result = deps.run("lsof", ["-ti", `:${String(port)}`, "-sTCP:LISTEN"], { + env: deps.env, + }); + if (result.status !== 0 && result.status !== 1) { + return { complete: false, pids: [] }; + } + return { complete: true, pids: [...new Set(parsePidLines(result.stdout))] }; +} + +function dockerCompatContainerForTarget(cmdline: string, port: number): string | null { + const tokens = cmdline.trim().split(/\s+/).filter(Boolean).map(cleanGatewayProcessToken); + const argv0 = tokens[0] ?? ""; + if ( + !DOCKER_DRIVER_GATEWAY_CONTAINER_RUNTIME_NAMES.has(path.basename(argv0)) || + tokens[1] !== "run" || + !tokens.slice(1).includes(DOCKER_DRIVER_GATEWAY_COMPAT_MOUNT_PATH) + ) { + return null; + } + const containerName = gatewayCompatContainerNameForPort(port); + const nameIndex = tokens.findIndex((token) => token === "--name"); + const inlineName = tokens.find((token) => token.startsWith("--name="))?.slice("--name=".length); + const explicitName = nameIndex >= 0 ? tokens[nameIndex + 1] : inlineName; + return explicitName === containerName ? containerName : null; +} + +type DockerCompatContainerIdentity = { + containerId: string; + containerName: string; + dockerEnv: NodeJS.ProcessEnv; + pid: number; +}; + +function dockerCompatContainerIdentity( + containerName: string, + dockerHost: string, + deps: HostGatewayProcessDeps, +): DockerCompatContainerIdentity | null { + const dockerEnv = { ...deps.env }; + for (const key of ["DOCKER_CERT_PATH", "DOCKER_CONFIG", "DOCKER_CONTEXT", "DOCKER_TLS_VERIFY"]) { + delete dockerEnv[key]; + } + dockerEnv.DOCKER_HOST = dockerHost; + const result = deps.run("docker", ["inspect", "--type", "container", containerName], { + env: dockerEnv, + }); + if (result.status !== 0) return null; + try { + const parsed = JSON.parse(result.stdout) as Array<{ + Args?: unknown; + HostConfig?: { NetworkMode?: unknown }; + Id?: unknown; + Name?: unknown; + Path?: unknown; + State?: { Pid?: unknown; Running?: unknown }; + }>; + if (!Array.isArray(parsed) || parsed.length !== 1) return null; + const container = parsed[0]; + const containerId = typeof container.Id === "string" ? container.Id : ""; + const containerPid = container.State?.Pid; + if ( + !/^[a-f0-9]{64}$/i.test(containerId) || + container.Name !== `/${containerName}` || + container.Path !== DOCKER_DRIVER_GATEWAY_COMPAT_MOUNT_PATH || + !Array.isArray(container.Args) || + container.Args.length !== 0 || + container.HostConfig?.NetworkMode !== "host" || + container.State?.Running !== true || + !Number.isSafeInteger(containerPid) || + Number(containerPid) <= 0 + ) { + return null; + } + return { + containerId, + containerName, + dockerEnv, + pid: Number(containerPid), + }; + } catch { + return null; + } +} + +function processEnvironment( + pid: number, + deps: HostGatewayProcessDeps, +): Record | null { + if (deps.readProcessEnvironment) return deps.readProcessEnvironment(pid); + try { + const entries = fs.readFileSync(`/proc/${String(pid)}/environ`, "utf-8").split("\0"); + const environment: Record = {}; + for (const entry of entries) { + const separator = entry.indexOf("="); + if (separator > 0) environment[entry.slice(0, separator)] = entry.slice(separator + 1); + } + return environment; + } catch { + return null; + } +} + export function hostGatewayCmdlineMatches( cmdline: string, gatewayBin: string | null | undefined, expectedOpenShellGateway?: OpenShellGatewayProcessTarget, + opts: { requireExpectedFlags?: boolean } = {}, ): boolean { - return sharedHostGatewayCmdlineMatches(cmdline, gatewayBin, expectedOpenShellGateway); + return sharedHostGatewayCmdlineMatches(cmdline, gatewayBin, expectedOpenShellGateway, opts); +} + +function normalizeExecutablePath(value: string): string { + try { + return fs.realpathSync.native(value); + } catch { + return path.resolve(value); + } +} + +function scopedGatewayOwnershipProof( + pid: number, + deps: HostGatewayProcessDeps, + options: StopHostGatewayOptions, + target: { name: string; port: number }, + stateDir: string, + pidFile: string, +): { + cmdline: string; + compatContainerIdentity?: DockerCompatContainerIdentity; + compatContainerName: string | null; + reason?: string; + startIdentity?: string; +} { + const markerPath = getDockerDriverGatewayRuntimeMarkerPath(stateDir); + const stateDirUid = ownedStateDirUid(stateDir); + const pidFileUid = regularFileUid(pidFile); + const markerUid = regularFileUid(markerPath); + if (stateDirUid === null) { + return { + cmdline: "", + compatContainerName: null, + reason: "gateway state directory is missing, symlinked, or not a directory", + }; + } + if (pidFileUid === null) { + return { cmdline: "", compatContainerName: null, reason: "PID file is not a regular file" }; + } + if (markerUid === null) { + return { + cmdline: "", + compatContainerName: null, + reason: "runtime marker is missing or not a regular file", + }; + } + if (stateDirUid !== pidFileUid || pidFileUid !== markerUid) { + return { + cmdline: "", + compatContainerName: null, + reason: "gateway state directory, PID file, and runtime marker have different owners", + }; + } + const currentUid = typeof process.getuid === "function" ? process.getuid() : null; + if (currentUid !== null && stateDirUid !== currentUid) { + return { + cmdline: "", + compatContainerName: null, + reason: "scoped gateway runtime evidence is not owned by the current user", + }; + } + if (readPidFile(pidFile) !== pid) { + return { + cmdline: "", + compatContainerName: null, + reason: "PID file identity changed while proving the scoped gateway target", + }; + } + + const marker = readDockerDriverGatewayRuntimeMarker(markerPath); + if (!marker) { + return { cmdline: "", compatContainerName: null, reason: "runtime marker is invalid" }; + } + if (marker.pid !== pid) { + return { + cmdline: "", + compatContainerName: null, + reason: `runtime marker PID ${String(marker.pid)} does not match PID file ${String(pid)}`, + }; + } + if (gatewayEndpointPort(marker.endpoint) !== target.port) { + return { + cmdline: "", + compatContainerName: null, + reason: `runtime marker endpoint does not identify port ${String(target.port)}`, + }; + } + if (marker.platform !== process.platform || marker.arch !== process.arch) { + return { + cmdline: "", + compatContainerName: null, + reason: "runtime marker platform identity does not match this host", + }; + } + if ( + marker.gatewayBin && + options.gatewayBin && + normalizeExecutablePath(marker.gatewayBin) !== normalizeExecutablePath(options.gatewayBin) + ) { + return { + cmdline: "", + compatContainerName: null, + reason: "runtime marker gateway executable does not match the cleanup target", + }; + } + + const ownerUid = pidUid(pid, deps); + if (ownerUid === null || ownerUid !== pidFileUid) { + return { + cmdline: "", + compatContainerName: null, + reason: "gateway process owner does not match the scoped runtime evidence owner", + }; + } + + const cmdline = processArgs(pid, deps); + if ( + !hostGatewayCmdlineMatches(cmdline, options.gatewayBin, target, { + requireExpectedFlags: true, + }) + ) { + return { + cmdline, + compatContainerName: null, + reason: `process command line does not prove gateway '${target.name}' on port ${String(target.port)}`, + }; + } + + const compatContainerName = dockerCompatContainerForTarget(cmdline, target.port); + if (!compatContainerName) { + if (!marker.gatewayBin) { + return { + cmdline, + compatContainerName, + reason: "runtime marker does not identify the direct gateway executable", + }; + } + const executable = processExecutable(pid, deps); + if ( + !executable || + normalizeExecutablePath(executable) !== normalizeExecutablePath(marker.gatewayBin) + ) { + return { + cmdline, + compatContainerName, + reason: "gateway process executable does not match the runtime marker", + }; + } + } + const startIdentity = processStartIdentity(pid, deps); + if (!startIdentity) { + return { + cmdline, + compatContainerName, + reason: "gateway process start identity could not be proven", + }; + } + const listeners = listeningPids(target.port, deps); + if (!listeners.complete) { + return { + cmdline, + compatContainerName, + reason: `listener ownership for port ${String(target.port)} could not be observed completely`, + }; + } + if (compatContainerName) { + const parentEnvironment = processEnvironment(pid, deps); + const parentDockerHost = parentEnvironment?.DOCKER_HOST?.trim() || null; + const provenDockerHost = marker.dockerHost ?? "unix:///var/run/docker.sock"; + const unsupportedDockerSelector = [ + "DOCKER_CERT_PATH", + "DOCKER_CONFIG", + "DOCKER_CONTEXT", + "DOCKER_TLS_VERIFY", + ].some((key) => Boolean(parentEnvironment?.[key]?.trim())); + if ( + marker.gatewayBin !== null || + !provenDockerHost.startsWith("unix:///") || + !parentEnvironment || + unsupportedDockerSelector || + (marker.dockerHost === null + ? parentDockerHost !== null + : parentDockerHost !== marker.dockerHost) + ) { + return { + cmdline, + compatContainerName, + reason: "compatibility gateway Docker daemon identity does not match the runtime marker", + }; + } + const compatContainerIdentity = dockerCompatContainerIdentity( + compatContainerName, + provenDockerHost, + deps, + ); + if (!compatContainerIdentity) { + return { + cmdline, + compatContainerName, + reason: `compatibility container '${compatContainerName}' identity could not be proven`, + }; + } + if (listeners.pids.length !== 1 || listeners.pids[0] !== compatContainerIdentity.pid) { + return { + cmdline, + compatContainerName, + reason: `compatibility container '${compatContainerName}' does not solely own the listener on port ${String(target.port)}`, + }; + } + return { cmdline, compatContainerIdentity, compatContainerName, startIdentity }; + } else if (listeners.pids.length !== 1 || listeners.pids[0] !== pid) { + return { + cmdline, + compatContainerName, + reason: `PID ${String(pid)} is not the sole listener owner for port ${String(target.port)}`, + }; + } + + return { cmdline, compatContainerName, startIdentity }; } function waitForExit( @@ -274,6 +701,45 @@ function tryStopPid( return "failed"; } +function tryStopScopedPid( + pid: number, + compatContainerIdentity: DockerCompatContainerIdentity | undefined, + expectedStartIdentity: string, + deps: HostGatewayProcessDeps, + options: Required>, +): "stopped" | "failed" | "identity-changed" { + const log = deps.log ?? ((message: string) => console.log(message)); + if (processStartIdentity(pid, deps) !== expectedStartIdentity) return "identity-changed"; + if (compatContainerIdentity) { + const removed = deps.run("docker", ["rm", "-f", compatContainerIdentity.containerId], { + env: compatContainerIdentity.dockerEnv, + }); + if (removed.status !== 0) { + const warn = deps.warn ?? ((message: string) => console.warn(message)); + const detail = removed.stderr.trim() || `status ${String(removed.status)}`; + warn( + `Failed to remove scoped gateway compatibility container '${compatContainerIdentity.containerName}': ${detail}`, + ); + return "failed"; + } + if (!waitForExit(pid, deps, options.killWaitMs, options.pollIntervalMs)) { + return "failed"; + } + } else { + // OpenShell 0.0.99 gracefully stops every managed Docker container in its + // configured namespace. Scoped teardown has already deleted this gateway's + // selected sandboxes, so SIGKILL avoids cross-stopping a sibling gateway's + // container while still targeting only the fully proven process. + deps.kill(pid, "SIGKILL"); + } + if (waitForExit(pid, deps, options.killWaitMs, options.pollIntervalMs)) { + log(`Stopped scoped host openshell-gateway process ${pid}`); + return "stopped"; + } + warnSudoRemediation(pid, deps); + return "failed"; +} + export function stopHostGatewayProcesses( depsOverrides: Partial = {}, options: StopHostGatewayOptions = {}, @@ -286,24 +752,69 @@ export function stopHostGatewayProcesses( const result: StopHostGatewayResult = { failed: [], orphanScanComplete: true, + ownershipFailures: [], skippedDeadPids: [], skippedNonMatchingPids: [], stopped: [], sudoRemediationPids: [], }; + const scopedGatewayStop = options.scopedGatewayStop ?? false; + const explicitPids = Array.from(options.pids ?? []).filter( + (pid): pid is number => Number.isInteger(pid) && pid > 0, + ); + let scopedTarget: { name: string; port: number } | null = null; + if (scopedGatewayStop) { + const port = Number(options.openShellGatewayPort); + const name = options.openShellGatewayName?.trim() ?? ""; + if ( + !Number.isInteger(port) || + port < 1 || + port > 65_535 || + !canonicalGatewayTargetMatches(name, port) + ) { + result.ownershipFailures.push( + "scoped gateway stop requires one canonical gateway name and port", + ); + return result; + } + if (options.usePidFile === false || explicitPids.length > 0) { + result.ownershipFailures.push( + "scoped gateway stop accepts only the selected gateway PID file", + ); + return result; + } + if (options.usePgrepFallback === true) { + result.ownershipFailures.push("scoped gateway stop forbids host-wide process discovery"); + return result; + } + scopedTarget = { name, port }; + } + if (options.usePidFile ?? true) { const pidFromFile = readPidFile(pidFile); if (pidFromFile !== null) { addPid(candidates, pidFromFile, "pid-file"); + } else if (scopedTarget) { + const markerPath = getDockerDriverGatewayRuntimeMarkerPath(stateDir); + if (fs.existsSync(pidFile) || fs.existsSync(markerPath)) { + result.ownershipFailures.push( + "scoped gateway PID/runtime evidence is incomplete or invalid", + ); + return result; + } + const listeners = listeningPids(scopedTarget.port, deps); + if (deps.isPortFree?.(scopedTarget.port) !== true || listeners.pids.length > 0) { + result.ownershipFailures.push( + `gateway port ${String(scopedTarget.port)} is occupied without PID-file ownership evidence`, + ); + return result; + } } else if (clearRuntimeState && fs.existsSync(pidFile)) { clearHostGatewayRuntimeFiles(stateDir, pidFile); } } - const explicitPids = Array.from(options.pids ?? []).filter( - (pid): pid is number => Number.isInteger(pid) && pid > 0, - ); for (const pid of explicitPids) addPid(candidates, pid, "explicit"); // When a caller passes explicit PIDs (e.g. drift-restart targeting one @@ -311,7 +822,9 @@ export function stopHostGatewayProcesses( // host. Otherwise an onboard drift could terminate an unrelated worktree's // gateway. Sweeping callers (uninstall, sandbox destroy of the last sandbox) // omit `pids` and so still get the pgrep fallback by default. - const useFallback = options.usePgrepFallback ?? explicitPids.length === 0; + const useFallback = scopedGatewayStop + ? false + : (options.usePgrepFallback ?? explicitPids.length === 0); let pgrepRan = false; if (useFallback) { const sweep = pgrepHostGatewayPids(deps); @@ -336,12 +849,90 @@ export function stopHostGatewayProcesses( for (const [pid, sources] of candidates) { if (!pidExists(pid, deps)) { result.skippedDeadPids.push(pid); + if (scopedTarget) { + const listeners = listeningPids(scopedTarget.port, deps); + if (deps.isPortFree?.(scopedTarget.port) !== true || listeners.pids.length > 0) { + result.ownershipFailures.push( + `recorded PID ${String(pid)} is dead but port ${String(scopedTarget.port)} remains occupied`, + ); + continue; + } + } if (clearRuntimeState && sources.has("pid-file") && !clearedRuntimeFiles) { clearHostGatewayRuntimeFiles(stateDir, pidFile); clearedRuntimeFiles = true; } continue; } + + if (scopedTarget) { + const proof = scopedGatewayOwnershipProof( + pid, + deps, + options, + scopedTarget, + stateDir, + pidFile, + ); + if (proof.reason) { + result.skippedNonMatchingPids.push(pid); + result.ownershipFailures.push(`PID ${String(pid)}: ${proof.reason}`); + continue; + } + const finalProof = scopedGatewayOwnershipProof( + pid, + deps, + options, + scopedTarget, + stateDir, + pidFile, + ); + if ( + finalProof.reason || + finalProof.cmdline !== proof.cmdline || + finalProof.startIdentity !== proof.startIdentity || + finalProof.compatContainerIdentity?.containerId !== + proof.compatContainerIdentity?.containerId + ) { + result.skippedNonMatchingPids.push(pid); + result.ownershipFailures.push( + `PID ${String(pid)}: gateway process identity changed immediately before signaling`, + ); + continue; + } + const scopedStop = tryStopScopedPid( + pid, + finalProof.compatContainerIdentity, + finalProof.startIdentity as string, + deps, + waitOptions, + ); + if (scopedStop === "identity-changed") { + result.skippedNonMatchingPids.push(pid); + result.ownershipFailures.push( + `PID ${String(pid)}: gateway process identity changed immediately before signaling`, + ); + continue; + } + if (scopedStop !== "stopped") { + result.failed.push(pid); + result.sudoRemediationPids.push(pid); + continue; + } + result.stopped.push(pid); + if (!deps.isPortFree?.(scopedTarget.port)) { + result.ownershipFailures.push( + `gateway port ${String(scopedTarget.port)} remains occupied after stopping PID ${String(pid)}`, + ); + continue; + } + if (clearRuntimeState && !clearedRuntimeFiles) { + clearHostGatewayRuntimeFiles(stateDir, pidFile); + clearedRuntimeFiles = true; + } + continue; + } + if ( !hostGatewayCmdlineMatches( processArgs(pid, deps), @@ -374,7 +965,7 @@ export function stopHostGatewayProcesses( } } - if (options.logNoProcesses && candidates.size === 0) { + if (options.logNoProcesses && candidates.size === 0 && result.ownershipFailures.length === 0) { if (useFallback && !pgrepRan) { // The pid-file branch found nothing and the pgrep fallback could not // run (typically `pgrep` is absent on a minimal image). Surface the diff --git a/src/lib/tunnel/gateway-port-release-test-helpers.ts b/src/lib/tunnel/gateway-port-release-test-helpers.ts index d68c0f62e1e..f56c95367e9 100644 --- a/src/lib/tunnel/gateway-port-release-test-helpers.ts +++ b/src/lib/tunnel/gateway-port-release-test-helpers.ts @@ -16,6 +16,7 @@ export function emptyStopResult( ): StopHostGatewayResult { return { failed: [], + ownershipFailures: [], skippedDeadPids: [], skippedNonMatchingPids: [], stopped: [], diff --git a/test/e2e/live/concurrent-gateway-ports.test.ts b/test/e2e/live/concurrent-gateway-ports.test.ts index 57ae4c8ad2d..15f5db172c7 100644 --- a/test/e2e/live/concurrent-gateway-ports.test.ts +++ b/test/e2e/live/concurrent-gateway-ports.test.ts @@ -10,7 +10,10 @@ */ import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { getTrustedActiveOpenShellGatewayUserServicePid } from "../../../src/lib/onboard/docker-driver-gateway-service.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; @@ -31,6 +34,20 @@ const PHASE_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_PHASE_TIMEOUT_MS ?? 1_2 const PROBE_ATTEMPTS = Number(process.env.NEMOCLAW_E2E_PROBE_ATTEMPTS ?? 12); const PROBE_DELAY_MS = Number(process.env.NEMOCLAW_E2E_PROBE_DELAY_SECONDS ?? 5) * 1_000; const TEST_TIMEOUT_MS = 90 * 60_000; +const POST_UNINSTALL_HEALTH_PROBES = 3; + +type GatewayProcessAuthority = "standalone-state" | "systemd-service"; + +interface GatewayProcessIdentity { + authority: GatewayProcessAuthority; + pid: number; +} + +interface CapturedProcessIdentity { + executable: string; + pid: number; + startIdentity: string; +} process.env.NEMOCLAW_CLI_BIN ??= CLI_ENTRYPOINT; validateSandboxName(SANDBOX_A); @@ -53,6 +70,103 @@ function gatewayNameForPort(port: string): string { return port === "8080" ? "nemoclaw" : `nemoclaw-${port}`; } +function gatewayStateDirForPort(port: string): string { + const numericPort = Number(port); + if (!Number.isInteger(numericPort) || numericPort < 1 || numericPort > 65_535) { + throw new Error(`invalid gateway port '${port}'`); + } + const leaf = port === "8080" ? "openshell-docker-gateway" : `openshell-docker-gateway-${port}`; + return path.join(process.env.HOME || os.homedir(), ".local", "state", "nemoclaw", leaf); +} + +function readGatewayPid(port: string): number | null { + try { + const raw = fs.readFileSync( + path.join(gatewayStateDirForPort(port), "openshell-gateway.pid"), + "utf-8", + ); + const pid = Number.parseInt(raw.trim(), 10); + return Number.isInteger(pid) && pid > 0 ? pid : null; + } catch { + return null; + } +} + +function readGatewayRuntimePid(port: string): number | null { + try { + const marker: unknown = JSON.parse( + fs.readFileSync(path.join(gatewayStateDirForPort(port), "runtime.json"), "utf-8"), + ); + if (!marker || typeof marker !== "object" || !("pid" in marker)) return null; + const pid = (marker as { pid?: unknown }).pid; + return typeof pid === "number" && Number.isInteger(pid) && pid > 0 ? pid : null; + } catch { + return null; + } +} + +function readStandaloneGatewayIdentity(port: string): GatewayProcessIdentity | null { + const pid = readGatewayPid(port); + const runtimePid = readGatewayRuntimePid(port); + if (pid === null && runtimePid === null) return null; + if (pid === null || runtimePid === null || pid !== runtimePid) { + throw new Error( + `gateway ${gatewayNameForPort(port)} has inconsistent standalone process state ` + + `(pid file: ${String(pid)}, runtime marker: ${String(runtimePid)})`, + ); + } + return { authority: "standalone-state", pid }; +} + +function readDefaultGatewayIdentity(): GatewayProcessIdentity { + const standalone = readStandaloneGatewayIdentity(GATEWAY_PORT_A); + if (standalone) return standalone; + + const pid = + process.platform === "linux" && GATEWAY_PORT_A === "8080" + ? getTrustedActiveOpenShellGatewayUserServicePid({ env: commandEnv() }) + : null; + if (pid === null) { + throw new Error( + `default gateway ${gatewayNameForPort(GATEWAY_PORT_A)} has neither matching ` + + "standalone PID/runtime state nor a trusted active systemd MainPID", + ); + } + return { authority: "systemd-service", pid }; +} + +function readAlternateGatewayIdentity(): GatewayProcessIdentity { + const identity = readStandaloneGatewayIdentity(GATEWAY_PORT_B); + if (!identity) { + throw new Error( + `alternate gateway ${gatewayNameForPort(GATEWAY_PORT_B)} is missing its standalone ` + + "PID/runtime ownership proof", + ); + } + return identity; +} + +function evidenceField(output: string, field: string): string | null { + const values = output + .split(/\r?\n/) + .filter((line) => line.startsWith(`${field}=`)) + .map((line) => line.slice(field.length + 1).trim()) + .filter(Boolean); + return values.length === 1 ? values[0] : null; +} + +function capturedProcessIdentity(result: ShellProbeResult): CapturedProcessIdentity | null { + const pidText = evidenceField(result.stdout, "active_pid"); + if (pidText === null) return null; + const pid = Number(pidText); + const executable = evidenceField(result.stdout, "process_executable"); + const startIdentity = evidenceField(result.stdout, "process_start_identity"); + if (!Number.isSafeInteger(pid) || pid <= 0 || !executable || !startIdentity) { + throw new Error(`incomplete gateway process identity evidence:\n${resultText(result)}`); + } + return { executable, pid, startIdentity }; +} + function openshellEnvForGateway(gatewayName: string): NodeJS.ProcessEnv { return commandEnv({ OPENSHELL_GATEWAY: gatewayName }); } @@ -91,6 +205,136 @@ async function command( }); } +async function captureGatewayEvidence( + host: HostCliClient, + sandbox: SandboxClient, + options: { + authority: GatewayProcessAuthority; + gatewayName: string; + knownPid?: number | null; + port: string; + stage: string; + }, +): Promise<{ + host: ShellProbeResult; + processIdentity: CapturedProcessIdentity | null; + sandbox: ShellProbeResult; +}> { + const stateDir = gatewayStateDirForPort(options.port); + const hostEvidence = await host.command( + "bash", + [ + "-lc", + [ + 'gateway="$1"', + 'port="$2"', + 'state_dir="$3"', + 'known_pid="$4"', + 'authority="$5"', + 'pid_file="$state_dir/openshell-gateway.pid"', + 'runtime_file="$state_dir/runtime.json"', + 'printf "gateway=%s\\nport=%s\\nstate_dir=%s\\nauthority=%s\\n" "$gateway" "$port" "$state_dir" "$authority"', + 'if [ -r "$pid_file" ]; then printf "pid_file="; cat "$pid_file"; else printf "pid_file=\\n"; fi', + 'if [ -r "$runtime_file" ]; then printf "runtime_marker=\\n"; cat "$runtime_file"; else printf "runtime_marker=\\n"; fi', + 'pid="$known_pid"', + 'if [ "$authority" = "standalone-state" ] && [ -r "$pid_file" ]; then pid="$(tr -d "[:space:]" < "$pid_file")"; fi', + 'if [ -n "$pid" ] && [ -r "/proc/$pid/cmdline" ]; then printf "proc_cmdline="; tr "\\000" " " < "/proc/$pid/cmdline"; printf "\\n"; fi', + 'if [ -n "$pid" ] && ps -p "$pid" -o pid= >/dev/null 2>&1; then', + ' printf "active_pid=%s\\n" "$pid"', + ' if [ -r "/proc/$pid/stat" ]; then', + ' proc_stat="$(cat "/proc/$pid/stat")"', + ' proc_stat="${proc_stat##*) }"', + " set -- $proc_stat", + " shift 19", + ' printf "process_start_identity=linux:%s\\n" "$1"', + " else", + ' process_started="$(ps -p "$pid" -o lstart= 2>/dev/null | sed -e "s/^[[:space:]]*//" -e "s/[[:space:]]*$//")"', + ' if [ -n "$process_started" ]; then printf "process_start_identity=ps:%s\\n" "$process_started"; fi', + " fi", + ' if [ -L "/proc/$pid/exe" ]; then', + ' printf "process_executable="; readlink "/proc/$pid/exe"; printf "\\n"', + " else", + ' process_executable="$(ps -p "$pid" -o comm= 2>/dev/null | sed -e "s/^[[:space:]]*//" -e "s/[[:space:]]*$//")"', + ' if [ -n "$process_executable" ]; then printf "process_executable=%s\\n" "$process_executable"; fi', + " fi", + ' ps -p "$pid" -o pid= -o ppid= -o user= -o command= 2>&1 || true', + "fi", + 'if command -v ss >/dev/null 2>&1; then ss -H -ltnp 2>&1 | awk -v port="$port" \'$4 ~ (":" port "$")\' || true; fi', + 'if command -v lsof >/dev/null 2>&1; then lsof -nP -a -iTCP:"$port" -sTCP:LISTEN 2>&1 || true; fi', + ].join("\n"), + "gateway-evidence", + options.gatewayName, + options.port, + stateDir, + options.knownPid ? String(options.knownPid) : "", + options.authority, + ], + { + artifactName: `${options.stage}-${options.gatewayName}-host-identity`, + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect(hostEvidence.exitCode, resultText(hostEvidence)).toBe(0); + + const sandboxEvidence = await sandbox.openshell(["sandbox", "list", "-g", options.gatewayName], { + artifactName: `${options.stage}-${options.gatewayName}-sandbox-phase`, + env: openshellEnvForGateway(options.gatewayName), + timeoutMs: 30_000, + }); + return { + host: hostEvidence, + processIdentity: capturedProcessIdentity(hostEvidence), + sandbox: sandboxEvidence, + }; +} + +async function captureGatewayPairEvidence( + host: HostCliClient, + sandbox: SandboxClient, + options: { + gatewayA: string; + gatewayB: string; + identityA: GatewayProcessIdentity; + identityB: GatewayProcessIdentity; + stage: string; + }, +): Promise<{ + gatewayA: { + host: ShellProbeResult; + processIdentity: CapturedProcessIdentity | null; + sandbox: ShellProbeResult; + }; + gatewayB: { + host: ShellProbeResult; + processIdentity: CapturedProcessIdentity | null; + sandbox: ShellProbeResult; + }; +}> { + const [gatewayA, gatewayB] = await Promise.all([ + captureGatewayEvidence(host, sandbox, { + authority: options.identityA.authority, + gatewayName: options.gatewayA, + knownPid: options.identityA.pid, + port: GATEWAY_PORT_A, + stage: options.stage, + }), + captureGatewayEvidence(host, sandbox, { + authority: options.identityB.authority, + gatewayName: options.gatewayB, + knownPid: options.identityB.pid, + port: GATEWAY_PORT_B, + stage: options.stage, + }), + ]); + await sandbox.openshell(["gateway", "list", "-o", "json"], { + artifactName: `${options.stage}-gateway-registrations`, + env: commandEnv(), + timeoutMs: 30_000, + }); + return { gatewayA, gatewayB }; +} + async function runOnboard( host: HostCliClient, sandboxName: string, @@ -198,6 +442,74 @@ async function expectPortNotListening( return result; } +async function expectSurvivingGatewayHealthyAcrossProbes( + host: HostCliClient, + sandbox: SandboxClient, + options: { + dashboardPort: string; + gatewayName: string; + gatewayPort: string; + sandboxName: string; + }, +): Promise { + const phases: string[] = []; + for (let attempt = 1; attempt <= POST_UNINSTALL_HEALTH_PROBES; attempt += 1) { + const suffix = String(attempt).padStart(2, "0"); + const sandboxList = await sandbox.openshell(["sandbox", "list", "-g", options.gatewayName], { + artifactName: `phase-4-survivor-probe-${suffix}-sandbox-phase`, + env: openshellEnvForGateway(options.gatewayName), + timeoutMs: 30_000, + }); + expect(sandboxList.exitCode, resultText(sandboxList)).toBe(0); + const phase = sandboxPhaseFromList(resultText(sandboxList), options.sandboxName) ?? "missing"; + phases.push(phase); + expect( + ["Ready", "Running"], + `survivor probe ${String(attempt)} observed ${options.sandboxName} phase '${phase}'`, + ).toContain(phase); + + await expectPortListening( + host, + options.gatewayPort, + `phase-4-survivor-probe-${suffix}-gateway-listener`, + ); + const scopedList = await command(host, ["list"], { + artifactName: `phase-4-survivor-probe-${suffix}-nemoclaw-list`, + env: commandEnv({ NEMOCLAW_GATEWAY_PORT: options.gatewayPort }), + timeoutMs: 60_000, + }); + expect(scopedList.exitCode, resultText(scopedList)).toBe(0); + expect(outputIncludesSandbox(scopedList.stdout, options.sandboxName), scopedList.stdout).toBe( + true, + ); + + const dashboard = await host.command( + "curl", + [ + "-sS", + "-L", + "--max-time", + "10", + "-o", + "/dev/null", + "-w", + "%{http_code}", + `http://127.0.0.1:${options.dashboardPort}/`, + ], + { + artifactName: `phase-4-survivor-probe-${suffix}-dashboard-http`, + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect(dashboard.exitCode, resultText(dashboard)).toBe(0); + expect(dashboard.stdout.trim()).toMatch(/^[23][0-9]{2}$/); + + if (attempt < POST_UNINSTALL_HEALTH_PROBES) await sleep(PROBE_DELAY_MS); + } + return phases; +} + async function prerequisiteOrSkip( host: HostCliClient, skip: (message: string) => never, @@ -473,6 +785,42 @@ test("concurrent gateway ports: onboards two sandboxes on isolated gateways and expect(dashboardB).not.toBe(dashboardA); progress.phase("uninstall alternate gateway without disrupting default"); + const gatewayIdentityA = readDefaultGatewayIdentity(); + const gatewayIdentityB = readAlternateGatewayIdentity(); + expect(gatewayIdentityA.pid).not.toBe(gatewayIdentityB.pid); + expect(gatewayIdentityB.authority).toBe("standalone-state"); + const beforeUninstallEvidence = await captureGatewayPairEvidence(host, sandbox, { + gatewayA, + gatewayB, + identityA: gatewayIdentityA, + identityB: gatewayIdentityB, + stage: "phase-4-before-uninstall", + }); + expect(beforeUninstallEvidence.gatewayA.processIdentity?.pid).toBe(gatewayIdentityA.pid); + expect(beforeUninstallEvidence.gatewayB.processIdentity?.pid).toBe(gatewayIdentityB.pid); + if (process.platform === "linux") { + if (gatewayIdentityA.authority === "standalone-state") { + expect(resultText(beforeUninstallEvidence.gatewayA.host)).toContain( + `openshell-gateway[nemoclaw=${gatewayA};port=${GATEWAY_PORT_A}]`, + ); + } + expect(resultText(beforeUninstallEvidence.gatewayB.host)).toContain( + `openshell-gateway[nemoclaw=${gatewayB};port=${GATEWAY_PORT_B}]`, + ); + expect(resultText(beforeUninstallEvidence.gatewayA.host)).toContain( + `active_pid=${String(gatewayIdentityA.pid)}`, + ); + expect(resultText(beforeUninstallEvidence.gatewayA.host)).not.toContain( + `active_pid=${String(gatewayIdentityB.pid)}`, + ); + expect(resultText(beforeUninstallEvidence.gatewayB.host)).toContain( + `active_pid=${String(gatewayIdentityB.pid)}`, + ); + expect(resultText(beforeUninstallEvidence.gatewayB.host)).not.toContain( + `active_pid=${String(gatewayIdentityA.pid)}`, + ); + } + const uninstallB = await command(host, ["uninstall", "--yes", "--destroy-user-data"], { artifactName: "phase-4-uninstall-gateway-b", env: commandEnv({ NEMOCLAW_GATEWAY_PORT: GATEWAY_PORT_B }), @@ -480,14 +828,41 @@ test("concurrent gateway ports: onboards two sandboxes on isolated gateways and }); expect(uninstallB.exitCode, resultText(uninstallB)).toBe(0); - const phaseAAfterUninstallB = await waitForSandboxReady( - sandbox, - SANDBOX_A, + const gatewayIdentityAAfterUninstall = readDefaultGatewayIdentity(); + expect(gatewayIdentityAAfterUninstall).toEqual(gatewayIdentityA); + const afterUninstallEvidence = await captureGatewayPairEvidence(host, sandbox, { gatewayA, - "phase-4-sandbox-a-still-ready-after-b-uninstall", + gatewayB, + identityA: gatewayIdentityAAfterUninstall, + identityB: gatewayIdentityB, + stage: "phase-4-after-uninstall", + }); + expect(readGatewayPid(GATEWAY_PORT_B)).toBeNull(); + expect(readGatewayRuntimePid(GATEWAY_PORT_B)).toBeNull(); + expect(afterUninstallEvidence.gatewayA.processIdentity).toEqual( + beforeUninstallEvidence.gatewayA.processIdentity, ); - expect(["Ready", "Running"]).toContain(phaseAAfterUninstallB); - await expectPortListening(host, GATEWAY_PORT_A, "phase-4-gateway-port-a-still-listening"); + expect(afterUninstallEvidence.gatewayB.processIdentity).toBeNull(); + if (process.platform === "linux") { + if (gatewayIdentityA.authority === "standalone-state") { + expect(resultText(afterUninstallEvidence.gatewayA.host)).toContain( + `openshell-gateway[nemoclaw=${gatewayA};port=${GATEWAY_PORT_A}]`, + ); + } + expect(resultText(afterUninstallEvidence.gatewayA.host)).toContain( + `active_pid=${String(gatewayIdentityA.pid)}`, + ); + expect(resultText(afterUninstallEvidence.gatewayB.host)).not.toContain( + `active_pid=${String(gatewayIdentityB.pid)}`, + ); + } + + const survivorPhases = await expectSurvivingGatewayHealthyAcrossProbes(host, sandbox, { + dashboardPort: dashboardA as string, + gatewayName: gatewayA, + gatewayPort: GATEWAY_PORT_A, + sandboxName: SANDBOX_A, + }); await expectPortNotListening(host, GATEWAY_PORT_B, "phase-4-gateway-port-b-stopped"); const listAAfterUninstallB = await command(host, ["list"], { @@ -523,7 +898,9 @@ test("concurrent gateway ports: onboards two sandboxes on isolated gateways and !outputIncludesSandbox(listGatewayB.stdout, SANDBOX_A), dashboardPortsDistinct: Boolean(dashboardA && dashboardB && dashboardA !== dashboardB), gatewayBUninstalled: uninstallB.exitCode === 0 && scopedStateRemoved.exitCode === 0, - sandboxAPreservedAfterUninstallB: ["Ready", "Running"].includes(phaseAAfterUninstallB), + sandboxAPreservedAfterUninstallB: + survivorPhases.length === POST_UNINSTALL_HEALTH_PROBES && + survivorPhases.every((phase) => phase === "Ready" || phase === "Running"), sharedCliPreserved: listAAfterUninstallB.exitCode === 0, }, }); From 9b23cd0d3dc632b19787dcb8d56d1cae9e5743e9 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sun, 9 Aug 2026 22:09:54 -0700 Subject: [PATCH 2/5] fix(uninstall): use docker adapters for scoped cleanup Signed-off-by: Prekshi Vyas --- ...run-plan-gateway-process-isolation.test.ts | 118 +++--- .../run-plan-gateway-service.test.ts | 393 +++++++++++------- src/lib/onboard/host-gateway-process.test.ts | 143 +++++-- src/lib/onboard/host-gateway-process.ts | 27 +- .../e2e/live/concurrent-gateway-ports.test.ts | 131 +++--- 5 files changed, 522 insertions(+), 290 deletions(-) diff --git a/src/lib/actions/uninstall/run-plan-gateway-process-isolation.test.ts b/src/lib/actions/uninstall/run-plan-gateway-process-isolation.test.ts index 66445c9c45d..b02c514b13e 100644 --- a/src/lib/actions/uninstall/run-plan-gateway-process-isolation.test.ts +++ b/src/lib/actions/uninstall/run-plan-gateway-process-isolation.test.ts @@ -18,6 +18,23 @@ function ok(stdout = ""): RunResult { return { status: 0, stdout, stderr: "" }; } +type RunResponder = () => RunResult; + +function commandSignature(command: string, args: readonly string[]): string { + return [command, ...args].join("\0"); +} + +function runFromResponses( + responses: ReadonlyMap, + calls: string[], +): NonNullable { + const fallback = () => ok(); + return (command, args) => { + calls.push([command, ...args].join(" ")); + return (responses.get(commandSignature(command, args)) ?? fallback)(); + }; +} + function withManagedGatewayAuthority(deps: UninstallRunDeps): UninstallRunDeps { const commandExists = deps.commandExists; return { @@ -146,6 +163,41 @@ describe("scoped uninstall gateway process isolation", () => { path.join(selectedGatewayRuntimeDir, "openshell-gateway.pid"), ).uid; let selectedAlive = true; + const gatewayList = JSON.stringify([ + { name: "nemoclaw" }, + { name: `nemoclaw-${String(selectedPort)}` }, + ]); + const stopped = { ...ok(), status: 1 }; + const runResponses = new Map([ + [commandSignature("openshell", ["gateway", "list", "-o", "json"]), () => ok(gatewayList)], + [ + commandSignature("lsof", ["-ti", `:${String(selectedPort)}`, "-sTCP:LISTEN"]), + () => (selectedAlive ? ok(`${String(selectedPid)}\n`) : stopped), + ], + [ + commandSignature("ps", ["-p", String(selectedPid), "-o", "pid="]), + () => (selectedAlive ? ok(`${String(selectedPid)}\n`) : stopped), + ], + [ + commandSignature("ps", ["-p", String(selectedPid), "-o", "uid="]), + () => ok(`${String(selectedUid)}\n`), + ], + [ + commandSignature("ps", ["-p", String(selectedPid), "-o", "comm="]), + () => ok("/opt/openshell-gateway\n"), + ], + [ + commandSignature("ps", ["-p", String(selectedPid), "-o", "lstart="]), + () => ok("fixture-start-identity\n"), + ], + [ + commandSignature("ps", ["-p", String(selectedPid), "-o", "args="]), + () => + ok( + `openshell-gateway[nemoclaw=nemoclaw-${String(selectedPort)};port=${String(selectedPort)}]\n`, + ), + ], + ]); const result = runPortUninstall( { assumeYes: true, @@ -167,40 +219,12 @@ describe("scoped uninstall gateway process isolation", () => { kill: (pid, signal) => { events.push(`kill ${String(pid)} ${String(signal)}`); signals.push({ pid, signal }); - if (pid !== selectedPid || signal !== "SIGKILL") return false; - selectedAlive = false; - return true; + const matchesSelectedGateway = pid === selectedPid && signal === "SIGKILL"; + selectedAlive = selectedAlive && !matchesSelectedGateway; + return matchesSelectedGateway; }, log: vi.fn(), - run: (command, args) => { - events.push([command, ...args].join(" ")); - if (command === "openshell" && args[0] === "gateway" && args[1] === "list") { - return ok( - JSON.stringify([ - { name: "nemoclaw" }, - { name: `nemoclaw-${String(selectedPort)}` }, - ]), - ); - } - if (command === "lsof" && args.includes(`:${String(selectedPort)}`)) { - return selectedAlive ? ok(`${String(selectedPid)}\n`) : { ...ok(), status: 1 }; - } - if (command === "ps" && args[1] === String(selectedPid)) { - if (args.includes("pid=")) { - return selectedAlive ? ok(`${String(selectedPid)}\n`) : { ...ok(), status: 1 }; - } - if (args.includes("uid=")) return ok(`${String(selectedUid)}\n`); - if (args.includes("comm=")) return ok("/opt/openshell-gateway\n"); - if (args.includes("lstart=")) return ok("fixture-start-identity\n"); - if (args.includes("args=")) { - return ok( - `openshell-gateway[nemoclaw=nemoclaw-${String(selectedPort)};port=${String(selectedPort)}]\n`, - ); - } - } - if (command === "pgrep") return ok(`${String(siblingPid)}\n${String(selectedPid)}\n`); - return ok(); - }, + run: runFromResponses(runResponses, events), runDocker: () => ok(), }, ); @@ -252,6 +276,17 @@ describe("scoped uninstall gateway process isolation", () => { const errors: string[] = []; const kill = vi.fn(() => true); const calls: string[] = []; + const gatewayList = JSON.stringify([ + { name: "nemoclaw" }, + { name: `nemoclaw-${String(selectedPort)}` }, + ]); + const runResponses = new Map([ + [commandSignature("openshell", ["gateway", "list", "-o", "json"]), () => ok(gatewayList)], + [ + commandSignature("ps", ["-p", String(siblingPid), "-o", "pid="]), + () => ok(`${String(siblingPid)}\n`), + ], + ]); const result = runPortUninstall( { assumeYes: true, @@ -268,24 +303,7 @@ describe("scoped uninstall gateway process isolation", () => { isTty: false, kill, log: vi.fn(), - run: (command, args) => { - calls.push([command, ...args].join(" ")); - if (command === "openshell" && args[0] === "gateway" && args[1] === "list") { - return ok( - JSON.stringify([ - { name: "nemoclaw" }, - { name: `nemoclaw-${String(selectedPort)}` }, - ]), - ); - } - if (command === "ps" && args[1] === String(siblingPid) && args.includes("pid=")) { - return ok(`${String(siblingPid)}\n`); - } - if (command === "pgrep") { - return ok(`${String(siblingPid)}\n${String(selectedMarkerPid)}\n`); - } - return ok(); - }, + run: runFromResponses(runResponses, calls), runDocker: () => ok(), }, ); diff --git a/src/lib/actions/uninstall/run-plan-gateway-service.test.ts b/src/lib/actions/uninstall/run-plan-gateway-service.test.ts index 59b22733a01..9185b5739f4 100644 --- a/src/lib/actions/uninstall/run-plan-gateway-service.test.ts +++ b/src/lib/actions/uninstall/run-plan-gateway-service.test.ts @@ -20,6 +20,42 @@ function ok(stdout = ""): RunResult { return { status: 0, stdout, stderr: "" }; } +type RunResponder = () => RunResult; + +function commandSignature(command: string, args: readonly string[]): string { + return [command, ...args].join("\0"); +} + +function systemctlShowSignature(serviceName: string): string { + return commandSignature("systemctl", [ + "--user", + "show", + serviceName, + "--property=FragmentPath", + "--property=ExecStart", + "--property=ExecStop", + "--property=ExecStopPost", + "--property=ActiveState", + "--property=MainPID", + "--property=Restart", + "--property=KillSignal", + "--property=KillMode", + ]); +} + +function runFromResponses( + responses: ReadonlyMap, + calls: string[][], + fallback: NonNullable = () => ok(), +): NonNullable { + return (command, args, options) => { + calls.push([command, ...args]); + return ( + responses.get(commandSignature(command, args)) ?? (() => fallback(command, args, options)) + )(); + }; +} + interface Fixture { env: NodeJS.ProcessEnv; home: string; @@ -136,6 +172,28 @@ function uninstall( run = () => ok(), ...overrides } = deps; + const servicePath = getNemoclawOpenShellGatewayUserServicePath(test.home, test.env); + const gatewayBin = `${test.home}/.local/bin/openshell-gateway`; + const inactiveSystemdShow = ok( + [ + `FragmentPath=${servicePath}`, + `ExecStart={ path=${gatewayBin} ; argv[]=${gatewayBin} ; }`, + "ExecStop=", + "ExecStopPost=", + "ActiveState=inactive", + "MainPID=0", + "Restart=on-failure", + "KillSignal=15", + "KillMode=control-group", + ].join("\n"), + ); + const defaultSystemdShows = new Map([ + [systemctlShowSignature(NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE), inactiveSystemdShow], + ]); + const gatewayListSignature = commandSignature("openshell", ["gateway", "list", "-o", "json"]); + const defaultResponses = new Map([ + [gatewayListSignature, ok(JSON.stringify(gateways))], + ]); return runUninstallPlan( { assumeYes: true, deleteModels: false, keepOpenShell }, { @@ -162,33 +220,13 @@ function uninstall( commandExists: (command) => command === "openshell" || command === "lsof" || commandExists(command), run: (command, args, options) => { - if (command === "openshell" && args[0] === "gateway" && args[1] === "list") { - return ok(JSON.stringify(gateways)); - } - const response = run(command, args, options); - if ( - command === "systemctl" && - args.includes("show") && - response.status === 0 && - response.stdout === "" - ) { - const servicePath = getNemoclawOpenShellGatewayUserServicePath(test.home, test.env); - const gatewayBin = `${test.home}/.local/bin/openshell-gateway`; - return ok( - [ - `FragmentPath=${servicePath}`, - `ExecStart={ path=${gatewayBin} ; argv[]=${gatewayBin} ; }`, - "ExecStop=", - "ExecStopPost=", - "ActiveState=inactive", - "MainPID=0", - "Restart=on-failure", - "KillSignal=15", - "KillMode=control-group", - ].join("\n"), - ); - } - return response; + const signature = commandSignature(command, args); + const response = defaultResponses.get(signature) ?? run(command, args, options); + const defaultSystemdShow = + response.status === 0 && response.stdout === "" + ? defaultSystemdShows.get(signature) + : undefined; + return defaultSystemdShow ?? response; }, }, ); @@ -328,6 +366,68 @@ describe("uninstall OpenShell gateway user service", () => { const kill = vi.fn(() => true); const readProcessExecutable = vi.fn(() => gatewayBin); const readProcessStartIdentity = vi.fn(() => "boot-identity:12345"); + const daemonReload = vi + .fn<() => RunResult>() + .mockImplementationOnce(() => { + events.push("daemon-reload"); + const dropInPath = path.join(`${servicePath}.d`, "99-nemoclaw-scoped-uninstall.conf"); + expect(fs.readFileSync(dropInPath, "utf-8")).toBe( + "[Service]\nRestart=no\nKillSignal=SIGKILL\nKillMode=control-group\n", + ); + scopedOverrideLoaded = true; + return ok(); + }) + .mockImplementation(() => { + events.push("daemon-reload"); + return ok(); + }); + const runResponses = new Map([ + [ + commandSignature("openshell", ["sandbox", "delete", "my-assistant"]), + () => { + events.push("sandbox-delete"); + return ok(); + }, + ], + [ + systemctlShowSignature(NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE), + () => + ok( + managedSystemdShow(test, { + active: !serviceStopped, + effectiveScopedStop: scopedOverrideLoaded, + mainPid, + }), + ), + ], + [commandSignature("systemctl", ["--user", "daemon-reload"]), daemonReload], + [ + commandSignature("systemctl", [ + "--user", + "disable", + "--now", + NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE, + ]), + () => { + events.push("disable-now"); + expect(scopedOverrideLoaded).toBe(true); + serviceStopped = true; + return ok(); + }, + ], + [ + commandSignature("ps", ["-p", String(mainPid), "-o", "uid="]), + () => ok(`${String(CURRENT_UID)}\n`), + ], + [ + commandSignature("ps", ["-p", String(mainPid), "-o", "pid="]), + () => (serviceStopped ? { status: 1, stdout: "", stderr: "" } : ok(`${mainPid}\n`)), + ], + [ + commandSignature("lsof", ["-ti", ":8080", "-sTCP:LISTEN"]), + () => (serviceStopped ? ok() : ok(`${mainPid}\n`)), + ], + ]); const result = uninstall( test, @@ -338,47 +438,7 @@ describe("uninstall OpenShell gateway user service", () => { kill, readProcessExecutable, readProcessStartIdentity, - run: (command, args) => { - calls.push([command, ...args]); - if (command === "openshell" && args[0] === "sandbox" && args[1] === "delete") { - events.push("sandbox-delete"); - return ok(); - } - if (command === "systemctl" && args.includes("show")) { - return ok( - managedSystemdShow(test, { - active: !serviceStopped, - effectiveScopedStop: scopedOverrideLoaded, - mainPid, - }), - ); - } - if (command === "systemctl" && args.includes("daemon-reload")) { - events.push("daemon-reload"); - if (!serviceStopped) { - const dropInPath = path.join(`${servicePath}.d`, "99-nemoclaw-scoped-uninstall.conf"); - expect(fs.readFileSync(dropInPath, "utf-8")).toBe( - "[Service]\nRestart=no\nKillSignal=SIGKILL\nKillMode=control-group\n", - ); - scopedOverrideLoaded = true; - } - return ok(); - } - if (command === "systemctl" && args.includes("disable")) { - events.push("disable-now"); - expect(scopedOverrideLoaded).toBe(true); - serviceStopped = true; - return ok(); - } - if (command === "ps" && args.at(-1) === "uid=") return ok(`${String(CURRENT_UID)}\n`); - if (command === "ps" && args.at(-1) === "pid=") { - return serviceStopped ? { status: 1, stdout: "", stderr: "" } : ok(`${mainPid}\n`); - } - if (command === "lsof" && args.includes(":8080")) { - return serviceStopped ? ok() : ok(`${mainPid}\n`); - } - return ok(); - }, + run: runFromResponses(runResponses, calls), }, [{ name: "nemoclaw" }, { name: "nemoclaw-8081" }], ); @@ -432,6 +492,48 @@ describe("uninstall OpenShell gateway user service", () => { const calls: string[][] = []; let scopedOverrideLoaded = false; let serviceStopped = false; + const runResponses = new Map([ + [ + systemctlShowSignature("openshell-gateway"), + () => + ok( + managedSystemdShow(test, { + active: !serviceStopped, + effectiveScopedStop: scopedOverrideLoaded, + execStartPath: gatewayBin, + fragmentPath: servicePath, + mainPid, + }), + ), + ], + [ + commandSignature("systemctl", ["--user", "daemon-reload"]), + () => { + scopedOverrideLoaded = fs.existsSync(dropInPath); + return ok(); + }, + ], + [ + commandSignature("systemctl", ["--user", "stop", "openshell-gateway"]), + () => { + expect(scopedOverrideLoaded).toBe(true); + serviceStopped = true; + return ok(); + }, + ], + [ + commandSignature("ps", ["-p", String(mainPid), "-o", "uid="]), + () => ok(`${String(CURRENT_UID)}\n`), + ], + [ + commandSignature("ps", ["-p", String(mainPid), "-o", "pid="]), + () => (serviceStopped ? { status: 1, stdout: "", stderr: "" } : ok(`${mainPid}\n`)), + ], + [ + commandSignature("lsof", ["-ti", ":8080", "-sTCP:LISTEN"]), + () => (serviceStopped ? ok() : ok(`${mainPid}\n`)), + ], + ]); const result = uninstall( test, @@ -444,37 +546,7 @@ describe("uninstall OpenShell gateway user service", () => { isPortFree: () => serviceStopped, readProcessExecutable: () => gatewayBin, readProcessStartIdentity: () => "boot-identity:package-service", - run: (command, args) => { - calls.push([command, ...args]); - if (command === "systemctl" && args.includes("show")) { - return ok( - managedSystemdShow(test, { - active: !serviceStopped, - effectiveScopedStop: scopedOverrideLoaded, - execStartPath: gatewayBin, - fragmentPath: servicePath, - mainPid, - }), - ); - } - if (command === "systemctl" && args.includes("daemon-reload")) { - scopedOverrideLoaded = fs.existsSync(dropInPath); - return ok(); - } - if (command === "systemctl" && args.includes("stop")) { - expect(scopedOverrideLoaded).toBe(true); - serviceStopped = true; - return ok(); - } - if (command === "ps" && args.at(-1) === "uid=") return ok(`${String(CURRENT_UID)}\n`); - if (command === "ps" && args.at(-1) === "pid=") { - return serviceStopped ? { status: 1, stdout: "", stderr: "" } : ok(`${mainPid}\n`); - } - if (command === "lsof" && args.includes(":8080")) { - return serviceStopped ? ok() : ok(`${mainPid}\n`); - } - return ok(); - }, + run: runFromResponses(runResponses, calls), }, [{ name: "nemoclaw" }, { name: "nemoclaw-8081" }], ); @@ -498,32 +570,54 @@ describe("uninstall OpenShell gateway user service", () => { let failFirstDropInRemoval = true; let scopedOverrideLoaded = false; let serviceStopped = false; - const run = (command: string, args: string[]) => { - if (command === "systemctl" && args.includes("show")) { - return ok( - managedSystemdShow(test, { - active: !serviceStopped, - effectiveScopedStop: scopedOverrideLoaded, - mainPid, - }), - ); - } - if (command === "systemctl" && args.includes("daemon-reload")) { - scopedOverrideLoaded = fs.existsSync(dropInPath); - return ok(); - } - if (command === "systemctl" && args.includes("disable")) { - serviceStopped = true; - return ok(); - } - if (command === "ps" && args.at(-1) === "uid=") return ok(`${String(CURRENT_UID)}\n`); - if (command === "ps" && args.at(-1) === "pid=") { - return serviceStopped ? { status: 1, stdout: "", stderr: "" } : ok(`${mainPid}\n`); - } - if (command === "lsof" && args.includes(":8080")) { - return serviceStopped ? ok() : ok(`${mainPid}\n`); - } - return ok(); + const calls: string[][] = []; + const runResponses = new Map([ + [ + systemctlShowSignature(NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE), + () => + ok( + managedSystemdShow(test, { + active: !serviceStopped, + effectiveScopedStop: scopedOverrideLoaded, + mainPid, + }), + ), + ], + [ + commandSignature("systemctl", ["--user", "daemon-reload"]), + () => { + scopedOverrideLoaded = fs.existsSync(dropInPath); + return ok(); + }, + ], + [ + commandSignature("systemctl", [ + "--user", + "disable", + "--now", + NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE, + ]), + () => { + serviceStopped = true; + return ok(); + }, + ], + [ + commandSignature("ps", ["-p", String(mainPid), "-o", "uid="]), + () => ok(`${String(CURRENT_UID)}\n`), + ], + [ + commandSignature("ps", ["-p", String(mainPid), "-o", "pid="]), + () => (serviceStopped ? { status: 1, stdout: "", stderr: "" } : ok(`${mainPid}\n`)), + ], + [ + commandSignature("lsof", ["-ti", ":8080", "-sTCP:LISTEN"]), + () => (serviceStopped ? ok() : ok(`${mainPid}\n`)), + ], + ]); + const run = runFromResponses(runResponses, calls); + const failDropInRemoval = (): never => { + throw new Error("injected drop-in removal failure"); }; const deps: Partial = { commandExists: (command) => command === "systemctl", @@ -531,11 +625,9 @@ describe("uninstall OpenShell gateway user service", () => { readProcessExecutable: () => gatewayBin, readProcessStartIdentity: () => "boot-identity:retry", rmSync: (target, options) => { - if (String(target) === dropInPath && failFirstDropInRemoval) { - failFirstDropInRemoval = false; - throw new Error("injected drop-in removal failure"); - } - fs.rmSync(target, options); + const failThisRemoval = String(target) === dropInPath && failFirstDropInRemoval; + failFirstDropInRemoval = failFirstDropInRemoval && !failThisRemoval; + return failThisRemoval ? failDropInRemoval() : fs.rmSync(target, options); }, run, }; @@ -578,6 +670,27 @@ describe("uninstall OpenShell gateway user service", () => { const mainPid = 41_201; const calls: string[][] = []; const kill = vi.fn(() => true); + const runResponses = new Map([ + [ + systemctlShowSignature(NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE), + () => + ok( + managedSystemdShow(test, { + active: true, + fragmentPath: identity.fragmentPath, + mainPid, + }), + ), + ], + [ + commandSignature("ps", ["-p", String(mainPid), "-o", "uid="]), + () => ok(`${String(identity.processUid ?? CURRENT_UID)}\n`), + ], + [ + commandSignature("lsof", ["-ti", ":8080", "-sTCP:LISTEN"]), + () => ok(`${identity.listenerPid}\n`), + ], + ]); const result = uninstall( test, @@ -588,25 +701,7 @@ describe("uninstall OpenShell gateway user service", () => { kill, readProcessExecutable: () => gatewayBin, readProcessStartIdentity: () => "boot-identity:12345", - run: (command, args) => { - calls.push([command, ...args]); - if (command === "systemctl" && args.includes("show")) { - return ok( - managedSystemdShow(test, { - active: true, - fragmentPath: identity.fragmentPath, - mainPid, - }), - ); - } - if (command === "ps" && args.at(-1) === "uid=") { - return ok(`${String(identity.processUid ?? CURRENT_UID)}\n`); - } - if (command === "lsof" && args.includes(":8080")) { - return ok(`${identity.listenerPid}\n`); - } - return ok(); - }, + run: runFromResponses(runResponses, calls), }, [{ name: "nemoclaw" }, { name: "nemoclaw-8081" }], ); diff --git a/src/lib/onboard/host-gateway-process.test.ts b/src/lib/onboard/host-gateway-process.test.ts index f72267669a0..a6f5e746818 100644 --- a/src/lib/onboard/host-gateway-process.test.ts +++ b/src/lib/onboard/host-gateway-process.test.ts @@ -81,7 +81,11 @@ type ScopedGatewayFixtureOptions = { markerPid?: number; markerPort?: number; omitMarker?: boolean; + omitPidFile?: boolean; pidFilePid?: number; + portFree?: boolean; + processAlive?: boolean; + processExecutable?: string | null; processUid?: number; startIdentities?: readonly string[]; usePgrepFallback?: boolean; @@ -108,17 +112,22 @@ function scopedGatewayFixture(options: ScopedGatewayFixtureOptions = {}) { options.cmdline ?? `${directGatewayBin} gateway start --name ${selectedName} --port ${String(selectedPort)}\n`; const compatibilityMode = selectedCmdline.includes("/opt/nemoclaw/openshell-gateway"); - fs.writeFileSync(selectedPidFile, `${String(options.pidFilePid ?? selectedPid)}\n`); + const writeSelectedPidFile = options.omitPidFile + ? () => undefined + : () => fs.writeFileSync(selectedPidFile, `${String(options.pidFilePid ?? selectedPid)}\n`); + writeSelectedPidFile(); fs.writeFileSync(siblingPidFile, `${String(siblingPid)}\n`); - if (!options.omitMarker) { - writeDockerDriverGatewayRuntimeMarkerForStateDir(selectedStateDir, { - desiredEnv: {}, - dockerHost: null, - endpoint: `https://127.0.0.1:${String(options.markerPort ?? selectedPort)}`, - gatewayBin: compatibilityMode ? null : directGatewayBin, - pid: options.markerPid ?? selectedPid, - }); - } + const writeSelectedMarker = options.omitMarker + ? () => undefined + : () => + writeDockerDriverGatewayRuntimeMarkerForStateDir(selectedStateDir, { + desiredEnv: {}, + dockerHost: null, + endpoint: `https://127.0.0.1:${String(options.markerPort ?? selectedPort)}`, + gatewayBin: compatibilityMode ? null : directGatewayBin, + pid: options.markerPid ?? selectedPid, + }); + writeSelectedMarker(); writeDockerDriverGatewayRuntimeMarkerForStateDir(siblingStateDir, { desiredEnv: {}, endpoint: "https://127.0.0.1:8080", @@ -126,7 +135,35 @@ function scopedGatewayFixture(options: ScopedGatewayFixtureOptions = {}) { }); const recordedPid = options.pidFilePid ?? selectedPid; - const exited = new Set(); + const exited = new Set(options.processAlive === false ? [recordedPid] : []); + const compatContainerPid = options.compatContainerPid; + const compatibilityResponses: [string, RunResult | ((args: string[]) => RunResult)][] = + compatContainerPid === undefined + ? [] + : [ + [ + `docker inspect --type container ${selectedCompatContainerName}`, + ok( + `${JSON.stringify([ + { + Args: [], + HostConfig: { NetworkMode: "host" }, + Id: selectedCompatContainerId, + Name: `/${selectedCompatContainerName}`, + Path: "/opt/nemoclaw/openshell-gateway", + State: { Pid: compatContainerPid, Running: true }, + }, + ])}\n`, + ), + ], + [ + `docker rm -f ${selectedCompatContainerId}`, + () => { + exited.add(recordedPid); + return ok(`${selectedCompatContainerName}\n`); + }, + ], + ]; const responses = new Map RunResult)>([ // A host-wide fallback would discover both gateways. Scoped teardown must // never execute this response. @@ -141,32 +178,15 @@ function scopedGatewayFixture(options: ScopedGatewayFixtureOptions = {}) { `lsof -ti :${String(selectedPort)} -sTCP:LISTEN`, ok((options.listenerPids ?? [recordedPid]).map(String).join("\n") + "\n"), ], + ...compatibilityResponses, ]); - if (options.compatContainerPid !== undefined) { - responses.set( - `docker inspect --type container ${selectedCompatContainerName}`, - ok( - `${JSON.stringify([ - { - Args: [], - HostConfig: { NetworkMode: "host" }, - Id: selectedCompatContainerId, - Name: `/${selectedCompatContainerName}`, - Path: "/opt/nemoclaw/openshell-gateway", - State: { Pid: options.compatContainerPid, Running: true }, - }, - ])}\n`, - ), - ); - responses.set(`docker rm -f ${selectedCompatContainerId}`, () => { - exited.add(recordedPid); - return ok(`${selectedCompatContainerName}\n`); - }); - } const { calls, run } = makeRun(responses); let startIdentityRead = 0; + const signalHandlers = new Map void>([ + ["SIGKILL", (pid) => void exited.add(pid)], + ]); const kill = vi.fn((pid, signal) => { - if (signal === "SIGKILL") exited.add(pid); + signalHandlers.get(signal)?.(pid); return true; }); @@ -176,16 +196,18 @@ function scopedGatewayFixture(options: ScopedGatewayFixtureOptions = {}) { kill, env: { USER: "tester" }, commandExists: () => true, - isPortFree: () => true, + dockerForceRm: (containerId) => run("docker", ["rm", "-f", containerId]), + dockerInspect: (args) => run("docker", ["inspect", ...args]), + isPortFree: () => options.portFree ?? true, log: vi.fn(), - readProcessExecutable: () => directGatewayBin, + readProcessExecutable: () => + options.processExecutable === undefined ? directGatewayBin : options.processExecutable, readProcessEnvironment: () => ({}), readProcessStartIdentity: (pid) => { - if (exited.has(pid)) return null; const identities = options.startIdentities ?? ["fixture-start-identity"]; const identity = identities[Math.min(startIdentityRead, identities.length - 1)] ?? null; startIdentityRead += 1; - return identity; + return exited.has(pid) ? null : identity; }, }, { @@ -708,6 +730,16 @@ describe("scoped host gateway stop isolation (#8663)", () => { options: { processUid: CURRENT_UID + 1 }, reason: "gateway process owner does not match the scoped runtime evidence owner", }, + { + label: "the process executable differs from the runtime marker", + options: { processExecutable: "/opt/foreign/openshell" }, + reason: "gateway process executable does not match the runtime marker", + }, + { + label: "the process executable has been deleted", + options: { processExecutable: "/opt/openshell/openshell (deleted)" }, + reason: "gateway process executable does not match the runtime marker", + }, { label: "the selected port listener belongs to the sibling PID", options: { listenerPids: [9_990_808] }, @@ -733,6 +765,43 @@ describe("scoped host gateway stop isolation (#8663)", () => { } }); + it("fails closed when the selected port is occupied without PID ownership evidence", () => { + const fixture = scopedGatewayFixture({ + listenerPids: [9_990_808], + omitMarker: true, + omitPidFile: true, + portFree: false, + }); + try { + expect(fixture.result.ownershipFailures).toEqual([ + "gateway port 18080 is occupied without PID-file ownership evidence", + ]); + expect(fixture.kill).not.toHaveBeenCalled(); + expect(fixture.calls.filter(({ command }) => command === "pgrep")).toEqual([]); + } finally { + fixture.cleanup(); + } + }); + + it("fails closed when the recorded PID is dead but the selected port remains occupied", () => { + const fixture = scopedGatewayFixture({ + listenerPids: [9_990_808], + portFree: false, + processAlive: false, + }); + try { + expect(fixture.result.skippedDeadPids).toEqual([fixture.selectedPid]); + expect(fixture.result.ownershipFailures).toEqual([ + `recorded PID ${String(fixture.selectedPid)} is dead but port 18080 remains occupied`, + ]); + expect(fixture.kill).not.toHaveBeenCalled(); + expect(fs.existsSync(fixture.selectedPidFile)).toBe(true); + expect(fs.existsSync(fixture.selectedRuntimeMarker)).toBe(true); + } finally { + fixture.cleanup(); + } + }); + it("rejects a requested host-wide fallback without scanning or signaling", () => { const fixture = scopedGatewayFixture({ usePgrepFallback: true }); try { diff --git a/src/lib/onboard/host-gateway-process.ts b/src/lib/onboard/host-gateway-process.ts index 40d25876a2b..d30656f7c11 100644 --- a/src/lib/onboard/host-gateway-process.ts +++ b/src/lib/onboard/host-gateway-process.ts @@ -6,6 +6,9 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { dockerForceRm as runDockerForceRm } from "../adapters/docker/container"; +import { dockerInspect as runDockerInspect } from "../adapters/docker/inspect"; +import type { DockerRunOptions, DockerRunResult } from "../adapters/docker/run"; import { waitUntil } from "../core/wait"; import { clearDockerDriverGatewayRuntimeMarker, @@ -33,6 +36,14 @@ export interface HostGatewayProcessDeps { kill: (pid: number, signal?: NodeJS.Signals | number) => boolean; env: NodeJS.ProcessEnv; commandExists?: (command: string) => boolean; + dockerForceRm: ( + containerName: string, + options?: DockerRunOptions, + ) => Pick; + dockerInspect: ( + args: readonly string[], + options?: DockerRunOptions, + ) => Pick; isPortFree?: (port: number) => boolean; log?: (message: string) => void; readProcessExecutable?: (pid: number) => string | null; @@ -144,6 +155,8 @@ function defaultDeps(overrides: Partial = {}): HostGatew kill: overrides.kill ?? defaultKill, env, commandExists: overrides.commandExists ?? ((cmd) => defaultCommandExists(cmd, env)), + dockerForceRm: overrides.dockerForceRm ?? runDockerForceRm, + dockerInspect: overrides.dockerInspect ?? runDockerInspect, isPortFree: overrides.isPortFree ?? ((port) => isHostPortFree(port)), log: overrides.log, readProcessExecutable: overrides.readProcessExecutable, @@ -320,12 +333,15 @@ function dockerCompatContainerIdentity( delete dockerEnv[key]; } dockerEnv.DOCKER_HOST = dockerHost; - const result = deps.run("docker", ["inspect", "--type", "container", containerName], { + const result = deps.dockerInspect(["--type", "container", containerName], { + encoding: "utf-8", env: dockerEnv, + ignoreError: true, + suppressOutput: true, }); if (result.status !== 0) return null; try { - const parsed = JSON.parse(result.stdout) as Array<{ + const parsed = JSON.parse(String(result.stdout ?? "")) as Array<{ Args?: unknown; HostConfig?: { NetworkMode?: unknown }; Id?: unknown; @@ -711,12 +727,15 @@ function tryStopScopedPid( const log = deps.log ?? ((message: string) => console.log(message)); if (processStartIdentity(pid, deps) !== expectedStartIdentity) return "identity-changed"; if (compatContainerIdentity) { - const removed = deps.run("docker", ["rm", "-f", compatContainerIdentity.containerId], { + const removed = deps.dockerForceRm(compatContainerIdentity.containerId, { + encoding: "utf-8", env: compatContainerIdentity.dockerEnv, + ignoreError: true, + suppressOutput: true, }); if (removed.status !== 0) { const warn = deps.warn ?? ((message: string) => console.warn(message)); - const detail = removed.stderr.trim() || `status ${String(removed.status)}`; + const detail = String(removed.stderr ?? "").trim() || `status ${String(removed.status)}`; warn( `Failed to remove scoped gateway compatibility container '${compatContainerIdentity.containerName}': ${detail}`, ); diff --git a/test/e2e/live/concurrent-gateway-ports.test.ts b/test/e2e/live/concurrent-gateway-ports.test.ts index 15f5db172c7..c376d47b975 100644 --- a/test/e2e/live/concurrent-gateway-ports.test.ts +++ b/test/e2e/live/concurrent-gateway-ports.test.ts @@ -24,6 +24,7 @@ import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compati import { CLI_DIST_ENTRYPOINT, CLI_ENTRYPOINT } from "../fixtures/paths.ts"; import { PollingError, pollUntil } from "../fixtures/polling.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { requireFixture } from "../support/require-fixture.ts"; const SANDBOX_A = process.env.NEMOCLAW_CGP_SANDBOX_A ?? "e2e-cgp-a"; const SANDBOX_B = process.env.NEMOCLAW_CGP_SANDBOX_B ?? "e2e-cgp-b"; @@ -72,9 +73,10 @@ function gatewayNameForPort(port: string): string { function gatewayStateDirForPort(port: string): string { const numericPort = Number(port); - if (!Number.isInteger(numericPort) || numericPort < 1 || numericPort > 65_535) { - throw new Error(`invalid gateway port '${port}'`); - } + requireFixture( + Number.isInteger(numericPort) && numericPort >= 1 && numericPort <= 65_535, + `invalid gateway port '${port}'`, + ); const leaf = port === "8080" ? "openshell-docker-gateway" : `openshell-docker-gateway-${port}`; return path.join(process.env.HOME || os.homedir(), ".local", "state", "nemoclaw", leaf); } @@ -97,8 +99,10 @@ function readGatewayRuntimePid(port: string): number | null { const marker: unknown = JSON.parse( fs.readFileSync(path.join(gatewayStateDirForPort(port), "runtime.json"), "utf-8"), ); - if (!marker || typeof marker !== "object" || !("pid" in marker)) return null; - const pid = (marker as { pid?: unknown }).pid; + const pid = + marker && typeof marker === "object" && "pid" in marker + ? (marker as { pid?: unknown }).pid + : null; return typeof pid === "number" && Number.isInteger(pid) && pid > 0 ? pid : null; } catch { return null; @@ -108,41 +112,39 @@ function readGatewayRuntimePid(port: string): number | null { function readStandaloneGatewayIdentity(port: string): GatewayProcessIdentity | null { const pid = readGatewayPid(port); const runtimePid = readGatewayRuntimePid(port); - if (pid === null && runtimePid === null) return null; - if (pid === null || runtimePid === null || pid !== runtimePid) { - throw new Error( - `gateway ${gatewayNameForPort(port)} has inconsistent standalone process state ` + - `(pid file: ${String(pid)}, runtime marker: ${String(runtimePid)})`, - ); - } - return { authority: "standalone-state", pid }; + const absent = pid === null && runtimePid === null; + requireFixture( + absent || (pid !== null && runtimePid !== null && pid === runtimePid), + `gateway ${gatewayNameForPort(port)} has inconsistent standalone process state ` + + `(pid file: ${String(pid)}, runtime marker: ${String(runtimePid)})`, + ); + return absent ? null : { authority: "standalone-state", pid: pid as number }; } -function readDefaultGatewayIdentity(): GatewayProcessIdentity { - const standalone = readStandaloneGatewayIdentity(GATEWAY_PORT_A); - if (standalone) return standalone; - +function readDefaultGatewayServiceIdentity(): GatewayProcessIdentity { const pid = process.platform === "linux" && GATEWAY_PORT_A === "8080" ? getTrustedActiveOpenShellGatewayUserServicePid({ env: commandEnv() }) : null; - if (pid === null) { - throw new Error( - `default gateway ${gatewayNameForPort(GATEWAY_PORT_A)} has neither matching ` + - "standalone PID/runtime state nor a trusted active systemd MainPID", - ); - } + requireFixture( + pid !== null, + `default gateway ${gatewayNameForPort(GATEWAY_PORT_A)} has neither matching ` + + "standalone PID/runtime state nor a trusted active systemd MainPID", + ); return { authority: "systemd-service", pid }; } +function readDefaultGatewayIdentity(): GatewayProcessIdentity { + return readStandaloneGatewayIdentity(GATEWAY_PORT_A) ?? readDefaultGatewayServiceIdentity(); +} + function readAlternateGatewayIdentity(): GatewayProcessIdentity { const identity = readStandaloneGatewayIdentity(GATEWAY_PORT_B); - if (!identity) { - throw new Error( - `alternate gateway ${gatewayNameForPort(GATEWAY_PORT_B)} is missing its standalone ` + - "PID/runtime ownership proof", - ); - } + requireFixture( + identity, + `alternate gateway ${gatewayNameForPort(GATEWAY_PORT_B)} is missing its standalone ` + + "PID/runtime ownership proof", + ); return identity; } @@ -155,18 +157,45 @@ function evidenceField(output: string, field: string): string | null { return values.length === 1 ? values[0] : null; } -function capturedProcessIdentity(result: ShellProbeResult): CapturedProcessIdentity | null { - const pidText = evidenceField(result.stdout, "active_pid"); - if (pidText === null) return null; +function parseCapturedProcessIdentity( + result: ShellProbeResult, + pidText: string, +): CapturedProcessIdentity { const pid = Number(pidText); const executable = evidenceField(result.stdout, "process_executable"); const startIdentity = evidenceField(result.stdout, "process_start_identity"); - if (!Number.isSafeInteger(pid) || pid <= 0 || !executable || !startIdentity) { - throw new Error(`incomplete gateway process identity evidence:\n${resultText(result)}`); - } + requireFixture( + Number.isSafeInteger(pid) && pid > 0 && executable && startIdentity, + `incomplete gateway process identity evidence:\n${resultText(result)}`, + ); return { executable, pid, startIdentity }; } +function capturedProcessIdentity(result: ShellProbeResult): CapturedProcessIdentity | null { + const pidText = evidenceField(result.stdout, "active_pid"); + return pidText === null ? null : parseCapturedProcessIdentity(result, pidText); +} + +function expectOnLinux(assertions: () => void): void { + (process.platform === "linux" ? assertions : () => undefined)(); +} + +function expectStandaloneGatewayArgv( + identity: GatewayProcessIdentity, + evidence: ShellProbeResult, + gatewayName: string, + gatewayPort: string, +): void { + const assertions: Record void> = { + "standalone-state": () => + expect(resultText(evidence)).toContain( + `openshell-gateway[nemoclaw=${gatewayName};port=${gatewayPort}]`, + ), + "systemd-service": () => undefined, + }; + assertions[identity.authority](); +} + function openshellEnvForGateway(gatewayName: string): NodeJS.ProcessEnv { return commandEnv({ OPENSHELL_GATEWAY: gatewayName }); } @@ -505,7 +534,7 @@ async function expectSurvivingGatewayHealthyAcrossProbes( expect(dashboard.exitCode, resultText(dashboard)).toBe(0); expect(dashboard.stdout.trim()).toMatch(/^[23][0-9]{2}$/); - if (attempt < POST_UNINSTALL_HEALTH_PROBES) await sleep(PROBE_DELAY_MS); + await (attempt < POST_UNINSTALL_HEALTH_PROBES ? sleep(PROBE_DELAY_MS) : Promise.resolve()); } return phases; } @@ -798,12 +827,13 @@ test("concurrent gateway ports: onboards two sandboxes on isolated gateways and }); expect(beforeUninstallEvidence.gatewayA.processIdentity?.pid).toBe(gatewayIdentityA.pid); expect(beforeUninstallEvidence.gatewayB.processIdentity?.pid).toBe(gatewayIdentityB.pid); - if (process.platform === "linux") { - if (gatewayIdentityA.authority === "standalone-state") { - expect(resultText(beforeUninstallEvidence.gatewayA.host)).toContain( - `openshell-gateway[nemoclaw=${gatewayA};port=${GATEWAY_PORT_A}]`, - ); - } + expectOnLinux(() => { + expectStandaloneGatewayArgv( + gatewayIdentityA, + beforeUninstallEvidence.gatewayA.host, + gatewayA, + GATEWAY_PORT_A, + ); expect(resultText(beforeUninstallEvidence.gatewayB.host)).toContain( `openshell-gateway[nemoclaw=${gatewayB};port=${GATEWAY_PORT_B}]`, ); @@ -819,7 +849,7 @@ test("concurrent gateway ports: onboards two sandboxes on isolated gateways and expect(resultText(beforeUninstallEvidence.gatewayB.host)).not.toContain( `active_pid=${String(gatewayIdentityA.pid)}`, ); - } + }); const uninstallB = await command(host, ["uninstall", "--yes", "--destroy-user-data"], { artifactName: "phase-4-uninstall-gateway-b", @@ -843,19 +873,20 @@ test("concurrent gateway ports: onboards two sandboxes on isolated gateways and beforeUninstallEvidence.gatewayA.processIdentity, ); expect(afterUninstallEvidence.gatewayB.processIdentity).toBeNull(); - if (process.platform === "linux") { - if (gatewayIdentityA.authority === "standalone-state") { - expect(resultText(afterUninstallEvidence.gatewayA.host)).toContain( - `openshell-gateway[nemoclaw=${gatewayA};port=${GATEWAY_PORT_A}]`, - ); - } + expectOnLinux(() => { + expectStandaloneGatewayArgv( + gatewayIdentityA, + afterUninstallEvidence.gatewayA.host, + gatewayA, + GATEWAY_PORT_A, + ); expect(resultText(afterUninstallEvidence.gatewayA.host)).toContain( `active_pid=${String(gatewayIdentityA.pid)}`, ); expect(resultText(afterUninstallEvidence.gatewayB.host)).not.toContain( `active_pid=${String(gatewayIdentityB.pid)}`, ); - } + }); const survivorPhases = await expectSurvivingGatewayHealthyAcrossProbes(host, sandbox, { dashboardPort: dashboardA as string, From eac996ce1d693ff519111fd4507e92f8740194fe Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sun, 9 Aug 2026 22:27:34 -0700 Subject: [PATCH 3/5] fix(uninstall): constrain docker adapter environment Signed-off-by: Prekshi Vyas --- src/lib/onboard/host-gateway-process.test.ts | 17 +++++++++++++++-- src/lib/onboard/host-gateway-process.ts | 10 +++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/lib/onboard/host-gateway-process.test.ts b/src/lib/onboard/host-gateway-process.test.ts index a6f5e746818..181989114ff 100644 --- a/src/lib/onboard/host-gateway-process.test.ts +++ b/src/lib/onboard/host-gateway-process.test.ts @@ -181,6 +181,7 @@ function scopedGatewayFixture(options: ScopedGatewayFixtureOptions = {}) { ...compatibilityResponses, ]); const { calls, run } = makeRun(responses); + const dockerAdapterCalls: Array<{ env: NodeJS.ProcessEnv | undefined; operation: string }> = []; let startIdentityRead = 0; const signalHandlers = new Map void>([ ["SIGKILL", (pid) => void exited.add(pid)], @@ -196,8 +197,14 @@ function scopedGatewayFixture(options: ScopedGatewayFixtureOptions = {}) { kill, env: { USER: "tester" }, commandExists: () => true, - dockerForceRm: (containerId) => run("docker", ["rm", "-f", containerId]), - dockerInspect: (args) => run("docker", ["inspect", ...args]), + dockerForceRm: (containerId, adapterOptions) => { + dockerAdapterCalls.push({ env: adapterOptions?.env, operation: "force-rm" }); + return run("docker", ["rm", "-f", containerId]); + }, + dockerInspect: (args, adapterOptions) => { + dockerAdapterCalls.push({ env: adapterOptions?.env, operation: "inspect" }); + return run("docker", ["inspect", ...args]); + }, isPortFree: () => options.portFree ?? true, log: vi.fn(), readProcessExecutable: () => @@ -229,6 +236,7 @@ function scopedGatewayFixture(options: ScopedGatewayFixtureOptions = {}) { fs.rmSync(siblingStateDir, { recursive: true, force: true }); }, kill, + dockerAdapterCalls, recordedPid, result, selectedPid, @@ -649,6 +657,11 @@ describe("scoped host gateway stop isolation (#8663)", () => { args: ["rm", "-f", fixture.selectedCompatContainerId], command: "docker", }); + expect(fixture.dockerAdapterCalls).toEqual([ + { env: { DOCKER_HOST: "unix:///var/run/docker.sock" }, operation: "inspect" }, + { env: { DOCKER_HOST: "unix:///var/run/docker.sock" }, operation: "inspect" }, + { env: { DOCKER_HOST: "unix:///var/run/docker.sock" }, operation: "force-rm" }, + ]); expect( dockerCalls.some(({ args }) => args.includes(fixture.siblingCompatContainerName)), ).toBe(false); diff --git a/src/lib/onboard/host-gateway-process.ts b/src/lib/onboard/host-gateway-process.ts index d30656f7c11..ae79cccc3a7 100644 --- a/src/lib/onboard/host-gateway-process.ts +++ b/src/lib/onboard/host-gateway-process.ts @@ -328,11 +328,11 @@ function dockerCompatContainerIdentity( dockerHost: string, deps: HostGatewayProcessDeps, ): DockerCompatContainerIdentity | null { - const dockerEnv = { ...deps.env }; - for (const key of ["DOCKER_CERT_PATH", "DOCKER_CONFIG", "DOCKER_CONTEXT", "DOCKER_TLS_VERIFY"]) { - delete dockerEnv[key]; - } - dockerEnv.DOCKER_HOST = dockerHost; + // Docker adapters apply the repository subprocess allowlist before spawning. + // Pass only the daemon identity we proved instead of treating the complete + // parent environment as an explicit override, which would reintroduce + // unrelated credentials that the adapter intentionally filters out. + const dockerEnv = { DOCKER_HOST: dockerHost }; const result = deps.dockerInspect(["--type", "container", containerName], { encoding: "utf-8", env: dockerEnv, From 0ae1bc1bfb4b1030af05cab8fa5c973b40dfeab9 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sun, 9 Aug 2026 22:34:54 -0700 Subject: [PATCH 4/5] refactor(uninstall): narrow scoped gateway isolation Signed-off-by: Apurv Kumaria --- docs/manage-sandboxes/uninstall-nemoclaw.mdx | 9 + docs/reference/commands.mdx | 9 + ...run-plan-gateway-process-isolation.test.ts | 333 --------- .../run-plan-gateway-segregation.test.ts | 8 +- .../run-plan-gateway-service.test.ts | 484 +----------- src/lib/actions/uninstall/run-plan.ts | 633 ++-------------- .../onboard/docker-driver-gateway-config.ts | 48 +- src/lib/onboard/docker-driver-gateway-env.ts | 2 + .../docker-driver-gateway-launch.test.ts | 18 +- .../onboard/docker-driver-gateway-launch.ts | 2 + .../docker-driver-gateway-prelaunch.test.ts | 1 - .../docker-driver-gateway-prelaunch.ts | 1 - src/lib/onboard/gateway-process-identity.ts | 1 - .../gateway-process-target-identity.ts | 19 +- .../host-gateway-process-target.test.ts | 163 +++- src/lib/onboard/host-gateway-process.test.ts | 424 ----------- src/lib/onboard/host-gateway-process.ts | 705 ++++-------------- .../gateway-port-release-test-helpers.ts | 1 - .../e2e/live/concurrent-gateway-ports.test.ts | 502 +++---------- 19 files changed, 525 insertions(+), 2838 deletions(-) delete mode 100644 src/lib/actions/uninstall/run-plan-gateway-process-isolation.test.ts diff --git a/docs/manage-sandboxes/uninstall-nemoclaw.mdx b/docs/manage-sandboxes/uninstall-nemoclaw.mdx index 5666e571071..b811975f94c 100644 --- a/docs/manage-sandboxes/uninstall-nemoclaw.mdx +++ b/docs/manage-sandboxes/uninstall-nemoclaw.mdx @@ -83,6 +83,15 @@ Rerun `NEMOCLAW_GATEWAY_PORT= $$nemoclaw uninstall` with the gateway port For an externally supervised authority, uninstall preserves the local gateway state used by the running process in both full and gateway-scoped cleanup. It also preserves the gateway process, supervisor resources, marked Linux unit, Docker resources, OpenShell binaries, and the declared external state directory. A custom-port uninstall does not stop or remove the default gateway service or its environment file. +Before scoped cleanup stops a host gateway process or managed default gateway service, NemoClaw requires two namespace proofs. +The selected gateway configuration and any running gateway process must use the state-root-specific OpenShell sandbox namespace that NemoClaw generated. +If either proof is absent, uninstall exits nonzero before it signals the host gateway. +NemoClaw preserves the gateway runtime evidence and local state. +Keep that state intact. +Restore the selected gateway through the supported install or onboarding recovery flow so it restarts with the generated configuration. +Verify every gateway with `openshell gateway list`. +Retry the scoped uninstall. +Do not add `sandbox_namespace` manually to a live gateway configuration because the running process can still be using its previous namespace. In this section, `` is `~/.nemoclaw/` for the default gateway or `~/.nemoclaw/gateways//` for a non-default gateway. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 2fd0fbdf684..b93683aeff9 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3892,6 +3892,15 @@ Rerun `NEMOCLAW_GATEWAY_PORT= $$nemoclaw uninstall` with the gateway port For an externally supervised authority, uninstall preserves the selected local gateway state in both full and gateway-scoped cleanup. It also preserves the gateway process, supervisor resources, marked Linux unit, Docker resources, OpenShell binaries, and the declared external state directory. A custom-port uninstall does not stop or remove the default gateway service or its environment file. +Before scoped cleanup stops a host gateway process or managed default gateway service, NemoClaw requires two namespace proofs. +The selected gateway configuration and any running gateway process must use the state-root-specific OpenShell sandbox namespace that NemoClaw generated. +If either proof is absent, uninstall exits nonzero before it signals the host gateway. +NemoClaw preserves the gateway runtime evidence and local state. +Keep that state intact. +Restore the selected gateway through the supported install or onboarding recovery flow so it restarts with the generated configuration. +Verify every gateway with `openshell gateway list`. +Retry the scoped uninstall. +Do not add `sandbox_namespace` manually to a live gateway configuration because the running process can still be using its previous namespace. ##### Uninstalling Every Gateway Port diff --git a/src/lib/actions/uninstall/run-plan-gateway-process-isolation.test.ts b/src/lib/actions/uninstall/run-plan-gateway-process-isolation.test.ts deleted file mode 100644 index b02c514b13e..00000000000 --- a/src/lib/actions/uninstall/run-plan-gateway-process-isolation.test.ts +++ /dev/null @@ -1,333 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { afterEach, describe, expect, it, vi } from "vitest"; -import { writeDockerDriverGatewayRuntimeMarkerForStateDir } from "../../onboard/docker-driver-gateway-runtime-marker"; -import { - type RunResult, - runUninstallPlan as runUninstallPlanBase, - type UninstallRunDeps, - type UninstallRunOptions, -} from "./run-plan"; - -function ok(stdout = ""): RunResult { - return { status: 0, stdout, stderr: "" }; -} - -type RunResponder = () => RunResult; - -function commandSignature(command: string, args: readonly string[]): string { - return [command, ...args].join("\0"); -} - -function runFromResponses( - responses: ReadonlyMap, - calls: string[], -): NonNullable { - const fallback = () => ok(); - return (command, args) => { - calls.push([command, ...args].join(" ")); - return (responses.get(commandSignature(command, args)) ?? fallback)(); - }; -} - -function withManagedGatewayAuthority(deps: UninstallRunDeps): UninstallRunDeps { - const commandExists = deps.commandExists; - return { - resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ - gatewayName, - gatewayPort, - mode: "nemoclaw-managed", - source: "standalone", - endpoint: null, - stateDir: null, - supervisor: null, - requiredCapabilities: [], - }), - ...deps, - isPortFree: deps.isPortFree ?? (() => true), - commandExists: (command) => command === "lsof" || (commandExists?.(command) ?? false), - }; -} - -function bindManagedGatewayAuthority(run: typeof runUninstallPlanBase) { - return (options: UninstallRunOptions, deps: UninstallRunDeps) => - run(options, withManagedGatewayAuthority(deps)); -} - -function writeScopedGatewayPairState(options: { - markerPid: number; - pidFilePid: number; - selectedPort: number; - siblingPid: number; - tmpHome: string; -}) { - const { markerPid, pidFilePid, selectedPort, siblingPid, tmpHome } = options; - const sharedStateDir = path.join(tmpHome, ".nemoclaw"); - const selectedStateDir = path.join(sharedStateDir, "gateways", String(selectedPort)); - const gatewayRuntimeRoot = path.join(tmpHome, ".local", "state", "nemoclaw"); - const selectedGatewayRuntimeDir = path.join( - gatewayRuntimeRoot, - `openshell-docker-gateway-${String(selectedPort)}`, - ); - const siblingGatewayRuntimeDir = path.join(gatewayRuntimeRoot, "openshell-docker-gateway"); - fs.mkdirSync(selectedStateDir, { recursive: true }); - fs.mkdirSync(selectedGatewayRuntimeDir, { recursive: true }); - fs.mkdirSync(siblingGatewayRuntimeDir, { recursive: true }); - fs.writeFileSync( - path.join(sharedStateDir, "sandboxes.json"), - JSON.stringify({ - defaultSandbox: "sibling-box", - sandboxes: { - "sibling-box": { - name: "sibling-box", - gatewayName: "nemoclaw", - gatewayPort: 8080, - }, - }, - }), - ); - fs.writeFileSync( - path.join(selectedStateDir, "sandboxes.json"), - JSON.stringify({ - defaultSandbox: "selected-box", - sandboxes: { - "selected-box": { - name: "selected-box", - gatewayName: `nemoclaw-${String(selectedPort)}`, - gatewayPort: selectedPort, - }, - }, - }), - ); - const pidFile = path.join(selectedGatewayRuntimeDir, "openshell-gateway.pid"); - fs.writeFileSync(pidFile, `${String(pidFilePid)}\n`); - writeDockerDriverGatewayRuntimeMarkerForStateDir(selectedGatewayRuntimeDir, { - desiredEnv: {}, - endpoint: `https://127.0.0.1:${String(selectedPort)}`, - gatewayBin: "/opt/openshell-gateway", - pid: markerPid, - }); - fs.writeFileSync(path.join(selectedGatewayRuntimeDir, "selected-state"), "keep\n"); - fs.writeFileSync( - path.join(siblingGatewayRuntimeDir, "openshell-gateway.pid"), - `${String(siblingPid)}\n`, - ); - fs.writeFileSync(path.join(siblingGatewayRuntimeDir, "sibling-state"), "keep\n"); - return { - pidFile, - selectedGatewayRuntimeDir, - selectedStateDir, - sharedStateDir, - siblingGatewayRuntimeDir, - }; -} - -afterEach(() => { - vi.unstubAllEnvs(); - vi.resetModules(); -}); - -describe("scoped uninstall gateway process isolation", () => { - it("proves and stops only the selected gateway process during scoped uninstall (#8663)", async () => { - const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-exact-process-")); - const selectedPort = 18_080; - const selectedPid = 987_650; - const siblingPid = 987_651; - try { - vi.stubEnv("NEMOCLAW_GATEWAY_PORT", String(selectedPort)); - vi.resetModules(); - const runPortUninstall = bindManagedGatewayAuthority( - (await import("./run-plan")).runUninstallPlan, - ); - const { - selectedGatewayRuntimeDir, - selectedStateDir, - sharedStateDir, - siblingGatewayRuntimeDir, - } = writeScopedGatewayPairState({ - markerPid: selectedPid, - pidFilePid: selectedPid, - selectedPort, - siblingPid, - tmpHome, - }); - - const events: string[] = []; - const signals: Array<{ pid: number; signal?: NodeJS.Signals | number }> = []; - const selectedUid = fs.statSync( - path.join(selectedGatewayRuntimeDir, "openshell-gateway.pid"), - ).uid; - let selectedAlive = true; - const gatewayList = JSON.stringify([ - { name: "nemoclaw" }, - { name: `nemoclaw-${String(selectedPort)}` }, - ]); - const stopped = { ...ok(), status: 1 }; - const runResponses = new Map([ - [commandSignature("openshell", ["gateway", "list", "-o", "json"]), () => ok(gatewayList)], - [ - commandSignature("lsof", ["-ti", `:${String(selectedPort)}`, "-sTCP:LISTEN"]), - () => (selectedAlive ? ok(`${String(selectedPid)}\n`) : stopped), - ], - [ - commandSignature("ps", ["-p", String(selectedPid), "-o", "pid="]), - () => (selectedAlive ? ok(`${String(selectedPid)}\n`) : stopped), - ], - [ - commandSignature("ps", ["-p", String(selectedPid), "-o", "uid="]), - () => ok(`${String(selectedUid)}\n`), - ], - [ - commandSignature("ps", ["-p", String(selectedPid), "-o", "comm="]), - () => ok("/opt/openshell-gateway\n"), - ], - [ - commandSignature("ps", ["-p", String(selectedPid), "-o", "lstart="]), - () => ok("fixture-start-identity\n"), - ], - [ - commandSignature("ps", ["-p", String(selectedPid), "-o", "args="]), - () => - ok( - `openshell-gateway[nemoclaw=nemoclaw-${String(selectedPort)};port=${String(selectedPort)}]\n`, - ), - ], - ]); - const result = runPortUninstall( - { - assumeYes: true, - deleteModels: false, - destroyUserData: true, - gatewayName: `nemoclaw-${String(selectedPort)}`, - keepOpenShell: false, - }, - { - commandExists: (command) => ["lsof", "openshell", "pgrep"].includes(command), - env: { - HOME: tmpHome, - LOGNAME: "tester", - NEMOCLAW_GATEWAY_PORT: String(selectedPort), - } as NodeJS.ProcessEnv, - existsSync: (target) => target.startsWith(tmpHome) && fs.existsSync(target), - isPortFree: () => !selectedAlive, - isTty: false, - kill: (pid, signal) => { - events.push(`kill ${String(pid)} ${String(signal)}`); - signals.push({ pid, signal }); - const matchesSelectedGateway = pid === selectedPid && signal === "SIGKILL"; - selectedAlive = selectedAlive && !matchesSelectedGateway; - return matchesSelectedGateway; - }, - log: vi.fn(), - run: runFromResponses(runResponses, events), - runDocker: () => ok(), - }, - ); - - expect(result.exitCode).toBe(0); - expect(signals).toEqual([{ pid: selectedPid, signal: "SIGKILL" }]); - expect(events.some((event) => event.startsWith("pgrep "))).toBe(false); - expect(events.indexOf("openshell sandbox delete selected-box")).toBeLessThan( - events.indexOf(`kill ${String(selectedPid)} SIGKILL`), - ); - expect(fs.existsSync(selectedStateDir)).toBe(false); - expect( - fs.readFileSync(path.join(siblingGatewayRuntimeDir, "openshell-gateway.pid"), "utf8"), - ).toBe(`${String(siblingPid)}\n`); - expect(fs.readFileSync(path.join(siblingGatewayRuntimeDir, "sibling-state"), "utf8")).toBe( - "keep\n", - ); - expect(fs.existsSync(path.join(sharedStateDir, "sandboxes.json"))).toBe(true); - } finally { - fs.rmSync(tmpHome, { recursive: true, force: true }); - } - }); - - it("fails closed and preserves runtime evidence when scoped PID identities cross-match (#8663)", async () => { - const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-crossed-process-")); - const selectedPort = 18_080; - const siblingPid = 987_652; - const selectedMarkerPid = 987_653; - try { - vi.stubEnv("NEMOCLAW_GATEWAY_PORT", String(selectedPort)); - vi.resetModules(); - const runPortUninstall = bindManagedGatewayAuthority( - (await import("./run-plan")).runUninstallPlan, - ); - const { - pidFile, - selectedGatewayRuntimeDir, - selectedStateDir, - sharedStateDir, - siblingGatewayRuntimeDir, - } = writeScopedGatewayPairState({ - markerPid: selectedMarkerPid, - pidFilePid: siblingPid, - selectedPort, - siblingPid, - tmpHome, - }); - - const errors: string[] = []; - const kill = vi.fn(() => true); - const calls: string[] = []; - const gatewayList = JSON.stringify([ - { name: "nemoclaw" }, - { name: `nemoclaw-${String(selectedPort)}` }, - ]); - const runResponses = new Map([ - [commandSignature("openshell", ["gateway", "list", "-o", "json"]), () => ok(gatewayList)], - [ - commandSignature("ps", ["-p", String(siblingPid), "-o", "pid="]), - () => ok(`${String(siblingPid)}\n`), - ], - ]); - const result = runPortUninstall( - { - assumeYes: true, - deleteModels: false, - destroyUserData: true, - gatewayName: `nemoclaw-${String(selectedPort)}`, - keepOpenShell: false, - }, - { - commandExists: (command) => ["lsof", "openshell", "pgrep"].includes(command), - env: { HOME: tmpHome, NEMOCLAW_GATEWAY_PORT: String(selectedPort) } as NodeJS.ProcessEnv, - error: (message) => errors.push(message), - existsSync: (target) => target.startsWith(tmpHome) && fs.existsSync(target), - isTty: false, - kill, - log: vi.fn(), - run: runFromResponses(runResponses, calls), - runDocker: () => ok(), - }, - ); - - expect(result.exitCode).toBe(1); - expect(kill).not.toHaveBeenCalled(); - expect(calls.some((call) => call.startsWith("pgrep "))).toBe(false); - expect(errors.join("\n")).toContain( - `runtime marker PID ${String(selectedMarkerPid)} does not match PID file ${String(siblingPid)}`, - ); - expect(fs.readFileSync(pidFile, "utf8")).toBe(`${String(siblingPid)}\n`); - expect(fs.readFileSync(path.join(selectedGatewayRuntimeDir, "selected-state"), "utf8")).toBe( - "keep\n", - ); - expect(fs.readFileSync(path.join(siblingGatewayRuntimeDir, "sibling-state"), "utf8")).toBe( - "keep\n", - ); - expect( - fs.readFileSync(path.join(siblingGatewayRuntimeDir, "openshell-gateway.pid"), "utf8"), - ).toBe(`${String(siblingPid)}\n`); - expect(fs.existsSync(selectedStateDir)).toBe(true); - expect(fs.existsSync(path.join(sharedStateDir, "sandboxes.json"))).toBe(true); - } finally { - fs.rmSync(tmpHome, { recursive: true, force: true }); - } - }); -}); diff --git a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts index 2d4d49cef1f..adc7388fd76 100644 --- a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts +++ b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; + import { NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER_LINE } from "../../onboard/docker-driver-gateway-service"; import { readGatewayRegistryFile } from "../../state/gateway-registry"; import { migrateLegacyPortState } from "../../state/legacy-port-migration"; @@ -21,8 +22,8 @@ function ok(stdout = ""): RunResult { } function withManagedGatewayAuthority(deps: UninstallRunDeps): UninstallRunDeps { - const commandExists = deps.commandExists; return { + isPortFree: () => true, resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ gatewayName, gatewayPort, @@ -34,11 +35,6 @@ function withManagedGatewayAuthority(deps: UninstallRunDeps): UninstallRunDeps { requiredCapabilities: [], }), ...deps, - isPortFree: deps.isPortFree ?? (() => true), - // Scoped teardown must distinguish an absent selected gateway from an - // unobservable listener. These unit fixtures model a successful empty - // lsof query unless a test overrides the command response with a PID. - commandExists: (command) => command === "lsof" || (commandExists?.(command) ?? false), }; } diff --git a/src/lib/actions/uninstall/run-plan-gateway-service.test.ts b/src/lib/actions/uninstall/run-plan-gateway-service.test.ts index 9185b5739f4..dc3cc11f020 100644 --- a/src/lib/actions/uninstall/run-plan-gateway-service.test.ts +++ b/src/lib/actions/uninstall/run-plan-gateway-service.test.ts @@ -20,42 +20,6 @@ function ok(stdout = ""): RunResult { return { status: 0, stdout, stderr: "" }; } -type RunResponder = () => RunResult; - -function commandSignature(command: string, args: readonly string[]): string { - return [command, ...args].join("\0"); -} - -function systemctlShowSignature(serviceName: string): string { - return commandSignature("systemctl", [ - "--user", - "show", - serviceName, - "--property=FragmentPath", - "--property=ExecStart", - "--property=ExecStop", - "--property=ExecStopPost", - "--property=ActiveState", - "--property=MainPID", - "--property=Restart", - "--property=KillSignal", - "--property=KillMode", - ]); -} - -function runFromResponses( - responses: ReadonlyMap, - calls: string[][], - fallback: NonNullable = () => ok(), -): NonNullable { - return (command, args, options) => { - calls.push([command, ...args]); - return ( - responses.get(commandSignature(command, args)) ?? (() => fallback(command, args, options)) - )(); - }; -} - interface Fixture { env: NodeJS.ProcessEnv; home: string; @@ -63,7 +27,6 @@ interface Fixture { } const tempRoots: string[] = []; -const CURRENT_UID = typeof process.getuid === "function" ? process.getuid() : 0; afterEach(() => { for (const root of tempRoots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); @@ -85,6 +48,7 @@ function fixture(useXdg = false): Fixture { } function writeManagedService(test: Fixture): string { + writeGatewayState(test); const servicePath = getNemoclawOpenShellGatewayUserServicePath(test.home, test.env); fs.mkdirSync(path.dirname(servicePath), { recursive: true }); fs.writeFileSync( @@ -130,77 +94,27 @@ function writeGatewayState(test: Fixture): string { "openshell-gateway.toml", ); fs.mkdirSync(path.dirname(configPath), { recursive: true }); - fs.writeFileSync(configPath, 'listen_address = "127.0.0.1:8080"\n'); + fs.writeFileSync( + configPath, + '[openshell.drivers.docker]\nsandbox_namespace = "nemoclaw-openshell-docker-gateway"\n', + ); return configPath; } -function managedSystemdShow( - test: Fixture, - options: { - active: boolean; - effectiveScopedStop?: boolean; - execStartPath?: string; - fragmentPath?: string; - mainPid: number; - }, -): string { - const gatewayBin = options.execStartPath ?? `${test.home}/.local/bin/openshell-gateway`; - const servicePath = - options.fragmentPath ?? getNemoclawOpenShellGatewayUserServicePath(test.home, test.env); - return [ - `FragmentPath=${servicePath}`, - `ExecStart={ path=${gatewayBin} ; argv[]=${gatewayBin} ; }`, - "ExecStop=", - "ExecStopPost=", - `ActiveState=${options.active ? "active" : "inactive"}`, - `MainPID=${String(options.active ? options.mainPid : 0)}`, - `Restart=${options.effectiveScopedStop ? "no" : "on-failure"}`, - `KillSignal=${options.effectiveScopedStop ? "SIGKILL" : "SIGTERM"}`, - "KillMode=control-group", - ].join("\n"); -} - function uninstall( test: Fixture, keepOpenShell: boolean, deps: Partial = {}, gateways: { name: string }[] = [{ name: "nemoclaw" }], ) { - const { - commandExists = () => false, - isPortFree = () => true, - run = () => ok(), - ...overrides - } = deps; - const servicePath = getNemoclawOpenShellGatewayUserServicePath(test.home, test.env); - const gatewayBin = `${test.home}/.local/bin/openshell-gateway`; - const inactiveSystemdShow = ok( - [ - `FragmentPath=${servicePath}`, - `ExecStart={ path=${gatewayBin} ; argv[]=${gatewayBin} ; }`, - "ExecStop=", - "ExecStopPost=", - "ActiveState=inactive", - "MainPID=0", - "Restart=on-failure", - "KillSignal=15", - "KillMode=control-group", - ].join("\n"), - ); - const defaultSystemdShows = new Map([ - [systemctlShowSignature(NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE), inactiveSystemdShow], - ]); - const gatewayListSignature = commandSignature("openshell", ["gateway", "list", "-o", "json"]); - const defaultResponses = new Map([ - [gatewayListSignature, ok(JSON.stringify(gateways))], - ]); + const { commandExists = () => false, run = () => ok(), ...overrides } = deps; return runUninstallPlan( { assumeYes: true, deleteModels: false, keepOpenShell }, { env: test.env, existsSync: (target) => String(target).startsWith(test.root) && fs.existsSync(target), + isPortFree: () => true, isTty: false, - isPortFree, platform: "linux", resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ gatewayName, @@ -215,19 +129,13 @@ function uninstall( rmSync: fs.rmSync, runDocker: () => ok(), ...overrides, - // A scoped uninstall with no PID evidence may only conclude that the - // gateway is absent after a complete empty listener observation. - commandExists: (command) => - command === "openshell" || command === "lsof" || commandExists(command), - run: (command, args, options) => { - const signature = commandSignature(command, args); - const response = defaultResponses.get(signature) ?? run(command, args, options); - const defaultSystemdShow = - response.status === 0 && response.stdout === "" - ? defaultSystemdShows.get(signature) - : undefined; - return defaultSystemdShow ?? response; - }, + commandExists: (command) => command === "openshell" || commandExists(command), + run: (command, args, options) => + command === "openshell" && args[0] === "gateway" && args[1] === "list" + ? ok(JSON.stringify(gateways)) + : command === "systemctl" && args.includes("--property=MainPID") + ? ok("0\n") + : run(command, args, options), }, ); } @@ -314,8 +222,8 @@ describe("uninstall OpenShell gateway user service", () => { run: (command, args) => { calls.push([command, ...args]); gatewayStopped ||= command === "systemctl" && args.includes("disable"); - // Scoped cleanup must finish its OpenShell calls before disabling - // the unit. The disable intentionally omits --now. + // `systemctl disable --now` also stops the OpenShell gateway service, + // so every scoped `openshell` call fails once the unit is disabled. return command === "openshell" && gatewayStopped ? { status: 1, stdout: "", stderr: "gateway unreachable" } : ok(); @@ -340,382 +248,32 @@ describe("uninstall OpenShell gateway user service", () => { expect(result.exitCode).toBe(0); expect(deletedAt).toBeGreaterThanOrEqual(0); expect(disabledAt).toBeGreaterThan(deletedAt); - expect(calls[disabledAt]).toEqual([ - "systemctl", - "--user", - "disable", - NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE, - ]); - expect(calls[disabledAt]).not.toContain("--now"); expect(dockerCalls).toContainEqual(["rm", "-f", "sandbox-id"]); expect(fs.existsSync(servicePath)).toBe(false); }); - it("proves and SIGKILL-stops only the active managed service after scoped sandbox cleanup (#8663)", () => { + it("does not signal a scoped service whose sandbox namespace is unproven (#8663)", () => { const test = fixture(true); - test.env.LOGNAME = "gateway-owner"; const servicePath = writeManagedService(test); - writeSelectedSandboxRegistry(test, "my-assistant"); - const gatewayBin = `${test.home}/.local/bin/openshell-gateway`; - const mainPid = 41_101; + fs.writeFileSync(writeGatewayState(test), "[openshell.drivers.docker]\n"); const calls: string[][] = []; - const events: string[] = []; - let scopedOverrideLoaded = false; - let serviceStopped = false; - const isPortFree = vi.fn(() => serviceStopped); - const kill = vi.fn(() => true); - const readProcessExecutable = vi.fn(() => gatewayBin); - const readProcessStartIdentity = vi.fn(() => "boot-identity:12345"); - const daemonReload = vi - .fn<() => RunResult>() - .mockImplementationOnce(() => { - events.push("daemon-reload"); - const dropInPath = path.join(`${servicePath}.d`, "99-nemoclaw-scoped-uninstall.conf"); - expect(fs.readFileSync(dropInPath, "utf-8")).toBe( - "[Service]\nRestart=no\nKillSignal=SIGKILL\nKillMode=control-group\n", - ); - scopedOverrideLoaded = true; - return ok(); - }) - .mockImplementation(() => { - events.push("daemon-reload"); - return ok(); - }); - const runResponses = new Map([ - [ - commandSignature("openshell", ["sandbox", "delete", "my-assistant"]), - () => { - events.push("sandbox-delete"); - return ok(); - }, - ], - [ - systemctlShowSignature(NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE), - () => - ok( - managedSystemdShow(test, { - active: !serviceStopped, - effectiveScopedStop: scopedOverrideLoaded, - mainPid, - }), - ), - ], - [commandSignature("systemctl", ["--user", "daemon-reload"]), daemonReload], - [ - commandSignature("systemctl", [ - "--user", - "disable", - "--now", - NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE, - ]), - () => { - events.push("disable-now"); - expect(scopedOverrideLoaded).toBe(true); - serviceStopped = true; - return ok(); - }, - ], - [ - commandSignature("ps", ["-p", String(mainPid), "-o", "uid="]), - () => ok(`${String(CURRENT_UID)}\n`), - ], - [ - commandSignature("ps", ["-p", String(mainPid), "-o", "pid="]), - () => (serviceStopped ? { status: 1, stdout: "", stderr: "" } : ok(`${mainPid}\n`)), - ], - [ - commandSignature("lsof", ["-ti", ":8080", "-sTCP:LISTEN"]), - () => (serviceStopped ? ok() : ok(`${mainPid}\n`)), - ], - ]); const result = uninstall( test, false, { commandExists: (command) => command === "systemctl", - isPortFree, - kill, - readProcessExecutable, - readProcessStartIdentity, - run: runFromResponses(runResponses, calls), - }, - [{ name: "nemoclaw" }, { name: "nemoclaw-8081" }], - ); - - expect(result.exitCode).toBe(0); - expect(events.slice(0, 3)).toEqual(["sandbox-delete", "daemon-reload", "disable-now"]); - expect(calls).toContainEqual([ - "systemctl", - "--user", - "disable", - "--now", - NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE, - ]); - expect(readProcessExecutable).toHaveBeenCalledTimes(2); - expect(readProcessExecutable).toHaveBeenNthCalledWith(1, mainPid); - expect(readProcessExecutable).toHaveBeenNthCalledWith(2, mainPid); - expect(readProcessStartIdentity).toHaveBeenCalledTimes(2); - expect(readProcessStartIdentity).toHaveBeenNthCalledWith(1, mainPid); - expect(readProcessStartIdentity).toHaveBeenNthCalledWith(2, mainPid); - expect(serviceStopped).toBe(true); - expect(isPortFree).toHaveBeenCalledWith(8080); - expect( - calls.some( - (call) => - call[0] === "ps" && - call[1] === "-p" && - call[2] === String(mainPid) && - call.at(-1) === "pid=", - ), - ).toBe(true); - expect(fs.existsSync(servicePath)).toBe(false); - expect(fs.existsSync(`${servicePath}.d`)).toBe(false); - expect(kill).not.toHaveBeenCalled(); - expect(calls.some((call) => call[0] === "pgrep")).toBe(false); - }); - - it("SIGKILL-stops but preserves a trusted package-owned gateway service (#8663)", () => { - const test = fixture(true); - test.env.LOGNAME = "gateway-owner"; - writeSelectedSandboxRegistry(test, "my-assistant"); - const servicePath = "/usr/lib/systemd/user/openshell-gateway.service"; - const gatewayBin = "/usr/bin/openshell-gateway"; - const mainPid = 41_301; - const dropInPath = path.join( - getOpenShellUserConfigHome(test.home, test.env), - "systemd", - "user", - "openshell-gateway.service.d", - "99-nemoclaw-scoped-uninstall.conf", - ); - const calls: string[][] = []; - let scopedOverrideLoaded = false; - let serviceStopped = false; - const runResponses = new Map([ - [ - systemctlShowSignature("openshell-gateway"), - () => - ok( - managedSystemdShow(test, { - active: !serviceStopped, - effectiveScopedStop: scopedOverrideLoaded, - execStartPath: gatewayBin, - fragmentPath: servicePath, - mainPid, - }), - ), - ], - [ - commandSignature("systemctl", ["--user", "daemon-reload"]), - () => { - scopedOverrideLoaded = fs.existsSync(dropInPath); - return ok(); - }, - ], - [ - commandSignature("systemctl", ["--user", "stop", "openshell-gateway"]), - () => { - expect(scopedOverrideLoaded).toBe(true); - serviceStopped = true; - return ok(); - }, - ], - [ - commandSignature("ps", ["-p", String(mainPid), "-o", "uid="]), - () => ok(`${String(CURRENT_UID)}\n`), - ], - [ - commandSignature("ps", ["-p", String(mainPid), "-o", "pid="]), - () => (serviceStopped ? { status: 1, stdout: "", stderr: "" } : ok(`${mainPid}\n`)), - ], - [ - commandSignature("lsof", ["-ti", ":8080", "-sTCP:LISTEN"]), - () => (serviceStopped ? ok() : ok(`${mainPid}\n`)), - ], - ]); - - const result = uninstall( - test, - false, - { - commandExists: (command) => command === "systemctl", - existsSync: (target) => - target === servicePath || - (String(target).startsWith(test.root) && fs.existsSync(String(target))), - isPortFree: () => serviceStopped, - readProcessExecutable: () => gatewayBin, - readProcessStartIdentity: () => "boot-identity:package-service", - run: runFromResponses(runResponses, calls), - }, - [{ name: "nemoclaw" }, { name: "nemoclaw-8081" }], - ); - - expect(result.exitCode).toBe(0); - expect(calls).toContainEqual(["systemctl", "--user", "stop", "openshell-gateway"]); - expect(calls.some((call) => call[0] === "systemctl" && call.includes("disable"))).toBe(false); - expect(calls.some((call) => call[0] === "pgrep")).toBe(false); - expect(serviceStopped).toBe(true); - expect(fs.existsSync(dropInPath)).toBe(false); - }); - - it("removes an owned scoped-stop override when retrying inactive service cleanup (#8663)", () => { - const test = fixture(true); - test.env.LOGNAME = "gateway-owner"; - const servicePath = writeManagedService(test); - writeSelectedSandboxRegistry(test, "my-assistant"); - const gatewayBin = `${test.home}/.local/bin/openshell-gateway`; - const mainPid = 41_302; - const dropInPath = path.join(`${servicePath}.d`, "99-nemoclaw-scoped-uninstall.conf"); - let failFirstDropInRemoval = true; - let scopedOverrideLoaded = false; - let serviceStopped = false; - const calls: string[][] = []; - const runResponses = new Map([ - [ - systemctlShowSignature(NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE), - () => - ok( - managedSystemdShow(test, { - active: !serviceStopped, - effectiveScopedStop: scopedOverrideLoaded, - mainPid, - }), - ), - ], - [ - commandSignature("systemctl", ["--user", "daemon-reload"]), - () => { - scopedOverrideLoaded = fs.existsSync(dropInPath); - return ok(); - }, - ], - [ - commandSignature("systemctl", [ - "--user", - "disable", - "--now", - NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE, - ]), - () => { - serviceStopped = true; + run: (command, args) => { + calls.push([command, ...args]); return ok(); }, - ], - [ - commandSignature("ps", ["-p", String(mainPid), "-o", "uid="]), - () => ok(`${String(CURRENT_UID)}\n`), - ], - [ - commandSignature("ps", ["-p", String(mainPid), "-o", "pid="]), - () => (serviceStopped ? { status: 1, stdout: "", stderr: "" } : ok(`${mainPid}\n`)), - ], - [ - commandSignature("lsof", ["-ti", ":8080", "-sTCP:LISTEN"]), - () => (serviceStopped ? ok() : ok(`${mainPid}\n`)), - ], - ]); - const run = runFromResponses(runResponses, calls); - const failDropInRemoval = (): never => { - throw new Error("injected drop-in removal failure"); - }; - const deps: Partial = { - commandExists: (command) => command === "systemctl", - isPortFree: () => serviceStopped, - readProcessExecutable: () => gatewayBin, - readProcessStartIdentity: () => "boot-identity:retry", - rmSync: (target, options) => { - const failThisRemoval = String(target) === dropInPath && failFirstDropInRemoval; - failFirstDropInRemoval = failFirstDropInRemoval && !failThisRemoval; - return failThisRemoval ? failDropInRemoval() : fs.rmSync(target, options); - }, - run, - }; - - const first = uninstall(test, false, deps, [{ name: "nemoclaw" }, { name: "nemoclaw-8081" }]); - - expect(first.exitCode).toBe(1); - expect(serviceStopped).toBe(true); - expect(fs.existsSync(servicePath)).toBe(true); - expect(fs.existsSync(dropInPath)).toBe(true); - - const retry = uninstall(test, false, deps, [{ name: "nemoclaw" }, { name: "nemoclaw-8081" }]); - - expect(retry.exitCode).toBe(0); - expect(fs.existsSync(servicePath)).toBe(false); - expect(fs.existsSync(dropInPath)).toBe(false); - }); - - it.each([ - { - label: "the port-8080 listener belongs to a sibling MainPID", - listenerPid: 41_202, - }, - { - fragmentPath: "/tmp/foreign-openshell-gateway.service", - label: "the loaded unit fragment is ambiguous", - listenerPid: 41_201, - }, - { - label: "the managed process owner differs from the current user", - listenerPid: 41_201, - processUid: CURRENT_UID + 1, - }, - ])("fails closed and preserves the active managed unit when $label (#8663)", (identity) => { - const test = fixture(true); - test.env.LOGNAME = "gateway-owner"; - const servicePath = writeManagedService(test); - writeSelectedSandboxRegistry(test, "my-assistant"); - const gatewayBin = `${test.home}/.local/bin/openshell-gateway`; - const mainPid = 41_201; - const calls: string[][] = []; - const kill = vi.fn(() => true); - const runResponses = new Map([ - [ - systemctlShowSignature(NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE), - () => - ok( - managedSystemdShow(test, { - active: true, - fragmentPath: identity.fragmentPath, - mainPid, - }), - ), - ], - [ - commandSignature("ps", ["-p", String(mainPid), "-o", "uid="]), - () => ok(`${String(identity.processUid ?? CURRENT_UID)}\n`), - ], - [ - commandSignature("lsof", ["-ti", ":8080", "-sTCP:LISTEN"]), - () => ok(`${identity.listenerPid}\n`), - ], - ]); - - const result = uninstall( - test, - false, - { - commandExists: (command) => command === "systemctl", - isPortFree: () => false, - kill, - readProcessExecutable: () => gatewayBin, - readProcessStartIdentity: () => "boot-identity:12345", - run: runFromResponses(runResponses, calls), }, [{ name: "nemoclaw" }, { name: "nemoclaw-8081" }], ); expect(result.exitCode).toBe(1); - expect(calls).toContainEqual(["openshell", "sandbox", "delete", "my-assistant"]); - expect(calls.some((call) => call[0] === "systemctl" && call.includes("disable"))).toBe(false); - expect(calls.some((call) => call[0] === "systemctl" && call.includes("daemon-reload"))).toBe( - false, - ); expect(fs.existsSync(servicePath)).toBe(true); - expect(fs.existsSync(`${servicePath}.d`)).toBe(false); - expect(kill).not.toHaveBeenCalled(); - expect(calls.some((call) => call[0] === "pgrep")).toBe(false); + expect(calls.some(([command]) => command === "systemctl")).toBe(false); }); it("preserves the marked Linux unit when scoped sandbox deletion fails (#8220)", () => { diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index eb10dcb6ccb..e5808f3940b 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -47,12 +47,9 @@ import { import { buildDockerGatewayDebEnvFile } from "../../onboard/docker-driver-gateway-env"; import { getNemoclawOpenShellGatewayUserServicePath, - getOpenShellGatewayUserServiceBinaryPaths, - getOpenShellGatewayUserServicePaths, getOpenShellUserConfigHome, NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE, NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER_LINE, - OPENSHELL_GATEWAY_USER_SERVICE, } from "../../onboard/docker-driver-gateway-service"; import { resolveGatewayName, resolveGatewayPortFromName } from "../../onboard/gateway-binding"; import { isExternallySupervised } from "../../onboard/gateway-ownership"; @@ -61,7 +58,8 @@ import { resolveGatewayTeardownAuthority, } from "../../onboard/gateway-teardown-authority"; import { - isHostPortFree, + hasStateScopedSandboxNamespace, + processUsesStateScopedSandboxNamespace, type StopHostGatewayOptions, stopHostGatewayProcesses, } from "../../onboard/host-gateway-process"; @@ -102,7 +100,6 @@ export interface UninstallRunDeps { error?: (message: string) => void; existsSync?: (target: string) => boolean; fs?: FileSystemDeps; - /** Test seam for confirming a scoped gateway released only its selected listener. */ isPortFree?: (port: number) => boolean; isTty?: boolean; kill?: (pid: number, signal?: NodeJS.Signals | number) => boolean; @@ -110,9 +107,7 @@ export interface UninstallRunDeps { openRegularFile?: typeof openRegularFileNoFollow; platform?: NodeJS.Platform; readProcessArgv?: (pid: number) => readonly string[] | null; - readProcessExecutable?: (pid: number) => string | null; readProcessEnvironment?: (pid: number) => Record | null; - readProcessStartIdentity?: (pid: number) => string | null; readLine?: () => string | null; requireCompleteGatewayProcessCleanup?: boolean; resolveGatewayTeardownAuthority?: GatewayTeardownAuthorityResolver; @@ -422,9 +417,7 @@ interface UninstallRuntime { openRegularFile: typeof openRegularFileNoFollow; platform: NodeJS.Platform; readProcessArgv: ((pid: number) => readonly string[] | null) | undefined; - readProcessExecutable: ((pid: number) => string | null) | undefined; readProcessEnvironment: ((pid: number) => Record | null) | undefined; - readProcessStartIdentity: ((pid: number) => string | null) | undefined; readLine: () => string | null; requireCompleteGatewayProcessCleanup: boolean; resolveGatewayTeardownAuthority: GatewayTeardownAuthorityResolver; @@ -463,9 +456,7 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime { openRegularFile: deps.openRegularFile ?? openRegularFileNoFollow, platform: deps.platform ?? process.platform, readProcessArgv: deps.readProcessArgv, - readProcessExecutable: deps.readProcessExecutable, readProcessEnvironment: deps.readProcessEnvironment, - readProcessStartIdentity: deps.readProcessStartIdentity, readLine: deps.readLine ?? readLineFromStdin, requireCompleteGatewayProcessCleanup: deps.requireCompleteGatewayProcessCleanup ?? false, resolveGatewayTeardownAuthority: @@ -785,14 +776,6 @@ function pidOwnedByCurrentUser(pid: number, runtime: UninstallRuntime): boolean return result.status === 0 && result.stdout.trim() === expected; } -function scopedServicePidOwnedByCurrentUser(pid: number, runtime: UninstallRuntime): boolean { - if (typeof process.getuid !== "function") return false; - const result = runtime.run("ps", ["-p", String(pid), "-o", "uid="], { env: runtime.env }); - if (result.status !== 0) return false; - const uid = Number.parseInt(result.stdout.trim(), 10); - return Number.isSafeInteger(uid) && uid === process.getuid(); -} - function tryStopOllamaProxyPid(pid: number, runtime: UninstallRuntime): boolean { // `runtime.kill()` only confirms the signal was sent; the proxy may ignore // SIGTERM, take time to clean up, or linger as a zombie. Verify the PID is @@ -987,7 +970,10 @@ function stopOrphanedOpenShell(runtime: UninstallRuntime): void { } } -function removeNemoclawOpenShellGatewayUserService(runtime: UninstallRuntime): boolean { +function removeNemoclawOpenShellGatewayUserService( + runtime: UninstallRuntime, + scopedStateDir?: string, +): boolean { if (runtime.platform !== "linux") return true; const servicePath = getNemoclawOpenShellGatewayUserServicePath( runtime.env.HOME || os.homedir(), @@ -1025,6 +1011,33 @@ function removeNemoclawOpenShellGatewayUserService(runtime: UninstallRuntime): b const hasSystemctl = runtime.commandExists("systemctl"); if (hasSystemctl) { + if (scopedStateDir !== undefined) { + const inspected = runtime.run( + "systemctl", + [ + "--user", + "show", + NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE, + "--property=MainPID", + "--value", + ], + { env: runtime.env }, + ); + const mainPid = Number(inspected.stdout.trim()); + if ( + !hasStateScopedSandboxNamespace(scopedStateDir) || + inspected.status !== 0 || + !inspected.stdout.trim() || + !Number.isSafeInteger(mainPid) || + mainPid < 0 || + (mainPid > 0 && !processUsesStateScopedSandboxNamespace(mainPid, scopedStateDir, runtime)) + ) { + runtime.warn( + "Refusing scoped gateway service stop because its loaded sandbox namespace cannot be proven.", + ); + return false; + } + } const disabled = runtime.run( "systemctl", ["--user", "disable", "--now", NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE], @@ -1060,549 +1073,6 @@ function removeNemoclawOpenShellGatewayUserService(runtime: UninstallRuntime): b return true; } -const SCOPED_GATEWAY_STOP_DROP_IN = "99-nemoclaw-scoped-uninstall.conf"; -const SCOPED_GATEWAY_STOP_DROP_IN_CONTENT = `[Service] -Restart=no -KillSignal=SIGKILL -KillMode=control-group -`; - -type ScopedGatewayServiceIdentity = { - active: boolean; - execStartPath: string; - fragmentPath: string; - killMode: string; - killSignal: string; - mainPid: number; - restart: string; -}; - -type ScopedGatewayServiceTarget = { - removeUnit: boolean; - serviceName: string; - trustedBinaryPaths: readonly string[]; - trustedUnitPaths: readonly string[]; -}; - -function parseSystemctlProperties(output: string): Record { - return Object.fromEntries( - output - .split(/\r?\n/) - .map((line) => { - const separator = line.indexOf("="); - return separator > 0 ? [line.slice(0, separator), line.slice(separator + 1).trim()] : null; - }) - .filter((entry): entry is [string, string] => entry !== null), - ); -} - -function systemdExecStartPath(value: string): string | null { - const candidate = /(?:^|[\s;])path=([^\s;]+)/.exec(value)?.[1]?.trim(); - return candidate && path.isAbsolute(candidate) ? path.normalize(candidate) : null; -} - -function declaredGatewayServiceBinary(contents: string): string | null { - const values = contents - .split(/\r?\n/) - .map((line) => /^ExecStart=(\S+)$/.exec(line.trim())?.[1] ?? null) - .filter((value): value is string => value !== null); - return values.length === 1 && path.isAbsolute(values[0]) ? path.normalize(values[0]) : null; -} - -function isTrustedManagedGatewayServiceBinary( - binaryPath: string, - runtime: UninstallRuntime, -): boolean { - const home = runtime.env.HOME || os.homedir(); - const configuredBinHome = runtime.env.XDG_BIN_HOME?.trim(); - const userBinHome = - configuredBinHome && path.isAbsolute(configuredBinHome) - ? path.normalize(configuredBinHome) - : path.join(home, ".local", "bin"); - return [ - path.join(userBinHome, "openshell-gateway"), - "/usr/local/bin/openshell-gateway", - "/usr/bin/openshell-gateway", - ].some((candidate) => path.normalize(candidate) === binaryPath); -} - -function inspectScopedGatewayService( - runtime: UninstallRuntime, - target: ScopedGatewayServiceTarget, -): ScopedGatewayServiceIdentity | null { - const result = runtime.run( - "systemctl", - [ - "--user", - "show", - target.serviceName, - "--property=FragmentPath", - "--property=ExecStart", - "--property=ExecStop", - "--property=ExecStopPost", - "--property=ActiveState", - "--property=MainPID", - "--property=Restart", - "--property=KillSignal", - "--property=KillMode", - ], - { env: runtime.env }, - ); - if (result.status !== 0) return null; - const properties = parseSystemctlProperties(result.stdout); - const fragmentPath = path.normalize(properties.FragmentPath ?? ""); - const execStartPath = systemdExecStartPath(properties.ExecStart ?? ""); - const mainPid = Number(properties.MainPID); - const activeState = properties.ActiveState; - if ( - !target.trustedUnitPaths.some((candidate) => path.normalize(candidate) === fragmentPath) || - !execStartPath || - !target.trustedBinaryPaths.some((candidate) => path.normalize(candidate) === execStartPath) || - (properties.ExecStop ?? "") !== "" || - (properties.ExecStopPost ?? "") !== "" || - (activeState !== "active" && activeState !== "inactive" && activeState !== "failed") || - !Number.isSafeInteger(mainPid) || - mainPid < 0 || - (activeState === "active" ? mainPid <= 0 : mainPid !== 0) - ) { - return null; - } - return { - active: activeState === "active", - execStartPath, - fragmentPath, - killMode: properties.KillMode ?? "", - killSignal: properties.KillSignal ?? "", - mainPid, - restart: properties.Restart ?? "", - }; -} - -function runtimeProcessExecutable(pid: number, runtime: UninstallRuntime): string | null { - if (runtime.readProcessExecutable) return runtime.readProcessExecutable(pid); - try { - return fs.realpathSync.native(`/proc/${String(pid)}/exe`); - } catch { - const result = runtime.run("lsof", ["-a", "-p", String(pid), "-d", "txt", "-Fn"], { - env: runtime.env, - }); - const executable = - result.status === 0 - ? result.stdout - .split(/\r?\n/) - .find((line) => line.startsWith("n/") && line.length > 2) - ?.slice(1) - : undefined; - return executable ?? null; - } -} - -function runtimeProcessStartIdentity(pid: number, runtime: UninstallRuntime): string | null { - if (runtime.readProcessStartIdentity) return runtime.readProcessStartIdentity(pid); - try { - const stat = fs.readFileSync(`/proc/${String(pid)}/stat`, "utf-8"); - const commandEnd = stat.lastIndexOf(")"); - if (commandEnd < 0) return null; - return ( - stat - .slice(commandEnd + 1) - .trim() - .split(/\s+/)[19] ?? null - ); - } catch { - const result = runtime.run("ps", ["-p", String(pid), "-o", "lstart="], { - env: runtime.env, - }); - return result.status === 0 && result.stdout.trim() ? result.stdout.trim() : null; - } -} - -function scopedGatewayListenerPids(runtime: UninstallRuntime, port: number): number[] | null { - if (!runtime.commandExists("lsof")) return null; - const result = runtime.run("lsof", ["-ti", `:${String(port)}`, "-sTCP:LISTEN"], { - env: runtime.env, - }); - if (result.status !== 0 && result.status !== 1) return null; - return [ - ...new Set( - splitNonEmptyLines(result.stdout) - .map((line) => Number.parseInt(line, 10)) - .filter((pid) => Number.isSafeInteger(pid) && pid > 0), - ), - ]; -} - -function scopedGatewayPortFree(runtime: UninstallRuntime, port: number): boolean { - return (runtime.isPortFree ?? isHostPortFree)(port); -} - -function normalizedExecutablePath(value: string): string { - try { - return fs.realpathSync.native(value); - } catch { - return path.normalize(value); - } -} - -type ScopedGatewayStopDropIn = { dir: string; path: string }; - -function scopedGatewayStopDropIn( - runtime: UninstallRuntime, - serviceName: string, -): ScopedGatewayStopDropIn { - const home = runtime.env.HOME || os.homedir(); - const userUnitDir = path.join(getOpenShellUserConfigHome(home, runtime.env), "systemd", "user"); - const dir = path.join(userUnitDir, `${serviceName}.service.d`); - return { dir, path: path.join(dir, SCOPED_GATEWAY_STOP_DROP_IN) }; -} - -function existingScopedGatewayStopDropIn( - runtime: UninstallRuntime, - serviceName: string, -): ScopedGatewayStopDropIn | null | false { - const dropIn = scopedGatewayStopDropIn(runtime, serviceName); - if (!fs.existsSync(dropIn.path)) return null; - try { - const currentUid = typeof process.getuid === "function" ? process.getuid() : null; - const dirStat = fs.lstatSync(dropIn.dir); - const fileStat = fs.lstatSync(dropIn.path); - if ( - dirStat.isSymbolicLink() || - !dirStat.isDirectory() || - fileStat.isSymbolicLink() || - !fileStat.isFile() || - (currentUid !== null && (dirStat.uid !== currentUid || fileStat.uid !== currentUid)) - ) { - return false; - } - const existing = runtime.openRegularFile(dropIn.path); - try { - return existing.readUtf8() === SCOPED_GATEWAY_STOP_DROP_IN_CONTENT ? dropIn : false; - } finally { - existing.close(); - } - } catch { - return false; - } -} - -function removeScopedGatewayStopDropIn( - runtime: UninstallRuntime, - dropInPath: string, - dropInDir: string, -): boolean { - try { - runtime.rmSync(dropInPath, { force: true }); - if (fs.existsSync(dropInDir) && fs.readdirSync(dropInDir).length === 0) { - runtime.rmSync(dropInDir, { force: true, recursive: true }); - } - return true; - } catch { - runtime.warn(`Failed to remove temporary scoped gateway service override ${dropInPath}`); - return false; - } -} - -function rollbackScopedGatewayStopDropIn( - runtime: UninstallRuntime, - dropIn: ScopedGatewayStopDropIn, -): boolean { - const removed = removeScopedGatewayStopDropIn(runtime, dropIn.path, dropIn.dir); - const reloaded = runtime.run("systemctl", ["--user", "daemon-reload"], { - env: runtime.env, - stdio: "ignore", - }); - if (!removed || reloaded.status !== 0) { - runtime.warn("Failed to roll back the temporary scoped gateway service override."); - return false; - } - return true; -} - -function installScopedGatewayStopDropIn( - runtime: UninstallRuntime, - serviceName: string, -): ScopedGatewayStopDropIn | null { - const dropIn = scopedGatewayStopDropIn(runtime, serviceName); - try { - if (fs.existsSync(dropIn.dir)) { - const stat = fs.lstatSync(dropIn.dir); - const currentUid = typeof process.getuid === "function" ? process.getuid() : null; - if ( - stat.isSymbolicLink() || - !stat.isDirectory() || - (currentUid !== null && stat.uid !== currentUid) - ) { - return null; - } - } else { - fs.mkdirSync(dropIn.dir, { mode: 0o700, recursive: true }); - } - const existing = existingScopedGatewayStopDropIn(runtime, serviceName); - if (existing === false) return null; - if (existing) return existing; - const created = runtime.openRegularFile(dropIn.path, { - create: true, - mode: 0o600, - writable: true, - }); - try { - created.replaceUtf8(SCOPED_GATEWAY_STOP_DROP_IN_CONTENT, 0o600); - } finally { - created.close(); - } - return existingScopedGatewayStopDropIn(runtime, serviceName) || null; - } catch { - return null; - } -} - -function removeManagedDefaultGatewayUserServiceScoped( - runtime: UninstallRuntime, - options: UninstallRunOptions, - externallySupervised: boolean, -): boolean { - if ( - options.keepOpenShell || - externallySupervised || - GATEWAY_PORT !== DEFAULT_GATEWAY_PORT || - runtime.platform !== "linux" - ) { - return true; - } - const nemoclawServicePath = getNemoclawOpenShellGatewayUserServicePath( - runtime.env.HOME || os.homedir(), - runtime.env, - ); - let target: ScopedGatewayServiceTarget | null = null; - if (runtime.existsSync(nemoclawServicePath)) { - let serviceContents: string; - try { - const service = runtime.openRegularFile(nemoclawServicePath); - try { - serviceContents = service.readUtf8(); - } finally { - service.close(); - } - } catch { - runtime.warn( - `Failed to validate ${nemoclawServicePath}; leaving gateway user service in place.`, - ); - return false; - } - if ( - !serviceContents - .split(/\r?\n/) - .some((line) => line.trimEnd() === NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER_LINE) - ) { - runtime.warn(`Leaving ${nemoclawServicePath} in place because it is not NemoClaw-managed.`); - return false; - } - const declaredBinary = declaredGatewayServiceBinary(serviceContents); - if (!declaredBinary || !isTrustedManagedGatewayServiceBinary(declaredBinary, runtime)) { - runtime.warn("The managed gateway service executable is not trusted; leaving it running."); - return false; - } - target = { - removeUnit: true, - serviceName: NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE, - trustedBinaryPaths: [declaredBinary], - trustedUnitPaths: [nemoclawServicePath], - }; - } else if ( - getOpenShellGatewayUserServicePaths().some((candidate) => runtime.existsSync(candidate)) - ) { - target = { - removeUnit: false, - serviceName: OPENSHELL_GATEWAY_USER_SERVICE, - trustedBinaryPaths: getOpenShellGatewayUserServiceBinaryPaths(), - trustedUnitPaths: getOpenShellGatewayUserServicePaths(), - }; - } - if (!target) return true; - if (!runtime.commandExists("systemctl")) { - runtime.warn("systemctl not found; cannot safely stop the scoped managed gateway service."); - return false; - } - const before = inspectScopedGatewayService(runtime, target); - if (!before) { - runtime.warn( - "Could not prove the scoped managed gateway service identity; leaving it running.", - ); - return false; - } - if (!before.active) { - if (target.removeUnit) { - // A staged NemoClaw unit can be inactive while onboarding has fallen - // back to the standalone per-gateway process. Disable only that unit; - // the PID/marker proof in the caller owns any live listener teardown. - const disabled = runtime.run("systemctl", ["--user", "disable", target.serviceName], { - env: runtime.env, - stdio: "ignore", - }); - const stillInactive = inspectScopedGatewayService(runtime, target); - if ( - disabled.status !== 0 || - !stillInactive || - stillInactive.active || - stillInactive.fragmentPath !== before.fragmentPath || - stillInactive.execStartPath !== before.execStartPath - ) { - runtime.warn("The inactive managed gateway service changed while disabling it."); - return false; - } - } - const leftoverDropIn = existingScopedGatewayStopDropIn(runtime, target.serviceName); - if (leftoverDropIn === false) { - runtime.warn("The temporary scoped gateway service override is not safely owned."); - return false; - } - if ( - leftoverDropIn && - !removeScopedGatewayStopDropIn(runtime, leftoverDropIn.path, leftoverDropIn.dir) - ) { - return false; - } - if (target.removeUnit) { - try { - runtime.rmSync(before.fragmentPath, { force: true }); - } catch { - runtime.run("systemctl", ["--user", "daemon-reload"], { - env: runtime.env, - stdio: "ignore", - }); - runtime.warn(`Failed to remove ${before.fragmentPath}; leaving it in place.`); - return false; - } - } - const reload = runtime.run("systemctl", ["--user", "daemon-reload"], { - env: runtime.env, - stdio: "ignore", - }); - if (reload.status !== 0) { - runtime.warn("Failed to reload the user systemd manager."); - return false; - } - runtime.log( - target.removeUnit - ? `Disabled and removed ${target.serviceName}.service` - : `Preserved inactive package-owned ${target.serviceName}.service`, - ); - return true; - } - - const listenerPids = scopedGatewayListenerPids(runtime, DEFAULT_GATEWAY_PORT); - const processExecutable = runtimeProcessExecutable(before.mainPid, runtime); - const startIdentity = runtimeProcessStartIdentity(before.mainPid, runtime); - if ( - !scopedServicePidOwnedByCurrentUser(before.mainPid, runtime) || - !processExecutable || - processExecutable.endsWith(" (deleted)") || - normalizedExecutablePath(processExecutable) !== - normalizedExecutablePath(before.execStartPath) || - !startIdentity || - listenerPids?.length !== 1 || - listenerPids[0] !== before.mainPid - ) { - runtime.warn( - "Could not prove the managed gateway process and listener ownership; leaving it running.", - ); - return false; - } - - const dropIn = installScopedGatewayStopDropIn(runtime, target.serviceName); - if (!dropIn) { - runtime.warn("Could not install the temporary scoped gateway SIGKILL override."); - return false; - } - const reload = runtime.run("systemctl", ["--user", "daemon-reload"], { - env: runtime.env, - stdio: "ignore", - }); - if (reload.status !== 0) { - rollbackScopedGatewayStopDropIn(runtime, dropIn); - runtime.warn("Failed to reload the user systemd manager for scoped gateway cleanup."); - return false; - } - const proven = inspectScopedGatewayService(runtime, target); - const listenerPidsAfterReload = scopedGatewayListenerPids(runtime, DEFAULT_GATEWAY_PORT); - const processExecutableAfterReload = runtimeProcessExecutable(before.mainPid, runtime); - if ( - !proven?.active || - proven.mainPid !== before.mainPid || - proven.fragmentPath !== before.fragmentPath || - proven.execStartPath !== before.execStartPath || - proven.restart !== "no" || - !["9", "KILL", "SIGKILL"].includes(proven.killSignal) || - proven.killMode !== "control-group" || - !scopedServicePidOwnedByCurrentUser(before.mainPid, runtime) || - !processExecutableAfterReload || - processExecutableAfterReload.endsWith(" (deleted)") || - normalizedExecutablePath(processExecutableAfterReload) !== - normalizedExecutablePath(before.execStartPath) || - runtimeProcessStartIdentity(before.mainPid, runtime) !== startIdentity || - listenerPidsAfterReload?.length !== 1 || - listenerPidsAfterReload[0] !== before.mainPid - ) { - rollbackScopedGatewayStopDropIn(runtime, dropIn); - runtime.warn( - "Managed gateway identity changed before the scoped service stop; leaving it running.", - ); - return false; - } - - const stopArgs = target.removeUnit - ? ["--user", "disable", "--now", target.serviceName] - : ["--user", "stop", target.serviceName]; - const stopped = runtime.run("systemctl", stopArgs, { env: runtime.env, stdio: "ignore" }); - if (stopped.status !== 0) { - rollbackScopedGatewayStopDropIn(runtime, dropIn); - runtime.warn(`Failed to stop ${target.serviceName}.service`); - return false; - } - const after = inspectScopedGatewayService(runtime, target); - if ( - !after || - after.active || - after.fragmentPath !== before.fragmentPath || - after.execStartPath !== before.execStartPath || - !waitForPidExit(before.mainPid, runtime, 1000) || - !scopedGatewayPortFree(runtime, DEFAULT_GATEWAY_PORT) - ) { - rollbackScopedGatewayStopDropIn(runtime, dropIn); - runtime.warn("Scoped managed gateway service stop did not release only its selected process."); - return false; - } - - try { - if (!removeScopedGatewayStopDropIn(runtime, dropIn.path, dropIn.dir)) return false; - if (target.removeUnit) runtime.rmSync(before.fragmentPath, { force: true }); - } catch { - runtime.run("systemctl", ["--user", "daemon-reload"], { - env: runtime.env, - stdio: "ignore", - }); - runtime.warn( - `Failed to finalize ${target.serviceName}.service; leaving remaining service state in place.`, - ); - return false; - } - const finalReload = runtime.run("systemctl", ["--user", "daemon-reload"], { - env: runtime.env, - stdio: "ignore", - }); - if (finalReload.status !== 0) { - runtime.warn("Failed to reload the user systemd manager."); - return false; - } - runtime.log( - target.removeUnit - ? `Stopped and removed ${target.serviceName}.service` - : `Stopped and preserved package-owned ${target.serviceName}.service`, - ); - return true; -} - // scripts/install.sh stages the NemoClaw-managed gateway user service only for // the default gateway port, so an uninstall run for any other port leaves it in // place. `--keep-openshell` and an externally supervised authority leave it in @@ -1611,10 +1081,13 @@ function removeManagedDefaultGatewayUserService( runtime: UninstallRuntime, options: UninstallRunOptions, externallySupervised: boolean, + selectedStateDir?: string, + scoped = false, ): boolean { - return options.keepOpenShell || externallySupervised || GATEWAY_PORT !== DEFAULT_GATEWAY_PORT - ? true - : removeNemoclawOpenShellGatewayUserService(runtime); + if (options.keepOpenShell || externallySupervised || GATEWAY_PORT !== DEFAULT_GATEWAY_PORT) { + return true; + } + return removeNemoclawOpenShellGatewayUserService(runtime, scoped ? selectedStateDir : undefined); } function removeNemoclawOpenShellGatewayEnv( @@ -2667,11 +2140,15 @@ function executePlan( return { ok: false }; } if (scopedToSelectedGateway && !options.keepOpenShell && !externallySupervised) { - // A marked default-port user service gets an exact, temporary - // systemd SIGKILL override. Standalone gateways use the PID/runtime - // ownership proof below. Neither path invokes OpenShell's shared - // graceful Docker cleanup. - if (!removeManagedDefaultGatewayUserServiceScoped(runtime, options, externallySupervised)) { + if ( + !removeManagedDefaultGatewayUserService( + runtime, + options, + externallySupervised, + paths.selectedGatewayLocalStateDir, + true, + ) + ) { return { ok: false }; } stopHostGatewayProcessesForUninstall(runtime, { @@ -2833,23 +2310,13 @@ function stopHostGatewayProcessesForUninstall( warn: runtime.warn, commandExists: runtime.commandExists, isPortFree: runtime.isPortFree, - readProcessExecutable: runtime.readProcessExecutable, readProcessEnvironment: runtime.readProcessEnvironment, - readProcessStartIdentity: runtime.readProcessStartIdentity, }, options, ); - const scopedIncomplete = - options.scopedGatewayStop === true && - (result.failed.length > 0 || - result.ownershipFailures.length > 0 || - result.skippedNonMatchingPids.length > 0); - if (scopedIncomplete) { - for (const failure of result.ownershipFailures) { - runtime.error(`Scoped gateway ownership check failed: ${failure}`); - } + if (options.scopedGatewayStop && (result.ownershipFailures?.length || result.failed.length)) { runtime.error( - "Cannot continue scoped uninstall because the selected gateway process was not proven and stopped.", + "Cannot prove ownership of or stop the selected host gateway process; retaining its runtime evidence.", ); throw new IncompleteHostGatewayCleanupError(); } diff --git a/src/lib/onboard/docker-driver-gateway-config.ts b/src/lib/onboard/docker-driver-gateway-config.ts index a38938225b5..191af0d3527 100644 --- a/src/lib/onboard/docker-driver-gateway-config.ts +++ b/src/lib/onboard/docker-driver-gateway-config.ts @@ -16,6 +16,7 @@ export { ensureDockerDriverGatewayJwtBundle } from "./docker-driver-gateway-jwt- // See docs/security/openshell-0.0.72-compatibility-review.mdx for the source-of-truth review. export const DOCKER_DRIVER_GATEWAY_CONFIG_NAME = "openshell-gateway.toml"; export const DOCKER_DRIVER_GATEWAY_JWT_TTL_SECS = 0; +export const NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV = "NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE"; function tomlString(value: string): string { return JSON.stringify(value); @@ -55,11 +56,54 @@ function cleanupStaleAtomicFileTemps(dir: string, basename: string): void { } } -function gatewayIdForStateDir(stateDir: string): string { +export function gatewayIdForStateDir(stateDir: string): string { const leaf = path.basename(path.resolve(stateDir)).replace(/[^A-Za-z0-9_.-]/g, "-"); return leaf ? `nemoclaw-${leaf}` : "nemoclaw"; } +/** Prove that a NemoClaw-owned gateway config uses its state-scoped namespace. */ +export function hasStateScopedSandboxNamespace(stateDir: string): boolean { + if (typeof process.getuid !== "function" || typeof fs.constants.O_NOFOLLOW !== "number") { + return false; + } + const configPath = path.join(stateDir, DOCKER_DRIVER_GATEWAY_CONFIG_NAME); + let descriptor: number | undefined; + try { + const state = fs.lstatSync(stateDir); + descriptor = fs.openSync(configPath, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + const config = fs.fstatSync(descriptor); + if ( + !state.isDirectory() || + state.isSymbolicLink() || + !config.isFile() || + config.nlink !== 1 || + state.uid !== process.getuid() || + config.uid !== state.uid || + config.size > 64 * 1024 + ) { + return false; + } + const expected = `sandbox_namespace = ${tomlString(gatewayIdForStateDir(stateDir))}`; + let inDriverTable = false; + const matches = fs + .readFileSync(descriptor, "utf-8") + .split(/\r?\n/) + .filter((line) => { + const trimmed = line.trim(); + if (trimmed.startsWith("[") && trimmed.endsWith("]")) { + inDriverTable = /^\[openshell\.drivers\.(?:docker|podman)\]$/.test(trimmed); + return false; + } + return inDriverTable && trimmed.startsWith("sandbox_namespace ="); + }); + return matches.length === 1 && matches[0]?.trim() === expected; + } catch { + return false; + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + } +} + function gatewayLocalTlsDir(gatewayEnv: Record): string { const localTlsDir = gatewayEnv.OPENSHELL_LOCAL_TLS_DIR?.trim(); if (!localTlsDir) { @@ -77,6 +121,7 @@ export function buildDockerDriverGatewayConfigToml( const driver = gatewayEnv.OPENSHELL_DRIVERS === "podman" ? "podman" : "docker"; const localTlsDir = jwtBundle ? gatewayLocalTlsDir(gatewayEnv) : undefined; const dockerEntries: [string, string | undefined][] = [ + ["sandbox_namespace", gatewayId], ["grpc_endpoint", gatewayEnv.OPENSHELL_GRPC_ENDPOINT], ["host_gateway_ip", driver === "podman" ? PORTABLE_HOST_GATEWAY_IP : undefined], ["socket_path", driver === "podman" ? gatewayEnv.OPENSHELL_PODMAN_SOCKET : undefined], @@ -169,5 +214,6 @@ export function prepareDockerDriverGatewayConfigEnv( gatewayEnv, sandboxBin, ); + gatewayEnv[NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV] = gatewayIdForStateDir(stateDir); return gatewayEnv; } diff --git a/src/lib/onboard/docker-driver-gateway-env.ts b/src/lib/onboard/docker-driver-gateway-env.ts index 573abe56f7d..f77ffd8bd8b 100644 --- a/src/lib/onboard/docker-driver-gateway-env.ts +++ b/src/lib/onboard/docker-driver-gateway-env.ts @@ -16,6 +16,7 @@ import { DEFAULT_GATEWAY_PORT, GATEWAY_PORT } from "../core/ports"; import { isSupportedGatewayDockerHost } from "../domain/docker-host"; import { DOCKER_DRIVER_GATEWAY_JWT_TTL_SECS, + NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV, prepareDockerDriverGatewayConfigEnv, } from "./docker-driver-gateway-config"; import { buildDockerDriverGatewayLocalTlsEnv } from "./docker-driver-gateway-local-tls"; @@ -52,6 +53,7 @@ export const DOCKER_DRIVER_GATEWAY_RUNTIME_ENV_KEYS = [ "OPENSHELL_GATEWAY_CONFIG", "OPENSHELL_VM_DRIVER_STATE_DIR", "OPENSHELL_DRIVER_DIR", + NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV, "NETAVARK_FW", ] as const; diff --git a/src/lib/onboard/docker-driver-gateway-launch.test.ts b/src/lib/onboard/docker-driver-gateway-launch.test.ts index e9a637cd048..41aa8e3f32f 100644 --- a/src/lib/onboard/docker-driver-gateway-launch.test.ts +++ b/src/lib/onboard/docker-driver-gateway-launch.test.ts @@ -6,7 +6,10 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; - +import { + gatewayIdForStateDir, + NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV, +} from "./docker-driver-gateway-config"; import { buildDockerDriverGatewayConfigToml, buildDockerDriverGatewayLaunch, @@ -100,12 +103,22 @@ describe("docker-driver-gateway-launch", () => { ); expect(toml).toContain('compute_drivers = ["docker"]'); + expect(toml).toContain('sandbox_namespace = "nemoclaw"'); expect(toml).toContain('grpc_endpoint = "https://127.0.0.1:8080"'); expect(toml).toContain('network_name = "openshell-docker"'); expect(toml).toContain('supervisor_image = "ghcr.io/nvidia/openshell/supervisor:0.0.44"'); expect(toml).toContain('supervisor_bin = "/home/shadeform/.local/bin/openshell-sandbox"'); }); + it("assigns different sandbox namespaces to different gateway state roots (#8663)", () => { + const defaultNamespace = gatewayIdForStateDir("/tmp/openshell-docker-gateway"); + const alternateNamespace = gatewayIdForStateDir("/tmp/openshell-docker-gateway-18080"); + + expect(defaultNamespace).toBe("nemoclaw-openshell-docker-gateway"); + expect(alternateNamespace).toBe("nemoclaw-openshell-docker-gateway-18080"); + expect(defaultNamespace).not.toBe(alternateNamespace); + }); + it("writes the exact rootless socket only for the Podman driver", () => { const toml = buildDockerDriverGatewayConfigToml({ OPENSHELL_DRIVERS: "podman", @@ -156,6 +169,9 @@ describe("docker-driver-gateway-launch", () => { expect(identity.launch?.mode).toBe("host"); expect(identity.driftGatewayBin).toBe(gatewayBin); expect(identity.desiredEnv.OPENSHELL_DOCKER_SUPERVISOR_BIN).toBe(sandboxBin); + expect(identity.desiredEnv[NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV]).toBe( + gatewayIdForStateDir(dir), + ); expect(identity.desiredEnv.OPENSHELL_GATEWAY_CONFIG).toBe( path.join(dir, "openshell-gateway.toml"), ); diff --git a/src/lib/onboard/docker-driver-gateway-launch.ts b/src/lib/onboard/docker-driver-gateway-launch.ts index 271875c8ee2..83ddd0ef0f4 100644 --- a/src/lib/onboard/docker-driver-gateway-launch.ts +++ b/src/lib/onboard/docker-driver-gateway-launch.ts @@ -12,6 +12,7 @@ import { } from "./docker-driver-gateway-compat"; import { buildDockerDriverGatewayConfigToml, + NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV, prepareDockerDriverGatewayConfigEnv, } from "./docker-driver-gateway-config"; import { @@ -177,6 +178,7 @@ export function buildDockerDriverGatewayRuntimeIdentity( ...Object.keys(options.gatewayEnv), "OPENSHELL_DOCKER_SUPERVISOR_BIN", "OPENSHELL_GATEWAY_CONFIG", + NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV, ]); const desiredEnv = Object.fromEntries( Object.entries(launch.env).filter( diff --git a/src/lib/onboard/docker-driver-gateway-prelaunch.test.ts b/src/lib/onboard/docker-driver-gateway-prelaunch.test.ts index 282c87bcfab..0f075d273d2 100644 --- a/src/lib/onboard/docker-driver-gateway-prelaunch.test.ts +++ b/src/lib/onboard/docker-driver-gateway-prelaunch.test.ts @@ -14,7 +14,6 @@ import type { StopHostGatewayOptions, StopHostGatewayResult } from "./host-gatew function emptyResult(overrides: Partial = {}): StopHostGatewayResult { return { failed: [], - ownershipFailures: [], skippedDeadPids: [], skippedNonMatchingPids: [], stopped: [], diff --git a/src/lib/onboard/docker-driver-gateway-prelaunch.ts b/src/lib/onboard/docker-driver-gateway-prelaunch.ts index 3f8abe920a4..1effffe9e3f 100644 --- a/src/lib/onboard/docker-driver-gateway-prelaunch.ts +++ b/src/lib/onboard/docker-driver-gateway-prelaunch.ts @@ -54,7 +54,6 @@ export interface ReapHostGatewayBeforeLaunchOptions { function emptyStopResult(): StopHostGatewayResult { return { failed: [], - ownershipFailures: [], skippedDeadPids: [], skippedNonMatchingPids: [], stopped: [], diff --git a/src/lib/onboard/gateway-process-identity.ts b/src/lib/onboard/gateway-process-identity.ts index 1aaf1fa95cb..835c5999598 100644 --- a/src/lib/onboard/gateway-process-identity.ts +++ b/src/lib/onboard/gateway-process-identity.ts @@ -14,7 +14,6 @@ import { export { buildOwnedHostGatewayArgv0, canonicalGatewayTargetMatches, - gatewayCompatContainerNameForPort, type OpenShellGatewayProcessTarget, } from "./gateway-process-target-identity"; diff --git a/src/lib/onboard/gateway-process-target-identity.ts b/src/lib/onboard/gateway-process-target-identity.ts index e78ec17caa1..2319172b4e2 100644 --- a/src/lib/onboard/gateway-process-target-identity.ts +++ b/src/lib/onboard/gateway-process-target-identity.ts @@ -46,11 +46,7 @@ export function gatewayTargetMatches( } export function canonicalGatewayTargetMatches(name: string, port: number): boolean { - return resolveGatewayName(port) === name; -} - -export function gatewayCompatContainerNameForPort(port: number): string { - return resolveGatewayCompatContainerName(port); + return resolveGatewayName(port) === name && resolveGatewayPortFromName(name) === port; } function cliFlagValue(tokens: string[], names: string[]): string | null { @@ -58,14 +54,9 @@ function cliFlagValue(tokens: string[], names: string[]): string | null { for (let index = 0; index < tokens.length; index += 1) { const token = tokens[index]; for (const name of names) { - if (token === name) { - const value = tokens[index + 1]; - if (!value) return null; - values.push(value); - } else if (token.startsWith(`${name}=`)) { - const value = token.slice(name.length + 1); - if (!value) return null; - values.push(value); + if (token === name && tokens[index + 1]) values.push(tokens[index + 1]); + else if (token.startsWith(`${name}=`) && token.length > name.length + 1) { + values.push(token.slice(name.length + 1)); } } } @@ -115,5 +106,5 @@ export function dockerCompatGatewayMatchesTarget( const port = Number(target.port); if (!Number.isInteger(port) || port < 1 || port > 65535) return false; if (target.name && target.name !== resolveGatewayName(port)) return false; - return cliFlagValue(tokens, ["--name"]) === gatewayCompatContainerNameForPort(port); + return cliFlagValue(tokens, ["--name"]) === resolveGatewayCompatContainerName(port); } diff --git a/src/lib/onboard/host-gateway-process-target.test.ts b/src/lib/onboard/host-gateway-process-target.test.ts index d9a951413ba..f395dd240a9 100644 --- a/src/lib/onboard/host-gateway-process-target.test.ts +++ b/src/lib/onboard/host-gateway-process-target.test.ts @@ -7,10 +7,14 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { + gatewayIdForStateDir, + NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV, +} from "./docker-driver-gateway-config"; +import { writeDockerDriverGatewayRuntimeMarkerForStateDir } from "./docker-driver-gateway-runtime-marker"; import { HOST_GATEWAY_PGREP_PATTERN, type HostGatewayProcessDeps, - hostGatewayCmdlineMatches, type RunResult, stopHostGatewayProcesses, } from "./host-gateway-process"; @@ -43,17 +47,95 @@ function makeRun(responses: Map): HostGatewayProcessDeps["r function psResponses( pid: number, opts: { - cmdline: string; + cmdline: string | (() => string); exited: Set; }, ): [string, RunResponse][] { return [ [`ps -p ${pid} -o pid=`, () => (opts.exited.has(pid) ? notFound() : ok(`${pid}\n`))], + [`ps -p ${pid} -o uid=`, staticResponse(ok(`${String(process.getuid?.() ?? 501)}\n`))], [`ps -p ${pid} -o user=`, staticResponse(ok("tester\n"))], - [`ps -p ${pid} -o args=`, staticResponse(ok(opts.cmdline))], + [ + `ps -p ${pid} -o args=`, + () => ok(typeof opts.cmdline === "function" ? opts.cmdline() : opts.cmdline), + ], ]; } +function stopScopedTarget( + overrides: { + cmdline?: string; + cmdlineAfterProof?: string; + markerPort?: number; + name?: string; + namespace?: string; + pidFilePid?: number; + port?: number; + } = {}, +) { + const selectedPid = 9_999_601; + const pid = overrides.pidFilePid ?? selectedPid; + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-scoped-target-")); + const pidFile = path.join(stateDir, "openshell-gateway.pid"); + fs.writeFileSync(pidFile, `${String(pid)}\n`); + fs.writeFileSync( + path.join(stateDir, "openshell-gateway.toml"), + `[openshell.drivers.docker]\nsandbox_namespace = "${gatewayIdForStateDir(stateDir)}"\n`, + ); + writeDockerDriverGatewayRuntimeMarkerForStateDir(stateDir, { + desiredEnv: {}, + endpoint: `https://127.0.0.1:${String(overrides.markerPort ?? 18080)}`, + pid: selectedPid, + }); + const exited = new Set(); + let cmdlineReads = 0; + const cmdline = overrides.cmdline ?? "openshell-gateway[nemoclaw=nemoclaw-18080;port=18080]"; + const run = vi.fn( + makeRun( + new Map([ + ...psResponses(pid, { + cmdline: () => { + cmdlineReads += 1; + return cmdlineReads > 1 && overrides.cmdlineAfterProof + ? overrides.cmdlineAfterProof + : cmdline; + }, + exited, + }), + ]), + ), + ); + const kill = vi.fn((killedPid, signal) => { + switch (signal) { + case "SIGTERM": + exited.add(killedPid); + break; + } + return true; + }); + const result = stopHostGatewayProcesses( + { + run, + kill, + env: {}, + isPortFree: () => true, + log: vi.fn(), + readProcessEnvironment: () => ({ + [NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV]: + overrides.namespace ?? gatewayIdForStateDir(stateDir), + }), + }, + { + openShellGatewayName: overrides.name ?? "nemoclaw-18080", + openShellGatewayPort: overrides.port ?? 18080, + scopedGatewayStop: true, + stateDir, + usePgrepFallback: false, + }, + ); + return { kill, pidFile, result, run }; +} + function stopTargetedPid(pid: number, cmdline: string, targeted = true) { const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-host-gateway-target-")); const pidFile = path.join(stateDir, "openshell-gateway.pid"); @@ -90,6 +172,44 @@ function stopTargetedPid(pid: number, cmdline: string, targeted = true) { } describe("stopHostGatewayProcesses target filtering", () => { + it("stops only the fully proven scoped PID without running pgrep (#8663)", () => { + const { kill, pidFile, result, run } = stopScopedTarget(); + + expect(result.stopped).toEqual([9_999_601]); + expect(result.ownershipFailures).toEqual([]); + expect(kill).toHaveBeenCalledWith(9_999_601, "SIGTERM"); + expect(run.mock.calls.some(([command]) => command === "pgrep")).toBe(false); + expect(fs.existsSync(pidFile)).toBe(false); + }); + + it.each([ + ["PID file", { pidFilePid: 9_999_602 }], + ["command line", { cmdline: "openshell-gateway[nemoclaw=nemoclaw;port=8080]" }], + ["gateway name", { name: "nemoclaw", port: 18080 }], + ["gateway port", { name: "nemoclaw", port: 8080 }], + ["loaded namespace", { namespace: "default" }], + ])("fails closed when a sibling cross-matches by %s (#8663)", (_case, overrides) => { + const { kill, pidFile, result } = stopScopedTarget(overrides); + + expect(result.stopped).toEqual([]); + expect(result.ownershipFailures?.length).toBe(1); + expect(kill).not.toHaveBeenCalled(); + expect(fs.existsSync(pidFile)).toBe(true); + }); + + it("revalidates process ownership immediately before signaling (#8663)", () => { + const { kill, pidFile, result } = stopScopedTarget({ + cmdlineAfterProof: "openshell-gateway[nemoclaw=nemoclaw;port=8080]", + }); + + expect(result.stopped).toEqual([]); + expect(result.ownershipFailures).toEqual([ + "PID 9999601: process ownership changed immediately before signaling", + ]); + expect(kill).not.toHaveBeenCalled(); + expect(fs.existsSync(pidFile)).toBe(true); + }); + it("accepts a matching OpenShell CLI gateway-start process for the cleanup target", () => { const { kill, pidFile, result } = stopTargetedPid( 9999553, @@ -183,40 +303,3 @@ describe("stopHostGatewayProcesses target filtering", () => { expect(fs.existsSync(pidFile)).toBe(false); }); }); - -describe("exact OpenShell gateway command-line identity (#8663)", () => { - const target = { name: "nemoclaw-18080", port: 18_080 }; - - it("accepts one exact gateway name and port", () => { - expect( - hostGatewayCmdlineMatches( - "/opt/openshell/openshell gateway start --name nemoclaw-18080 --port 18080", - null, - target, - { requireExpectedFlags: true }, - ), - ).toBe(true); - }); - - it.each([ - ["missing gateway name", "/opt/openshell/openshell gateway start --port 18080"], - ["missing gateway port", "/opt/openshell/openshell gateway start --name nemoclaw-18080"], - ["sibling gateway name", "/opt/openshell/openshell gateway start --name nemoclaw --port 18080"], - [ - "sibling gateway port", - "/opt/openshell/openshell gateway start --name nemoclaw-18080 --port 8080", - ], - [ - "duplicate gateway names", - "/opt/openshell/openshell gateway start --name nemoclaw-18080 --name nemoclaw --port 18080", - ], - [ - "duplicate gateway ports", - "/opt/openshell/openshell gateway start --name nemoclaw-18080 --port 18080 --port 8080", - ], - ])("rejects %s", (_case, cmdline) => { - expect(hostGatewayCmdlineMatches(cmdline, null, target, { requireExpectedFlags: true })).toBe( - false, - ); - }); -}); diff --git a/src/lib/onboard/host-gateway-process.test.ts b/src/lib/onboard/host-gateway-process.test.ts index 181989114ff..99b4a97ba49 100644 --- a/src/lib/onboard/host-gateway-process.test.ts +++ b/src/lib/onboard/host-gateway-process.test.ts @@ -7,10 +7,6 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; -import { - getDockerDriverGatewayRuntimeMarkerPath, - writeDockerDriverGatewayRuntimeMarkerForStateDir, -} from "./docker-driver-gateway-runtime-marker"; import { clearHostGatewayRuntimeFiles, HOST_GATEWAY_PGREP_PATTERN, @@ -72,185 +68,6 @@ function psResponses( ]; } -const CURRENT_UID = typeof process.getuid === "function" ? process.getuid() : 1_000; - -type ScopedGatewayFixtureOptions = { - cmdline?: string; - compatContainerPid?: number; - listenerPids?: readonly number[]; - markerPid?: number; - markerPort?: number; - omitMarker?: boolean; - omitPidFile?: boolean; - pidFilePid?: number; - portFree?: boolean; - processAlive?: boolean; - processExecutable?: string | null; - processUid?: number; - startIdentities?: readonly string[]; - usePgrepFallback?: boolean; -}; - -function scopedGatewayFixture(options: ScopedGatewayFixtureOptions = {}) { - const selectedPid = 9_991_880; - const siblingPid = 9_990_808; - const selectedPort = 18_080; - const selectedName = "nemoclaw-18080"; - const directGatewayBin = "/opt/openshell/openshell"; - const selectedCompatContainerName = "nemoclaw-openshell-gateway-18080"; - const selectedCompatContainerId = "a".repeat(64); - const siblingCompatContainerName = "nemoclaw-openshell-gateway"; - const selectedStateDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-host-gateway-scoped-selected-"), - ); - const siblingStateDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-host-gateway-scoped-sibling-"), - ); - const selectedPidFile = path.join(selectedStateDir, "openshell-gateway.pid"); - const siblingPidFile = path.join(siblingStateDir, "openshell-gateway.pid"); - const selectedCmdline = - options.cmdline ?? - `${directGatewayBin} gateway start --name ${selectedName} --port ${String(selectedPort)}\n`; - const compatibilityMode = selectedCmdline.includes("/opt/nemoclaw/openshell-gateway"); - const writeSelectedPidFile = options.omitPidFile - ? () => undefined - : () => fs.writeFileSync(selectedPidFile, `${String(options.pidFilePid ?? selectedPid)}\n`); - writeSelectedPidFile(); - fs.writeFileSync(siblingPidFile, `${String(siblingPid)}\n`); - const writeSelectedMarker = options.omitMarker - ? () => undefined - : () => - writeDockerDriverGatewayRuntimeMarkerForStateDir(selectedStateDir, { - desiredEnv: {}, - dockerHost: null, - endpoint: `https://127.0.0.1:${String(options.markerPort ?? selectedPort)}`, - gatewayBin: compatibilityMode ? null : directGatewayBin, - pid: options.markerPid ?? selectedPid, - }); - writeSelectedMarker(); - writeDockerDriverGatewayRuntimeMarkerForStateDir(siblingStateDir, { - desiredEnv: {}, - endpoint: "https://127.0.0.1:8080", - pid: siblingPid, - }); - - const recordedPid = options.pidFilePid ?? selectedPid; - const exited = new Set(options.processAlive === false ? [recordedPid] : []); - const compatContainerPid = options.compatContainerPid; - const compatibilityResponses: [string, RunResult | ((args: string[]) => RunResult)][] = - compatContainerPid === undefined - ? [] - : [ - [ - `docker inspect --type container ${selectedCompatContainerName}`, - ok( - `${JSON.stringify([ - { - Args: [], - HostConfig: { NetworkMode: "host" }, - Id: selectedCompatContainerId, - Name: `/${selectedCompatContainerName}`, - Path: "/opt/nemoclaw/openshell-gateway", - State: { Pid: compatContainerPid, Running: true }, - }, - ])}\n`, - ), - ], - [ - `docker rm -f ${selectedCompatContainerId}`, - () => { - exited.add(recordedPid); - return ok(`${selectedCompatContainerName}\n`); - }, - ], - ]; - const responses = new Map RunResult)>([ - // A host-wide fallback would discover both gateways. Scoped teardown must - // never execute this response. - [PGREP_KEY, ok(`${String(siblingPid)}\n${String(selectedPid)}\n`)], - [ - `ps -p ${String(recordedPid)} -o pid=`, - () => (exited.has(recordedPid) ? notFound() : ok(`${String(recordedPid)}\n`)), - ], - [`ps -p ${String(recordedPid)} -o uid=`, ok(`${String(options.processUid ?? CURRENT_UID)}\n`)], - [`ps -p ${String(recordedPid)} -o args=`, ok(selectedCmdline)], - [ - `lsof -ti :${String(selectedPort)} -sTCP:LISTEN`, - ok((options.listenerPids ?? [recordedPid]).map(String).join("\n") + "\n"), - ], - ...compatibilityResponses, - ]); - const { calls, run } = makeRun(responses); - const dockerAdapterCalls: Array<{ env: NodeJS.ProcessEnv | undefined; operation: string }> = []; - let startIdentityRead = 0; - const signalHandlers = new Map void>([ - ["SIGKILL", (pid) => void exited.add(pid)], - ]); - const kill = vi.fn((pid, signal) => { - signalHandlers.get(signal)?.(pid); - return true; - }); - - const result = stopHostGatewayProcesses( - { - run, - kill, - env: { USER: "tester" }, - commandExists: () => true, - dockerForceRm: (containerId, adapterOptions) => { - dockerAdapterCalls.push({ env: adapterOptions?.env, operation: "force-rm" }); - return run("docker", ["rm", "-f", containerId]); - }, - dockerInspect: (args, adapterOptions) => { - dockerAdapterCalls.push({ env: adapterOptions?.env, operation: "inspect" }); - return run("docker", ["inspect", ...args]); - }, - isPortFree: () => options.portFree ?? true, - log: vi.fn(), - readProcessExecutable: () => - options.processExecutable === undefined ? directGatewayBin : options.processExecutable, - readProcessEnvironment: () => ({}), - readProcessStartIdentity: (pid) => { - const identities = options.startIdentities ?? ["fixture-start-identity"]; - const identity = identities[Math.min(startIdentityRead, identities.length - 1)] ?? null; - startIdentityRead += 1; - return exited.has(pid) ? null : identity; - }, - }, - { - killWaitMs: 0, - gatewayBin: directGatewayBin, - openShellGatewayName: selectedName, - openShellGatewayPort: selectedPort, - pollIntervalMs: 0, - scopedGatewayStop: true, - stateDir: selectedStateDir, - usePgrepFallback: options.usePgrepFallback, - }, - ); - - return { - calls, - cleanup: () => { - fs.rmSync(selectedStateDir, { recursive: true, force: true }); - fs.rmSync(siblingStateDir, { recursive: true, force: true }); - }, - kill, - dockerAdapterCalls, - recordedPid, - result, - selectedPid, - selectedPidFile, - selectedCompatContainerName, - selectedCompatContainerId, - selectedRuntimeMarker: getDockerDriverGatewayRuntimeMarkerPath(selectedStateDir), - siblingCompatContainerName, - siblingPid, - siblingPidFile, - siblingRuntimeMarker: getDockerDriverGatewayRuntimeMarkerPath(siblingStateDir), - }; -} - describe("host gateway cleanup boundaries", () => { it.each([ ["free", 0, true], @@ -607,244 +424,3 @@ describe("stopHostGatewayProcesses", () => { expect(fs.existsSync(pidFile)).toBe(false); }); }); - -describe("scoped host gateway stop isolation (#8663)", () => { - it("stops only the selected gateway after proving its PID, marker, owner, command line, and listener", () => { - const fixture = scopedGatewayFixture(); - try { - expect(fixture.result).toMatchObject({ - failed: [], - ownershipFailures: [], - skippedNonMatchingPids: [], - stopped: [fixture.selectedPid], - }); - expect(fixture.kill.mock.calls).toEqual([[fixture.selectedPid, "SIGKILL"]]); - expect(fixture.kill).not.toHaveBeenCalledWith(fixture.siblingPid, expect.anything()); - expect(fixture.calls.filter(({ command }) => command === "pgrep")).toEqual([]); - expect(fs.existsSync(fixture.selectedPidFile)).toBe(false); - expect(fs.existsSync(fixture.selectedRuntimeMarker)).toBe(false); - expect(fs.readFileSync(fixture.siblingPidFile, "utf-8")).toBe( - `${String(fixture.siblingPid)}\n`, - ); - expect(fs.existsSync(fixture.siblingRuntimeMarker)).toBe(true); - } finally { - fixture.cleanup(); - } - }); - - it("removes only the selected per-port Docker compatibility container after correlating its listener", () => { - const compatContainerPid = 7_771_880; - const fixture = scopedGatewayFixture({ - cmdline: - "/usr/local/bin/docker run --rm --name nemoclaw-openshell-gateway-18080 --network host ubuntu:24.04 /opt/nemoclaw/openshell-gateway\n", - compatContainerPid, - listenerPids: [compatContainerPid], - }); - try { - expect(fixture.result).toMatchObject({ - failed: [], - ownershipFailures: [], - skippedNonMatchingPids: [], - stopped: [fixture.selectedPid], - }); - expect(fixture.kill).not.toHaveBeenCalled(); - const dockerCalls = fixture.calls.filter(({ command }) => command === "docker"); - expect(dockerCalls).toContainEqual({ - args: ["inspect", "--type", "container", fixture.selectedCompatContainerName], - command: "docker", - }); - expect(dockerCalls).toContainEqual({ - args: ["rm", "-f", fixture.selectedCompatContainerId], - command: "docker", - }); - expect(fixture.dockerAdapterCalls).toEqual([ - { env: { DOCKER_HOST: "unix:///var/run/docker.sock" }, operation: "inspect" }, - { env: { DOCKER_HOST: "unix:///var/run/docker.sock" }, operation: "inspect" }, - { env: { DOCKER_HOST: "unix:///var/run/docker.sock" }, operation: "force-rm" }, - ]); - expect( - dockerCalls.some(({ args }) => args.includes(fixture.siblingCompatContainerName)), - ).toBe(false); - expect(fs.existsSync(fixture.selectedPidFile)).toBe(false); - expect(fs.existsSync(fixture.selectedRuntimeMarker)).toBe(false); - expect(fs.existsSync(fixture.siblingPidFile)).toBe(true); - expect(fs.existsSync(fixture.siblingRuntimeMarker)).toBe(true); - } finally { - fixture.cleanup(); - } - }); - - it("preserves selected state when the Docker compatibility container does not own the selected listener", () => { - const compatContainerPid = 7_771_880; - const siblingContainerPid = 7_770_808; - const fixture = scopedGatewayFixture({ - cmdline: - "/usr/local/bin/docker run --rm --name nemoclaw-openshell-gateway-18080 --network host ubuntu:24.04 /opt/nemoclaw/openshell-gateway\n", - compatContainerPid, - listenerPids: [siblingContainerPid], - }); - try { - expect(fixture.result.stopped).toEqual([]); - expect(fixture.result.skippedNonMatchingPids).toEqual([fixture.selectedPid]); - expect(fixture.result.ownershipFailures).toEqual([ - `PID ${String(fixture.selectedPid)}: compatibility container '${fixture.selectedCompatContainerName}' does not solely own the listener on port 18080`, - ]); - expect(fixture.kill).not.toHaveBeenCalled(); - const dockerCalls = fixture.calls.filter(({ command }) => command === "docker"); - expect(dockerCalls).toContainEqual({ - args: ["inspect", "--type", "container", fixture.selectedCompatContainerName], - command: "docker", - }); - expect(dockerCalls.some(({ args }) => args[0] === "rm")).toBe(false); - expect( - dockerCalls.some(({ args }) => args.includes(fixture.siblingCompatContainerName)), - ).toBe(false); - expect(fs.existsSync(fixture.selectedPidFile)).toBe(true); - expect(fs.existsSync(fixture.selectedRuntimeMarker)).toBe(true); - expect(fs.existsSync(fixture.siblingPidFile)).toBe(true); - expect(fs.existsSync(fixture.siblingRuntimeMarker)).toBe(true); - } finally { - fixture.cleanup(); - } - }); - - it.each([ - { - label: "the selected PID file points at the sibling PID", - options: { pidFilePid: 9_990_808 }, - reason: "runtime marker PID 9991880 does not match PID file 9990808", - }, - { - label: "the process command line names the sibling gateway", - options: { - cmdline: "/opt/openshell/openshell gateway start --name nemoclaw --port 18080\n", - }, - reason: "process command line does not prove gateway 'nemoclaw-18080' on port 18080", - }, - { - label: "the process command line names the selected gateway on the sibling port", - options: { - cmdline: "/opt/openshell/openshell gateway start --name nemoclaw-18080 --port 8080\n", - }, - reason: "process command line does not prove gateway 'nemoclaw-18080' on port 18080", - }, - { - label: "the runtime marker identifies the sibling port", - options: { markerPort: 8_080 }, - reason: "runtime marker endpoint does not identify port 18080", - }, - { - label: "the runtime marker is missing", - options: { omitMarker: true }, - reason: "runtime marker is missing or not a regular file", - }, - { - label: "the process owner differs from the runtime evidence owner", - options: { processUid: CURRENT_UID + 1 }, - reason: "gateway process owner does not match the scoped runtime evidence owner", - }, - { - label: "the process executable differs from the runtime marker", - options: { processExecutable: "/opt/foreign/openshell" }, - reason: "gateway process executable does not match the runtime marker", - }, - { - label: "the process executable has been deleted", - options: { processExecutable: "/opt/openshell/openshell (deleted)" }, - reason: "gateway process executable does not match the runtime marker", - }, - { - label: "the selected port listener belongs to the sibling PID", - options: { listenerPids: [9_990_808] }, - reason: "PID 9991880 is not the sole listener owner for port 18080", - }, - ] as const)("fails closed and preserves evidence when $label", ({ options, reason }) => { - const fixture = scopedGatewayFixture(options); - try { - expect(fixture.result.stopped).toEqual([]); - expect(fixture.result.failed).toEqual([]); - expect(fixture.result.skippedNonMatchingPids).toEqual([fixture.recordedPid]); - expect(fixture.result.ownershipFailures).toEqual([ - `PID ${String(fixture.recordedPid)}: ${reason}`, - ]); - expect(fixture.kill).not.toHaveBeenCalled(); - expect(fixture.calls.filter(({ command }) => command === "pgrep")).toEqual([]); - expect(fs.existsSync(fixture.selectedPidFile)).toBe(true); - expect(fs.existsSync(fixture.selectedRuntimeMarker)).toBe(!options.omitMarker); - expect(fs.existsSync(fixture.siblingPidFile)).toBe(true); - expect(fs.existsSync(fixture.siblingRuntimeMarker)).toBe(true); - } finally { - fixture.cleanup(); - } - }); - - it("fails closed when the selected port is occupied without PID ownership evidence", () => { - const fixture = scopedGatewayFixture({ - listenerPids: [9_990_808], - omitMarker: true, - omitPidFile: true, - portFree: false, - }); - try { - expect(fixture.result.ownershipFailures).toEqual([ - "gateway port 18080 is occupied without PID-file ownership evidence", - ]); - expect(fixture.kill).not.toHaveBeenCalled(); - expect(fixture.calls.filter(({ command }) => command === "pgrep")).toEqual([]); - } finally { - fixture.cleanup(); - } - }); - - it("fails closed when the recorded PID is dead but the selected port remains occupied", () => { - const fixture = scopedGatewayFixture({ - listenerPids: [9_990_808], - portFree: false, - processAlive: false, - }); - try { - expect(fixture.result.skippedDeadPids).toEqual([fixture.selectedPid]); - expect(fixture.result.ownershipFailures).toEqual([ - `recorded PID ${String(fixture.selectedPid)} is dead but port 18080 remains occupied`, - ]); - expect(fixture.kill).not.toHaveBeenCalled(); - expect(fs.existsSync(fixture.selectedPidFile)).toBe(true); - expect(fs.existsSync(fixture.selectedRuntimeMarker)).toBe(true); - } finally { - fixture.cleanup(); - } - }); - - it("rejects a requested host-wide fallback without scanning or signaling", () => { - const fixture = scopedGatewayFixture({ usePgrepFallback: true }); - try { - expect(fixture.result.ownershipFailures).toEqual([ - "scoped gateway stop forbids host-wide process discovery", - ]); - expect(fixture.calls.filter(({ command }) => command === "pgrep")).toEqual([]); - expect(fixture.kill).not.toHaveBeenCalled(); - expect(fs.existsSync(fixture.selectedPidFile)).toBe(true); - expect(fs.existsSync(fixture.selectedRuntimeMarker)).toBe(true); - } finally { - fixture.cleanup(); - } - }); - - it("fails closed when the selected process identity changes immediately before signaling", () => { - const fixture = scopedGatewayFixture({ startIdentities: ["original", "replacement"] }); - try { - expect(fixture.result.stopped).toEqual([]); - expect(fixture.result.skippedNonMatchingPids).toEqual([fixture.selectedPid]); - expect(fixture.result.ownershipFailures).toEqual([ - `PID ${String(fixture.selectedPid)}: gateway process identity changed immediately before signaling`, - ]); - expect(fixture.kill).not.toHaveBeenCalled(); - expect(fs.existsSync(fixture.selectedPidFile)).toBe(true); - expect(fs.existsSync(fixture.selectedRuntimeMarker)).toBe(true); - expect(fs.existsSync(fixture.siblingPidFile)).toBe(true); - expect(fs.existsSync(fixture.siblingRuntimeMarker)).toBe(true); - } finally { - fixture.cleanup(); - } - }); -}); diff --git a/src/lib/onboard/host-gateway-process.ts b/src/lib/onboard/host-gateway-process.ts index ae79cccc3a7..cf72536be9b 100644 --- a/src/lib/onboard/host-gateway-process.ts +++ b/src/lib/onboard/host-gateway-process.ts @@ -6,25 +6,25 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { dockerForceRm as runDockerForceRm } from "../adapters/docker/container"; -import { dockerInspect as runDockerInspect } from "../adapters/docker/inspect"; -import type { DockerRunOptions, DockerRunResult } from "../adapters/docker/run"; import { waitUntil } from "../core/wait"; +import { + gatewayIdForStateDir, + hasStateScopedSandboxNamespace, + NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV, +} from "./docker-driver-gateway-config"; import { clearDockerDriverGatewayRuntimeMarker, getDockerDriverGatewayRuntimeMarkerPath, - readDockerDriverGatewayRuntimeMarker, + parseDockerDriverGatewayRuntimeMarker, } from "./docker-driver-gateway-runtime-marker"; import { canonicalGatewayTargetMatches, - cleanGatewayProcessToken, - DOCKER_DRIVER_GATEWAY_COMPAT_MOUNT_PATH, - DOCKER_DRIVER_GATEWAY_CONTAINER_RUNTIME_NAMES, - gatewayCompatContainerNameForPort, type OpenShellGatewayProcessTarget, hostGatewayCmdlineMatches as sharedHostGatewayCmdlineMatches, } from "./gateway-process-identity"; +export { hasStateScopedSandboxNamespace } from "./docker-driver-gateway-config"; + export interface RunResult { status: number | null; stdout: string; @@ -36,19 +36,9 @@ export interface HostGatewayProcessDeps { kill: (pid: number, signal?: NodeJS.Signals | number) => boolean; env: NodeJS.ProcessEnv; commandExists?: (command: string) => boolean; - dockerForceRm: ( - containerName: string, - options?: DockerRunOptions, - ) => Pick; - dockerInspect: ( - args: readonly string[], - options?: DockerRunOptions, - ) => Pick; isPortFree?: (port: number) => boolean; log?: (message: string) => void; - readProcessExecutable?: (pid: number) => string | null; readProcessEnvironment?: (pid: number) => Record | null; - readProcessStartIdentity?: (pid: number) => string | null; warn?: (message: string) => void; } @@ -65,10 +55,7 @@ export interface StopHostGatewayOptions { pollIntervalMs?: number; /** Keep PID/runtime evidence when a PID-file process does not match the cleanup target. */ preserveRuntimeFilesOnNonMatching?: boolean; - /** - * Stop one gateway without invoking OpenShell's shared Docker shutdown cleanup. - * Requires exact per-gateway PID, runtime-marker, owner, cmdline, and listener proof. - */ + /** Restrict cleanup to one fully proven PID-file gateway. */ scopedGatewayStop?: boolean; stateDir?: string; termWaitMs?: number; @@ -81,7 +68,7 @@ export interface StopHostGatewayResult { failed: number[]; /** Whether a requested pgrep fallback completed with a usable result. */ orphanScanComplete?: boolean; - ownershipFailures: string[]; + ownershipFailures?: string[]; skippedDeadPids: number[]; skippedNonMatchingPids: number[]; stopped: number[]; @@ -155,13 +142,9 @@ function defaultDeps(overrides: Partial = {}): HostGatew kill: overrides.kill ?? defaultKill, env, commandExists: overrides.commandExists ?? ((cmd) => defaultCommandExists(cmd, env)), - dockerForceRm: overrides.dockerForceRm ?? runDockerForceRm, - dockerInspect: overrides.dockerInspect ?? runDockerInspect, isPortFree: overrides.isPortFree ?? ((port) => isHostPortFree(port)), log: overrides.log, - readProcessExecutable: overrides.readProcessExecutable, readProcessEnvironment: overrides.readProcessEnvironment, - readProcessStartIdentity: overrides.readProcessStartIdentity, warn: overrides.warn, }; } @@ -197,47 +180,6 @@ function processArgs(pid: number, deps: HostGatewayProcessDeps): string { return result.status === 0 ? result.stdout.trim() : ""; } -function processExecutable(pid: number, deps: HostGatewayProcessDeps): string | null { - if (deps.readProcessExecutable) return deps.readProcessExecutable(pid); - try { - return fs.readlinkSync(`/proc/${String(pid)}/exe`); - } catch { - const lsof = deps.run("lsof", ["-a", "-p", String(pid), "-d", "txt", "-Fn"], { - env: deps.env, - }); - const lsofPath = - lsof.status === 0 - ? lsof.stdout - .split(/\r?\n/) - .find((line) => line.startsWith("n/") && line.length > 2) - ?.slice(1) - : undefined; - if (lsofPath) return lsofPath; - const result = deps.run("ps", ["-p", String(pid), "-o", "comm="], { env: deps.env }); - const executable = result.status === 0 ? result.stdout.trim() : ""; - return executable && path.isAbsolute(executable) ? executable : null; - } -} - -function processStartIdentity(pid: number, deps: HostGatewayProcessDeps): string | null { - if (deps.readProcessStartIdentity) return deps.readProcessStartIdentity(pid); - try { - const stat = fs.readFileSync(`/proc/${String(pid)}/stat`, "utf-8"); - const commandEnd = stat.lastIndexOf(")"); - if (commandEnd < 0) return null; - return ( - stat - .slice(commandEnd + 1) - .trim() - .split(/\s+/)[19] ?? null - ); - } catch { - const result = deps.run("ps", ["-p", String(pid), "-o", "lstart="], { env: deps.env }); - const started = result.status === 0 ? result.stdout.trim() : ""; - return started ? started : null; - } -} - function pidExists(pid: number, deps: HostGatewayProcessDeps): boolean { return deps.run("ps", ["-p", String(pid), "-o", "pid="], { env: deps.env }).status === 0; } @@ -248,151 +190,55 @@ function pidOwner(pid: number, deps: HostGatewayProcessDeps): string | null { return result.stdout.trim() || null; } -function pidUid(pid: number, deps: HostGatewayProcessDeps): number | null { - const result = deps.run("ps", ["-p", String(pid), "-o", "uid="], { env: deps.env }); - if (result.status !== 0) return null; - const uid = Number.parseInt(result.stdout.trim(), 10); - return Number.isInteger(uid) && uid >= 0 ? uid : null; -} - -function regularFileUid(filePath: string): number | null { - try { - const stat = fs.lstatSync(filePath); - return stat.isFile() && !stat.isSymbolicLink() ? stat.uid : null; - } catch { - return null; - } -} - -function ownedStateDirUid(stateDir: string): number | null { - try { - const stat = fs.lstatSync(stateDir); - return stat.isDirectory() && !stat.isSymbolicLink() ? stat.uid : null; - } catch { - return null; - } -} - -function gatewayEndpointPort(endpoint: string): number | null { +function readOwnedRuntimeFile(filePath: string, uid: number): string | null { + if (typeof fs.constants.O_NOFOLLOW !== "number") return null; + let descriptor: number | undefined; try { - const parsed = new URL(endpoint); - const port = Number.parseInt(parsed.port, 10); - return Number.isInteger(port) && port > 0 && port <= 65_535 ? port : null; - } catch { - return null; - } -} - -function listeningPids( - port: number, - deps: HostGatewayProcessDeps, -): { complete: boolean; pids: number[] } { - if (deps.commandExists && !deps.commandExists("lsof")) { - return { complete: false, pids: [] }; - } - const result = deps.run("lsof", ["-ti", `:${String(port)}`, "-sTCP:LISTEN"], { - env: deps.env, - }); - if (result.status !== 0 && result.status !== 1) { - return { complete: false, pids: [] }; - } - return { complete: true, pids: [...new Set(parsePidLines(result.stdout))] }; -} - -function dockerCompatContainerForTarget(cmdline: string, port: number): string | null { - const tokens = cmdline.trim().split(/\s+/).filter(Boolean).map(cleanGatewayProcessToken); - const argv0 = tokens[0] ?? ""; - if ( - !DOCKER_DRIVER_GATEWAY_CONTAINER_RUNTIME_NAMES.has(path.basename(argv0)) || - tokens[1] !== "run" || - !tokens.slice(1).includes(DOCKER_DRIVER_GATEWAY_COMPAT_MOUNT_PATH) - ) { - return null; - } - const containerName = gatewayCompatContainerNameForPort(port); - const nameIndex = tokens.findIndex((token) => token === "--name"); - const inlineName = tokens.find((token) => token.startsWith("--name="))?.slice("--name=".length); - const explicitName = nameIndex >= 0 ? tokens[nameIndex + 1] : inlineName; - return explicitName === containerName ? containerName : null; -} - -type DockerCompatContainerIdentity = { - containerId: string; - containerName: string; - dockerEnv: NodeJS.ProcessEnv; - pid: number; -}; - -function dockerCompatContainerIdentity( - containerName: string, - dockerHost: string, - deps: HostGatewayProcessDeps, -): DockerCompatContainerIdentity | null { - // Docker adapters apply the repository subprocess allowlist before spawning. - // Pass only the daemon identity we proved instead of treating the complete - // parent environment as an explicit override, which would reintroduce - // unrelated credentials that the adapter intentionally filters out. - const dockerEnv = { DOCKER_HOST: dockerHost }; - const result = deps.dockerInspect(["--type", "container", containerName], { - encoding: "utf-8", - env: dockerEnv, - ignoreError: true, - suppressOutput: true, - }); - if (result.status !== 0) return null; - try { - const parsed = JSON.parse(String(result.stdout ?? "")) as Array<{ - Args?: unknown; - HostConfig?: { NetworkMode?: unknown }; - Id?: unknown; - Name?: unknown; - Path?: unknown; - State?: { Pid?: unknown; Running?: unknown }; - }>; - if (!Array.isArray(parsed) || parsed.length !== 1) return null; - const container = parsed[0]; - const containerId = typeof container.Id === "string" ? container.Id : ""; - const containerPid = container.State?.Pid; - if ( - !/^[a-f0-9]{64}$/i.test(containerId) || - container.Name !== `/${containerName}` || - container.Path !== DOCKER_DRIVER_GATEWAY_COMPAT_MOUNT_PATH || - !Array.isArray(container.Args) || - container.Args.length !== 0 || - container.HostConfig?.NetworkMode !== "host" || - container.State?.Running !== true || - !Number.isSafeInteger(containerPid) || - Number(containerPid) <= 0 - ) { + descriptor = fs.openSync(filePath, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + const stat = fs.fstatSync(descriptor); + if (!stat.isFile() || stat.nlink !== 1 || stat.uid !== uid || stat.size > 64 * 1024) return null; - } - return { - containerId, - containerName, - dockerEnv, - pid: Number(containerPid), - }; + return fs.readFileSync(descriptor, "utf-8"); } catch { return null; + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); } } -function processEnvironment( +export function processUsesStateScopedSandboxNamespace( pid: number, - deps: HostGatewayProcessDeps, -): Record | null { - if (deps.readProcessEnvironment) return deps.readProcessEnvironment(pid); - try { - const entries = fs.readFileSync(`/proc/${String(pid)}/environ`, "utf-8").split("\0"); - const environment: Record = {}; - for (const entry of entries) { - const separator = entry.indexOf("="); - if (separator > 0) environment[entry.slice(0, separator)] = entry.slice(separator + 1); + stateDir: string, + deps: Pick, +): boolean { + const uid = typeof process.getuid === "function" ? process.getuid() : -1; + const owner = deps.run("ps", ["-p", String(pid), "-o", "uid="], { env: deps.env }); + if (owner.status !== 0 || Number(owner.stdout.trim()) !== uid) return false; + let environment = deps.readProcessEnvironment?.(pid) ?? null; + if (!environment) { + try { + environment = Object.fromEntries( + fs + .readFileSync(`/proc/${String(pid)}/environ`, "utf-8") + .split("\0") + .filter(Boolean) + .map((entry) => [ + entry.slice(0, entry.indexOf("=")), + entry.slice(entry.indexOf("=") + 1), + ]), + ); + } catch { + const command = deps.run("ps", ["eww", "-p", String(pid), "-o", "command="], { + env: deps.env, + }); + const prefix = `${NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV}=`; + const value = command.stdout.split(/\s+/).find((token) => token.startsWith(prefix)); + environment = value + ? { [NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV]: value.slice(prefix.length) } + : null; } - return environment; - } catch { - return null; } + return environment?.[NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV] === gatewayIdForStateDir(stateDir); } export function hostGatewayCmdlineMatches( @@ -404,222 +250,48 @@ export function hostGatewayCmdlineMatches( return sharedHostGatewayCmdlineMatches(cmdline, gatewayBin, expectedOpenShellGateway, opts); } -function normalizeExecutablePath(value: string): string { - try { - return fs.realpathSync.native(value); - } catch { - return path.resolve(value); - } -} - -function scopedGatewayOwnershipProof( +function scopedGatewayOwnershipFailure( pid: number, deps: HostGatewayProcessDeps, options: StopHostGatewayOptions, - target: { name: string; port: number }, stateDir: string, pidFile: string, -): { - cmdline: string; - compatContainerIdentity?: DockerCompatContainerIdentity; - compatContainerName: string | null; - reason?: string; - startIdentity?: string; -} { - const markerPath = getDockerDriverGatewayRuntimeMarkerPath(stateDir); - const stateDirUid = ownedStateDirUid(stateDir); - const pidFileUid = regularFileUid(pidFile); - const markerUid = regularFileUid(markerPath); - if (stateDirUid === null) { - return { - cmdline: "", - compatContainerName: null, - reason: "gateway state directory is missing, symlinked, or not a directory", - }; - } - if (pidFileUid === null) { - return { cmdline: "", compatContainerName: null, reason: "PID file is not a regular file" }; - } - if (markerUid === null) { - return { - cmdline: "", - compatContainerName: null, - reason: "runtime marker is missing or not a regular file", - }; - } - if (stateDirUid !== pidFileUid || pidFileUid !== markerUid) { - return { - cmdline: "", - compatContainerName: null, - reason: "gateway state directory, PID file, and runtime marker have different owners", - }; - } - const currentUid = typeof process.getuid === "function" ? process.getuid() : null; - if (currentUid !== null && stateDirUid !== currentUid) { - return { - cmdline: "", - compatContainerName: null, - reason: "scoped gateway runtime evidence is not owned by the current user", - }; - } - if (readPidFile(pidFile) !== pid) { - return { - cmdline: "", - compatContainerName: null, - reason: "PID file identity changed while proving the scoped gateway target", - }; - } - - const marker = readDockerDriverGatewayRuntimeMarker(markerPath); - if (!marker) { - return { cmdline: "", compatContainerName: null, reason: "runtime marker is invalid" }; - } - if (marker.pid !== pid) { - return { - cmdline: "", - compatContainerName: null, - reason: `runtime marker PID ${String(marker.pid)} does not match PID file ${String(pid)}`, - }; - } - if (gatewayEndpointPort(marker.endpoint) !== target.port) { - return { - cmdline: "", - compatContainerName: null, - reason: `runtime marker endpoint does not identify port ${String(target.port)}`, - }; - } - if (marker.platform !== process.platform || marker.arch !== process.arch) { - return { - cmdline: "", - compatContainerName: null, - reason: "runtime marker platform identity does not match this host", - }; + target: { name: string; port: number }, +): string | null { + if (!hasStateScopedSandboxNamespace(stateDir)) { + return "gateway config does not prove an isolated sandbox namespace"; + } + const uid = typeof process.getuid === "function" ? process.getuid() : -1; + const pidText = readOwnedRuntimeFile(pidFile, uid); + const markerText = readOwnedRuntimeFile(getDockerDriverGatewayRuntimeMarkerPath(stateDir), uid); + const marker = markerText ? parseDockerDriverGatewayRuntimeMarker(markerText) : null; + if (Number(pidText?.trim()) !== pid || marker?.pid !== pid) { + return "PID file and runtime marker do not identify the same process"; + } + let markerPort = 0; + try { + markerPort = Number(new URL(marker.endpoint).port); + } catch { + return "runtime marker endpoint is invalid"; } if ( - marker.gatewayBin && - options.gatewayBin && - normalizeExecutablePath(marker.gatewayBin) !== normalizeExecutablePath(options.gatewayBin) + markerPort !== target.port || + marker.platform !== process.platform || + marker.arch !== process.arch ) { - return { - cmdline: "", - compatContainerName: null, - reason: "runtime marker gateway executable does not match the cleanup target", - }; + return "runtime marker does not identify the selected gateway"; } - - const ownerUid = pidUid(pid, deps); - if (ownerUid === null || ownerUid !== pidFileUid) { - return { - cmdline: "", - compatContainerName: null, - reason: "gateway process owner does not match the scoped runtime evidence owner", - }; + if (!processUsesStateScopedSandboxNamespace(pid, stateDir, deps)) { + return "gateway process owner and loaded sandbox namespace cannot be proven"; } - - const cmdline = processArgs(pid, deps); if ( - !hostGatewayCmdlineMatches(cmdline, options.gatewayBin, target, { + !hostGatewayCmdlineMatches(processArgs(pid, deps), options.gatewayBin, target, { requireExpectedFlags: true, }) ) { - return { - cmdline, - compatContainerName: null, - reason: `process command line does not prove gateway '${target.name}' on port ${String(target.port)}`, - }; - } - - const compatContainerName = dockerCompatContainerForTarget(cmdline, target.port); - if (!compatContainerName) { - if (!marker.gatewayBin) { - return { - cmdline, - compatContainerName, - reason: "runtime marker does not identify the direct gateway executable", - }; - } - const executable = processExecutable(pid, deps); - if ( - !executable || - normalizeExecutablePath(executable) !== normalizeExecutablePath(marker.gatewayBin) - ) { - return { - cmdline, - compatContainerName, - reason: "gateway process executable does not match the runtime marker", - }; - } - } - const startIdentity = processStartIdentity(pid, deps); - if (!startIdentity) { - return { - cmdline, - compatContainerName, - reason: "gateway process start identity could not be proven", - }; + return "process command line does not identify the selected gateway name and port"; } - const listeners = listeningPids(target.port, deps); - if (!listeners.complete) { - return { - cmdline, - compatContainerName, - reason: `listener ownership for port ${String(target.port)} could not be observed completely`, - }; - } - if (compatContainerName) { - const parentEnvironment = processEnvironment(pid, deps); - const parentDockerHost = parentEnvironment?.DOCKER_HOST?.trim() || null; - const provenDockerHost = marker.dockerHost ?? "unix:///var/run/docker.sock"; - const unsupportedDockerSelector = [ - "DOCKER_CERT_PATH", - "DOCKER_CONFIG", - "DOCKER_CONTEXT", - "DOCKER_TLS_VERIFY", - ].some((key) => Boolean(parentEnvironment?.[key]?.trim())); - if ( - marker.gatewayBin !== null || - !provenDockerHost.startsWith("unix:///") || - !parentEnvironment || - unsupportedDockerSelector || - (marker.dockerHost === null - ? parentDockerHost !== null - : parentDockerHost !== marker.dockerHost) - ) { - return { - cmdline, - compatContainerName, - reason: "compatibility gateway Docker daemon identity does not match the runtime marker", - }; - } - const compatContainerIdentity = dockerCompatContainerIdentity( - compatContainerName, - provenDockerHost, - deps, - ); - if (!compatContainerIdentity) { - return { - cmdline, - compatContainerName, - reason: `compatibility container '${compatContainerName}' identity could not be proven`, - }; - } - if (listeners.pids.length !== 1 || listeners.pids[0] !== compatContainerIdentity.pid) { - return { - cmdline, - compatContainerName, - reason: `compatibility container '${compatContainerName}' does not solely own the listener on port ${String(target.port)}`, - }; - } - return { cmdline, compatContainerIdentity, compatContainerName, startIdentity }; - } else if (listeners.pids.length !== 1 || listeners.pids[0] !== pid) { - return { - cmdline, - compatContainerName, - reason: `PID ${String(pid)} is not the sole listener owner for port ${String(target.port)}`, - }; - } - - return { cmdline, compatContainerName, startIdentity }; + return null; } function waitForExit( @@ -701,13 +373,16 @@ function tryStopPid( pid: number, deps: HostGatewayProcessDeps, options: Required>, -): "stopped" | "failed" { + canSignal?: () => boolean, +): "stopped" | "failed" | "identity-changed" { const log = deps.log ?? ((message: string) => console.log(message)); + if (canSignal && !canSignal()) return "identity-changed"; deps.kill(pid, "SIGTERM"); if (waitForExit(pid, deps, options.termWaitMs, options.pollIntervalMs)) { log(`Stopped host openshell-gateway process ${pid}`); return "stopped"; } + if (canSignal && !canSignal()) return "identity-changed"; deps.kill(pid, "SIGKILL"); if (waitForExit(pid, deps, options.killWaitMs, options.pollIntervalMs)) { log(`Stopped host openshell-gateway process ${pid} (after SIGKILL)`); @@ -717,48 +392,6 @@ function tryStopPid( return "failed"; } -function tryStopScopedPid( - pid: number, - compatContainerIdentity: DockerCompatContainerIdentity | undefined, - expectedStartIdentity: string, - deps: HostGatewayProcessDeps, - options: Required>, -): "stopped" | "failed" | "identity-changed" { - const log = deps.log ?? ((message: string) => console.log(message)); - if (processStartIdentity(pid, deps) !== expectedStartIdentity) return "identity-changed"; - if (compatContainerIdentity) { - const removed = deps.dockerForceRm(compatContainerIdentity.containerId, { - encoding: "utf-8", - env: compatContainerIdentity.dockerEnv, - ignoreError: true, - suppressOutput: true, - }); - if (removed.status !== 0) { - const warn = deps.warn ?? ((message: string) => console.warn(message)); - const detail = String(removed.stderr ?? "").trim() || `status ${String(removed.status)}`; - warn( - `Failed to remove scoped gateway compatibility container '${compatContainerIdentity.containerName}': ${detail}`, - ); - return "failed"; - } - if (!waitForExit(pid, deps, options.killWaitMs, options.pollIntervalMs)) { - return "failed"; - } - } else { - // OpenShell 0.0.99 gracefully stops every managed Docker container in its - // configured namespace. Scoped teardown has already deleted this gateway's - // selected sandboxes, so SIGKILL avoids cross-stopping a sibling gateway's - // container while still targeting only the fully proven process. - deps.kill(pid, "SIGKILL"); - } - if (waitForExit(pid, deps, options.killWaitMs, options.pollIntervalMs)) { - log(`Stopped scoped host openshell-gateway process ${pid}`); - return "stopped"; - } - warnSudoRemediation(pid, deps); - return "failed"; -} - export function stopHostGatewayProcesses( depsOverrides: Partial = {}, options: StopHostGatewayOptions = {}, @@ -778,57 +411,44 @@ export function stopHostGatewayProcesses( sudoRemediationPids: [], }; - const scopedGatewayStop = options.scopedGatewayStop ?? false; const explicitPids = Array.from(options.pids ?? []).filter( (pid): pid is number => Number.isInteger(pid) && pid > 0, ); - let scopedTarget: { name: string; port: number } | null = null; - if (scopedGatewayStop) { - const port = Number(options.openShellGatewayPort); - const name = options.openShellGatewayName?.trim() ?? ""; - if ( - !Number.isInteger(port) || - port < 1 || - port > 65_535 || - !canonicalGatewayTargetMatches(name, port) - ) { - result.ownershipFailures.push( - "scoped gateway stop requires one canonical gateway name and port", - ); - return result; - } - if (options.usePidFile === false || explicitPids.length > 0) { - result.ownershipFailures.push( - "scoped gateway stop accepts only the selected gateway PID file", - ); - return result; - } - if (options.usePgrepFallback === true) { - result.ownershipFailures.push("scoped gateway stop forbids host-wide process discovery"); - return result; - } - scopedTarget = { name, port }; + const scopedPort = Number(options.openShellGatewayPort); + const scopedName = options.openShellGatewayName?.trim() ?? ""; + const rejectScoped = (reason: string, pid?: number): StopHostGatewayResult => { + if (pid) result.skippedNonMatchingPids.push(pid); + (result.ownershipFailures ??= []).push(pid ? `PID ${String(pid)}: ${reason}` : reason); + return result; + }; + if ( + options.scopedGatewayStop && + (!Number.isInteger(scopedPort) || + scopedPort < 1 || + scopedPort > 65_535 || + !canonicalGatewayTargetMatches(scopedName, scopedPort) || + options.usePidFile === false || + options.usePgrepFallback === true || + explicitPids.length > 0) + ) { + return rejectScoped("scoped cleanup requires one canonical name, port, and PID file"); } if (options.usePidFile ?? true) { const pidFromFile = readPidFile(pidFile); if (pidFromFile !== null) { addPid(candidates, pidFromFile, "pid-file"); - } else if (scopedTarget) { - const markerPath = getDockerDriverGatewayRuntimeMarkerPath(stateDir); - if (fs.existsSync(pidFile) || fs.existsSync(markerPath)) { - result.ownershipFailures.push( - "scoped gateway PID/runtime evidence is incomplete or invalid", - ); - return result; - } - const listeners = listeningPids(scopedTarget.port, deps); - if (deps.isPortFree?.(scopedTarget.port) !== true || listeners.pids.length > 0) { - result.ownershipFailures.push( - `gateway port ${String(scopedTarget.port)} is occupied without PID-file ownership evidence`, - ); - return result; + } else if (options.scopedGatewayStop) { + if ( + fs.existsSync(pidFile) || + fs.existsSync(getDockerDriverGatewayRuntimeMarkerPath(stateDir)) || + deps.isPortFree?.(scopedPort) !== true + ) { + return rejectScoped("selected gateway has incomplete ownership evidence"); } + if (options.logNoProcesses) + (deps.log ?? console.log)("No host openshell-gateway processes found"); + return result; } else if (clearRuntimeState && fs.existsSync(pidFile)) { clearHostGatewayRuntimeFiles(stateDir, pidFile); } @@ -841,7 +461,7 @@ export function stopHostGatewayProcesses( // host. Otherwise an onboard drift could terminate an unrelated worktree's // gateway. Sweeping callers (uninstall, sandbox destroy of the last sandbox) // omit `pids` and so still get the pgrep fallback by default. - const useFallback = scopedGatewayStop + const useFallback = options.scopedGatewayStop ? false : (options.usePgrepFallback ?? explicitPids.length === 0); let pgrepRan = false; @@ -868,14 +488,8 @@ export function stopHostGatewayProcesses( for (const [pid, sources] of candidates) { if (!pidExists(pid, deps)) { result.skippedDeadPids.push(pid); - if (scopedTarget) { - const listeners = listeningPids(scopedTarget.port, deps); - if (deps.isPortFree?.(scopedTarget.port) !== true || listeners.pids.length > 0) { - result.ownershipFailures.push( - `recorded PID ${String(pid)} is dead but port ${String(scopedTarget.port)} remains occupied`, - ); - continue; - } + if (options.scopedGatewayStop && deps.isPortFree?.(scopedPort) !== true) { + return rejectScoped("recorded process is dead but its selected port remains occupied"); } if (clearRuntimeState && sources.has("pid-file") && !clearedRuntimeFiles) { clearHostGatewayRuntimeFiles(stateDir, pidFile); @@ -883,76 +497,15 @@ export function stopHostGatewayProcesses( } continue; } - - if (scopedTarget) { - const proof = scopedGatewayOwnershipProof( - pid, - deps, - options, - scopedTarget, - stateDir, - pidFile, - ); - if (proof.reason) { - result.skippedNonMatchingPids.push(pid); - result.ownershipFailures.push(`PID ${String(pid)}: ${proof.reason}`); - continue; - } - const finalProof = scopedGatewayOwnershipProof( - pid, - deps, - options, - scopedTarget, - stateDir, - pidFile, - ); - if ( - finalProof.reason || - finalProof.cmdline !== proof.cmdline || - finalProof.startIdentity !== proof.startIdentity || - finalProof.compatContainerIdentity?.containerId !== - proof.compatContainerIdentity?.containerId - ) { - result.skippedNonMatchingPids.push(pid); - result.ownershipFailures.push( - `PID ${String(pid)}: gateway process identity changed immediately before signaling`, - ); - continue; - } - const scopedStop = tryStopScopedPid( - pid, - finalProof.compatContainerIdentity, - finalProof.startIdentity as string, - deps, - waitOptions, - ); - if (scopedStop === "identity-changed") { - result.skippedNonMatchingPids.push(pid); - result.ownershipFailures.push( - `PID ${String(pid)}: gateway process identity changed immediately before signaling`, - ); - continue; - } - if (scopedStop !== "stopped") { - result.failed.push(pid); - result.sudoRemediationPids.push(pid); - continue; - } - result.stopped.push(pid); - if (!deps.isPortFree?.(scopedTarget.port)) { - result.ownershipFailures.push( - `gateway port ${String(scopedTarget.port)} remains occupied after stopping PID ${String(pid)}`, - ); - continue; - } - if (clearRuntimeState && !clearedRuntimeFiles) { - clearHostGatewayRuntimeFiles(stateDir, pidFile); - clearedRuntimeFiles = true; - } - continue; + if (options.scopedGatewayStop) { + const reason = scopedGatewayOwnershipFailure(pid, deps, options, stateDir, pidFile, { + name: scopedName, + port: scopedPort, + }); + if (reason) return rejectScoped(reason, pid); } - if ( + !options.scopedGatewayStop && !hostGatewayCmdlineMatches( processArgs(pid, deps), options.gatewayBin, @@ -972,8 +525,26 @@ export function stopHostGatewayProcesses( continue; } - if (tryStopPid(pid, deps, waitOptions) === "stopped") { + const stopResult = tryStopPid( + pid, + deps, + waitOptions, + options.scopedGatewayStop + ? () => + scopedGatewayOwnershipFailure(pid, deps, options, stateDir, pidFile, { + name: scopedName, + port: scopedPort, + }) === null + : undefined, + ); + if (stopResult === "identity-changed") { + return rejectScoped("process ownership changed immediately before signaling", pid); + } + if (stopResult === "stopped") { result.stopped.push(pid); + if (options.scopedGatewayStop && deps.isPortFree?.(scopedPort) !== true) { + return rejectScoped("selected gateway port remains occupied after its process stopped"); + } if (clearRuntimeState && !clearedRuntimeFiles) { clearHostGatewayRuntimeFiles(stateDir, pidFile); clearedRuntimeFiles = true; @@ -984,7 +555,7 @@ export function stopHostGatewayProcesses( } } - if (options.logNoProcesses && candidates.size === 0 && result.ownershipFailures.length === 0) { + if (options.logNoProcesses && candidates.size === 0) { if (useFallback && !pgrepRan) { // The pid-file branch found nothing and the pgrep fallback could not // run (typically `pgrep` is absent on a minimal image). Surface the diff --git a/src/lib/tunnel/gateway-port-release-test-helpers.ts b/src/lib/tunnel/gateway-port-release-test-helpers.ts index f56c95367e9..d68c0f62e1e 100644 --- a/src/lib/tunnel/gateway-port-release-test-helpers.ts +++ b/src/lib/tunnel/gateway-port-release-test-helpers.ts @@ -16,7 +16,6 @@ export function emptyStopResult( ): StopHostGatewayResult { return { failed: [], - ownershipFailures: [], skippedDeadPids: [], skippedNonMatchingPids: [], stopped: [], diff --git a/test/e2e/live/concurrent-gateway-ports.test.ts b/test/e2e/live/concurrent-gateway-ports.test.ts index c376d47b975..15fc46b93a7 100644 --- a/test/e2e/live/concurrent-gateway-ports.test.ts +++ b/test/e2e/live/concurrent-gateway-ports.test.ts @@ -10,10 +10,7 @@ */ import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { getTrustedActiveOpenShellGatewayUserServicePid } from "../../../src/lib/onboard/docker-driver-gateway-service.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; @@ -24,7 +21,6 @@ import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compati import { CLI_DIST_ENTRYPOINT, CLI_ENTRYPOINT } from "../fixtures/paths.ts"; import { PollingError, pollUntil } from "../fixtures/polling.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; -import { requireFixture } from "../support/require-fixture.ts"; const SANDBOX_A = process.env.NEMOCLAW_CGP_SANDBOX_A ?? "e2e-cgp-a"; const SANDBOX_B = process.env.NEMOCLAW_CGP_SANDBOX_B ?? "e2e-cgp-b"; @@ -37,19 +33,6 @@ const PROBE_DELAY_MS = Number(process.env.NEMOCLAW_E2E_PROBE_DELAY_SECONDS ?? 5) const TEST_TIMEOUT_MS = 90 * 60_000; const POST_UNINSTALL_HEALTH_PROBES = 3; -type GatewayProcessAuthority = "standalone-state" | "systemd-service"; - -interface GatewayProcessIdentity { - authority: GatewayProcessAuthority; - pid: number; -} - -interface CapturedProcessIdentity { - executable: string; - pid: number; - startIdentity: string; -} - process.env.NEMOCLAW_CLI_BIN ??= CLI_ENTRYPOINT; validateSandboxName(SANDBOX_A); validateSandboxName(SANDBOX_B); @@ -71,131 +54,6 @@ function gatewayNameForPort(port: string): string { return port === "8080" ? "nemoclaw" : `nemoclaw-${port}`; } -function gatewayStateDirForPort(port: string): string { - const numericPort = Number(port); - requireFixture( - Number.isInteger(numericPort) && numericPort >= 1 && numericPort <= 65_535, - `invalid gateway port '${port}'`, - ); - const leaf = port === "8080" ? "openshell-docker-gateway" : `openshell-docker-gateway-${port}`; - return path.join(process.env.HOME || os.homedir(), ".local", "state", "nemoclaw", leaf); -} - -function readGatewayPid(port: string): number | null { - try { - const raw = fs.readFileSync( - path.join(gatewayStateDirForPort(port), "openshell-gateway.pid"), - "utf-8", - ); - const pid = Number.parseInt(raw.trim(), 10); - return Number.isInteger(pid) && pid > 0 ? pid : null; - } catch { - return null; - } -} - -function readGatewayRuntimePid(port: string): number | null { - try { - const marker: unknown = JSON.parse( - fs.readFileSync(path.join(gatewayStateDirForPort(port), "runtime.json"), "utf-8"), - ); - const pid = - marker && typeof marker === "object" && "pid" in marker - ? (marker as { pid?: unknown }).pid - : null; - return typeof pid === "number" && Number.isInteger(pid) && pid > 0 ? pid : null; - } catch { - return null; - } -} - -function readStandaloneGatewayIdentity(port: string): GatewayProcessIdentity | null { - const pid = readGatewayPid(port); - const runtimePid = readGatewayRuntimePid(port); - const absent = pid === null && runtimePid === null; - requireFixture( - absent || (pid !== null && runtimePid !== null && pid === runtimePid), - `gateway ${gatewayNameForPort(port)} has inconsistent standalone process state ` + - `(pid file: ${String(pid)}, runtime marker: ${String(runtimePid)})`, - ); - return absent ? null : { authority: "standalone-state", pid: pid as number }; -} - -function readDefaultGatewayServiceIdentity(): GatewayProcessIdentity { - const pid = - process.platform === "linux" && GATEWAY_PORT_A === "8080" - ? getTrustedActiveOpenShellGatewayUserServicePid({ env: commandEnv() }) - : null; - requireFixture( - pid !== null, - `default gateway ${gatewayNameForPort(GATEWAY_PORT_A)} has neither matching ` + - "standalone PID/runtime state nor a trusted active systemd MainPID", - ); - return { authority: "systemd-service", pid }; -} - -function readDefaultGatewayIdentity(): GatewayProcessIdentity { - return readStandaloneGatewayIdentity(GATEWAY_PORT_A) ?? readDefaultGatewayServiceIdentity(); -} - -function readAlternateGatewayIdentity(): GatewayProcessIdentity { - const identity = readStandaloneGatewayIdentity(GATEWAY_PORT_B); - requireFixture( - identity, - `alternate gateway ${gatewayNameForPort(GATEWAY_PORT_B)} is missing its standalone ` + - "PID/runtime ownership proof", - ); - return identity; -} - -function evidenceField(output: string, field: string): string | null { - const values = output - .split(/\r?\n/) - .filter((line) => line.startsWith(`${field}=`)) - .map((line) => line.slice(field.length + 1).trim()) - .filter(Boolean); - return values.length === 1 ? values[0] : null; -} - -function parseCapturedProcessIdentity( - result: ShellProbeResult, - pidText: string, -): CapturedProcessIdentity { - const pid = Number(pidText); - const executable = evidenceField(result.stdout, "process_executable"); - const startIdentity = evidenceField(result.stdout, "process_start_identity"); - requireFixture( - Number.isSafeInteger(pid) && pid > 0 && executable && startIdentity, - `incomplete gateway process identity evidence:\n${resultText(result)}`, - ); - return { executable, pid, startIdentity }; -} - -function capturedProcessIdentity(result: ShellProbeResult): CapturedProcessIdentity | null { - const pidText = evidenceField(result.stdout, "active_pid"); - return pidText === null ? null : parseCapturedProcessIdentity(result, pidText); -} - -function expectOnLinux(assertions: () => void): void { - (process.platform === "linux" ? assertions : () => undefined)(); -} - -function expectStandaloneGatewayArgv( - identity: GatewayProcessIdentity, - evidence: ShellProbeResult, - gatewayName: string, - gatewayPort: string, -): void { - const assertions: Record void> = { - "standalone-state": () => - expect(resultText(evidence)).toContain( - `openshell-gateway[nemoclaw=${gatewayName};port=${gatewayPort}]`, - ), - "systemd-service": () => undefined, - }; - assertions[identity.authority](); -} - function openshellEnvForGateway(gatewayName: string): NodeJS.ProcessEnv { return commandEnv({ OPENSHELL_GATEWAY: gatewayName }); } @@ -234,134 +92,49 @@ async function command( }); } +function evidencePid(evidence: string, gateway: string): string | undefined { + return evidence + .split(`gateway=${gateway}\n`)[1] + ?.split("gateway=")[0] + ?.match(/^active_pid=(\d+)$/m)?.[1]; +} + async function captureGatewayEvidence( host: HostCliClient, sandbox: SandboxClient, - options: { - authority: GatewayProcessAuthority; - gatewayName: string; - knownPid?: number | null; - port: string; - stage: string; - }, -): Promise<{ - host: ShellProbeResult; - processIdentity: CapturedProcessIdentity | null; - sandbox: ShellProbeResult; -}> { - const stateDir = gatewayStateDirForPort(options.port); - const hostEvidence = await host.command( + gateways: readonly (readonly [string, string])[], + stage: string, +): Promise { + const script = [ + 'for spec in "$@"; do', + ' gateway="${spec%%:*}"; port="${spec##*:}"; leaf=openshell-docker-gateway', + ' test "$port" = 8080 || leaf="$leaf-$port"', + ' state="$HOME/.local/state/nemoclaw/$leaf"; pid=""', + ' test ! -r "$state/openshell-gateway.pid" || pid="$(tr -d "[:space:]" < "$state/openshell-gateway.pid")"', + ' if test -z "$pid" && test "$port" = 8080 && command -v systemctl >/dev/null; then for service in openshell-gateway nemoclaw-openshell-gateway; do candidate="$(systemctl --user show "$service" --property=MainPID --value 2>/dev/null || true)"; test "${candidate:-0}" -le 0 || { pid="$candidate"; break; }; done; fi', + ' printf "gateway=%s\\nport=%s\\npid_file=%s\\n" "$gateway" "$port" "${pid:-}"', + ' if test -n "$pid" && ps -p "$pid" >/dev/null 2>&1; then printf "active_pid=%s\\n" "$pid"; ps -p "$pid" -o pid=,ppid=,uid=,lstart=,args=; fi', + ' printf "listeners=\\n"; ss -H -ltnp 2>&1 | grep -E "[:.]$port\\b" || true', + ' printf "runtime=\\n"; test ! -r "$state/runtime.json" || cat "$state/runtime.json"', + ' printf "namespace=\\n"; test ! -r "$state/openshell-gateway.toml" || grep "^sandbox_namespace" "$state/openshell-gateway.toml" || true', + "done", + ].join("\n"); + const evidence = await host.command( "bash", - [ - "-lc", - [ - 'gateway="$1"', - 'port="$2"', - 'state_dir="$3"', - 'known_pid="$4"', - 'authority="$5"', - 'pid_file="$state_dir/openshell-gateway.pid"', - 'runtime_file="$state_dir/runtime.json"', - 'printf "gateway=%s\\nport=%s\\nstate_dir=%s\\nauthority=%s\\n" "$gateway" "$port" "$state_dir" "$authority"', - 'if [ -r "$pid_file" ]; then printf "pid_file="; cat "$pid_file"; else printf "pid_file=\\n"; fi', - 'if [ -r "$runtime_file" ]; then printf "runtime_marker=\\n"; cat "$runtime_file"; else printf "runtime_marker=\\n"; fi', - 'pid="$known_pid"', - 'if [ "$authority" = "standalone-state" ] && [ -r "$pid_file" ]; then pid="$(tr -d "[:space:]" < "$pid_file")"; fi', - 'if [ -n "$pid" ] && [ -r "/proc/$pid/cmdline" ]; then printf "proc_cmdline="; tr "\\000" " " < "/proc/$pid/cmdline"; printf "\\n"; fi', - 'if [ -n "$pid" ] && ps -p "$pid" -o pid= >/dev/null 2>&1; then', - ' printf "active_pid=%s\\n" "$pid"', - ' if [ -r "/proc/$pid/stat" ]; then', - ' proc_stat="$(cat "/proc/$pid/stat")"', - ' proc_stat="${proc_stat##*) }"', - " set -- $proc_stat", - " shift 19", - ' printf "process_start_identity=linux:%s\\n" "$1"', - " else", - ' process_started="$(ps -p "$pid" -o lstart= 2>/dev/null | sed -e "s/^[[:space:]]*//" -e "s/[[:space:]]*$//")"', - ' if [ -n "$process_started" ]; then printf "process_start_identity=ps:%s\\n" "$process_started"; fi', - " fi", - ' if [ -L "/proc/$pid/exe" ]; then', - ' printf "process_executable="; readlink "/proc/$pid/exe"; printf "\\n"', - " else", - ' process_executable="$(ps -p "$pid" -o comm= 2>/dev/null | sed -e "s/^[[:space:]]*//" -e "s/[[:space:]]*$//")"', - ' if [ -n "$process_executable" ]; then printf "process_executable=%s\\n" "$process_executable"; fi', - " fi", - ' ps -p "$pid" -o pid= -o ppid= -o user= -o command= 2>&1 || true', - "fi", - 'if command -v ss >/dev/null 2>&1; then ss -H -ltnp 2>&1 | awk -v port="$port" \'$4 ~ (":" port "$")\' || true; fi', - 'if command -v lsof >/dev/null 2>&1; then lsof -nP -a -iTCP:"$port" -sTCP:LISTEN 2>&1 || true; fi', - ].join("\n"), - "gateway-evidence", - options.gatewayName, - options.port, - stateDir, - options.knownPid ? String(options.knownPid) : "", - options.authority, - ], - { - artifactName: `${options.stage}-${options.gatewayName}-host-identity`, - env: commandEnv(), - timeoutMs: 30_000, - }, + ["-lc", script, "gateway-evidence", ...gateways.map(([name, port]) => `${name}:${port}`)], + { artifactName: `${stage}-gateway-processes`, env: commandEnv(), timeoutMs: 30_000 }, ); - expect(hostEvidence.exitCode, resultText(hostEvidence)).toBe(0); - - const sandboxEvidence = await sandbox.openshell(["sandbox", "list", "-g", options.gatewayName], { - artifactName: `${options.stage}-${options.gatewayName}-sandbox-phase`, - env: openshellEnvForGateway(options.gatewayName), - timeoutMs: 30_000, - }); - return { - host: hostEvidence, - processIdentity: capturedProcessIdentity(hostEvidence), - sandbox: sandboxEvidence, - }; -} - -async function captureGatewayPairEvidence( - host: HostCliClient, - sandbox: SandboxClient, - options: { - gatewayA: string; - gatewayB: string; - identityA: GatewayProcessIdentity; - identityB: GatewayProcessIdentity; - stage: string; - }, -): Promise<{ - gatewayA: { - host: ShellProbeResult; - processIdentity: CapturedProcessIdentity | null; - sandbox: ShellProbeResult; - }; - gatewayB: { - host: ShellProbeResult; - processIdentity: CapturedProcessIdentity | null; - sandbox: ShellProbeResult; - }; -}> { - const [gatewayA, gatewayB] = await Promise.all([ - captureGatewayEvidence(host, sandbox, { - authority: options.identityA.authority, - gatewayName: options.gatewayA, - knownPid: options.identityA.pid, - port: GATEWAY_PORT_A, - stage: options.stage, - }), - captureGatewayEvidence(host, sandbox, { - authority: options.identityB.authority, - gatewayName: options.gatewayB, - knownPid: options.identityB.pid, - port: GATEWAY_PORT_B, - stage: options.stage, - }), - ]); - await sandbox.openshell(["gateway", "list", "-o", "json"], { - artifactName: `${options.stage}-gateway-registrations`, - env: commandEnv(), - timeoutMs: 30_000, - }); - return { gatewayA, gatewayB }; + expect(evidence.exitCode, resultText(evidence)).toBe(0); + await Promise.all( + gateways.map(([name]) => + sandbox.openshell(["sandbox", "list", "-g", name], { + artifactName: `${stage}-${name}-sandbox-phase`, + env: openshellEnvForGateway(name), + timeoutMs: 30_000, + }), + ), + ); + return resultText(evidence); } async function runOnboard( @@ -471,74 +244,6 @@ async function expectPortNotListening( return result; } -async function expectSurvivingGatewayHealthyAcrossProbes( - host: HostCliClient, - sandbox: SandboxClient, - options: { - dashboardPort: string; - gatewayName: string; - gatewayPort: string; - sandboxName: string; - }, -): Promise { - const phases: string[] = []; - for (let attempt = 1; attempt <= POST_UNINSTALL_HEALTH_PROBES; attempt += 1) { - const suffix = String(attempt).padStart(2, "0"); - const sandboxList = await sandbox.openshell(["sandbox", "list", "-g", options.gatewayName], { - artifactName: `phase-4-survivor-probe-${suffix}-sandbox-phase`, - env: openshellEnvForGateway(options.gatewayName), - timeoutMs: 30_000, - }); - expect(sandboxList.exitCode, resultText(sandboxList)).toBe(0); - const phase = sandboxPhaseFromList(resultText(sandboxList), options.sandboxName) ?? "missing"; - phases.push(phase); - expect( - ["Ready", "Running"], - `survivor probe ${String(attempt)} observed ${options.sandboxName} phase '${phase}'`, - ).toContain(phase); - - await expectPortListening( - host, - options.gatewayPort, - `phase-4-survivor-probe-${suffix}-gateway-listener`, - ); - const scopedList = await command(host, ["list"], { - artifactName: `phase-4-survivor-probe-${suffix}-nemoclaw-list`, - env: commandEnv({ NEMOCLAW_GATEWAY_PORT: options.gatewayPort }), - timeoutMs: 60_000, - }); - expect(scopedList.exitCode, resultText(scopedList)).toBe(0); - expect(outputIncludesSandbox(scopedList.stdout, options.sandboxName), scopedList.stdout).toBe( - true, - ); - - const dashboard = await host.command( - "curl", - [ - "-sS", - "-L", - "--max-time", - "10", - "-o", - "/dev/null", - "-w", - "%{http_code}", - `http://127.0.0.1:${options.dashboardPort}/`, - ], - { - artifactName: `phase-4-survivor-probe-${suffix}-dashboard-http`, - env: commandEnv(), - timeoutMs: 30_000, - }, - ); - expect(dashboard.exitCode, resultText(dashboard)).toBe(0); - expect(dashboard.stdout.trim()).toMatch(/^[23][0-9]{2}$/); - - await (attempt < POST_UNINSTALL_HEALTH_PROBES ? sleep(PROBE_DELAY_MS) : Promise.resolve()); - } - return phases; -} - async function prerequisiteOrSkip( host: HostCliClient, skip: (message: string) => never, @@ -814,42 +519,21 @@ test("concurrent gateway ports: onboards two sandboxes on isolated gateways and expect(dashboardB).not.toBe(dashboardA); progress.phase("uninstall alternate gateway without disrupting default"); - const gatewayIdentityA = readDefaultGatewayIdentity(); - const gatewayIdentityB = readAlternateGatewayIdentity(); - expect(gatewayIdentityA.pid).not.toBe(gatewayIdentityB.pid); - expect(gatewayIdentityB.authority).toBe("standalone-state"); - const beforeUninstallEvidence = await captureGatewayPairEvidence(host, sandbox, { - gatewayA, - gatewayB, - identityA: gatewayIdentityA, - identityB: gatewayIdentityB, - stage: "phase-4-before-uninstall", - }); - expect(beforeUninstallEvidence.gatewayA.processIdentity?.pid).toBe(gatewayIdentityA.pid); - expect(beforeUninstallEvidence.gatewayB.processIdentity?.pid).toBe(gatewayIdentityB.pid); - expectOnLinux(() => { - expectStandaloneGatewayArgv( - gatewayIdentityA, - beforeUninstallEvidence.gatewayA.host, - gatewayA, - GATEWAY_PORT_A, - ); - expect(resultText(beforeUninstallEvidence.gatewayB.host)).toContain( - `openshell-gateway[nemoclaw=${gatewayB};port=${GATEWAY_PORT_B}]`, - ); - expect(resultText(beforeUninstallEvidence.gatewayA.host)).toContain( - `active_pid=${String(gatewayIdentityA.pid)}`, - ); - expect(resultText(beforeUninstallEvidence.gatewayA.host)).not.toContain( - `active_pid=${String(gatewayIdentityB.pid)}`, - ); - expect(resultText(beforeUninstallEvidence.gatewayB.host)).toContain( - `active_pid=${String(gatewayIdentityB.pid)}`, - ); - expect(resultText(beforeUninstallEvidence.gatewayB.host)).not.toContain( - `active_pid=${String(gatewayIdentityA.pid)}`, - ); - }); + const gatewayPair = [ + [gatewayA, GATEWAY_PORT_A], + [gatewayB, GATEWAY_PORT_B], + ] as const; + const beforeEvidence = await captureGatewayEvidence( + host, + sandbox, + gatewayPair, + "phase-4-before-uninstall", + ); + const pidA = evidencePid(beforeEvidence, gatewayA); + const pidB = evidencePid(beforeEvidence, gatewayB); + expect(pidA).toMatch(/^\d+$/); + expect(pidB).toMatch(/^\d+$/); + expect(pidA).not.toBe(pidB); const uninstallB = await command(host, ["uninstall", "--yes", "--destroy-user-data"], { artifactName: "phase-4-uninstall-gateway-b", @@ -858,42 +542,56 @@ test("concurrent gateway ports: onboards two sandboxes on isolated gateways and }); expect(uninstallB.exitCode, resultText(uninstallB)).toBe(0); - const gatewayIdentityAAfterUninstall = readDefaultGatewayIdentity(); - expect(gatewayIdentityAAfterUninstall).toEqual(gatewayIdentityA); - const afterUninstallEvidence = await captureGatewayPairEvidence(host, sandbox, { - gatewayA, - gatewayB, - identityA: gatewayIdentityAAfterUninstall, - identityB: gatewayIdentityB, - stage: "phase-4-after-uninstall", - }); - expect(readGatewayPid(GATEWAY_PORT_B)).toBeNull(); - expect(readGatewayRuntimePid(GATEWAY_PORT_B)).toBeNull(); - expect(afterUninstallEvidence.gatewayA.processIdentity).toEqual( - beforeUninstallEvidence.gatewayA.processIdentity, + const afterEvidence = await captureGatewayEvidence( + host, + sandbox, + gatewayPair, + "phase-4-after-uninstall", ); - expect(afterUninstallEvidence.gatewayB.processIdentity).toBeNull(); - expectOnLinux(() => { - expectStandaloneGatewayArgv( - gatewayIdentityA, - afterUninstallEvidence.gatewayA.host, - gatewayA, - GATEWAY_PORT_A, - ); - expect(resultText(afterUninstallEvidence.gatewayA.host)).toContain( - `active_pid=${String(gatewayIdentityA.pid)}`, + expect(evidencePid(afterEvidence, gatewayA)).toBe(pidA); + expect(evidencePid(afterEvidence, gatewayB)).toBeUndefined(); + + const survivorPhases: string[] = []; + for (let probe = 1; probe <= POST_UNINSTALL_HEALTH_PROBES; probe += 1) { + survivorPhases.push( + await waitForSandboxReady( + sandbox, + SANDBOX_A, + gatewayA, + `phase-4-survivor-probe-${String(probe)}`, + ), ); - expect(resultText(afterUninstallEvidence.gatewayB.host)).not.toContain( - `active_pid=${String(gatewayIdentityB.pid)}`, + await expectPortListening(host, GATEWAY_PORT_A, `phase-4-survivor-port-${String(probe)}`); + const scopedList = await command(host, ["list"], { + artifactName: `phase-4-survivor-list-${String(probe)}`, + env: commandEnv({ NEMOCLAW_GATEWAY_PORT: GATEWAY_PORT_A }), + timeoutMs: 60_000, + }); + expect(scopedList.exitCode, resultText(scopedList)).toBe(0); + expect(outputIncludesSandbox(scopedList.stdout, SANDBOX_A), scopedList.stdout).toBe(true); + const dashboardProbe = await host.command( + "curl", + [ + "-sS", + "-L", + "--max-time", + "10", + "-o", + "/dev/null", + "-w", + "%{http_code}", + `http://127.0.0.1:${DASHBOARD_PORT_A}/`, + ], + { + artifactName: `phase-4-survivor-dashboard-${String(probe)}`, + env: commandEnv(), + timeoutMs: 30_000, + }, ); - }); - - const survivorPhases = await expectSurvivingGatewayHealthyAcrossProbes(host, sandbox, { - dashboardPort: dashboardA as string, - gatewayName: gatewayA, - gatewayPort: GATEWAY_PORT_A, - sandboxName: SANDBOX_A, - }); + expect(dashboardProbe.exitCode, resultText(dashboardProbe)).toBe(0); + expect(dashboardProbe.stdout.trim()).toMatch(/^[23][0-9]{2}$/); + await (probe < POST_UNINSTALL_HEALTH_PROBES ? sleep(PROBE_DELAY_MS) : Promise.resolve()); + } await expectPortNotListening(host, GATEWAY_PORT_B, "phase-4-gateway-port-b-stopped"); const listAAfterUninstallB = await command(host, ["list"], { From e69e94d699c57b7a37299d771017095866dc1eff Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sun, 9 Aug 2026 23:37:05 -0700 Subject: [PATCH 5/5] fix(onboard): scope gateway namespaces to Docker Signed-off-by: Apurv Kumaria --- docs/manage-sandboxes/uninstall-nemoclaw.mdx | 10 ++++++---- docs/reference/commands.mdx | 10 ++++++---- .../run-plan-gateway-service.test.ts | 13 ++++-------- .../onboard/docker-driver-gateway-config.ts | 18 +++++++++++------ .../docker-driver-gateway-launch.test.ts | 15 +++++++++----- .../onboard/docker-driver-gateway-launch.ts | 3 +++ .../e2e/live/concurrent-gateway-ports.test.ts | 20 +++++++++---------- 7 files changed, 51 insertions(+), 38 deletions(-) diff --git a/docs/manage-sandboxes/uninstall-nemoclaw.mdx b/docs/manage-sandboxes/uninstall-nemoclaw.mdx index b811975f94c..0c770a3e001 100644 --- a/docs/manage-sandboxes/uninstall-nemoclaw.mdx +++ b/docs/manage-sandboxes/uninstall-nemoclaw.mdx @@ -83,12 +83,14 @@ Rerun `NEMOCLAW_GATEWAY_PORT= $$nemoclaw uninstall` with the gateway port For an externally supervised authority, uninstall preserves the local gateway state used by the running process in both full and gateway-scoped cleanup. It also preserves the gateway process, supervisor resources, marked Linux unit, Docker resources, OpenShell binaries, and the declared external state directory. A custom-port uninstall does not stop or remove the default gateway service or its environment file. -Before scoped cleanup stops a host gateway process or managed default gateway service, NemoClaw requires two namespace proofs. -The selected gateway configuration and any running gateway process must use the state-root-specific OpenShell sandbox namespace that NemoClaw generated. -If either proof is absent, uninstall exits nonzero before it signals the host gateway. +Before scoped cleanup stops a Docker gateway process, including a managed default gateway service, NemoClaw requires two Docker namespace proofs. +The selected Docker gateway configuration and any running gateway process must use the state-root-specific OpenShell sandbox namespace that NemoClaw generated. +Because the supported OpenShell Podman schema does not expose `sandbox_namespace`, scoped Podman uninstall fails closed before signaling and preserves the gateway runtime evidence and local state. +Full single-gateway Podman uninstall continues to use normal graceful teardown. +For Docker, if either proof is absent, uninstall exits nonzero before it signals the host gateway. NemoClaw preserves the gateway runtime evidence and local state. Keep that state intact. -Restore the selected gateway through the supported install or onboarding recovery flow so it restarts with the generated configuration. +Restore the selected Docker gateway through the supported install or onboarding recovery flow so it restarts with the generated configuration. Verify every gateway with `openshell gateway list`. Retry the scoped uninstall. Do not add `sandbox_namespace` manually to a live gateway configuration because the running process can still be using its previous namespace. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index b93683aeff9..f1956c936ff 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3892,12 +3892,14 @@ Rerun `NEMOCLAW_GATEWAY_PORT= $$nemoclaw uninstall` with the gateway port For an externally supervised authority, uninstall preserves the selected local gateway state in both full and gateway-scoped cleanup. It also preserves the gateway process, supervisor resources, marked Linux unit, Docker resources, OpenShell binaries, and the declared external state directory. A custom-port uninstall does not stop or remove the default gateway service or its environment file. -Before scoped cleanup stops a host gateway process or managed default gateway service, NemoClaw requires two namespace proofs. -The selected gateway configuration and any running gateway process must use the state-root-specific OpenShell sandbox namespace that NemoClaw generated. -If either proof is absent, uninstall exits nonzero before it signals the host gateway. +Before scoped cleanup stops a Docker gateway process, including a managed default gateway service, NemoClaw requires two Docker namespace proofs. +The selected Docker gateway configuration and any running gateway process must use the state-root-specific OpenShell sandbox namespace that NemoClaw generated. +Because the supported OpenShell Podman schema does not expose `sandbox_namespace`, scoped Podman uninstall fails closed before signaling and preserves the gateway runtime evidence and local state. +Full single-gateway Podman uninstall continues to use normal graceful teardown. +For Docker, if either proof is absent, uninstall exits nonzero before it signals the host gateway. NemoClaw preserves the gateway runtime evidence and local state. Keep that state intact. -Restore the selected gateway through the supported install or onboarding recovery flow so it restarts with the generated configuration. +Restore the selected Docker gateway through the supported install or onboarding recovery flow so it restarts with the generated configuration. Verify every gateway with `openshell gateway list`. Retry the scoped uninstall. Do not add `sandbox_namespace` manually to a live gateway configuration because the running process can still be using its previous namespace. diff --git a/src/lib/actions/uninstall/run-plan-gateway-service.test.ts b/src/lib/actions/uninstall/run-plan-gateway-service.test.ts index dc3cc11f020..1d7dd550674 100644 --- a/src/lib/actions/uninstall/run-plan-gateway-service.test.ts +++ b/src/lib/actions/uninstall/run-plan-gateway-service.test.ts @@ -7,6 +7,7 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { gatewayIdForStateDir } from "../../onboard/docker-driver-gateway-config"; import { getNemoclawOpenShellGatewayUserServicePath, getOpenShellUserConfigHome, @@ -85,18 +86,12 @@ function writeSelectedSandboxRegistry(test: Fixture, sandboxName: string): strin } function writeGatewayState(test: Fixture): string { - const configPath = path.join( - test.home, - ".local", - "state", - "nemoclaw", - "openshell-docker-gateway", - "openshell-gateway.toml", - ); + const stateDir = path.join(test.home, ".local", "state", "nemoclaw", "openshell-docker-gateway"); + const configPath = path.join(stateDir, "openshell-gateway.toml"); fs.mkdirSync(path.dirname(configPath), { recursive: true }); fs.writeFileSync( configPath, - '[openshell.drivers.docker]\nsandbox_namespace = "nemoclaw-openshell-docker-gateway"\n', + `[openshell.drivers.docker]\nsandbox_namespace = "${gatewayIdForStateDir(stateDir)}"\n`, ); return configPath; } diff --git a/src/lib/onboard/docker-driver-gateway-config.ts b/src/lib/onboard/docker-driver-gateway-config.ts index 191af0d3527..771b6fd066b 100644 --- a/src/lib/onboard/docker-driver-gateway-config.ts +++ b/src/lib/onboard/docker-driver-gateway-config.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { randomBytes } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { @@ -58,10 +58,12 @@ function cleanupStaleAtomicFileTemps(dir: string, basename: string): void { export function gatewayIdForStateDir(stateDir: string): string { const leaf = path.basename(path.resolve(stateDir)).replace(/[^A-Za-z0-9_.-]/g, "-"); - return leaf ? `nemoclaw-${leaf}` : "nemoclaw"; + const scope = `${String(process.getuid?.() ?? "unknown")}\0${path.resolve(stateDir)}`; + const suffix = createHash("sha256").update(scope).digest("hex").slice(0, 12); + return `nemoclaw-${leaf || "gateway"}-${suffix}`; } -/** Prove that a NemoClaw-owned gateway config uses its state-scoped namespace. */ +/** Prove that a NemoClaw-owned Docker gateway config uses its state-scoped namespace. */ export function hasStateScopedSandboxNamespace(stateDir: string): boolean { if (typeof process.getuid !== "function" || typeof fs.constants.O_NOFOLLOW !== "number") { return false; @@ -91,7 +93,7 @@ export function hasStateScopedSandboxNamespace(stateDir: string): boolean { .filter((line) => { const trimmed = line.trim(); if (trimmed.startsWith("[") && trimmed.endsWith("]")) { - inDriverTable = /^\[openshell\.drivers\.(?:docker|podman)\]$/.test(trimmed); + inDriverTable = trimmed === "[openshell.drivers.docker]"; return false; } return inDriverTable && trimmed.startsWith("sandbox_namespace ="); @@ -121,7 +123,7 @@ export function buildDockerDriverGatewayConfigToml( const driver = gatewayEnv.OPENSHELL_DRIVERS === "podman" ? "podman" : "docker"; const localTlsDir = jwtBundle ? gatewayLocalTlsDir(gatewayEnv) : undefined; const dockerEntries: [string, string | undefined][] = [ - ["sandbox_namespace", gatewayId], + ["sandbox_namespace", driver === "docker" ? gatewayId : undefined], ["grpc_endpoint", gatewayEnv.OPENSHELL_GRPC_ENDPOINT], ["host_gateway_ip", driver === "podman" ? PORTABLE_HOST_GATEWAY_IP : undefined], ["socket_path", driver === "podman" ? gatewayEnv.OPENSHELL_PODMAN_SOCKET : undefined], @@ -214,6 +216,10 @@ export function prepareDockerDriverGatewayConfigEnv( gatewayEnv, sandboxBin, ); - gatewayEnv[NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV] = gatewayIdForStateDir(stateDir); + if (gatewayEnv.OPENSHELL_DRIVERS === "podman") { + delete gatewayEnv[NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV]; + } else { + gatewayEnv[NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV] = gatewayIdForStateDir(stateDir); + } return gatewayEnv; } diff --git a/src/lib/onboard/docker-driver-gateway-launch.test.ts b/src/lib/onboard/docker-driver-gateway-launch.test.ts index 41aa8e3f32f..772dc5aeffa 100644 --- a/src/lib/onboard/docker-driver-gateway-launch.test.ts +++ b/src/lib/onboard/docker-driver-gateway-launch.test.ts @@ -114,9 +114,9 @@ describe("docker-driver-gateway-launch", () => { const defaultNamespace = gatewayIdForStateDir("/tmp/openshell-docker-gateway"); const alternateNamespace = gatewayIdForStateDir("/tmp/openshell-docker-gateway-18080"); - expect(defaultNamespace).toBe("nemoclaw-openshell-docker-gateway"); - expect(alternateNamespace).toBe("nemoclaw-openshell-docker-gateway-18080"); + expect(defaultNamespace).toMatch(/^nemoclaw-openshell-docker-gateway-[a-f0-9]{12}$/); expect(defaultNamespace).not.toBe(alternateNamespace); + expect(gatewayIdForStateDir("/tmp/a/gateway")).not.toBe(gatewayIdForStateDir("/tmp/b/gateway")); }); it("writes the exact rootless socket only for the Podman driver", () => { @@ -130,6 +130,7 @@ describe("docker-driver-gateway-launch", () => { expect(toml).toContain("[openshell.drivers.podman]"); expect(toml).toContain('socket_path = "/run/user/1001/podman/podman.sock"'); + expect(toml).not.toContain("sandbox_namespace"); }); it("rejects wildcard binds for direct host gateway launches", () => { @@ -239,20 +240,24 @@ describe("docker-driver-gateway-launch", () => { }); }); - it("scrubs stale auth-disable env from direct host gateway launches", () => { + it("scrubs stale internal env from direct host gateway launches", () => { withTempBinaries(({ dir, gatewayBin }) => { const launch = buildDockerDriverGatewayLaunch({ gatewayBin, stateDir: dir, platform: "linux", - env: { OPENSHELL_DISABLE_GATEWAY_AUTH: "true" }, + env: { + OPENSHELL_DISABLE_GATEWAY_AUTH: "true", + [NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV]: "stale", + }, hostGlibcVersion: "2.39", requiredGlibcVersions: ["2.39"], - gatewayEnv: { OPENSHELL_DRIVERS: "docker" }, + gatewayEnv: { OPENSHELL_DRIVERS: "podman" }, }); expect(launch.mode).toBe("host"); expect(launch.env.OPENSHELL_DISABLE_GATEWAY_AUTH).toBeUndefined(); + expect(launch.env[NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV]).toBeUndefined(); }); }); }); diff --git a/src/lib/onboard/docker-driver-gateway-launch.ts b/src/lib/onboard/docker-driver-gateway-launch.ts index 83ddd0ef0f4..43847d53f16 100644 --- a/src/lib/onboard/docker-driver-gateway-launch.ts +++ b/src/lib/onboard/docker-driver-gateway-launch.ts @@ -115,6 +115,9 @@ function buildGatewayProcessEnv( if (!("OPENSHELL_DISABLE_GATEWAY_AUTH" in gatewayEnv)) { delete env.OPENSHELL_DISABLE_GATEWAY_AUTH; } + if (!(NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV in gatewayEnv)) { + delete env[NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV]; + } return env; } diff --git a/test/e2e/live/concurrent-gateway-ports.test.ts b/test/e2e/live/concurrent-gateway-ports.test.ts index 15fc46b93a7..49464ebca24 100644 --- a/test/e2e/live/concurrent-gateway-ports.test.ts +++ b/test/e2e/live/concurrent-gateway-ports.test.ts @@ -92,11 +92,11 @@ async function command( }); } -function evidencePid(evidence: string, gateway: string): string | undefined { +function gatewayProcessEvidence(evidence: string, gateway: string): string | undefined { return evidence .split(`gateway=${gateway}\n`)[1] ?.split("gateway=")[0] - ?.match(/^active_pid=(\d+)$/m)?.[1]; + ?.match(/^active_pid=\d+\nexecutable=.*\n.*$/m)?.[0]; } async function captureGatewayEvidence( @@ -113,7 +113,7 @@ async function captureGatewayEvidence( ' test ! -r "$state/openshell-gateway.pid" || pid="$(tr -d "[:space:]" < "$state/openshell-gateway.pid")"', ' if test -z "$pid" && test "$port" = 8080 && command -v systemctl >/dev/null; then for service in openshell-gateway nemoclaw-openshell-gateway; do candidate="$(systemctl --user show "$service" --property=MainPID --value 2>/dev/null || true)"; test "${candidate:-0}" -le 0 || { pid="$candidate"; break; }; done; fi', ' printf "gateway=%s\\nport=%s\\npid_file=%s\\n" "$gateway" "$port" "${pid:-}"', - ' if test -n "$pid" && ps -p "$pid" >/dev/null 2>&1; then printf "active_pid=%s\\n" "$pid"; ps -p "$pid" -o pid=,ppid=,uid=,lstart=,args=; fi', + ' if test -n "$pid" && ps -p "$pid" >/dev/null 2>&1; then printf "active_pid=%s\\nexecutable=%s\\n" "$pid" "$(readlink -f "/proc/$pid/exe" 2>/dev/null || true)"; ps -p "$pid" -o pid=,ppid=,uid=,lstart=,args=; fi', ' printf "listeners=\\n"; ss -H -ltnp 2>&1 | grep -E "[:.]$port\\b" || true', ' printf "runtime=\\n"; test ! -r "$state/runtime.json" || cat "$state/runtime.json"', ' printf "namespace=\\n"; test ! -r "$state/openshell-gateway.toml" || grep "^sandbox_namespace" "$state/openshell-gateway.toml" || true', @@ -529,11 +529,11 @@ test("concurrent gateway ports: onboards two sandboxes on isolated gateways and gatewayPair, "phase-4-before-uninstall", ); - const pidA = evidencePid(beforeEvidence, gatewayA); - const pidB = evidencePid(beforeEvidence, gatewayB); - expect(pidA).toMatch(/^\d+$/); - expect(pidB).toMatch(/^\d+$/); - expect(pidA).not.toBe(pidB); + const processA = gatewayProcessEvidence(beforeEvidence, gatewayA); + const processB = gatewayProcessEvidence(beforeEvidence, gatewayB); + expect(processA).toBeDefined(); + expect(processB).toBeDefined(); + expect(processA).not.toBe(processB); const uninstallB = await command(host, ["uninstall", "--yes", "--destroy-user-data"], { artifactName: "phase-4-uninstall-gateway-b", @@ -548,8 +548,8 @@ test("concurrent gateway ports: onboards two sandboxes on isolated gateways and gatewayPair, "phase-4-after-uninstall", ); - expect(evidencePid(afterEvidence, gatewayA)).toBe(pidA); - expect(evidencePid(afterEvidence, gatewayB)).toBeUndefined(); + expect(gatewayProcessEvidence(afterEvidence, gatewayA)).toBe(processA); + expect(gatewayProcessEvidence(afterEvidence, gatewayB)).toBeUndefined(); const survivorPhases: string[] = []; for (let probe = 1; probe <= POST_UNINSTALL_HEALTH_PROBES; probe += 1) {