diff --git a/docs/manage-sandboxes/uninstall-nemoclaw.mdx b/docs/manage-sandboxes/uninstall-nemoclaw.mdx index 5666e571071..0c770a3e001 100644 --- a/docs/manage-sandboxes/uninstall-nemoclaw.mdx +++ b/docs/manage-sandboxes/uninstall-nemoclaw.mdx @@ -83,6 +83,17 @@ Rerun `NEMOCLAW_GATEWAY_PORT= $$nemoclaw uninstall` with the gateway port For an externally supervised authority, uninstall preserves the local gateway state used by the running process in both full and gateway-scoped cleanup. It also preserves the gateway process, supervisor resources, marked Linux unit, Docker resources, OpenShell binaries, and the declared external state directory. A custom-port uninstall does not stop or remove the default gateway service or its environment file. +Before scoped cleanup stops a Docker gateway process, including a managed default gateway service, NemoClaw requires two Docker namespace proofs. +The selected Docker gateway configuration and any running gateway process must use the state-root-specific OpenShell sandbox namespace that NemoClaw generated. +Because the supported OpenShell Podman schema does not expose `sandbox_namespace`, scoped Podman uninstall fails closed before signaling and preserves the gateway runtime evidence and local state. +Full single-gateway Podman uninstall continues to use normal graceful teardown. +For Docker, if either proof is absent, uninstall exits nonzero before it signals the host gateway. +NemoClaw preserves the gateway runtime evidence and local state. +Keep that state intact. +Restore the selected Docker gateway through the supported install or onboarding recovery flow so it restarts with the generated configuration. +Verify every gateway with `openshell gateway list`. +Retry the scoped uninstall. +Do not add `sandbox_namespace` manually to a live gateway configuration because the running process can still be using its previous namespace. In this section, `` is `~/.nemoclaw/` for the default gateway or `~/.nemoclaw/gateways//` for a non-default gateway. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 2fd0fbdf684..f1956c936ff 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3892,6 +3892,17 @@ Rerun `NEMOCLAW_GATEWAY_PORT= $$nemoclaw uninstall` with the gateway port For an externally supervised authority, uninstall preserves the selected local gateway state in both full and gateway-scoped cleanup. It also preserves the gateway process, supervisor resources, marked Linux unit, Docker resources, OpenShell binaries, and the declared external state directory. A custom-port uninstall does not stop or remove the default gateway service or its environment file. +Before scoped cleanup stops a Docker gateway process, including a managed default gateway service, NemoClaw requires two Docker namespace proofs. +The selected Docker gateway configuration and any running gateway process must use the state-root-specific OpenShell sandbox namespace that NemoClaw generated. +Because the supported OpenShell Podman schema does not expose `sandbox_namespace`, scoped Podman uninstall fails closed before signaling and preserves the gateway runtime evidence and local state. +Full single-gateway Podman uninstall continues to use normal graceful teardown. +For Docker, if either proof is absent, uninstall exits nonzero before it signals the host gateway. +NemoClaw preserves the gateway runtime evidence and local state. +Keep that state intact. +Restore the selected Docker gateway through the supported install or onboarding recovery flow so it restarts with the generated configuration. +Verify every gateway with `openshell gateway list`. +Retry the scoped uninstall. +Do not add `sandbox_namespace` manually to a live gateway configuration because the running process can still be using its previous namespace. ##### Uninstalling Every Gateway Port diff --git a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts index 48cddd87208..adc7388fd76 100644 --- a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts +++ b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts @@ -23,6 +23,7 @@ function ok(stdout = ""): RunResult { function withManagedGatewayAuthority(deps: UninstallRunDeps): UninstallRunDeps { return { + isPortFree: () => true, resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ gatewayName, gatewayPort, diff --git a/src/lib/actions/uninstall/run-plan-gateway-service.test.ts b/src/lib/actions/uninstall/run-plan-gateway-service.test.ts index 84e88b18730..1d7dd550674 100644 --- a/src/lib/actions/uninstall/run-plan-gateway-service.test.ts +++ b/src/lib/actions/uninstall/run-plan-gateway-service.test.ts @@ -7,6 +7,7 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { gatewayIdForStateDir } from "../../onboard/docker-driver-gateway-config"; import { getNemoclawOpenShellGatewayUserServicePath, getOpenShellUserConfigHome, @@ -48,6 +49,7 @@ function fixture(useXdg = false): Fixture { } function writeManagedService(test: Fixture): string { + writeGatewayState(test); const servicePath = getNemoclawOpenShellGatewayUserServicePath(test.home, test.env); fs.mkdirSync(path.dirname(servicePath), { recursive: true }); fs.writeFileSync( @@ -84,16 +86,13 @@ function writeSelectedSandboxRegistry(test: Fixture, sandboxName: string): strin } function writeGatewayState(test: Fixture): string { - const configPath = path.join( - test.home, - ".local", - "state", - "nemoclaw", - "openshell-docker-gateway", - "openshell-gateway.toml", - ); + const stateDir = path.join(test.home, ".local", "state", "nemoclaw", "openshell-docker-gateway"); + const configPath = path.join(stateDir, "openshell-gateway.toml"); fs.mkdirSync(path.dirname(configPath), { recursive: true }); - fs.writeFileSync(configPath, 'listen_address = "127.0.0.1:8080"\n'); + fs.writeFileSync( + configPath, + `[openshell.drivers.docker]\nsandbox_namespace = "${gatewayIdForStateDir(stateDir)}"\n`, + ); return configPath; } @@ -109,6 +108,7 @@ function uninstall( { env: test.env, existsSync: (target) => String(target).startsWith(test.root) && fs.existsSync(target), + isPortFree: () => true, isTty: false, platform: "linux", resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ @@ -128,7 +128,9 @@ function uninstall( run: (command, args, options) => command === "openshell" && args[0] === "gateway" && args[1] === "list" ? ok(JSON.stringify(gateways)) - : run(command, args, options), + : command === "systemctl" && args.includes("--property=MainPID") + ? ok("0\n") + : run(command, args, options), }, ); } @@ -245,6 +247,30 @@ describe("uninstall OpenShell gateway user service", () => { expect(fs.existsSync(servicePath)).toBe(false); }); + it("does not signal a scoped service whose sandbox namespace is unproven (#8663)", () => { + const test = fixture(true); + const servicePath = writeManagedService(test); + fs.writeFileSync(writeGatewayState(test), "[openshell.drivers.docker]\n"); + const calls: string[][] = []; + + const result = uninstall( + test, + false, + { + commandExists: (command) => command === "systemctl", + run: (command, args) => { + calls.push([command, ...args]); + return ok(); + }, + }, + [{ name: "nemoclaw" }, { name: "nemoclaw-8081" }], + ); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(servicePath)).toBe(true); + expect(calls.some(([command]) => command === "systemctl")).toBe(false); + }); + it("preserves the marked Linux unit when scoped sandbox deletion fails (#8220)", () => { const test = fixture(true); const servicePath = writeManagedService(test); diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index b2e6f5be30d..e5808f3940b 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -58,6 +58,8 @@ import { resolveGatewayTeardownAuthority, } from "../../onboard/gateway-teardown-authority"; import { + hasStateScopedSandboxNamespace, + processUsesStateScopedSandboxNamespace, type StopHostGatewayOptions, stopHostGatewayProcesses, } from "../../onboard/host-gateway-process"; @@ -98,12 +100,14 @@ export interface UninstallRunDeps { error?: (message: string) => void; existsSync?: (target: string) => boolean; fs?: FileSystemDeps; + isPortFree?: (port: number) => boolean; isTty?: boolean; kill?: (pid: number, signal?: NodeJS.Signals | number) => boolean; log?: (message: string) => void; openRegularFile?: typeof openRegularFileNoFollow; platform?: NodeJS.Platform; readProcessArgv?: (pid: number) => readonly string[] | null; + readProcessEnvironment?: (pid: number) => Record | null; readLine?: () => string | null; requireCompleteGatewayProcessCleanup?: boolean; resolveGatewayTeardownAuthority?: GatewayTeardownAuthorityResolver; @@ -406,12 +410,14 @@ interface UninstallRuntime { env: NodeJS.ProcessEnv; error: (message: string) => void; existsSync: (target: string) => boolean; + isPortFree: ((port: number) => boolean) | undefined; isTty: boolean; kill: (pid: number, signal?: NodeJS.Signals | number) => boolean; log: (message: string) => void; openRegularFile: typeof openRegularFileNoFollow; platform: NodeJS.Platform; readProcessArgv: ((pid: number) => readonly string[] | null) | undefined; + readProcessEnvironment: ((pid: number) => Record | null) | undefined; readLine: () => string | null; requireCompleteGatewayProcessCleanup: boolean; resolveGatewayTeardownAuthority: GatewayTeardownAuthorityResolver; @@ -432,6 +438,7 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime { env, error: deps.error ?? ((message) => console.error(message)), existsSync: deps.existsSync ?? ((target) => fs.existsSync(target)), + isPortFree: deps.isPortFree, // Side-effect-free TTY check + EAGAIN-tolerant reader; the // process.stdin/non-blocking-fd hazard is documented in core/stdin.ts. isTty: deps.isTty ?? isStdinTty(), @@ -449,6 +456,7 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime { openRegularFile: deps.openRegularFile ?? openRegularFileNoFollow, platform: deps.platform ?? process.platform, readProcessArgv: deps.readProcessArgv, + readProcessEnvironment: deps.readProcessEnvironment, readLine: deps.readLine ?? readLineFromStdin, requireCompleteGatewayProcessCleanup: deps.requireCompleteGatewayProcessCleanup ?? false, resolveGatewayTeardownAuthority: @@ -962,7 +970,10 @@ function stopOrphanedOpenShell(runtime: UninstallRuntime): void { } } -function removeNemoclawOpenShellGatewayUserService(runtime: UninstallRuntime): boolean { +function removeNemoclawOpenShellGatewayUserService( + runtime: UninstallRuntime, + scopedStateDir?: string, +): boolean { if (runtime.platform !== "linux") return true; const servicePath = getNemoclawOpenShellGatewayUserServicePath( runtime.env.HOME || os.homedir(), @@ -1000,6 +1011,33 @@ function removeNemoclawOpenShellGatewayUserService(runtime: UninstallRuntime): b const hasSystemctl = runtime.commandExists("systemctl"); if (hasSystemctl) { + if (scopedStateDir !== undefined) { + const inspected = runtime.run( + "systemctl", + [ + "--user", + "show", + NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE, + "--property=MainPID", + "--value", + ], + { env: runtime.env }, + ); + const mainPid = Number(inspected.stdout.trim()); + if ( + !hasStateScopedSandboxNamespace(scopedStateDir) || + inspected.status !== 0 || + !inspected.stdout.trim() || + !Number.isSafeInteger(mainPid) || + mainPid < 0 || + (mainPid > 0 && !processUsesStateScopedSandboxNamespace(mainPid, scopedStateDir, runtime)) + ) { + runtime.warn( + "Refusing scoped gateway service stop because its loaded sandbox namespace cannot be proven.", + ); + return false; + } + } const disabled = runtime.run( "systemctl", ["--user", "disable", "--now", NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE], @@ -1043,10 +1081,13 @@ function removeManagedDefaultGatewayUserService( runtime: UninstallRuntime, options: UninstallRunOptions, externallySupervised: boolean, + selectedStateDir?: string, + scoped = false, ): boolean { - return options.keepOpenShell || externallySupervised || GATEWAY_PORT !== DEFAULT_GATEWAY_PORT - ? true - : removeNemoclawOpenShellGatewayUserService(runtime); + if (options.keepOpenShell || externallySupervised || GATEWAY_PORT !== DEFAULT_GATEWAY_PORT) { + return true; + } + return removeNemoclawOpenShellGatewayUserService(runtime, scoped ? selectedStateDir : undefined); } function removeNemoclawOpenShellGatewayEnv( @@ -2099,7 +2140,15 @@ function executePlan( return { ok: false }; } if (scopedToSelectedGateway && !options.keepOpenShell && !externallySupervised) { - if (!removeManagedDefaultGatewayUserService(runtime, options, externallySupervised)) { + if ( + !removeManagedDefaultGatewayUserService( + runtime, + options, + externallySupervised, + paths.selectedGatewayLocalStateDir, + true, + ) + ) { return { ok: false }; } stopHostGatewayProcessesForUninstall(runtime, { @@ -2108,7 +2157,9 @@ function executePlan( openShellGatewayName: options.gatewayName || resolveGatewayName(GATEWAY_PORT), openShellGatewayPort: GATEWAY_PORT, preserveRuntimeFilesOnNonMatching: true, + scopedGatewayStop: true, stateDir: paths.selectedGatewayLocalStateDir, + usePgrepFallback: false, }); } else if (scopedToSelectedGateway && externallySupervised) { runtime.log("Kept the externally supervised OpenShell gateway process running."); @@ -2258,9 +2309,17 @@ function stopHostGatewayProcessesForUninstall( log: runtime.log, warn: runtime.warn, commandExists: runtime.commandExists, + isPortFree: runtime.isPortFree, + readProcessEnvironment: runtime.readProcessEnvironment, }, options, ); + if (options.scopedGatewayStop && (result.ownershipFailures?.length || result.failed.length)) { + runtime.error( + "Cannot prove ownership of or stop the selected host gateway process; retaining its runtime evidence.", + ); + throw new IncompleteHostGatewayCleanupError(); + } if (!runtime.requireCompleteGatewayProcessCleanup) return; if (result.failed.length === 0 && result.orphanScanComplete !== false) return; runtime.error("Cannot continue uninstall because host gateway process cleanup did not complete."); diff --git a/src/lib/onboard/docker-driver-gateway-config.ts b/src/lib/onboard/docker-driver-gateway-config.ts index a38938225b5..771b6fd066b 100644 --- a/src/lib/onboard/docker-driver-gateway-config.ts +++ b/src/lib/onboard/docker-driver-gateway-config.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { randomBytes } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { @@ -16,6 +16,7 @@ export { ensureDockerDriverGatewayJwtBundle } from "./docker-driver-gateway-jwt- // See docs/security/openshell-0.0.72-compatibility-review.mdx for the source-of-truth review. export const DOCKER_DRIVER_GATEWAY_CONFIG_NAME = "openshell-gateway.toml"; export const DOCKER_DRIVER_GATEWAY_JWT_TTL_SECS = 0; +export const NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV = "NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE"; function tomlString(value: string): string { return JSON.stringify(value); @@ -55,9 +56,54 @@ function cleanupStaleAtomicFileTemps(dir: string, basename: string): void { } } -function gatewayIdForStateDir(stateDir: string): string { +export function gatewayIdForStateDir(stateDir: string): string { const leaf = path.basename(path.resolve(stateDir)).replace(/[^A-Za-z0-9_.-]/g, "-"); - return leaf ? `nemoclaw-${leaf}` : "nemoclaw"; + const scope = `${String(process.getuid?.() ?? "unknown")}\0${path.resolve(stateDir)}`; + const suffix = createHash("sha256").update(scope).digest("hex").slice(0, 12); + return `nemoclaw-${leaf || "gateway"}-${suffix}`; +} + +/** Prove that a NemoClaw-owned Docker gateway config uses its state-scoped namespace. */ +export function hasStateScopedSandboxNamespace(stateDir: string): boolean { + if (typeof process.getuid !== "function" || typeof fs.constants.O_NOFOLLOW !== "number") { + return false; + } + const configPath = path.join(stateDir, DOCKER_DRIVER_GATEWAY_CONFIG_NAME); + let descriptor: number | undefined; + try { + const state = fs.lstatSync(stateDir); + descriptor = fs.openSync(configPath, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + const config = fs.fstatSync(descriptor); + if ( + !state.isDirectory() || + state.isSymbolicLink() || + !config.isFile() || + config.nlink !== 1 || + state.uid !== process.getuid() || + config.uid !== state.uid || + config.size > 64 * 1024 + ) { + return false; + } + const expected = `sandbox_namespace = ${tomlString(gatewayIdForStateDir(stateDir))}`; + let inDriverTable = false; + const matches = fs + .readFileSync(descriptor, "utf-8") + .split(/\r?\n/) + .filter((line) => { + const trimmed = line.trim(); + if (trimmed.startsWith("[") && trimmed.endsWith("]")) { + inDriverTable = trimmed === "[openshell.drivers.docker]"; + return false; + } + return inDriverTable && trimmed.startsWith("sandbox_namespace ="); + }); + return matches.length === 1 && matches[0]?.trim() === expected; + } catch { + return false; + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + } } function gatewayLocalTlsDir(gatewayEnv: Record): string { @@ -77,6 +123,7 @@ export function buildDockerDriverGatewayConfigToml( const driver = gatewayEnv.OPENSHELL_DRIVERS === "podman" ? "podman" : "docker"; const localTlsDir = jwtBundle ? gatewayLocalTlsDir(gatewayEnv) : undefined; const dockerEntries: [string, string | undefined][] = [ + ["sandbox_namespace", driver === "docker" ? gatewayId : undefined], ["grpc_endpoint", gatewayEnv.OPENSHELL_GRPC_ENDPOINT], ["host_gateway_ip", driver === "podman" ? PORTABLE_HOST_GATEWAY_IP : undefined], ["socket_path", driver === "podman" ? gatewayEnv.OPENSHELL_PODMAN_SOCKET : undefined], @@ -169,5 +216,10 @@ export function prepareDockerDriverGatewayConfigEnv( gatewayEnv, sandboxBin, ); + if (gatewayEnv.OPENSHELL_DRIVERS === "podman") { + delete gatewayEnv[NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV]; + } else { + gatewayEnv[NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV] = gatewayIdForStateDir(stateDir); + } return gatewayEnv; } diff --git a/src/lib/onboard/docker-driver-gateway-env.ts b/src/lib/onboard/docker-driver-gateway-env.ts index 573abe56f7d..f77ffd8bd8b 100644 --- a/src/lib/onboard/docker-driver-gateway-env.ts +++ b/src/lib/onboard/docker-driver-gateway-env.ts @@ -16,6 +16,7 @@ import { DEFAULT_GATEWAY_PORT, GATEWAY_PORT } from "../core/ports"; import { isSupportedGatewayDockerHost } from "../domain/docker-host"; import { DOCKER_DRIVER_GATEWAY_JWT_TTL_SECS, + NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV, prepareDockerDriverGatewayConfigEnv, } from "./docker-driver-gateway-config"; import { buildDockerDriverGatewayLocalTlsEnv } from "./docker-driver-gateway-local-tls"; @@ -52,6 +53,7 @@ export const DOCKER_DRIVER_GATEWAY_RUNTIME_ENV_KEYS = [ "OPENSHELL_GATEWAY_CONFIG", "OPENSHELL_VM_DRIVER_STATE_DIR", "OPENSHELL_DRIVER_DIR", + NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV, "NETAVARK_FW", ] as const; diff --git a/src/lib/onboard/docker-driver-gateway-launch.test.ts b/src/lib/onboard/docker-driver-gateway-launch.test.ts index e9a637cd048..772dc5aeffa 100644 --- a/src/lib/onboard/docker-driver-gateway-launch.test.ts +++ b/src/lib/onboard/docker-driver-gateway-launch.test.ts @@ -6,7 +6,10 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; - +import { + gatewayIdForStateDir, + NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV, +} from "./docker-driver-gateway-config"; import { buildDockerDriverGatewayConfigToml, buildDockerDriverGatewayLaunch, @@ -100,12 +103,22 @@ describe("docker-driver-gateway-launch", () => { ); expect(toml).toContain('compute_drivers = ["docker"]'); + expect(toml).toContain('sandbox_namespace = "nemoclaw"'); expect(toml).toContain('grpc_endpoint = "https://127.0.0.1:8080"'); expect(toml).toContain('network_name = "openshell-docker"'); expect(toml).toContain('supervisor_image = "ghcr.io/nvidia/openshell/supervisor:0.0.44"'); expect(toml).toContain('supervisor_bin = "/home/shadeform/.local/bin/openshell-sandbox"'); }); + it("assigns different sandbox namespaces to different gateway state roots (#8663)", () => { + const defaultNamespace = gatewayIdForStateDir("/tmp/openshell-docker-gateway"); + const alternateNamespace = gatewayIdForStateDir("/tmp/openshell-docker-gateway-18080"); + + expect(defaultNamespace).toMatch(/^nemoclaw-openshell-docker-gateway-[a-f0-9]{12}$/); + expect(defaultNamespace).not.toBe(alternateNamespace); + expect(gatewayIdForStateDir("/tmp/a/gateway")).not.toBe(gatewayIdForStateDir("/tmp/b/gateway")); + }); + it("writes the exact rootless socket only for the Podman driver", () => { const toml = buildDockerDriverGatewayConfigToml({ OPENSHELL_DRIVERS: "podman", @@ -117,6 +130,7 @@ describe("docker-driver-gateway-launch", () => { expect(toml).toContain("[openshell.drivers.podman]"); expect(toml).toContain('socket_path = "/run/user/1001/podman/podman.sock"'); + expect(toml).not.toContain("sandbox_namespace"); }); it("rejects wildcard binds for direct host gateway launches", () => { @@ -156,6 +170,9 @@ describe("docker-driver-gateway-launch", () => { expect(identity.launch?.mode).toBe("host"); expect(identity.driftGatewayBin).toBe(gatewayBin); expect(identity.desiredEnv.OPENSHELL_DOCKER_SUPERVISOR_BIN).toBe(sandboxBin); + expect(identity.desiredEnv[NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV]).toBe( + gatewayIdForStateDir(dir), + ); expect(identity.desiredEnv.OPENSHELL_GATEWAY_CONFIG).toBe( path.join(dir, "openshell-gateway.toml"), ); @@ -223,20 +240,24 @@ describe("docker-driver-gateway-launch", () => { }); }); - it("scrubs stale auth-disable env from direct host gateway launches", () => { + it("scrubs stale internal env from direct host gateway launches", () => { withTempBinaries(({ dir, gatewayBin }) => { const launch = buildDockerDriverGatewayLaunch({ gatewayBin, stateDir: dir, platform: "linux", - env: { OPENSHELL_DISABLE_GATEWAY_AUTH: "true" }, + env: { + OPENSHELL_DISABLE_GATEWAY_AUTH: "true", + [NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV]: "stale", + }, hostGlibcVersion: "2.39", requiredGlibcVersions: ["2.39"], - gatewayEnv: { OPENSHELL_DRIVERS: "docker" }, + gatewayEnv: { OPENSHELL_DRIVERS: "podman" }, }); expect(launch.mode).toBe("host"); expect(launch.env.OPENSHELL_DISABLE_GATEWAY_AUTH).toBeUndefined(); + expect(launch.env[NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV]).toBeUndefined(); }); }); }); diff --git a/src/lib/onboard/docker-driver-gateway-launch.ts b/src/lib/onboard/docker-driver-gateway-launch.ts index 271875c8ee2..43847d53f16 100644 --- a/src/lib/onboard/docker-driver-gateway-launch.ts +++ b/src/lib/onboard/docker-driver-gateway-launch.ts @@ -12,6 +12,7 @@ import { } from "./docker-driver-gateway-compat"; import { buildDockerDriverGatewayConfigToml, + NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV, prepareDockerDriverGatewayConfigEnv, } from "./docker-driver-gateway-config"; import { @@ -114,6 +115,9 @@ function buildGatewayProcessEnv( if (!("OPENSHELL_DISABLE_GATEWAY_AUTH" in gatewayEnv)) { delete env.OPENSHELL_DISABLE_GATEWAY_AUTH; } + if (!(NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV in gatewayEnv)) { + delete env[NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV]; + } return env; } @@ -177,6 +181,7 @@ export function buildDockerDriverGatewayRuntimeIdentity( ...Object.keys(options.gatewayEnv), "OPENSHELL_DOCKER_SUPERVISOR_BIN", "OPENSHELL_GATEWAY_CONFIG", + NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV, ]); const desiredEnv = Object.fromEntries( Object.entries(launch.env).filter( diff --git a/src/lib/onboard/gateway-process-identity.ts b/src/lib/onboard/gateway-process-identity.ts index 4f221fe186c..835c5999598 100644 --- a/src/lib/onboard/gateway-process-identity.ts +++ b/src/lib/onboard/gateway-process-identity.ts @@ -13,6 +13,7 @@ import { export { buildOwnedHostGatewayArgv0, + canonicalGatewayTargetMatches, type OpenShellGatewayProcessTarget, } from "./gateway-process-target-identity"; @@ -43,6 +44,7 @@ export function gatewayProcessCmdlineMatches( opts: { expectedOpenShellGateway?: OpenShellGatewayProcessTarget; processNames?: ReadonlySet; + requireExpectedFlags?: boolean; resolveExecutablePath?: ResolveExecutablePath; } = {}, ): boolean { @@ -59,12 +61,12 @@ export function gatewayProcessCmdlineMatches( if (processNames.has(base)) { if (processNames.has("openshell-gateway") && base === "openshell-gateway") { return openShellGatewayMatchesTarget(tokens, opts.expectedOpenShellGateway, { - requireExpectedFlags: false, + requireExpectedFlags: opts.requireExpectedFlags ?? false, }); } if (base === "openclaw-gateway") { return openShellGatewayMatchesTarget(tokens, opts.expectedOpenShellGateway, { - requireExpectedFlags: true, + requireExpectedFlags: opts.requireExpectedFlags ?? true, }); } return true; @@ -76,7 +78,7 @@ export function gatewayProcessCmdlineMatches( tokens[2] === "start" ) { return openShellGatewayMatchesTarget(tokens, opts.expectedOpenShellGateway, { - requireExpectedFlags: true, + requireExpectedFlags: opts.requireExpectedFlags ?? true, }); } @@ -86,7 +88,7 @@ export function gatewayProcessCmdlineMatches( const expected = normalize(gatewayBin); if (actual && expected && actual === expected) { return openShellGatewayMatchesTarget(tokens, opts.expectedOpenShellGateway, { - requireExpectedFlags: false, + requireExpectedFlags: opts.requireExpectedFlags ?? false, }); } } @@ -108,8 +110,10 @@ export function hostGatewayCmdlineMatches( cmdline: string, gatewayBin: string | null | undefined, expectedOpenShellGateway?: OpenShellGatewayProcessTarget, + opts: { requireExpectedFlags?: boolean } = {}, ): boolean { return gatewayProcessCmdlineMatches(cmdline, gatewayBin, { expectedOpenShellGateway, + requireExpectedFlags: opts.requireExpectedFlags, }); } diff --git a/src/lib/onboard/gateway-process-target-identity.ts b/src/lib/onboard/gateway-process-target-identity.ts index e446c6be2b8..2319172b4e2 100644 --- a/src/lib/onboard/gateway-process-target-identity.ts +++ b/src/lib/onboard/gateway-process-target-identity.ts @@ -45,15 +45,22 @@ export function gatewayTargetMatches( return true; } +export function canonicalGatewayTargetMatches(name: string, port: number): boolean { + return resolveGatewayName(port) === name && resolveGatewayPortFromName(name) === port; +} + function cliFlagValue(tokens: string[], names: string[]): string | null { + const values: string[] = []; for (let index = 0; index < tokens.length; index += 1) { const token = tokens[index]; for (const name of names) { - if (token === name) return tokens[index + 1] ?? null; - if (token.startsWith(`${name}=`)) return token.slice(name.length + 1); + if (token === name && tokens[index + 1]) values.push(tokens[index + 1]); + else if (token.startsWith(`${name}=`) && token.length > name.length + 1) { + values.push(token.slice(name.length + 1)); + } } } - return null; + return values.length === 1 ? values[0] : null; } export function openShellGatewayMatchesTarget( diff --git a/src/lib/onboard/host-gateway-process-target.test.ts b/src/lib/onboard/host-gateway-process-target.test.ts index 80d23009608..f395dd240a9 100644 --- a/src/lib/onboard/host-gateway-process-target.test.ts +++ b/src/lib/onboard/host-gateway-process-target.test.ts @@ -7,6 +7,11 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { + gatewayIdForStateDir, + NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV, +} from "./docker-driver-gateway-config"; +import { writeDockerDriverGatewayRuntimeMarkerForStateDir } from "./docker-driver-gateway-runtime-marker"; import { HOST_GATEWAY_PGREP_PATTERN, type HostGatewayProcessDeps, @@ -42,17 +47,95 @@ function makeRun(responses: Map): HostGatewayProcessDeps["r function psResponses( pid: number, opts: { - cmdline: string; + cmdline: string | (() => string); exited: Set; }, ): [string, RunResponse][] { return [ [`ps -p ${pid} -o pid=`, () => (opts.exited.has(pid) ? notFound() : ok(`${pid}\n`))], + [`ps -p ${pid} -o uid=`, staticResponse(ok(`${String(process.getuid?.() ?? 501)}\n`))], [`ps -p ${pid} -o user=`, staticResponse(ok("tester\n"))], - [`ps -p ${pid} -o args=`, staticResponse(ok(opts.cmdline))], + [ + `ps -p ${pid} -o args=`, + () => ok(typeof opts.cmdline === "function" ? opts.cmdline() : opts.cmdline), + ], ]; } +function stopScopedTarget( + overrides: { + cmdline?: string; + cmdlineAfterProof?: string; + markerPort?: number; + name?: string; + namespace?: string; + pidFilePid?: number; + port?: number; + } = {}, +) { + const selectedPid = 9_999_601; + const pid = overrides.pidFilePid ?? selectedPid; + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-scoped-target-")); + const pidFile = path.join(stateDir, "openshell-gateway.pid"); + fs.writeFileSync(pidFile, `${String(pid)}\n`); + fs.writeFileSync( + path.join(stateDir, "openshell-gateway.toml"), + `[openshell.drivers.docker]\nsandbox_namespace = "${gatewayIdForStateDir(stateDir)}"\n`, + ); + writeDockerDriverGatewayRuntimeMarkerForStateDir(stateDir, { + desiredEnv: {}, + endpoint: `https://127.0.0.1:${String(overrides.markerPort ?? 18080)}`, + pid: selectedPid, + }); + const exited = new Set(); + let cmdlineReads = 0; + const cmdline = overrides.cmdline ?? "openshell-gateway[nemoclaw=nemoclaw-18080;port=18080]"; + const run = vi.fn( + makeRun( + new Map([ + ...psResponses(pid, { + cmdline: () => { + cmdlineReads += 1; + return cmdlineReads > 1 && overrides.cmdlineAfterProof + ? overrides.cmdlineAfterProof + : cmdline; + }, + exited, + }), + ]), + ), + ); + const kill = vi.fn((killedPid, signal) => { + switch (signal) { + case "SIGTERM": + exited.add(killedPid); + break; + } + return true; + }); + const result = stopHostGatewayProcesses( + { + run, + kill, + env: {}, + isPortFree: () => true, + log: vi.fn(), + readProcessEnvironment: () => ({ + [NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV]: + overrides.namespace ?? gatewayIdForStateDir(stateDir), + }), + }, + { + openShellGatewayName: overrides.name ?? "nemoclaw-18080", + openShellGatewayPort: overrides.port ?? 18080, + scopedGatewayStop: true, + stateDir, + usePgrepFallback: false, + }, + ); + return { kill, pidFile, result, run }; +} + function stopTargetedPid(pid: number, cmdline: string, targeted = true) { const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-host-gateway-target-")); const pidFile = path.join(stateDir, "openshell-gateway.pid"); @@ -89,6 +172,44 @@ function stopTargetedPid(pid: number, cmdline: string, targeted = true) { } describe("stopHostGatewayProcesses target filtering", () => { + it("stops only the fully proven scoped PID without running pgrep (#8663)", () => { + const { kill, pidFile, result, run } = stopScopedTarget(); + + expect(result.stopped).toEqual([9_999_601]); + expect(result.ownershipFailures).toEqual([]); + expect(kill).toHaveBeenCalledWith(9_999_601, "SIGTERM"); + expect(run.mock.calls.some(([command]) => command === "pgrep")).toBe(false); + expect(fs.existsSync(pidFile)).toBe(false); + }); + + it.each([ + ["PID file", { pidFilePid: 9_999_602 }], + ["command line", { cmdline: "openshell-gateway[nemoclaw=nemoclaw;port=8080]" }], + ["gateway name", { name: "nemoclaw", port: 18080 }], + ["gateway port", { name: "nemoclaw", port: 8080 }], + ["loaded namespace", { namespace: "default" }], + ])("fails closed when a sibling cross-matches by %s (#8663)", (_case, overrides) => { + const { kill, pidFile, result } = stopScopedTarget(overrides); + + expect(result.stopped).toEqual([]); + expect(result.ownershipFailures?.length).toBe(1); + expect(kill).not.toHaveBeenCalled(); + expect(fs.existsSync(pidFile)).toBe(true); + }); + + it("revalidates process ownership immediately before signaling (#8663)", () => { + const { kill, pidFile, result } = stopScopedTarget({ + cmdlineAfterProof: "openshell-gateway[nemoclaw=nemoclaw;port=8080]", + }); + + expect(result.stopped).toEqual([]); + expect(result.ownershipFailures).toEqual([ + "PID 9999601: process ownership changed immediately before signaling", + ]); + expect(kill).not.toHaveBeenCalled(); + expect(fs.existsSync(pidFile)).toBe(true); + }); + it("accepts a matching OpenShell CLI gateway-start process for the cleanup target", () => { const { kill, pidFile, result } = stopTargetedPid( 9999553, diff --git a/src/lib/onboard/host-gateway-process.ts b/src/lib/onboard/host-gateway-process.ts index 11c3e217702..cf72536be9b 100644 --- a/src/lib/onboard/host-gateway-process.ts +++ b/src/lib/onboard/host-gateway-process.ts @@ -7,12 +7,24 @@ import os from "node:os"; import path from "node:path"; import { waitUntil } from "../core/wait"; -import { clearDockerDriverGatewayRuntimeMarker } from "./docker-driver-gateway-runtime-marker"; import { + gatewayIdForStateDir, + hasStateScopedSandboxNamespace, + NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV, +} from "./docker-driver-gateway-config"; +import { + clearDockerDriverGatewayRuntimeMarker, + getDockerDriverGatewayRuntimeMarkerPath, + parseDockerDriverGatewayRuntimeMarker, +} from "./docker-driver-gateway-runtime-marker"; +import { + canonicalGatewayTargetMatches, type OpenShellGatewayProcessTarget, hostGatewayCmdlineMatches as sharedHostGatewayCmdlineMatches, } from "./gateway-process-identity"; +export { hasStateScopedSandboxNamespace } from "./docker-driver-gateway-config"; + export interface RunResult { status: number | null; stdout: string; @@ -24,7 +36,9 @@ export interface HostGatewayProcessDeps { kill: (pid: number, signal?: NodeJS.Signals | number) => boolean; env: NodeJS.ProcessEnv; commandExists?: (command: string) => boolean; + isPortFree?: (port: number) => boolean; log?: (message: string) => void; + readProcessEnvironment?: (pid: number) => Record | null; warn?: (message: string) => void; } @@ -41,6 +55,8 @@ export interface StopHostGatewayOptions { pollIntervalMs?: number; /** Keep PID/runtime evidence when a PID-file process does not match the cleanup target. */ preserveRuntimeFilesOnNonMatching?: boolean; + /** Restrict cleanup to one fully proven PID-file gateway. */ + scopedGatewayStop?: boolean; stateDir?: string; termWaitMs?: number; /** Whether to read and act on the resolved pid file. */ @@ -52,6 +68,7 @@ export interface StopHostGatewayResult { failed: number[]; /** Whether a requested pgrep fallback completed with a usable result. */ orphanScanComplete?: boolean; + ownershipFailures?: string[]; skippedDeadPids: number[]; skippedNonMatchingPids: number[]; stopped: number[]; @@ -125,7 +142,9 @@ function defaultDeps(overrides: Partial = {}): HostGatew kill: overrides.kill ?? defaultKill, env, commandExists: overrides.commandExists ?? ((cmd) => defaultCommandExists(cmd, env)), + isPortFree: overrides.isPortFree ?? ((port) => isHostPortFree(port)), log: overrides.log, + readProcessEnvironment: overrides.readProcessEnvironment, warn: overrides.warn, }; } @@ -171,12 +190,108 @@ function pidOwner(pid: number, deps: HostGatewayProcessDeps): string | null { return result.stdout.trim() || null; } +function readOwnedRuntimeFile(filePath: string, uid: number): string | null { + if (typeof fs.constants.O_NOFOLLOW !== "number") return null; + let descriptor: number | undefined; + try { + descriptor = fs.openSync(filePath, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + const stat = fs.fstatSync(descriptor); + if (!stat.isFile() || stat.nlink !== 1 || stat.uid !== uid || stat.size > 64 * 1024) + return null; + return fs.readFileSync(descriptor, "utf-8"); + } catch { + return null; + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + } +} + +export function processUsesStateScopedSandboxNamespace( + pid: number, + stateDir: string, + deps: Pick, +): boolean { + const uid = typeof process.getuid === "function" ? process.getuid() : -1; + const owner = deps.run("ps", ["-p", String(pid), "-o", "uid="], { env: deps.env }); + if (owner.status !== 0 || Number(owner.stdout.trim()) !== uid) return false; + let environment = deps.readProcessEnvironment?.(pid) ?? null; + if (!environment) { + try { + environment = Object.fromEntries( + fs + .readFileSync(`/proc/${String(pid)}/environ`, "utf-8") + .split("\0") + .filter(Boolean) + .map((entry) => [ + entry.slice(0, entry.indexOf("=")), + entry.slice(entry.indexOf("=") + 1), + ]), + ); + } catch { + const command = deps.run("ps", ["eww", "-p", String(pid), "-o", "command="], { + env: deps.env, + }); + const prefix = `${NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV}=`; + const value = command.stdout.split(/\s+/).find((token) => token.startsWith(prefix)); + environment = value + ? { [NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV]: value.slice(prefix.length) } + : null; + } + } + return environment?.[NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV] === gatewayIdForStateDir(stateDir); +} + export function hostGatewayCmdlineMatches( cmdline: string, gatewayBin: string | null | undefined, expectedOpenShellGateway?: OpenShellGatewayProcessTarget, + opts: { requireExpectedFlags?: boolean } = {}, ): boolean { - return sharedHostGatewayCmdlineMatches(cmdline, gatewayBin, expectedOpenShellGateway); + return sharedHostGatewayCmdlineMatches(cmdline, gatewayBin, expectedOpenShellGateway, opts); +} + +function scopedGatewayOwnershipFailure( + pid: number, + deps: HostGatewayProcessDeps, + options: StopHostGatewayOptions, + stateDir: string, + pidFile: string, + target: { name: string; port: number }, +): string | null { + if (!hasStateScopedSandboxNamespace(stateDir)) { + return "gateway config does not prove an isolated sandbox namespace"; + } + const uid = typeof process.getuid === "function" ? process.getuid() : -1; + const pidText = readOwnedRuntimeFile(pidFile, uid); + const markerText = readOwnedRuntimeFile(getDockerDriverGatewayRuntimeMarkerPath(stateDir), uid); + const marker = markerText ? parseDockerDriverGatewayRuntimeMarker(markerText) : null; + if (Number(pidText?.trim()) !== pid || marker?.pid !== pid) { + return "PID file and runtime marker do not identify the same process"; + } + let markerPort = 0; + try { + markerPort = Number(new URL(marker.endpoint).port); + } catch { + return "runtime marker endpoint is invalid"; + } + if ( + markerPort !== target.port || + marker.platform !== process.platform || + marker.arch !== process.arch + ) { + return "runtime marker does not identify the selected gateway"; + } + if (!processUsesStateScopedSandboxNamespace(pid, stateDir, deps)) { + return "gateway process owner and loaded sandbox namespace cannot be proven"; + } + if ( + !hostGatewayCmdlineMatches(processArgs(pid, deps), options.gatewayBin, target, { + requireExpectedFlags: true, + }) + ) { + return "process command line does not identify the selected gateway name and port"; + } + return null; } function waitForExit( @@ -258,13 +373,16 @@ function tryStopPid( pid: number, deps: HostGatewayProcessDeps, options: Required>, -): "stopped" | "failed" { + canSignal?: () => boolean, +): "stopped" | "failed" | "identity-changed" { const log = deps.log ?? ((message: string) => console.log(message)); + if (canSignal && !canSignal()) return "identity-changed"; deps.kill(pid, "SIGTERM"); if (waitForExit(pid, deps, options.termWaitMs, options.pollIntervalMs)) { log(`Stopped host openshell-gateway process ${pid}`); return "stopped"; } + if (canSignal && !canSignal()) return "identity-changed"; deps.kill(pid, "SIGKILL"); if (waitForExit(pid, deps, options.killWaitMs, options.pollIntervalMs)) { log(`Stopped host openshell-gateway process ${pid} (after SIGKILL)`); @@ -286,24 +404,56 @@ export function stopHostGatewayProcesses( const result: StopHostGatewayResult = { failed: [], orphanScanComplete: true, + ownershipFailures: [], skippedDeadPids: [], skippedNonMatchingPids: [], stopped: [], sudoRemediationPids: [], }; + const explicitPids = Array.from(options.pids ?? []).filter( + (pid): pid is number => Number.isInteger(pid) && pid > 0, + ); + const scopedPort = Number(options.openShellGatewayPort); + const scopedName = options.openShellGatewayName?.trim() ?? ""; + const rejectScoped = (reason: string, pid?: number): StopHostGatewayResult => { + if (pid) result.skippedNonMatchingPids.push(pid); + (result.ownershipFailures ??= []).push(pid ? `PID ${String(pid)}: ${reason}` : reason); + return result; + }; + if ( + options.scopedGatewayStop && + (!Number.isInteger(scopedPort) || + scopedPort < 1 || + scopedPort > 65_535 || + !canonicalGatewayTargetMatches(scopedName, scopedPort) || + options.usePidFile === false || + options.usePgrepFallback === true || + explicitPids.length > 0) + ) { + return rejectScoped("scoped cleanup requires one canonical name, port, and PID file"); + } + if (options.usePidFile ?? true) { const pidFromFile = readPidFile(pidFile); if (pidFromFile !== null) { addPid(candidates, pidFromFile, "pid-file"); + } else if (options.scopedGatewayStop) { + if ( + fs.existsSync(pidFile) || + fs.existsSync(getDockerDriverGatewayRuntimeMarkerPath(stateDir)) || + deps.isPortFree?.(scopedPort) !== true + ) { + return rejectScoped("selected gateway has incomplete ownership evidence"); + } + if (options.logNoProcesses) + (deps.log ?? console.log)("No host openshell-gateway processes found"); + return result; } else if (clearRuntimeState && fs.existsSync(pidFile)) { clearHostGatewayRuntimeFiles(stateDir, pidFile); } } - const explicitPids = Array.from(options.pids ?? []).filter( - (pid): pid is number => Number.isInteger(pid) && pid > 0, - ); for (const pid of explicitPids) addPid(candidates, pid, "explicit"); // When a caller passes explicit PIDs (e.g. drift-restart targeting one @@ -311,7 +461,9 @@ export function stopHostGatewayProcesses( // host. Otherwise an onboard drift could terminate an unrelated worktree's // gateway. Sweeping callers (uninstall, sandbox destroy of the last sandbox) // omit `pids` and so still get the pgrep fallback by default. - const useFallback = options.usePgrepFallback ?? explicitPids.length === 0; + const useFallback = options.scopedGatewayStop + ? false + : (options.usePgrepFallback ?? explicitPids.length === 0); let pgrepRan = false; if (useFallback) { const sweep = pgrepHostGatewayPids(deps); @@ -336,13 +488,24 @@ export function stopHostGatewayProcesses( for (const [pid, sources] of candidates) { if (!pidExists(pid, deps)) { result.skippedDeadPids.push(pid); + if (options.scopedGatewayStop && deps.isPortFree?.(scopedPort) !== true) { + return rejectScoped("recorded process is dead but its selected port remains occupied"); + } if (clearRuntimeState && sources.has("pid-file") && !clearedRuntimeFiles) { clearHostGatewayRuntimeFiles(stateDir, pidFile); clearedRuntimeFiles = true; } continue; } + if (options.scopedGatewayStop) { + const reason = scopedGatewayOwnershipFailure(pid, deps, options, stateDir, pidFile, { + name: scopedName, + port: scopedPort, + }); + if (reason) return rejectScoped(reason, pid); + } if ( + !options.scopedGatewayStop && !hostGatewayCmdlineMatches( processArgs(pid, deps), options.gatewayBin, @@ -362,8 +525,26 @@ export function stopHostGatewayProcesses( continue; } - if (tryStopPid(pid, deps, waitOptions) === "stopped") { + const stopResult = tryStopPid( + pid, + deps, + waitOptions, + options.scopedGatewayStop + ? () => + scopedGatewayOwnershipFailure(pid, deps, options, stateDir, pidFile, { + name: scopedName, + port: scopedPort, + }) === null + : undefined, + ); + if (stopResult === "identity-changed") { + return rejectScoped("process ownership changed immediately before signaling", pid); + } + if (stopResult === "stopped") { result.stopped.push(pid); + if (options.scopedGatewayStop && deps.isPortFree?.(scopedPort) !== true) { + return rejectScoped("selected gateway port remains occupied after its process stopped"); + } if (clearRuntimeState && !clearedRuntimeFiles) { clearHostGatewayRuntimeFiles(stateDir, pidFile); clearedRuntimeFiles = true; diff --git a/test/e2e/live/concurrent-gateway-ports.test.ts b/test/e2e/live/concurrent-gateway-ports.test.ts index 57ae4c8ad2d..49464ebca24 100644 --- a/test/e2e/live/concurrent-gateway-ports.test.ts +++ b/test/e2e/live/concurrent-gateway-ports.test.ts @@ -31,6 +31,7 @@ const PHASE_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_PHASE_TIMEOUT_MS ?? 1_2 const PROBE_ATTEMPTS = Number(process.env.NEMOCLAW_E2E_PROBE_ATTEMPTS ?? 12); const PROBE_DELAY_MS = Number(process.env.NEMOCLAW_E2E_PROBE_DELAY_SECONDS ?? 5) * 1_000; const TEST_TIMEOUT_MS = 90 * 60_000; +const POST_UNINSTALL_HEALTH_PROBES = 3; process.env.NEMOCLAW_CLI_BIN ??= CLI_ENTRYPOINT; validateSandboxName(SANDBOX_A); @@ -91,6 +92,51 @@ async function command( }); } +function gatewayProcessEvidence(evidence: string, gateway: string): string | undefined { + return evidence + .split(`gateway=${gateway}\n`)[1] + ?.split("gateway=")[0] + ?.match(/^active_pid=\d+\nexecutable=.*\n.*$/m)?.[0]; +} + +async function captureGatewayEvidence( + host: HostCliClient, + sandbox: SandboxClient, + gateways: readonly (readonly [string, string])[], + stage: string, +): Promise { + const script = [ + 'for spec in "$@"; do', + ' gateway="${spec%%:*}"; port="${spec##*:}"; leaf=openshell-docker-gateway', + ' test "$port" = 8080 || leaf="$leaf-$port"', + ' state="$HOME/.local/state/nemoclaw/$leaf"; pid=""', + ' test ! -r "$state/openshell-gateway.pid" || pid="$(tr -d "[:space:]" < "$state/openshell-gateway.pid")"', + ' if test -z "$pid" && test "$port" = 8080 && command -v systemctl >/dev/null; then for service in openshell-gateway nemoclaw-openshell-gateway; do candidate="$(systemctl --user show "$service" --property=MainPID --value 2>/dev/null || true)"; test "${candidate:-0}" -le 0 || { pid="$candidate"; break; }; done; fi', + ' printf "gateway=%s\\nport=%s\\npid_file=%s\\n" "$gateway" "$port" "${pid:-}"', + ' if test -n "$pid" && ps -p "$pid" >/dev/null 2>&1; then printf "active_pid=%s\\nexecutable=%s\\n" "$pid" "$(readlink -f "/proc/$pid/exe" 2>/dev/null || true)"; ps -p "$pid" -o pid=,ppid=,uid=,lstart=,args=; fi', + ' printf "listeners=\\n"; ss -H -ltnp 2>&1 | grep -E "[:.]$port\\b" || true', + ' printf "runtime=\\n"; test ! -r "$state/runtime.json" || cat "$state/runtime.json"', + ' printf "namespace=\\n"; test ! -r "$state/openshell-gateway.toml" || grep "^sandbox_namespace" "$state/openshell-gateway.toml" || true', + "done", + ].join("\n"); + const evidence = await host.command( + "bash", + ["-lc", script, "gateway-evidence", ...gateways.map(([name, port]) => `${name}:${port}`)], + { artifactName: `${stage}-gateway-processes`, env: commandEnv(), timeoutMs: 30_000 }, + ); + expect(evidence.exitCode, resultText(evidence)).toBe(0); + await Promise.all( + gateways.map(([name]) => + sandbox.openshell(["sandbox", "list", "-g", name], { + artifactName: `${stage}-${name}-sandbox-phase`, + env: openshellEnvForGateway(name), + timeoutMs: 30_000, + }), + ), + ); + return resultText(evidence); +} + async function runOnboard( host: HostCliClient, sandboxName: string, @@ -473,6 +519,22 @@ test("concurrent gateway ports: onboards two sandboxes on isolated gateways and expect(dashboardB).not.toBe(dashboardA); progress.phase("uninstall alternate gateway without disrupting default"); + const gatewayPair = [ + [gatewayA, GATEWAY_PORT_A], + [gatewayB, GATEWAY_PORT_B], + ] as const; + const beforeEvidence = await captureGatewayEvidence( + host, + sandbox, + gatewayPair, + "phase-4-before-uninstall", + ); + const processA = gatewayProcessEvidence(beforeEvidence, gatewayA); + const processB = gatewayProcessEvidence(beforeEvidence, gatewayB); + expect(processA).toBeDefined(); + expect(processB).toBeDefined(); + expect(processA).not.toBe(processB); + const uninstallB = await command(host, ["uninstall", "--yes", "--destroy-user-data"], { artifactName: "phase-4-uninstall-gateway-b", env: commandEnv({ NEMOCLAW_GATEWAY_PORT: GATEWAY_PORT_B }), @@ -480,14 +542,56 @@ test("concurrent gateway ports: onboards two sandboxes on isolated gateways and }); expect(uninstallB.exitCode, resultText(uninstallB)).toBe(0); - const phaseAAfterUninstallB = await waitForSandboxReady( + const afterEvidence = await captureGatewayEvidence( + host, sandbox, - SANDBOX_A, - gatewayA, - "phase-4-sandbox-a-still-ready-after-b-uninstall", + gatewayPair, + "phase-4-after-uninstall", ); - expect(["Ready", "Running"]).toContain(phaseAAfterUninstallB); - await expectPortListening(host, GATEWAY_PORT_A, "phase-4-gateway-port-a-still-listening"); + expect(gatewayProcessEvidence(afterEvidence, gatewayA)).toBe(processA); + expect(gatewayProcessEvidence(afterEvidence, gatewayB)).toBeUndefined(); + + const survivorPhases: string[] = []; + for (let probe = 1; probe <= POST_UNINSTALL_HEALTH_PROBES; probe += 1) { + survivorPhases.push( + await waitForSandboxReady( + sandbox, + SANDBOX_A, + gatewayA, + `phase-4-survivor-probe-${String(probe)}`, + ), + ); + await expectPortListening(host, GATEWAY_PORT_A, `phase-4-survivor-port-${String(probe)}`); + const scopedList = await command(host, ["list"], { + artifactName: `phase-4-survivor-list-${String(probe)}`, + env: commandEnv({ NEMOCLAW_GATEWAY_PORT: GATEWAY_PORT_A }), + timeoutMs: 60_000, + }); + expect(scopedList.exitCode, resultText(scopedList)).toBe(0); + expect(outputIncludesSandbox(scopedList.stdout, SANDBOX_A), scopedList.stdout).toBe(true); + const dashboardProbe = await host.command( + "curl", + [ + "-sS", + "-L", + "--max-time", + "10", + "-o", + "/dev/null", + "-w", + "%{http_code}", + `http://127.0.0.1:${DASHBOARD_PORT_A}/`, + ], + { + artifactName: `phase-4-survivor-dashboard-${String(probe)}`, + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect(dashboardProbe.exitCode, resultText(dashboardProbe)).toBe(0); + expect(dashboardProbe.stdout.trim()).toMatch(/^[23][0-9]{2}$/); + await (probe < POST_UNINSTALL_HEALTH_PROBES ? sleep(PROBE_DELAY_MS) : Promise.resolve()); + } await expectPortNotListening(host, GATEWAY_PORT_B, "phase-4-gateway-port-b-stopped"); const listAAfterUninstallB = await command(host, ["list"], { @@ -523,7 +627,9 @@ test("concurrent gateway ports: onboards two sandboxes on isolated gateways and !outputIncludesSandbox(listGatewayB.stdout, SANDBOX_A), dashboardPortsDistinct: Boolean(dashboardA && dashboardB && dashboardA !== dashboardB), gatewayBUninstalled: uninstallB.exitCode === 0 && scopedStateRemoved.exitCode === 0, - sandboxAPreservedAfterUninstallB: ["Ready", "Running"].includes(phaseAAfterUninstallB), + sandboxAPreservedAfterUninstallB: + survivorPhases.length === POST_UNINSTALL_HEALTH_PROBES && + survivorPhases.every((phase) => phase === "Ready" || phase === "Running"), sharedCliPreserved: listAAfterUninstallB.exitCode === 0, }, });