diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index f4d83aee124..28ed79a99e2 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1755,6 +1755,9 @@ Cleaning up the gateway after the last sandbox also purges the shared cluster vo If final gateway cleanup finds a live PID-file process whose command line does not prove it owns the target gateway, `destroy` exits non-zero after sandbox and registry deletion and skips gateway and volume removal. NemoClaw preserves the per-gateway PID file and runtime marker so you can inspect the process. Stop only the listener that matches the target gateway, then rerun `destroy` to converge cleanup. +When the default-port gateway runs under the packaged OpenShell gateway service, gateway cleanup stops that service before it reaps host processes, so the gateway port is released instead of being rebound by the service manager. +The service is stopped, not disabled or removed, and the next onboarding run starts it again. +If the service cannot be stopped, `destroy` exits non-zero after sandbox and registry deletion, prints the status command for the service, and skips gateway and volume removal. If the OpenShell gateway is unreachable and the sandbox has no managed MCP ownership state, `--force` removes only NemoClaw's local registry entry and local artifacts. Gateway-side deletion remains unconfirmed, shared host-service and gateway teardown are skipped, and the sandbox and retained volume may still exist if the gateway returns. Start the gateway with `$$nemoclaw status` and retry destroy when you need a confirmed deletion. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 6cad47f36ff..c6c10500b11 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -993,6 +993,31 @@ Create a snapshot first when the sandbox is reachable enough to back up state. For details, refer to [Create and Restore Snapshots](../manage-sandboxes/state-and-backups/create-and-restore-snapshots). +### Gateway Port Stays Bound After Destroying the Last Sandbox + +Destroying the final sandbox with `--cleanup-gateway` stops the packaged OpenShell gateway service before it reaps host gateway processes, so the gateway port is released. +The service is stopped, not disabled or removed, and the next onboarding run starts it again. +If the service cannot be stopped, `destroy` exits non-zero and prints the status command for the service. +Stop the service yourself, then rerun `destroy`. + +On Apple Silicon macOS with Homebrew: + +```bash +brew services stop openshell +``` + +On Linux, use the service name that matches the install. For package installs: + +```bash +systemctl --user stop openshell-gateway +``` + +For tarball installs: + +```bash +systemctl --user stop nemoclaw-openshell-gateway +``` + ### `gateway restart` or `recover` reports `privileged control unavailable` diff --git a/src/lib/actions/sandbox/destroy-gateway.test.ts b/src/lib/actions/sandbox/destroy-gateway.test.ts index 7f10d13bf33..1560ac89951 100644 --- a/src/lib/actions/sandbox/destroy-gateway.test.ts +++ b/src/lib/actions/sandbox/destroy-gateway.test.ts @@ -28,6 +28,46 @@ vi.mock("../../onboard/stale-gateway-cleanup", () => ({ import { cleanupGatewayAfterLastSandbox } from "./destroy-gateway"; +function packagedServiceOwner({ + gatewayName, + gatewayPort, +}: { + gatewayName: string; + gatewayPort: number; +}) { + return { + gatewayName, + gatewayPort, + mode: "nemoclaw-managed" as const, + source: "packaged-service" as const, + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }; +} + +function serviceStopResult(stopped: boolean, reason?: string) { + return { + attempted: true, + manager: "systemd" as const, + serviceName: "nemoclaw-openshell-gateway", + statusCommand: "systemctl --user status nemoclaw-openshell-gateway", + stopped, + ...(reason === undefined ? {} : { reason }), + }; +} + +function idleHostReaperResult() { + return { + failed: [], + skippedDeadPids: [], + skippedNonMatchingPids: [], + stopped: [], + sudoRemediationPids: [], + }; +} + describe("cleanupGatewayAfterLastSandbox", () => { beforeEach(() => { mocks.resolveGatewayTeardownAuthority.mockImplementation( @@ -253,6 +293,71 @@ describe("cleanupGatewayAfterLastSandbox", () => { expect(mocks.dockerRemoveVolumesByPrefix).not.toHaveBeenCalled(); }); + it("stops the packaged gateway service before the host reaper on final destroy (#7904)", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + vi.spyOn(os, "homedir").mockReturnValue("/home/tester"); + mocks.resolveGatewayTeardownAuthority.mockImplementationOnce(packagedServiceOwner); + const events: string[] = []; + mocks.stopHostGatewayProcesses.mockImplementationOnce(() => { + events.push("host-reaper"); + return idleHostReaperResult(); + }); + const stopService = vi.fn(() => { + events.push("service-stop"); + return serviceStopResult(true); + }); + const runOpenshell = vi.fn(() => ({ status: 0, stdout: "", stderr: "" })); + + cleanupGatewayAfterLastSandbox("nemoclaw", runOpenshell, { + stopOpenShellGatewayUserService: stopService, + }); + + expect(events).toEqual(["service-stop", "host-reaper"]); + expect(runOpenshell).toHaveBeenCalledWith(["gateway", "remove", "nemoclaw"], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + expect(mocks.dockerRemoveVolumesByPrefix).toHaveBeenCalledWith("openshell-cluster-nemoclaw", { + ignoreError: true, + }); + }); + + it("fails destroy when the packaged gateway service survives the stop (#7904)", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + vi.spyOn(os, "homedir").mockReturnValue("/home/tester"); + mocks.resolveGatewayTeardownAuthority.mockImplementationOnce(packagedServiceOwner); + const stopService = vi.fn(() => + serviceStopResult(false, "systemctl --user stop nemoclaw-openshell-gateway failed: timeout"), + ); + const runOpenshell = vi.fn(() => ({ status: 0, stdout: "", stderr: "" })); + + expect(() => + cleanupGatewayAfterLastSandbox("nemoclaw", runOpenshell, { + stopOpenShellGatewayUserService: stopService, + }), + ).toThrow("systemctl --user status nemoclaw-openshell-gateway"); + expect(mocks.stopHostGatewayProcesses).not.toHaveBeenCalled(); + expect(runOpenshell).not.toHaveBeenCalledWith( + ["gateway", "remove", "nemoclaw"], + expect.anything(), + ); + expect(mocks.dockerRemoveVolumesByPrefix).not.toHaveBeenCalled(); + }); + + it("leaves the service manager alone for a standalone NemoClaw gateway (#7904)", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + vi.spyOn(os, "homedir").mockReturnValue("/home/tester"); + const stopService = vi.fn(() => serviceStopResult(true)); + const runOpenshell = vi.fn(() => ({ status: 0, stdout: "", stderr: "" })); + + cleanupGatewayAfterLastSandbox("nemoclaw", runOpenshell, { + stopOpenShellGatewayUserService: stopService, + }); + + expect(stopService).not.toHaveBeenCalled(); + expect(mocks.stopHostGatewayProcesses).toHaveBeenCalledOnce(); + }); + it.each([ [ "host reaper", diff --git a/src/lib/actions/sandbox/destroy-gateway.ts b/src/lib/actions/sandbox/destroy-gateway.ts index 32c495fe2ce..dc1823dc724 100644 --- a/src/lib/actions/sandbox/destroy-gateway.ts +++ b/src/lib/actions/sandbox/destroy-gateway.ts @@ -7,6 +7,7 @@ import path from "node:path"; import { dockerRemoveVolumesByPrefix } from "../../adapters/docker/volume"; import { OPENSHELL_OPERATION_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; import { DASHBOARD_PORT } from "../../core/ports"; +import { stopOpenShellGatewayUserService } from "../../onboard/docker-driver-gateway-service"; import { resolveGatewayPortFromName, resolveGatewayStateDirName, @@ -28,6 +29,7 @@ const DASHBOARD_FORWARD_PORT = String(DASHBOARD_PORT); export interface CleanupGatewayDeps { resolveGatewayTeardownAuthority?: GatewayTeardownAuthorityResolver; + stopOpenShellGatewayUserService?: typeof stopOpenShellGatewayUserService; } // Compute the Docker-driver gateway state directory that belongs to @@ -105,6 +107,16 @@ export function cleanupGatewayAfterLastSandbox( // ports the live openshell tracks; this catches orphans whose openshell // record was lost across upgrades or failed onboards. stopStaleDashboardListeners(); + if (!externallySupervised && owner.source === "packaged-service") { + const stopService = deps.stopOpenShellGatewayUserService ?? stopOpenShellGatewayUserService; + const serviceStop = stopService(); + if (serviceStop.attempted && !serviceStop.stopped) { + throw new Error( + `Failed to stop the packaged OpenShell gateway service '${serviceStop.serviceName}' that owns gateway '${gatewayName}': ${serviceStop.reason}. ` + + `Check: ${serviceStop.statusCommand}. Stop the service, then rerun destroy.`, + ); + } + } if (!externallySupervised && (process.platform === "linux" || process.platform === "darwin")) { // Sandbox destroy is conservative: only stop the host gateway whose PID // file we wrote during onboard. Disable the pgrep sweep so a stray diff --git a/src/lib/onboard/docker-driver-gateway-service.test.ts b/src/lib/onboard/docker-driver-gateway-service.test.ts index 85a56eb2170..0e9b7f66f41 100644 --- a/src/lib/onboard/docker-driver-gateway-service.test.ts +++ b/src/lib/onboard/docker-driver-gateway-service.test.ts @@ -15,6 +15,7 @@ import { type SpawnSyncLikeResult, startOpenShellGatewayUserService, startPackageManagedDockerDriverGateway, + stopOpenShellGatewayUserService, } from "./docker-driver-gateway-service"; const STATUS_CONNECTED = ` @@ -552,4 +553,129 @@ describe("docker-driver-gateway-service", () => { ).rejects.toThrow("configured 1s health deadline"); expect(clear).not.toHaveBeenCalled(); }); + + it("stops the trusted systemd gateway unit without disabling it (#7904)", () => { + const events: string[] = []; + const home = "/home/nvidia"; + const servicePath = `${home}/.config/systemd/user/nemoclaw-openshell-gateway.service`; + const gatewayBin = `${home}/.local/bin/openshell-gateway`; + + const result = stopOpenShellGatewayUserService({ + commandExists: (command) => command === "systemctl", + env: { HOME: home }, + existsSync: (candidate) => candidate === servicePath, + home, + lstatSync: nonSymlinkStat, + platform: "linux", + readFileSync: () => `# ${NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER}\n`, + spawnSyncImpl: systemdSpawn(events, servicePath, gatewayBin), + }); + + expect(result).toEqual({ + attempted: true, + manager: "systemd", + serviceName: "nemoclaw-openshell-gateway", + statusCommand: "systemctl --user status nemoclaw-openshell-gateway", + stopped: true, + }); + expect(events).toEqual([ + "show nemoclaw-openshell-gateway --property=FragmentPath --property=ExecStart", + "stop nemoclaw-openshell-gateway", + ]); + }); + + it("refuses to stop a systemd unit that no longer has the trusted identity (#7904)", () => { + const events: string[] = []; + const home = "/home/nvidia"; + const servicePath = `${home}/.config/systemd/user/nemoclaw-openshell-gateway.service`; + + const result = stopOpenShellGatewayUserService({ + commandExists: (command) => command === "systemctl", + env: { HOME: home }, + existsSync: (candidate) => candidate === servicePath, + home, + lstatSync: nonSymlinkStat, + platform: "linux", + readFileSync: () => `# ${NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER}\n`, + spawnSyncImpl: systemdSpawn( + events, + `${home}/.config/systemd/user/unrelated.service`, + "/usr/bin/unrelated", + ), + }); + + expect(result).toMatchObject({ attempted: true, stopped: false }); + expect(result.reason).toContain("service identity is not a trusted OpenShell gateway"); + expect(events).toEqual([ + "show nemoclaw-openshell-gateway --property=FragmentPath --property=ExecStart", + ]); + }); + + it("stops the official Homebrew gateway service on macOS (#7904)", () => { + const events: string[] = []; + const brew = vi.fn((_command: string, args: string[]) => { + events.push(args.join(" ")); + return args[0] === "info" ? officialFormulaInfo() : spawnResult(); + }); + + const result = stopOpenShellGatewayUserService({ + commandExists: (command) => command === "brew", + platform: "darwin", + spawnSyncImpl: brew, + }); + + expect(result).toEqual({ + attempted: true, + manager: "homebrew", + serviceName: "openshell", + statusCommand: "brew services info openshell", + stopped: true, + }); + expect(events.at(-1)).toBe("services stop openshell"); + }); + + it("reports the failing stop command when the gateway service survives (#7904)", () => { + const home = "/home/nvidia"; + const servicePath = `${home}/.config/systemd/user/nemoclaw-openshell-gateway.service`; + const gatewayBin = `${home}/.local/bin/openshell-gateway`; + + const result = stopOpenShellGatewayUserService({ + commandExists: (command) => command === "systemctl", + env: { HOME: home }, + existsSync: (candidate) => candidate === servicePath, + home, + lstatSync: nonSymlinkStat, + platform: "linux", + readFileSync: () => `# ${NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER}\n`, + spawnSyncImpl: vi.fn((_command: string, args: string[]) => + args.includes("show") + ? spawnResult(0, "", trustedShowOutput(servicePath, gatewayBin)) + : spawnResult(1, "Job for nemoclaw-openshell-gateway.service failed"), + ), + }); + + expect(result).toMatchObject({ + attempted: true, + stopped: false, + statusCommand: "systemctl --user status nemoclaw-openshell-gateway", + }); + expect(result.reason).toContain( + "systemctl --user stop nemoclaw-openshell-gateway failed: Job for", + ); + }); + + it.each([ + [ + "no service is installed", + { existsSync: () => false, platform: "linux" as const }, + "service not installed", + ], + ["the platform has no service manager", { platform: "win32" as const }, "unsupported platform"], + ])("reports nothing to stop when %s (#7904)", (_case, opts, reason) => { + expect(stopOpenShellGatewayUserService({ commandExists: () => true, ...opts })).toEqual({ + attempted: false, + reason, + stopped: false, + }); + }); }); diff --git a/src/lib/onboard/docker-driver-gateway-service.ts b/src/lib/onboard/docker-driver-gateway-service.ts index 78e47ce04df..7e41084336b 100644 --- a/src/lib/onboard/docker-driver-gateway-service.ts +++ b/src/lib/onboard/docker-driver-gateway-service.ts @@ -48,6 +48,15 @@ export interface OpenShellGatewayUserServiceStartResult { started: boolean; } +export interface OpenShellGatewayUserServiceStopResult { + attempted: boolean; + manager?: "homebrew" | "systemd"; + reason?: string; + serviceName?: string; + statusCommand?: string; + stopped: boolean; +} + export interface SpawnSyncLikeResult { error?: Error; status: number | null; @@ -200,6 +209,19 @@ function runBrew( return runCommand("brew", args, opts); } +function runStopService( + service: OpenShellGatewayUserServiceTarget, + opts: Required>, +) { + return service.manager === "homebrew" + ? runBrew(["services", "stop", service.serviceName], opts) + : runSystemctlUser(["stop", service.serviceName], opts); +} + +function stopServiceCommandName(service: OpenShellGatewayUserServiceTarget): string { + return service.manager === "homebrew" ? "brew" : "systemctl"; +} + function readTextFileIfPresent( filePath: string, opts: Pick = {}, @@ -539,7 +561,7 @@ export function startOpenShellGatewayUserService( reason: "service not installed", }; } - const command = service.manager === "homebrew" ? "brew" : "systemctl"; + const command = stopServiceCommandName(service); if (!commandExists(command)) { return serviceFailure(service, `${command} is not available`, true); } @@ -590,10 +612,7 @@ export function startOpenShellGatewayUserService( ); if (envFailure) return envFailure; - const stop = - service.manager === "homebrew" - ? runBrew(["services", "stop", service.serviceName], { env, spawnSyncImpl }) - : runSystemctlUser(["stop", service.serviceName], { env, spawnSyncImpl }); + const stop = runStopService(service, { env, spawnSyncImpl }); if (!stop.ok) { const prefix = service.manager === "homebrew" ? "brew services stop" : "systemctl --user stop"; return serviceFailure( @@ -642,6 +661,40 @@ export function startOpenShellGatewayUserService( }; } +export function stopOpenShellGatewayUserService( + opts: OpenShellGatewayUserServiceOptions = {}, +): OpenShellGatewayUserServiceStopResult { + const platform = opts.platform ?? process.platform; + if (platform !== "linux" && platform !== "darwin") { + return { attempted: false, stopped: false, reason: "unsupported platform" }; + } + const env = opts.env ?? process.env; + const home = effectiveHome(opts.home, opts.env); + const commandExists = opts.commandExists ?? ((command) => defaultCommandExists(command, env)); + const spawnSyncImpl = opts.spawnSyncImpl ?? spawnSync; + const service = resolveOpenShellGatewayUserService({ ...opts, env, home }); + if (!service) return { attempted: false, stopped: false, reason: "service not installed" }; + + const describe = (stopped: boolean, reason?: string): OpenShellGatewayUserServiceStopResult => ({ + attempted: true, + manager: service.manager, + serviceName: service.serviceName, + statusCommand: service.statusCommand, + stopped, + ...(reason === undefined ? {} : { reason }), + }); + const command = stopServiceCommandName(service); + if (!commandExists(command)) return describe(false, `${command} is not available`); + if (service.manager === "systemd") { + const identity = validateSystemdServiceIdentity(service, { env, spawnSyncImpl }); + if (!identity.ok) return describe(false, identity.reason ?? "service identity is invalid"); + } + const stop = runStopService(service, { env, spawnSyncImpl }); + if (stop.ok) return describe(true); + const prefix = service.manager === "homebrew" ? "brew services stop" : "systemctl --user stop"; + return describe(false, `${prefix} ${service.serviceName} failed: ${stop.reason}`); +} + export async function startPackageManagedDockerDriverGateway({ clearDockerDriverGatewayRuntimeFiles, exitOnFailure, diff --git a/test/cli/destroy-gateway-cleanup.test.ts b/test/cli/destroy-gateway-cleanup.test.ts index f923c24aa93..b469f3b1051 100644 --- a/test/cli/destroy-gateway-cleanup.test.ts +++ b/test/cli/destroy-gateway-cleanup.test.ts @@ -319,6 +319,94 @@ describe("CLI dispatch", () => { }, ); + it.runIf(process.platform === "linux")( + "stops the packaged gateway service so the port is free after the final destroy (#7904)", + testTimeoutOptions(30_000), + () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-destroy-service-")); + const localBin = path.join(home, "bin"); + const registryDir = path.join(home, ".nemoclaw"); + const configHome = path.join(home, ".config"); + const binHome = path.join(home, ".local", "bin"); + const unitDir = path.join(configHome, "systemd", "user"); + const unitPath = path.join(unitDir, "nemoclaw-openshell-gateway.service"); + const gatewayBin = path.join(binHome, "openshell-gateway"); + const openshellLog = path.join(home, "openshell.log"); + const systemctlLog = path.join(home, "systemctl.log"); + fs.mkdirSync(localBin, { recursive: true }); + fs.mkdirSync(registryDir, { recursive: true }); + fs.mkdirSync(unitDir, { recursive: true }); + fs.writeFileSync( + unitPath, + ["[Unit]", "# NEMOCLAW_MANAGED_OPENSHELL_GATEWAY=1", "[Service]"].join("\n"), + ); + fs.writeFileSync( + path.join(registryDir, "sandboxes.json"), + JSON.stringify({ + sandboxes: { + alpha: { + name: "alpha", + model: "test-model", + provider: "nvidia-prod", + gpuEnabled: false, + policies: [], + }, + }, + defaultSandbox: "alpha", + }), + { mode: 0o600 }, + ); + fs.writeFileSync( + path.join(localBin, "openshell"), + [ + "#!/bin/sh", + `log_file=${JSON.stringify(openshellLog)}`, + 'if [ "$1" = "sandbox" ] && [ "$2" = "list" ]; then', + ' printf "NAME STATUS\\n" >> "$log_file"', + " exit 0", + "fi", + 'printf \'%s\\n\' "$*" >> "$log_file"', + "exit 0", + ].join("\n"), + { mode: 0o755 }, + ); + fs.writeFileSync( + path.join(localBin, "systemctl"), + [ + "#!/bin/sh", + `log_file=${JSON.stringify(systemctlLog)}`, + 'printf \'%s\\n\' "$*" >> "$log_file"', + 'if [ "$2" = "show" ]; then', + ` printf 'FragmentPath=%s\\n' ${JSON.stringify(unitPath)}`, + ` printf 'ExecStart={ path=%s ; argv[]=%s ; }\\n' ${JSON.stringify(gatewayBin)} ${JSON.stringify(gatewayBin)}`, + "fi", + "exit 0", + ].join("\n"), + { mode: 0o755 }, + ); + fs.writeFileSync(path.join(localBin, "docker"), "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + fs.writeFileSync(path.join(localBin, "pgrep"), "#!/bin/sh\nexit 1\n", { mode: 0o755 }); + fs.writeFileSync(path.join(localBin, "lsof"), "#!/bin/sh\nexit 1\n", { mode: 0o755 }); + + const r = runWithEnv( + "alpha destroy -y --cleanup-gateway", + { + HOME: home, + PATH: `${localBin}:${process.env.PATH || ""}`, + XDG_BIN_HOME: binHome, + XDG_CONFIG_HOME: configHome, + }, + 30_000, + ); + + expect(r.code, r.out).toBe(0); + const systemctlOutput = fs.readFileSync(systemctlLog, "utf8"); + expect(systemctlOutput).toContain("--user stop nemoclaw-openshell-gateway\n"); + expect(systemctlOutput).not.toContain("disable"); + expect(fs.readFileSync(openshellLog, "utf8")).toContain("gateway remove nemoclaw"); + }, + ); + it("keeps the gateway runtime when other sandboxes still exist", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-destroy-shared-")); const localBin = path.join(home, "bin");