Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions docs/reference/commands.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -1755,6 +1755,9 @@ Cleaning up the gateway after the last sandbox also purges the shared cluster vo
If final gateway cleanup finds a live PID-file process whose command line does not prove it owns the target gateway, `destroy` exits non-zero after sandbox and registry deletion and skips gateway and volume removal.
NemoClaw preserves the per-gateway PID file and runtime marker so you can inspect the process.
Stop only the listener that matches the target gateway, then rerun `destroy` to converge cleanup.
When the default-port gateway runs under the packaged OpenShell gateway service, gateway cleanup stops that service before it reaps host processes, so the gateway port is released instead of being rebound by the service manager.
The service is stopped, not disabled or removed, and the next onboarding run starts it again.
If the service cannot be stopped, `destroy` exits non-zero after sandbox and registry deletion, prints the status command for the service, and skips gateway and volume removal.
If the OpenShell gateway is unreachable and the sandbox has no managed MCP ownership state, `--force` removes only NemoClaw's local registry entry and local artifacts.
Gateway-side deletion remains unconfirmed, shared host-service and gateway teardown are skipped, and the sandbox and retained volume may still exist if the gateway returns.
Start the gateway with `$$nemoclaw <name> status` and retry destroy when you need a confirmed deletion.
Expand Down
25 changes: 25 additions & 0 deletions docs/reference/troubleshooting.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -993,6 +993,31 @@ Create a snapshot first when the sandbox is reachable enough to back up state.
For details, refer to [Create and Restore Snapshots](../manage-sandboxes/state-and-backups/create-and-restore-snapshots).
</Warning>

### Gateway Port Stays Bound After Destroying the Last Sandbox

Destroying the final sandbox with `--cleanup-gateway` stops the packaged OpenShell gateway service before it reaps host gateway processes, so the gateway port is released.
The service is stopped, not disabled or removed, and the next onboarding run starts it again.
If the service cannot be stopped, `destroy` exits non-zero and prints the status command for the service.
Stop the service yourself, then rerun `destroy`.

On Apple Silicon macOS with Homebrew:

```bash
brew services stop openshell
```

On Linux, use the service name that matches the install. For package installs:

```bash
systemctl --user stop openshell-gateway
```

For tarball installs:

```bash
systemctl --user stop nemoclaw-openshell-gateway
```

<AgentOnly variant="openclaw,hermes">

### `gateway restart` or `recover` reports `privileged control unavailable`
Expand Down
105 changes: 105 additions & 0 deletions src/lib/actions/sandbox/destroy-gateway.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,46 @@ vi.mock("../../onboard/stale-gateway-cleanup", () => ({

import { cleanupGatewayAfterLastSandbox } from "./destroy-gateway";

function packagedServiceOwner({
gatewayName,
gatewayPort,
}: {
gatewayName: string;
gatewayPort: number;
}) {
return {
gatewayName,
gatewayPort,
mode: "nemoclaw-managed" as const,
source: "packaged-service" as const,
endpoint: null,
stateDir: null,
supervisor: null,
requiredCapabilities: [],
};
}

function serviceStopResult(stopped: boolean, reason?: string) {
return {
attempted: true,
manager: "systemd" as const,
serviceName: "nemoclaw-openshell-gateway",
statusCommand: "systemctl --user status nemoclaw-openshell-gateway",
stopped,
...(reason === undefined ? {} : { reason }),
};
}

function idleHostReaperResult() {
return {
failed: [],
skippedDeadPids: [],
skippedNonMatchingPids: [],
stopped: [],
sudoRemediationPids: [],
};
}

describe("cleanupGatewayAfterLastSandbox", () => {
beforeEach(() => {
mocks.resolveGatewayTeardownAuthority.mockImplementation(
Expand Down Expand Up @@ -253,6 +293,71 @@ describe("cleanupGatewayAfterLastSandbox", () => {
expect(mocks.dockerRemoveVolumesByPrefix).not.toHaveBeenCalled();
});

it("stops the packaged gateway service before the host reaper on final destroy (#7904)", () => {
vi.spyOn(process, "platform", "get").mockReturnValue("linux");
vi.spyOn(os, "homedir").mockReturnValue("/home/tester");
mocks.resolveGatewayTeardownAuthority.mockImplementationOnce(packagedServiceOwner);
const events: string[] = [];
mocks.stopHostGatewayProcesses.mockImplementationOnce(() => {
events.push("host-reaper");
return idleHostReaperResult();
});
const stopService = vi.fn(() => {
events.push("service-stop");
return serviceStopResult(true);
});
const runOpenshell = vi.fn(() => ({ status: 0, stdout: "", stderr: "" }));

cleanupGatewayAfterLastSandbox("nemoclaw", runOpenshell, {
stopOpenShellGatewayUserService: stopService,
});

expect(events).toEqual(["service-stop", "host-reaper"]);
expect(runOpenshell).toHaveBeenCalledWith(["gateway", "remove", "nemoclaw"], {
ignoreError: true,
stdio: ["ignore", "pipe", "pipe"],
});
expect(mocks.dockerRemoveVolumesByPrefix).toHaveBeenCalledWith("openshell-cluster-nemoclaw", {
ignoreError: true,
});
});

it("fails destroy when the packaged gateway service survives the stop (#7904)", () => {
vi.spyOn(process, "platform", "get").mockReturnValue("darwin");
vi.spyOn(os, "homedir").mockReturnValue("/home/tester");
mocks.resolveGatewayTeardownAuthority.mockImplementationOnce(packagedServiceOwner);
const stopService = vi.fn(() =>
serviceStopResult(false, "systemctl --user stop nemoclaw-openshell-gateway failed: timeout"),
);
const runOpenshell = vi.fn(() => ({ status: 0, stdout: "", stderr: "" }));

expect(() =>
cleanupGatewayAfterLastSandbox("nemoclaw", runOpenshell, {
stopOpenShellGatewayUserService: stopService,
}),
).toThrow("systemctl --user status nemoclaw-openshell-gateway");
expect(mocks.stopHostGatewayProcesses).not.toHaveBeenCalled();
expect(runOpenshell).not.toHaveBeenCalledWith(
["gateway", "remove", "nemoclaw"],
expect.anything(),
);
expect(mocks.dockerRemoveVolumesByPrefix).not.toHaveBeenCalled();
});

it("leaves the service manager alone for a standalone NemoClaw gateway (#7904)", () => {
vi.spyOn(process, "platform", "get").mockReturnValue("linux");
vi.spyOn(os, "homedir").mockReturnValue("/home/tester");
const stopService = vi.fn(() => serviceStopResult(true));
const runOpenshell = vi.fn(() => ({ status: 0, stdout: "", stderr: "" }));

cleanupGatewayAfterLastSandbox("nemoclaw", runOpenshell, {
stopOpenShellGatewayUserService: stopService,
});

expect(stopService).not.toHaveBeenCalled();
expect(mocks.stopHostGatewayProcesses).toHaveBeenCalledOnce();
});

it.each([
[
"host reaper",
Expand Down
12 changes: 12 additions & 0 deletions src/lib/actions/sandbox/destroy-gateway.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import path from "node:path";
import { dockerRemoveVolumesByPrefix } from "../../adapters/docker/volume";
import { OPENSHELL_OPERATION_TIMEOUT_MS } from "../../adapters/openshell/timeouts";
import { DASHBOARD_PORT } from "../../core/ports";
import { stopOpenShellGatewayUserService } from "../../onboard/docker-driver-gateway-service";
import {
resolveGatewayPortFromName,
resolveGatewayStateDirName,
Expand All @@ -28,6 +29,7 @@ const DASHBOARD_FORWARD_PORT = String(DASHBOARD_PORT);

export interface CleanupGatewayDeps {
resolveGatewayTeardownAuthority?: GatewayTeardownAuthorityResolver;
stopOpenShellGatewayUserService?: typeof stopOpenShellGatewayUserService;
}

// Compute the Docker-driver gateway state directory that belongs to
Expand Down Expand Up @@ -105,6 +107,16 @@ export function cleanupGatewayAfterLastSandbox(
// ports the live openshell tracks; this catches orphans whose openshell
// record was lost across upgrades or failed onboards.
stopStaleDashboardListeners();
if (!externallySupervised && owner.source === "packaged-service") {
const stopService = deps.stopOpenShellGatewayUserService ?? stopOpenShellGatewayUserService;
const serviceStop = stopService();
if (serviceStop.attempted && !serviceStop.stopped) {
throw new Error(
`Failed to stop the packaged OpenShell gateway service '${serviceStop.serviceName}' that owns gateway '${gatewayName}': ${serviceStop.reason}. ` +
`Check: ${serviceStop.statusCommand}. Stop the service, then rerun destroy.`,
);
}
}
if (!externallySupervised && (process.platform === "linux" || process.platform === "darwin")) {
// Sandbox destroy is conservative: only stop the host gateway whose PID
// file we wrote during onboard. Disable the pgrep sweep so a stray
Expand Down
126 changes: 126 additions & 0 deletions src/lib/onboard/docker-driver-gateway-service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
type SpawnSyncLikeResult,
startOpenShellGatewayUserService,
startPackageManagedDockerDriverGateway,
stopOpenShellGatewayUserService,
} from "./docker-driver-gateway-service";

const STATUS_CONNECTED = `
Expand Down Expand Up @@ -552,4 +553,129 @@ describe("docker-driver-gateway-service", () => {
).rejects.toThrow("configured 1s health deadline");
expect(clear).not.toHaveBeenCalled();
});

it("stops the trusted systemd gateway unit without disabling it (#7904)", () => {
const events: string[] = [];
const home = "/home/nvidia";
const servicePath = `${home}/.config/systemd/user/nemoclaw-openshell-gateway.service`;
const gatewayBin = `${home}/.local/bin/openshell-gateway`;

const result = stopOpenShellGatewayUserService({
commandExists: (command) => command === "systemctl",
env: { HOME: home },
existsSync: (candidate) => candidate === servicePath,
home,
lstatSync: nonSymlinkStat,
platform: "linux",
readFileSync: () => `# ${NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER}\n`,
spawnSyncImpl: systemdSpawn(events, servicePath, gatewayBin),
});

expect(result).toEqual({
attempted: true,
manager: "systemd",
serviceName: "nemoclaw-openshell-gateway",
statusCommand: "systemctl --user status nemoclaw-openshell-gateway",
stopped: true,
});
expect(events).toEqual([
"show nemoclaw-openshell-gateway --property=FragmentPath --property=ExecStart",
"stop nemoclaw-openshell-gateway",
]);
});

it("refuses to stop a systemd unit that no longer has the trusted identity (#7904)", () => {
const events: string[] = [];
const home = "/home/nvidia";
const servicePath = `${home}/.config/systemd/user/nemoclaw-openshell-gateway.service`;

const result = stopOpenShellGatewayUserService({
commandExists: (command) => command === "systemctl",
env: { HOME: home },
existsSync: (candidate) => candidate === servicePath,
home,
lstatSync: nonSymlinkStat,
platform: "linux",
readFileSync: () => `# ${NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER}\n`,
spawnSyncImpl: systemdSpawn(
events,
`${home}/.config/systemd/user/unrelated.service`,
"/usr/bin/unrelated",
),
});

expect(result).toMatchObject({ attempted: true, stopped: false });
expect(result.reason).toContain("service identity is not a trusted OpenShell gateway");
expect(events).toEqual([
"show nemoclaw-openshell-gateway --property=FragmentPath --property=ExecStart",
]);
});

it("stops the official Homebrew gateway service on macOS (#7904)", () => {
const events: string[] = [];
const brew = vi.fn((_command: string, args: string[]) => {
events.push(args.join(" "));
return args[0] === "info" ? officialFormulaInfo() : spawnResult();
});

const result = stopOpenShellGatewayUserService({
commandExists: (command) => command === "brew",
platform: "darwin",
spawnSyncImpl: brew,
});

expect(result).toEqual({
attempted: true,
manager: "homebrew",
serviceName: "openshell",
statusCommand: "brew services info openshell",
stopped: true,
});
expect(events.at(-1)).toBe("services stop openshell");
});

it("reports the failing stop command when the gateway service survives (#7904)", () => {
const home = "/home/nvidia";
const servicePath = `${home}/.config/systemd/user/nemoclaw-openshell-gateway.service`;
const gatewayBin = `${home}/.local/bin/openshell-gateway`;

const result = stopOpenShellGatewayUserService({
commandExists: (command) => command === "systemctl",
env: { HOME: home },
existsSync: (candidate) => candidate === servicePath,
home,
lstatSync: nonSymlinkStat,
platform: "linux",
readFileSync: () => `# ${NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER}\n`,
spawnSyncImpl: vi.fn((_command: string, args: string[]) =>
args.includes("show")
? spawnResult(0, "", trustedShowOutput(servicePath, gatewayBin))
: spawnResult(1, "Job for nemoclaw-openshell-gateway.service failed"),
),
});

expect(result).toMatchObject({
attempted: true,
stopped: false,
statusCommand: "systemctl --user status nemoclaw-openshell-gateway",
});
expect(result.reason).toContain(
"systemctl --user stop nemoclaw-openshell-gateway failed: Job for",
);
});

it.each([
[
"no service is installed",
{ existsSync: () => false, platform: "linux" as const },
"service not installed",
],
["the platform has no service manager", { platform: "win32" as const }, "unsupported platform"],
])("reports nothing to stop when %s (#7904)", (_case, opts, reason) => {
expect(stopOpenShellGatewayUserService({ commandExists: () => true, ...opts })).toEqual({
attempted: false,
reason,
stopped: false,
});
});
});
Loading
Loading