Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ci/source-architecture-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
"src/lib/actions/sandbox/process-recovery.ts": 21,
"src/lib/actions/sandbox/rebuild-pipeline.ts": 28,
"src/lib/actions/sandbox/snapshot.ts": 39,
"src/lib/actions/uninstall/run-plan.ts": 25,
"src/lib/actions/uninstall/run-plan.ts": 26,
"src/lib/inference/onboard-probes.ts": 21,
"src/lib/inference/vllm.ts": 21,
"src/lib/onboard.ts": 225,
Expand Down
2 changes: 2 additions & 0 deletions docs/manage-sandboxes/uninstall-nemoclaw.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,8 @@ If validation, runtime cleanup, or temporary-claim retirement fails, uninstall e
Resolve the reported SSH, Docker, peer-host, or claim error before you retry.
Runtime cleanup can partially complete before an error, so inspect every cluster node before retrying.
When sibling gateways remain, it removes only the selected gateway's resources and port-scoped state and preserves those shared host resources.
A recorded sandbox that OpenShell reports as already removed leaves nothing to delete, so the scoped path reports it, finishes the remaining gateway cleanup, and exits `0`.
A sandbox that OpenShell cannot reach or refuses to delete still leaves state behind, so uninstall keeps the selected gateway state for a retry and exits nonzero.
This gateway-scoped path leaves a managed distributed runtime running.
It preserves the cleanup receipt, copied SSH bindings, temporary discovery claims, and host-global API key.
This preservation also applies when you pass `--destroy-user-data`.
Expand Down
189 changes: 189 additions & 0 deletions src/lib/actions/uninstall/run-plan-absent-sandbox.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,189 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import fs from "node:fs";
import os from "node:os";
import path from "node:path";

import { afterEach, describe, expect, it, vi } from "vitest";

import { readGatewayRegistryFile } from "../../state/gateway-registry";
import { migrateLegacyPortState } from "../../state/legacy-port-migration";
import {
type RunResult,
runUninstallPlan as runUninstallPlanBase,
type UninstallRunDeps,
type UninstallRunOptions,
} from "./run-plan";

function ok(stdout = ""): RunResult {
return { status: 0, stdout, stderr: "" };
}

function withManagedGatewayAuthority(deps: UninstallRunDeps): UninstallRunDeps {
return {
resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({
gatewayName,
gatewayPort,
mode: "nemoclaw-managed",
source: gatewayPort === 8080 ? "packaged-service" : "standalone",
endpoint: null,
stateDir: null,
supervisor: null,
requiredCapabilities: [],
}),
...deps,
};
}

function bindManagedGatewayAuthority(run: typeof runUninstallPlanBase) {
return (options: UninstallRunOptions, deps: UninstallRunDeps) =>
run(options, withManagedGatewayAuthority(deps));
}

afterEach(() => {
vi.unstubAllEnvs();
vi.resetModules();
});

describe("uninstall sandbox delete outcomes (#7906)", () => {
it.each([
{
case: "already removed",
deleteResponse: {
status: 1,
stdout: "",
stderr: "Error: status: NotFound, sandbox 'selected-box' not found",
},
expectedExitCode: 0,
expectedWarning: "OpenShell sandbox 'selected-box' already removed",
expectsPreservedState: false,
},
{
case: "unreachable",
deleteResponse: {
status: 1,
stdout: "",
stderr: "error trying to connect: tcp connect error: Connection refused (os error 111)",
},
expectedExitCode: 1,
expectedWarning: "OpenShell sandbox 'selected-box' could not be removed or was unreachable",
expectsPreservedState: true,
},
{
case: "rejected for an unrecognized reason",
deleteResponse: { status: 1, stdout: "", stderr: "Error: sandbox is still finalizing" },
expectedExitCode: 1,
expectedWarning: "OpenShell sandbox 'selected-box' could not be removed or was unreachable",
expectsPreservedState: true,
},
])("completes selected-gateway cleanup when the recorded sandbox is $case", async ({
deleteResponse,
expectedExitCode,
expectedWarning,
expectsPreservedState,
}) => {
const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-absent-sandbox-"));
const selectedPort = 9123;
try {
vi.stubEnv("NEMOCLAW_GATEWAY_PORT", String(selectedPort));
vi.resetModules();
const runPortUninstall = bindManagedGatewayAuthority(
(await import("./run-plan")).runUninstallPlan,
);
const shared = path.join(tmpHome, ".nemoclaw");
const sharedRegistryFile = path.join(shared, "sandboxes.json");
fs.mkdirSync(shared, { recursive: true });
fs.writeFileSync(
sharedRegistryFile,
JSON.stringify({
defaultSandbox: "selected-box",
sandboxes: {
"selected-box": {
name: "selected-box",
gatewayName: `nemoclaw-${String(selectedPort)}`,
gatewayPort: selectedPort,
},
"sibling-box": {
name: "sibling-box",
gatewayName: "nemoclaw-9124",
gatewayPort: 9124,
},
},
}),
);
migrateLegacyPortState({ gatewayPort: selectedPort, home: tmpHome });
const calls: string[][] = [];
const warnings: string[] = [];

const result = runPortUninstall(
{
assumeYes: true,
deleteModels: false,
destroyUserData: true,
gatewayName: `nemoclaw-${String(selectedPort)}`,
keepOpenShell: false,
},
{
commandExists: (command) => command === "openshell",
env: {
HOME: tmpHome,
NEMOCLAW_GATEWAY_PORT: String(selectedPort),
} as NodeJS.ProcessEnv,
error: (line) => warnings.push(line),
existsSync: (target) => target.startsWith(tmpHome) && fs.existsSync(target),
isTty: false,
log: vi.fn(),
run: (_command, args) => {
calls.push(args);
return args[0] === "sandbox" && args[1] === "delete" ? deleteResponse : ok();
},
runDocker: () => ok(""),
},
);

expect(result.exitCode).toBe(expectedExitCode);
expect(warnings.join("\n")).toContain(expectedWarning);
expect(calls).toContainEqual(["sandbox", "delete", "selected-box"]);
expect(
calls.some(
(args) =>
args[0] === "gateway" &&
args[1] === "remove" &&
args[2] === `nemoclaw-${String(selectedPort)}`,
),
).toBe(!expectsPreservedState);
expect(fs.existsSync(path.join(shared, "gateways", String(selectedPort)))).toBe(
expectsPreservedState,
);
expect(warnings.join("\n").includes("Selected gateway cleanup was incomplete")).toBe(
expectsPreservedState,
);
const selectedRegistryFile = path.join(
shared,
"gateways",
String(selectedPort),
"sandboxes.json",
);
const selectedSandboxes = readGatewayRegistryFile(tmpHome, selectedRegistryFile)?.sandboxes;
expect(selectedSandboxes?.["selected-box"]).toEqual(
expectsPreservedState
? {
name: "selected-box",
gatewayName: `nemoclaw-${String(selectedPort)}`,
gatewayPort: selectedPort,
}
: undefined,
);
expect(readGatewayRegistryFile(tmpHome, sharedRegistryFile)?.sandboxes).toEqual({
"sibling-box": {
name: "sibling-box",
gatewayName: "nemoclaw-9124",
gatewayPort: 9124,
},
});
} finally {
fs.rmSync(tmpHome, { recursive: true, force: true });
}
});
});
31 changes: 24 additions & 7 deletions src/lib/actions/uninstall/run-plan.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,14 @@ import { isErrnoException } from "../../core/errno";
import { DEFAULT_GATEWAY_PORT, GATEWAY_PORT } from "../../core/ports";
import { isStdinTty, readLineFromStdin } from "../../core/stdin";
import { sleepMs } from "../../core/wait";
import { getSandboxDeleteOutcome } from "../../domain/sandbox/destroy";
import {
gatewayDestroySkipMessage,
OPENSHELL_SANDBOXES_DELETE_SKIP_MESSAGE,
preservedRegistryUnrecoverableWarnings,
providerDeleteSkipMessage,
sandboxDeleteAbsentMessage,
sandboxDeleteFailureMessage,
} from "../../domain/uninstall/messaging";
import {
defaultUninstallPaths,
Expand Down Expand Up @@ -574,6 +577,22 @@ function runOptional(
return false;
}

function deleteSelectedGatewaySandbox(runtime: UninstallRuntime, sandboxName: string): boolean {
const result = runtime.run("openshell", ["sandbox", "delete", sandboxName], {
env: runtime.env,
});
if (result.status === 0) {
runtime.log(`Deleted OpenShell sandbox '${sandboxName}'`);
return true;
}
if (getSandboxDeleteOutcome(result).alreadyGone) {
runtime.warn(sandboxDeleteAbsentMessage(sandboxName));
return true;
}
runtime.warn(sandboxDeleteFailureMessage(sandboxName));
return false;
}

const GATEWAY_ALREADY_ABSENT =
/gateway[^\n]*(?:does not exist|not found)|No (?:active )?gateway|No gateway metadata found/i;
const GATEWAY_REMOVE_UNSUPPORTED =
Expand Down Expand Up @@ -1144,13 +1163,11 @@ function removeOpenShellResources(
let removedSelectedResources = true;
for (const sandboxName of sandboxNames) {
removedSelectedResources =
runOptional(
runtime,
`Deleted OpenShell sandbox '${sandboxName}'`,
"openshell",
["sandbox", "delete", sandboxName],
{ onSkip: `OpenShell sandbox '${sandboxName}' was already absent or unreachable` },
) && removedSelectedResources;
deleteSelectedGatewaySandbox(runtime, sandboxName) && removedSelectedResources;
Comment thread
apurvvkumaria marked this conversation as resolved.
}
if (!removedSelectedResources) {
runtime.warn("Selected gateway cleanup was incomplete; preserving its state for retry.");
return false;
}
removedSelectedResources =
removeGatewayRegistration(runtime, gatewayLabel, !externallySupervised) &&
Expand Down
8 changes: 8 additions & 0 deletions src/lib/domain/uninstall/messaging.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,14 @@ export function gatewayDestroySkipMessage(gatewayLabel: string): string {
return `Gateway '${gatewayLabel}' already removed or unreachable`;
}

export function sandboxDeleteAbsentMessage(sandboxName: string): string {
return `OpenShell sandbox '${sandboxName}' already removed`;
}

export function sandboxDeleteFailureMessage(sandboxName: string): string {
return `OpenShell sandbox '${sandboxName}' could not be removed or was unreachable`;
}

/**
* Warnings shown when uninstall preserves sandboxes.json: uninstall removes
* the gateway registration, provider registrations, and Docker image the
Expand Down
43 changes: 43 additions & 0 deletions test/uninstall.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,49 @@ exit 0
}
}, 60_000);

it("completes selected-gateway cleanup and exits 0 when the recorded sandbox is already absent (#7906)", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-absent-sandbox-"));
const fakeBin = path.join(tmp, "bin");
writeFakeTools(fakeBin);
fs.writeFileSync(
path.join(fakeBin, "openshell"),
`#!/usr/bin/env bash
case "$*" in
"gateway list -o json") printf '[{"name":"nemoclaw"},{"name":"nemoclaw-9124"}]\\n' ;;
"sandbox delete my-assistant")
printf "Error: status: NotFound, sandbox 'my-assistant' not found\\n" >&2
exit 1
;;
esac
exit 0
`,
{ mode: 0o755 },
);
const stateDir = path.join(tmp, ".nemoclaw");
fs.mkdirSync(stateDir, { recursive: true });
fs.writeFileSync(
path.join(stateDir, "sandboxes.json"),
JSON.stringify({
defaultSandbox: "my-assistant",
sandboxes: {
"my-assistant": { name: "my-assistant", gatewayName: "nemoclaw", gatewayPort: 8080 },
sibling: { name: "sibling", gatewayName: "nemoclaw-9124", gatewayPort: 9124 },
},
}),
);
try {
const result = runUninstall(tmp, ["--yes", "--destroy-user-data"]);
const output = `${result.stdout}${result.stderr}`;

expect(result.status, output).toBe(0);
expect(output).toMatch(/OpenShell sandbox 'my-assistant' already removed/);
expect(output).not.toMatch(/Selected gateway cleanup was incomplete/);
expect(output).not.toMatch(/Uninstall completed with errors/);
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
}, 60_000);

it("uses NemoHermes branding for --yes when Hermes is active", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemohermes-uninstall-yes-"));
writeFakeTools(path.join(tmp, "bin"));
Expand Down
Loading