Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/reference/commands.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -4082,7 +4082,7 @@ For a managed dual-Station vLLM runtime, full uninstall revalidates the exact re
If that cleanup fails, uninstall exits nonzero, preserves its owner-only cleanup receipt, and tells you to resolve the reported peer error before retrying.
Pair cleanup can partially complete before an error; verify both Stations before the retry.
For an authenticated host-local vLLM runtime, full uninstall verifies the exact named container, NemoClaw ownership label, persisted API key, and authentication fingerprint before removing the container by its inspected ID.
When that ownership state is missing, full uninstall removes the reserved `nemoclaw-vllm` container only when Docker reports its NemoClaw managed label and a valid container ID.
When both the host-local runtime receipt and API key are missing, full uninstall removes the reserved `nemoclaw-vllm` container only when Docker reports its NemoClaw managed label and a valid container ID.
An unlabeled container or malformed inspection remains in place and stops the remaining uninstall steps.
For managed llama.cpp, full uninstall verifies the exact named container and network ownership before removing both resources by their inspected IDs.
These host-local checks run before NemoClaw deletes their state.
Expand Down
166 changes: 161 additions & 5 deletions src/lib/actions/uninstall/run-plan-local-model-profile.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,22 @@ const ORPHANED_VLLM_INSPECT_ARGS = [
"nemoclaw-vllm",
];

const ORPHANED_VLLM_ABSENCE_ARGS = [
"container",
"ls",
"--all",
"--no-trunc",
"--filter",
"name=^/nemoclaw-vllm$",
"--format",
"{{.ID}}",
];

const RESERVED_INFERENCE_NAMES_ARGS = ["ps", "-a", "--format", "{{.Names}}"];

function runUninstallPlan(options: UninstallRunOptions, deps: UninstallRunDeps) {
const commandExists = deps.commandExists;
const hasExplicitDocker = deps.runDocker !== undefined;
return runUninstallPlanBase(options, {
resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({
gatewayName,
Expand All @@ -53,6 +66,16 @@ function runUninstallPlan(options: UninstallRunOptions, deps: UninstallRunDeps)
requiredCapabilities: [],
}),
...deps,
commandExists: (command) =>
(!hasExplicitDocker && command === "docker") || commandExists?.(command) === true,
runDocker:
deps.runDocker ??
dockerResults(
new Map([
[JSON.stringify(ORPHANED_VLLM_INSPECT_ARGS), notFound()],
[JSON.stringify(ORPHANED_VLLM_ABSENCE_ARGS), ok()],
]),
),
});
}

Expand Down Expand Up @@ -144,7 +167,7 @@ describe("uninstall local model profile cleanup", () => {

expect(result.exitCode).toBe(1);
expect(runDocker.mock.calls.some(([args]) => args[0] === "rm")).toBe(false);
expect(errors.join("\n")).toContain("remains after ownership-aware cleanup");
expect(errors.join("\n")).toContain("Could not verify NemoClaw ownership");
});

it("preserves an orphaned host-local vLLM container after malformed inspection output (#8981)", () => {
Expand Down Expand Up @@ -172,7 +195,135 @@ describe("uninstall local model profile cleanup", () => {

expect(result.exitCode).toBe(1);
expect(runDocker.mock.calls.some(([args]) => args[0] === "rm")).toBe(false);
expect(errors.join("\n")).toContain("remains after ownership-aware cleanup");
expect(errors.join("\n")).toContain("Could not verify NemoClaw ownership");
});

it("stops orphan cleanup after an empty successful inspection (#8981)", () => {
const errors: string[] = [];
const runDocker = dockerResults(new Map([[JSON.stringify(ORPHANED_VLLM_INSPECT_ARGS), ok()]]));

const result = runUninstallPlan(
{ assumeYes: true, deleteModels: false, keepOpenShell: true },
{
commandExists: (command) => command === "openshell" || command === "docker",
env: { HOME: "/tmp/nemoclaw-uninstall-empty-vllm-inspection" } as NodeJS.ProcessEnv,
existsSync: () => false,
error: (message) => errors.push(message),
isTty: false,
log: () => {},
run: vi.fn(okWithKnownGatewayList),
runDocker,
},
);

expect(result.exitCode).toBe(1);
expect(runDocker).toHaveBeenCalledTimes(1);
expect(errors.join("\n")).toContain("Could not verify NemoClaw ownership");
});

it("stops orphan cleanup when Docker is unavailable (#8981)", () => {
const errors: string[] = [];
const runDocker = vi.fn((_args: string[]) => ok());

const result = runUninstallPlan(
{ assumeYes: true, deleteModels: false, keepOpenShell: true },
{
commandExists: (command) => command === "openshell",
env: { HOME: "/tmp/nemoclaw-uninstall-vllm-no-docker" } as NodeJS.ProcessEnv,
existsSync: () => false,
error: (message) => errors.push(message),
isTty: false,
log: () => {},
run: vi.fn(okWithKnownGatewayList),
runDocker,
},
);

expect(result.exitCode).toBe(1);
expect(runDocker).not.toHaveBeenCalled();
expect(errors.join("\n")).toContain("Docker is unavailable");
});

it("continues orphan cleanup only after Docker proves the container is absent (#8981)", () => {
const runDocker = dockerResults(
new Map([
[JSON.stringify(ORPHANED_VLLM_INSPECT_ARGS), notFound()],
[JSON.stringify(ORPHANED_VLLM_ABSENCE_ARGS), ok()],
]),
);

const result = runUninstallPlan(
{ assumeYes: true, deleteModels: false, keepOpenShell: true },
{
commandExists: (command) => command === "openshell" || command === "docker",
env: { HOME: "/tmp/nemoclaw-uninstall-vllm-absent" } as NodeJS.ProcessEnv,
existsSync: () => false,
isTty: false,
log: () => {},
run: vi.fn(okWithKnownGatewayList),
runDocker,
},
);

expect(result.exitCode).toBe(0);
expect(runDocker).toHaveBeenCalledWith(
ORPHANED_VLLM_ABSENCE_ARGS,
expect.objectContaining({ timeout: 10_000 }),
);
});

it("stops orphan cleanup when Docker cannot prove the container is absent (#8981)", () => {
const errors: string[] = [];
const runDocker = dockerResults(
new Map([
[JSON.stringify(ORPHANED_VLLM_INSPECT_ARGS), notFound()],
[JSON.stringify(ORPHANED_VLLM_ABSENCE_ARGS), notFound()],
]),
);

const result = runUninstallPlan(
{ assumeYes: true, deleteModels: false, keepOpenShell: true },
{
commandExists: (command) => command === "openshell" || command === "docker",
env: { HOME: "/tmp/nemoclaw-uninstall-vllm-ambiguous" } as NodeJS.ProcessEnv,
existsSync: () => false,
error: (message) => errors.push(message),
isTty: false,
log: () => {},
run: vi.fn(okWithKnownGatewayList),
runDocker,
},
);

expect(result.exitCode).toBe(1);
expect(runDocker.mock.calls.some(([args]) => args[0] === "ps")).toBe(false);
expect(errors.join("\n")).toContain("Could not confirm");
});

it("uses exact cleanup when the host-local vLLM receipt remains without its API key (#8981)", () => {
const errors: string[] = [];
const runDocker = vi.fn((_args: string[]) => ok());
const runLocalModelRuntimeCleanup = vi.fn(() => notFound());

const result = runUninstallPlan(
{ assumeYes: true, deleteModels: false, keepOpenShell: true },
{
commandExists: (command) => command === "openshell" || command === "docker",
env: { HOME: "/tmp/nemoclaw-uninstall-receipted-vllm" } as NodeJS.ProcessEnv,
existsSync: (target) => String(target).endsWith("/host-local-vllm-runtime.json"),
error: (message) => errors.push(message),
isTty: false,
log: () => {},
run: vi.fn(okWithKnownGatewayList),
runDocker,
runLocalModelRuntimeCleanup,
},
);

expect(result.exitCode).toBe(1);
expect(runLocalModelRuntimeCleanup).toHaveBeenCalledOnce();
expect(runDocker).not.toHaveBeenCalled();
expect(errors.join("\n")).toContain("Host-local model runtime cleanup did not complete");
});

it("stops uninstall when orphaned host-local vLLM removal fails (#8981)", () => {
Expand Down Expand Up @@ -221,6 +372,7 @@ describe("uninstall local model profile cleanup", () => {
]),
notFound(),
],
[JSON.stringify(ORPHANED_VLLM_ABSENCE_ARGS), ok()],
[JSON.stringify(["ps", "-a", "--format", "{{.Names}}"]), ok("nemoclaw-llama-cpp\n")],
[
JSON.stringify(["ps", "-a", "--format", "{{.ID}} {{.Image}} {{.Names}}"]),
Expand Down Expand Up @@ -275,9 +427,13 @@ describe("uninstall local model profile cleanup", () => {
isTty: false,
log: () => {},
run: vi.fn(okWithKnownGatewayList),
runDocker: vi.fn((args: string[]) =>
args[0] === "ps" && args.at(-1) === "{{.Names}}" ? notFound() : ok(),
),
runDocker: vi.fn((args: string[]) => {
if (JSON.stringify(args) === JSON.stringify(ORPHANED_VLLM_INSPECT_ARGS)) {
return notFound();
}
if (JSON.stringify(args) === JSON.stringify(ORPHANED_VLLM_ABSENCE_ARGS)) return ok();
return args[0] === "ps" && args.at(-1) === "{{.Names}}" ? notFound() : ok();
}),
},
);

Expand Down
46 changes: 41 additions & 5 deletions src/lib/actions/uninstall/run-plan.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import {
cleanupManagedLlamaCppRuntimeForSandbox,
HOST_LOCAL_VLLM_CONTAINER_NAME,
HOST_LOCAL_VLLM_MANAGED_LABEL,
HOST_LOCAL_VLLM_RUNTIME_RECEIPT_FILE,
type ManagedLlamaCppCleanupTarget,
resolveManagedLlamaCppCleanupTarget,
} from "../../inference/local-model-profile/cleanup";
Expand Down Expand Up @@ -1592,12 +1593,20 @@ function removeHostLocalModelRuntimes(paths: UninstallPaths, runtime: UninstallR
const sharedRoot = path.dirname(paths.managedSwapMarkerPath);
const hasLlamaState = runtime.existsSync(path.join(sharedRoot, "managed-llama-cpp"));
const hasManagedKey = runtime.existsSync(path.join(sharedRoot, MANAGED_VLLM_API_KEY_FILE));
const hasHostLocalReceipt = runtime.existsSync(
path.join(sharedRoot, HOST_LOCAL_VLLM_RUNTIME_RECEIPT_FILE),
);
const hasDistributedReceipt = [
MANAGED_CLUSTER_VLLM_RUNTIME_RECEIPT_FILE,
DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE,
].some((name) => runtime.existsSync(path.join(sharedRoot, name)));
if (!hasLlamaState && (!hasManagedKey || hasDistributedReceipt)) {
if (!hasManagedKey && !hasDistributedReceipt && !removeOrphanedManagedHostLocalVllm(runtime)) {
if (!hasLlamaState && ((!hasManagedKey && !hasHostLocalReceipt) || hasDistributedReceipt)) {
if (
!hasManagedKey &&
!hasHostLocalReceipt &&
!hasDistributedReceipt &&
!removeOrphanedManagedHostLocalVllm(runtime)
) {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
return false;
}
return true;
Expand All @@ -1614,7 +1623,12 @@ function removeHostLocalModelRuntimes(paths: UninstallPaths, runtime: UninstallR
}

function removeOrphanedManagedHostLocalVllm(runtime: UninstallRuntime): boolean {
if (!runtime.commandExists("docker")) return true;
if (!runtime.commandExists("docker")) {
runtime.error(
`Docker is unavailable, so NemoClaw could not confirm that '${HOST_LOCAL_VLLM_CONTAINER_NAME}' is absent. NemoClaw did not start the remaining uninstall steps.`,
);
return false;
}
const inspection = runtime.runDocker(
[
"container",
Expand All @@ -1625,10 +1639,32 @@ function removeOrphanedManagedHostLocalVllm(runtime: UninstallRuntime): boolean
],
{ env: runtime.env, timeout: 10_000 },
);
if (inspection.status !== 0 || !inspection.stdout.trim()) return true;
if (inspection.status !== 0) {
const absence = runtime.runDocker(
[
"container",
"ls",
"--all",
"--no-trunc",
"--filter",
`name=^/${HOST_LOCAL_VLLM_CONTAINER_NAME}$`,
"--format",
"{{.ID}}",
],
{ env: runtime.env, timeout: 10_000 },
);
if (absence.status === 0 && !absence.stdout.trim()) return true;
runtime.error(
`Could not confirm that orphaned managed inference container '${HOST_LOCAL_VLLM_CONTAINER_NAME}' is absent. NemoClaw did not start the remaining uninstall steps.`,
);
return false;
}
const [containerId, managedLabel, ...extra] = inspection.stdout.trim().split(/\s+/);
if (!/^[0-9a-f]{64}$/u.test(containerId ?? "") || managedLabel !== "true" || extra.length > 0) {
return true;
runtime.error(
`Could not verify NemoClaw ownership of orphaned managed inference container '${HOST_LOCAL_VLLM_CONTAINER_NAME}'. NemoClaw did not start the remaining uninstall steps.`,
);
return false;
}
const removal = runtime.runDocker(["rm", "-f", containerId], {
env: runtime.env,
Expand Down
6 changes: 5 additions & 1 deletion src/lib/inference/local-model-profile/cleanup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,11 @@ import {
} from "../serving/vllm-host-local-lifecycle";
import { loadManagedVllmApiKey, managedVllmStateDir } from "../vllm-api-key";

export { HOST_LOCAL_VLLM_CONTAINER_NAME, HOST_LOCAL_VLLM_MANAGED_LABEL };
export {
HOST_LOCAL_VLLM_CONTAINER_NAME,
HOST_LOCAL_VLLM_MANAGED_LABEL,
HOST_LOCAL_VLLM_RUNTIME_RECEIPT_FILE,
};

const LLAMA_MANAGED_LABEL = "io.nvidia.nemoclaw.host-local-inference.managed";
const LLAMA_PROVIDER_LABEL = "io.nvidia.nemoclaw.host-local-inference.provider";
Expand Down
Loading