Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
2fc5c50
fix(uninstall): preserve failed llama.cpp cleanup state
prekshivyas Aug 19, 2026
edcb705
Merge remote-tracking branch 'origin/main' into codex/fix-9575-llama-…
prekshivyas Aug 19, 2026
6432af6
test(uninstall): preserve sibling model authority
prekshivyas Aug 20, 2026
e58cd1d
Merge remote-tracking branch 'origin/main' into codex/fix-9575-llama-…
prekshivyas Aug 20, 2026
b0c5291
test(inference): qualify vllm serving-port fixtures
prekshivyas Aug 20, 2026
870496b
test(uninstall): assert preserved llama ownership
prekshivyas Aug 20, 2026
78e300b
test(inference): qualify Spark onboarder fixtures
prekshivyas Aug 20, 2026
021e380
Merge branch 'main' into codex/fix-9575-llama-cpp-uninstall
prekshivyas Aug 20, 2026
3cc81e4
test(package): allow full CLI docs crawl
prekshivyas Aug 20, 2026
235370b
Merge branch 'codex/fix-9575-llama-cpp-uninstall' of https://github.c…
prekshivyas Aug 20, 2026
e2ede40
Merge remote-tracking branch 'origin/main' into codex/fix-9575-llama-…
prekshivyas Aug 20, 2026
e0501c8
fix(inference): lease llama bridge cleanup
prekshivyas Aug 20, 2026
c1bc00c
fix(ci): compile catalog for CPU delegation proof
prekshivyas Aug 20, 2026
af50bee
Merge remote-tracking branch 'origin/main' into codex/fix-9575-llama-…
prekshivyas Aug 20, 2026
6a7c509
test(ci): avoid shape-locking proof workflow
prekshivyas Aug 20, 2026
e808d7d
Merge remote-tracking branch 'origin/main' into codex/fix-9575-llama-…
prekshivyas Aug 20, 2026
fbfe522
fix(uninstall): preserve model stores after cleanup failure
senthilr-nv Aug 20, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 18 additions & 9 deletions src/lib/actions/uninstall/run-plan-local-model-profile.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -620,13 +620,17 @@ describe("uninstall local model profile cleanup", () => {
}
});

it("preserves selected gateway authority when scoped cleanup leaves ownership state", () => {
it("continues unrelated uninstall after managed llama.cpp cleanup fails (#9575)", () => {
const tmpHome = fs.realpathSync(
fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-llama-fail-")),
);
const stateDir = publishManagedLlamaOwner(tmpHome, 8080, "selected-sandbox");
writeScopedGatewayState(tmpHome);
const unrelatedState = path.join(tmpHome, ".nemoclaw", "unrelated-state.json");
fs.writeFileSync(unrelatedState, "{}\n", { mode: 0o600 });
const errors: string[] = [];
const logs: string[] = [];
const runLocalModelRuntimeCleanup = vi.fn(() => ok());
try {
const result = runUninstallPlan(
{ assumeYes: true, deleteModels: false, keepOpenShell: true },
Expand All @@ -637,19 +641,24 @@ describe("uninstall local model profile cleanup", () => {
existsSync: fs.existsSync,
isPortFree: () => true,
isTty: false,
log: () => {},
run: vi.fn((command: string, args: string[]) =>
command === "openshell" && args[0] === "gateway" && args[1] === "list"
? ok(JSON.stringify([{ name: "nemoclaw" }, { name: "nemoclaw-9000" }]))
: ok(),
),
runManagedLlamaCppRuntimeCleanup: vi.fn(() => ok()),
log: (message) => logs.push(message),
run: vi.fn(okWithKnownGatewayList),
runLocalModelRuntimeCleanup,
runManagedLlamaCppRuntimeCleanup: vi.fn(() => ({
status: 1,
stdout: "",
stderr: "qualified endpoint changed",
})),
}),
);

expect(result.exitCode).toBe(1);
expect(fs.existsSync(stateDir)).toBe(true);
expect(errors.join("\n")).toContain("returned without retiring its ownership state");
expect(fs.existsSync(path.join(stateDir, "owner.json"))).toBe(true);
expect(fs.existsSync(unrelatedState)).toBe(false);
expect(runLocalModelRuntimeCleanup).not.toHaveBeenCalled();
expect(logs.some((message) => message.endsWith("State and binaries"))).toBe(true);
expect(errors.join("\n")).toContain("continue unrelated uninstall steps");
} finally {
fs.rmSync(tmpHome, { recursive: true, force: true });
}
Expand Down
73 changes: 56 additions & 17 deletions src/lib/actions/uninstall/run-plan.ts
Original file line number Diff line number Diff line change
Expand Up @@ -600,6 +600,7 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime {
deps.runManagedLlamaCppRuntimeCleanup ??
((sandboxName, gatewayPort) => {
const result = cleanupManagedLlamaCppRuntimeForSandbox(sandboxName, {
env,
gatewayPort,
homeDir: env.HOME || os.homedir(),
});
Expand Down Expand Up @@ -2027,49 +2028,59 @@ function managedLlamaCppCleanupTargets(
}
}

interface ManagedLlamaCppCleanupOutcome {
readonly failedStateDirs: readonly string[];
readonly ok: boolean;
}

function removeManagedLlamaCppRuntimes(
runtime: UninstallRuntime,
scopedToSelectedGateway: boolean,
): boolean {
): ManagedLlamaCppCleanupOutcome {
const targets = managedLlamaCppCleanupTargets(runtime, scopedToSelectedGateway);
if (targets === null) return false;
if (targets === null) return { failedStateDirs: [], ok: false };
for (const target of targets) {
const result = runtime.runManagedLlamaCppRuntimeCleanup(target.sandboxName, target.gatewayPort);
for (const removed of splitNonEmptyLines(result.stdout)) runtime.log(`Removed ${removed}`);
if (result.status !== 0) {
runtime.error(
`Managed llama.cpp cleanup for sandbox '${target.sandboxName}' on gateway port ${String(target.gatewayPort)} did not complete: ${result.stderr.trim() || "unknown cleanup error"}. NemoClaw did not start the remaining uninstall steps.`,
`Managed llama.cpp cleanup for sandbox '${target.sandboxName}' on gateway port ${String(target.gatewayPort)} did not complete: ${result.stderr.trim() || "unknown cleanup error"}. NemoClaw will preserve its ownership state and continue unrelated uninstall steps.`,
);
return false;
return { failedStateDirs: [target.stateDir], ok: true };
}
if (fs.lstatSync(target.stateDir, { throwIfNoEntry: false }) !== undefined) {
runtime.error(
`Managed llama.cpp cleanup for sandbox '${target.sandboxName}' on gateway port ${String(target.gatewayPort)} returned without retiring its ownership state. NemoClaw did not start the remaining uninstall steps.`,
`Managed llama.cpp cleanup for sandbox '${target.sandboxName}' on gateway port ${String(target.gatewayPort)} returned without retiring its ownership state. NemoClaw will preserve its ownership state and continue unrelated uninstall steps.`,
);
return false;
return { failedStateDirs: [target.stateDir], ok: true };
}
}
return true;
return { failedStateDirs: [], ok: true };
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}

function removeManagedModelRuntimes(
paths: UninstallPaths,
runtime: UninstallRuntime,
scopedToSelectedGateway: boolean,
): boolean {
if (!removeManagedLlamaCppRuntimes(runtime, scopedToSelectedGateway)) return false;
if (scopedToSelectedGateway) return true;
): ManagedLlamaCppCleanupOutcome {
const llama = removeManagedLlamaCppRuntimes(runtime, scopedToSelectedGateway);
if (!llama.ok || llama.failedStateDirs.length > 0) return llama;
if (scopedToSelectedGateway) return llama;
const sharedRoot = path.dirname(paths.managedSwapMarkerPath);
const hasDistributedReceipt = [
MANAGED_CLUSTER_VLLM_RUNTIME_RECEIPT_FILE,
DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE,
].some((name) => runtime.existsSync(path.join(sharedRoot, name)));
if (!removeManagedDistributedVllmRuntime(paths, runtime, !hasDistributedReceipt)) return false;
if (!removeHostLocalModelRuntimes(paths, runtime)) return false;
if (!removeManagedDistributedVllmRuntime(paths, runtime, !hasDistributedReceipt)) {
return { failedStateDirs: [], ok: false };
}
if (!removeHostLocalModelRuntimes(paths, runtime)) {
return { failedStateDirs: [], ok: false };
}
if (!hasDistributedReceipt) {
removePath(path.join(sharedRoot, MANAGED_VLLM_API_KEY_FILE), runtime);
}
if (!runtime.commandExists("docker")) return true;
if (!runtime.commandExists("docker")) return llama;
const inventory = runtime.runDocker(["ps", "-a", "--format", "{{.Names}}"], {
env: runtime.env,
timeout: 10_000,
Expand All @@ -2078,16 +2089,29 @@ function removeManagedModelRuntimes(
runtime.error(
"Docker could not inventory reserved managed inference container names. NemoClaw refused the remaining uninstall steps so it cannot report incomplete cleanup as success.",
);
return false;
return { failedStateDirs: [], ok: false };
}
const residual = splitNonEmptyLines(inventory.stdout).find((name) =>
MANAGED_INFERENCE_CONTAINER_NAME_PATTERN.test(name),
);
if (!residual) return true;
if (!residual) return llama;
runtime.error(
`Managed inference container '${residual}' remains after ownership-aware cleanup. NemoClaw refused the remaining uninstall steps; restore its ownership state or remove it after manual review, then retry.`,
);
return false;
return { failedStateDirs: [], ok: false };
}

function recordManagedModelCleanup(
paths: UninstallPaths,
runtime: UninstallRuntime,
scopedToSelectedGateway: boolean,
failedStateDirs: string[],
onPartialFailure: () => void,
): boolean {
const result = removeManagedModelRuntimes(paths, runtime, scopedToSelectedGateway);
failedStateDirs.push(...result.failedStateDirs);
if (result.failedStateDirs.length > 0) onPartialFailure();
return result.ok;
}

function removeDockerContainers(runtime: UninstallRuntime, gatewayName?: string): void {
Expand Down Expand Up @@ -2752,6 +2776,7 @@ function executePlan(
return { ok: false };
}
let ok = true;
const failedManagedLlamaStateDirs: string[] = [];
const branding = runtimeBranding(runtime);
const preserveSharedOpenShell =
options.keepOpenShell || externallySupervised || portableRuntimeCleanup;
Expand Down Expand Up @@ -2793,7 +2818,15 @@ function executePlan(
if (step.name === "Stopping services") {
if (
!portableRuntimeCleanup &&
!removeManagedModelRuntimes(paths, runtime, scopedToSelectedGateway)
!recordManagedModelCleanup(
paths,
runtime,
scopedToSelectedGateway,
failedManagedLlamaStateDirs,
() => {
ok = false;
},
)
) {
return { ok: false };
}
Expand Down Expand Up @@ -2957,6 +2990,12 @@ function executePlan(
[
...preserveUnderStateDir,
...portableStateEntries,
...failedManagedLlamaStateDirs.flatMap((stateDir) => {
const relative = path.relative(paths.nemoclawStateDir, stateDir);
return relative && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative)
? [relative.split(path.sep)[0]!]
: [];
}),
...(selectedIsDefault
? [GATEWAYS_SUBDIR, path.basename(paths.managedSwapMarkerPath)]
: []),
Expand Down
4 changes: 4 additions & 0 deletions src/lib/inference/local-model-profile/cleanup.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -582,17 +582,21 @@ describe("host-local model cleanup", () => {
const homeDir = temporaryHome();
const original = engineHarness({ authorityId: "docker:original" });
const changed = engineHarness({ authorityId: "docker:changed" });
const privateBridge = privateBridgeFixture();
createManagedState(homeDir, original.engine);

const result = cleanupLocalModelRuntimes({
homeDir,
engine: changed.engine,
privateBridge,
});

expect(result).toMatchObject({
ok: false,
reason: expect.stringContaining("endpoint"),
});
expect(privateBridge.stopTransaction).toHaveBeenCalledExactlyOnceWith(TRANSACTION_ID);
expect(privateBridge.assertStopped).toHaveBeenCalledExactlyOnceWith(TRANSACTION_ID);
expect(changed.capture).not.toHaveBeenCalled();
expect(fs.existsSync(managedLlamaCppStatePaths(homeDir).stateDir)).toBe(true);
});
Expand Down
12 changes: 12 additions & 0 deletions src/lib/inference/local-model-profile/cleanup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ export interface LocalModelRuntimeCleanupOptions {
sandboxName?: string;
env?: NodeJS.ProcessEnv;
engine?: ContainerEngine;
privateBridge?: DockerLlamaCppPrivateBridgeController;
deps?: Partial<CleanupDeps>;
}

Expand Down Expand Up @@ -469,6 +470,7 @@ function cleanupLlamaCpp(
sandboxName?: string;
env?: NodeJS.ProcessEnv;
engine?: ContainerEngine;
privateBridge?: DockerLlamaCppPrivateBridgeController;
} = {},
): boolean {
const paths = managedLlamaCppStatePaths(homeDir, options.gatewayPort);
Expand Down Expand Up @@ -547,6 +549,13 @@ function cleanupLlamaCpp(
throw new Error("managed llama.cpp finalized receipt is missing");
}

const privateBridge =
options.privateBridge ??
(process.platform === "linux" ? createDockerLlamaCppPrivateBridgeController() : undefined);
if (privateBridge) {
privateBridge.stopTransaction(journal.transactionId);
privateBridge.assertStopped(journal.transactionId);
}
const engine = options.engine ?? createManagedLlamaCppEngine(options.env ?? process.env);
requireQualifiedEngine(receipt?.engineAuthority ?? journal.engineAuthority, engine);
requireEngineSuccess(
Expand Down Expand Up @@ -627,6 +636,7 @@ export interface ManagedLlamaCppSandboxCleanupOptions {
readonly gatewayPort?: number;
readonly env?: NodeJS.ProcessEnv;
readonly engine?: ContainerEngine;
readonly privateBridge?: DockerLlamaCppPrivateBridgeController;
readonly deps?: Partial<CleanupDeps>;
}

Expand Down Expand Up @@ -906,6 +916,7 @@ export function cleanupManagedLlamaCppRuntimeForSandbox(
sandboxName,
env: options.env,
engine: options.engine,
privateBridge: options.privateBridge,
});
preserveSharedHuggingFaceCache(homeDir, preserved);
return { ok: true, removed, preserved };
Expand Down Expand Up @@ -955,6 +966,7 @@ export function cleanupLocalModelRuntimes(
sandboxName: options.sandboxName,
env: options.env,
engine: options.engine,
privateBridge: options.privateBridge,
});
}
preserveSharedHuggingFaceCache(homeDir, preserved);
Expand Down
Loading